fix(fs-search): drop exhausted glob sample groups
Keep only groups with another path in the active round. This bounds skewed sampling by paths visited instead of rescanning every singleton for every late-group item.
This commit is contained in:
@@ -150,27 +150,35 @@ function topLevelSegment(path: string): string {
|
||||
* @returns the page grouped by top-level entry, with the shown/total top-level spread.
|
||||
*/
|
||||
export function sampleAcrossTopLevel(paths: readonly string[], maxItems: number, root = '.'): GlobSample {
|
||||
type ActiveGroup = { key: string; items: string[]; index: number; current: string }
|
||||
const groups = new Map<string, string[]>()
|
||||
let active: ActiveGroup[] = []
|
||||
for (const path of paths) {
|
||||
const key = topLevelSegment(relativeToSearchRoot(path, root))
|
||||
const group = groups.get(key)
|
||||
if (group === undefined) groups.set(key, [path])
|
||||
else group.push(path)
|
||||
if (group === undefined) {
|
||||
const items = [path]
|
||||
groups.set(key, items)
|
||||
active.push({ key, items, index: 0, current: path })
|
||||
} else {
|
||||
group.push(path)
|
||||
}
|
||||
}
|
||||
let rounds = 0
|
||||
for (const group of groups.values()) rounds = Math.max(rounds, group.length)
|
||||
const taken = new Map<string, string[]>()
|
||||
let count = 0
|
||||
for (let round = 0; round < rounds && count < maxItems; round += 1) {
|
||||
for (const [key, group] of groups) {
|
||||
while (active.length > 0 && count < maxItems) {
|
||||
const nextActive: ActiveGroup[] = []
|
||||
for (const { key, items, index, current } of active) {
|
||||
if (count >= maxItems) break
|
||||
const path = group[round]
|
||||
if (path === undefined) continue
|
||||
count += 1
|
||||
const bucket = taken.get(key)
|
||||
if (bucket === undefined) taken.set(key, [path])
|
||||
else bucket.push(path)
|
||||
if (bucket === undefined) taken.set(key, [current])
|
||||
else bucket.push(current)
|
||||
const nextIndex = index + 1
|
||||
const nextPath = items[nextIndex]
|
||||
if (nextPath !== undefined) nextActive.push({ key, items, index: nextIndex, current: nextPath })
|
||||
}
|
||||
active = nextActive
|
||||
}
|
||||
return { items: [...taken.values()].flat(), shown: taken.size, total: groups.size }
|
||||
}
|
||||
|
||||
@@ -538,6 +538,19 @@ describe('cross-directory sampling', () => {
|
||||
expect(sampleAcrossTopLevel(paths, 3)).toEqual({ items: ['solo/a', 'many/b', 'many/c'], shown: 2, total: 2 })
|
||||
})
|
||||
|
||||
it('does not rescan exhausted entries while filling a skewed page', () => {
|
||||
const singletonCount = 12_500
|
||||
const paths = [
|
||||
...Array.from({ length: singletonCount }, (_, index) => `group-${index}/only`),
|
||||
...Array.from({ length: singletonCount }, (_, index) => `late/${index}`),
|
||||
]
|
||||
expect(sampleAcrossTopLevel(paths, paths.length - 1)).toMatchObject({
|
||||
shown: singletonCount + 1,
|
||||
total: singletonCount + 1,
|
||||
items: { length: paths.length - 1 },
|
||||
})
|
||||
}, 500)
|
||||
|
||||
it('reports the entries it could not reach when the page is smaller than the top level', () => {
|
||||
const paths = ['a/1', 'b/1', 'c/1', 'd/1']
|
||||
expect(sampleAcrossTopLevel(paths, 2)).toEqual({ items: ['a/1', 'b/1'], shown: 2, total: 4 })
|
||||
|
||||
Reference in New Issue
Block a user