Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion layers/registry/server/utils/github-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -587,8 +587,15 @@ export async function getTree(
// into the inventory: sync uses absent paths to retire Skills.
const rootSha = initial.data.sha
const incomplete: TreeResponse = { sha: rootSha, tree: [], truncated: true }
// The discarded recursive response can hold 100,000 entries. Release it
// before reading subtrees, so both inventories never share the working set.
initial.data = incomplete
const pending = [{ sha: rootSha, prefix: '', recursive: false }]
const entries: TreeEntry[] = []
// scienceide_env exhausted Worker memory on five expansion attempts. Bound
// retained strings as well as objects; long paths defeat an entry-only cap.
const maxRetainedChars = 8 * 1024 * 1024
let retainedChars = 0
let rateLimit = initial.rateLimit
let requests = 1
while (pending.length > 0) {
Expand All @@ -613,10 +620,13 @@ export async function getTree(
pending.push({ ...next, recursive: false })
continue
}
if (entries.length + result.data.tree.length > 250_000)
if (entries.length + result.data.tree.length > 100_000)
return { ...result, data: incomplete, rateLimit, notModified: false }
for (const entry of result.data.tree) {
const path = next.prefix ? `${next.prefix}/${entry.path}` : entry.path
retainedChars += path.length + entry.sha.length
if (retainedChars > maxRetainedChars)
return { ...result, data: incomplete, rateLimit, notModified: false }
entries.push({ path, type: entry.type, sha: entry.sha, size: entry.size })
if (!next.recursive && entry.type === 'tree')
pending.push({ sha: entry.sha, prefix: path, recursive: true })
Expand Down
19 changes: 19 additions & 0 deletions test/unit/github-client-tree.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,23 @@ describe('getTree', () => {
expect(fetchMock.mock.calls.length).toBeGreaterThan(1)
expect(fetchMock.mock.calls.length).toBeLessThanOrEqual(64)
})

it('stops expansion when long paths exceed the retained tree budget', async () => {
const subtree = Array.from({ length: 20_000 }, (_, i) => ({
path: `${'directory/'.repeat(50)}${i}/SKILL.md`,
type: 'blob',
sha: 'a'.repeat(40),
}))
vi.stubGlobal('fetch', vi.fn(async (url: string) => Response.json(
url.endsWith('root?recursive=1')
? { sha: 'root', truncated: true, tree: [] }
: url.endsWith('root')
? { sha: 'root', tree: [{ path: 'vendor', type: 'tree', sha: 'subtree' }] }
: { sha: 'subtree', tree: subtree },
)))

const result = await getTree('acme', 'large', 'root', {}, { expandTruncated: true })

expect(result.data).toEqual({ sha: 'root', tree: [], truncated: true })
})
})
Loading