diff --git a/packages/typescript/src/api/async/api.ts b/packages/typescript/src/api/async/api.ts index 35866d4c53132..d91df45d68322 100644 --- a/packages/typescript/src/api/async/api.ts +++ b/packages/typescript/src/api/async/api.ts @@ -47,6 +47,7 @@ import { parseNodeHandle, readParseOptionsKey, readSourceFileHash, + readSourceFileLease, RemoteSourceFile, } from "../node/node.ts"; import { Wtf8Decoder } from "../node/wtf8.ts"; @@ -327,6 +328,7 @@ export class API implements FormatDiagnosticsHo private initialized: boolean = false; private initializing: Promise | undefined; private activeSnapshots: Map = new Map(); + private activeSourceFileLeases: Map = new Map(); readonly printer: Printer; readonly internal: InternalAPI; @@ -422,22 +424,57 @@ export class API implements FormatDiagnosticsHo return this.client.apiRequest("parseJsonConfigFileContent", { json, ...options }); } - async createSourceFile(fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): Promise { + /** + * Create and retain a source file independently of a program. + * Dispose the returned lease when the source file no longer needs to remain available remotely. + */ + async createSourceFile(fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): Promise { await this.ensureInitialized(); const data = await this.client.apiRequestBinary("createSourceFile", { fileName, sourceText, options }); if (!data) { throw new Error("createSourceFile returned no source file"); } - return new RemoteSourceFile(data, this.decoder, this.client.getTimingCollector()) as unknown as SourceFile; + return this.retainSourceFileResponse(data); } - async createSourceFileFromFile(file: DocumentIdentifier, options: CreateSourceFileOptions = {}): Promise { + /** + * Read, create, and retain a source file independently of a program. + * Dispose the returned lease when the source file no longer needs to remain available remotely. + */ + async createSourceFileFromFile(file: DocumentIdentifier, options: CreateSourceFileOptions = {}): Promise { await this.ensureInitialized(); const data = await this.client.apiRequestBinary("createSourceFileFromFile", { fileName: resolveFileName(file), options }); if (!data) { throw new Error("createSourceFileFromFile returned no source file"); } - return new RemoteSourceFile(data, this.decoder, this.client.getTimingCollector()) as unknown as SourceFile; + return this.retainSourceFileResponse(data); + } + + private retainSourceFileResponse(data: Uint8Array): RetainedSourceFile { + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + const lease = readSourceFileLease(view); + try { + const decoded = new RemoteSourceFile(data, this.decoder, this.client.getTimingCollector()) as unknown as SourceFile; + const sourceFile = this.sourceFileCache.setForLease(decoded.path, decoded, readParseOptionsKey(view), readSourceFileHash(view), lease); + const retained = new RetainedSourceFile(sourceFile, lease, this.client, () => { + this.activeSourceFileLeases.delete(lease); + this.sourceFileCache.releaseLease(lease); + }); + this.activeSourceFileLeases.set(lease, retained); + return retained; + } + catch (error) { + // @sync-skip-block-start + void this.client.apiRequest("releaseSourceFile", { lease }).catch(() => {}); + // @sync-skip-block-end + // @sync-only-start + // try { + // this.client.apiRequest("releaseSourceFile", { lease }); + // } + // catch {} + // @sync-only-end + throw error; + } } async transpileModule(input: string, options: TranspileOptions = {}): Promise { @@ -608,15 +645,21 @@ export class API implements FormatDiagnosticsHo async close(): Promise { await this.initializing?.catch(() => {}); // @sync-skip - // Dispose all active snapshots try { - for (const snapshot of [...this.activeSnapshots.values()]) { - await snapshot.dispose(); + for (const retained of [...this.activeSourceFileLeases.values()]) { + await retained.dispose(); } - this.sourceFileCache.clear(); } finally { - await this.client.close(); // always close the underlying connection + try { + for (const snapshot of [...this.activeSnapshots.values()]) { + await snapshot.dispose(); + } + this.sourceFileCache.clear(); + } + finally { + await this.client.close(); // always close the underlying connection + } } } @@ -708,6 +751,42 @@ export class API implements FormatDiagnosticsHo type EnsureInitialized = () => Promise; // @sync: type EnsureInitialized = (() => void) & { gen(): Generator; }; +/** An independently retained source file and its disposable remote-lifetime lease. */ +export class RetainedSourceFile { + readonly sourceFile: SourceFile; + private readonly lease: number; + private readonly client: Client; + private readonly onDispose: () => void; + private disposed = false; + private disposePromise: Promise | undefined; + + constructor(sourceFile: SourceFile, lease: number, client: Client, onDispose: () => void) { + this.sourceFile = sourceFile; + this.lease = lease; + this.client = client; + this.onDispose = onDispose; + } + + [globalThis.Symbol.asyncDispose](): Promise { // @sync: [globalThis.Symbol.dispose](): void { + return this.dispose(); // @sync: this.dispose(); + } + + dispose(): Promise { + return this.disposePromise ??= this.disposeWorker(); + } + + private async disposeWorker(): Promise { + if (this.disposed) return; + this.disposed = true; + try { + await this.client.apiRequest("releaseSourceFile", { lease: this.lease }); + } + finally { + this.onDispose(); + } + } +} + export class InternalAPI { private client: Client; private ensureInitialized: EnsureInitialized; diff --git a/packages/typescript/src/api/node/node.infrastructure.ts b/packages/typescript/src/api/node/node.infrastructure.ts index 5ce47a9f3d06f..fedbddff297d0 100644 --- a/packages/typescript/src/api/node/node.infrastructure.ts +++ b/packages/typescript/src/api/node/node.infrastructure.ts @@ -11,6 +11,7 @@ import { HEADER_OFFSET_HASH_LO0, HEADER_OFFSET_HASH_LO1, HEADER_OFFSET_PARSE_OPTIONS, + HEADER_OFFSET_SOURCE_FILE_LEASE, NODE_DATA_TYPE_CHILDREN, NODE_DATA_TYPE_EXTENDED, NODE_DATA_TYPE_STRING, @@ -90,6 +91,17 @@ export function readParseOptionsKey(data: DataView): string { return data.getUint32(HEADER_OFFSET_PARSE_OPTIONS, true).toString(); } +export function readSourceFileLease(data: DataView): number { + const lease = data.getBigUint64(HEADER_OFFSET_SOURCE_FILE_LEASE, true); + if (lease === 0n) { + throw new Error("Source file response has no lease"); + } + if (lease > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new Error(`Source file lease ${lease} exceeds the maximum safe integer`); + } + return Number(lease); +} + function hex8(n: number): string { return (n >>> 0).toString(16).padStart(8, "0"); } diff --git a/packages/typescript/src/api/node/node.ts b/packages/typescript/src/api/node/node.ts index 65247c6d267bd..bf25377d43f2b 100644 --- a/packages/typescript/src/api/node/node.ts +++ b/packages/typescript/src/api/node/node.ts @@ -38,7 +38,7 @@ import { Wtf8Decoder } from "./wtf8.ts"; // Re-export everything consumers need from the other two files. export { RemoteNode, RemoteNodeList } from "./node.generated.ts"; -export { readParseOptionsKey, readSourceFileHash, RemoteNodeBase } from "./node.infrastructure.ts"; +export { readParseOptionsKey, readSourceFileHash, readSourceFileLease, RemoteNodeBase } from "./node.infrastructure.ts"; const sourceFileExtendedDataOffsets = { Text: 0, diff --git a/packages/typescript/src/api/node/protocol.ts b/packages/typescript/src/api/node/protocol.ts index 52ff7e831124f..9218b00d13e3d 100644 --- a/packages/typescript/src/api/node/protocol.ts +++ b/packages/typescript/src/api/node/protocol.ts @@ -1,4 +1,4 @@ -export const PROTOCOL_VERSION = 8; +export const PROTOCOL_VERSION = 9; export const HEADER_OFFSET_METADATA = 0; export const HEADER_OFFSET_HASH_LO0 = 4; @@ -11,7 +11,10 @@ export const HEADER_OFFSET_STRING_TABLE = 28; export const HEADER_OFFSET_EXTENDED_DATA = 32; export const HEADER_OFFSET_STRUCTURED_DATA = 36; export const HEADER_OFFSET_NODES = 40; -export const HEADER_SIZE = 44; +export const HEADER_OFFSET_SOURCE_FILE_ID = 44; +export const HEADER_OFFSET_SOURCE_FILE_LEASE = 52; +export const HEADER_OFFSET_BINDER_DATA = 60; +export const HEADER_SIZE = 64; export const NODE_LEN = 28; diff --git a/packages/typescript/src/api/proto.generated.ts b/packages/typescript/src/api/proto.generated.ts index 864a448f045f5..25b7badfa1a5b 100644 --- a/packages/typescript/src/api/proto.generated.ts +++ b/packages/typescript/src/api/proto.generated.ts @@ -21,6 +21,7 @@ export type APIMethod = { params: TParams; result: TResult; }; export interface APIMethodInfo { release: APIMethod; + releaseSourceFile: APIMethod; batchRequests: APIMethod; initialize: APIMethod; createSnapshot: APIMethod; @@ -201,6 +202,10 @@ export interface ReleaseParams { snapshot: number; } +export interface ReleaseSourceFileParams { + lease: number; +} + export interface BatchRequestsParams { requests: readonly BatchRequest[] | null; continuationToken?: string | undefined; @@ -1186,6 +1191,7 @@ export interface BatchRequest { | "readConfigFile" | "release" | "releaseModuleResolver" + | "releaseSourceFile" | "resolveModuleName" | "resolveName" | "saveHeapProfile" @@ -1354,6 +1360,7 @@ export interface BatchResponse { | "readConfigFile" | "release" | "releaseModuleResolver" + | "releaseSourceFile" | "resolveModuleName" | "resolveName" | "saveHeapProfile" diff --git a/packages/typescript/src/api/sourceFileCache.ts b/packages/typescript/src/api/sourceFileCache.ts index 62ad83eea341a..1dfb3124b8b7c 100644 --- a/packages/typescript/src/api/sourceFileCache.ts +++ b/packages/typescript/src/api/sourceFileCache.ts @@ -7,8 +7,12 @@ import type { SnapshotChanges } from "./proto.ts"; /** * Builds a composite ref key from a snapshot ID and project ID. */ -function refKey(snapshotId: number, projectId: string): string { - return `${snapshotId}:${projectId}`; +function snapshotRefKey(snapshotId: number, projectId: string): string { + return `snapshot:${snapshotId}:${projectId}`; +} + +function leaseRefKey(leaseId: number): string { + return `lease:${leaseId}`; } /** @@ -21,20 +25,19 @@ export interface CachedSourceFile { contentHash: string; /** The parse options key that was used to create this file */ parseOptionsKey: string; - /** Set of (snapshot, project) ref keys that reference this entry */ + /** Set of snapshot/project or direct-lease ref keys that reference this entry */ refs: Set; } /** - * Client-side cache for source files keyed by (path, parseOptionsKey, contentHash). + * Client-side cache for source files keyed by (path, fileName, scriptKind, parseOptionsKey, contentHash). * * Supports multiple versions of the same file at the same path (e.g., from * different snapshots with different file contents). Each version is identified - * by its content hash and parse options key. + * by its script kind, content hash, and parse options key. * - * Entries are ref-counted by (snapshot, project) pairs. When a snapshot is - * disposed, all refs for that snapshot across all projects are released, - * and entries with no remaining references are evicted. + * Entries are ref-counted by (snapshot, project) pairs and direct source-file + * leases. Releasing an owner evicts entries with no remaining references. * * When a new snapshot is created, unchanged cache entries from the previous * snapshot are retained per-project. Only files within changed or removed @@ -45,6 +48,8 @@ export class SourceFileCache { private cache: Map = new Map(); /** Map from snapshotId to (projectId → Set of paths fetched through that project) */ private snapshotProjectPaths: Map>> = new Map(); + /** Map from direct lease ID to its retained path */ + private leasePaths: Map = new Map(); /** * Get a cached source file already retained for the given (snapshot, project) pair. @@ -58,7 +63,7 @@ export class SourceFileCache { getRetained(path: Path, snapshotId: number, projectId: string): SourceFile | undefined { const entries = this.cache.get(path); if (!entries) return undefined; - const key = refKey(snapshotId, projectId); + const key = snapshotRefKey(snapshotId, projectId); const entry = entries.find(e => e.refs.has(key)); return entry?.file; } @@ -68,21 +73,42 @@ export class SourceFileCache { * Returns the cached file — which may be an existing entry if the hash matches. */ set(path: Path, file: SourceFile, parseOptionsKey: string, contentHash: string, snapshotId: number, projectId: string): SourceFile { + const result = this.setWithRef(path, file, parseOptionsKey, contentHash, snapshotRefKey(snapshotId, projectId)); + this.trackPath(snapshotId, projectId, path); + return result; + } + + /** + * Store a source file in the cache and retain it for a direct lease. + * Returns the cached file so leased and program-owned files share identity. + */ + setForLease(path: Path, file: SourceFile, parseOptionsKey: string, contentHash: string, leaseId: number): SourceFile { + if (this.leasePaths.has(leaseId)) { + throw new Error(`Source file lease ${leaseId} is already cached`); + } + const result = this.setWithRef(path, file, parseOptionsKey, contentHash, leaseRefKey(leaseId)); + this.leasePaths.set(leaseId, path); + return result; + } + + private setWithRef(path: Path, file: SourceFile, parseOptionsKey: string, contentHash: string, ref: string): SourceFile { let entries = this.cache.get(path); if (!entries) { entries = []; this.cache.set(path, entries); } - const ref = refKey(snapshotId, projectId); // Check if we already have this exact version - const existing = entries.find(e => e.parseOptionsKey === parseOptionsKey && e.contentHash === contentHash); + const existing = entries.find(e => + e.file.fileName === file.fileName && + e.file.scriptKind === file.scriptKind && + e.parseOptionsKey === parseOptionsKey && + e.contentHash === contentHash + ); if (existing) { existing.refs.add(ref); - this.trackPath(snapshotId, projectId, path); return existing.file; } entries.push({ file, contentHash, parseOptionsKey, refs: new Set([ref]) }); - this.trackPath(snapshotId, projectId, path); return file; } @@ -111,8 +137,8 @@ export class SourceFileCache { for (const p of projectChanges.deletedFiles ?? []) invalidPaths.add(p); } - const prevRef = refKey(previousSnapshotId, projectId); - const newRef = refKey(newSnapshotId, projectId); + const prevRef = snapshotRefKey(previousSnapshotId, projectId); + const newRef = snapshotRefKey(newSnapshotId, projectId); for (const path of paths) { if (invalidPaths?.has(path)) continue; @@ -137,24 +163,35 @@ export class SourceFileCache { const projectMap = this.snapshotProjectPaths.get(snapshotId); if (!projectMap) return; for (const [projectId, paths] of projectMap) { - const key = refKey(snapshotId, projectId); + const key = snapshotRefKey(snapshotId, projectId); for (const path of paths) { - const entries = this.cache.get(path); - if (!entries) continue; - for (let i = entries.length - 1; i >= 0; i--) { - entries[i].refs.delete(key); - if (entries[i].refs.size === 0) { - entries.splice(i, 1); - } - } - if (entries.length === 0) { - this.cache.delete(path); - } + this.releaseRef(path, key); } } this.snapshotProjectPaths.delete(snapshotId); } + releaseLease(leaseId: number): void { + const path = this.leasePaths.get(leaseId); + if (path === undefined) return; + this.releaseRef(path, leaseRefKey(leaseId)); + this.leasePaths.delete(leaseId); + } + + private releaseRef(path: Path, ref: string): void { + const entries = this.cache.get(path); + if (!entries) return; + for (let i = entries.length - 1; i >= 0; i--) { + entries[i].refs.delete(ref); + if (entries[i].refs.size === 0) { + entries.splice(i, 1); + } + } + if (entries.length === 0) { + this.cache.delete(path); + } + } + private trackPath(snapshotId: number, projectId: string, path: Path): void { let projectMap = this.snapshotProjectPaths.get(snapshotId); if (!projectMap) { @@ -175,6 +212,7 @@ export class SourceFileCache { clear(): void { this.cache.clear(); this.snapshotProjectPaths.clear(); + this.leasePaths.clear(); } /** diff --git a/packages/typescript/src/api/sync/api.ts b/packages/typescript/src/api/sync/api.ts index 351dbfc7605a9..04f7f104eb410 100644 --- a/packages/typescript/src/api/sync/api.ts +++ b/packages/typescript/src/api/sync/api.ts @@ -64,6 +64,7 @@ import { parseNodeHandle, readParseOptionsKey, readSourceFileHash, + readSourceFileLease, RemoteSourceFile, } from "../node/node.ts"; import { Wtf8Decoder } from "../node/wtf8.ts"; @@ -346,6 +347,7 @@ export class API implements FormatDiagnosticsHo private initialized: boolean = false; private initializing: void | undefined; private activeSnapshots: Map = new Map(); + private activeSourceFileLeases: Map = new Map(); readonly printer: Printer; readonly internal: InternalAPI; @@ -559,60 +561,90 @@ export class API implements FormatDiagnosticsHo ); } + /** + * Create and retain a source file independently of a program. + * Dispose the returned lease when the source file no longer needs to remain available remotely. + */ get createSourceFile(): { - (fileName: string, sourceText: string, options?: CreateSourceFileOptions): SourceFile; - gen(fileName: string, sourceText: string, options?: CreateSourceFileOptions): Generator; + (fileName: string, sourceText: string, options?: CreateSourceFileOptions): RetainedSourceFile; + gen(fileName: string, sourceText: string, options?: CreateSourceFileOptions): Generator; } { const owner = this; return cacheGeneratorMethod( owner, "createSourceFile", - function (fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): SourceFile { + function (fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): RetainedSourceFile { owner.ensureInitialized(); const data = owner.client.apiRequestBinary("createSourceFile", { fileName, sourceText, options }); if (!data) { throw new Error("createSourceFile returned no source file"); } - return new RemoteSourceFile(data, owner.decoder, owner.client.getTimingCollector()) as unknown as SourceFile; + return owner.retainSourceFileResponse(data); }, - function* (fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): Generator { + function* (fileName: string, sourceText: string, options: CreateSourceFileOptions = {}): Generator { yield* owner.ensureInitialized.gen(); const data = sourceFileResponseToUint8Array(yield* apiRequest("createSourceFile", { fileName, sourceText, options })); if (!data) { throw new Error("createSourceFile returned no source file"); } - return new RemoteSourceFile(data, owner.decoder, owner.client.getTimingCollector()) as unknown as SourceFile; + return owner.retainSourceFileResponse(data); }, ); } + /** + * Read, create, and retain a source file independently of a program. + * Dispose the returned lease when the source file no longer needs to remain available remotely. + */ get createSourceFileFromFile(): { - (file: DocumentIdentifier, options?: CreateSourceFileOptions): SourceFile; - gen(file: DocumentIdentifier, options?: CreateSourceFileOptions): Generator; + (file: DocumentIdentifier, options?: CreateSourceFileOptions): RetainedSourceFile; + gen(file: DocumentIdentifier, options?: CreateSourceFileOptions): Generator; } { const owner = this; return cacheGeneratorMethod( owner, "createSourceFileFromFile", - function (file: DocumentIdentifier, options: CreateSourceFileOptions = {}): SourceFile { + function (file: DocumentIdentifier, options: CreateSourceFileOptions = {}): RetainedSourceFile { owner.ensureInitialized(); const data = owner.client.apiRequestBinary("createSourceFileFromFile", { fileName: resolveFileName(file), options }); if (!data) { throw new Error("createSourceFileFromFile returned no source file"); } - return new RemoteSourceFile(data, owner.decoder, owner.client.getTimingCollector()) as unknown as SourceFile; + return owner.retainSourceFileResponse(data); }, - function* (file: DocumentIdentifier, options: CreateSourceFileOptions = {}): Generator { + function* (file: DocumentIdentifier, options: CreateSourceFileOptions = {}): Generator { yield* owner.ensureInitialized.gen(); const data = sourceFileResponseToUint8Array(yield* apiRequest("createSourceFileFromFile", { fileName: resolveFileName(file), options })); if (!data) { throw new Error("createSourceFileFromFile returned no source file"); } - return new RemoteSourceFile(data, owner.decoder, owner.client.getTimingCollector()) as unknown as SourceFile; + return owner.retainSourceFileResponse(data); }, ); } + private retainSourceFileResponse(data: Uint8Array): RetainedSourceFile { + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + const lease = readSourceFileLease(view); + try { + const decoded = new RemoteSourceFile(data, this.decoder, this.client.getTimingCollector()) as unknown as SourceFile; + const sourceFile = this.sourceFileCache.setForLease(decoded.path, decoded, readParseOptionsKey(view), readSourceFileHash(view), lease); + const retained = new RetainedSourceFile(sourceFile, lease, this.client, () => { + this.activeSourceFileLeases.delete(lease); + this.sourceFileCache.releaseLease(lease); + }); + this.activeSourceFileLeases.set(lease, retained); + return retained; + } + catch (error) { + try { + this.client.apiRequest("releaseSourceFile", { lease }); + } + catch {} + throw error; + } + } + get transpileModule(): { (input: string, options?: TranspileOptions): TranspileOutput; gen(input: string, options?: TranspileOptions): Generator; @@ -960,27 +992,39 @@ export class API implements FormatDiagnosticsHo owner, "close", function (): void { - // Dispose all active snapshots try { - for (const snapshot of [...owner.activeSnapshots.values()]) { - snapshot.dispose(); + for (const retained of [...owner.activeSourceFileLeases.values()]) { + retained.dispose(); } - owner.sourceFileCache.clear(); } finally { - owner.client.close(); // always close the underlying connection + try { + for (const snapshot of [...owner.activeSnapshots.values()]) { + snapshot.dispose(); + } + owner.sourceFileCache.clear(); + } + finally { + owner.client.close(); // always close the underlying connection + } } }, function* (): Generator { - // Dispose all active snapshots try { - for (const snapshot of [...owner.activeSnapshots.values()]) { - yield* snapshot.dispose.gen(); + for (const retained of [...owner.activeSourceFileLeases.values()]) { + yield* retained.dispose.gen(); } - owner.sourceFileCache.clear(); } finally { - owner.client.close(); // always close the underlying connection + try { + for (const snapshot of [...owner.activeSnapshots.values()]) { + yield* snapshot.dispose.gen(); + } + owner.sourceFileCache.clear(); + } + finally { + owner.client.close(); // always close the underlying connection + } } }, ); @@ -1179,6 +1223,75 @@ export class API implements FormatDiagnosticsHo type EnsureInitialized = (() => void) & { gen(): Generator; }; +/** An independently retained source file and its disposable remote-lifetime lease. */ +export class RetainedSourceFile { + readonly sourceFile: SourceFile; + private readonly lease: number; + private readonly client: Client; + private readonly onDispose: () => void; + private disposed = false; + private disposePromise: void | undefined; + + constructor(sourceFile: SourceFile, lease: number, client: Client, onDispose: () => void) { + this.sourceFile = sourceFile; + this.lease = lease; + this.client = client; + this.onDispose = onDispose; + } + + [globalThis.Symbol.dispose](): void { + this.dispose(); + } + + get dispose(): { + (): void; + gen(): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "dispose", + function (): void { + return owner.disposePromise ??= owner.disposeWorker(); + }, + function* (): Generator { + return owner.disposePromise ??= yield* owner.disposeWorker.gen(); + }, + ); + } + + private get disposeWorker(): { + (): void; + gen(): Generator; + } { + const owner = this; + return cacheGeneratorMethod( + owner, + "disposeWorker", + function (): void { + if (owner.disposed) return; + owner.disposed = true; + try { + owner.client.apiRequest("releaseSourceFile", { lease: owner.lease }); + } + finally { + owner.onDispose(); + } + }, + function* (): Generator { + if (owner.disposed) return; + owner.disposed = true; + try { + yield* apiRequest("releaseSourceFile", { lease: owner.lease }); + } + finally { + owner.onDispose(); + } + }, + ); + } +} + export class InternalAPI { private client: Client; private ensureInitialized: EnsureInitialized; diff --git a/packages/typescript/test/async/api.test.ts b/packages/typescript/test/async/api.test.ts index a4d58eb8c0480..a468d5c4286e2 100644 --- a/packages/typescript/test/async/api.test.ts +++ b/packages/typescript/test/async/api.test.ts @@ -379,7 +379,8 @@ describe("API", { concurrency }, () => { test("createSourceFile", async () => { await using api = spawnAPI(); const sourceText = "export const element =
;"; - const sourceFile = await api.createSourceFile("/component.tsx", sourceText); + await using retained = await api.createSourceFile("/component.tsx", sourceText); + const sourceFile = retained.sourceFile; assert.equal(sourceFile.fileName, "/component.tsx"); assert.match(sourceFile.path, /\/component\.tsx$/); assert.equal(sourceFile.text, sourceText); @@ -387,14 +388,35 @@ describe("API", { concurrency }, () => { assert.equal(sourceFile.statements.length, 1); assert.strictEqual(sourceFile.statements[0].parent, sourceFile); - assert.equal((await api.createSourceFile("", "")).scriptKind, ScriptKind.TS); - assert.equal((await api.createSourceFile(".", "")).scriptKind, ScriptKind.TS); + await using retainedAgain = await api.createSourceFile("/component.tsx", sourceText); + assert.strictEqual(retainedAgain.sourceFile, sourceFile); - const overridden = await api.createSourceFile("/component.txt", sourceText, { scriptKind: ScriptKind.TSX }); - assert.equal(overridden.scriptKind, ScriptKind.TSX); - assert.equal(overridden.statements.length, 1); + await using empty = await api.createSourceFile("", ""); + assert.equal(empty.sourceFile.scriptKind, ScriptKind.TS); + await using dot = await api.createSourceFile(".", ""); + assert.equal(dot.sourceFile.scriptKind, ScriptKind.TS); + + await using overridden = await api.createSourceFile("/component.txt", sourceText, { scriptKind: ScriptKind.TSX }); + assert.equal(overridden.sourceFile.scriptKind, ScriptKind.TSX); + assert.equal(overridden.sourceFile.statements.length, 1); + await using defaultKind = await api.createSourceFile("/component.txt", sourceText); + assert.notStrictEqual(defaultKind.sourceFile, overridden.sourceFile); + + await using upperCase = await api.createSourceFile("/CaseSensitive.ts", ""); + await using lowerCase = await api.createSourceFile("/casesensitive.ts", ""); + assert.notStrictEqual(lowerCase.sourceFile, upperCase.sourceFile); + assert.equal(upperCase.sourceFile.fileName, "/CaseSensitive.ts"); + assert.equal(lowerCase.sourceFile.fileName, "/casesensitive.ts"); await assert.rejects(api.createSourceFile("/invalid.ts", "", { scriptKind: 999 as ScriptKind }), /invalid scriptKind 999/); // @sync: assert.throws(() => api.createSourceFile("/invalid.ts", "", { scriptKind: 999 as ScriptKind }), /invalid scriptKind 999/); + + // Each lease can be disposed repeatedly without throwing or releasing another lease. + const firstDispose = retained.dispose(); + const secondDispose = retained.dispose(); + assert.strictEqual(firstDispose, secondDispose); // @sync-skip + await firstDispose; // @sync: retained.dispose(); + await using retainedAfterDispose = await api.createSourceFile("/component.tsx", sourceText); + assert.strictEqual(retainedAfterDispose.sourceFile, retainedAgain.sourceFile); }); test("createSourceFile can be used with a compatible program", async () => { @@ -402,7 +424,8 @@ describe("API", { concurrency }, () => { await using api = spawnAPI({ "/component.tsx": sourceText, }); - const sourceFile = await api.createSourceFile("/component.tsx", sourceText); + await using retained = await api.createSourceFile("/component.tsx", sourceText); + const sourceFile = retained.sourceFile; const snapshot = await api.createSnapshot({ openFiles: ["/component.tsx"] }); const project = snapshot.getProjects()[0]; assert.equal((await api.printer.printNode(sourceFile)).trimEnd(), sourceText); @@ -411,11 +434,24 @@ describe("API", { concurrency }, () => { await snapshot.dispose(); }); + test("createSourceFile shares identity with a matching program file", async () => { + const sourceText = "export declare const element: number;"; + await using api = spawnAPI({ + "/component.d.ts": sourceText, + }); + await using retained = await api.createSourceFile("/component.d.ts", sourceText); + const snapshot = await api.createSnapshot({ openFiles: ["/component.d.ts"] }); + const project = snapshot.getProjects()[0]; + assert.strictEqual(await project.program.getSourceFile("/component.d.ts"), retained.sourceFile); + await snapshot.dispose(); + }); + test("createSourceFileFromFile", async () => { await using api = spawnAPI({ "/input.ts": "export const fromFile = 1;", }); - const fromFile = await api.createSourceFileFromFile({ uri: "file:///input.ts" }); + await using retained = await api.createSourceFileFromFile({ uri: "file:///input.ts" }); + const fromFile = retained.sourceFile; assert.equal(fromFile.fileName, "/input.ts"); assert.equal(fromFile.text, "export const fromFile = 1;"); assert.equal(fromFile.scriptKind, ScriptKind.TS); @@ -2423,6 +2459,39 @@ describe("Snapshot disposal", { concurrency }, () => { assert.ok(snap1.isDisposed()); assert.ok(snap2.isDisposed()); }); + + // @sync-skip-block-start + test("api.close waits for source file disposal already in progress", async () => { + const api = spawnAPI(); + const retained = await api.createSourceFile("/retained.ts", "export {};"); + const client = (api as unknown as { + client: { apiRequest(method: string, params: unknown): Promise; }; + }).client; + const apiRequest = client.apiRequest.bind(client); + let releaseStarted!: () => void; + let finishRelease!: () => void; + const started = new Promise(resolve => releaseStarted = resolve); + const finish = new Promise(resolve => finishRelease = resolve); + client.apiRequest = async (method, params) => { + if (method === "releaseSourceFile") { + releaseStarted(); + await finish; + } + return apiRequest(method, params); + }; + + const disposePromise = retained.dispose(); + await started; + let closed = false; + const closePromise = api.close().then(() => closed = true); + await Promise.resolve(); + assert.equal(closed, false); + finishRelease(); + await disposePromise; + await closePromise; + assert.equal(closed, true); + }); + // @sync-skip-block-end }); describe("Source file cache keying across projects", { concurrency }, () => { diff --git a/packages/typescript/test/encoder.test.ts b/packages/typescript/test/encoder.test.ts index 7570d240300a4..50826e6ca618c 100644 --- a/packages/typescript/test/encoder.test.ts +++ b/packages/typescript/test/encoder.test.ts @@ -43,8 +43,10 @@ import { } from "../src/api/node/node.ts"; import { HEADER_OFFSET_NODES, + HEADER_SIZE, NODE_LEN, NODE_OFFSET_DATA, + PROTOCOL_VERSION, } from "../src/api/node/protocol.ts"; import { Wtf8Decoder } from "../src/api/node/wtf8.ts"; import { areTestsFiltered } from "./testUtils.ts"; @@ -71,7 +73,8 @@ describe("Encoder", { concurrency }, () => { // Verify header const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength); const metadata = view.getUint32(0, true); - assert.strictEqual(metadata >>> 24, 8, "protocol version should be 8"); + assert.strictEqual(metadata >>> 24, PROTOCOL_VERSION); + assert.strictEqual(HEADER_SIZE, 64); // Verify we can decode it const decoded = decode(encoded); @@ -193,11 +196,11 @@ describe("Encoder", { concurrency }, () => { assert.strictEqual(rootKind, SyntaxKind.IfStatement); }); - test("protocol version is 8", () => { + test("protocol version matches", () => { const sf = makeSF("", "/test.ts", []); const encoded = encodeSourceFile(sf); const view = new DataView(encoded.buffer, encoded.byteOffset, encoded.byteLength); - assert.strictEqual(view.getUint32(0, true) >>> 24, 8); + assert.strictEqual(view.getUint32(0, true) >>> 24, PROTOCOL_VERSION); }); test("encodes source files without content mapping metadata", () => { diff --git a/packages/typescript/test/sync/api-generators.test.ts b/packages/typescript/test/sync/api-generators.test.ts index 81ada366568c7..d25bc6435e683 100644 --- a/packages/typescript/test/sync/api-generators.test.ts +++ b/packages/typescript/test/sync/api-generators.test.ts @@ -44,6 +44,7 @@ import { type NodeHandle, type Program, type Project, + type RetainedSourceFile, type Signature, SignatureKind, type Snapshot, @@ -314,6 +315,16 @@ function assertSourceFilesEquivalent(actual: SourceFile, expected: SourceFile, m assert.equal(actual.text, expected.text, message); } +function assertRetainedSourceFilesEquivalent(actual: RetainedSourceFile, expected: RetainedSourceFile, message?: string): void { + try { + assertSourceFilesEquivalent(actual.sourceFile, expected.sourceFile, message); + } + finally { + actual.dispose(); + expected.dispose(); + } +} + function assertOptionalSourceFilesEquivalent(actual: SourceFile | undefined, expected: SourceFile | undefined, message?: string): void { assertOptionalEquivalent(actual, expected, assertSourceFilesEquivalent, message); } @@ -427,8 +438,12 @@ describe("API - generator batching", { concurrency: areTestsFiltered() }, () => api.createSourceFile.gen("/generated.ts", "export const generated = true;"), api.createSourceFileFromFile.gen("/src/index.ts"), )); - assert.equal(fromText.text, "export const generated = true;"); - assert.equal(fromFile.text, parityFiles["/src/index.ts"]); + context.after(() => { + fromText.dispose(); + fromFile.dispose(); + }); + assert.equal(fromText.sourceFile.text, "export const generated = true;"); + assert.equal(fromFile.sourceFile.text, parityFiles["/src/index.ts"]); assert.deepEqual(requestBatches, [ ["initialize"], ["createSourceFile", "createSourceFileFromFile"], @@ -1540,8 +1555,8 @@ describe("API - generator batching", { concurrency: areTestsFiltered() }, () => parityCase("API", "readConfigFile", api.readConfigFile, assertDeepEquivalent, "/tsconfig.json"), parityCase("API", "parseJsonConfigFileContent", api.parseJsonConfigFileContent, assertDeepEquivalent, { compilerOptions: { strict: true } }, { configDirectory: "/" }), parityCase("API", "parseJsonConfigFileContent", api.parseJsonConfigFileContent, assertDeepEquivalent, { extends: "./base.json" }, { configFileName: "/tsconfig.json" }), - parityCase("API", "createSourceFile", api.createSourceFile, assertSourceFilesEquivalent, "/generated.ts", "export const generated = true;"), - parityCase("API", "createSourceFileFromFile", api.createSourceFileFromFile, assertSourceFilesEquivalent, "/src/index.ts"), + parityCase("API", "createSourceFile", api.createSourceFile, assertRetainedSourceFilesEquivalent, "/generated.ts", "export const generated = true;"), + parityCase("API", "createSourceFileFromFile", api.createSourceFileFromFile, assertRetainedSourceFilesEquivalent, "/src/index.ts"), parityCase("API", "transpileModule", api.transpileModule, assertDeepEquivalent, "export const value: number = 1;", { compilerOptions: { module: 99 } }), parityCase("API", "transpileModuleFromFile", api.transpileModuleFromFile, assertDeepEquivalent, "/src/index.ts"), parityCase("API", "transpileDeclaration", api.transpileDeclaration, assertDeepEquivalent, "export function declared(value: string): number { return value.length; }"), diff --git a/packages/typescript/test/sync/api.test.ts b/packages/typescript/test/sync/api.test.ts index 977687b3365fe..e5f8ec43ece02 100644 --- a/packages/typescript/test/sync/api.test.ts +++ b/packages/typescript/test/sync/api.test.ts @@ -368,7 +368,8 @@ describe("API", { concurrency }, () => { test("createSourceFile", () => { using api = spawnAPI(); const sourceText = "export const element =
;"; - const sourceFile = api.createSourceFile("/component.tsx", sourceText); + using retained = api.createSourceFile("/component.tsx", sourceText); + const sourceFile = retained.sourceFile; assert.equal(sourceFile.fileName, "/component.tsx"); assert.match(sourceFile.path, /\/component\.tsx$/); assert.equal(sourceFile.text, sourceText); @@ -376,14 +377,34 @@ describe("API", { concurrency }, () => { assert.equal(sourceFile.statements.length, 1); assert.strictEqual(sourceFile.statements[0].parent, sourceFile); - assert.equal((api.createSourceFile("", "")).scriptKind, ScriptKind.TS); - assert.equal((api.createSourceFile(".", "")).scriptKind, ScriptKind.TS); + using retainedAgain = api.createSourceFile("/component.tsx", sourceText); + assert.strictEqual(retainedAgain.sourceFile, sourceFile); - const overridden = api.createSourceFile("/component.txt", sourceText, { scriptKind: ScriptKind.TSX }); - assert.equal(overridden.scriptKind, ScriptKind.TSX); - assert.equal(overridden.statements.length, 1); + using empty = api.createSourceFile("", ""); + assert.equal(empty.sourceFile.scriptKind, ScriptKind.TS); + using dot = api.createSourceFile(".", ""); + assert.equal(dot.sourceFile.scriptKind, ScriptKind.TS); + + using overridden = api.createSourceFile("/component.txt", sourceText, { scriptKind: ScriptKind.TSX }); + assert.equal(overridden.sourceFile.scriptKind, ScriptKind.TSX); + assert.equal(overridden.sourceFile.statements.length, 1); + using defaultKind = api.createSourceFile("/component.txt", sourceText); + assert.notStrictEqual(defaultKind.sourceFile, overridden.sourceFile); + + using upperCase = api.createSourceFile("/CaseSensitive.ts", ""); + using lowerCase = api.createSourceFile("/casesensitive.ts", ""); + assert.notStrictEqual(lowerCase.sourceFile, upperCase.sourceFile); + assert.equal(upperCase.sourceFile.fileName, "/CaseSensitive.ts"); + assert.equal(lowerCase.sourceFile.fileName, "/casesensitive.ts"); assert.throws(() => api.createSourceFile("/invalid.ts", "", { scriptKind: 999 as ScriptKind }), /invalid scriptKind 999/); + + // Each lease can be disposed repeatedly without throwing or releasing another lease. + const firstDispose = retained.dispose(); + const secondDispose = retained.dispose(); + retained.dispose(); + using retainedAfterDispose = api.createSourceFile("/component.tsx", sourceText); + assert.strictEqual(retainedAfterDispose.sourceFile, retainedAgain.sourceFile); }); test("createSourceFile can be used with a compatible program", () => { @@ -391,7 +412,8 @@ describe("API", { concurrency }, () => { using api = spawnAPI({ "/component.tsx": sourceText, }); - const sourceFile = api.createSourceFile("/component.tsx", sourceText); + using retained = api.createSourceFile("/component.tsx", sourceText); + const sourceFile = retained.sourceFile; const snapshot = api.createSnapshot({ openFiles: ["/component.tsx"] }); const project = snapshot.getProjects()[0]; assert.equal((api.printer.printNode(sourceFile)).trimEnd(), sourceText); @@ -400,11 +422,24 @@ describe("API", { concurrency }, () => { snapshot.dispose(); }); + test("createSourceFile shares identity with a matching program file", () => { + const sourceText = "export declare const element: number;"; + using api = spawnAPI({ + "/component.d.ts": sourceText, + }); + using retained = api.createSourceFile("/component.d.ts", sourceText); + const snapshot = api.createSnapshot({ openFiles: ["/component.d.ts"] }); + const project = snapshot.getProjects()[0]; + assert.strictEqual(project.program.getSourceFile("/component.d.ts"), retained.sourceFile); + snapshot.dispose(); + }); + test("createSourceFileFromFile", () => { using api = spawnAPI({ "/input.ts": "export const fromFile = 1;", }); - const fromFile = api.createSourceFileFromFile({ uri: "file:///input.ts" }); + using retained = api.createSourceFileFromFile({ uri: "file:///input.ts" }); + const fromFile = retained.sourceFile; assert.equal(fromFile.fileName, "/input.ts"); assert.equal(fromFile.text, "export const fromFile = 1;"); assert.equal(fromFile.scriptKind, ScriptKind.TS); diff --git a/tsc/internal/api/encoder/encoder.go b/tsc/internal/api/encoder/encoder.go index d76f67beef3fe..0f90a4b0fac68 100644 --- a/tsc/internal/api/encoder/encoder.go +++ b/tsc/internal/api/encoder/encoder.go @@ -59,11 +59,16 @@ const ( HeaderOffsetExtendedData HeaderOffsetStructuredData HeaderOffsetNodes + HeaderOffsetSourceFileID + _ + HeaderOffsetSourceFileLease + _ + HeaderOffsetBinderData HeaderSize ) const ( - ProtocolVersion uint8 = 8 + ProtocolVersion uint8 = 9 ) // Source File Binary Format @@ -79,14 +84,14 @@ const ( // // | Section | Length | Description | // | ------------------ | ------------------ | ----------------------------------------------------------------------------------------------- | -// | Header | 44 bytes | Contains the content hash, parse options, flags, and byte offsets to the start of each section. | +// | Header | 64 bytes | Contains the content hash, parse options, flags, file metadata, and byte offsets to the start of each section. | // | String offsets | 8 bytes per string | Pairs of starting byte offsets and ending byte offsets into the **string data** section. | // | String data | variable | UTF-8 encoded string data. | // | Extended node data | variable | Extra data for some kinds of nodes. | // | Structured data | variable | Msgpack-encoded metadata blobs (e.g. file references). | // | Nodes | 28 bytes per node | Defines the AST structure of the file, with references to strings and extended data. | // -// Header (44 bytes) +// Header (64 bytes) // ----------------- // // The header contains the following fields: @@ -102,6 +107,9 @@ const ( // | 32-35 | uint32 | Byte offset to extended node data section | // | 36-39 | uint32 | Byte offset to structured data section | // | 40-43 | uint32 | Byte offset to nodes section | +// | 44-51 | uint64 | Source file ID (0 = none) | +// | 52-59 | uint64 | Source file lease ID (0 = none) | +// | 60-63 | uint32 | Byte offset to binder data (0 = none) | // // String offsets (8 bytes per string) // ----------------------------------- @@ -422,6 +430,11 @@ func EncodeSourceFile(sourceFile *ast.SourceFile) ([]byte, *NodeIndexTable, erro return data, nodeTable, nil } +// SetSourceFileLease sets the session-scoped lease ID in an encoded source file. +func SetSourceFileLease(data []byte, lease uint64) { + binary.LittleEndian.PutUint64(data[HeaderOffsetSourceFileLease:], lease) +} + // EncodeNode encodes an arbitrary AST node and its descendants into the binary format. // The sourceFile is needed to provide the source text for efficient string encoding. // When encoding a non-SourceFile node, the header hash and parse options fields will be zero. @@ -621,6 +634,9 @@ func encodeTree(rootNode *ast.Node, sourceFile *ast.SourceFile) ([]byte, *NodeIn uint32(offsetExtendedData), uint32(offsetStructuredData), uint32(offsetNodes), + 0, 0, // source file ID + 0, 0, // source file lease ID + 0, // binder data offset } var headerBytes, strsBytes []byte diff --git a/tsc/internal/api/encoder/encoder_test.go b/tsc/internal/api/encoder/encoder_test.go index 4ac3e5ba036a6..1731ac8a05097 100644 --- a/tsc/internal/api/encoder/encoder_test.go +++ b/tsc/internal/api/encoder/encoder_test.go @@ -37,8 +37,8 @@ func TestEncodeSourceFile(t *testing.T) { func TestEncodeContentMapperSourceFileMetadata(t *testing.T) { t.Parallel() - if encoder.ProtocolVersion != 8 { - t.Fatalf("protocol version = %d, want 8", encoder.ProtocolVersion) + if encoder.ProtocolVersion != 9 { + t.Fatalf("protocol version = %d, want 9", encoder.ProtocolVersion) } sourceFile := parser.ParseSourceFile(ast.SourceFileParseOptions{ FileName: "/component.vue", diff --git a/tsc/internal/api/proto.go b/tsc/internal/api/proto.go index 936b3cd0107c7..91413d3132af7 100644 --- a/tsc/internal/api/proto.go +++ b/tsc/internal/api/proto.go @@ -34,12 +34,13 @@ var ( type Method string type ( - SnapshotID uint64 - ModuleResolverID uint64 - SymbolID uint64 - TypeID uint32 - SignatureID uint64 - NodeHandle string + SnapshotID uint64 + ModuleResolverID uint64 + SourceFileLeaseID uint64 + SymbolID uint64 + TypeID uint32 + SignatureID uint64 + NodeHandle string ) func SymbolHandle(symbol *ast.Symbol) SymbolID { @@ -55,7 +56,8 @@ func SignatureHandle(sig *checker.Signature) SignatureID { } const ( - MethodRelease Method = "release" + MethodRelease Method = "release" + MethodReleaseSourceFile Method = "releaseSourceFile" MethodBatchRequests Method = "batchRequests" @@ -553,6 +555,7 @@ type OpenedFileOperationResult struct { var unmarshalers = map[Method]func([]byte) (any, error){ MethodBatchRequests: unmarshallerFor[BatchRequestsParams], MethodRelease: unmarshallerFor[ReleaseParams], + MethodReleaseSourceFile: unmarshallerFor[ReleaseSourceFileParams], MethodInitialize: noParams, MethodCreateSnapshot: unmarshallerFor[CreateSnapshotParams], MethodUpdateSnapshot: unmarshallerFor[UpdateSnapshotParams], @@ -867,6 +870,10 @@ type ReleaseParams struct { Snapshot SnapshotID `json:"snapshot"` } +type ReleaseSourceFileParams struct { + Lease SourceFileLeaseID `json:"lease"` +} + type ProfileParams struct { Dir string `json:"dir"` } diff --git a/tsc/internal/api/session.go b/tsc/internal/api/session.go index d240576da24ce..19ff6f88c1c9e 100644 --- a/tsc/internal/api/session.go +++ b/tsc/internal/api/session.go @@ -31,7 +31,6 @@ import ( "github.com/microsoft/TypeScript/tsc/internal/lsp/lsproto" "github.com/microsoft/TypeScript/tsc/internal/module" "github.com/microsoft/TypeScript/tsc/internal/nodebuilder" - "github.com/microsoft/TypeScript/tsc/internal/parser" "github.com/microsoft/TypeScript/tsc/internal/pprof" "github.com/microsoft/TypeScript/tsc/internal/printer" "github.com/microsoft/TypeScript/tsc/internal/project" @@ -442,6 +441,9 @@ type Session struct { nextProgramResolutionContextID atomic.Uint64 programResolutionContexts map[uint64]*programResolutionContext programResolutionContextsMu sync.RWMutex + sourceFileLeases map[SourceFileLeaseID]*project.SourceFileLease + sourceFileLeasesMu sync.Mutex + nextSourceFileLeaseID atomic.Uint64 conn ipc.Conn cpuProfiler pprof.CPUProfiler @@ -491,6 +493,7 @@ func newSession(snapshotHost *project.SnapshotHost, withLocale func(context.Cont snapshots: make(map[SnapshotID]*snapshotData), moduleResolvers: make(map[ModuleResolverID]*moduleResolverRegistration), programResolutionContexts: make(map[uint64]*programResolutionContext), + sourceFileLeases: make(map[SourceFileLeaseID]*project.SourceFileLease), } if options != nil { s.useBinaryResponses = options.UseBinaryResponses @@ -699,6 +702,8 @@ func (s *Session) HandleRequest(ctx context.Context, method string, params json. return s.handleBatchRequests(ctx, parsed.(*BatchRequestsParams)) case string(MethodRelease): return s.handleRelease(ctx, parsed.(*ReleaseParams)) + case string(MethodReleaseSourceFile): + return s.handleReleaseSourceFile(parsed.(*ReleaseSourceFileParams)) case string(MethodInitialize): return s.handleInitialize(ctx) case string(MethodCreateSnapshot): @@ -1674,11 +1679,11 @@ func (s *Session) handleTranspile(ctx context.Context, params *TranspileParams, // @gen-proto-result: SourceFileResponse func (s *Session) handleCreateSourceFile(ctx context.Context, params *CreateSourceFileParams) (any, error) { - sourceFile, err := s.createSourceFile(params.FileName, params.SourceText, params.Options) + lease, err := s.createSourceFile(params.FileName, params.SourceText, params.Options) if err != nil { return nil, err } - return s.encodeSourceFileResponse(sourceFile) + return s.encodeLeasedSourceFile(lease) } // @gen-proto-result: SourceFileResponse @@ -1688,14 +1693,14 @@ func (s *Session) handleCreateSourceFileFromFile(ctx context.Context, params *Cr if !ok { return nil, fmt.Errorf("%w: could not read file %q", ErrClientError, fileName) } - sourceFile, err := s.createSourceFile(fileName, sourceText, params.Options) + lease, err := s.createSourceFile(fileName, sourceText, params.Options) if err != nil { return nil, err } - return s.encodeSourceFileResponse(sourceFile) + return s.encodeLeasedSourceFile(lease) } -func (s *Session) createSourceFile(fileName string, sourceText string, options CreateSourceFileOptions) (*ast.SourceFile, error) { +func (s *Session) createSourceFile(fileName string, sourceText string, options CreateSourceFileOptions) (*project.SourceFileLease, error) { scriptKind := options.ScriptKind if scriptKind == core.ScriptKindUnknown { scriptKind = core.EnsureScriptKindFromFileName(fileName) @@ -1704,12 +1709,63 @@ func (s *Session) createSourceFile(fileName string, sourceText string, options C return nil, fmt.Errorf("%w: invalid scriptKind %d", ErrClientError, scriptKind) } fileName = tspath.GetNormalizedAbsolutePath(fileName, s.GetCurrentDirectory()) - return parser.ParseSourceFile(ast.SourceFileParseOptions{ + return s.acquireSourceFile(ast.SourceFileParseOptions{ FileName: fileName, Path: s.toPath(fileName), }, sourceText, scriptKind), nil } +func (s *Session) acquireSourceFile(options ast.SourceFileParseOptions, sourceText string, scriptKind core.ScriptKind) *project.SourceFileLease { + return s.snapshotHost.AcquireSourceFile(options, sourceText, scriptKind) +} + +func (s *Session) encodeLeasedSourceFile(lease *project.SourceFileLease) (any, error) { + data, _, err := encoder.EncodeSourceFile(lease.SourceFile()) + if err != nil { + lease.Release() + return nil, fmt.Errorf("failed to encode source file: %w", err) + } + id := SourceFileLeaseID(s.nextSourceFileLeaseID.Add(1)) + encoder.SetSourceFileLease(data, uint64(id)) + s.sourceFileLeasesMu.Lock() + s.sourceFileLeases[id] = lease + s.sourceFileLeasesMu.Unlock() + if s.useBinaryResponses { + return RawBinary(data), nil + } + return &SourceFileResponse{Data: base64.StdEncoding.EncodeToString(data)}, nil +} + +func (s *Session) handleReleaseSourceFile(params *ReleaseSourceFileParams) (any, error) { + if params == nil || params.Lease == 0 { + return nil, fmt.Errorf("%w: empty source file lease", ErrClientError) + } + s.sourceFileLeasesMu.Lock() + lease := s.sourceFileLeases[params.Lease] + if lease != nil { + delete(s.sourceFileLeases, params.Lease) + } + s.sourceFileLeasesMu.Unlock() + if lease == nil { + return nil, fmt.Errorf("%w: source file lease %d not found", ErrClientError, params.Lease) + } + lease.Release() + return true, nil +} + +func (s *Session) releaseSourceFileLeases() { + s.sourceFileLeasesMu.Lock() + leases := make([]*project.SourceFileLease, 0, len(s.sourceFileLeases)) + for _, lease := range s.sourceFileLeases { + leases = append(leases, lease) + } + clear(s.sourceFileLeases) + s.sourceFileLeasesMu.Unlock() + for _, lease := range leases { + lease.Release() + } +} + func isValidCreateSourceFileScriptKind(scriptKind core.ScriptKind) bool { switch scriptKind { case core.ScriptKindJS, core.ScriptKindJSX, core.ScriptKindTS, core.ScriptKindTSX, core.ScriptKindJSON: @@ -4473,6 +4529,7 @@ func (s *Session) createSnapshotOperationResponse(snapshot *project.Snapshot, re func (s *Session) Close() { s.closeOnce.Do(func() { s.releaseLanguageServerRefs() + s.releaseSourceFileLeases() s.snapshotsMu.Lock() snapshots := make([]*project.Snapshot, 0, len(s.snapshots)) diff --git a/tsc/internal/api/session_createsourcefile_test.go b/tsc/internal/api/session_createsourcefile_test.go index 3b10071c5a89e..077c17e832922 100644 --- a/tsc/internal/api/session_createsourcefile_test.go +++ b/tsc/internal/api/session_createsourcefile_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/microsoft/TypeScript/tsc/internal/core" + "github.com/microsoft/TypeScript/tsc/internal/lsp/lsproto" "github.com/microsoft/TypeScript/tsc/internal/testutil/projecttestutil" "gotest.tools/v3/assert" ) @@ -22,43 +23,114 @@ func TestCreateSourceFile(t *testing.T) { t.Run("text", func(t *testing.T) { t.Parallel() - sourceFile, err := session.createSourceFile( + lease, err := session.createSourceFile( "src/input.tsx", `export const element =
;`, CreateSourceFileOptions{}, ) assert.NilError(t, err) + t.Cleanup(lease.Release) + sourceFile := lease.SourceFile() assert.Equal(t, sourceFile.FileName(), "/src/input.tsx") assert.Equal(t, string(sourceFile.Path()), "/src/input.tsx") assert.Equal(t, sourceFile.Text(), `export const element =
;`) assert.Equal(t, sourceFile.ScriptKind, core.ScriptKindTSX) assert.Equal(t, len(sourceFile.Statements.Nodes), 1) - assert.Assert(t, !sourceFile.IsBound()) + assert.Assert(t, sourceFile.IsBound()) }) t.Run("script kind override", func(t *testing.T) { t.Parallel() - sourceFile, err := session.createSourceFile( + lease, err := session.createSourceFile( "/src/component.txt", `export const element =
;`, CreateSourceFileOptions{ScriptKind: core.ScriptKindTSX}, ) assert.NilError(t, err) + t.Cleanup(lease.Release) + sourceFile := lease.SourceFile() assert.Equal(t, sourceFile.ScriptKind, core.ScriptKindTSX) assert.Equal(t, len(sourceFile.Diagnostics()), 0) }) + t.Run("shares parse cache with programs", func(t *testing.T) { + t.Parallel() + + const fileName = "/src/shared.ts" + const sourceText = `export const shared = 1;` + cacheProjectSession, _ := projecttestutil.Setup(map[string]any{fileName: sourceText}) + t.Cleanup(cacheProjectSession.Close) + cacheSession := NewLSPSession(cacheProjectSession, nil) + t.Cleanup(cacheSession.Close) + + cacheProjectSession.DidOpenFile(context.Background(), "file:///src/shared.ts", 1, sourceText, lsproto.LanguageKindTypeScript) + languageService, err := cacheProjectSession.GetLanguageService(context.Background(), "file:///src/shared.ts") + assert.NilError(t, err) + cacheProjectSession.WaitForBackgroundTasks() + + programFile := languageService.GetProgram().GetSourceFile(fileName) + direct := cacheSession.acquireSourceFile(programFile.ParseOptions(), sourceText, programFile.ScriptKind) + t.Cleanup(direct.Release) + assert.Assert(t, programFile == direct.SourceFile()) + }) + + t.Run("lease release", func(t *testing.T) { + t.Parallel() + + findLease := func(fileName string) SourceFileLeaseID { + session.sourceFileLeasesMu.Lock() + defer session.sourceFileLeasesMu.Unlock() + for id, lease := range session.sourceFileLeases { + if lease.SourceFile().FileName() == fileName { + return id + } + } + return 0 + } + + first, err := session.handleCreateSourceFile(context.Background(), &CreateSourceFileParams{ + FileName: "/src/lease-1.ts", + SourceText: "export {};", + }) + assert.NilError(t, err) + assert.Assert(t, first != nil) + firstLease := findLease("/src/lease-1.ts") + assert.Assert(t, firstLease != 0) + + second, err := session.handleCreateSourceFile(context.Background(), &CreateSourceFileParams{ + FileName: "/src/lease-2.ts", + SourceText: "export {};", + }) + assert.NilError(t, err) + assert.Assert(t, second != nil) + secondLease := findLease("/src/lease-2.ts") + assert.Assert(t, secondLease != 0) + assert.Assert(t, firstLease != secondLease) + + _, err = session.handleReleaseSourceFile(&ReleaseSourceFileParams{Lease: firstLease}) + assert.NilError(t, err) + assert.Equal(t, findLease("/src/lease-1.ts"), SourceFileLeaseID(0)) + + _, err = session.handleReleaseSourceFile(&ReleaseSourceFileParams{Lease: firstLease}) + assert.ErrorContains(t, err, "source file lease") + + _, err = session.handleReleaseSourceFile(&ReleaseSourceFileParams{Lease: secondLease}) + assert.NilError(t, err) + }) + t.Run("unknown extension defaults to TypeScript", func(t *testing.T) { t.Parallel() - sourceFile, err := session.createSourceFile( + lease, err := session.createSourceFile( "/src/component.txt", `export const value: string = "ok";`, CreateSourceFileOptions{}, ) assert.NilError(t, err) + t.Cleanup(lease.Release) + sourceFile := lease.SourceFile() assert.Equal(t, sourceFile.ScriptKind, core.ScriptKindTS) assert.Equal(t, len(sourceFile.Diagnostics()), 0) }) diff --git a/tsc/internal/project/snapshothost.go b/tsc/internal/project/snapshothost.go index 3f2d97d52ca77..59f95187c7d26 100644 --- a/tsc/internal/project/snapshothost.go +++ b/tsc/internal/project/snapshothost.go @@ -3,9 +3,12 @@ package project import ( "context" "slices" + "sync" "sync/atomic" + "github.com/microsoft/TypeScript/tsc/internal/ast" "github.com/microsoft/TypeScript/tsc/internal/contentmapper" + "github.com/microsoft/TypeScript/tsc/internal/core" "github.com/microsoft/TypeScript/tsc/internal/ls/lsutil" "github.com/microsoft/TypeScript/tsc/internal/lsp/lsproto" "github.com/microsoft/TypeScript/tsc/internal/project/logging" @@ -28,10 +31,37 @@ type SnapshotHost struct { snapshotID atomic.Uint64 } +type SourceFileLease struct { + cache *ParseCache + key ParseCacheKey + sourceFile *ast.SourceFile + releaseOnce sync.Once +} + +func (l *SourceFileLease) SourceFile() *ast.SourceFile { + return l.sourceFile +} + +func (l *SourceFileLease) Release() { + l.releaseOnce.Do(func() { + l.cache.Deref(l.key) + }) +} + func (s *SnapshotHost) nextSnapshotID() uint64 { return s.snapshotID.Add(1) } +func (s *SnapshotHost) AcquireSourceFile(options ast.SourceFileParseOptions, text string, scriptKind core.ScriptKind) *SourceFileLease { + fileHandle := NewCachedFileHandle(options.FileName, text) + key := NewParseCacheKey(options, fileHandle.Hash(), scriptKind) + return &SourceFileLease{ + cache: s.parseCache, + key: key, + sourceFile: s.parseCache.Acquire(key, fileHandle), + } +} + func NewSnapshotHost(init *SessionInit) *SnapshotHost { currentDirectory := init.Options.CurrentDirectory useCaseSensitiveFileNames := init.FS.UseCaseSensitiveFileNames()