Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion packages/cli/src/analyzers/analysis/fileScanner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { Dirent, Stats } from "node:fs";
import { readdir, readFile, stat } from "node:fs/promises";
import { extname, join, relative } from "node:path";
import pLimit from "p-limit";
import { shouldIgnorePath } from "./filterEngine.js";
import { clearGitignoreCache, shouldIgnorePath } from "./filterEngine.js";

export type ScannedFile = {
path: string;
Expand All @@ -17,6 +17,9 @@ export async function scanFiles(projectRoot: string): Promise<ScannedFile[]> {
const files: ScannedFile[] = [];
const limit = pLimit(50);

// Ignore files are cached per process; re-read them for every scan.
clearGitignoreCache();

// The limiter only guards actual filesystem I/O (readdir, stat + readFile).
// visit() must NOT run inside limit(): a directory task that awaits its
// children while holding a slot starves those children once 50 directories
Expand Down
89 changes: 60 additions & 29 deletions packages/cli/src/analyzers/analysis/filterEngine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,41 +58,75 @@ const IGNORED_FILES = new Set([
"bun.lockb"
]);

const gitignoreCache = new Map<string, ReturnType<typeof ignore>>();
type IgnoreMatcher = ReturnType<typeof ignore>;

function loadGitignore(projectRoot: string): void {
const resolvedRoot = path.resolve(projectRoot);
if (gitignoreCache.has(resolvedRoot)) return;
// One matcher per directory that holds a .gitignore (null = checked, none).
const gitignoreCache = new Map<string, IgnoreMatcher | null>();
const rootMatcherCache = new Map<string, IgnoreMatcher>();

const ig = ignore();
/** Forget cached ignore files, e.g. before a fresh scan of an edited project. */
export function clearGitignoreCache(): void {
gitignoreCache.clear();
rootMatcherCache.clear();
}

function readIgnoreFile(directory: string): IgnoreMatcher | null {
const cached = gitignoreCache.get(directory);
if (cached !== undefined) return cached;

const gitignorePath = path.join(resolvedRoot, ".gitignore");
let matcher: IgnoreMatcher | null = null;
try {
ig.add(fs.readFileSync(gitignorePath, "utf8"));
matcher = ignore().add(fs.readFileSync(path.join(directory, ".gitignore"), "utf8"));
} catch {
// no .gitignore
// no .gitignore in this directory
}

const excludePath = path.join(resolvedRoot, ".git", "info", "exclude");
try {
ig.add(fs.readFileSync(excludePath, "utf8"));
} catch {
// no .git/info/exclude
gitignoreCache.set(directory, matcher);
return matcher;
}

// Root rules: .git/info/exclude has the lowest precedence, then the root .gitignore.
function getRootMatcher(resolvedRoot: string): IgnoreMatcher {
const cached = rootMatcherCache.get(resolvedRoot);
if (cached) return cached;

const ig = ignore();
for (const ruleFile of [path.join(".git", "info", "exclude"), ".gitignore"]) {
try {
ig.add(fs.readFileSync(path.join(resolvedRoot, ruleFile), "utf8"));
} catch {
// rule file not present
}
}

gitignoreCache.set(resolvedRoot, ig);
rootMatcherCache.set(resolvedRoot, ig);
return ig;
}

function getGitignore(projectRoot: string): ReturnType<typeof ignore> {
const resolvedRoot = path.resolve(projectRoot);
if (!gitignoreCache.has(resolvedRoot)) {
loadGitignore(resolvedRoot);
// Git applies each .gitignore relative to its own directory, and rules in a
// deeper file override rules from the directories above it.
function isGitIgnored(p: string, isDirectory: boolean, resolvedRoot: string): boolean {
const target = isDirectory ? `${p}/` : p;
const rootResult = getRootMatcher(resolvedRoot).test(target);
let ignored = rootResult.ignored && !rootResult.unignored;

const segments = p.split("/");
for (let depth = 1; depth < segments.length; depth += 1) {
const matcher = readIgnoreFile(path.join(resolvedRoot, ...segments.slice(0, depth)));
if (!matcher) continue;

const result = matcher.test(`${segments.slice(depth).join("/")}${isDirectory ? "/" : ""}`);
if (result.ignored) ignored = true;
else if (result.unignored) ignored = false;
}
return gitignoreCache.get(resolvedRoot)!;

return ignored;
}

const ENV_FILE_SEGMENT = /^\.env(\.|$)/;

export function shouldIgnorePath(p: string, isDirectory: boolean, projectRoot?: string): boolean {
const resolvedRoot = projectRoot ?? process.cwd();
const resolvedRoot = path.resolve(projectRoot ?? process.cwd());

const segments = p.split("/");
if (segments.some((segment) => IGNORED_DIRECTORIES.has(segment))) {
Expand All @@ -103,22 +137,19 @@ export function shouldIgnorePath(p: string, isDirectory: boolean, projectRoot?:
return true;
}

if (p.startsWith(".env") || p.includes("/.env")) {
return true;
}

if (p.includes("public/assets/") || p.endsWith(".min.js") || p.endsWith(".min.ts")) {
// .env, .env.local, ... but not look-alikes such as .envrc or .environment.ts
if (segments.some((segment) => ENV_FILE_SEGMENT.test(segment))) {
return true;
}

if (!isDirectory && [...IGNORED_EXTENSIONS].some((ext) => p.endsWith(ext))) {
const lowerPath = p.toLowerCase();
if (/(^|\/)public\/assets\//.test(lowerPath) || lowerPath.endsWith(".min.js") || lowerPath.endsWith(".min.ts")) {
return true;
}

const ig = getGitignore(resolvedRoot);
if (ig.ignores(isDirectory ? `${p}/` : p)) {
if (!isDirectory && [...IGNORED_EXTENSIONS].some((ext) => lowerPath.endsWith(ext))) {
return true;
}

return false;
return isGitIgnored(p, isDirectory, resolvedRoot);
}
4 changes: 4 additions & 0 deletions packages/cli/src/analyzers/analysis/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ export {
shouldIgnorePath,
} from "./filterEngine.js";

export {
parseJsonText,
} from "./jsonText.js";

export {
PrismaExtractor,
} from "./extractors/prismaExtractor.js";
Expand Down
8 changes: 8 additions & 0 deletions packages/cli/src/analyzers/analysis/jsonText.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/**
* JSON.parse that tolerates a leading UTF-8 byte order mark. Windows editors
* write BOM-prefixed package.json files that npm accepts but plain JSON.parse
* rejects, which silently emptied every manifest-based detector.
*/
export function parseJsonText(content: string): unknown {
return JSON.parse(content.charCodeAt(0) === 0xfeff ? content.slice(1) : content);
}
59 changes: 46 additions & 13 deletions packages/cli/src/analyzers/analysis/tsMorphAnalyzer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,12 +94,29 @@ export class TsMorphAnalyzer implements FileAnalyzer {
if (specifier) imports.add(specifier);
}
for (const call of sourceFile.getDescendantsOfKind(SyntaxKind.CallExpression)) {
if (call.getExpression().getText() !== "require") continue;
const callee = call.getExpression();
// require("x") and dynamic import("x"); lazy routes and code splitting
// are only reachable through the latter.
if (callee.getKind() !== SyntaxKind.ImportKeyword && callee.getText() !== "require") continue;
const argument = call.getArguments()[0];
if (argument && Node.isStringLiteral(argument)) {
if (argument && (Node.isStringLiteral(argument) || Node.isNoSubstitutionTemplateLiteral(argument))) {
imports.add(argument.getLiteralValue());
}
}
for (const declaration of sourceFile.getDescendantsOfKind(SyntaxKind.ImportEqualsDeclaration)) {
const reference = declaration.getModuleReference();
if (Node.isExternalModuleReference(reference)) {
const expression = reference.getExpression();
if (expression && Node.isStringLiteral(expression)) imports.add(expression.getLiteralValue());
}
}
for (const importType of sourceFile.getDescendantsOfKind(SyntaxKind.ImportType)) {
const argument = importType.getArgument();
if (Node.isLiteralTypeNode(argument)) {
const literal = argument.getLiteral();
if (Node.isStringLiteral(literal)) imports.add(literal.getLiteralValue());
}
}

// --- Exports ---
const exports = [...sourceFile.getExportedDeclarations().keys()].sort();
Expand All @@ -109,6 +126,8 @@ export class TsMorphAnalyzer implements FileAnalyzer {

for (const statement of sourceFile.getStatements()) {
if (Node.isFunctionDeclaration(statement) && statement.getName()) {
// Overload signatures share the implementation's name; list it once.
if (statement.isOverload()) continue;
symbols.push({
name: statement.getNameOrThrow(),
kind: "function",
Expand All @@ -128,6 +147,7 @@ export class TsMorphAnalyzer implements FileAnalyzer {
exported: classExported
});
for (const method of statement.getMethods()) {
if (method.isOverload()) continue;
symbols.push({
name: method.getName(),
kind: "method",
Expand Down Expand Up @@ -160,17 +180,21 @@ export class TsMorphAnalyzer implements FileAnalyzer {
) {
for (const declaration of statement.getDeclarations()) {
const initializer = declaration.getInitializer();
symbols.push({
name: declaration.getName(),
kind: "const",
line: declaration.getStartLineNumber(),
exported: statement.isExported(),
async: Boolean(
initializer
&& (Node.isArrowFunction(initializer) || Node.isFunctionExpression(initializer))
&& initializer.isAsync()
)
});
const isAsyncFunction = Boolean(
initializer
&& (Node.isArrowFunction(initializer) || Node.isFunctionExpression(initializer))
&& initializer.isAsync()
);
// `const { a, b } = x` declares a and b, not a symbol named "{ a, b }".
for (const name of declaredNames(declaration.getNameNode())) {
symbols.push({
name,
kind: "const",
line: declaration.getStartLineNumber(),
exported: statement.isExported(),
async: isAsyncFunction
});
}
}
}
}
Expand Down Expand Up @@ -207,6 +231,15 @@ export class TsMorphAnalyzer implements FileAnalyzer {
// Helpers
// ---------------------------------------------------------------------------

function declaredNames(nameNode: Node): string[] {
if (Node.isIdentifier(nameNode)) return [nameNode.getText()];
return nameNode
.getDescendantsOfKind(SyntaxKind.BindingElement)
.map((element) => element.getNameNode())
.filter((node) => Node.isIdentifier(node))
.map((node) => node.getText());
}

function replaceExtension(filePath: string, newExt: string): string {
const lastDot = filePath.lastIndexOf(".");
if (lastDot === -1) return filePath + newExt;
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/analyzers/detectors/databaseDetector.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ScannedFile } from "../analysis/index.js";
import { parseJsonText, type ScannedFile } from "../analysis/index.js";
import { isArchitectureSource } from "../graph/index.js";

export type DatabaseInfo = {
Expand Down Expand Up @@ -75,7 +75,7 @@ function readDependencies(files: ScannedFile[]): Set<string> {

for (const file of files.filter((item) => item.path.endsWith("package.json"))) {
try {
const parsed = JSON.parse(file.content) as {
const parsed = parseJsonText(file.content) as {
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
};
Expand Down
7 changes: 4 additions & 3 deletions packages/cli/src/analyzers/detectors/frameworkDetector.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { ScannedFile } from "../analysis/index.js";
import { parseJsonText, type ScannedFile } from "../analysis/index.js";
import { isArchitectureSource } from "../graph/index.js";

export const FRONTEND_FRAMEWORKS = [
Expand Down Expand Up @@ -65,6 +65,7 @@ export function detectFrameworks(files: ScannedFile[]): DetectedFramework[] {
// Vue UI library (vue only in peerDependencies) from being labeled an app.
const hasVueRuntime = "@vitejs/plugin-vue" in dependencies
|| "@vitejs/plugin-vue2" in dependencies
|| "@vue/cli-service" in dependencies
|| "vue-cli-service" in dependencies;
if (!("nuxt" in dependencies) && "vue" in dependencies && hasVueRuntime) {
detected.add("vue");
Expand Down Expand Up @@ -158,7 +159,7 @@ export function detectFrameworks(files: ScannedFile[]): DetectedFramework[] {

function isParseableJson(content: string): boolean {
try {
JSON.parse(content);
parseJsonText(content);
return true;
} catch {
return false;
Expand All @@ -181,7 +182,7 @@ function readManifestDependencies(files: ScannedFile[]): Array<Record<string, st

function readDependencies(content: string): Record<string, string> {
try {
const parsed = JSON.parse(content) as {
const parsed = parseJsonText(content) as {
dependencies?: Record<string, string>;
devDependencies?: Record<string, string>;
};
Expand Down
Loading
Loading