fix: keep generated files and project paperwork out of the code index

docs/capabilities.md

@@ -20,6 +20,7 @@ Config-registered project roots (from `~/.codebase-context/config.json`) are loa
 Per-project config overrides supported today:
 
 - `projects[].excludePatterns`: merged with the built-in exclusion set for that project at index time
+- `projects[].includeExcluded`: root-relative source globs explicitly restored through built-in exclusions, `excludePatterns` and Git ignores (for example, `[".agents/tools/**/*.ts"]`). Normal `include` patterns narrow the scan and cannot override exclusions. Binary/unsupported files, files over the size limit and paths outside the project remain excluded.
 - `projects[].analyzerHints.analyzer`: prefers a registered analyzer by name for that project and falls back safely when the name is missing or invalid
 - `projects[].analyzerHints.extensions`: adds project-local source extensions for indexing and auto-refresh watching without changing defaults for other projects
 - `projects[].parsing.maxChunks`: sets the maximum number of searchable chunks indexed for that project
@@ -262,6 +263,12 @@ Impact is 2-hop transitive: direct importers (hop 1) and their importers (hop 2)
 
 ## Indexing
 
+Inside a Git worktree, indexing uses `git ls-files -co --exclude-standard`: tracked files and untracked files that are not ignored remain eligible, including nested `.gitignore`, `.git/info/exclude` and Git's global excludes. Tracked files stay eligible even if a later ignore rule matches them. Outside Git (or without Git installed), the existing root `.gitignore` behavior remains. Library callers can set `respectGitignore: false` to disable Git filtering.
+
+The index additionally skips `out`, `.work`, `.agents`, `.codex`, `.cursor`, `.github/agents`, lockfiles (`*.lock`, `*.lockb`, `package-lock.json`, `npm-shrinkwrap.json`, `pnpm-lock.yaml`, `packages.lock.json`), minified JavaScript/CSS and source maps. These new filters apply only to indexing, preserving file-watcher and project-discovery behavior. This lets watchers observe explicitly restored source in those directories. Previously excluded directories such as `coverage` still require a manual `reindex` after edits when opted in.
+
+To index selected excluded source, add `"includeExcluded": [".agents/tools/**/*.ts"]` to that project's entry in `~/.codebase-context/config.json` (with its absolute `root`). Both MCP indexing and CLI `reindex` use it. Library callers pass the same array as `config.includeExcluded` to `CodebaseIndexer`. Use narrow globs: these files become searchable even when Git ignores them. Run a full `reindex` after changing selection settings to remove previously indexed junk.
+
 - Initial: full scan → chunking (50 lines, 0 overlap) → embedding → vector DB (LanceDB) + keyword index (Fuse.js)
 - Incremental: SHA-256 manifest diffing, selective embed/delete, full intelligence regeneration
 - Auto-refresh (MCP server mode only): chokidar file watcher triggers incremental reindex after a debounce on any source file change — `node_modules/`, `.git/`, `dist/`, and `.codebase-context/` are excluded. One-shot CLI runs skip the watcher entirely.

src/cli.ts

@@ -138,7 +138,12 @@ async function initToolContext(): Promise<ToolContext> {
       let lastLoggedProgress = { phase: '', percentage: -1 };
       const indexer = new CodebaseIndexer({
         rootPath,
-        ...(projectConfig?.parsing ? { config: { parsing: projectConfig.parsing } } : {}),
+        config: {
+          ...(projectConfig?.parsing ? { parsing: projectConfig.parsing } : {}),
+          ...(projectConfig?.includeExcluded
+            ? { includeExcluded: projectConfig.includeExcluded }
+            : {})
+        },
         incrementalOnly,
         onProgress: (progress) => {
           const shouldLog =

src/constants/codebase-context.ts

@@ -56,6 +56,23 @@ export const EXCLUDED_GLOB_PATTERNS: readonly string[] = EXCLUDED_DIRECTORY_NAME
   (dir) => `**/${dir}/**`
 );
 
+/** Index-only non-source files. Watchers still observe them for live updates. */
+export const INDEX_ONLY_EXCLUDED_GLOB_PATTERNS: readonly string[] = [
+  ...['out', '.work', '.agents', '.codex', '.cursor'].map((dir) => `**/${dir}/**`),
+  '**/.github/agents/**',
+  '**/.github/FUNDING.yml',
+  '**/.github/ISSUE_TEMPLATE/**',
+  '**/*.lock',
+  '**/*.lockb',
+  '**/package-lock.json',
+  '**/npm-shrinkwrap.json',
+  '**/pnpm-lock.yaml',
+  '**/packages.lock.json',
+  '**/*.min.js',
+  '**/*.min.css',
+  '**/*.map'
+];
+
 /**
  * Additional directories skipped only during project discovery (not generated
  * code, just not useful roots to recurse into).

src/core/indexer.ts

@@ -4,6 +4,8 @@
  */
 
 import { randomUUID } from 'crypto';
+import { execFile, spawn } from 'node:child_process';
+import { promisify } from 'node:util';
 import { promises as fs } from 'fs';
 import path from 'path';
 import { glob } from 'glob';
@@ -23,6 +25,7 @@ import {
 import { analyzerRegistry } from './analyzer-registry.js';
 import type { AnalyzerSelectionOptions } from './analyzer-registry.js';
 import { getSupportedExtensions, isBinaryFile, isCodeFile } from '../utils/language-detection.js';
+import { gitEnvironmentWithoutLocalRepositoryContext } from '../utils/git-environment.js';
 import {
   getEmbeddingProvider,
   getConfiguredDimensions,
@@ -42,6 +45,7 @@ import { getFileCommitDates } from '../utils/git-dates.js';
 import {
   CODEBASE_CONTEXT_DIRNAME,
   EXCLUDED_GLOB_PATTERNS,
+  INDEX_ONLY_EXCLUDED_GLOB_PATTERNS,
   HEALTH_FILENAME,
   INDEX_FORMAT_VERSION,
   INDEXING_STATS_FILENAME,
@@ -58,6 +62,7 @@ import { deriveCodebaseHealth } from '../health/derive.js';
 const STAGING_DIRNAME = '.staging';
 const PREVIOUS_DIRNAME = '.previous';
 const DEFAULT_MAX_CHUNKS = 5000;
+const execFileAsync = promisify(execFile);
 
 import {
   computeFileHashes,
@@ -1091,13 +1096,65 @@ export class CodebaseIndexer {
     }
   }
 
+  private async gitFiles(): Promise<Set<string> | null> {
+    let gitEnvironment: NodeJS.ProcessEnv;
+
+    try {
+      gitEnvironment = gitEnvironmentWithoutLocalRepositoryContext();
+      const { stdout } = await execFileAsync('git', ['rev-parse', '--is-inside-work-tree'], {
+        cwd: this.rootPath,
+        env: { ...gitEnvironment, LC_ALL: 'C' }
+      });
+      if (stdout.trim() !== 'true') return null;
+    } catch (error) {
+      const gitError = error as { code?: string | number; stderr?: string };
+      // Keep the existing root-.gitignore fallback outside Git or without Git installed.
+      if (
+        gitError.code === 'ENOENT' ||
+        (gitError.code === 128 && gitError.stderr?.includes('not a git repository'))
+      )
+        return null;
+      throw error;
+    }
+
+    return new Promise((resolve, reject) => {
+      const git = spawn('git', ['ls-files', '-co', '--exclude-standard', '-z'], {
+        cwd: this.rootPath,
+        env: gitEnvironment,
+        stdio: ['ignore', 'pipe', 'pipe']
+      });
+      const files = new Set<string>();
+      let pending = '';
+      let stderr = '';
+      git.stdout.setEncoding('utf8');
+      git.stdout.on('data', (chunk: string) => {
+        const paths = (pending + chunk).split('\0');
+        pending = paths.pop() ?? '';
+        for (const file of paths) if (file) files.add(file);
+      });
+      git.stderr.setEncoding('utf8');
+      git.stderr.on('data', (chunk: string) => {
+        stderr = (stderr + chunk).slice(-4096);
+      });
+      git.on('error', reject);
+      git.on('close', (code) => {
+        if (code !== 0) {
+          reject(new Error(`Git file discovery failed (${code}): ${stderr.trim()}`));
+        } else {
+          resolve(files);
+        }
+      });
+    });
+  }
+
   private async scanFiles(): Promise<string[]> {
     const files: string[] = [];
     const seen = new Set<string>();
 
-    // Read .gitignore if respecting it
+    const gitFiles = this.config.respectGitignore ? await this.gitFiles() : null;
+    // Outside Git, preserve the existing root-only .gitignore behavior.
     let ig: ReturnType<typeof ignore.default> | null = null;
-    if (this.config.respectGitignore) {
+    if (this.config.respectGitignore && !gitFiles) {
       try {
         const gitignorePath = path.join(this.rootPath, '.gitignore');
         const gitignoreContent = await fs.readFile(gitignorePath, 'utf-8');
@@ -1108,14 +1165,17 @@ export class CodebaseIndexer {
     }
 
     // Scan with glob
-    const includePatterns = this.getIncludePatterns();
-    const excludePatterns = this.config.exclude || [];
+    const excludePatterns = [...INDEX_ONLY_EXCLUDED_GLOB_PATTERNS, ...(this.config.exclude || [])];
 
-    for (const pattern of includePatterns) {
-      const matches = await glob(pattern, {
+    for (const selection of [
+      { patterns: this.getIncludePatterns(), explicit: false },
+      { patterns: this.config.includeExcluded ?? [], explicit: true }
+    ]) {
+      const matches = await glob(selection.patterns, {
         cwd: this.rootPath,
         absolute: true,
-        ignore: excludePatterns,
+        ignore: selection.explicit ? [] : excludePatterns,
+        dot: selection.explicit || gitFiles !== null,
         nodir: true
       });
 
@@ -1124,12 +1184,13 @@ export class CodebaseIndexer {
         if (seen.has(normalizedFile)) {
           continue;
         }
-        seen.add(normalizedFile);
-
-        const relativePath = path.relative(this.rootPath, file);
+        const relativePath = path.relative(this.rootPath, file).replace(/\\/g, '/');
+        if (relativePath.startsWith('../') || path.isAbsolute(relativePath)) continue;
 
-        // Check gitignore
-        if (ig && ig.ignores(relativePath)) {
+        if (
+          !selection.explicit &&
+          ((gitFiles && !gitFiles.has(relativePath)) || (ig && ig.ignores(relativePath)))
+        ) {
           continue;
         }
 
@@ -1149,6 +1210,7 @@ export class CodebaseIndexer {
           continue;
         }
 
+        seen.add(normalizedFile);
         files.push(file);
       }
     }

src/index.ts

@@ -1253,21 +1253,19 @@ async function performIndexingOnce(
     let lastLoggedProgress = { phase: '', percentage: -1 };
     const indexer = new CodebaseIndexer({
       rootPath: project.rootPath,
-      ...(project.runtimeOverrides.extraExcludePatterns?.length
-        ? {
-            config: {
-              exclude: [
-                ...EXCLUDED_GLOB_PATTERNS,
-                ...project.runtimeOverrides.extraExcludePatterns
-              ],
-              ...(project.runtimeOverrides.maxChunks
-                ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } }
-                : {})
+      config: {
+        ...(project.runtimeOverrides.extraExcludePatterns?.length
+          ? {
+              exclude: [...EXCLUDED_GLOB_PATTERNS, ...project.runtimeOverrides.extraExcludePatterns]
             }
-          }
-        : project.runtimeOverrides.maxChunks
-          ? { config: { parsing: { maxChunks: project.runtimeOverrides.maxChunks } } }
           : {}),
+        ...(project.runtimeOverrides.includeExcluded?.length
+          ? { includeExcluded: project.runtimeOverrides.includeExcluded }
+          : {}),
+        ...(project.runtimeOverrides.maxChunks
+          ? { parsing: { maxChunks: project.runtimeOverrides.maxChunks } }
+          : {})
+      },
       ...(project.runtimeOverrides.preferredAnalyzer ||
       project.runtimeOverrides.extraSourceExtensions?.length
         ? {
@@ -1650,6 +1648,9 @@ function buildProjectRuntimeOverrides(projectConfig: ProjectConfig): ProjectRunt
   if (projectConfig.excludePatterns?.length) {
     runtimeOverrides.extraExcludePatterns = [...projectConfig.excludePatterns];
   }
+  if (projectConfig.includeExcluded?.length) {
+    runtimeOverrides.includeExcluded = [...projectConfig.includeExcluded];
+  }
 
   if (projectConfig.parsing?.maxChunks) {
     runtimeOverrides.maxChunks = projectConfig.parsing.maxChunks;

src/project-state.ts

@@ -14,6 +14,8 @@ import type { ToolPaths, IndexState } from './tools/types.js';
 export interface ProjectRuntimeOverrides {
   /** Extra glob exclusion patterns merged with the default index-time exclusions. */
   extraExcludePatterns?: string[];
+  /** Source globs explicitly restored through exclusions and Git ignores. */
+  includeExcluded?: string[];
   /** Maximum chunks retained in semantic and keyword indexes for this project. */
   maxChunks?: number;
   /** Analyzer name to prefer for this project without mutating global registry order. */

src/server/config.ts

@@ -5,6 +5,7 @@ import path from 'node:path';
 export interface ProjectConfig {
   root: string;
   excludePatterns?: string[];
+  includeExcluded?: string[];
   parsing?: {
     maxChunks?: number;
   };
@@ -89,6 +90,10 @@ export async function loadServerConfig(): Promise<ServerConfig | null> {
         if (excludePatterns) {
           parsedProject.excludePatterns = excludePatterns;
         }
+        const includeExcluded = parseStringArray(project.includeExcluded);
+        if (includeExcluded) {
+          parsedProject.includeExcluded = includeExcluded;
+        }
 
         if (
           typeof project.parsing === 'object' &&

src/types/index.ts

@@ -470,6 +470,8 @@ export interface CodebaseConfig {
   // File filtering
   include?: string[];
   exclude?: string[];
+  /** Root-relative source globs that override exclusions and Git ignores. */
+  includeExcluded?: string[];
   respectGitignore?: boolean;
 
   // Parsing options

src/utils/git-dates.ts

@@ -5,6 +5,7 @@
 
 import { exec } from 'child_process';
 import { promisify } from 'util';
+import { gitEnvironmentWithoutLocalRepositoryContext } from './git-environment.js';
 
 const execAsync = promisify(exec);
 const commitDateCache = new Map<string, Map<string, Date>>();
@@ -46,6 +47,7 @@ export async function getFileCommitDates(rootPath: string): Promise<Map<string,
     // Format: ":::ISO_DATE" followed by affected files on new lines
     const { stdout } = await execAsync('git log --format=":::%cd" --name-only --date=iso-strict', {
       cwd: rootPath,
+      env: gitEnvironmentWithoutLocalRepositoryContext(),
       maxBuffer: 50 * 1024 * 1024 // 50MB buffer for large repos
     });
 

src/utils/git-environment.ts

@@ -0,0 +1,27 @@
+import { execFileSync } from 'node:child_process';
+
+let localEnvironmentVariableNames: string[] | undefined;
+
+function getLocalEnvironmentVariableNames(): string[] {
+  if (!localEnvironmentVariableNames) {
+    const output = execFileSync('git', ['rev-parse', '--local-env-vars'], {
+      encoding: 'utf8'
+    });
+    localEnvironmentVariableNames = output.split(/\r?\n/).filter(Boolean);
+  }
+
+  return localEnvironmentVariableNames;
+}
+
+/** Remove Git's repository-local context while preserving global configuration. */
+export function gitEnvironmentWithoutLocalRepositoryContext(
+  environment: NodeJS.ProcessEnv = process.env
+): NodeJS.ProcessEnv {
+  const sanitized = { ...environment };
+
+  for (const name of getLocalEnvironmentVariableNames()) {
+    delete sanitized[name];
+  }
+
+  return sanitized;
+}

tests/file-watcher.test.ts

@@ -16,6 +16,31 @@ describe('FileWatcher', () => {
     await rmWithRetries(tempDir);
   });
 
+  it('still observes source in index-only exclusions so opt-in files can refresh', async () => {
+    let resolveReady!: () => void;
+    let resolveChanged!: () => void;
+    const ready = new Promise<void>((resolve) => {
+      resolveReady = resolve;
+    });
+    const changed = new Promise<void>((resolve) => {
+      resolveChanged = resolve;
+    });
+    const stop = startFileWatcher({
+      rootPath: tempDir,
+      debounceMs: 50,
+      onReady: resolveReady,
+      onChanged: resolveChanged
+    });
+    try {
+      await ready;
+      await fs.mkdir(path.join(tempDir, '.agents'));
+      await fs.writeFile(path.join(tempDir, '.agents', 'source.ts'), 'export const fresh = 1;');
+      await changed;
+    } finally {
+      stop();
+    }
+  }, 8000);
+
   it('triggers onChanged after debounce window', async () => {
     const debounceMs = 400;
     let callCount = 0;
@@ -29,7 +54,9 @@ describe('FileWatcher', () => {
       rootPath: tempDir,
       debounceMs,
       onReady: () => resolveReady(),
-      onChanged: () => { callCount++; },
+      onChanged: () => {
+        callCount++;
+      }
     });
 
     try {
@@ -57,7 +84,9 @@ describe('FileWatcher', () => {
       rootPath: tempDir,
       debounceMs,
       onReady: () => resolveReady(),
-      onChanged: () => { callCount++; },
+      onChanged: () => {
+        callCount++;
+      }
     });
 
     try {
@@ -89,7 +118,9 @@ describe('FileWatcher', () => {
       rootPath: tempDir,
       debounceMs,
       onReady: () => resolveReady(),
-      onChanged: () => { callCount++; },
+      onChanged: () => {
+        callCount++;
+      }
     });
 
     // Give chokidar a moment to finish initializing before the first write