Site search: find symbols by name, rank by reader intent, add hand-marked index entries

builder/cpu-worker.mjs

@@ -244,7 +244,8 @@ const handlers = {
     // global indices during consolidation).
     const searchEntries = deriveSearchEntries(chunk, env.site)
       .map(e => ({ doc: e.doc, title: e.title, content: e.content,
-                   url: e.url, relUrl: e.relUrl }));
+                   url: e.url, relUrl: e.relUrl,
+                   index: e.index, index_also: e.index_also }));
 
     return {
       pages: chunk.map(p => ({

builder/offline.mjs

@@ -281,62 +281,144 @@ const JTD_NAVLINK_REPLACEMENT = `function navLink() {
 }`;
 
 const JTD_INITSEARCH_FN_REPLACEMENT = `function initSearch() {
-  // Patched by _plugins/offlinify.rb for file:// compatibility.
-  // The upstream version fires XMLHttpRequest for search-data.json,
-  // which browsers block under file://. We instead read the index
-  // from a global the offline copy preloads via <script src=>.
-  var docs = window.SEARCH_DATA;
-  if (!docs) {
-    console.log('Offlinify: window.SEARCH_DATA not found; ensure search-data.js loads before just-the-docs.js');
-    return;
-  }
-  // Rebuild each doc.url from doc.relUrl (no baseurl prefix) so
-  // search-result clicks land on the right file regardless of
-  // whatever baseurl the site was built with. Upstream sets
-  // \`link.href = doc.url\`, so this is the value users navigate
-  // to.
-  var siteRoot = window.OFFLINE_SITE_ROOT || '';
-  for (var i in docs) {
-    var rel = docs[i].relUrl;
-    if (typeof rel === 'string' && rel.charAt(0) === '/') {
-      var hash = '';
-      var hashIdx = rel.indexOf('#');
-      if (hashIdx !== -1) {
-        hash = rel.slice(hashIdx);
-        rel = rel.slice(0, hashIdx);
+  // Patched by _plugins/offlinify.rb for file:// compatibility, and for
+  // the lazy index build (WIP.Search.md rollout step 5C): initSearch()
+  // only defines how to build the index -- loadIndex() below -- and hands
+  // that to searchLoaded(), which wires up the search box right away but
+  // doesn't call loadIndex() until the first keystroke. window.SEARCH_DATA
+  // is already sitting in memory (preloaded via <script src=> before this
+  // file runs), so there's no fetch to defer here, only the build itself
+  // -- the same ~1.3s / 240MB-heap cost the online copy defers, paid only
+  // by readers who open search.
+  function loadIndex(onSuccess, onError) {
+    try {
+      // The upstream version fires XMLHttpRequest for search-data.json,
+      // which browsers block under file://. We instead read the index
+      // from a global the offline copy preloads via <script src=>.
+      var docs = window.SEARCH_DATA;
+      if (!docs) {
+        console.log('Offlinify: window.SEARCH_DATA not found; ensure search-data.js loads before just-the-docs.js');
+        onError();
+        return;
       }
-      rel = rel.slice(1); // strip leading /
-      if (rel.endsWith('/')) {
-        rel = rel + 'index.html';
-      } else {
-        var lastSlash = rel.lastIndexOf('/');
-        var lastSeg = lastSlash === -1 ? rel : rel.slice(lastSlash + 1);
-        if (lastSeg.indexOf('.') === -1) rel = rel + '.html';
+      // Rebuild each doc.url from doc.relUrl (no baseurl prefix) so
+      // search-result clicks land on the right file regardless of
+      // whatever baseurl the site was built with. Upstream sets
+      // \`link.href = doc.url\`, so this is the value users navigate
+      // to. Reading from doc.relUrl each time (not doc.url) keeps this
+      // idempotent, since a failed load can retry and run it again.
+      var siteRoot = window.OFFLINE_SITE_ROOT || '';
+      for (var i in docs) {
+        var rel = docs[i].relUrl;
+        if (typeof rel === 'string' && rel.charAt(0) === '/') {
+          var hash = '';
+          var hashIdx = rel.indexOf('#');
+          if (hashIdx !== -1) {
+            hash = rel.slice(hashIdx);
+            rel = rel.slice(0, hashIdx);
+          }
+          rel = rel.slice(1); // strip leading /
+          if (rel.endsWith('/')) {
+            rel = rel + 'index.html';
+          } else {
+            var lastSlash = rel.lastIndexOf('/');
+            var lastSeg = lastSlash === -1 ? rel : rel.slice(lastSlash + 1);
+            if (lastSeg.indexOf('.') === -1) rel = rel + '.html';
+          }
+          docs[i].url = siteRoot + rel + hash;
+        }
       }
-      docs[i].url = siteRoot + rel + hash;
-    }
-  }
 
-  lunr.tokenizer.separator = /[\\s\\-\\/]+/;
-
-  var index = lunr(function(){
-    this.ref('id');
-    this.field('title', { boost: 200 });
-    this.field('content', { boost: 2 });
-    this.field('relUrl');
-    this.metadataWhitelist = ['position'];
-
-    for (var i in docs) {
-      this.add({
-        id: i,
-        title: docs[i].title,
-        content: docs[i].content,
-        relUrl: docs[i].relUrl
+      // Mirrors the online build's dot-run-split patch (see
+      // builder/vendor/just-the-docs/README.md and WIP.Search.md's
+      // "Design" section): wrap lunr.tokenizer once so a run of 2+ dots
+      // ("Do...Loop", "For Each...Next") tokenises as spaces instead of
+      // one opaque token, for both the index and the query. Installed
+      // once, since lunr is a global singleton and loadIndex() can run
+      // again after a failed load; the wrapper carries \`separator\`
+      // itself because the original tokenizer reads
+      // \`lunr.tokenizer.separator\` at call time, which after this
+      // reassignment resolves to the wrapper's own property. Each token's
+      // HTML entities are decoded after the split, by the online copy's
+      // decodeTokenEntities().
+      if (!lunr.tokenizer.dotRunSplit) {
+        var originalTokenizer = lunr.tokenizer;
+        var dotRunSplitTokenizer = function (input) {
+          if (typeof input === 'string') {
+            input = input.replace(/\\.{2,}/g, function (m) {
+              return new Array(m.length + 1).join(' ');
+            });
+          }
+          return originalTokenizer(input).map(decodeTokenEntities);
+        };
+        dotRunSplitTokenizer.dotRunSplit = true;
+        dotRunSplitTokenizer.separator = /[\\s\\-\\/]+/;
+        lunr.tokenizer = dotRunSplitTokenizer;
+      }
+      // Mirrors the online build's token-set key patch
+      // (separateTokenSetKeys() is the online copy's), and its set-union
+      // patch (accumulateSetUnions(), likewise).
+      separateTokenSetKeys();
+      accumulateSetUnions();
+
+      // Mirrors the online build's stem-twin patch (stemTwins() and
+      // qualifiedField() are the online copy's).
+      var twins = stemTwins(docs);
+      var index = lunr(function(){
+        this.ref('id');
+        this.field('title', { boost: 200 });
+        this.field('content', { boost: 2 });
+        // Mirrors the online build's just-the-docs.js patch (see
+        // builder/vendor/just-the-docs/README.md and WIP.Search.md's
+        // "Design" §2): the same two symbol-index fields, at the same
+        // boosts. test/search.test.mjs extracts the field/boost list from
+        // both this string and the vendored just-the-docs.js source and
+        // asserts they agree, so the two copies cannot drift apart silently.
+        this.field('names', { boost: 100 });
+        this.field('qualified', { boost: 500 });
+        // Mirrors the online build's exact-name, primary-name and
+        // page-title patches (exactName() is the online copy's, which this
+        // file keeps).
+        this.field('exact', { boost: 50 });
+        this.field('primary', { boost: 1000 });
+        this.field('page', { boost: 5 });
+        // Mirrors the online build's index-term patch (indexField(),
+        // indexedContent() and pinIndexFieldLengths() are the online
+        // copy's).
+        this.field('index', { boost: 1000 });
+        this.field('relUrl');
+        this.metadataWhitelist = ['position'];
+        pinIndexFieldLengths(this);
+        // Mirrors the online build's stop-word patch (step 5A): keep
+        // lunr's default stop words in the index, since many are
+        // twinBASIC keywords (Do, For, If, Is, On, With, Each...) that the
+        // search pipeline's query side never dropped.
+        this.pipeline.remove(lunr.stopWordFilter);
+
+        for (var i in docs) {
+          this.add({
+            id: i,
+            title: docs[i].title,
+            content: indexedContent(docs[i]),
+            names: docs[i].names || '',
+            qualified: qualifiedField(docs[i], twins),
+            exact: (docs[i].names || '').split(/\\s+/).filter(Boolean).map(exactName).join(' '),
+            primary: (docs[i].primary || '').split(/\\s+/).filter(Boolean).map(exactName).join(' '),
+            page: docs[i].doc || '',
+            index: indexField(docs[i]),
+            relUrl: docs[i].relUrl
+          });
+        }
       });
+
+      onSuccess(index, docs);
+    } catch (e) {
+      console.log('Error building search index: ' + e);
+      onError();
     }
-  });
+  }
 
-  searchLoaded(index, docs);
+  searchLoaded(loadIndex);
 }`;
 
 // §6.9  patchJustTheDocsJs -- regex-substitute navLink() + initSearch().

builder/render.mjs

@@ -59,7 +59,11 @@ function renderPage(page, md) {
     return page.rawContent;
   }
   const source = applyPreRenderRewrites(page.rawContent, md);
-  let html = md.render(source, { page });
+  const env = { page };
+  let html = md.render(source, env);
+  // Lifted off the headings by searchIndexMarksPlugin; search.mjs reads
+  // them when it splits this page into entries.
+  if (env.searchIndexMarks) page.searchIndexMarks = env.searchIndexMarks;
   html = normaliseVoidTags(html);
   html = padEmptyCells(html);
   return html;
@@ -433,6 +437,7 @@ export function createMarkdownIt(ctx) {
   md.use(footnote);
   configureFootnotes(md);
   md.use(headerIdPlugin);
+  md.use(searchIndexMarksPlugin);
   md.use(headingLevelNormalizePlugin);
   md.use(tocPlugin);
   md.use(relativeLinksPlugin, ctx);
@@ -1249,6 +1254,48 @@ function headerIdPlugin(md) {
   });
 }
 
+// Hand-marked search index entries (WIP.Search.md, "What shipped, third
+// round: the index pilot"). A heading's `{: index="..." }` names the terms
+// a reader looks up to find that section, as a book's index would, and
+// `{: index_also="..." }` the terms it is a strong second answer for. Both
+// are lifted off the heading here, once header-id has given it its id, into
+// env.searchIndexMarks as `{ id, index, index_also }` with the raw
+// attribute values, so they never reach the HTML. search.mjs parses the
+// values and attaches them to the entry holding that heading. On any other
+// element the attribute would be published and do nothing, so it fails the
+// build instead.
+const SEARCH_INDEX_ATTRS = ["index", "index_also"];
+
+function searchIndexMarksPlugin(md) {
+  md.core.ruler.after("header-id", "search-index-marks", (state) => {
+    const misplaced = (t) => {
+      const name = SEARCH_INDEX_ATTRS.find((a) => t.attrGet(a) !== null);
+      if (!name) return;
+      throw new Error(
+        `${state.env?.page?.srcRel ?? "(unknown page)"}: \`{: ${name}="..." }\` on ` +
+        `${t.tag ? `<${t.tag}>` : t.type}; only a heading can carry a search index entry. ` +
+        "For the whole page, use `index:` in the front matter.");
+    };
+    for (const t of state.tokens) {
+      if (t.type === "heading_open") {
+        const mark = { id: t.attrGet("id") };
+        let found = false;
+        for (const a of SEARCH_INDEX_ATTRS) {
+          const v = t.attrGet(a);
+          if (v === null) continue;
+          mark[a] = v;
+          found = true;
+          t.attrs.splice(t.attrIndex(a), 1);
+        }
+        if (found) (state.env.searchIndexMarks ??= []).push(mark);
+      } else if (t.attrs) {
+        misplaced(t);
+      }
+      for (const c of t.children ?? []) if (c.attrs) misplaced(c);
+    }
+  });
+}
+
 function headingText(children) {
   // Concatenate the visible text content from inline tokens. Skip markup
   // wrappers (em, strong, link openers) and pick up text + code spans --