perf: ~32% faster highlighting, fix silent token loss, memory and gutter bugs

src/Escape.php

@@ -26,15 +26,26 @@ public static function terminal(string $input): string
         );
     }
 
+    /**
+     * Escapes HTML special characters and turns tokens back into raw markup in a single pass.
+     * The escape and token keys are disjoint and strtr never rescans replaced text,
+     * so this equals escaping first and reversing the tokens afterwards.
+     */
+    private const array HTML_MAP = [
+        '&' => '&',
+        '<' => '&lt;',
+        '>' => '&gt;',
+        '"' => '&quot;',
+        '❶' => '&',
+        '❷' => '<',
+        '❸' => '>',
+        '❹' => '"',
+        self::INJECTION_TOKEN => '',
+    ];
+
     public static function html(string $input): string
     {
-        return self::reverse(
-            str_replace(
-                ['&', '<', '>', '"'],
-                ['&amp;', '&lt;', '&gt;', '&quot;'],
-                $input,
-            ),
-        );
+        return strtr($input, self::HTML_MAP);
     }
 
     public static function tokens(string $input): string
@@ -45,13 +56,4 @@ public static function tokens(string $input): string
             $input,
         );
     }
-
-    private static function reverse(string $input): string
-    {
-        return str_replace(
-            self::TOKEN_KEYS,
-            self::TOKEN_VALUES,
-            $input,
-        );
-    }
 }

src/Highlighter.php

@@ -46,7 +46,17 @@
 use Tempest\Highlight\Tokens\GroupTokens;
 use Tempest\Highlight\Tokens\ParseTokens;
 use Tempest\Highlight\Tokens\RenderTokens;
-
+use Tempest\Highlight\Tokens\TokenType;
+use WeakMap;
+
+/**
+ * @phpstan-type LanguageCacheEntry array{
+ *     patterns: array<int|string, array{?string, Pattern, ?TokenType, ?string}>,
+ *     patternRequirements: array<int|string, string[]>,
+ *     before: list<array{Injection, string[]|null}>,
+ *     after: list<array{Injection, string[]|null}>,
+ * }
+ */
 final class Highlighter
 {
     private array $languages = [];
@@ -56,12 +66,18 @@ final class Highlighter
     private readonly ParseTokens $parseTokens;
     private readonly GroupTokens $groupTokens;
     private readonly RenderTokens $renderTokens;
-    private array $patterns = [];
+    /**
+     * Per-language caches, keyed by the language instance itself so entries
+     * are dropped when a language is freed and can never be served to a
+     * different language that reuses its object id. Clones share this map,
+     * which is safe because entries depend only on the language.
+     *
+     * @var WeakMap<Language, LanguageCacheEntry>
+     */
+    private WeakMap $languageCache;
     private array $afterInjections = [];
-    /** @var array<int, Injection[]> */
-    private array $beforeInjectionsCache = [];
-    /** @var array<int, Injection[]> */
-    private array $afterInjectionsCache = [];
+    /** @var array<class-string, string[]|null> */
+    private array $requiredSubstrings = [];
     private ?self $nestedHighlighter = null;
 
     public function __construct(
@@ -105,6 +121,7 @@ public function __construct(
             ->addLanguage(new JsxLanguage())
             ->addLanguage(new TsxLanguage());
 
+        $this->languageCache = new WeakMap();
         $this->parseTokens = new ParseTokens();
         $this->groupTokens = new GroupTokens();
         $this->renderTokens = new RenderTokens($this->theme);
@@ -203,8 +220,14 @@ private function parseContent(string $content, Language $language): string
         $tokens = [];
         $nestedHighlighter = $this->getNestedHighlighter();
 
+        $languageCache = $this->getLanguageCache($language);
+
         // Before Injections
-        foreach ($this->getBeforeInjections($language) as $injection) {
+        foreach ($languageCache['before'] as [$injection, $requiredSubstrings]) {
+            if ($requiredSubstrings !== null && ! $this->containsAny($content, $requiredSubstrings)) {
+                continue;
+            }
+
             $parsedInjection = $injection->parse($content, $nestedHighlighter);
             $content = $parsedInjection->content;
 
@@ -214,19 +237,34 @@ private function parseContent(string $content, Language $language): string
         }
 
         // Patterns
-        foreach (
-            $this->parseTokens->parse($content, $this->getPatterns($language)) as $token
-        ) {
+        $patterns = $languageCache['patterns'];
+
+        foreach ($languageCache['patternRequirements'] as $key => $requiredSubstrings) {
+            if (! $this->containsAny($content, $requiredSubstrings)) {
+                unset($patterns[$key]);
+            }
+        }
+
+        foreach ($this->parseTokens->parseCompiled($content, $patterns) as $token) {
             $tokens[] = $token;
         }
 
         $groupedTokens = ($this->groupTokens)($tokens);
         $content = ($this->renderTokens)($content, $groupedTokens);
 
         // After Injections
-        foreach ($this->getAfterInjections($language) as $injection) {
-            $parsedInjection = $injection->parse($content, $nestedHighlighter);
-            $content = $parsedInjection->content;
+        if (! $this->isNested) {
+            foreach ($languageCache['after'] as [$injection, $requiredSubstrings]) {
+                if ($requiredSubstrings !== null && ! $this->containsAny($content, $requiredSubstrings)) {
+                    continue;
+                }
+
+                $content = $injection->parse($content, $nestedHighlighter)->content;
+            }
+
+            if ($this->gutterInjection instanceof GutterInjection) {
+                $content = $this->gutterInjection->parse($content, $nestedHighlighter)->content;
+            }
         }
 
         if ($this->isNested) {
@@ -237,61 +275,42 @@ private function parseContent(string $content, Language $language): string
     }
 
     /**
-     * @return Injection[]
+     * @return LanguageCacheEntry
      */
-    private function getBeforeInjections(Language $language): array
+    private function getLanguageCache(Language $language): array
     {
-        $languageId = spl_object_id($language);
-
-        if (isset($this->beforeInjectionsCache[$languageId])) {
-            return $this->beforeInjectionsCache[$languageId];
-        }
-
-        $this->buildInjectionCaches($language);
-
-        return $this->beforeInjectionsCache[$languageId];
-    }
-
-    /**
-     * @return Injection[]
-     */
-    private function getAfterInjections(Language $language): array
-    {
-        if ($this->isNested) {
-            return [];
-        }
-
-        $languageId = spl_object_id($language);
-
-        if (! isset($this->afterInjectionsCache[$languageId])) {
-            $this->buildInjectionCaches($language);
-        }
-
-        $afterInjections = $this->afterInjectionsCache[$languageId];
-
-        if ($this->gutterInjection instanceof GutterInjection) {
-            $afterInjections[] = $this->gutterInjection;
+        if (isset($this->languageCache[$language])) {
+            return $this->languageCache[$language];
         }
 
-        return $afterInjections;
-    }
-
-    private function buildInjectionCaches(Language $language): void
-    {
-        $languageId = spl_object_id($language);
         $before = [];
         $after = [];
 
         foreach ($language->getInjections() as $injection) {
             if ($this->isAfterInjection($injection)) {
-                $after[] = $injection;
+                $after[] = [$injection, $this->getRequiredSubstrings($injection)];
             } else {
-                $before[] = $injection;
+                $before[] = [$injection, $this->getRequiredSubstrings($injection)];
+            }
+        }
+
+        $patterns = $this->parseTokens->compile($language->getPatterns());
+        $patternRequirements = [];
+
+        foreach ($patterns as $key => [, $pattern]) {
+            $requiredSubstrings = $this->getRequiredSubstrings($pattern);
+
+            if ($requiredSubstrings !== null) {
+                $patternRequirements[$key] = $requiredSubstrings;
             }
         }
 
-        $this->beforeInjectionsCache[$languageId] = $before;
-        $this->afterInjectionsCache[$languageId] = $after;
+        return $this->languageCache[$language] = [
+            'patterns' => $patterns,
+            'patternRequirements' => $patternRequirements,
+            'before' => $before,
+            'after' => $after,
+        ];
     }
 
     private function normalizeNewline(string $subject): string
@@ -303,18 +322,41 @@ private function normalizeNewline(string $subject): string
         return str_replace(["\r\n", "\r"], "\n", $subject);
     }
 
-    private function getPatterns(Language $language): array
-    {
-        $languageId = spl_object_id($language);
-
-        return $this->patterns[$languageId] ??= $language->getPatterns();
-    }
-
     private function isAfterInjection(Injection $injection): bool
     {
         $class = $injection::class;
 
         return $this->afterInjections[$class] ??=
             new ReflectionClass($class)->getAttributes(After::class) !== [];
     }
+
+    /**
+     * @return string[]|null
+     */
+    private function getRequiredSubstrings(Injection|Pattern $instance): ?array
+    {
+        $class = $instance::class;
+
+        if (! array_key_exists($class, $this->requiredSubstrings)) {
+            $attribute = new ReflectionClass($class)->getAttributes(RequiresSubstring::class)[0] ?? null;
+
+            $this->requiredSubstrings[$class] = $attribute?->newInstance()->substrings ?: null;
+        }
+
+        return $this->requiredSubstrings[$class];
+    }
+
+    /**
+     * @param string[] $substrings
+     */
+    private function containsAny(string $content, array $substrings): bool
+    {
+        foreach ($substrings as $substring) {
+            if (str_contains($content, $substring)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
 }

src/Languages/Base/Injections/AdditionInjection.php

@@ -9,8 +9,10 @@
 use Tempest\Highlight\Highlighter;
 use Tempest\Highlight\Injection;
 use Tempest\Highlight\ParsedInjection;
+use Tempest\Highlight\RequiresSubstring;
 
 #[After]
+#[RequiresSubstring('{+', '+}', "\r")]
 final readonly class AdditionInjection implements Injection
 {
     private const string NEWLINE = "\n";
@@ -22,30 +24,40 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio
         $content = str_replace('❷span class=❹ignore❹❸{+❷/span❸', '{+', $content);
         $content = str_replace('❷span class=❹ignore❹❸+}❷/span❸', '+}', $content);
 
-        preg_match_all('/(\{\+)((.|\n)*?)(\+})/', $content, $matches, PREG_OFFSET_CAPTURE);
+        $firstOpener = strpos($content, '{+');
+        $lastCloser = $firstOpener === false ? false : strrpos($content, '+}', $firstOpener);
 
-        $parsedOffset = 0;
+        if ($lastCloser === false) {
+            return new ParsedInjection($content);
+        }
+
+        // No match can end after the last closer, so unclosed openers past it are never rescanned to the end.
+        preg_match_all('/(\{\+)([\s\S]*?)(\+})/', substr($content, 0, $lastCloser + 2), $matches, PREG_OFFSET_CAPTURE);
+
+        $parsedContent = '';
+        $parsedUntil = 0;
+        $startingLineNumber = 1;
+        $linesCountedUntil = 0;
 
         $open = Escape::tokens('<span class="hl-addition">');
         $close = Escape::tokens('</span>');
 
-        foreach ($matches[0] as $match) {
-            $matchedContent = $match[0];
-            $offset = $match[1];
-            $replacementOffset = $offset + $parsedOffset;
-
+        foreach ($matches[0] as [$matchedContent, $offset]) {
             $parsedMatchedContent = str_replace(
                 ['{+', self::NEWLINE, '+}'],
                 [$open, $close . self::NEWLINE . $open, $close],
                 $matchedContent,
             );
 
             if (($gutter = $highlighter->getGutterInjection()) instanceof GutterInjection) {
-                $startingLineNumber = substr_count(
+                $startingLineNumber += substr_count(
                     haystack: $content,
                     needle: self::NEWLINE,
-                    length: $replacementOffset,
-                ) + 1;
+                    offset: $linesCountedUntil,
+                    length: $offset - $linesCountedUntil,
+                );
+
+                $linesCountedUntil = $offset;
 
                 $totalAmountOfLines = substr_count(
                     haystack: $parsedMatchedContent,
@@ -59,17 +71,11 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio
                 }
             }
 
-            $content = substr_replace(
-                $content,
-                $parsedMatchedContent,
-                $replacementOffset,
-                strlen($matchedContent),
-            );
-
-            $parsedOffset += strlen($parsedMatchedContent) - strlen($matchedContent);
+            $parsedContent .= substr($content, $parsedUntil, $offset - $parsedUntil) . $parsedMatchedContent;
+            $parsedUntil = $offset + strlen($matchedContent);
         }
 
-        return new ParsedInjection($content);
+        return new ParsedInjection($parsedContent . substr($content, $parsedUntil));
     }
 
     private function normalizeLineEndings(string $content): string

src/Languages/Base/Injections/BlurInjection.php

@@ -6,7 +6,9 @@
 
 use Tempest\Highlight\Injection;
 use Tempest\Highlight\Languages\Base\IsHighlightInjection;
+use Tempest\Highlight\RequiresSubstring;
 
+#[RequiresSubstring('{~')]
 final readonly class BlurInjection implements Injection
 {
     use IsHighlightInjection;