From c64e020572794fd7ef45eecd27500f8ef2d04b71 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:46:54 +0400 Subject: [PATCH 01/15] perf(escape): escape and restore tokens in a single strtr pass Escape::html ran nine sequential str_replace passes over the full rendered output. The escape keys and token keys are disjoint and strtr never rescans replaced text, so one strtr pass gives identical output at about a third of the cost. --- src/Escape.php | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/Escape.php b/src/Escape.php index 46774e04..0bfe59b7 100644 --- a/src/Escape.php +++ b/src/Escape.php @@ -26,15 +26,26 @@ public static function terminal(string $input): string ); } + /** + * Escapes HTML special characters and turns tokens back into raw markup in a single pass. + * The escape and token keys are disjoint and strtr never rescans replaced text, + * so this equals escaping first and reversing the tokens afterwards. + */ + private const array HTML_MAP = [ + '&' => '&', + '<' => '<', + '>' => '>', + '"' => '"', + '❶' => '&', + '❷' => '<', + '❸' => '>', + '❹' => '"', + self::INJECTION_TOKEN => '', + ]; + public static function html(string $input): string { - return self::reverse( - str_replace( - ['&', '<', '>', '"'], - ['&', '<', '>', '"'], - $input, - ), - ); + return strtr($input, self::HTML_MAP); } public static function tokens(string $input): string @@ -45,13 +56,4 @@ public static function tokens(string $input): string $input, ); } - - private static function reverse(string $input): string - { - return str_replace( - self::TOKEN_KEYS, - self::TOKEN_VALUES, - $input, - ); - } } From 5e63325a27d631c346fc2f0e2e61d2468417bbe1 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:46:54 +0400 Subject: [PATCH 02/15] perf(twig,gdscript): match keywords with one combined pattern Twig tags and GDScript ran one preg_match_all per keyword (38 and 36 patterns). Every keyword is a whole word bounded by \b, so a single alternation yields exactly the same tokens with one regex pass. --- src/Languages/Gdscript/GdscriptLanguage.php | 76 +++++++++--------- .../CombinedGdscriptKeywordPattern.php | 50 ++++++++++++ .../Patterns/CombinedTwigKeywordPattern.php | 50 ++++++++++++ src/Languages/Twig/TwigTagLanguage.php | 79 ++++++++++--------- 4 files changed, 179 insertions(+), 76 deletions(-) create mode 100644 src/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPattern.php create mode 100644 src/Languages/Twig/Patterns/CombinedTwigKeywordPattern.php diff --git a/src/Languages/Gdscript/GdscriptLanguage.php b/src/Languages/Gdscript/GdscriptLanguage.php index 9e4fd79c..0713d50d 100644 --- a/src/Languages/Gdscript/GdscriptLanguage.php +++ b/src/Languages/Gdscript/GdscriptLanguage.php @@ -9,11 +9,11 @@ use Tempest\Highlight\Languages\Gdscript\Patterns\AnnotationPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\AsTypePattern; use Tempest\Highlight\Languages\Gdscript\Patterns\ClassNamePattern; +use Tempest\Highlight\Languages\Gdscript\Patterns\CombinedGdscriptKeywordPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\DoubleQuoteValuePattern; use Tempest\Highlight\Languages\Gdscript\Patterns\ExtendsPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\FunctionCallPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\FunctionNamePattern; -use Tempest\Highlight\Languages\Gdscript\Patterns\KeywordPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\OperatorPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\PropertyAccessPattern; use Tempest\Highlight\Languages\Gdscript\Patterns\ReturnTypePattern; @@ -76,42 +76,44 @@ public function getPatterns(): array new OperatorPattern('>'), new OperatorPattern('!'), - new KeywordPattern('if'), - new KeywordPattern('elif'), - new KeywordPattern('else'), - new KeywordPattern('for'), - new KeywordPattern('while'), - new KeywordPattern('match'), - new KeywordPattern('break'), - new KeywordPattern('continue'), - new KeywordPattern('pass'), - new KeywordPattern('return'), - new KeywordPattern('class'), - new KeywordPattern('class_name'), - new KeywordPattern('extends'), - new KeywordPattern('is'), - new KeywordPattern('not'), - new KeywordPattern('and'), - new KeywordPattern('or'), - new KeywordPattern('in'), - new KeywordPattern('as'), - new KeywordPattern('self'), - new KeywordPattern('signal'), - new KeywordPattern('func'), - new KeywordPattern('static'), - new KeywordPattern('const'), - new KeywordPattern('enum'), - new KeywordPattern('var'), - new KeywordPattern('breakpoint'), - new KeywordPattern('preload'), - new KeywordPattern('await'), - new KeywordPattern('yield'), - new KeywordPattern('assert'), - new KeywordPattern('super'), - new KeywordPattern('PI'), - new KeywordPattern('INF'), - new KeywordPattern('TAU'), - new KeywordPattern('NAN'), + new CombinedGdscriptKeywordPattern([ + 'if', + 'elif', + 'else', + 'for', + 'while', + 'match', + 'break', + 'continue', + 'pass', + 'return', + 'class', + 'class_name', + 'extends', + 'is', + 'not', + 'and', + 'or', + 'in', + 'as', + 'self', + 'signal', + 'func', + 'static', + 'const', + 'enum', + 'var', + 'breakpoint', + 'preload', + 'await', + 'yield', + 'assert', + 'super', + 'PI', + 'INF', + 'TAU', + 'NAN', + ]), new SinglelineCommentPattern(), diff --git a/src/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPattern.php b/src/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPattern.php new file mode 100644 index 00000000..7e848a94 --- /dev/null +++ b/src/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPattern.php @@ -0,0 +1,50 @@ + strlen($b) <=> strlen($a)); + + $alternation = implode( + '|', + array_map( + static fn (string $keyword): string => preg_quote($keyword, '/'), + $keywords, + ), + ); + + $this->compiledPattern = "/\b(?(?:{$alternation}))\b/"; + } + + public function match(string $content): array + { + preg_match_all($this->compiledPattern, $content, $matches, PREG_OFFSET_CAPTURE); + + return $matches; + } + + public function getTokenType(): TokenTypeEnum + { + return TokenTypeEnum::KEYWORD; + } +} diff --git a/src/Languages/Twig/Patterns/CombinedTwigKeywordPattern.php b/src/Languages/Twig/Patterns/CombinedTwigKeywordPattern.php new file mode 100644 index 00000000..c6097602 --- /dev/null +++ b/src/Languages/Twig/Patterns/CombinedTwigKeywordPattern.php @@ -0,0 +1,50 @@ + strlen($b) <=> strlen($a)); + + $alternation = implode( + '|', + array_map( + static fn (string $keyword): string => preg_quote($keyword, '/'), + $keywords, + ), + ); + + $this->compiledPattern = "/\b(?(?:{$alternation}))\b/"; + } + + public function match(string $content): array + { + preg_match_all($this->compiledPattern, $content, $matches, PREG_OFFSET_CAPTURE); + + return $matches; + } + + public function getTokenType(): TokenTypeEnum + { + return TokenTypeEnum::KEYWORD; + } +} diff --git a/src/Languages/Twig/TwigTagLanguage.php b/src/Languages/Twig/TwigTagLanguage.php index 3d8c1f6d..4389db9d 100644 --- a/src/Languages/Twig/TwigTagLanguage.php +++ b/src/Languages/Twig/TwigTagLanguage.php @@ -6,10 +6,10 @@ use Override; use Tempest\Highlight\Languages\Base\BaseLanguage; +use Tempest\Highlight\Languages\Twig\Patterns\CombinedTwigKeywordPattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigArrayKeyPattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigDoubleQuoteValuePattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigFilterPattern; -use Tempest\Highlight\Languages\Twig\Patterns\TwigKeywordPattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigMethodPattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigNamedArgumentPattern; use Tempest\Highlight\Languages\Twig\Patterns\TwigPropertyPattern; @@ -40,44 +40,45 @@ public function getPatterns(): array new TwigSingleQuoteValuePattern(), // KEYWORDS - new TwigKeywordPattern('set'), - new TwigKeywordPattern('apply'), - new TwigKeywordPattern('autoescape'), - new TwigKeywordPattern('endautoescape'), - new TwigKeywordPattern('block'), - new TwigKeywordPattern('endblock'), - new TwigKeywordPattern('cache'), - new TwigKeywordPattern('deprecated'), - new TwigKeywordPattern('do'), - new TwigKeywordPattern('embed'), - new TwigKeywordPattern('extends'), - new TwigKeywordPattern('flush'), - new TwigKeywordPattern('for'), - new TwigKeywordPattern('endfor'), - new TwigKeywordPattern('from'), - new TwigKeywordPattern('if'), - new TwigKeywordPattern('else'), - new TwigKeywordPattern('elseif'), - new TwigKeywordPattern('endif'), - new TwigKeywordPattern('import'), - new TwigKeywordPattern('include'), - new TwigKeywordPattern('macro'), - new TwigKeywordPattern('endmacro'), - new TwigKeywordPattern('sandbox'), - new TwigKeywordPattern('set'), - new TwigKeywordPattern('use'), - new TwigKeywordPattern('verbatim'), - new TwigKeywordPattern('with'), - new TwigKeywordPattern('end'), - new TwigKeywordPattern('as'), - new TwigKeywordPattern('in'), - new TwigKeywordPattern('endapply'), - new TwigKeywordPattern('endcache'), - new TwigKeywordPattern('endembed'), - new TwigKeywordPattern('endset'), - new TwigKeywordPattern('endsandbox'), - new TwigKeywordPattern('endverbatim'), - new TwigKeywordPattern('endwith'), + new CombinedTwigKeywordPattern([ + 'set', + 'apply', + 'autoescape', + 'endautoescape', + 'block', + 'endblock', + 'cache', + 'deprecated', + 'do', + 'embed', + 'extends', + 'flush', + 'for', + 'endfor', + 'from', + 'if', + 'else', + 'elseif', + 'endif', + 'import', + 'include', + 'macro', + 'endmacro', + 'sandbox', + 'use', + 'verbatim', + 'with', + 'end', + 'as', + 'in', + 'endapply', + 'endcache', + 'endembed', + 'endset', + 'endsandbox', + 'endverbatim', + 'endwith', + ]), ]; } } From 8169d96376e6217df177e30cb7b12ba8ba6db060 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:46:55 +0400 Subject: [PATCH 03/15] fix(highlighter): key language caches by instance, reuse injected languages The pattern and injection caches were keyed by spl_object_id. Injections created a new language object for every match, so the caches missed on each nested parse, and a freed object's id could be reused by a different language, which then got the stale patterns. Store the caches in a WeakMap keyed by the language instance, and let the four injections that built a language per match reuse one instance. --- src/Highlighter.php | 55 +++++++++---------- .../Injections/JavaScriptInHtmlInjection.php | 5 +- .../JavaScript/Injections/JsDocInjection.php | 5 +- .../Twig/Injections/TwigEchoInjection.php | 5 +- .../Twig/Injections/TwigTagInjection.php | 5 +- tests/HighlighterTest.php | 22 ++++++++ 6 files changed, 65 insertions(+), 32 deletions(-) diff --git a/src/Highlighter.php b/src/Highlighter.php index 843cf5b6..6f248a5d 100644 --- a/src/Highlighter.php +++ b/src/Highlighter.php @@ -46,6 +46,7 @@ use Tempest\Highlight\Tokens\GroupTokens; use Tempest\Highlight\Tokens\ParseTokens; use Tempest\Highlight\Tokens\RenderTokens; +use WeakMap; final class Highlighter { @@ -56,12 +57,16 @@ final class Highlighter private readonly ParseTokens $parseTokens; private readonly GroupTokens $groupTokens; private readonly RenderTokens $renderTokens; - private array $patterns = []; + /** + * Per-language caches, keyed by the language instance itself so entries + * are dropped when a language is freed and can never be served to a + * different language that reuses its object id. Clones share this map, + * which is safe because entries depend only on the language. + * + * @var WeakMap, before: Injection[], after: Injection[]}> + */ + private WeakMap $languageCache; private array $afterInjections = []; - /** @var array */ - private array $beforeInjectionsCache = []; - /** @var array */ - private array $afterInjectionsCache = []; private ?self $nestedHighlighter = null; public function __construct( @@ -105,6 +110,7 @@ public function __construct( ->addLanguage(new JsxLanguage()) ->addLanguage(new TsxLanguage()); + $this->languageCache = new WeakMap(); $this->parseTokens = new ParseTokens(); $this->groupTokens = new GroupTokens(); $this->renderTokens = new RenderTokens($this->theme); @@ -241,15 +247,7 @@ private function parseContent(string $content, Language $language): string */ private function getBeforeInjections(Language $language): array { - $languageId = spl_object_id($language); - - if (isset($this->beforeInjectionsCache[$languageId])) { - return $this->beforeInjectionsCache[$languageId]; - } - - $this->buildInjectionCaches($language); - - return $this->beforeInjectionsCache[$languageId]; + return $this->getLanguageCache($language)['before']; } /** @@ -261,13 +259,7 @@ private function getAfterInjections(Language $language): array return []; } - $languageId = spl_object_id($language); - - if (! isset($this->afterInjectionsCache[$languageId])) { - $this->buildInjectionCaches($language); - } - - $afterInjections = $this->afterInjectionsCache[$languageId]; + $afterInjections = $this->getLanguageCache($language)['after']; if ($this->gutterInjection instanceof GutterInjection) { $afterInjections[] = $this->gutterInjection; @@ -276,9 +268,15 @@ private function getAfterInjections(Language $language): array return $afterInjections; } - private function buildInjectionCaches(Language $language): void + /** + * @return array{patterns: array, before: Injection[], after: Injection[]} + */ + private function getLanguageCache(Language $language): array { - $languageId = spl_object_id($language); + if (isset($this->languageCache[$language])) { + return $this->languageCache[$language]; + } + $before = []; $after = []; @@ -290,8 +288,11 @@ private function buildInjectionCaches(Language $language): void } } - $this->beforeInjectionsCache[$languageId] = $before; - $this->afterInjectionsCache[$languageId] = $after; + return $this->languageCache[$language] = [ + 'patterns' => $language->getPatterns(), + 'before' => $before, + 'after' => $after, + ]; } private function normalizeNewline(string $subject): string @@ -305,9 +306,7 @@ private function normalizeNewline(string $subject): string private function getPatterns(Language $language): array { - $languageId = spl_object_id($language); - - return $this->patterns[$languageId] ??= $language->getPatterns(); + return $this->getLanguageCache($language)['patterns']; } private function isAfterInjection(Injection $injection): bool diff --git a/src/Languages/Html/Injections/JavaScriptInHtmlInjection.php b/src/Languages/Html/Injections/JavaScriptInHtmlInjection.php index ef37d9f8..253dfbef 100644 --- a/src/Languages/Html/Injections/JavaScriptInHtmlInjection.php +++ b/src/Languages/Html/Injections/JavaScriptInHtmlInjection.php @@ -20,6 +20,9 @@ public function getPattern(): string public function parseContent(string $content, Highlighter $highlighter): string { - return $highlighter->parse($content, new JavaScriptLanguage()); + // One shared instance, so the highlighter's per-language caches hit on every match + static $language = new JavaScriptLanguage(); + + return $highlighter->parse($content, $language); } } diff --git a/src/Languages/JavaScript/Injections/JsDocInjection.php b/src/Languages/JavaScript/Injections/JsDocInjection.php index 717fe07f..3c002c1d 100644 --- a/src/Languages/JavaScript/Injections/JsDocInjection.php +++ b/src/Languages/JavaScript/Injections/JsDocInjection.php @@ -20,6 +20,9 @@ public function getPattern(): string public function parseContent(string $content, Highlighter $highlighter): string { - return $highlighter->parse($content, new JsDocLanguage()); + // One shared instance, so the highlighter's per-language caches hit on every match + static $language = new JsDocLanguage(); + + return $highlighter->parse($content, $language); } } diff --git a/src/Languages/Twig/Injections/TwigEchoInjection.php b/src/Languages/Twig/Injections/TwigEchoInjection.php index 480cf227..8248c153 100644 --- a/src/Languages/Twig/Injections/TwigEchoInjection.php +++ b/src/Languages/Twig/Injections/TwigEchoInjection.php @@ -21,6 +21,9 @@ public function getPattern(): string public function parseContent(string $content, Highlighter $highlighter): string { - return Escape::injection($highlighter->parse($content, new TwigEchoLanguage())); + // One shared instance, so the highlighter's per-language caches hit on every match + static $language = new TwigEchoLanguage(); + + return Escape::injection($highlighter->parse($content, $language)); } } diff --git a/src/Languages/Twig/Injections/TwigTagInjection.php b/src/Languages/Twig/Injections/TwigTagInjection.php index 5a73bb3a..ef556c35 100644 --- a/src/Languages/Twig/Injections/TwigTagInjection.php +++ b/src/Languages/Twig/Injections/TwigTagInjection.php @@ -21,6 +21,9 @@ public function getPattern(): string public function parseContent(string $content, Highlighter $highlighter): string { - return Escape::injection($highlighter->parse($content, new TwigTagLanguage())); + // One shared instance, so the highlighter's per-language caches hit on every match + static $language = new TwigTagLanguage(); + + return Escape::injection($highlighter->parse($content, $language)); } } diff --git a/tests/HighlighterTest.php b/tests/HighlighterTest.php index f92e672e..d6651523 100644 --- a/tests/HighlighterTest.php +++ b/tests/HighlighterTest.php @@ -12,6 +12,8 @@ use Tempest\Highlight\IsInjection; use Tempest\Highlight\Language; use Tempest\Highlight\Languages\Php\PhpLanguage; +use Tempest\Highlight\Languages\Sql\SqlLanguage; +use Tempest\Highlight\Languages\Text\TextLanguage; use Tempest\Highlight\Themes\InlineTheme; use Tempest\Highlight\Tokens\TokenTypeEnum; @@ -171,4 +173,24 @@ public function text_language_does_not_use_the_configured_fallback_language(): v $this->assertSame('resources/list', $highlighter->parse('resources/list', 'text')); $this->assertSame('Sec-Fetch-Site', $highlighter->parse('Sec-Fetch-Site', 'txt')); } + + #[Test] + public function freed_language_cache_is_not_reused_by_a_new_language(): void + { + $base = new Highlighter(); + $base->parse('SELECT * FROM users', new SqlLanguage()); + + $highlighter = $base->withGutter(); + unset($base); + + // Drops the last reference to the SqlLanguage, so PHP can hand its + // object id to the next object, here the PhpLanguage below + $highlighter->setCurrentLanguage(new TextLanguage()); + $highlighted = $highlighter->parse('assertSame( + '1<?php echo $foo;', + $highlighted, + ); + } } From c85befb087fda7684d61ecfad1a1c61cd43921ce Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:46:55 +0400 Subject: [PATCH 04/15] fix(patterns): stop large multiline blocks from losing their highlighting Patterns used (.|\n)*? to match across lines. PCRE handles the alternation group with one backtracking frame per character, so a block of roughly 50KB or more exhausts the JIT stack. preg_match_all then returns false and the block is silently left unhighlighted. [\s\S] matches the same characters as (.|\n) without the group, so the output is unchanged and the size limit is gone. --- .../Base/Injections/AdditionInjection.php | 2 +- .../Base/Injections/CustomClassInjection.php | 2 +- .../Base/Injections/DeletionInjection.php | 2 +- src/Languages/Base/IsHighlightInjection.php | 2 +- .../Base/Patterns/InjectionTokenPattern.php | 2 +- .../Blade/Injections/BladePhpInjection.php | 2 +- .../Blade/Patterns/BladeCommentPattern.php | 2 +- .../Css/Patterns/CssCommentPattern.php | 2 +- .../Ellison/Injections/ParserInjection.php | 2 +- .../CssAttributeInHtmlInjection.php | 2 +- .../Html/Injections/CssInHtmlInjection.php | 2 +- .../Injections/JavaScriptInHtmlInjection.php | 2 +- .../Html/Injections/PhpInHtmlInjection.php | 2 +- .../JavaScript/Injections/JsDocInjection.php | 2 +- .../Patterns/JsMultilineCommentPattern.php | 2 +- .../PhpAttributeInstanceInjection.php | 2 +- .../Php/Injections/PhpDocCommentInjection.php | 2 +- .../PhpFunctionParametersInjection.php | 2 +- .../Php/Injections/PhpHeredocInjection.php | 2 +- .../MultilineSingleDocCommentPattern.php | 2 +- .../PyTripleDoubleQuoteStringPattern.php | 2 +- .../PyTripleSingleQuoteStringPattern.php | 2 +- .../Patterns/SqlMultilineCommentPattern.php | 2 +- .../Twig/Injections/TwigEchoInjection.php | 2 +- .../Twig/Injections/TwigTagInjection.php | 2 +- .../Twig/Patterns/TwigCommentPattern.php | 2 +- .../Xml/Patterns/XmlCommentPattern.php | 2 +- src/Themes/InlineTheme.php | 2 +- tests/HighlighterTest.php | 25 +++++++++++++++++++ 29 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/Languages/Base/Injections/AdditionInjection.php b/src/Languages/Base/Injections/AdditionInjection.php index a18fa557..2e0a3037 100644 --- a/src/Languages/Base/Injections/AdditionInjection.php +++ b/src/Languages/Base/Injections/AdditionInjection.php @@ -22,7 +22,7 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio $content = str_replace('❷span class=❹ignore❹❸{+❷/span❸', '{+', $content); $content = str_replace('❷span class=❹ignore❹❸+}❷/span❸', '+}', $content); - preg_match_all('/(\{\+)((.|\n)*?)(\+})/', $content, $matches, PREG_OFFSET_CAPTURE); + preg_match_all('/(\{\+)([\s\S]*?)(\+})/', $content, $matches, PREG_OFFSET_CAPTURE); $parsedOffset = 0; diff --git a/src/Languages/Base/Injections/CustomClassInjection.php b/src/Languages/Base/Injections/CustomClassInjection.php index 2edb9a72..ad5ef86f 100644 --- a/src/Languages/Base/Injections/CustomClassInjection.php +++ b/src/Languages/Base/Injections/CustomClassInjection.php @@ -14,7 +14,7 @@ { public function parse(string $content, Highlighter $highlighter): ParsedInjection { - $pattern = '/(?{\:(?[\w-]+)\:)(?(.|\n)*?)(?:})/'; + $pattern = '/(?{\:(?[\w-]+)\:)(?[\s\S]*?)(?:})/'; $tokens = []; diff --git a/src/Languages/Base/Injections/DeletionInjection.php b/src/Languages/Base/Injections/DeletionInjection.php index 7b96959e..04462d90 100644 --- a/src/Languages/Base/Injections/DeletionInjection.php +++ b/src/Languages/Base/Injections/DeletionInjection.php @@ -22,7 +22,7 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio $content = str_replace('❷span class=❹ignore❹❸{-❷/span❸', '{-', $content); $content = str_replace('❷span class=❹ignore❹❸-}❷/span❸', '-}', $content); - preg_match_all('/(?getToken(); - $pattern = '/(?{' . $token . ')(?(.|\n)*?)(?' . $token . '})(?!})/'; + $pattern = '/(?{' . $token . ')(?[\s\S]*?)(?' . $token . '})(?!})/'; preg_match_all($pattern, $content, $matches, PREG_OFFSET_CAPTURE); diff --git a/src/Languages/Base/Patterns/InjectionTokenPattern.php b/src/Languages/Base/Patterns/InjectionTokenPattern.php index d3fc47f9..7fa2ae28 100644 --- a/src/Languages/Base/Patterns/InjectionTokenPattern.php +++ b/src/Languages/Base/Patterns/InjectionTokenPattern.php @@ -15,7 +15,7 @@ public function getPattern(): string { - return Escape::INJECTION_TOKEN . '(?(.|\n)*?)' . Escape::INJECTION_TOKEN; + return Escape::INJECTION_TOKEN . '(?[\s\S]*?)' . Escape::INJECTION_TOKEN; } public function getTokenType(): TokenTypeEnum diff --git a/src/Languages/Blade/Injections/BladePhpInjection.php b/src/Languages/Blade/Injections/BladePhpInjection.php index a3bd9bf5..e15a0ca2 100644 --- a/src/Languages/Blade/Injections/BladePhpInjection.php +++ b/src/Languages/Blade/Injections/BladePhpInjection.php @@ -14,7 +14,7 @@ public function getPattern(): string { - return '\@php(?(.|\n)*?)\@endphp'; + return '\@php(?[\s\S]*?)\@endphp'; } public function parseContent(string $content, Highlighter $highlighter): string diff --git a/src/Languages/Blade/Patterns/BladeCommentPattern.php b/src/Languages/Blade/Patterns/BladeCommentPattern.php index 7a69df23..eaabefe2 100644 --- a/src/Languages/Blade/Patterns/BladeCommentPattern.php +++ b/src/Languages/Blade/Patterns/BladeCommentPattern.php @@ -19,7 +19,7 @@ public function getPattern(): string { - return '(?\{\{\-\-(.|\n)*?\-\-\}\})'; + return '(?\{\{\-\-[\s\S]*?\-\-\}\})'; } public function getTokenType(): TokenTypeEnum diff --git a/src/Languages/Css/Patterns/CssCommentPattern.php b/src/Languages/Css/Patterns/CssCommentPattern.php index af72ec6b..9cafeb71 100644 --- a/src/Languages/Css/Patterns/CssCommentPattern.php +++ b/src/Languages/Css/Patterns/CssCommentPattern.php @@ -22,7 +22,7 @@ public function getPattern(): string { - return '/(?\/\*(.|\n)*?\*\/)/m'; + return '/(?\/\*[\s\S]*?\*\/)/m'; } public function getTokenType(): TokenTypeEnum diff --git a/src/Languages/Ellison/Injections/ParserInjection.php b/src/Languages/Ellison/Injections/ParserInjection.php index b60adad0..96ef981a 100644 --- a/src/Languages/Ellison/Injections/ParserInjection.php +++ b/src/Languages/Ellison/Injections/ParserInjection.php @@ -16,7 +16,7 @@ public function getPattern(): string { - return '(?(.|\n)*)'; + return '(?[\s\S]*)'; } public function parseContent(string $content, Highlighter $highlighter): string diff --git a/src/Languages/Html/Injections/CssAttributeInHtmlInjection.php b/src/Languages/Html/Injections/CssAttributeInHtmlInjection.php index f53a3c38..5bf302de 100644 --- a/src/Languages/Html/Injections/CssAttributeInHtmlInjection.php +++ b/src/Languages/Html/Injections/CssAttributeInHtmlInjection.php @@ -14,7 +14,7 @@ public function getPattern(): string { - return 'style="(?(.|\n)*?)"'; + return 'style="(?[\s\S]*?)"'; } public function parseContent(string $content, Highlighter $highlighter): string diff --git a/src/Languages/Html/Injections/CssInHtmlInjection.php b/src/Languages/Html/Injections/CssInHtmlInjection.php index 5f51cabc..092c7178 100644 --- a/src/Languages/Html/Injections/CssInHtmlInjection.php +++ b/src/Languages/Html/Injections/CssInHtmlInjection.php @@ -14,7 +14,7 @@ public function getPattern(): string { - return '')] final readonly class CssInHtmlInjection implements Injection { use IsInjection; public function getPattern(): string { - return '')] final class VueStyleScopedInjection implements Injection { use IsInjection; diff --git a/src/Languages/Vue/Injections/VueStyleScssInjection.php b/src/Languages/Vue/Injections/VueStyleScssInjection.php index b333ba3a..1182dac2 100644 --- a/src/Languages/Vue/Injections/VueStyleScssInjection.php +++ b/src/Languages/Vue/Injections/VueStyleScssInjection.php @@ -7,7 +7,9 @@ use Tempest\Highlight\Highlighter; use Tempest\Highlight\Injection; use Tempest\Highlight\IsInjection; +use Tempest\Highlight\RequiresSubstring; +#[RequiresSubstring('lang="s')] final class VueStyleScssInjection implements Injection { use IsInjection; diff --git a/src/Languages/Vue/Injections/VueTypeScriptInjection.php b/src/Languages/Vue/Injections/VueTypeScriptInjection.php index a9e45c17..77f67661 100644 --- a/src/Languages/Vue/Injections/VueTypeScriptInjection.php +++ b/src/Languages/Vue/Injections/VueTypeScriptInjection.php @@ -7,7 +7,9 @@ use Tempest\Highlight\Highlighter; use Tempest\Highlight\Injection; use Tempest\Highlight\IsInjection; +use Tempest\Highlight\RequiresSubstring; +#[RequiresSubstring('lang="ts"')] final class VueTypeScriptInjection implements Injection { use IsInjection; diff --git a/tests/Languages/Php/Injections/PhpGenericTypeInjectionTest.php b/tests/Languages/Php/Injections/PhpGenericTypeInjectionTest.php new file mode 100644 index 00000000..6dc850ee --- /dev/null +++ b/tests/Languages/Php/Injections/PhpGenericTypeInjectionTest.php @@ -0,0 +1,25 @@ + Escape::tokens('') . $type . Escape::tokens(''); + + $parsed = new PhpGenericTypeInjection()->parse(' * Wraps in ', new Highlighter()); + + $this->assertSame( + ' * Wraps <' . $generic('b') . '> in <' . $generic('span') . '>', + $parsed->content, + ); + } +} From 948bd4a6c315cf3daccf5db7dc64ff9576e16794 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:22:15 +0400 Subject: [PATCH 10/15] fix(tokens): only freeze immutable patterns from this package Compiled regexes were reused for any IsPattern class under the Tempest\Highlight namespace. KeywordPattern::caseInsensitive() and custom classes in that namespace could change their pattern between parses, and the old regex kept being used. Freeze only classes whose file lives in this package's src directory and whose properties are all readonly. OperatorPattern and TwigKeywordPattern never change after construction, so their properties are now readonly and they keep the fast path. --- .../Php/Patterns/OperatorPattern.php | 2 +- .../Twig/Patterns/TwigKeywordPattern.php | 2 +- src/Tokens/ParseTokens.php | 26 +++++++++++--- tests/Tokens/CompiledPatternsTest.php | 35 +++++++++++++++++++ tests/Tokens/Fixtures/ConfigurablePattern.php | 33 +++++++++++++++++ 5 files changed, 91 insertions(+), 7 deletions(-) create mode 100644 tests/Tokens/Fixtures/ConfigurablePattern.php diff --git a/src/Languages/Php/Patterns/OperatorPattern.php b/src/Languages/Php/Patterns/OperatorPattern.php index c7b4526a..501582a3 100644 --- a/src/Languages/Php/Patterns/OperatorPattern.php +++ b/src/Languages/Php/Patterns/OperatorPattern.php @@ -12,7 +12,7 @@ final class OperatorPattern implements Pattern { use IsPattern; - public function __construct(private string $operator) + public function __construct(private readonly string $operator) { } diff --git a/src/Languages/Twig/Patterns/TwigKeywordPattern.php b/src/Languages/Twig/Patterns/TwigKeywordPattern.php index 2f351c71..bf0712be 100644 --- a/src/Languages/Twig/Patterns/TwigKeywordPattern.php +++ b/src/Languages/Twig/Patterns/TwigKeywordPattern.php @@ -12,7 +12,7 @@ final class TwigKeywordPattern implements Pattern { use IsPattern; - public function __construct(private string $keyword) + public function __construct(private readonly string $keyword) { } diff --git a/src/Tokens/ParseTokens.php b/src/Tokens/ParseTokens.php index 04e7b9fc..ee0fb80e 100644 --- a/src/Tokens/ParseTokens.php +++ b/src/Tokens/ParseTokens.php @@ -30,7 +30,7 @@ public function parse(string $content, array $patterns): array } /** - * Built-in patterns that use IsPattern::match() get their final regex and token type resolved once. + * Patterns from this package that use IsPattern::match() and only hold readonly state get their final regex and token type resolved once. * Every other pattern keeps calling its own match() and getTokenType() on each parse. * * @internal @@ -142,12 +142,28 @@ private function usesBuiltInMatch(Pattern $pattern): bool if (! isset($builtIn[$class])) { $reflection = new ReflectionClass($class); - // Anonymous classes are named after their interface, so they would pass the namespace check - $builtIn[$class] = str_starts_with($class, 'Tempest\\Highlight\\') - && ! $reflection->isAnonymous() - && $reflection->getMethod('match')->getFileName() === new ReflectionClass(IsPattern::class)->getFileName(); + $builtIn[$class] = ! $reflection->isAnonymous() + && str_starts_with((string) $reflection->getFileName(), dirname(__DIR__) . DIRECTORY_SEPARATOR) + && $reflection->getMethod('match')->getFileName() === new ReflectionClass(IsPattern::class)->getFileName() + && $this->hasOnlyReadonlyState($reflection); } return $builtIn[$class]; } + + /** + * @param ReflectionClass $reflection + */ + private function hasOnlyReadonlyState(ReflectionClass $reflection): bool + { + do { + foreach ($reflection->getProperties() as $property) { + if (! $property->isReadOnly()) { + return false; + } + } + } while ($reflection = $reflection->getParentClass()); + + return true; + } } diff --git a/tests/Tokens/CompiledPatternsTest.php b/tests/Tokens/CompiledPatternsTest.php index 7bfec126..b0035c9d 100644 --- a/tests/Tokens/CompiledPatternsTest.php +++ b/tests/Tokens/CompiledPatternsTest.php @@ -4,12 +4,15 @@ namespace Tempest\Highlight\Tests\Tokens; +use ArrayObject; use PHPUnit\Framework\Attributes\Test; use PHPUnit\Framework\TestCase; use Tempest\Highlight\Highlighter; use Tempest\Highlight\IsPattern; use Tempest\Highlight\Language; +use Tempest\Highlight\Languages\Php\Patterns\KeywordPattern; use Tempest\Highlight\Pattern; +use Tempest\Highlight\Tests\Tokens\Fixtures\ConfigurablePattern; use Tempest\Highlight\Tests\Tokens\Fixtures\CustomMatchPattern; use Tempest\Highlight\Tokens\TokenTypeEnum; @@ -57,6 +60,38 @@ public function getTokenType(): TokenTypeEnum $this->assertSame('foo bar', $highlighter->parse('foo bar', $language)); } + #[Test] + public function built_in_patterns_with_mutable_state_are_asked_again_on_every_parse(): void + { + $pattern = new KeywordPattern('select'); + $highlighter = new Highlighter(); + $language = $this->language([$pattern]); + + $this->assertSame('select SELECT;', $highlighter->parse('select SELECT;', $language)); + + $pattern->caseInsensitive(); + + $this->assertSame( + 'select SELECT;', + $highlighter->parse('select SELECT;', $language), + ); + } + + #[Test] + public function patterns_outside_the_package_are_asked_again_on_every_parse_even_in_its_namespace(): void + { + $config = new ArrayObject(['word' => 'foo', 'type' => TokenTypeEnum::KEYWORD]); + $highlighter = new Highlighter(); + $language = $this->language([new ConfigurablePattern($config)]); + + $this->assertSame('foo bar', $highlighter->parse('foo bar', $language)); + + $config['word'] = 'bar'; + $config['type'] = TokenTypeEnum::TYPE; + + $this->assertSame('foo bar', $highlighter->parse('foo bar', $language)); + } + /** * @param Pattern[] $patterns */ diff --git a/tests/Tokens/Fixtures/ConfigurablePattern.php b/tests/Tokens/Fixtures/ConfigurablePattern.php new file mode 100644 index 00000000..4003ebf7 --- /dev/null +++ b/tests/Tokens/Fixtures/ConfigurablePattern.php @@ -0,0 +1,33 @@ + $config + */ + public function __construct( + private ArrayObject $config, + ) { + } + + public function getPattern(): string + { + return "(?{$this->config['word']})"; + } + + public function getTokenType(): TokenTypeEnum + { + return $this->config['type']; + } +} From 1090e1d11597e17fb7062273fc07b7754b6b39dd Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:22:29 +0400 Subject: [PATCH 11/15] fix(tokens): compare packed sort keys as integers SORT_NUMERIC compares integers as doubles. Packed keys above 2^53, reached from inputs of about 95MB, could compare equal, so a shorter token stayed ahead of a longer one starting at the same offset. The default flag compares two integers exactly. --- src/Tokens/GroupTokens.php | 2 +- tests/Tokens/GroupTokensTest.php | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/Tokens/GroupTokens.php b/src/Tokens/GroupTokens.php index efd506e7..2fb9bc0e 100644 --- a/src/Tokens/GroupTokens.php +++ b/src/Tokens/GroupTokens.php @@ -112,7 +112,7 @@ private function sortTokens(array $tokens): array $keys[] = ($token->start - $low) * $range + ($high - $token->end); } - asort($keys, SORT_NUMERIC); + asort($keys); $sorted = []; diff --git a/tests/Tokens/GroupTokensTest.php b/tests/Tokens/GroupTokensTest.php index c6c8b6f7..e3507a32 100644 --- a/tests/Tokens/GroupTokensTest.php +++ b/tests/Tokens/GroupTokensTest.php @@ -46,4 +46,19 @@ public function test_exact_overlaps_within_children(): void $this->assertCount(1, $tokens); $this->assertCount(3, $tokens[0]->children); } + + public function test_longer_token_wins_at_offsets_beyond_float_precision(): void + { + $tokens = [ + new Token(0, 'a', TokenTypeEnum::KEYWORD), + new Token(100_000_000, '/', TokenTypeEnum::OPERATOR), + new Token(100_000_001, '/', TokenTypeEnum::OPERATOR), + new Token(100_000_000, '//', TokenTypeEnum::COMMENT), + ]; + + $grouped = (new GroupTokens())($tokens); + + $this->assertCount(2, $grouped); + $this->assertSame(TokenTypeEnum::COMMENT, $grouped[1]->type); + } } From 15370218cd48dbf8a4118759f8f11b233a004208 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:22:41 +0400 Subject: [PATCH 12/15] fix(graphql): stop rescanning unclosed strings An unclosed string with escaped quotes took quadratic time: after the string failed, matching restarted at every escaped quote inside it and scanned the same text again. A 262KB input took about 2 seconds. Restarts inside the scanned body always fail the same way, so (*SKIP) resumes after it. Fuzzing 200k inputs against the previous regex found no difference in matches. --- .../Graphql/Patterns/GraphqlStringPattern.php | 3 ++- tests/Languages/Graphql/GraphqlLongLineTest.php | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Languages/Graphql/Patterns/GraphqlStringPattern.php b/src/Languages/Graphql/Patterns/GraphqlStringPattern.php index 82aa014e..40e62849 100644 --- a/src/Languages/Graphql/Patterns/GraphqlStringPattern.php +++ b/src/Languages/Graphql/Patterns/GraphqlStringPattern.php @@ -16,7 +16,8 @@ final class GraphqlStringPattern implements Pattern public function getPattern(): string { - return '"(?(?:[^"\\\\]++|\\\\.)*+)"'; + // An unclosed string would be rescanned from each escaped quote inside it, so skip past it + return '"(?(?:[^"\\\\]++|\\\\.)*+)(?:"|(*SKIP)(*FAIL))'; } public function getTokenType(): TokenTypeEnum diff --git a/tests/Languages/Graphql/GraphqlLongLineTest.php b/tests/Languages/Graphql/GraphqlLongLineTest.php index 92aa2dd0..8ff5e09f 100644 --- a/tests/Languages/Graphql/GraphqlLongLineTest.php +++ b/tests/Languages/Graphql/GraphqlLongLineTest.php @@ -28,4 +28,17 @@ public function test_long_word_keeps_later_fields(): void $highlighter->parse(str_repeat('a', 600000) . "\nfield: String", 'graphql'), ); } + + public function test_unterminated_string_with_escaped_quotes_is_parsed_in_linear_time(): void + { + $highlighter = new Highlighter(); + $content = 'f(a: "' . str_repeat('text \\" ', 32768) . "\\\n" . 'f(b: "x")'; + + $startedAt = hrtime(true); + $output = $highlighter->parse($content, 'graphql'); + $elapsedSeconds = (hrtime(true) - $startedAt) / 1_000_000_000; + + $this->assertStringContainsString('"x"', $output); + $this->assertLessThan(1.0, $elapsedSeconds); + } } From 966486641c9a3f6061b18cf2bc67fdb6b780f92a Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:23:05 +0400 Subject: [PATCH 13/15] fix(python): only treat _ as a keyword in case patterns Restoring the missing alternation also enabled the _ branch wherever a colon follows, so parameters, annotated names, dict keys, lambdas and slices styled _ as a keyword. Require a preceding "case" so only the wildcard in "case _:" matches. --- .../Python/Patterns/PyKeywordPattern.php | 2 +- .../Python/PythonPatternBugsTest.php | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/Languages/Python/Patterns/PyKeywordPattern.php b/src/Languages/Python/Patterns/PyKeywordPattern.php index ddab76aa..d27a2da7 100644 --- a/src/Languages/Python/Patterns/PyKeywordPattern.php +++ b/src/Languages/Python/Patterns/PyKeywordPattern.php @@ -25,7 +25,7 @@ public function getPattern(): string { $keywords = implode('|', $this->keywords); - return "\b(?(?:_(?=\s*:)|{$keywords}))\b"; + return "\b(?(?:(?<=\bcase\s)_(?=\s*:)|{$keywords}))\b"; } public function getTokenType(): TokenTypeEnum diff --git a/tests/Languages/Python/PythonPatternBugsTest.php b/tests/Languages/Python/PythonPatternBugsTest.php index 9b61390a..f53fa279 100644 --- a/tests/Languages/Python/PythonPatternBugsTest.php +++ b/tests/Languages/Python/PythonPatternBugsTest.php @@ -29,5 +29,30 @@ public static function provide_highlight_cases(): iterable 'case _:', 'case _:', ]; + + yield 'wildcard case in match block' => [ + "match x:\n case _:\n pass", + "match x:\n case _:\n pass", + ]; + + yield 'underscore parameter' => [ + 'def consume(_: int): pass', + 'def consume(_: int): pass', + ]; + + yield 'underscore annotated variable' => [ + '_: int = 1', + '_: int = 1', + ]; + + yield 'underscore dictionary key' => [ + 'mapping = {_: "translated"}', + 'mapping = {_: "translated"}', + ]; + + yield 'underscore lambda parameter' => [ + 'lambda _: 0', + 'lambda _: 0', + ]; } } From 98ce35e302af2eaa60240288dc2dc67fde9a84d1 Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:24:43 +0400 Subject: [PATCH 14/15] test(twig,gdscript): cover the combined keyword patterns Both patterns take their keyword list in the constructor, so the PatternTest attribute discovery cannot build them. Cover multiple matches with offsets, word boundaries, case, duplicates, the empty list, and equality with one pattern per keyword on the bench fixture. --- .../CombinedGdscriptKeywordPatternTest.php | 78 +++++++++++++++++++ .../CombinedTwigKeywordPatternTest.php | 78 +++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tests/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPatternTest.php create mode 100644 tests/Languages/Twig/Patterns/CombinedTwigKeywordPatternTest.php diff --git a/tests/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPatternTest.php b/tests/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPatternTest.php new file mode 100644 index 00000000..54501847 --- /dev/null +++ b/tests/Languages/Gdscript/Patterns/CombinedGdscriptKeywordPatternTest.php @@ -0,0 +1,78 @@ + $expected + */ + #[DataProvider('provide_matches_with_offsets_cases')] + public function test_matches_with_offsets(string $content, array $expected): void + { + $matches = new CombinedGdscriptKeywordPattern(self::KEYWORDS)->match($content); + + $this->assertSame($expected, $matches['match']); + } + + public static function provide_matches_with_offsets_cases(): iterable + { + yield 'multiple matches' => ['if x is not y and z', [['if', 0], ['is', 5], ['not', 8], ['and', 14]]]; + yield 'keyword that prefixes another' => ['class_name Foo', [['class_name', 0]]]; + yield 'keyword inside identifier' => ['classy inside island variable PIE', []]; + yield 'keyword next to underscore or digit' => ['var_x func2 _self', []]; + yield 'keyword next to punctuation' => ['self.x(PI)', [['self', 0], ['PI', 7]]]; + yield 'case sensitive' => ['Func VAR pi inf', []]; + yield 'no keywords' => ['x = 1', []]; + } + + public function test_matches_the_same_tokens_as_one_pattern_per_keyword(): void + { + $content = file_get_contents(__DIR__ . '/../../../Bench/Fixtures/gdscript.txt'); + + $expected = []; + + foreach (self::KEYWORDS as $keyword) { + foreach (new KeywordPattern($keyword)->match($content)['match'] as $match) { + $expected[] = $match; + } + } + + $actual = new CombinedGdscriptKeywordPattern(self::KEYWORDS)->match($content)['match']; + + usort($expected, static fn (array $a, array $b): int => $a[1] <=> $b[1]); + + $this->assertNotEmpty($actual); + $this->assertSame($expected, $actual); + } + + public function test_duplicate_keywords_match_once(): void + { + $matches = new CombinedGdscriptKeywordPattern(['var', 'var'])->match('var x'); + + $this->assertSame([['var', 0]], $matches['match']); + } + + public function test_token_type_is_keyword(): void + { + $this->assertSame(TokenTypeEnum::KEYWORD, new CombinedGdscriptKeywordPattern(['var'])->getTokenType()); + } + + public function test_requires_a_keyword(): void + { + $this->expectException(InvalidArgumentException::class); + + new CombinedGdscriptKeywordPattern([]); + } +} diff --git a/tests/Languages/Twig/Patterns/CombinedTwigKeywordPatternTest.php b/tests/Languages/Twig/Patterns/CombinedTwigKeywordPatternTest.php new file mode 100644 index 00000000..dac0f6b7 --- /dev/null +++ b/tests/Languages/Twig/Patterns/CombinedTwigKeywordPatternTest.php @@ -0,0 +1,78 @@ + $expected + */ + #[DataProvider('provide_matches_with_offsets_cases')] + public function test_matches_with_offsets(string $content, array $expected): void + { + $matches = new CombinedTwigKeywordPattern(self::KEYWORDS)->match($content); + + $this->assertSame($expected, $matches['match']); + } + + public static function provide_matches_with_offsets_cases(): iterable + { + yield 'multiple matches' => ['for item in items', [['for', 0], ['in', 9]]]; + yield 'keyword that prefixes another' => ['endif end endfor', [['endif', 0], ['end', 6], ['endfor', 10]]]; + yield 'keyword inside identifier' => ['settings ending format inside', []]; + yield 'keyword next to underscore or digit' => ['set_x if1 _for', []]; + yield 'keyword next to punctuation' => ['{%if(x)%}{%endif%}', [['if', 2], ['endif', 11]]]; + yield 'case sensitive' => ['SET If FOR', []]; + yield 'no keywords' => ['{{ name }}', []]; + } + + public function test_matches_the_same_tokens_as_one_pattern_per_keyword(): void + { + $content = file_get_contents(__DIR__ . '/../../../Bench/Fixtures/twig.txt'); + + $expected = []; + + foreach (self::KEYWORDS as $keyword) { + foreach (new TwigKeywordPattern($keyword)->match($content)['match'] as $match) { + $expected[] = $match; + } + } + + $actual = new CombinedTwigKeywordPattern(self::KEYWORDS)->match($content)['match']; + + usort($expected, static fn (array $a, array $b): int => $a[1] <=> $b[1]); + + $this->assertNotEmpty($actual); + $this->assertSame($expected, $actual); + } + + public function test_duplicate_keywords_match_once(): void + { + $matches = new CombinedTwigKeywordPattern(['if', 'if'])->match('if x'); + + $this->assertSame([['if', 0]], $matches['match']); + } + + public function test_token_type_is_keyword(): void + { + $this->assertSame(TokenTypeEnum::KEYWORD, new CombinedTwigKeywordPattern(['if'])->getTokenType()); + } + + public function test_requires_a_keyword(): void + { + $this->expectException(InvalidArgumentException::class); + + new CombinedTwigKeywordPattern([]); + } +} From 6be53a001e8b33ec9f0d74efc19879705b13f10e Mon Sep 17 00:00:00 2001 From: Arthur Minasyan <29732308+ArtMin96@users.noreply.github.com> Date: Fri, 23 Oct 2026 12:55:50 +0400 Subject: [PATCH 15/15] fix(python): highlight case wildcards from the case pattern context The `(?<=\bcase\s)_` lookbehind missed `case _:` and `case \t_:`, took its context from a comment ending in "case", and ignored wildcards in or, sequence, star, class and mapping patterns. PyCaseWildcardPattern scans `case` clauses whose enclosing block is a `match` statement (or the top level of a snippet) and marks `_` names in the pattern, skipping strings, comments, guards and bodies. Its results match the wildcards Python's ast reports across 2,467 generated programs. --- .../Python/Patterns/PyCaseWildcardPattern.php | 287 ++++++++++++++++++ .../Python/Patterns/PyKeywordPattern.php | 2 +- src/Languages/Python/PythonLanguage.php | 2 + .../Python/PythonPatternBugsTest.php | 100 ++++++ 4 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 src/Languages/Python/Patterns/PyCaseWildcardPattern.php diff --git a/src/Languages/Python/Patterns/PyCaseWildcardPattern.php b/src/Languages/Python/Patterns/PyCaseWildcardPattern.php new file mode 100644 index 00000000..0198e931 --- /dev/null +++ b/src/Languages/Python/Patterns/PyCaseWildcardPattern.php @@ -0,0 +1,287 @@ + []]; + } + + $strings = $this->multiLineStrings($content); + $string = 0; + $wildcards = []; + $blocks = []; + $scannedUntil = 0; + $length = strlen($content); + $lineStart = 0; + + while ($lineStart < $length) { + while (isset($strings[$string]) && $strings[$string][1] <= $lineStart) { + $string++; + } + + $insideString = ($strings[$string][0] ?? $lineStart) < $lineStart; + + // Lines that continue an earlier header, clause or string are not statements and do not affect indentation + if ($insideString || $lineStart < $scannedUntil) { + $lineStart = $this->nextLine($content, max($scannedUntil, $insideString ? $strings[$string][1] : 0)); + + continue; + } + + $indentLength = strspn($content, " \t\f", $lineStart); + $first = $lineStart + $indentLength; + + if (str_contains("#\r\n", $content[$first] ?? "\n")) { + $lineStart = $this->nextLine($content, $first); + + continue; + } + + $indent = $this->indentWidth(substr($content, $lineStart, $indentLength)); + + while ($blocks !== [] && $blocks[array_key_last($blocks)][0] >= $indent) { + array_pop($blocks); + } + + $enclosedByMatch = $blocks === [] || $blocks[array_key_last($blocks)][1]; + $isMatchHeader = false; + + if (preg_match('/\G(?:match|case)(?![\w\x80-\xff])/', $content, $keyword, 0, $first)) { + [$scannedUntil, $isStatement, $clauseWildcards] = $this->scan($content, $first + strlen($keyword[0]), $keyword[0] === 'case'); + + if ($keyword[0] === 'match') { + $isMatchHeader = $isStatement; + } elseif ($isStatement && $enclosedByMatch) { + array_push($wildcards, ...$clauseWildcards); + } + } + + $blocks[] = [$indent, $isMatchHeader]; + $lineStart = $this->nextLine($content, max($first, $scannedUntil)); + } + + return ['match' => $wildcards]; + } + + public function getTokenType(): TokenTypeEnum + { + return TokenTypeEnum::KEYWORD; + } + + /** + * Scans a `match` or `case` header from just after its keyword up to its top-level colon. + * + * @return array{int, bool, array} + */ + private function scan(string $content, int $index, bool $inPattern): array + { + $index += strspn($content, " \t\f", $index); + + if (str_contains("=.:)]},#\r\n", $content[$index] ?? "\n")) { + return [$index, false, []]; + } + + $length = strlen($content); + $depth = 0; + $previous = ''; + $wildcards = []; + + while ($index < $length) { + $char = $content[$index]; + + if ($char === ' ' || $char === "\t" || $char === "\f") { + $index++; + + continue; + } + + if ($char === "\n" || $char === "\r") { + if ($depth === 0) { + return [$index, false, []]; + } + + $index++; + + continue; + } + + if ($char === '#') { + $index += strcspn($content, "\r\n", $index); + + continue; + } + + if ($char === '\\') { + $index = $this->skipEscape($content, $index); + + continue; + } + + if ($char === ':' && $depth === 0) { + return [$index, true, $wildcards]; + } + + if ($char === '"' || $char === "'") { + $index = $this->skipString($content, $index); + $previous = '"'; + + continue; + } + + if (preg_match('/\G[\w\x80-\xff]+/', $content, $word, 0, $index)) { + $word = $word[0]; + $next = $index + strlen($word); + + if (($content[$next] ?? '') === '"' || ($content[$next] ?? '') === "'") { + if (preg_match('/^[rbuft]{1,2}$/i', $word)) { + $index = $this->skipString($content, $next); + $previous = '"'; + + continue; + } + } + + if ($word === 'if' && $depth === 0) { + $inPattern = false; + } elseif ($inPattern && $word === '_' && $previous !== '.' && ! preg_match('/\G[ \t\f]*(?:=(?!=)|\.)/', $content, $nameOrAttribute, 0, $next)) { + $wildcards[] = ['_', $index]; + } + + $index = $next; + $previous = 'a'; + + continue; + } + + // A pattern never assigns, calls or subscripts at the top level, so this is an ordinary statement using `case` as a name + if ($inPattern && $depth === 0 && ( + ($char === '=' && $previous !== '=' && ($content[$index + 1] ?? '') !== '=') + || (($char === '(' || $char === '[') && in_array($previous, [')', ']', '}'], true)) + )) { + return [$index, false, []]; + } + + if ($char === '(' || $char === '[' || $char === '{') { + $depth++; + } elseif ($char === ')' || $char === ']' || $char === '}') { + $depth = max(0, $depth - 1); + } + + $previous = $char; + $index++; + } + + return [$index, false, []]; + } + + /** + * @return array start and end offsets of the strings that span several lines + */ + private function multiLineStrings(string $content): array + { + // Comments and single-line strings are skipped, so only strings that may span lines are returned + preg_match_all( + <<<'REGEX' + / + (?: \#[^\r\n]*+ + | "(?!"")(?:[^"\\\r\n]++|\\[^\r\n])*+(?:"|(?=[\r\n]|\z)) + | '(?!'')(?:[^'\\\r\n]++|\\[^\r\n])*+(?:'|(?=[\r\n]|\z)) + ) (*SKIP)(*FAIL) + | """(?:[^"\\]++|\\[\s\S]?|"(?!""))*+(?:"""|\z) + | '''(?:[^'\\]++|\\[\s\S]?|'(?!''))*+(?:'''|\z) + | "(?:[^"\\\r\n]++|\\[\s\S]?)*+"? + | '(?:[^'\\\r\n]++|\\[\s\S]?)*+'? + /x + REGEX, + $content, + $tokens, + PREG_OFFSET_CAPTURE, + ); + + $strings = []; + + foreach ($tokens[0] as [$token, $offset]) { + if (strpbrk($token, "\r\n") !== false) { + $strings[] = [$offset, $offset + strlen($token)]; + } + } + + return $strings; + } + + private function skipString(string $content, int $index): int + { + $quote = $content[$index]; + $delimiter = substr($content, $index, 3) === $quote . $quote . $quote ? $quote . $quote . $quote : $quote; + $index += strlen($delimiter); + $length = strlen($content); + + while ($index < $length) { + $index += strcspn($content, "\\\r\n" . $quote, $index); + $char = $content[$index] ?? ''; + + if ($char === '\\') { + $index = $this->skipEscape($content, $index); + } elseif ($char !== $quote) { + // An unterminated single-line string ends at the newline + if ($delimiter === $quote) { + return $index; + } + + $index++; + } elseif (substr_compare($content, $delimiter, $index, strlen($delimiter)) === 0) { + return $index + strlen($delimiter); + } else { + $index++; + } + } + + return $length; + } + + private function nextLine(string $content, int $offset): int + { + $offset += strcspn($content, "\r\n", $offset); + + return $offset + (substr($content, $offset, 2) === "\r\n" ? 2 : 1); + } + + private function skipEscape(string $content, int $index): int + { + return $index + (substr_compare($content, "\\\r\n", $index, 3) === 0 ? 3 : 2); + } + + private function indentWidth(string $indentation): int + { + $width = 0; + + for ($i = 0, $length = strlen($indentation); $i < $length; $i++) { + $width = match ($indentation[$i]) { + "\t" => intdiv($width, 8) * 8 + 8, + "\f" => 0, + default => $width + 1, + }; + } + + return $width; + } +} diff --git a/src/Languages/Python/Patterns/PyKeywordPattern.php b/src/Languages/Python/Patterns/PyKeywordPattern.php index d27a2da7..ce43cb61 100644 --- a/src/Languages/Python/Patterns/PyKeywordPattern.php +++ b/src/Languages/Python/Patterns/PyKeywordPattern.php @@ -25,7 +25,7 @@ public function getPattern(): string { $keywords = implode('|', $this->keywords); - return "\b(?(?:(?<=\bcase\s)_(?=\s*:)|{$keywords}))\b"; + return "\b(?(?:{$keywords}))\b"; } public function getTokenType(): TokenTypeEnum diff --git a/src/Languages/Python/PythonLanguage.php b/src/Languages/Python/PythonLanguage.php index e9dc4ae2..a2a4d18e 100644 --- a/src/Languages/Python/PythonLanguage.php +++ b/src/Languages/Python/PythonLanguage.php @@ -10,6 +10,7 @@ use Tempest\Highlight\Languages\Php\Patterns\SingleQuoteValuePattern; use Tempest\Highlight\Languages\Python\Patterns\PyArgumentPattern; use Tempest\Highlight\Languages\Python\Patterns\PyBuiltinPattern; +use Tempest\Highlight\Languages\Python\Patterns\PyCaseWildcardPattern; use Tempest\Highlight\Languages\Python\Patterns\PyClassNamePattern; use Tempest\Highlight\Languages\Python\Patterns\PyCommentPattern; use Tempest\Highlight\Languages\Python\Patterns\PyDecoratorPattern; @@ -50,6 +51,7 @@ public function getPatterns(): array // KEYWORDS new PyKeywordPattern(), + new PyCaseWildcardPattern(), // PROPERTY new PyClassNamePattern(), diff --git a/tests/Languages/Python/PythonPatternBugsTest.php b/tests/Languages/Python/PythonPatternBugsTest.php index f53fa279..beffc4dc 100644 --- a/tests/Languages/Python/PythonPatternBugsTest.php +++ b/tests/Languages/Python/PythonPatternBugsTest.php @@ -54,5 +54,105 @@ public static function provide_highlight_cases(): iterable 'lambda _: 0', 'lambda _: 0', ]; + + yield 'wildcard case after two spaces' => [ + "match value:\n case _:\n pass", + "match value:\n case _:\n pass", + ]; + + yield 'wildcard case after space and tab' => [ + "match value:\n case \t_:\n pass", + "match value:\n case \t_:\n pass", + ]; + + yield 'underscore after comment ending in case' => [ + "# Special case\n_: int = 1", + "# Special case\n_: int = 1", + ]; + + yield 'wildcard in or pattern' => [ + "match x:\n case 1 | _:\n pass", + "match x:\n case 1 | _:\n pass", + ]; + + yield 'wildcard in sequence pattern' => [ + "match x:\n case (_, x):\n pass", + "match x:\n case (_, x):\n pass", + ]; + + yield 'wildcard star pattern' => [ + "match x:\n case [x, *_]:\n pass", + "match x:\n case [x, *_]:\n pass", + ]; + + yield 'wildcard in class pattern' => [ + "match x:\n case Point(x=_):\n pass", + "match x:\n case Point(x=_):\n pass", + ]; + + yield 'wildcard in mapping pattern' => [ + "match x:\n case {\"k\": _}:\n pass", + "match x:\n case {"k": _}:\n pass", + ]; + + yield 'underscore in guard' => [ + "match x:\n case _ if _ > 0:\n pass", + "match x:\n case _ if _ > 0:\n pass", + ]; + + yield 'underscore in one-line case body' => [ + "match x:\n case _: return _", + "match x:\n case _: return _", + ]; + + yield 'underscore in case string' => [ + "match x:\n case \"_\":\n pass", + "match x:\n case "_":\n pass", + ]; + + yield 'underscore in case comment' => [ + "match x:\n case _: # _ comment\n pass", + "match x:\n case _: # _ comment\n pass", + ]; + + yield 'underscore value pattern attribute' => [ + "match x:\n case Color._:\n pass", + "match x:\n case Color._:\n pass", + ]; + + yield 'case identifier with lambda' => [ + "case = lambda _: _", + "case = lambda _: _", + ]; + + yield 'case outside match block' => [ + "if x:\n case _:\n pass", + "if x:\n case _:\n pass", + ]; + + yield 'case subscript assigned a lambda' => [ + "case = {}\ncase[_] = lambda _: _", + "case = {}\ncase[_] = lambda _: _", + ]; + + yield 'case call subscript annotation' => [ + "case(_)[0]: int = 1", + "case(_)[0]: int = 1", + ]; + + yield 'underscore attribute in mapping key' => [ + "match x:\n case {_.y: _, 1: z}:\n pass", + "match x:\n case {_.y: _, 1: z}:\n pass", + ]; + + yield 'wildcard after multi-line match subject' => [ + "match (x,\ny):\n case _:\n pass", + "match (x,\ny):\n case _:\n pass", + ]; + + yield 'wildcard after multi-line string in case body' => [ + "match x:\n case 1:\n s = \"\"\"\ndoc\n\"\"\"\n case _:\n pass", + "match x:\n case 1:\n s = """\ndoc\n"""\n case _:\n pass", + ]; } }