fix: render highlight markers correctly around nested and injected tokens

src/Highlighter.php

@@ -43,9 +43,13 @@
 use Tempest\Highlight\Languages\Xml\XmlLanguage;
 use Tempest\Highlight\Languages\Yaml\YamlLanguage;
 use Tempest\Highlight\Themes\CssTheme;
+use Tempest\Highlight\Tokens\DynamicTokenType;
 use Tempest\Highlight\Tokens\GroupTokens;
 use Tempest\Highlight\Tokens\ParseTokens;
 use Tempest\Highlight\Tokens\RenderTokens;
+use Tempest\Highlight\Tokens\ShiftTokens;
+use Tempest\Highlight\Tokens\Token;
+use Tempest\Highlight\Tokens\TokenTypeEnum;
 
 final class Highlighter
 {
@@ -55,6 +59,7 @@ final class Highlighter
     private bool $isNested = false;
     private readonly ParseTokens $parseTokens;
     private readonly GroupTokens $groupTokens;
+    private readonly ShiftTokens $shiftTokens;
     private readonly RenderTokens $renderTokens;
     private array $patterns = [];
     private array $afterInjections = [];
@@ -107,6 +112,7 @@ public function __construct(
 
         $this->parseTokens = new ParseTokens();
         $this->groupTokens = new GroupTokens();
+        $this->shiftTokens = new ShiftTokens();
         $this->renderTokens = new RenderTokens($this->theme);
     }
 
@@ -206,6 +212,11 @@ private function parseContent(string $content, Language $language): string
         // Before Injections
         foreach ($this->getBeforeInjections($language) as $injection) {
             $parsedInjection = $injection->parse($content, $nestedHighlighter);
+
+            if ($tokens !== [] && $parsedInjection->replacements !== []) {
+                $tokens = ($this->shiftTokens)($tokens, $parsedInjection->replacements, $content, $parsedInjection->content);
+            }
+
             $content = $parsedInjection->content;
 
             foreach ($parsedInjection->tokens as $token) {
@@ -214,10 +225,12 @@ private function parseContent(string $content, Language $language): string
         }
 
         // Patterns
-        foreach (
-            $this->parseTokens->parse($content, $this->getPatterns($language)) as $token
-        ) {
-            $tokens[] = $token;
+        if ($tokens === []) {
+            $tokens = $this->parseTokens->parse($content, $this->getPatterns($language));
+        } else {
+            foreach ($this->parsePatternsAroundMarkers($content, $tokens, $language) as $token) {
+                $tokens[] = $token;
+            }
         }
 
         $groupedTokens = ($this->groupTokens)($tokens);
@@ -236,6 +249,99 @@ private function parseContent(string $content, Language $language): string
         return $this->theme->escape($content);
     }
 
+    /**
+     * Markers like {_ and _} stay in the content until it's rendered, as hidden tokens, but patterns shouldn't
+     * see them: `{_function` should match the `function` keyword, and `pre{_class_}post` shouldn't match `class`.
+     * Patterns run on the content without markers, and the tokens they find are moved back onto the real content.
+     *
+     * Highlighted parts (from markers or custom classes) take precedence: a token that only partially
+     * overlaps one of them is dropped, so that the highlighted part can be rendered as a whole.
+     *
+     * @param Token[] $tokens Tokens found by injections
+     * @return Token[]
+     */
+    private function parsePatternsAroundMarkers(string $content, array $tokens, Language $language): array
+    {
+        $markers = [];
+        $highlights = [];
+
+        foreach ($tokens as $token) {
+            if ($token->type === TokenTypeEnum::HIDDEN) {
+                $markers[] = $token;
+            } elseif ($token->type instanceof DynamicTokenType) {
+                $highlights[] = $token;
+            }
+        }
+
+        usort($markers, static fn (Token $a, Token $b): int => $a->start <=> $b->start);
+
+        $contentWithoutMarkers = '';
+        $removedMarkers = [];
+        $cursor = 0;
+
+        foreach ($markers as $marker) {
+            if ($marker->start < $cursor) {
+                continue;
+            }
+
+            $contentWithoutMarkers .= substr($content, $cursor, $marker->start - $cursor);
+            $removedMarkers[] = $marker;
+            $cursor = $marker->end;
+        }
+
+        $contentWithoutMarkers .= substr($content, $cursor);
+
+        $patternTokens = [];
+
+        foreach ($this->parseTokens->parse($contentWithoutMarkers, $this->getPatterns($language)) as $token) {
+            if ($removedMarkers !== []) {
+                // A marker right at the start of a token comes before it, a marker right at its end comes after it
+                $start = $this->offsetWithMarkers($token->start, $removedMarkers, includeMarkersAtOffset: true);
+                $end = $this->offsetWithMarkers($token->end, $removedMarkers, includeMarkersAtOffset: false);
+
+                $token = new Token(
+                    offset: $start,
+                    value: substr($content, $start, $end - $start),
+                    type: $token->type,
+                    pattern: $token->pattern,
+                );
+            }
+
+            foreach ($highlights as $highlight) {
+                $crossesStart = $token->start < $highlight->start && $highlight->start < $token->end && $token->end < $highlight->end;
+                $crossesEnd = $highlight->start < $token->start && $token->start < $highlight->end && $highlight->end < $token->end;
+
+                if ($crossesStart || $crossesEnd) {
+                    continue 2;
+                }
+            }
+
+            $patternTokens[] = $token;
+        }
+
+        return $patternTokens;
+    }
+
+    /**
+     * @param Token[] $markers
+     */
+    private function offsetWithMarkers(int $offset, array $markers, bool $includeMarkersAtOffset): int
+    {
+        $removedLength = 0;
+
+        foreach ($markers as $marker) {
+            $offsetWithoutMarkers = $marker->start - $removedLength;
+
+            if ($offsetWithoutMarkers > $offset || ($offsetWithoutMarkers === $offset && ! $includeMarkersAtOffset)) {
+                break;
+            }
+
+            $removedLength += $marker->length;
+        }
+
+        return $offset + $removedLength;
+    }
+
     /**
      * @return Injection[]
      */

src/IsInjection.php

@@ -18,24 +18,51 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio
             $pattern = "/{$pattern}/";
         }
 
+        $replacements = [];
+
         $result = preg_replace_callback(
             pattern: $pattern,
-            callback: function (array $matches) use ($highlighter): string {
-                $content = $matches['match'] ?? '';
+            callback: function (array $matches) use ($highlighter, &$replacements): string {
+                [$fullMatch, $offset] = $matches[0];
+
+                $content = $matches['match'][0] ?? '';
 
                 if ($content === '' || $content === '0') {
-                    return $matches[0];
+                    return $fullMatch;
+                }
+
+                $parsedContent = $this->parseContent($content, $highlighter);
+
+                if ($parsedContent === $content) {
+                    return $fullMatch;
+                }
+
+                // Only the parsed content is rewritten, the rest of the match stays the same.
+                // Every occurrence that str_replace() will replace is reported on its own.
+                $position = 0;
+
+                while (($position = strpos($fullMatch, $content, $position)) !== false) {
+                    $replacements[] = [$offset + $position, strlen($content), strlen($parsedContent)];
+                    $position += strlen($content);
                 }
 
                 return str_replace(
                     search: $content,
-                    replace: $this->parseContent($content, $highlighter),
-                    subject: $matches[0],
+                    replace: $parsedContent,
+                    subject: $fullMatch,
                 );
             },
             subject: $content,
+            flags: PREG_OFFSET_CAPTURE,
         );
 
-        return new ParsedInjection($result ?? $content);
+        if ($result === null) {
+            return new ParsedInjection($content);
+        }
+
+        return new ParsedInjection(
+            content: $result,
+            replacements: $replacements,
+        );
     }
 }

src/ParsedInjection.php

@@ -14,13 +14,19 @@
  * to all other tokens detected by patterns, and thus follow all token rules.
  * They are grouped and checked on whether tokens can be contained by other tokens.
  * This offers more flexibility from the injection's point of view, and in same cases lead to more accurate highlighting.
+ *
+ * When an injection rewrites parts of the content, it can list those parts as replacements
+ * (offset and length in the original content, and the length of what replaced it),
+ * so that tokens found by earlier injections can be moved along with the content.
  */
 final readonly class ParsedInjection
 {
     public function __construct(
         public string $content,
         /** @var Token[] */
         public array $tokens = [],
+        /** @var array<array{int, int, int}> */
+        public array $replacements = [],
     ) {
     }
 }

src/Tokens/GroupTokens.php

@@ -34,6 +34,8 @@ public function __invoke(array $tokens): array
             $token = $tokens[$i];
             $token->resetHierarchy();
             $tokenEnd = $token->end;
+            $lastChildEnd = 0;
+            $childrenOverlap = false;
 
             // Since tokens are sorted by start, only check subsequent tokens
             // that could overlap (start < token->end)
@@ -53,13 +55,36 @@ public function __invoke(array $tokens): array
                 // At this point we know: token->start <= compareToken->start < token->end
                 // and they are not equal (different indices, and sorted order means
                 // same-start tokens differ in end). This means containsOrOverlaps is true.
-                if ($token->canContain($compareToken)) {
+                // A token that runs past the end of this one can't be nested in it,
+                // so it's dropped like any other token that can't be contained.
+                if (
+                    $compareToken->end <= $tokenEnd
+                    && $token->canContain($compareToken)
+                ) {
+                    if ($compareToken->start < $lastChildEnd) {
+                        $childrenOverlap = true;
+                    }
+
+                    $lastChildEnd = max($lastChildEnd, $compareToken->end);
+
                     $token->addChild($compareToken);
                 }
 
                 $removed[$j] = true;
             }
 
+            // Children can overlap each other as well (for example, a string and
+            // the keywords inside it, both wrapped in {_ _}), so they need to be
+            // grouped the same way before they can be rendered.
+            if ($childrenOverlap) {
+                $children = $token->children;
+                $token->children = [];
+
+                foreach ($this($children) as $child) {
+                    $token->addChild($child);
+                }
+            }
+
             if (! $token->parent instanceof Token) {
                 $groupedTokens[] = $token;
             }