From c903a060bc530891543f115737c4674b84bb8bae Mon Sep 17 00:00:00 2001 From: Akiyo974 Date: Tue, 20 Oct 2026 08:49:35 -0400 Subject: [PATCH 1/2] fix: render highlight markers correctly around nested and injected tokens --- src/Highlighter.php | 157 +++++++++++++++++- src/IsInjection.php | 30 +++- src/ParsedInjection.php | 6 + src/Tokens/GroupTokens.php | 27 ++- src/Tokens/Token.php | 18 +- tests/Languages/Base/HighlightMarkersTest.php | 129 ++++++++++++++ tests/Languages/Php/PhpLanguageTest.php | 2 +- tests/Tokens/GroupTokensTest.php | 46 +++++ 8 files changed, 402 insertions(+), 13 deletions(-) create mode 100644 tests/Languages/Base/HighlightMarkersTest.php diff --git a/src/Highlighter.php b/src/Highlighter.php index 843cf5b6..e4f14ed4 100644 --- a/src/Highlighter.php +++ b/src/Highlighter.php @@ -46,6 +46,8 @@ use Tempest\Highlight\Tokens\GroupTokens; use Tempest\Highlight\Tokens\ParseTokens; use Tempest\Highlight\Tokens\RenderTokens; +use Tempest\Highlight\Tokens\Token; +use Tempest\Highlight\Tokens\TokenTypeEnum; final class Highlighter { @@ -208,16 +210,34 @@ private function parseContent(string $content, Language $language): string $parsedInjection = $injection->parse($content, $nestedHighlighter); $content = $parsedInjection->content; + if ($tokens !== [] && $parsedInjection->replacements !== []) { + $tokens = $this->shiftTokens($tokens, $parsedInjection->replacements, $content); + } + foreach ($parsedInjection->tokens as $token) { $tokens[] = $token; } } // Patterns - foreach ( - $this->parseTokens->parse($content, $this->getPatterns($language)) as $token - ) { - $tokens[] = $token; + $markers = $this->getMarkers($tokens); + + if ($markers === []) { + foreach ( + $this->parseTokens->parse($content, $this->getPatterns($language)) as $token + ) { + $tokens[] = $token; + } + } else { + foreach ( + $this->parseTokens->parse($this->hideMarkers($content, $markers), $this->getPatterns($language)) as $token + ) { + $token = $this->trimMarkers($token, $markers, $content); + + if ($token instanceof Token) { + $tokens[] = $token; + } + } } $groupedTokens = ($this->groupTokens)($tokens); @@ -236,6 +256,135 @@ private function parseContent(string $content, Language $language): string return $this->theme->escape($content); } + /** + * Markers like {_ and _} stay in the content until it's rendered, as hidden tokens. + * + * @param Token[] $tokens + * @return Token[] + */ + private function getMarkers(array $tokens): array + { + $markers = []; + + foreach ($tokens as $token) { + if ($token->type === TokenTypeEnum::HIDDEN) { + $markers[] = $token; + } + } + + if (count($markers) > 2) { + usort($markers, static fn (Token $a, Token $b): int => $a->start <=> $b->start); + } + + return $markers; + } + + /** + * Patterns shouldn't see markers: `{_function` should still match the `function` keyword. + * They are replaced by spaces of the same length, so that all offsets stay the same. + * + * @param Token[] $markers + */ + private function hideMarkers(string $content, array $markers): string + { + foreach ($markers as $marker) { + $content = substr_replace($content, str_repeat(' ', $marker->length), $marker->offset, $marker->length); + } + + return $content; + } + + /** + * A token found by a pattern shouldn't start or end on a marker, otherwise it would + * swallow the marker and break the highlight around it. Its value is also taken from + * the real content again, since it was matched against the content with hidden markers. + * + * @param Token[] $markers + */ + private function trimMarkers(Token $token, array $markers, string $content): ?Token + { + $start = $token->start; + $end = $token->end; + + foreach ($markers as $marker) { + if ($marker->start <= $start && $start < $marker->end) { + $start = $marker->end; + } + } + + foreach (array_reverse($markers) as $marker) { + if ($marker->start < $end && $end <= $marker->end) { + $end = $marker->start; + } + } + + if ($start >= $end) { + return null; + } + + return new Token( + offset: $start, + value: substr($content, $start, $end - $start), + type: $token->type, + pattern: $token->pattern, + ); + } + + /** + * Moves tokens found by earlier injections (like {_ _} or {:class: :}) + * so that they still point to the same code after a later injection rewrote parts of it. + * Tokens that lie inside a rewritten part, or only partially cover one, + * are dropped: the injection already highlighted that part of the code itself. + * + * @param Token[] $tokens + * @param array $replacements + * @return Token[] + */ + private function shiftTokens(array $tokens, array $replacements, string $content): array + { + $shiftedTokens = []; + + foreach ($tokens as $token) { + $startDelta = 0; + $endDelta = 0; + + foreach ($replacements as [$offset, $oldLength, $newLength]) { + $delta = $newLength - $oldLength; + + if ($offset + $oldLength <= $token->start) { + $startDelta += $delta; + $endDelta += $delta; + + continue; + } + + if ($offset >= $token->end) { + break; + } + + if ($offset >= $token->start && $offset + $oldLength <= $token->end) { + $endDelta += $delta; + + continue; + } + + continue 2; + } + + $start = $token->start + $startDelta; + $end = $token->end + $endDelta; + + $shiftedTokens[] = new Token( + offset: $start, + value: substr($content, $start, $end - $start), + type: $token->type, + pattern: $token->pattern, + ); + } + + return $shiftedTokens; + } + /** * @return Injection[] */ diff --git a/src/IsInjection.php b/src/IsInjection.php index 5b8d6b0e..43c30e4f 100644 --- a/src/IsInjection.php +++ b/src/IsInjection.php @@ -18,24 +18,42 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio $pattern = "/{$pattern}/"; } + $replacements = []; + $result = preg_replace_callback( pattern: $pattern, - callback: function (array $matches) use ($highlighter): string { - $content = $matches['match'] ?? ''; + callback: function (array $matches) use ($highlighter, &$replacements): string { + [$fullMatch, $offset] = $matches[0]; + + $content = $matches['match'][0] ?? ''; if ($content === '' || $content === '0') { - return $matches[0]; + return $fullMatch; } - return str_replace( + $replacement = str_replace( search: $content, replace: $this->parseContent($content, $highlighter), - subject: $matches[0], + subject: $fullMatch, ); + + if ($replacement !== $fullMatch) { + $replacements[] = [$offset, strlen($fullMatch), strlen($replacement)]; + } + + return $replacement; }, subject: $content, + flags: PREG_OFFSET_CAPTURE, ); - return new ParsedInjection($result ?? $content); + if ($result === null) { + return new ParsedInjection($content); + } + + return new ParsedInjection( + content: $result, + replacements: $replacements, + ); } } diff --git a/src/ParsedInjection.php b/src/ParsedInjection.php index 12c77e67..c0520135 100644 --- a/src/ParsedInjection.php +++ b/src/ParsedInjection.php @@ -14,6 +14,10 @@ * to all other tokens detected by patterns, and thus follow all token rules. * They are grouped and checked on whether tokens can be contained by other tokens. * This offers more flexibility from the injection's point of view, and in same cases lead to more accurate highlighting. + * + * When an injection rewrites parts of the content, it can list those parts as replacements + * (offset and length in the original content, and the length of what replaced it), + * so that tokens found by earlier injections can be moved along with the content. */ final readonly class ParsedInjection { @@ -21,6 +25,8 @@ public function __construct( public string $content, /** @var Token[] */ public array $tokens = [], + /** @var array */ + public array $replacements = [], ) { } } diff --git a/src/Tokens/GroupTokens.php b/src/Tokens/GroupTokens.php index 970c8077..df50f4d2 100644 --- a/src/Tokens/GroupTokens.php +++ b/src/Tokens/GroupTokens.php @@ -34,6 +34,8 @@ public function __invoke(array $tokens): array $token = $tokens[$i]; $token->resetHierarchy(); $tokenEnd = $token->end; + $lastChildEnd = 0; + $childrenOverlap = false; // Since tokens are sorted by start, only check subsequent tokens // that could overlap (start < token->end) @@ -53,13 +55,36 @@ public function __invoke(array $tokens): array // At this point we know: token->start <= compareToken->start < token->end // and they are not equal (different indices, and sorted order means // same-start tokens differ in end). This means containsOrOverlaps is true. - if ($token->canContain($compareToken)) { + // A token that runs past the end of this one can't be nested in it, + // so it's dropped like any other token that can't be contained. + if ( + $compareToken->end <= $tokenEnd + && $token->canContain($compareToken) + ) { + if ($compareToken->start < $lastChildEnd) { + $childrenOverlap = true; + } + + $lastChildEnd = max($lastChildEnd, $compareToken->end); + $token->addChild($compareToken); } $removed[$j] = true; } + // Children can overlap each other as well (for example, a string and + // the keywords inside it, both wrapped in {_ _}), so they need to be + // grouped the same way before they can be rendered. + if ($childrenOverlap) { + $children = $token->children; + $token->children = []; + + foreach ($this($children) as $child) { + $token->addChild($child); + } + } + if (! $token->parent instanceof Token) { $groupedTokens[] = $token; } diff --git a/src/Tokens/Token.php b/src/Tokens/Token.php index 626f9f70..a5cd463a 100644 --- a/src/Tokens/Token.php +++ b/src/Tokens/Token.php @@ -74,15 +74,31 @@ public function canContain(Token $otherToken): bool { foreach ($this->children as $childToken) { // If there's already a child token at the exact position, - // we're not going to allow another child to be added + // we're not going to allow another child to be added, + // unless one of them is a highlight marker wrapping the other if ( $childToken->start === $otherToken->start && $childToken->end === $otherToken->end + && ! $childToken->type instanceof DynamicTokenType + && ! $otherToken->type instanceof DynamicTokenType ) { return false; } } + // Highlight markers like {_ _} or {:class: :} may be placed anywhere, + // including inside strings or comments, so built-in token types can + // always contain them. + if ( + ( + $otherToken->type === TokenTypeEnum::HIDDEN + || $otherToken->type instanceof DynamicTokenType + ) + && $this->type instanceof TokenTypeEnum + ) { + return true; + } + return $this->type->canContain($otherToken->type); } } diff --git a/tests/Languages/Base/HighlightMarkersTest.php b/tests/Languages/Base/HighlightMarkersTest.php new file mode 100644 index 00000000..dfd47f9f --- /dev/null +++ b/tests/Languages/Base/HighlightMarkersTest.php @@ -0,0 +1,129 @@ +assertSame( + $expected, + (new Highlighter())->parse($content, $language), + ); + } + + public static function provide_markers_wrap_highlighted_code_cases(): iterable + { + return [ + 'emphasize around a string with sql keywords (#148)' => [ + 'php', + '$foo = {_"SELECT * FROM users"_};', + '$foo = {_"SELECT * FROM users"_};', + ], + 'strong around a string with sql keywords' => [ + 'php', + '$foo = {*"SELECT * FROM users"*};', + '$foo = {*"SELECT * FROM users"*};', + ], + 'blur around a string with sql keywords' => [ + 'php', + '$foo = {~"SELECT * FROM users"~};', + '$foo = {~"SELECT * FROM users"~};', + ], + 'custom class around a string with sql keywords' => [ + 'php', + '$foo = {:hl-x:"SELECT * FROM users":};', + '$foo = "SELECT * FROM users";', + ], + 'emphasize around a whole line' => [ + 'php', + '{_$foo = "SELECT * FROM users";_}', + '{_$foo = "SELECT * FROM users";_}', + ], + 'emphasize inside a string' => [ + 'php', + '$foo = "Hello {_world_}";', + '$foo = "Hello {_world_}";', + ], + 'nested markers' => [ + 'php', + '{*{_"SELECT * FROM users"_}*}', + '{*{_"SELECT * FROM users"_}*}', + ], + 'emphasize around a function with parameters' => [ + 'php', + '{_public function foo(string $bar): Baz {}_}', + '{_public function foo(string $bar): Baz {}_}', + ], + 'custom class around a function with parameters (#166)' => [ + 'php', + '{:hl-x:function ($test) {:}', + 'function ($test) {', + ], + 'custom class around a closed function with parameters (#166)' => [ + 'php', + '{:hl-x:function ($test) {}:}', + 'function ($test) {}', + ], + 'emphasize around html with css in an attribute' => [ + 'html', + '{_
hi
_}', + '{_<div style="color: red">hi</div>_}', + ], + 'emphasize inside an html attribute' => [ + 'html', + '
hi
', + '<div style="{_color: red_}">hi</div>', + ], + 'emphasize around blade' => [ + 'blade', + '{_{{ $foo }} @if($x) y @endif_}', + '{_{{ $foo }} @if($x) <b>y</b> @endif_}', + ], + ]; + } + + #[DataProvider('provide_wrapping_code_in_markers_does_not_change_its_highlighting_cases')] + public function test_wrapping_code_in_markers_does_not_change_its_highlighting(string $language, string $content): void + { + $highlighter = new Highlighter(); + + $this->assertSame( + '{_' + . $highlighter->parse($content, $language) + . '_}', + $highlighter->parse('{_' . $content . '_}', $language), + ); + } + + public static function provide_wrapping_code_in_markers_does_not_change_its_highlighting_cases(): iterable + { + return [ + ['php', '$foo = "SELECT * FROM users";'], + ['php', 'public function foo(string $bar): Baz {}'], + ['php', 'function ($test) {}'], + ['html', '
hi
'], + ['blade', '{{ $foo }} @if($x) y @endif'], + ['javascript', 'const f = (a) => `x ${a}`;'], + ['css', '.a { color: red; }'], + ['sql', 'SELECT * FROM users WHERE id = "x"'], + ['python', 'query = "SELECT * FROM users"'], + ]; + } + + public function test_markers_work_with_gutter(): void + { + $highlighter = (new Highlighter())->withGutter(); + + $parsed = $highlighter->parse('{:hl-x:function ($test) {}:}', 'php'); + + $this->assertStringContainsString('($test) {}', $parsed); + } +} diff --git a/tests/Languages/Php/PhpLanguageTest.php b/tests/Languages/Php/PhpLanguageTest.php index 3fce0942..36d2bfc9 100644 --- a/tests/Languages/Php/PhpLanguageTest.php +++ b/tests/Languages/Php/PhpLanguageTest.php @@ -57,7 +57,7 @@ public static function provide_highlight_cases(): iterable ['$foo = false;', '$foo = false;'], ['/** @var Foo $var */', '/** @var Foo $var */'], ['{~}): Foo {}~}', '{~}): Foo {}~}'], - ['{~class~} Foo {}', '{~class~} Foo {}'], + ['{~class~} Foo {}', '{~class~} Foo {}'], ['#[ConsoleCommand()]', '#[ConsoleCommand()]'], ['#[ConsoleCommand]', '#[ConsoleCommand]'], ['#[ConsoleCommand(foo: [])]', '#[ConsoleCommand(foo: [])]'], diff --git a/tests/Tokens/GroupTokensTest.php b/tests/Tokens/GroupTokensTest.php index c6c8b6f7..5b4e1776 100644 --- a/tests/Tokens/GroupTokensTest.php +++ b/tests/Tokens/GroupTokensTest.php @@ -46,4 +46,50 @@ public function test_exact_overlaps_within_children(): void $this->assertCount(1, $tokens); $this->assertCount(3, $tokens[0]->children); } + + public function test_children_are_grouped_as_well(): void + { + $tokens = [ + new Token(0, '"SELECT * FROM users"', new DynamicTokenType('hl-em')), + new Token(0, '"SELECT * FROM users"', TokenTypeEnum::VALUE), + new Token(1, 'SELECT', TokenTypeEnum::PROPERTY), + new Token(10, 'FROM', TokenTypeEnum::PROPERTY), + ]; + + $grouped = (new GroupTokens())($tokens); + + $this->assertCount(1, $grouped); + $this->assertCount(1, $grouped[0]->children); + $this->assertSame(TokenTypeEnum::VALUE, $grouped[0]->children[0]->type); + $this->assertSame([], $grouped[0]->children[0]->children); + } + + public function test_token_running_past_its_parent_is_dropped(): void + { + $tokens = [ + new Token(0, '$foo = "SEL', new DynamicTokenType('hl-em')), + new Token(7, '"SELECT"', TokenTypeEnum::VALUE), + ]; + + $grouped = (new GroupTokens())($tokens); + + $this->assertCount(1, $grouped); + $this->assertSame([], $grouped[0]->children); + } + + public function test_markers_can_be_placed_inside_any_token(): void + { + $tokens = [ + new Token(0, '"Hello {_world_}"', TokenTypeEnum::VALUE), + new Token(7, '{_', TokenTypeEnum::HIDDEN), + new Token(9, 'world', new DynamicTokenType('hl-em')), + new Token(14, '_}', TokenTypeEnum::HIDDEN), + ]; + + $grouped = (new GroupTokens())($tokens); + + $this->assertCount(1, $grouped); + $this->assertCount(3, $grouped[0]->children); + $this->assertSame('hl-em', $grouped[0]->children[1]->type->getValue()); + } } From 9c1e283fe5aa943ae5026e141bd4d23d3a690e11 Mon Sep 17 00:00:00 2001 From: Akiyo974 Date: Tue, 20 Oct 2026 09:12:06 -0400 Subject: [PATCH 2/2] fix: keep markers next to injections and inside identifiers --- src/Highlighter.php | 173 ++++------ src/IsInjection.php | 25 +- src/Tokens/ShiftTokens.php | 317 ++++++++++++++++++ tests/Languages/Base/HighlightMarkersTest.php | 50 +++ tests/Languages/Php/PhpLanguageTest.php | 2 +- 5 files changed, 450 insertions(+), 117 deletions(-) create mode 100644 src/Tokens/ShiftTokens.php diff --git a/src/Highlighter.php b/src/Highlighter.php index e4f14ed4..6b65e535 100644 --- a/src/Highlighter.php +++ b/src/Highlighter.php @@ -43,9 +43,11 @@ use Tempest\Highlight\Languages\Xml\XmlLanguage; use Tempest\Highlight\Languages\Yaml\YamlLanguage; use Tempest\Highlight\Themes\CssTheme; +use Tempest\Highlight\Tokens\DynamicTokenType; use Tempest\Highlight\Tokens\GroupTokens; use Tempest\Highlight\Tokens\ParseTokens; use Tempest\Highlight\Tokens\RenderTokens; +use Tempest\Highlight\Tokens\ShiftTokens; use Tempest\Highlight\Tokens\Token; use Tempest\Highlight\Tokens\TokenTypeEnum; @@ -57,6 +59,7 @@ final class Highlighter private bool $isNested = false; private readonly ParseTokens $parseTokens; private readonly GroupTokens $groupTokens; + private readonly ShiftTokens $shiftTokens; private readonly RenderTokens $renderTokens; private array $patterns = []; private array $afterInjections = []; @@ -109,6 +112,7 @@ public function __construct( $this->parseTokens = new ParseTokens(); $this->groupTokens = new GroupTokens(); + $this->shiftTokens = new ShiftTokens(); $this->renderTokens = new RenderTokens($this->theme); } @@ -208,35 +212,24 @@ private function parseContent(string $content, Language $language): string // Before Injections foreach ($this->getBeforeInjections($language) as $injection) { $parsedInjection = $injection->parse($content, $nestedHighlighter); - $content = $parsedInjection->content; if ($tokens !== [] && $parsedInjection->replacements !== []) { - $tokens = $this->shiftTokens($tokens, $parsedInjection->replacements, $content); + $tokens = ($this->shiftTokens)($tokens, $parsedInjection->replacements, $content, $parsedInjection->content); } + $content = $parsedInjection->content; + foreach ($parsedInjection->tokens as $token) { $tokens[] = $token; } } // Patterns - $markers = $this->getMarkers($tokens); - - if ($markers === []) { - foreach ( - $this->parseTokens->parse($content, $this->getPatterns($language)) as $token - ) { - $tokens[] = $token; - } + if ($tokens === []) { + $tokens = $this->parseTokens->parse($content, $this->getPatterns($language)); } else { - foreach ( - $this->parseTokens->parse($this->hideMarkers($content, $markers), $this->getPatterns($language)) as $token - ) { - $token = $this->trimMarkers($token, $markers, $content); - - if ($token instanceof Token) { - $tokens[] = $token; - } + foreach ($this->parsePatternsAroundMarkers($content, $tokens, $language) as $token) { + $tokens[] = $token; } } @@ -257,132 +250,96 @@ private function parseContent(string $content, Language $language): string } /** - * Markers like {_ and _} stay in the content until it's rendered, as hidden tokens. + * Markers like {_ and _} stay in the content until it's rendered, as hidden tokens, but patterns shouldn't + * see them: `{_function` should match the `function` keyword, and `pre{_class_}post` shouldn't match `class`. + * Patterns run on the content without markers, and the tokens they find are moved back onto the real content. * - * @param Token[] $tokens + * Highlighted parts (from markers or custom classes) take precedence: a token that only partially + * overlaps one of them is dropped, so that the highlighted part can be rendered as a whole. + * + * @param Token[] $tokens Tokens found by injections * @return Token[] */ - private function getMarkers(array $tokens): array + private function parsePatternsAroundMarkers(string $content, array $tokens, Language $language): array { $markers = []; + $highlights = []; foreach ($tokens as $token) { if ($token->type === TokenTypeEnum::HIDDEN) { $markers[] = $token; + } elseif ($token->type instanceof DynamicTokenType) { + $highlights[] = $token; } } - if (count($markers) > 2) { - usort($markers, static fn (Token $a, Token $b): int => $a->start <=> $b->start); - } + usort($markers, static fn (Token $a, Token $b): int => $a->start <=> $b->start); - return $markers; - } + $contentWithoutMarkers = ''; + $removedMarkers = []; + $cursor = 0; - /** - * Patterns shouldn't see markers: `{_function` should still match the `function` keyword. - * They are replaced by spaces of the same length, so that all offsets stay the same. - * - * @param Token[] $markers - */ - private function hideMarkers(string $content, array $markers): string - { foreach ($markers as $marker) { - $content = substr_replace($content, str_repeat(' ', $marker->length), $marker->offset, $marker->length); + if ($marker->start < $cursor) { + continue; + } + + $contentWithoutMarkers .= substr($content, $cursor, $marker->start - $cursor); + $removedMarkers[] = $marker; + $cursor = $marker->end; } - return $content; - } + $contentWithoutMarkers .= substr($content, $cursor); - /** - * A token found by a pattern shouldn't start or end on a marker, otherwise it would - * swallow the marker and break the highlight around it. Its value is also taken from - * the real content again, since it was matched against the content with hidden markers. - * - * @param Token[] $markers - */ - private function trimMarkers(Token $token, array $markers, string $content): ?Token - { - $start = $token->start; - $end = $token->end; + $patternTokens = []; - foreach ($markers as $marker) { - if ($marker->start <= $start && $start < $marker->end) { - $start = $marker->end; + foreach ($this->parseTokens->parse($contentWithoutMarkers, $this->getPatterns($language)) as $token) { + if ($removedMarkers !== []) { + // A marker right at the start of a token comes before it, a marker right at its end comes after it + $start = $this->offsetWithMarkers($token->start, $removedMarkers, includeMarkersAtOffset: true); + $end = $this->offsetWithMarkers($token->end, $removedMarkers, includeMarkersAtOffset: false); + + $token = new Token( + offset: $start, + value: substr($content, $start, $end - $start), + type: $token->type, + pattern: $token->pattern, + ); } - } - foreach (array_reverse($markers) as $marker) { - if ($marker->start < $end && $end <= $marker->end) { - $end = $marker->start; + foreach ($highlights as $highlight) { + $crossesStart = $token->start < $highlight->start && $highlight->start < $token->end && $token->end < $highlight->end; + $crossesEnd = $highlight->start < $token->start && $token->start < $highlight->end && $highlight->end < $token->end; + + if ($crossesStart || $crossesEnd) { + continue 2; + } } - } - if ($start >= $end) { - return null; + $patternTokens[] = $token; } - return new Token( - offset: $start, - value: substr($content, $start, $end - $start), - type: $token->type, - pattern: $token->pattern, - ); + return $patternTokens; } /** - * Moves tokens found by earlier injections (like {_ _} or {:class: :}) - * so that they still point to the same code after a later injection rewrote parts of it. - * Tokens that lie inside a rewritten part, or only partially cover one, - * are dropped: the injection already highlighted that part of the code itself. - * - * @param Token[] $tokens - * @param array $replacements - * @return Token[] + * @param Token[] $markers */ - private function shiftTokens(array $tokens, array $replacements, string $content): array + private function offsetWithMarkers(int $offset, array $markers, bool $includeMarkersAtOffset): int { - $shiftedTokens = []; + $removedLength = 0; - foreach ($tokens as $token) { - $startDelta = 0; - $endDelta = 0; - - foreach ($replacements as [$offset, $oldLength, $newLength]) { - $delta = $newLength - $oldLength; - - if ($offset + $oldLength <= $token->start) { - $startDelta += $delta; - $endDelta += $delta; - - continue; - } - - if ($offset >= $token->end) { - break; - } - - if ($offset >= $token->start && $offset + $oldLength <= $token->end) { - $endDelta += $delta; - - continue; - } + foreach ($markers as $marker) { + $offsetWithoutMarkers = $marker->start - $removedLength; - continue 2; + if ($offsetWithoutMarkers > $offset || ($offsetWithoutMarkers === $offset && ! $includeMarkersAtOffset)) { + break; } - $start = $token->start + $startDelta; - $end = $token->end + $endDelta; - - $shiftedTokens[] = new Token( - offset: $start, - value: substr($content, $start, $end - $start), - type: $token->type, - pattern: $token->pattern, - ); + $removedLength += $marker->length; } - return $shiftedTokens; + return $offset + $removedLength; } /** diff --git a/src/IsInjection.php b/src/IsInjection.php index 43c30e4f..cc542e17 100644 --- a/src/IsInjection.php +++ b/src/IsInjection.php @@ -31,17 +31,26 @@ public function parse(string $content, Highlighter $highlighter): ParsedInjectio return $fullMatch; } - $replacement = str_replace( - search: $content, - replace: $this->parseContent($content, $highlighter), - subject: $fullMatch, - ); + $parsedContent = $this->parseContent($content, $highlighter); + + if ($parsedContent === $content) { + return $fullMatch; + } + + // Only the parsed content is rewritten, the rest of the match stays the same. + // Every occurrence that str_replace() will replace is reported on its own. + $position = 0; - if ($replacement !== $fullMatch) { - $replacements[] = [$offset, strlen($fullMatch), strlen($replacement)]; + while (($position = strpos($fullMatch, $content, $position)) !== false) { + $replacements[] = [$offset + $position, strlen($content), strlen($parsedContent)]; + $position += strlen($content); } - return $replacement; + return str_replace( + search: $content, + replace: $parsedContent, + subject: $fullMatch, + ); }, subject: $content, flags: PREG_OFFSET_CAPTURE, diff --git a/src/Tokens/ShiftTokens.php b/src/Tokens/ShiftTokens.php new file mode 100644 index 00000000..65b9d3fa --- /dev/null +++ b/src/Tokens/ShiftTokens.php @@ -0,0 +1,317 @@ + $replacements Offset and length in the old content, and the length of what replaced it + * @return Token[] + */ + public function __invoke(array $tokens, array $replacements, string $oldContent, string $newContent): array + { + $markerPairs = $this->getMarkerPairs($tokens); + $alignments = []; + $shiftedTokens = []; + + foreach ($tokens as $token) { + $range = $this->shift($token, $replacements, $oldContent, $newContent, $markerPairs[spl_object_id($token)] ?? null, $alignments); + + if ($range === null) { + continue; + } + + [$start, $end] = $range; + + $shiftedTokens[] = new Token( + offset: $start, + value: substr($newContent, $start, $end - $start), + type: $token->type, + pattern: $token->pattern, + ); + } + + return $shiftedTokens; + } + + /** + * Markers like {_ _} create three tokens: the hidden opening marker, the highlighted code, and the hidden closing marker. + * This returns the range covered by all three, for each of those tokens. + * + * @param Token[] $tokens + * @return array + */ + private function getMarkerPairs(array $tokens): array + { + $markersByStart = []; + $markersByEnd = []; + + foreach ($tokens as $token) { + if ($token->type === TokenTypeEnum::HIDDEN) { + $markersByStart[$token->start] = $token; + $markersByEnd[$token->end] = $token; + } + } + + $markerPairs = []; + + foreach ($tokens as $token) { + if (! $token->type instanceof DynamicTokenType) { + continue; + } + + $opening = $markersByEnd[$token->start] ?? null; + $closing = $markersByStart[$token->end] ?? null; + + if ($opening === null || $closing === null) { + continue; + } + + $range = [$opening->start, $closing->end]; + + $markerPairs[spl_object_id($opening)] = $range; + $markerPairs[spl_object_id($token)] = $range; + $markerPairs[spl_object_id($closing)] = $range; + } + + return $markerPairs; + } + + /** + * @param array $replacements + * @param array{int, int}|null $markerPair + * @param array|null> $alignments + * @return array{int, int}|null + */ + private function shift(Token $token, array $replacements, string $oldContent, string $newContent, ?array $markerPair, array &$alignments): ?array + { + $isHighlight = $token->type === TokenTypeEnum::HIDDEN || $token->type instanceof DynamicTokenType; + $delta = 0; + $start = null; + $end = null; + $startIsMapped = false; + $endIsMapped = false; + + foreach ($replacements as $index => [$offset, $oldLength, $newLength]) { + $replacementEnd = $offset + $oldLength; + + // Both markers of a pair like {_ _} are part of the rewritten code, so the injection has already highlighted them + if ($markerPair !== null && $markerPair[0] >= $offset && $markerPair[1] <= $replacementEnd) { + return null; + } + + if ($start === null && $token->start < $replacementEnd) { + // A token that starts right where the rewritten code starts, but ends inside of it, + // is moved inside of the rewritten code as a whole. + $startsInside = $token->start > $offset + || ($token->start === $offset && $token->end < $replacementEnd); + + if (! $startsInside) { + $start = $token->start + $delta; + } else { + // The token starts inside the rewritten code. Custom classes don't have their delimiters anymore, + // and a marker without its other half wasn't highlighted by the injection, so they are moved onto the rewritten code. + $alignment = $isHighlight ? $this->getAlignment($index, $replacements, $oldContent, $newContent, $alignments) : null; + + if ($alignment === null) { + return null; + } + + $start = $offset + $delta + $alignment[$token->start - $offset]; + $startIsMapped = true; + } + } + + if ($token->end <= $offset) { + $end = $token->end + $delta; + + break; + } + + if ($token->end < $replacementEnd || ($startIsMapped && $token->end === $replacementEnd)) { + // The token ends inside the rewritten code + $alignment = $isHighlight ? $this->getAlignment($index, $replacements, $oldContent, $newContent, $alignments) : null; + + if ($alignment === null) { + return null; + } + + $end = $offset + $delta + $alignment[$token->end - $offset - 1] + 1; + $endIsMapped = true; + + break; + } + + $delta += $newLength - $oldLength; + } + + $start ??= $token->start + $delta; + $end ??= $token->end + $delta; + + if (! $startIsMapped && ! $endIsMapped) { + return [$start, $end]; + } + + if ($this->isBalanced(substr($newContent, $start, $end - $start))) { + return [$start, $end]; + } + + // Include the opening markup right before the token and the closing markup right after it, + // for example when a custom class wraps several tokens that were highlighted by the injection. + while ($startIsMapped && ($markupStart = $this->openingMarkupEndingAt($newContent, $start)) !== null) { + $start = $markupStart; + } + + while ($endIsMapped && ($markupEnd = $this->closingMarkupStartingAt($newContent, $end)) !== null) { + $end = $markupEnd; + } + + if ($this->isBalanced(substr($newContent, $start, $end - $start))) { + return [$start, $end]; + } + + return null; + } + + /** + * The rewritten code is the original code with highlighting markup added to it, + * so every position in the original code can be found back by skipping that markup. + * + * @param array $replacements + * @param array|null> $alignments + * @return array|null Positions in the rewritten code, by position in the original code + */ + private function getAlignment(int $index, array $replacements, string $oldContent, string $newContent, array &$alignments): ?array + { + if (array_key_exists($index, $alignments)) { + return $alignments[$index]; + } + + $delta = 0; + + for ($i = 0; $i < $index; $i++) { + $delta += $replacements[$i][2] - $replacements[$i][1]; + } + + [$offset, $oldLength, $newLength] = $replacements[$index]; + + $original = substr($oldContent, $offset, $oldLength); + $rewritten = substr($newContent, $offset + $delta, $newLength); + + $positions = []; + $originalPosition = 0; + $rewrittenPosition = 0; + + while ($rewrittenPosition < $newLength) { + $markupEnd = $this->markupEndingAfter($rewritten, $rewrittenPosition); + + if ($markupEnd !== null) { + $rewrittenPosition = $markupEnd; + + continue; + } + + if (($original[$originalPosition] ?? null) !== $rewritten[$rewrittenPosition]) { + return $alignments[$index] = null; + } + + $positions[$originalPosition] = $rewrittenPosition; + $originalPosition++; + $rewrittenPosition++; + } + + if ($originalPosition !== $oldLength) { + return $alignments[$index] = null; + } + + return $alignments[$index] = $positions; + } + + private function markupEndingAfter(string $content, int $position): ?int + { + $character = substr($content, $position, 3); + + if ($character === Escape::INJECTION_TOKEN) { + return $position + 3; + } + + if ($character === '❷') { + $tagEnd = strpos($content, '❸', $position); + + return $tagEnd === false ? null : $tagEnd + 3; + } + + return null; + } + + private function openingMarkupEndingAt(string $content, int $position): ?int + { + if ($position < 3) { + return null; + } + + $previous = substr($content, $position - 3, 3); + + if ($previous === Escape::INJECTION_TOKEN) { + return $position - 3; + } + + if ($previous === '❸') { + $tagStart = strrpos(substr($content, 0, $position), '❷'); + + if ($tagStart === false || substr($content, $tagStart, 4) === '❷/') { + return null; + } + + return $tagStart; + } + + return null; + } + + private function closingMarkupStartingAt(string $content, int $position): ?int + { + $next = substr($content, $position, 4); + + if (str_starts_with($next, Escape::INJECTION_TOKEN)) { + return $position + 3; + } + + if ($next === '❷/') { + $tagEnd = strpos($content, '❸', $position); + + return $tagEnd === false ? null : $tagEnd + 3; + } + + return null; + } + + /** + * Whether the markup inside this content is properly nested, so that it can be wrapped in another tag. + */ + private function isBalanced(string $content): bool + { + $depth = 0; + + preg_match_all('/❷(\/?)/', $content, $tags); + + foreach ($tags[1] as $closing) { + $depth += $closing === '/' ? -1 : 1; + + if ($depth < 0) { + return false; + } + } + + return $depth === 0; + } +} diff --git a/tests/Languages/Base/HighlightMarkersTest.php b/tests/Languages/Base/HighlightMarkersTest.php index dfd47f9f..9d5f255f 100644 --- a/tests/Languages/Base/HighlightMarkersTest.php +++ b/tests/Languages/Base/HighlightMarkersTest.php @@ -87,6 +87,56 @@ public static function provide_markers_wrap_highlighted_code_cases(): iterable '{_{{ $foo }} @if($x) y @endif_}', '{_{{ $foo }} @if($x) <b>y</b> @endif_}', ], + 'emphasize inside an identifier' => [ + 'php', + 'pre{_class_}post', + 'pre{_class_}post', + ], + 'emphasize at the start of an identifier' => [ + 'php', + '{_pre_}class', + '{_pre_}class', + ], + 'custom class next to an injection' => [ + 'php', + '{:hl-x:function:} ($test) {}', + 'function ($test) {}', + ], + 'emphasize in a tag next to an embedded script' => [ + 'vue', + '', + '<script lang="ts" id="{_demo_}">const x = 1;</script>', + ], + 'custom class inside an embedded language' => [ + 'html', + '
hi
', + '<div style="color: red">hi</div>', + ], + 'custom class around several tokens of an embedded language' => [ + 'html', + '
hi
', + '<div style="color: red">hi</div>', + ], + 'custom classes inside function parameters' => [ + 'php', + 'function foo({:hl-x:$a:}, {:hl-y:$b:}) {}', + 'function foo($a, $b) {}', + ], + 'custom class around function parameters' => [ + 'php', + 'function foo({:hl-x:string $bar:}) {}', + 'function foo(string $bar) {}', + ], + 'marker pair split by an embedded language' => [ + 'blade', + '{~Welcome, {{~} $user->name }}', + '{~Welcome, {{~} $user->name }}', + ], + 'custom class starting inside an embedded language' => [ + 'svelte', + 'let { initial, onchan{:hl-x:ge }: { initi:}al } = $props();', + 'let { initial, onchange }: { initial } = $props();', + ], ]; } diff --git a/tests/Languages/Php/PhpLanguageTest.php b/tests/Languages/Php/PhpLanguageTest.php index 36d2bfc9..ab5c1fd4 100644 --- a/tests/Languages/Php/PhpLanguageTest.php +++ b/tests/Languages/Php/PhpLanguageTest.php @@ -57,7 +57,7 @@ public static function provide_highlight_cases(): iterable ['$foo = false;', '$foo = false;'], ['/** @var Foo $var */', '/** @var Foo $var */'], ['{~}): Foo {}~}', '{~}): Foo {}~}'], - ['{~class~} Foo {}', '{~class~} Foo {}'], + ['{~class~} Foo {}', '{~class~} Foo {}'], ['#[ConsoleCommand()]', '#[ConsoleCommand()]'], ['#[ConsoleCommand]', '#[ConsoleCommand]'], ['#[ConsoleCommand(foo: [])]', '#[ConsoleCommand(foo: [])]'],