gh-153569: Simplify tokenizer state

Lib/test/test_fstring.py

@@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self):
                             f"{prefix[-1]}-string: expecting '}}' to close '{{' "
                             f"on line {lineno}")
 
+    def test_unclosed_replacement_field_quote_line(self):
+        for prefix in ('f', 't', 'rf', 'rt'):
+            for quote in ('"', "'"):
+                triple = quote * 3
+                for suffix in ('', '\nx'):
+                    source = prefix + triple + '{1' + triple + suffix
+                    with self.subTest(source=source):
+                        with self.assertRaises(SyntaxError) as cm:
+                            compile(source, '<test>', 'exec')
+                        self.assertEqual(
+                            cm.exception.msg,
+                            f"{prefix[-1]}-string: expecting '}}'")
+                        self.assertEqual(cm.exception.lineno, 1)
+
     @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI")
     def test_mismatched_parens(self):
         self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' "

Lib/test/test_repl.py

@@ -187,16 +187,19 @@ def read_until(marker, start=0):
     def test_lexer_buffer_realloc_with_null_start(self):
         # gh-144759: NULL pointer arithmetic when the lexer buffer grows
         # while parsing long input.
-        long_value = "a" * 2000
+        long_value = "é漢" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'
         print(x)
         """)
-        p = spawn_repl()
-        p.stdin.write(user_input)
-        output = kill_python(p)
-        self.assertEqual(p.returncode, 0)
-        self.assertIn(long_value, output)
+        for newline in ("\n", "\r\n"):
+            with self.subTest(newline=newline):
+                p = spawn_repl(encoding="utf-8")
+                # Bypass Windows text-mode translation of CRLF to CRCRLF.
+                p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8"))
+                output = kill_python(p)
+                self.assertEqual(p.returncode, 0)
+                self.assertIn(long_value, output)
 
     @cpython_only
     def test_multiline_fstring_source_reallocation(self):

Lib/test/test_source_encoding.py

@@ -135,6 +135,11 @@ def test_stateful_file_decoder_spans_lines(self):
         )
         self._assert_python_file_ok(source)
 
+    @support.requires_subprocess()
+    def test_stateful_file_decoder_preserves_buffered_text(self):
+        source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n"
+        self._assert_python_file_ok(source)
+
     @support.requires_subprocess()
     def test_stateful_file_decoder_finalizes_before_implicit_newline(self):
         source = b"# coding: hz\n# ~{1dA?"

Lib/test/test_tokenize.py

@@ -8,6 +8,7 @@
 import token
 import tokenize
 import unittest
+import weakref
 from io import BytesIO, StringIO
 from textwrap import dedent
 from unittest import TestCase, mock
@@ -2254,6 +2255,18 @@ def check_tokenize(self, s, expected):
             )
             self.assertEqual(result, expected.rstrip().splitlines())
 
+    def test_readline_reference_cycle(self):
+        class Readline:
+            def __call__(self):
+                return ""
+
+        readline = Readline()
+        readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
+        ref = weakref.ref(readline)
+        del readline
+        support.gc_collect()
+        self.assertIsNone(ref())
+
     def test_encoding(self):
         def readline(encoding):
             yield "1+1".encode(encoding)
@@ -2327,6 +2340,20 @@ def test_utf8_decoder_spans_readline_calls(self):
             tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""),
         ])
 
+    def test_utf8_decoder_spans_many_readline_calls(self):
+        for prefix in (b"", b"previous\n"):
+            with self.subTest(prefix=prefix):
+                chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100
+                          + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""])
+                source = b"".join(chunks)
+                expected = list(_tokenize.TokenizerIter(
+                    BytesIO(source).readline, encoding="utf-8", extra_tokens=True
+                ))
+                tokens = list(_tokenize.TokenizerIter(
+                    iter(chunks).__next__, encoding="utf-8", extra_tokens=True
+                ))
+                self.assertEqual(tokens, expected)
+
     def test_utf8_decoder_replaces_incomplete_input_at_eof(self):
         expected = [
             tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"),
@@ -2382,6 +2409,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self):
         )
         self.assertEqual(readline.call_count, 2)
 
+    def test_readline_memory_error_in_string(self):
+        readline = mock.Mock(side_effect=['"""first\n', MemoryError])
+        iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)
+        with self.assertRaises(MemoryError):
+            next(iterator)
+
     def test_readline_callback_is_not_read_ahead(self):
         readline = mock.Mock(side_effect=["x\n", "y\n", ""])
         iterator = _tokenize.TokenizerIter(readline, extra_tokens=True)

Lib/test/test_type_comments.py

@@ -338,6 +338,16 @@ def test_ignores(self):
         tree = self.classic_parse(ignores)
         self.assertEqual(tree.type_ignores, [])
 
+    def test_many_ignores(self):
+        comment_count = 25
+        tags = [f"[tag_{index}]" for index in range(comment_count)]
+        source = "".join(f"pass  # type: ignore{tag}\n" for tag in tags)
+        for tree in self.parse_all(source):
+            self.assertEqual(
+                [(item.lineno, item.tag) for item in tree.type_ignores],
+                list(enumerate(tags, start=1)),
+            )
+
     def test_longargs(self):
         for tree in self.parse_all(longargs, minver=8):
             for t in tree.body:

Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst

@@ -0,0 +1,4 @@
+Correct missing-brace error messages for multiline f-strings and t-strings.
+Improve tokenization of UTF-8 input split across many readline calls.
+Preserve source order when reading files with stateful encodings.
+Preserve input errors when scanning string literals.

Modules/_testinternalcapi/tokenizer.c

@@ -38,6 +38,12 @@ static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
+    const char multiple_lines[] = "a\nb\n";
+    const char first_line[] = "alpha\n";
+    const char second_line[] = "\xce\xb2\n";
+    const char expected[] = "alpha\n\xce\xb2\n";
+    const char tail[] = "tail";
+    const char terminated_line[] = "x\n";
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
@@ -46,23 +52,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     }
 
     if (check_system_error(
-            _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
+            _PyTok_SourceAppendLine(&source, "", 0) < 0,
             "accepted empty source line") < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, multiple_lines, sizeof(multiple_lines) - 1) < 0,
                 "accepted multiple source lines") < 0 ||
-            check_system_error(
-                _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0,
-                "accepted missing implicit newline") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "alpha\n", 6, 0) == 0,
+                      &source, first_line, sizeof(first_line) - 1) == 0,
                   "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
-                      &source, "\xce\xb2\n", 3, 1) == 6,
-                  "wrong second source offset") < 0 ||
-            check(!_PyTok_SourceLineIsImplicit(&source, 1) &&
-                      _PyTok_SourceLineIsImplicit(&source, 2),
-                  "wrong implicit newline flags") < 0) {
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      (Py_ssize_t)sizeof(first_line) - 1,
+                  "wrong second source offset") < 0) {
         goto error;
     }
 
@@ -74,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
-    if (check(source.len == 9 &&
-                  memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
+    if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 &&
+                  memcmp(source.bytes, expected, sizeof(expected)) == 0,
               "wrong source contents") < 0) {
         goto error;
     }
 
     _PyTok_SourceClear(&source);
-    if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 ||
+    if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 ||
             check_system_error(
-                _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0,
+                _PyTok_SourceAppendLine(
+                    &source, terminated_line, sizeof(terminated_line) - 1) < 0,
                 "appended after unterminated source line") < 0) {
         goto error;
     }
@@ -104,27 +107,35 @@ static PyObject *
 test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
                              PyObject *Py_UNUSED(args))
 {
+    enum { LINE_COUNT = 2 };
+    const char first_line[] = "x\n";
+    const char second_line[] = "y\n";
+    const char tail[] = "tail";
+    const char final_line[] = "z\n";
+    const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1);
+    const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1);
+    const _PyTok_Off discarded_len = first_batch_len + second_batch_len;
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
-    for (int i = 0; i < 260; i++) {
-        if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) {
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) {
             goto error;
         }
     }
     char *bytes = source.bytes;
     _PyTok_Off capacity = source.cap;
     _PyTok_SourceDiscard(&source);
-    if (check(source.base_offset == 520 && source.len == 0 &&
+    if (check(source.base_offset == first_batch_len && source.len == 0 &&
                   source.nlines == 0 && source.bytes == bytes &&
                   source.cap == capacity && source.bytes[0] == '\0',
               "discard did not preserve source allocation") < 0) {
         goto error;
     }
-    for (int i = 0; i < 260; i++) {
-        if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i,
-                  "wrong source offset after discard") < 0 ||
-                check(!_PyTok_SourceLineIsImplicit(&source, i + 1),
-                      "discard preserved implicit newline flag") < 0) {
+    for (int i = 0; i < LINE_COUNT; i++) {
+        if (check(_PyTok_SourceAppendLine(
+                      &source, second_line, sizeof(second_line) - 1) ==
+                      first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i,
+                  "wrong source offset after discard") < 0) {
             goto error;
         }
     }
@@ -133,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module),
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, tail, sizeof(tail) - 1) == discarded_len,
               "wrong source offset after repeated discard") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044,
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) ==
+                  discarded_len + (Py_ssize_t)sizeof(tail) - 1,
               "cannot append after discarding unterminated line") < 0) {
         goto error;
     }
     _PyTok_SourceDiscard(&source);
     source.base_offset = PY_SSIZE_T_MAX - 1;
-    if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 &&
+    if (check(_PyTok_SourceAppendLine(
+                  &source, final_line, sizeof(final_line) - 1) < 0 &&
                   PyErr_ExceptionMatches(PyExc_MemoryError),
               "accepted overflowing logical source offset") < 0) {
         goto error;

Parser/lexer/lexer.c

@@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token
 
                 // Handle valid f or t string creation:
                 if (saw_f || saw_t) {
-                    return _PyLexer_scan_fstring_start(tok, token, c);
+                    ftstring_kind kind = saw_t
+                        ? (saw_r ? RAW_TSTRING : TSTRING)
+                        : (saw_r ? RAW_FSTRING : FSTRING);
+                    return _PyLexer_scan_fstring_start(tok, token, c, kind);
                 }
                 return _PyLexer_scan_string(tok, token, c);
             }

Parser/lexer/lexer_internal.h

@@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr(
 void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *);
 int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);
 int _PyLexer_scan_number(struct tok_state *, struct token *, int, int);
-int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int);
+int _PyLexer_scan_fstring_start(
+    struct tok_state *, struct token *, int, ftstring_kind);
 int _PyLexer_scan_string(struct tok_state *, struct token *, int);
 int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *);
 int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *);

Parser/lexer/string.c

@@ -13,7 +13,7 @@
 {
     tok->diagnostic = (_PyTokenizer_Diagnostic){
         .location = {location.lineno, location.byte_col + 1},
-        .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp),
+        .text_span = {start - location.byte_col, tok->inp},
     };
     int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1);
     token->start_loc = location;
@@ -83,7 +83,7 @@
     PyObject *res;
     if (comments != NULL && comments->count > 0) {
         Py_ssize_t stripped_size = expr_len;
         _PyTok_Off previous_end = state->expr_span.start;
         Py_ssize_t comment_count = 0;
         for (Py_ssize_t i = 0; i < comments->count; i++) {
             _PyTok_Span comment = comments->spans[i];
@@ -254,7 +254,8 @@
 }
 
 int
-_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c)
+_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token,
+                            int c, ftstring_kind kind)
 {
     _PyTok_Off p_start = -1;
     _PyTok_Off p_end = -1;
@@ -293,30 +294,9 @@
     state->start_loc = tok->start_loc;
     state->expr_span = (_PyTok_Span){-1, -1};
 
-    int raw = 0;
-    int tstring = 0;
-    switch (*_PyLexer_BufferPointer(tok, tok->start)) {
-        case 'T':
-        case 't':
-            raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r';
-            tstring = 1;
-            break;
-        case 'F':
-        case 'f':
-            raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r';
-            break;
-        case 'R':
-        case 'r':
-            raw = 1;
-            tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't';
-            break;
-        default:
-            Py_UNREACHABLE();
-    }
-    state->kind = tstring
-        ? (raw ? RAW_TSTRING : TSTRING)
-        : (raw ? RAW_FSTRING : FSTRING);
-    return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
+    state->kind = kind;
+    return _PyLexer_IsTString(kind)
+        ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START);
 }
 
 int
@@ -355,6 +335,9 @@
             break;
         }
         if (c == EOF || (quote_size == 1 && c == '\n')) {
+            if (tok_failed(tok)) {
+                return MAKE_TOKEN(ERRORTOKEN);
+            }
             int end_lineno = tok->lineno;
             _PyTok_Loc location = tok->start_loc;
             const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col;
@@ -370,7 +353,7 @@
                     assert(level >= 0 && level < tok->level);
                     assert(tok->parenstack[level] == '{');
                     int lineno = tok->parenlinenostack[level];
-                    if (lineno != tok->lineno) {
+                    if (lineno != location.lineno) {
                         _PyTokenizer_syntaxerror_at(
                             tok, line, cursor_offset, location.lineno, -1, -1,
                             "%c-string: expecting '}' to close '{' on line %d",