diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 63006266bcd039e..72c80b8bfbfecc5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self): f"{prefix[-1]}-string: expecting '}}' to close '{{' " f"on line {lineno}") + def test_unclosed_replacement_field_quote_line(self): + for prefix in ('f', 't', 'rf', 'rt'): + for quote in ('"', "'"): + triple = quote * 3 + for suffix in ('', '\nx'): + source = prefix + triple + '{1' + triple + suffix + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual( + cm.exception.msg, + f"{prefix[-1]}-string: expecting '}}'") + self.assertEqual(cm.exception.lineno, 1) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 21e603c561d73c0..3525dee84493acc 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -187,16 +187,19 @@ def read_until(marker, start=0): def test_lexer_buffer_realloc_with_null_start(self): # gh-144759: NULL pointer arithmetic when the lexer buffer grows # while parsing long input. - long_value = "a" * 2000 + long_value = "é漢" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' print(x) """) - p = spawn_repl() - p.stdin.write(user_input) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(long_value, output) + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + # Bypass Windows text-mode translation of CRLF to CRCRLF. + p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8")) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) @cpython_only def test_multiline_fstring_source_reallocation(self): diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index ec98e609c4e98f9..63f3126c99925cc 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -135,6 +135,11 @@ def test_stateful_file_decoder_spans_lines(self): ) self._assert_python_file_ok(source) + @support.requires_subprocess() + def test_stateful_file_decoder_preserves_buffered_text(self): + source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n" + self._assert_python_file_ok(source) + @support.requires_subprocess() def test_stateful_file_decoder_finalizes_before_implicit_newline(self): source = b"# coding: hz\n# ~{1dA?" diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 53215eceeb8aed3..413fcfc41da6f3e 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -8,6 +8,7 @@ import token import tokenize import unittest +import weakref from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock @@ -2254,6 +2255,18 @@ def check_tokenize(self, s, expected): ) self.assertEqual(result, expected.rstrip().splitlines()) + def test_readline_reference_cycle(self): + class Readline: + def __call__(self): + return "" + + readline = Readline() + readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + ref = weakref.ref(readline) + del readline + support.gc_collect() + self.assertIsNone(ref()) + def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) @@ -2327,6 +2340,20 @@ def test_utf8_decoder_spans_readline_calls(self): tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""), ]) + def test_utf8_decoder_spans_many_readline_calls(self): + for prefix in (b"", b"previous\n"): + with self.subTest(prefix=prefix): + chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100 + + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""]) + source = b"".join(chunks) + expected = list(_tokenize.TokenizerIter( + BytesIO(source).readline, encoding="utf-8", extra_tokens=True + )) + tokens = list(_tokenize.TokenizerIter( + iter(chunks).__next__, encoding="utf-8", extra_tokens=True + )) + self.assertEqual(tokens, expected) + def test_utf8_decoder_replaces_incomplete_input_at_eof(self): expected = [ tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"), @@ -2382,6 +2409,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self): ) self.assertEqual(readline.call_count, 2) + def test_readline_memory_error_in_string(self): + readline = mock.Mock(side_effect=['"""first\n', MemoryError]) + iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + with self.assertRaises(MemoryError): + next(iterator) + def test_readline_callback_is_not_read_ahead(self): readline = mock.Mock(side_effect=["x\n", "y\n", ""]) iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index d827ac271085bdd..832a1e283e0c279 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -338,6 +338,16 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) + def test_many_ignores(self): + comment_count = 25 + tags = [f"[tag_{index}]" for index in range(comment_count)] + source = "".join(f"pass # type: ignore{tag}\n" for tag in tags) + for tree in self.parse_all(source): + self.assertEqual( + [(item.lineno, item.tag) for item in tree.type_ignores], + list(enumerate(tags, start=1)), + ) + def test_longargs(self): for tree in self.parse_all(longargs, minver=8): for t in tree.body: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst new file mode 100644 index 000000000000000..7bd6b023fb32da5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -0,0 +1,4 @@ +Correct missing-brace error messages for multiline f-strings and t-strings. +Improve tokenization of UTF-8 input split across many readline calls. +Preserve source order when reading files with stateful encodings. +Preserve input errors when scanning string literals. diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 1f89c12f223c7d9..0c2bdee2e803b22 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -38,6 +38,12 @@ static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + const char multiple_lines[] = "a\nb\n"; + const char first_line[] = "alpha\n"; + const char second_line[] = "\xce\xb2\n"; + const char expected[] = "alpha\n\xce\xb2\n"; + const char tail[] = "tail"; + const char terminated_line[] = "x\n"; _PyTok_SourceText source; _PyTok_SourceInit(&source); @@ -46,23 +52,19 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } if (check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, + _PyTok_SourceAppendLine( + &source, multiple_lines, sizeof(multiple_lines) - 1) < 0, "accepted multiple source lines") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6, 0) == 0, + &source, first_line, sizeof(first_line) - 1) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong implicit newline flags") < 0) { + &source, second_line, sizeof(second_line) - 1) == + (Py_ssize_t)sizeof(first_line) - 1, + "wrong second source offset") < 0) { goto error; } @@ -74,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(source.len == 9 && - memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 && + memcmp(source.bytes, expected, sizeof(expected)) == 0, "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || + if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, + _PyTok_SourceAppendLine( + &source, terminated_line, sizeof(terminated_line) - 1) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -104,27 +107,35 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + enum { LINE_COUNT = 2 }; + const char first_line[] = "x\n"; + const char second_line[] = "y\n"; + const char tail[] = "tail"; + const char final_line[] = "z\n"; + const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1); + const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1); + const _PyTok_Off discarded_len = first_batch_len + second_batch_len; _PyTok_SourceText source; _PyTok_SourceInit(&source); - for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) { goto error; } } char *bytes = source.bytes; _PyTok_Off capacity = source.cap; _PyTok_SourceDiscard(&source); - if (check(source.base_offset == 520 && source.len == 0 && + if (check(source.base_offset == first_batch_len && source.len == 0 && source.nlines == 0 && source.bytes == bytes && source.cap == capacity && source.bytes[0] == '\0', "discard did not preserve source allocation") < 0) { goto error; } - for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, - "wrong source offset after discard") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, i + 1), - "discard preserved implicit newline flag") < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (check(_PyTok_SourceAppendLine( + &source, second_line, sizeof(second_line) - 1) == + first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i, + "wrong source offset after discard") < 0) { goto error; } } @@ -133,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + if (check(_PyTok_SourceAppendLine( + &source, tail, sizeof(tail) - 1) == discarded_len, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) == + discarded_len + (Py_ssize_t)sizeof(tail) - 1, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index f27fee8d61b9085..070760deefd58b4 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token // Handle valid f or t string creation: if (saw_f || saw_t) { - return _PyLexer_scan_fstring_start(tok, token, c); + ftstring_kind kind = saw_t + ? (saw_r ? RAW_TSTRING : TSTRING) + : (saw_r ? RAW_FSTRING : FSTRING); + return _PyLexer_scan_fstring_start(tok, token, c, kind); } return _PyLexer_scan_string(tok, token, c); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index f0d9576d95f8ff3..652512a3e88c22d 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr( void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); -int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_fstring_start( + struct tok_state *, struct token *, int, ftstring_kind); int _PyLexer_scan_string(struct tok_state *, struct token *, int); int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a48cedf270362ab..b481ee42f7b33b2 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -13,7 +13,7 @@ string_error_token(struct tok_state *tok, struct token *token, { tok->diagnostic = (_PyTokenizer_Diagnostic){ .location = {location.lineno, location.byte_col + 1}, - .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + .text_span = {start - location.byte_col, tok->inp}, }; int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); token->start_loc = location; @@ -254,7 +254,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, } int -_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, + int c, ftstring_kind kind) { _PyTok_Off p_start = -1; _PyTok_Off p_end = -1; @@ -293,30 +294,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) state->start_loc = tok->start_loc; state->expr_span = (_PyTok_Span){-1, -1}; - int raw = 0; - int tstring = 0; - switch (*_PyLexer_BufferPointer(tok, tok->start)) { - case 'T': - case 't': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - tstring = 1; - break; - case 'F': - case 'f': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - break; - case 'R': - case 'r': - raw = 1; - tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't'; - break; - default: - Py_UNREACHABLE(); - } - state->kind = tstring - ? (raw ? RAW_TSTRING : TSTRING) - : (raw ? RAW_FSTRING : FSTRING); - return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = kind; + return _PyLexer_IsTString(kind) + ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int @@ -355,6 +335,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { + return MAKE_TOKEN(ERRORTOKEN); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col; @@ -370,7 +353,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(level >= 0 && level < tok->level); assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; - if (lineno != tok->lineno) { + if (lineno != location.lineno) { _PyTokenizer_syntaxerror_at( tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", diff --git a/Parser/parser.c b/Parser/parser.c index b3d73dd4fafb762..09ed5b81eeb9105 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27282,14 +27282,8 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // NEWLINE if (p->error_indicator) { @@ -27304,8 +27298,7 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27349,14 +27342,8 @@ _loop1_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // statement if (p->error_indicator) { @@ -27371,8 +27358,7 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27421,14 +27407,8 @@ _loop0_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ';' simple_stmt if (p->error_indicator) { @@ -27452,8 +27432,7 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27975,14 +27954,8 @@ _loop1_12_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -27997,8 +27970,7 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28047,14 +28019,8 @@ _loop0_13_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' NAME if (p->error_indicator) { @@ -28078,8 +28044,7 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28267,14 +28232,8 @@ _loop0_17_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28289,8 +28248,7 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28334,14 +28292,8 @@ _loop1_18_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28356,8 +28308,7 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28406,14 +28357,8 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' import_from_as_name if (p->error_indicator) { @@ -28437,8 +28382,7 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28569,14 +28513,8 @@ _loop0_22_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_as_name if (p->error_indicator) { @@ -28600,8 +28538,7 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28686,14 +28623,8 @@ _loop1_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('@' named_expression NEWLINE) if (p->error_indicator) { @@ -28708,8 +28639,7 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28853,14 +28783,8 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -28875,8 +28799,7 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28920,14 +28843,8 @@ _loop0_28_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -28942,8 +28859,7 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28987,14 +28903,8 @@ _loop1_29_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -29009,8 +28919,7 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29059,14 +28968,8 @@ _loop1_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29081,8 +28984,7 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29131,14 +29033,8 @@ _loop0_31_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29153,8 +29049,7 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29198,14 +29093,8 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29220,8 +29109,7 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29270,14 +29158,8 @@ _loop0_33_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' with_item if (p->error_indicator) { @@ -29301,8 +29183,7 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29463,14 +29344,8 @@ _loop1_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_block if (p->error_indicator) { @@ -29485,8 +29360,7 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29535,14 +29409,8 @@ _loop1_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_star_block if (p->error_indicator) { @@ -29557,8 +29425,7 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29607,14 +29474,8 @@ _loop1_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // case_block if (p->error_indicator) { @@ -29629,8 +29490,7 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29679,14 +29539,8 @@ _loop0_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // '|' closed_pattern if (p->error_indicator) { @@ -29710,8 +29564,7 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30005,14 +29858,8 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' maybe_star_pattern if (p->error_indicator) { @@ -30036,8 +29883,7 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30122,14 +29968,8 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' key_value_pattern if (p->error_indicator) { @@ -30153,8 +29993,7 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30296,14 +30135,8 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' pattern if (p->error_indicator) { @@ -30327,8 +30160,7 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30413,14 +30245,8 @@ _loop0_51_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' keyword_pattern if (p->error_indicator) { @@ -30444,8 +30270,7 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30530,14 +30355,8 @@ _loop0_53_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' type_param if (p->error_indicator) { @@ -30561,8 +30380,7 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30647,14 +30465,8 @@ _loop1_55_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' expression) if (p->error_indicator) { @@ -30669,8 +30481,7 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30719,14 +30530,8 @@ _loop1_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_expression) if (p->error_indicator) { @@ -30741,8 +30546,7 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30791,14 +30595,8 @@ _loop0_57_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' star_named_expression if (p->error_indicator) { @@ -30822,8 +30620,7 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30908,14 +30705,8 @@ _loop1_59_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('or' conjunction) if (p->error_indicator) { @@ -30930,8 +30721,7 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30980,14 +30770,8 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('and' inversion) if (p->error_indicator) { @@ -31002,8 +30786,7 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31052,14 +30835,8 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // compare_op_bitwise_or_pair if (p->error_indicator) { @@ -31074,8 +30851,7 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31167,14 +30943,8 @@ _loop0_63_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (slice | starred_expression) if (p->error_indicator) { @@ -31198,8 +30968,7 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31615,14 +31384,8 @@ _loop0_70_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31637,8 +31400,7 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31682,14 +31444,8 @@ _loop0_71_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31704,8 +31460,7 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31749,14 +31504,8 @@ _loop1_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31771,8 +31520,7 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31821,14 +31569,8 @@ _loop1_73_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31843,8 +31585,7 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31893,14 +31634,8 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31915,8 +31650,7 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31960,14 +31694,8 @@ _loop1_75_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31982,8 +31710,7 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32032,14 +31759,8 @@ _loop0_76_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_format_spec if (p->error_indicator) { @@ -32054,8 +31775,7 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32099,14 +31819,8 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_middle if (p->error_indicator) { @@ -32121,8 +31835,7 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32166,14 +31879,8 @@ _loop0_78_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_format_spec if (p->error_indicator) { @@ -32188,8 +31895,7 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32233,14 +31939,8 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_middle if (p->error_indicator) { @@ -32255,8 +31955,7 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32300,14 +31999,8 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (fstring | string) if (p->error_indicator) { @@ -32322,8 +32015,7 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32372,14 +32064,8 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring if (p->error_indicator) { @@ -32394,8 +32080,7 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32444,14 +32129,8 @@ _loop0_82_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' display_item if (p->error_indicator) { @@ -32475,8 +32154,7 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32610,14 +32288,8 @@ _loop0_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' double_starred_kvpair if (p->error_indicator) { @@ -32641,8 +32313,7 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32727,14 +32398,8 @@ _loop1_87_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // for_if_clause if (p->error_indicator) { @@ -32749,8 +32414,7 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32799,14 +32463,8 @@ _loop0_88_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('if' disjunction) if (p->error_indicator) { @@ -32821,8 +32479,7 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32944,14 +32601,8 @@ _loop0_90_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { @@ -32975,8 +32626,7 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33108,14 +32758,8 @@ _loop0_93_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_starred if (p->error_indicator) { @@ -33139,8 +32783,7 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33271,14 +32914,8 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_double_starred if (p->error_indicator) { @@ -33302,8 +32939,7 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33388,14 +33024,8 @@ _loop0_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_target) if (p->error_indicator) { @@ -33410,8 +33040,7 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33571,14 +33200,8 @@ _loop0_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' del_target if (p->error_indicator) { @@ -33602,8 +33225,7 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33688,14 +33310,8 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' expression if (p->error_indicator) { @@ -33719,8 +33335,7 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33905,14 +33520,8 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression !'=') if (p->error_indicator) { @@ -33936,8 +33545,7 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34221,14 +33829,8 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (!STRING expression_without_invalid) if (p->error_indicator) { @@ -34243,8 +33845,7 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34676,14 +34277,8 @@ _loop0_119_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // star_named_expressions if (p->error_indicator) { @@ -34698,8 +34293,7 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34743,14 +34337,8 @@ _loop0_120_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -34765,8 +34353,7 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35288,14 +34875,8 @@ _loop0_129_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' lambda_param if (p->error_indicator) { @@ -35319,8 +34900,7 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35624,14 +35204,8 @@ _loop0_135_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_name if (p->error_indicator) { @@ -35655,8 +35229,7 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35782,14 +35355,8 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expression ['as' star_target]) if (p->error_indicator) { @@ -35813,8 +35380,7 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35899,14 +35465,8 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expressions ['as' star_target]) if (p->error_indicator) { @@ -35930,8 +35490,7 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36073,14 +35632,8 @@ _loop0_143_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // block if (p->error_indicator) { @@ -36095,8 +35648,7 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -37726,14 +37278,8 @@ _loop0_172_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' bitwise_or) if (p->error_indicator) { @@ -37748,8 +37294,7 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity *= 2; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; diff --git a/Parser/pegen.c b/Parser/pegen.c index 5edf5542c7a8f99..35c85d1a5a557a4 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -12,6 +12,8 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define PEGEN_ARRAY_GROWTH_FACTOR 2 +#define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { const char *key; // Borrowed from arena-owned token bytes. @@ -35,6 +37,9 @@ Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) { const unsigned char *data = (const unsigned char*)PyUnicode_AsUTF8(line); + if (data == NULL) { + return -1; + } Py_ssize_t len = 0; while (col_offset < end_col_offset) { @@ -143,21 +148,35 @@ init_normalization(Parser *p) return 1; } -static int -growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { - assert(initial_size > 0); - arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items)); - arr->size = initial_size; - arr->num_items = 0; - - return arr->items != NULL; +void ** +_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity) +{ + if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*buffer))) { + return NULL; + } + Py_ssize_t new_capacity = *capacity == 0 + ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **new_buffer = PyMem_Realloc(buffer, new_capacity * sizeof(*buffer)); + if (new_buffer != NULL) { + *capacity = new_capacity; + } + return new_buffer; } static int -growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { +growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) +{ if (arr->num_items >= arr->size) { - size_t new_size = arr->size * 2; - void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); + if (arr->size > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*arr->items))) { + return 0; + } + size_t new_size = arr->size == 0 + ? TYPE_IGNORE_INITIAL_CAPACITY + : arr->size * PEGEN_ARRAY_GROWTH_FACTOR; + void *new_items_array = PyMem_Realloc( + arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; } @@ -172,8 +191,9 @@ growable_comment_array_add(growable_comment_array *arr, int lineno, char *commen } static void -growable_comment_array_deallocate(growable_comment_array *arr) { - for (unsigned i = 0; i < arr->num_items; i++) { +growable_comment_array_deallocate(growable_comment_array *arr) +{ + for (size_t i = 0; i < arr->num_items; i++) { PyMem_Free(arr->items[i].comment); } PyMem_Free(arr->items); @@ -273,12 +293,13 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to static int _resize_tokens_array(Parser *p) { - if (p->size > INT_MAX / 2 || - (size_t)p->size > PY_SSIZE_T_MAX / (2 * sizeof(*p->tokens))) { + if (p->size > INT_MAX / PEGEN_ARRAY_GROWTH_FACTOR || + (size_t)p->size > PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*p->tokens))) { PyErr_NoMemory(); return -1; } - int newsize = p->size * 2; + int newsize = p->size * PEGEN_ARRAY_GROWTH_FACTOR; Token **new_tokens = PyMem_Realloc(p->tokens, (size_t)newsize * sizeof(Token *)); if (new_tokens == NULL) { PyErr_NoMemory(); @@ -320,6 +341,7 @@ _PyPegen_fill_token(Parser *p) tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) { + PyMem_Free(tag); PyErr_NoMemory(); goto error; } @@ -943,12 +965,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, PyMem_Free(p); return (Parser *) PyErr_NoMemory(); } - if (!growable_comment_array_init(&p->type_ignore_comments, 10)) { - PyMem_Free(p->tokens[0]); - PyMem_Free(p->tokens); - PyMem_Free(p); - return (Parser *) PyErr_NoMemory(); - } + p->type_ignore_comments = (growable_comment_array){0}; p->mark = 0; p->fill = 0; @@ -995,7 +1012,7 @@ _PyPegen_Parser_Free(Parser *p) Py_XDECREF(p->normalize); // Resizes allocate blocks starting at indices 1, 2, 4, and so on. PyMem_Free(p->tokens[0]); - for (int i = 1; i < p->size; i *= 2) { + for (int i = 1; i < p->size; i *= PEGEN_ARRAY_GROWTH_FACTOR) { PyMem_Free(p->tokens[i]); } PyMem_Free(p->tokens); diff --git a/Parser/pegen.h b/Parser/pegen.h index eb3f1d8a3041a7c..825cc9197bfabf3 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -190,6 +190,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +// Return NULL on failure, leaving the buffer and capacity unchanged. +void **_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 74ab56c912ae1af..6997a1158e9310f 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -327,6 +327,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number); + error_line = NULL; if (!tmp) { goto error; } diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 69c3bb371add960..0af8cc31a3b947f 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -8,6 +8,14 @@ #include "helpers.h" #include "../lexer/state.h" +#define UNICODE_BOM 0xFEFF +#define UTF16_LE_BOM "\xff\xfe" +#define UTF16_BE_BOM "\xfe\xff" +#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) +#define UTF8_BOM "\xef\xbb\xbf" +#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) +#define NORMALIZED_LINE_INITIAL_CAPACITY 256 + char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) { @@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return -1; } if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 && - PyUnicode_ReadChar(unicode, 0) == 0xFEFF) { - utf8 += 3; - utf8_len -= 3; + PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) { + utf8 += UTF8_BOM_LENGTH; + utf8_len -= UTF8_BOM_LENGTH; } chunk_release_data(chunk); chunk->owner = unicode; @@ -72,12 +80,12 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller must provide len + 2 bytes: normalization can append a final -// newline and always writes a NUL terminator. +// The caller provides len + add_final_newline + 1 bytes for an optional +// final newline and the NUL terminator. static void normalize_newlines_into(char *result, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len, int *implicit_newline) + Py_ssize_t *out_len) { Py_ssize_t write = 0; if (preserve_crlf || memchr(data, '\r', len) == NULL) { @@ -97,33 +105,27 @@ normalize_newlines_into(char *result, const char *data, Py_ssize_t len, result[write++] = c; } } - int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; - if (implicit) { + if (add_final_newline && write > 0 && result[write - 1] != '\n') { result[write++] = '\n'; } result[write] = '\0'; *out_len = write; - if (implicit_newline != NULL) { - *implicit_newline = implicit; - } } char * -_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, - int add_final_newline, Py_ssize_t *out_len, - int *implicit_newline) +_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, + Py_ssize_t *out_len) { - if (len > PY_SSIZE_T_MAX - 2) { + if (len < 0 || len == PY_SSIZE_T_MAX) { PyErr_NoMemory(); return NULL; } - char *result = PyMem_Malloc((size_t)len + 2); + char *result = PyMem_Malloc((size_t)len + 1); if (result == NULL) { PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, preserve_crlf, - add_final_newline, out_len, implicit_newline); + normalize_newlines_into(result, data, len, 0, 0, out_len); return result; } @@ -158,11 +160,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) return 0; } } - for (; i + 6 < len; i++) { - if (memcmp(line + i, "coding", 6) != 0) { + const Py_ssize_t marker_len = sizeof("coding") - 1; + for (; i < len - marker_len; i++) { + if (memcmp(line + i, "coding", marker_len) != 0) { continue; } - const char *cursor = line + i + 6; + const char *cursor = line + i + marker_len; if (*cursor != ':' && *cursor != '=') { continue; } @@ -179,37 +182,36 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) if (cursor == start) { continue; } - char *found = _PyTok_CopyBytes(start, cursor - start); - if (found == NULL) { - return -1; - } - char normalized[13]; - int n; - for (n = 0; n < 12 && found[n] != '\0'; n++) { - normalized[n] = found[n] == '_' ? '-' : Py_TOLOWER(found[n]); + Py_ssize_t encoding_len = cursor - start; + // Include the longest recognized prefix and its NUL terminator. + char normalized[sizeof("iso-latin-1-")]; + const Py_ssize_t prefix_len = Py_MIN( + encoding_len, (Py_ssize_t)sizeof(normalized) - 1); + Py_ssize_t n; + for (n = 0; n < prefix_len; n++) { + normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; - const char *canonical = found; + const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || - strncmp(normalized, "utf-8-", 6) == 0) { + strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) { canonical = "utf-8"; } else if (strcmp(normalized, "latin-1") == 0 || strcmp(normalized, "iso-8859-1") == 0 || strcmp(normalized, "iso-latin-1") == 0 || - strncmp(normalized, "latin-1-", 8) == 0 || - strncmp(normalized, "iso-8859-1-", 11) == 0 || - strncmp(normalized, "iso-latin-1-", 12) == 0) { + strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 || + strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 || + strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) { canonical = "iso-8859-1"; } - if (canonical != found) { - PyMem_Free(found); - found = _PyTok_CopyBytes(canonical, strlen(canonical)); - if (found == NULL) { - return -1; - } + if (canonical != start) { + encoding_len = strlen(canonical); + } + *encoding = _PyTok_CopyBytes(canonical, encoding_len); + if (*encoding == NULL) { + return -1; } - *encoding = found; *scan_next = 0; return 0; } @@ -221,17 +223,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *second, int final, Py_ssize_t *bom_len) { - int bom = first->len >= 3 && - (unsigned char)first->data[0] == 0xEF && - (unsigned char)first->data[1] == 0xBB && - (unsigned char)first->data[2] == 0xBF; - *bom_len = bom ? 3 : 0; + int bom = first->len >= UTF8_BOM_LENGTH && + memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0; + *bom_len = bom ? UTF8_BOM_LENGTH : 0; char *cookie = NULL; int scan_next = 0; int cookie_line = 1; - const char *first_data = first->data + (bom ? 3 : 0); - Py_ssize_t first_len = first->len - (bom ? 3 : 0); + const char *first_data = first->data + *bom_len; + Py_ssize_t first_len = first->len - *bom_len; if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) { return _PYTOK_ENCODING_ERROR; } @@ -256,8 +256,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, } if (bom && strcmp(cookie, "utf-8") != 0) { const _PyTok_Chunk *line = cookie_line == 2 ? second : first; - const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); - Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); + const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0); + Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0); int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { @@ -269,11 +269,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } - if (!bom && _PyTok_SetEncoding(tok, cookie) < 0) { - PyMem_Free(cookie); - return _PYTOK_ENCODING_ERROR; - } - PyMem_Free(cookie); + PyMem_Free(tok->encoding); + tok->encoding = cookie; return _PYTOK_ENCODING_DONE; } @@ -329,37 +326,40 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, data[pos + raw_line_len - 1] == '\r'; int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; - int normalize = add_newline || - (!preserve_crlf && - memchr(data + pos, '\r', raw_line_len) != NULL); - const char *line = data + pos; Py_ssize_t line_len = raw_line_len; - int implicit = 0; + // raw_line_length stops at the first CR or LF, so any CR is in + // the line terminator. + int normalize = add_newline || + (!preserve_crlf && + (line[line_len - 1] == '\r' || + (line_len > 1 && line[line_len - 2] == '\r'))); if (normalize) { - if (line_len > PY_SSIZE_T_MAX - 2) { + if (line_len > PY_SSIZE_T_MAX - add_newline - 1) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) { + Py_ssize_t needed = line_len + add_newline + 1; + if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, + NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } normalize_newlines_into(normalized, line, line_len, preserve_crlf, add_newline, - &line_len, &implicit); + &line_len); line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len, implicit); + &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; goto error; } + tok->reader->prepared_final_newline_is_implicit = add_newline; pos += raw_line_len; } PyMem_Free(normalized); @@ -472,11 +472,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final) return 0; } int strip_bom = reader->kind == _PYTOK_READER_READLINE && - chunk->len >= 2 && - (((unsigned char)chunk->data[0] == 0xFF && - (unsigned char)chunk->data[1] == 0xFE) || - ((unsigned char)chunk->data[0] == 0xFE && - (unsigned char)chunk->data[1] == 0xFF)); + chunk->len >= UTF16_BOM_LENGTH && + (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 || + memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0); PyObject *input; if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT && PyBytes_Check(chunk->owner) && diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c003deba18dd8f2..6a0b7615f8df5de 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" #include "pycore_pystate.h" +#include "pycore_runtime.h" #include "errcode.h" #include "helpers.h" @@ -8,6 +9,8 @@ #include "reader_internal.h" #include "../lexer/state.h" +#define READER_BUFFER_GROWTH_FACTOR 2 + #ifdef HAVE_UNISTD_H # include #endif @@ -37,6 +40,24 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } +int +_PyTokenizer_Traverse(struct tok_state *tok, visitproc visit, void *arg) +{ + Py_VISIT(tok->filename); + Py_VISIT(tok->module); + _PyTok_Reader *reader = tok->reader; + Py_VISIT(reader->readline); + Py_VISIT(reader->decoder); + for (int i = 0; + i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) { + _PyTok_Chunk *chunk = &reader->prefetched_lines[i]; + if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT) { + Py_VISIT(chunk->owner); + } + } + return 0; +} + int _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, Py_ssize_t initial_capacity) @@ -48,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, // Grow geometrically to avoid reallocating for every longer line. Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= READER_BUFFER_GROWTH_FACTOR; } char *resized = PyMem_Realloc(*buffer, cap); if (resized == NULL) { @@ -67,10 +88,14 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { + if (len == 0) { + return 0; + } if (reader->decoded_pos > 0) { Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, (size_t)remaining); + reader->decoded_scan -= reader->decoded_pos; reader->decoded_pos = 0; reader->decoded_len = remaining; } @@ -104,13 +129,17 @@ static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); + assert(reader->decoded_scan >= reader->decoded_pos && + reader->decoded_scan <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } char *start = reader->decoded + reader->decoded_pos; - char *newline = memchr(start, '\n', - reader->decoded_len - reader->decoded_pos); + // Previously scanned bytes cannot contain a newline. + char *newline = memchr(reader->decoded + reader->decoded_scan, '\n', + reader->decoded_len - reader->decoded_scan); if (newline == NULL) { + reader->decoded_scan = reader->decoded_len; return 0; } Py_ssize_t len = newline - start + 1; @@ -118,10 +147,12 @@ pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) chunk->len = len; chunk->ownership = _PYTOK_CHUNK_BORROWED; reader->decoded_pos += len; + reader->decoded_scan = reader->decoded_pos; chunk->implicit_newline = reader->decoded_pos == reader->decoded_len && reader->decoded_tail_is_implicit; if (reader->decoded_pos == reader->decoded_len) { reader->decoded_pos = reader->decoded_len = 0; + reader->decoded_scan = 0; reader->decoded_tail_is_implicit = 0; } return 1; @@ -142,7 +173,6 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } - int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -151,8 +181,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)start; chunk->len = tok->source.bytes + end - start; chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = _PyTok_SourceLineIsImplicit( - &tok->source, lineno); + chunk->implicit_newline = end == tok->source.len && + tok->reader->prepared_final_newline_is_implicit; return _PYTOK_READ_LINE; } @@ -162,9 +192,12 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_Reader *reader = tok->reader; Py_ssize_t len = 0; for (;;) { - if (len > PY_SSIZE_T_MAX - BUFSIZ || - _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, - len + BUFSIZ, BUFSIZ) < 0) { + if (len > PY_SSIZE_T_MAX - BUFSIZ) { + PyErr_NoMemory(); + return _PYTOK_READ_ERROR; + } + if (_PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, + len + BUFSIZ, BUFSIZ) < 0) { return _PYTOK_READ_ERROR; } int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX); @@ -182,7 +215,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) break; } } - int implicit = len == 0 || reader->file_buffer[len - 1] != '\n'; + int implicit = reader->file_buffer[len - 1] != '\n'; chunk->data = reader->file_buffer; chunk->len = len; chunk->implicit_newline = implicit; @@ -325,7 +358,8 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } int decoded = _PyTok_DecodeChunk(tok, &input, 0); - if (decoded == 0 && chunk_is_line(&input)) { + if (decoded == 0 && reader->decoded_pos == reader->decoded_len && + chunk_is_line(&input)) { *chunk = input; return _PYTOK_READ_LINE; } @@ -466,9 +500,6 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_ERROR; } } - if (pop_decoded_line(reader, chunk)) { - return _PYTOK_READ_LINE; - } } } @@ -503,9 +534,12 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_ChunkClear(&decoded); return _PYTOK_READ_ERROR; } + if (memchr(decoded.data, '\r', decoded.len) == NULL) { + *chunk = decoded; + return _PYTOK_READ_LINE; + } chunk->data = _PyTok_NormalizeNewlines( - decoded.data, decoded.len, 0, 0, - &chunk->len, NULL); + decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); @@ -618,8 +652,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) reset_streaming_buffer(tok); } _PyTok_Off source_start = _PyTok_SourceAppendLine( - &tok->source, chunk.data, chunk.len, - chunk.implicit_newline); + &tok->source, chunk.data, chunk.len); if (source_start < 0) { _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) @@ -780,7 +813,16 @@ static FILE * fdopen_borrow(int fd) { int copy = _Py_dup(fd); - return copy < 0 ? NULL : fdopen(copy, "r"); + if (copy < 0) { + return NULL; + } + FILE *fp = fdopen(copy, "r"); + if (fp == NULL) { + int saved_errno = errno; + close(copy); + errno = saved_errno; + } + return fp; } #endif @@ -796,21 +838,16 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) fclose(fp); return NULL; } - tok->filename = filename != NULL - ? Py_NewRef(filename) : PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; - } + _Py_DECLARE_STR(anon_string, ""); + tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); if (initialize_file(tok) < 0) { fclose(fp); _PyTokenizer_Free(tok); return NULL; } fclose(fp); - char *encoding = tok->encoding == NULL - ? NULL : _PyTok_CopyBytes(tok->encoding, strlen(tok->encoding)); + char *encoding = tok->encoding; + tok->encoding = NULL; _PyTokenizer_Free(tok); return encoding; } diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 390b6ed9d8581a5..f74063de8d653a4 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -3,6 +3,8 @@ #include "Python.h" +#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2 + typedef enum { _PYTOK_READER_PREPARED, _PYTOK_READER_FILE, @@ -46,13 +48,15 @@ typedef struct _PyTok_Reader { char *file_buffer; Py_ssize_t file_buffer_cap; - _PyTok_Chunk prefetched_lines[2]; + _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES]; char *decoded; Py_ssize_t decoded_pos; + Py_ssize_t decoded_scan; // First byte not yet scanned for a newline. Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; + unsigned char prepared_final_newline_is_implicit; unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; @@ -67,8 +71,7 @@ int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t); char *_PyTok_CopyBytes(const char *, Py_ssize_t); int _PyTok_DecodeOnce( struct tok_state *, _PyTok_Chunk *, const char *, const char *); -char *_PyTok_NormalizeNewlines( - const char *, Py_ssize_t, int, int, Py_ssize_t *, int *); +char *_PyTok_NormalizeNewlines(const char *, Py_ssize_t, Py_ssize_t *); void _PyTok_ChunkClear(_PyTok_Chunk *); int _PyTok_SetEncoding(struct tok_state *, const char *); _PyTok_EncodingResult _PyTok_DetectEncoding( diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d69eab93923e81c..ebb7b05507c56ec 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -1,7 +1,10 @@ #include "Python.h" +#include "pycore_pymem.h" // PYMEM_DEADBYTE #include "source.h" +#define SOURCE_BUFFER_GROWTH_FACTOR 2 + void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -12,7 +15,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -25,10 +27,6 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) if (source->bytes != NULL) { source->bytes[0] = '\0'; } - if (source->implicit_lines != NULL) { - Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); - memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); - } source->nlines = 0; } @@ -40,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= SOURCE_BUFFER_GROWTH_FACTOR; } char *bytes; #ifdef Py_DEBUG @@ -62,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } #ifdef Py_DEBUG if (source->bytes != NULL) { - memset(source->bytes, 0xDD, source->cap); + memset(source->bytes, PYMEM_DEADBYTE, source->cap); PyMem_Free(source->bytes); } #endif @@ -71,35 +69,9 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_implicit_lines(_PyTok_SourceText *source, int nlines) -{ - Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); - if (needed <= source->implicit_cap) { - return 0; - } - Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16; - while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { - cap = needed; - break; - } - cap *= 2; - } - unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap); - if (lines == NULL) { - PyErr_NoMemory(); - return -1; - } - memset(lines + source->implicit_cap, 0, cap - source->implicit_cap); - source->implicit_lines = lines; - source->implicit_cap = cap; - return 0; -} - static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { if (len <= 0 || bytes == NULL || (source->nlines > 0 && @@ -108,8 +80,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, return -1; } const char *newline = memchr(bytes, '\n', len); - if ((newline != NULL && newline != bytes + len - 1) || - (implicit_newline && newline == NULL)) { + if (newline != NULL && newline != bytes + len - 1) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } @@ -123,9 +94,9 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, _PyTok_Off _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { - if (validate_line(source, bytes, len, implicit_newline) < 0) { + if (validate_line(source, bytes, len) < 0) { return -1; } if (source->len > PY_SSIZE_T_MAX - len - 1 || @@ -133,9 +104,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, PyErr_NoMemory(); return -1; } - int nlines = source->nlines + 1; - if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || - reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -143,11 +112,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (implicit_newline) { - source->implicit_lines[(nlines - 1) / 8] |= - (unsigned char)(1U << ((nlines - 1) & 7)); - } - source->nlines = nlines; + source->nlines++; return source->base_offset + start; } @@ -170,14 +135,3 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, *len = (newline != NULL ? newline : end) - line; return line; } - -int -_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) -{ - if (lineno < 1 || lineno > source->nlines || - (lineno - 1) / 8 >= source->implicit_cap) { - return 0; - } - return (source->implicit_lines[(lineno - 1) / 8] >> - ((lineno - 1) & 7)) & 1; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 9576419aade1d46..66980a41fb7c481 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -10,9 +10,7 @@ typedef struct { _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - unsigned char *implicit_lines; int nlines; - Py_ssize_t implicit_cap; } _PyTok_SourceText; static inline const char * @@ -29,19 +27,14 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the - final line. implicit_newline means that the final newline was synthesized. - The input must not point into source storage. */ + final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( - _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, - int implicit_newline); + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* Return false for invalid line numbers and the virtual EOF line. */ -PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( - const _PyTok_SourceText *, int); #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 82a84830d7cbfe7..2ef87add3646c3c 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -58,6 +58,7 @@ typedef struct { Errors are returned as ERRORTOKEN, with or without a Python exception. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); +int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); void _PyTokenizer_raise_init_error(PyObject *filename); void _PyToken_Init(struct token *); static inline void diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h index c12b6de449c7008..b488cb01618d35f 100644 --- a/Parser/tokenizer/types.h +++ b/Parser/tokenizer/types.h @@ -18,12 +18,6 @@ typedef struct { int byte_col; } _PyTok_Loc; -static inline _PyTok_Span -_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) -{ - return (_PyTok_Span){start, end}; -} - static inline int _PyTok_SpanIsValid(_PyTok_Span span) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 3802709732e29da..c6f690e27724311 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -37,7 +37,6 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; - Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -57,21 +56,18 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding) /*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/ { - tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - PyObject *filename = PyUnicode_FromString(""); - if (filename == NULL) { + struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding); + if (tok == NULL) { return NULL; } - self->tok = _PyTokenizer_FromReadline(readline, encoding); - if (self->tok == NULL) { - Py_DECREF(filename); + _Py_DECLARE_STR(anon_string, ""); + _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL); + tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); + if (self == NULL) { + _PyTokenizer_Free(tok); return NULL; } - _PyTokenizer_SetContext(self->tok, filename, NULL); - Py_DECREF(filename); + self->tok = tok; _PyTokenizer_SetOptions(self->tok, extra_tokens, 0); self->extra_tokens = extra_tokens; self->done = 0; @@ -79,18 +75,15 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; - self->last_end_lineno = 0; return (PyObject *)self; } -static int +static void _tokenizer_error(tokenizeriterobject *it) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - if (PyErr_Occurred()) { - return -1; - } + assert(!PyErr_Occurred()); const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; @@ -105,19 +98,17 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SyntaxLocationObject( info.filename, info.location.lineno, (int)Py_MAX(0, info.input_span.end - info.input_span.start)); - return -1; + return; case E_DEDENT: msg = "unindent does not match any outer indentation level"; errtype = PyExc_IndentationError; break; case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; + PyErr_SetNone(PyExc_KeyboardInterrupt); + return; case E_NOMEM: PyErr_NoMemory(); - return -1; + return; case E_TABSPACE: errtype = PyExc_TabError; msg = "inconsistent use of tabs and spaces in indentation"; @@ -138,7 +129,6 @@ _tokenizer_error(tokenizeriterobject *it) PyObject* error_line = NULL; PyObject* tmp = NULL; PyObject* value = NULL; - int result = 0; Py_ssize_t input_size; const char *input = _PyTokenizer_SpanView( @@ -148,30 +138,25 @@ _tokenizer_error(tokenizeriterobject *it) size -= 1; // Remove the newline character from the end of the line error_line = PyUnicode_DecodeUTF8(input, size, "replace"); if (!error_line) { - result = -1; goto exit; } Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size); if (offset == -1) { - result = -1; goto exit; } tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); if (!tmp) { - result = -1; goto exit; } errstr = PyUnicode_FromString(msg); if (!errstr) { - result = -1; goto exit; } value = _PyTuple_FromPair(errstr, tmp); if (!value) { - result = -1; goto exit; } @@ -182,7 +167,6 @@ _tokenizer_error(tokenizeriterobject *it) Py_XDECREF(error_line); Py_XDECREF(tmp); Py_XDECREF(value); - return result; } static PyObject * @@ -206,7 +190,7 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, return line; } -static void +static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, const char *token_start, const char *line_start, const char *end_line_start, PyObject *line, int line_changed, @@ -222,6 +206,9 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + if (*col_offset < 0) { + return -1; + } it->byte_col_offset_diff = byte_offset - *col_offset; } else { @@ -234,16 +221,23 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + if (token_col_offset < 0) { + return -1; + } *end_col_offset = *col_offset + token_col_offset; it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { - *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( + line, end_line_start - line_start, token_end - line_start); + if (*end_col_offset < 0) { + return -1; + } it->byte_col_offset_diff += end_byte_offset - *end_col_offset; } } it->last_lineno = lineno; - it->last_end_lineno = end_lineno; + return 0; } static PyObject * @@ -266,9 +260,8 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done || type == ERRORTOKEN) { + if (it->done) { PyErr_SetString(PyExc_StopIteration, "EOF"); - it->done = 1; goto exit; } _PyToken_View view; @@ -313,8 +306,11 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, &token, token_start, view.line, view.end_line, line, - line_changed, &col_offset, &end_col_offset); + if (_get_col_offsets(it, &token, token_start, view.line, view.end_line, line, + line_changed, &col_offset, &end_col_offset) < 0) { + Py_DECREF(str); + goto exit; + } if (it->extra_tokens) { if (is_trailing_token) { @@ -327,8 +323,8 @@ tokenizeriter_next(PyObject *op) type = OP; } else if (type == NEWLINE) { - Py_DECREF(str); if (!view.implicit_newline) { + Py_DECREF(str); assert(token_start != NULL); if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); @@ -346,7 +342,6 @@ tokenizeriter_next(PyObject *op) } if (str == NULL) { - Py_DECREF(line); goto exit; } } @@ -367,15 +362,26 @@ tokenizeriter_dealloc(PyObject *op) { tokenizeriterobject *it = (tokenizeriterobject*)op; PyTypeObject *tp = Py_TYPE(it); + PyObject_GC_UnTrack(it); Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } +static int +tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg) +{ + tokenizeriterobject *it = (tokenizeriterobject *)op; + Py_VISIT(Py_TYPE(it)); + Py_VISIT(it->last_line); + return _PyTokenizer_Traverse(it->tok, visit, arg); +} + static PyType_Slot tokenizeriter_slots[] = { {Py_tp_new, tokenizeriter_new}, {Py_tp_dealloc, tokenizeriter_dealloc}, + {Py_tp_traverse, tokenizeriter_traverse}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, tokenizeriter_next}, @@ -385,7 +391,7 @@ static PyType_Slot tokenizeriter_slots[] = { static PyType_Spec tokenizeriter_spec = { .name = "_tokenize.TokenizerIter", .basicsize = sizeof(tokenizeriterobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = tokenizeriter_slots, }; diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 6dd598beb1b39c4..a5a07d02dbc029e 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -31,12 +31,6 @@ def emit(self, value: str) -> None: self.writer.print("p->level--;") self.writer.print(f"return {value};") - def check_memory(self, expr: str) -> None: - self.writer.print(f"if ({expr}) {{") - with self.writer.indent(): - self.no_memory() - self.writer.print("}") - def no_memory(self) -> None: self.writer.print("p->error_indicator = 1;") self.writer.print("PyErr_NoMemory();") @@ -55,17 +49,15 @@ def __init__(self, writer: CWriter, returns: _CReturnEmitter): self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: - self._print("void **_children = PyMem_Malloc(sizeof(void *));") - self._returns.check_memory("!_children") - self._print("Py_ssize_t _children_capacity = 1;") + self._print("void **_children = NULL;") + self._print("Py_ssize_t _children_capacity = 0;") self._print("Py_ssize_t _n = 0;") def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity *= 2;") self._print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + "void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity);" ) self._check_memory("!_new_children") self._print("_children = _new_children;")