From a7684aaccf5ee1842109302a38ed0d4252c970b5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:46:29 +0100 Subject: [PATCH 01/29] Use diagnostic line for missing-brace errors --- Lib/test/test_fstring.py | 14 ++++++++++++++ ...-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/lexer/string.c | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 63006266bcd039e..72c80b8bfbfecc5 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -645,6 +645,20 @@ def test_unclosed_multiline_replacement_field(self): f"{prefix[-1]}-string: expecting '}}' to close '{{' " f"on line {lineno}") + def test_unclosed_replacement_field_quote_line(self): + for prefix in ('f', 't', 'rf', 'rt'): + for quote in ('"', "'"): + triple = quote * 3 + for suffix in ('', '\nx'): + source = prefix + triple + '{1' + triple + suffix + with self.subTest(source=source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + self.assertEqual( + cm.exception.msg, + f"{prefix[-1]}-string: expecting '}}'") + self.assertEqual(cm.exception.lineno, 1) + @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") def test_mismatched_parens(self): self.assertAllRaise(SyntaxError, r"closing parenthesis '\}' " diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst new file mode 100644 index 000000000000000..2b0aae1a3d2b4c4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -0,0 +1 @@ +Correct missing-brace error messages for multiline f-strings and t-strings. diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index a48cedf270362ab..41dc3c66edff09f 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -370,7 +370,7 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) assert(level >= 0 && level < tok->level); assert(tok->parenstack[level] == '{'); int lineno = tok->parenlinenostack[level]; - if (lineno != tok->lineno) { + if (lineno != location.lineno) { _PyTokenizer_syntaxerror_at( tok, line, cursor_offset, location.lineno, -1, -1, "%c-string: expecting '}' to close '{' on line %d", From f276100b7d9f33b3f06db60f34e7f79bcca15ab6 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:58:16 +0100 Subject: [PATCH 02/29] Keep implicit prepared newline metadata in the reader --- Modules/_testinternalcapi/tokenizer.c | 34 ++++++--------- Parser/tokenizer/decoder.c | 3 +- Parser/tokenizer/reader.c | 8 ++-- Parser/tokenizer/reader_internal.h | 1 + Parser/tokenizer/source.c | 61 +++------------------------ Parser/tokenizer/source.h | 11 +---- 6 files changed, 27 insertions(+), 91 deletions(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 1f89c12f223c7d9..fd131abe985937c 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -46,23 +46,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } if (check_system_error( - _PyTok_SourceAppendLine(&source, "", 0, 0) < 0, + _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4, 0) < 0, + _PyTok_SourceAppendLine(&source, "a\nb\n", 4) < 0, "accepted multiple source lines") < 0 || - check_system_error( - _PyTok_SourceAppendLine(&source, "a", 1, 1) < 0, - "accepted missing implicit newline") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6, 0) == 0, + &source, "alpha\n", 6) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3, 1) == 6, - "wrong second source offset") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, 1) && - _PyTok_SourceLineIsImplicit(&source, 2), - "wrong implicit newline flags") < 0) { + &source, "\xce\xb2\n", 3) == 6, + "wrong second source offset") < 0) { goto error; } @@ -81,9 +75,9 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4, 0) < 0 || + if (_PyTok_SourceAppendLine(&source, "tail", 4) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2, 0) < 0, + _PyTok_SourceAppendLine(&source, "x\n", 2) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -107,7 +101,7 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), _PyTok_SourceText source; _PyTok_SourceInit(&source); for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2, 1) < 0) { + if (_PyTok_SourceAppendLine(&source, "x\n", 2) < 0) { goto error; } } @@ -121,10 +115,8 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2, 0) == 520 + 2 * i, - "wrong source offset after discard") < 0 || - check(!_PyTok_SourceLineIsImplicit(&source, i + 1), - "discard preserved implicit newline flag") < 0) { + if (check(_PyTok_SourceAppendLine(&source, "y\n", 2) == 520 + 2 * i, + "wrong source offset after discard") < 0) { goto error; } } @@ -133,18 +125,18 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4, 0) == 1040, + if (check(_PyTok_SourceAppendLine(&source, "tail", 4) == 1040, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) == 1044, + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) == 1044, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2, 0) < 0 && + if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 69c3bb371add960..d3c3f7202e9e336 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -354,12 +354,13 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( - &tok->source, line, line_len, implicit); + &tok->source, line, line_len); if (appended < 0) { tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) ? E_NOMEM : E_ERROR; goto error; } + tok->reader->prepared_final_newline_is_implicit = implicit; pos += raw_line_len; } PyMem_Free(normalized); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c003deba18dd8f2..fd7225f63b37660 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -142,7 +142,6 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) if (tok->lineno >= tok->source.nlines) { return _PYTOK_READ_EOF; } - int lineno = tok->lineno + 1; const char *start = _PyLexer_BufferPointer(tok, tok->inp); const char *newline = memchr( start, '\n', tok->source.bytes + tok->source.len - start); @@ -151,8 +150,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk) chunk->data = (char *)start; chunk->len = tok->source.bytes + end - start; chunk->ownership = _PYTOK_CHUNK_BORROWED; - chunk->implicit_newline = _PyTok_SourceLineIsImplicit( - &tok->source, lineno); + chunk->implicit_newline = end == tok->source.len && + tok->reader->prepared_final_newline_is_implicit; return _PYTOK_READ_LINE; } @@ -618,8 +617,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok) reset_streaming_buffer(tok); } _PyTok_Off source_start = _PyTok_SourceAppendLine( - &tok->source, chunk.data, chunk.len, - chunk.implicit_newline); + &tok->source, chunk.data, chunk.len); if (source_start < 0) { _PyTok_ChunkClear(&chunk); tok->done = PyErr_ExceptionMatches(PyExc_MemoryError) diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 390b6ed9d8581a5..9638fef88cd593b 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -53,6 +53,7 @@ typedef struct _PyTok_Reader { Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; + unsigned char prepared_final_newline_is_implicit; unsigned char decoded_tail_is_implicit; unsigned char file_initialized; unsigned char file_eof; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d69eab93923e81c..d08e09c597d7c68 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -12,7 +12,6 @@ void _PyTok_SourceClear(_PyTok_SourceText *source) { PyMem_Free(source->bytes); - PyMem_Free(source->implicit_lines); _PyTok_SourceInit(source); } @@ -25,10 +24,6 @@ _PyTok_SourceDiscard(_PyTok_SourceText *source) if (source->bytes != NULL) { source->bytes[0] = '\0'; } - if (source->implicit_lines != NULL) { - Py_ssize_t used = source->nlines / 8 + (source->nlines % 8 != 0); - memset(source->implicit_lines, 0, Py_MIN(used, source->implicit_cap)); - } source->nlines = 0; } @@ -71,35 +66,9 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) return 0; } -static int -reserve_implicit_lines(_PyTok_SourceText *source, int nlines) -{ - Py_ssize_t needed = nlines / 8 + (nlines % 8 != 0); - if (needed <= source->implicit_cap) { - return 0; - } - Py_ssize_t cap = source->implicit_cap > 0 ? source->implicit_cap : 16; - while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { - cap = needed; - break; - } - cap *= 2; - } - unsigned char *lines = PyMem_Realloc(source->implicit_lines, cap); - if (lines == NULL) { - PyErr_NoMemory(); - return -1; - } - memset(lines + source->implicit_cap, 0, cap - source->implicit_cap); - source->implicit_lines = lines; - source->implicit_cap = cap; - return 0; -} - static int validate_line(const _PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { if (len <= 0 || bytes == NULL || (source->nlines > 0 && @@ -108,8 +77,7 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, return -1; } const char *newline = memchr(bytes, '\n', len); - if ((newline != NULL && newline != bytes + len - 1) || - (implicit_newline && newline == NULL)) { + if (newline != NULL && newline != bytes + len - 1) { PyErr_SetString(PyExc_SystemError, "invalid tokenizer source line"); return -1; } @@ -123,9 +91,9 @@ validate_line(const _PyTok_SourceText *source, const char *bytes, _PyTok_Off _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, - Py_ssize_t len, int implicit_newline) + Py_ssize_t len) { - if (validate_line(source, bytes, len, implicit_newline) < 0) { + if (validate_line(source, bytes, len) < 0) { return -1; } if (source->len > PY_SSIZE_T_MAX - len - 1 || @@ -133,9 +101,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, PyErr_NoMemory(); return -1; } - int nlines = source->nlines + 1; - if ((implicit_newline && reserve_implicit_lines(source, nlines) < 0) || - reserve_bytes(source, source->len + len + 1) < 0) { + if (reserve_bytes(source, source->len + len + 1) < 0) { return -1; } @@ -143,11 +109,7 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes, memcpy(source->bytes + start, bytes, len); source->len += len; source->bytes[source->len] = '\0'; - if (implicit_newline) { - source->implicit_lines[(nlines - 1) / 8] |= - (unsigned char)(1U << ((nlines - 1) & 7)); - } - source->nlines = nlines; + source->nlines++; return source->base_offset + start; } @@ -170,14 +132,3 @@ _PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno, *len = (newline != NULL ? newline : end) - line; return line; } - -int -_PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno) -{ - if (lineno < 1 || lineno > source->nlines || - (lineno - 1) / 8 >= source->implicit_cap) { - return 0; - } - return (source->implicit_lines[(lineno - 1) / 8] >> - ((lineno - 1) & 7)) & 1; -} diff --git a/Parser/tokenizer/source.h b/Parser/tokenizer/source.h index 9576419aade1d46..66980a41fb7c481 100644 --- a/Parser/tokenizer/source.h +++ b/Parser/tokenizer/source.h @@ -10,9 +10,7 @@ typedef struct { _PyTok_Off base_offset; _PyTok_Off len; _PyTok_Off cap; - unsigned char *implicit_lines; int nlines; - Py_ssize_t implicit_cap; } _PyTok_SourceText; static inline const char * @@ -29,19 +27,14 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *); PyAPI_FUNC(void) _PyTok_SourceDiscard(_PyTok_SourceText *); /* Append one nonempty logical line and return its start offset. The input may contain one newline, as its final byte. An unterminated line must be the - final line. implicit_newline means that the final newline was synthesized. - The input must not point into source storage. */ + final line. The input must not point into source storage. */ PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine( - _PyTok_SourceText *source, const char *bytes, Py_ssize_t len, - int implicit_newline); + _PyTok_SourceText *source, const char *bytes, Py_ssize_t len); /* Return borrowed bytes excluding '\n', writing the byte length to *len. Line numbers are 1-based and clamp to the first or final line; a trailing '\n' adds an empty final line. The view need not be NUL-terminated. This does not set an exception. Append, discard, and clear invalidate the view. */ PyAPI_FUNC(const char *) _PyTok_SourceLineView( const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len); -/* Return false for invalid line numbers and the virtual EOF line. */ -PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit( - const _PyTok_SourceText *, int); #endif From 5d287599174b102d6ece90e6592766235f8ed1b7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:02 +0100 Subject: [PATCH 03/29] Reuse parsed formatted string kind in lexer --- Parser/lexer/lexer.c | 5 ++++- Parser/lexer/lexer_internal.h | 3 ++- Parser/lexer/string.c | 30 +++++------------------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index f27fee8d61b9085..070760deefd58b4 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -314,7 +314,10 @@ _PyLexer_get_normal(struct tok_state *tok, ftstring_state *current, struct token // Handle valid f or t string creation: if (saw_f || saw_t) { - return _PyLexer_scan_fstring_start(tok, token, c); + ftstring_kind kind = saw_t + ? (saw_r ? RAW_TSTRING : TSTRING) + : (saw_r ? RAW_FSTRING : FSTRING); + return _PyLexer_scan_fstring_start(tok, token, c, kind); } return _PyLexer_scan_string(tok, token, c); } diff --git a/Parser/lexer/lexer_internal.h b/Parser/lexer/lexer_internal.h index f0d9576d95f8ff3..652512a3e88c22d 100644 --- a/Parser/lexer/lexer_internal.h +++ b/Parser/lexer/lexer_internal.h @@ -66,7 +66,8 @@ int _PyLexer_close_ftstring_expr( void _PyLexer_mark_ftstring_debug(struct tok_state *, ftstring_state *); int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int); int _PyLexer_scan_number(struct tok_state *, struct token *, int, int); -int _PyLexer_scan_fstring_start(struct tok_state *, struct token *, int); +int _PyLexer_scan_fstring_start( + struct tok_state *, struct token *, int, ftstring_kind); int _PyLexer_scan_string(struct tok_state *, struct token *, int); int _PyLexer_get_normal(struct tok_state *, ftstring_state *, struct token *); int _PyLexer_get_ftstring(struct tok_state *, ftstring_state *, struct token *); diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 41dc3c66edff09f..00c6543341ce75c 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -254,7 +254,8 @@ _PyLexer_check_string_prefixes(struct tok_state *tok, } int -_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) +_PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, + int c, ftstring_kind kind) { _PyTok_Off p_start = -1; _PyTok_Off p_end = -1; @@ -293,30 +294,9 @@ _PyLexer_scan_fstring_start(struct tok_state *tok, struct token *token, int c) state->start_loc = tok->start_loc; state->expr_span = (_PyTok_Span){-1, -1}; - int raw = 0; - int tstring = 0; - switch (*_PyLexer_BufferPointer(tok, tok->start)) { - case 'T': - case 't': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - tstring = 1; - break; - case 'F': - case 'f': - raw = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 'r'; - break; - case 'R': - case 'r': - raw = 1; - tstring = Py_TOLOWER(_PyLexer_BufferPointer(tok, tok->start)[1]) == 't'; - break; - default: - Py_UNREACHABLE(); - } - state->kind = tstring - ? (raw ? RAW_TSTRING : TSTRING) - : (raw ? RAW_FSTRING : FSTRING); - return tstring ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); + state->kind = kind; + return _PyLexer_IsTString(kind) + ? MAKE_TOKEN(TSTRING_START) : MAKE_TOKEN(FSTRING_START); } int From 3fdd4986bb1c6f2c7109b297f4e8a5e7f1ff703b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 04/29] Transfer owned tokenizer encoding strings --- Parser/tokenizer/decoder.c | 31 ++++++++++++------------------- Parser/tokenizer/reader.c | 14 +++++--------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index d3c3f7202e9e336..1d438406bc7b461 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -179,17 +179,14 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) if (cursor == start) { continue; } - char *found = _PyTok_CopyBytes(start, cursor - start); - if (found == NULL) { - return -1; - } + Py_ssize_t encoding_len = cursor - start; char normalized[13]; int n; - for (n = 0; n < 12 && found[n] != '\0'; n++) { - normalized[n] = found[n] == '_' ? '-' : Py_TOLOWER(found[n]); + for (n = 0; n < 12 && n < encoding_len; n++) { + normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; - const char *canonical = found; + const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || strncmp(normalized, "utf-8-", 6) == 0) { canonical = "utf-8"; @@ -202,14 +199,13 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) strncmp(normalized, "iso-latin-1-", 12) == 0) { canonical = "iso-8859-1"; } - if (canonical != found) { - PyMem_Free(found); - found = _PyTok_CopyBytes(canonical, strlen(canonical)); - if (found == NULL) { - return -1; - } + if (canonical != start) { + encoding_len = strlen(canonical); + } + *encoding = _PyTok_CopyBytes(canonical, encoding_len); + if (*encoding == NULL) { + return -1; } - *encoding = found; *scan_next = 0; return 0; } @@ -269,11 +265,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, PyMem_Free(cookie); return _PYTOK_ENCODING_ERROR; } - if (!bom && _PyTok_SetEncoding(tok, cookie) < 0) { - PyMem_Free(cookie); - return _PYTOK_ENCODING_ERROR; - } - PyMem_Free(cookie); + PyMem_Free(tok->encoding); + tok->encoding = cookie; return _PYTOK_ENCODING_DONE; } diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index fd7225f63b37660..222118577f01e85 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -1,6 +1,7 @@ #include "Python.h" #include "pycore_fileutils.h" #include "pycore_pystate.h" +#include "pycore_runtime.h" #include "errcode.h" #include "helpers.h" @@ -794,21 +795,16 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) fclose(fp); return NULL; } - tok->filename = filename != NULL - ? Py_NewRef(filename) : PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return NULL; - } + _Py_DECLARE_STR(anon_string, ""); + tok->filename = Py_NewRef(filename != NULL ? filename : &_Py_STR(anon_string)); if (initialize_file(tok) < 0) { fclose(fp); _PyTokenizer_Free(tok); return NULL; } fclose(fp); - char *encoding = tok->encoding == NULL - ? NULL : _PyTok_CopyBytes(tok->encoding, strlen(tok->encoding)); + char *encoding = tok->encoding; + tok->encoding = NULL; _PyTokenizer_Free(tok); return encoding; } From b0ab20bd447399ac65bde89ecb82909ba400f154 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 05/29] Remove unused tokenizer state and redundant branches --- Parser/lexer/string.c | 2 +- Parser/tokenizer/reader.c | 5 +---- Parser/tokenizer/types.h | 6 ------ Python/Python-tokenize.c | 29 +++++++---------------------- 4 files changed, 9 insertions(+), 33 deletions(-) diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 00c6543341ce75c..3c38e1ab7c32cec 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -13,7 +13,7 @@ string_error_token(struct tok_state *tok, struct token *token, { tok->diagnostic = (_PyTokenizer_Diagnostic){ .location = {location.lineno, location.byte_col + 1}, - .text_span = _PyTok_SpanFromBounds(start - location.byte_col, tok->inp), + .text_span = {start - location.byte_col, tok->inp}, }; int type = _PyLexer_token_setup(tok, token, ERRORTOKEN, -1, -1); token->start_loc = location; diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 222118577f01e85..75e3bda929884fc 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -182,7 +182,7 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) break; } } - int implicit = len == 0 || reader->file_buffer[len - 1] != '\n'; + int implicit = reader->file_buffer[len - 1] != '\n'; chunk->data = reader->file_buffer; chunk->len = len; chunk->implicit_newline = implicit; @@ -466,9 +466,6 @@ next_readline(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_ERROR; } } - if (pop_decoded_line(reader, chunk)) { - return _PYTOK_READ_LINE; - } } } diff --git a/Parser/tokenizer/types.h b/Parser/tokenizer/types.h index c12b6de449c7008..b488cb01618d35f 100644 --- a/Parser/tokenizer/types.h +++ b/Parser/tokenizer/types.h @@ -18,12 +18,6 @@ typedef struct { int byte_col; } _PyTok_Loc; -static inline _PyTok_Span -_PyTok_SpanFromBounds(_PyTok_Off start, _PyTok_Off end) -{ - return (_PyTok_Span){start, end}; -} - static inline int _PyTok_SpanIsValid(_PyTok_Span span) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 3802709732e29da..ced88dc9ecebfb5 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -37,7 +37,6 @@ typedef struct /* Needed to cache line for performance */ PyObject *last_line; Py_ssize_t last_lineno; - Py_ssize_t last_end_lineno; Py_ssize_t byte_col_offset_diff; } tokenizeriterobject; @@ -79,18 +78,15 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, self->last_line = NULL; self->byte_col_offset_diff = 0; self->last_lineno = 0; - self->last_end_lineno = 0; return (PyObject *)self; } -static int +static void _tokenizer_error(tokenizeriterobject *it) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it); - if (PyErr_Occurred()) { - return -1; - } + assert(!PyErr_Occurred()); const char *msg = NULL; PyObject* errtype = PyExc_SyntaxError; @@ -105,19 +101,17 @@ _tokenizer_error(tokenizeriterobject *it) PyErr_SyntaxLocationObject( info.filename, info.location.lineno, (int)Py_MAX(0, info.input_span.end - info.input_span.start)); - return -1; + return; case E_DEDENT: msg = "unindent does not match any outer indentation level"; errtype = PyExc_IndentationError; break; case E_INTR: - if (!PyErr_Occurred()) { - PyErr_SetNone(PyExc_KeyboardInterrupt); - } - return -1; + PyErr_SetNone(PyExc_KeyboardInterrupt); + return; case E_NOMEM: PyErr_NoMemory(); - return -1; + return; case E_TABSPACE: errtype = PyExc_TabError; msg = "inconsistent use of tabs and spaces in indentation"; @@ -138,7 +132,6 @@ _tokenizer_error(tokenizeriterobject *it) PyObject* error_line = NULL; PyObject* tmp = NULL; PyObject* value = NULL; - int result = 0; Py_ssize_t input_size; const char *input = _PyTokenizer_SpanView( @@ -148,30 +141,25 @@ _tokenizer_error(tokenizeriterobject *it) size -= 1; // Remove the newline character from the end of the line error_line = PyUnicode_DecodeUTF8(input, size, "replace"); if (!error_line) { - result = -1; goto exit; } Py_ssize_t offset = _PyPegen_byte_offset_to_character_offset(error_line, input_size); if (offset == -1) { - result = -1; goto exit; } tmp = Py_BuildValue("(OnnOOO)", info.filename, info.location.lineno, offset, error_line, Py_None, Py_None); if (!tmp) { - result = -1; goto exit; } errstr = PyUnicode_FromString(msg); if (!errstr) { - result = -1; goto exit; } value = _PyTuple_FromPair(errstr, tmp); if (!value) { - result = -1; goto exit; } @@ -182,7 +170,6 @@ _tokenizer_error(tokenizeriterobject *it) Py_XDECREF(error_line); Py_XDECREF(tmp); Py_XDECREF(value); - return result; } static PyObject * @@ -243,7 +230,6 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, } } it->last_lineno = lineno; - it->last_end_lineno = end_lineno; } static PyObject * @@ -266,9 +252,8 @@ tokenizeriter_next(PyObject *op) } goto exit; } - if (it->done || type == ERRORTOKEN) { + if (it->done) { PyErr_SetString(PyExc_StopIteration, "EOF"); - it->done = 1; goto exit; } _PyToken_View view; From 8b0d8dd8456bad80b82da12389e0369caa51f17b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:56:26 +0100 Subject: [PATCH 06/29] Avoid rescanning partial decoded tokenizer lines --- Lib/test/test_tokenize.py | 14 ++++++++++++++ ...-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/tokenizer/reader.c | 11 +++++++++-- Parser/tokenizer/reader_internal.h | 1 + 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 53215eceeb8aed3..725d8c7376dacd8 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2327,6 +2327,20 @@ def test_utf8_decoder_spans_readline_calls(self): tokenize.TokenInfo(token.ENDMARKER, "", (2, 0), (2, 0), ""), ]) + def test_utf8_decoder_spans_many_readline_calls(self): + for prefix in (b"", b"previous\n"): + with self.subTest(prefix=prefix): + chunks = ([prefix + b"x\xc3"] + [b"\xa9\xc3"] * 100 + + [b"\xa9\n", b"z\xc3", b"\xa9\n", b""]) + source = b"".join(chunks) + expected = list(_tokenize.TokenizerIter( + BytesIO(source).readline, encoding="utf-8", extra_tokens=True + )) + tokens = list(_tokenize.TokenizerIter( + iter(chunks).__next__, encoding="utf-8", extra_tokens=True + )) + self.assertEqual(tokens, expected) + def test_utf8_decoder_replaces_incomplete_input_at_eof(self): expected = [ tokenize.TokenInfo(token.NAME, "x�", (1, 0), (1, 2), "x�"), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index 2b0aae1a3d2b4c4..4ae20f3436d238d 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1 +1,2 @@ Correct missing-brace error messages for multiline f-strings and t-strings. +Improve tokenization of UTF-8 input split across many readline calls. diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 75e3bda929884fc..c686b39a99567d0 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -72,6 +72,7 @@ append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, (size_t)remaining); + reader->decoded_scan -= reader->decoded_pos; reader->decoded_pos = 0; reader->decoded_len = remaining; } @@ -105,13 +106,17 @@ static int pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) { assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len); + assert(reader->decoded_scan >= reader->decoded_pos && + reader->decoded_scan <= reader->decoded_len); if (reader->decoded_pos == reader->decoded_len) { return 0; } char *start = reader->decoded + reader->decoded_pos; - char *newline = memchr(start, '\n', - reader->decoded_len - reader->decoded_pos); + // Previously scanned bytes cannot contain a newline. + char *newline = memchr(reader->decoded + reader->decoded_scan, '\n', + reader->decoded_len - reader->decoded_scan); if (newline == NULL) { + reader->decoded_scan = reader->decoded_len; return 0; } Py_ssize_t len = newline - start + 1; @@ -119,10 +124,12 @@ pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk) chunk->len = len; chunk->ownership = _PYTOK_CHUNK_BORROWED; reader->decoded_pos += len; + reader->decoded_scan = reader->decoded_pos; chunk->implicit_newline = reader->decoded_pos == reader->decoded_len && reader->decoded_tail_is_implicit; if (reader->decoded_pos == reader->decoded_len) { reader->decoded_pos = reader->decoded_len = 0; + reader->decoded_scan = 0; reader->decoded_tail_is_implicit = 0; } return 1; diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 9638fef88cd593b..fff2cc9db86be40 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -50,6 +50,7 @@ typedef struct _PyTok_Reader { char *decoded; Py_ssize_t decoded_pos; + Py_ssize_t decoded_scan; Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; From b913bf8cf5b57e49867988fdc13b5b53daf94551 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:58:01 +0100 Subject: [PATCH 07/29] Reuse prepared tokenizer line terminators for CR detection --- Parser/tokenizer/decoder.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 1d438406bc7b461..653b5748b5729ad 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -322,12 +322,14 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, data[pos + raw_line_len - 1] == '\r'; int add_newline = add_final_newline && pos + raw_line_len == len && !terminated; - int normalize = add_newline || - (!preserve_crlf && - memchr(data + pos, '\r', raw_line_len) != NULL); - const char *line = data + pos; Py_ssize_t line_len = raw_line_len; + // raw_line_length stops at the first CR or LF, so any CR is in + // the line terminator. + int normalize = add_newline || + (!preserve_crlf && + (line[line_len - 1] == '\r' || + (line_len > 1 && line[line_len - 2] == '\r'))); int implicit = 0; if (normalize) { if (line_len > PY_SSIZE_T_MAX - 2) { From 65a1f41d64e6eb5c3da97d64a678ab722a77e5b8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:00:36 +0100 Subject: [PATCH 08/29] Reuse interactive tokenizer chunks without carriage returns --- Lib/test/test_repl.py | 13 +++++++++++++ Parser/tokenizer/reader.c | 4 ++++ 2 files changed, 17 insertions(+) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 21e603c561d73c0..e1ce17b4dd4256c 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -214,6 +214,19 @@ def test_multiline_fstring_source_reallocation(self): self.assertEqual(p.returncode, 0) self.assertIn(">>> 3\n>>> ", output) + def test_long_non_ascii_input(self): + value = "é漢" * 5000 + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + p.stdin.write( + f"value = {value!r}{newline}" + f"print(len(value), value[:2]){newline}" + ) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(">>> 10000 é漢\n>>> ", output) + def test_close_stdin(self): user_input = dedent(''' import os diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index c686b39a99567d0..68fdb2f37e4c927 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -507,6 +507,10 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_ChunkClear(&decoded); return _PYTOK_READ_ERROR; } + if (memchr(decoded.data, '\r', decoded.len) == NULL) { + *chunk = decoded; + return _PYTOK_READ_LINE; + } chunk->data = _PyTok_NormalizeNewlines( decoded.data, decoded.len, 0, 0, &chunk->len, NULL); From d807f73ff7ae3aa99871e8ac5510a6458651173b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:12 +0100 Subject: [PATCH 09/29] Release tokenizer constructor resources on allocation failure --- Python/Python-tokenize.c | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index ced88dc9ecebfb5..882ef134c9e8f91 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -56,21 +56,18 @@ tokenizeriter_new_impl(PyTypeObject *type, PyObject *readline, int extra_tokens, const char *encoding) /*[clinic end generated code: output=7501a1211683ce16 input=f7dddf8a613ae8bd]*/ { - tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); - if (self == NULL) { - return NULL; - } - PyObject *filename = PyUnicode_FromString(""); - if (filename == NULL) { + struct tok_state *tok = _PyTokenizer_FromReadline(readline, encoding); + if (tok == NULL) { return NULL; } - self->tok = _PyTokenizer_FromReadline(readline, encoding); - if (self->tok == NULL) { - Py_DECREF(filename); + _Py_DECLARE_STR(anon_string, ""); + _PyTokenizer_SetContext(tok, &_Py_STR(anon_string), NULL); + tokenizeriterobject *self = (tokenizeriterobject *)type->tp_alloc(type, 0); + if (self == NULL) { + _PyTokenizer_Free(tok); return NULL; } - _PyTokenizer_SetContext(self->tok, filename, NULL); - Py_DECREF(filename); + self->tok = tok; _PyTokenizer_SetOptions(self->tok, extra_tokens, 0); self->extra_tokens = extra_tokens; self->done = 0; From 98a96e55f8dc14615085a0d7b4763722b6fbbf72 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 17:57:50 +0100 Subject: [PATCH 10/29] Narrow interactive newline normalization wrapper --- Parser/tokenizer/decoder.c | 21 +++++++-------------- Parser/tokenizer/reader.c | 3 +-- Parser/tokenizer/reader_internal.h | 3 +-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index 653b5748b5729ad..d30d5e7a0227c5e 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -77,7 +77,7 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, static void normalize_newlines_into(char *result, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline, - Py_ssize_t *out_len, int *implicit_newline) + Py_ssize_t *out_len) { Py_ssize_t write = 0; if (preserve_crlf || memchr(data, '\r', len) == NULL) { @@ -97,21 +97,16 @@ normalize_newlines_into(char *result, const char *data, Py_ssize_t len, result[write++] = c; } } - int implicit = add_final_newline && write > 0 && result[write - 1] != '\n'; - if (implicit) { + if (add_final_newline && write > 0 && result[write - 1] != '\n') { result[write++] = '\n'; } result[write] = '\0'; *out_len = write; - if (implicit_newline != NULL) { - *implicit_newline = implicit; - } } char * -_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, - int add_final_newline, Py_ssize_t *out_len, - int *implicit_newline) +_PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, + Py_ssize_t *out_len) { if (len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); @@ -122,8 +117,7 @@ _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, int preserve_crlf, PyErr_NoMemory(); return NULL; } - normalize_newlines_into(result, data, len, preserve_crlf, - add_final_newline, out_len, implicit_newline); + normalize_newlines_into(result, data, len, 0, 0, out_len); return result; } @@ -330,7 +324,6 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, (!preserve_crlf && (line[line_len - 1] == '\r' || (line_len > 1 && line[line_len - 2] == '\r'))); - int implicit = 0; if (normalize) { if (line_len > PY_SSIZE_T_MAX - 2) { PyErr_NoMemory(); @@ -345,7 +338,7 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, } normalize_newlines_into(normalized, line, line_len, preserve_crlf, add_newline, - &line_len, &implicit); + &line_len); line = normalized; } _PyTok_Off appended = _PyTok_SourceAppendLine( @@ -355,7 +348,7 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, ? E_NOMEM : E_ERROR; goto error; } - tok->reader->prepared_final_newline_is_implicit = implicit; + tok->reader->prepared_final_newline_is_implicit = add_newline; pos += raw_line_len; } PyMem_Free(normalized); diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 68fdb2f37e4c927..9859c650248287c 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -512,8 +512,7 @@ next_interactive(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } chunk->data = _PyTok_NormalizeNewlines( - decoded.data, decoded.len, 0, 0, - &chunk->len, NULL); + decoded.data, decoded.len, &chunk->len); _PyTok_ChunkClear(&decoded); if (chunk->data == NULL) { PyErr_NoMemory(); diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index fff2cc9db86be40..26e5957b68af70c 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -69,8 +69,7 @@ int _PyTok_ReserveBuffer(char **, Py_ssize_t *, Py_ssize_t, Py_ssize_t); char *_PyTok_CopyBytes(const char *, Py_ssize_t); int _PyTok_DecodeOnce( struct tok_state *, _PyTok_Chunk *, const char *, const char *); -char *_PyTok_NormalizeNewlines( - const char *, Py_ssize_t, int, int, Py_ssize_t *, int *); +char *_PyTok_NormalizeNewlines(const char *, Py_ssize_t, Py_ssize_t *); void _PyTok_ChunkClear(_PyTok_Chunk *); int _PyTok_SetEncoding(struct tok_state *, const char *); _PyTok_EncodingResult _PyTok_DetectEncoding( From 6eb2e4a7878e8826446419a7ae95a1c635836366 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:03 +0100 Subject: [PATCH 11/29] Close duplicated tokenizer descriptors when fdopen fails --- Parser/tokenizer/reader.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 9859c650248287c..494aca408615760 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -786,7 +786,16 @@ static FILE * fdopen_borrow(int fd) { int copy = _Py_dup(fd); - return copy < 0 ? NULL : fdopen(copy, "r"); + if (copy < 0) { + return NULL; + } + FILE *fp = fdopen(copy, "r"); + if (fp == NULL) { + int saved_errno = errno; + close(copy); + errno = saved_errno; + } + return fp; } #endif From f20095351115a57e51febad52d7a4070d19a1ae3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:03 +0100 Subject: [PATCH 12/29] Preserve cached tokenizer lines when newline allocation fails --- Python/Python-tokenize.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 882ef134c9e8f91..cedb1f7e3f160e4 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -309,8 +309,8 @@ tokenizeriter_next(PyObject *op) type = OP; } else if (type == NEWLINE) { - Py_DECREF(str); if (!view.implicit_newline) { + Py_DECREF(str); assert(token_start != NULL); if (token_start[0] == '\r') { str = PyUnicode_FromString("\r\n"); @@ -328,7 +328,6 @@ tokenizeriter_next(PyObject *op) } if (str == NULL) { - Py_DECREF(line); goto exit; } } From f01c604188e76b587a48fb45ac3eca65d05a52f8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:40:54 +0100 Subject: [PATCH 13/29] Collect cycles involving tokenizer readline callbacks --- Lib/test/test_tokenize.py | 13 +++++++++++++ Parser/tokenizer/reader.c | 18 ++++++++++++++++++ Parser/tokenizer/tokenizer.h | 1 + Python/Python-tokenize.c | 13 ++++++++++++- 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 725d8c7376dacd8..1602489f3d52db2 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -8,6 +8,7 @@ import token import tokenize import unittest +import weakref from io import BytesIO, StringIO from textwrap import dedent from unittest import TestCase, mock @@ -2254,6 +2255,18 @@ def check_tokenize(self, s, expected): ) self.assertEqual(result, expected.rstrip().splitlines()) + def test_readline_reference_cycle(self): + class Readline: + def __call__(self): + return "" + + readline = Readline() + readline.iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + ref = weakref.ref(readline) + del readline + support.gc_collect() + self.assertIsNone(ref()) + def test_encoding(self): def readline(encoding): yield "1+1".encode(encoding) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 494aca408615760..298a9e64a46188f 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -38,6 +38,24 @@ _PyTok_ReaderFree(struct tok_state *tok) tok->reader = NULL; } +int +_PyTokenizer_Traverse(struct tok_state *tok, visitproc visit, void *arg) +{ + Py_VISIT(tok->filename); + Py_VISIT(tok->module); + _PyTok_Reader *reader = tok->reader; + Py_VISIT(reader->readline); + Py_VISIT(reader->decoder); + for (int i = 0; + i < (int)Py_ARRAY_LENGTH(reader->prefetched_lines); i++) { + _PyTok_Chunk *chunk = &reader->prefetched_lines[i]; + if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT) { + Py_VISIT(chunk->owner); + } + } + return 0; +} + int _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, Py_ssize_t initial_capacity) diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index 82a84830d7cbfe7..2ef87add3646c3c 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -58,6 +58,7 @@ typedef struct { Errors are returned as ERRORTOKEN, with or without a Python exception. */ void _PyTokenizer_Get(struct tok_state *, struct token *); void _PyTokenizer_Free(struct tok_state *); +int _PyTokenizer_Traverse(struct tok_state *, visitproc, void *); void _PyTokenizer_raise_init_error(PyObject *filename); void _PyToken_Init(struct token *); static inline void diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index cedb1f7e3f160e4..a10200bcd5ff6df 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -348,15 +348,26 @@ tokenizeriter_dealloc(PyObject *op) { tokenizeriterobject *it = (tokenizeriterobject*)op; PyTypeObject *tp = Py_TYPE(it); + PyObject_GC_UnTrack(it); Py_XDECREF(it->last_line); _PyTokenizer_Free(it->tok); tp->tp_free(it); Py_DECREF(tp); } +static int +tokenizeriter_traverse(PyObject *op, visitproc visit, void *arg) +{ + tokenizeriterobject *it = (tokenizeriterobject *)op; + Py_VISIT(Py_TYPE(it)); + Py_VISIT(it->last_line); + return _PyTokenizer_Traverse(it->tok, visit, arg); +} + static PyType_Slot tokenizeriter_slots[] = { {Py_tp_new, tokenizeriter_new}, {Py_tp_dealloc, tokenizeriter_dealloc}, + {Py_tp_traverse, tokenizeriter_traverse}, {Py_tp_getattro, PyObject_GenericGetAttr}, {Py_tp_iter, PyObject_SelfIter}, {Py_tp_iternext, tokenizeriter_next}, @@ -366,7 +377,7 @@ static PyType_Slot tokenizeriter_slots[] = { static PyType_Spec tokenizeriter_spec = { .name = "_tokenize.TokenizerIter", .basicsize = sizeof(tokenizeriterobject), - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE | Py_TPFLAGS_HAVE_GC), .slots = tokenizeriter_slots, }; From f3e60767adfed1ea1f539e3b3e60a7e300ff585b Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:57:59 +0100 Subject: [PATCH 14/29] Preserve buffered text when decoding source files --- Lib/test/test_source_encoding.py | 5 +++++ .../2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/tokenizer/reader.c | 3 ++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index ec98e609c4e98f9..63f3126c99925cc 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -135,6 +135,11 @@ def test_stateful_file_decoder_spans_lines(self): ) self._assert_python_file_ok(source) + @support.requires_subprocess() + def test_stateful_file_decoder_preserves_buffered_text(self): + source = b"# coding: hz\nx~\ny = 1\nassert xy == 1\n" + self._assert_python_file_ok(source) + @support.requires_subprocess() def test_stateful_file_decoder_finalizes_before_implicit_newline(self): source = b"# coding: hz\n# ~{1dA?" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index 4ae20f3436d238d..a1b233bc758f951 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1,2 +1,3 @@ Correct missing-brace error messages for multiline f-strings and t-strings. Improve tokenization of UTF-8 input split across many readline calls. +Preserve source order when reading files with stateful encodings. diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 298a9e64a46188f..e692ae8ac2a24fe 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -350,7 +350,8 @@ next_file(struct tok_state *tok, _PyTok_Chunk *chunk) return _PYTOK_READ_LINE; } int decoded = _PyTok_DecodeChunk(tok, &input, 0); - if (decoded == 0 && chunk_is_line(&input)) { + if (decoded == 0 && reader->decoded_pos == reader->decoded_len && + chunk_is_line(&input)) { *chunk = input; return _PYTOK_READ_LINE; } From ec6259a52df7321345237a51338c3ecdd8889657 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:58:00 +0100 Subject: [PATCH 15/29] Clear the transferred syntax error line before cleanup --- Parser/pegen_errors.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index 74ab56c912ae1af..6997a1158e9310f 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -327,6 +327,7 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, } tmp = Py_BuildValue("(OnnNnn)", info.filename, lineno, col_number, error_line, end_lineno, end_col_number); + error_line = NULL; if (!tmp) { goto error; } From 30f8f525d178ac41d6be3c805368629f95e9c249 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 18:58:00 +0100 Subject: [PATCH 16/29] Propagate tokenizer column conversion failures --- Parser/pegen.c | 3 +++ Python/Python-tokenize.c | 19 ++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 5edf5542c7a8f99..8a45435edaa7307 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -35,6 +35,9 @@ Py_ssize_t _PyPegen_byte_offset_to_character_offset_line(PyObject *line, Py_ssize_t col_offset, Py_ssize_t end_col_offset) { const unsigned char *data = (const unsigned char*)PyUnicode_AsUTF8(line); + if (data == NULL) { + return -1; + } Py_ssize_t len = 0; while (col_offset < end_col_offset) { diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index a10200bcd5ff6df..f9b8a949ab5649e 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -190,7 +190,7 @@ _get_current_line(tokenizeriterobject *it, int current_lineno, return line; } -static void +static int _get_col_offsets(tokenizeriterobject *it, const struct token *token, const char *token_start, const char *line_start, const char *end_line_start, PyObject *line, int line_changed, @@ -206,6 +206,9 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, byte_offset = token_start - line_start; if (line_changed) { *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset); + if (*col_offset < 0) { + return -1; + } it->byte_col_offset_diff = byte_offset - *col_offset; } else { @@ -218,15 +221,22 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, if (lineno == end_lineno) { // Avoid rescanning the prefix of a very long line. Py_ssize_t token_col_offset = _PyPegen_byte_offset_to_character_offset_line(line, byte_offset, end_byte_offset); + if (token_col_offset < 0) { + return -1; + } *end_col_offset = *col_offset + token_col_offset; it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + if (*end_col_offset < 0) { + return -1; + } it->byte_col_offset_diff += end_byte_offset - *end_col_offset; } } it->last_lineno = lineno; + return 0; } static PyObject * @@ -295,8 +305,11 @@ tokenizeriter_next(PyObject *op) Py_ssize_t end_lineno = token.end_loc.lineno; Py_ssize_t col_offset = -1; Py_ssize_t end_col_offset = -1; - _get_col_offsets(it, &token, token_start, view.line, view.end_line, line, - line_changed, &col_offset, &end_col_offset); + if (_get_col_offsets(it, &token, token_start, view.line, view.end_line, line, + line_changed, &col_offset, &end_col_offset) < 0) { + Py_DECREF(str); + goto exit; + } if (it->extra_tokens) { if (is_trailing_token) { From 5ad32394ce370375574d26c69376e923a72ed2ad Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:21:09 +0100 Subject: [PATCH 17/29] Allocate parser repetition buffers on first match --- Parser/parser.c | 780 +++++------------- .../peg_generator/pegen/c_generator_rules.py | 13 +- 2 files changed, 198 insertions(+), 595 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index b3d73dd4fafb762..c3b5c1592cee71f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27282,14 +27282,8 @@ _loop0_1_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // NEWLINE if (p->error_indicator) { @@ -27304,7 +27298,7 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -27349,14 +27343,8 @@ _loop1_2_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // statement if (p->error_indicator) { @@ -27371,7 +27359,7 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -27421,14 +27409,8 @@ _loop0_3_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ';' simple_stmt if (p->error_indicator) { @@ -27452,7 +27434,7 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -27975,14 +27957,8 @@ _loop1_12_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -27997,7 +27973,7 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28047,14 +28023,8 @@ _loop0_13_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' NAME if (p->error_indicator) { @@ -28078,7 +28048,7 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28267,14 +28237,8 @@ _loop0_17_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28289,7 +28253,7 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28334,14 +28298,8 @@ _loop1_18_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('.' | '...') if (p->error_indicator) { @@ -28356,7 +28314,7 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28406,14 +28364,8 @@ _loop0_19_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' import_from_as_name if (p->error_indicator) { @@ -28437,7 +28389,7 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28569,14 +28521,8 @@ _loop0_22_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_as_name if (p->error_indicator) { @@ -28600,7 +28546,7 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28686,14 +28632,8 @@ _loop1_24_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('@' named_expression NEWLINE) if (p->error_indicator) { @@ -28708,7 +28648,7 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28853,14 +28793,8 @@ _loop0_27_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -28875,7 +28809,7 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28920,14 +28854,8 @@ _loop0_28_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -28942,7 +28870,7 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -28987,14 +28915,8 @@ _loop1_29_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_no_default if (p->error_indicator) { @@ -29009,7 +28931,7 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29059,14 +28981,8 @@ _loop1_30_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_with_default if (p->error_indicator) { @@ -29081,7 +28997,7 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29131,14 +29047,8 @@ _loop0_31_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29153,7 +29063,7 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29198,14 +29108,8 @@ _loop1_32_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // param_maybe_default if (p->error_indicator) { @@ -29220,7 +29124,7 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29270,14 +29174,8 @@ _loop0_33_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' with_item if (p->error_indicator) { @@ -29301,7 +29199,7 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29463,14 +29361,8 @@ _loop1_36_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_block if (p->error_indicator) { @@ -29485,7 +29377,7 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29535,14 +29427,8 @@ _loop1_37_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // except_star_block if (p->error_indicator) { @@ -29557,7 +29443,7 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29607,14 +29493,8 @@ _loop1_38_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // case_block if (p->error_indicator) { @@ -29629,7 +29509,7 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -29679,14 +29559,8 @@ _loop0_39_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // '|' closed_pattern if (p->error_indicator) { @@ -29710,7 +29584,7 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30005,14 +29879,8 @@ _loop0_44_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' maybe_star_pattern if (p->error_indicator) { @@ -30036,7 +29904,7 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30122,14 +29990,8 @@ _loop0_46_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' key_value_pattern if (p->error_indicator) { @@ -30153,7 +30015,7 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30296,14 +30158,8 @@ _loop0_49_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' pattern if (p->error_indicator) { @@ -30327,7 +30183,7 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30413,14 +30269,8 @@ _loop0_51_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' keyword_pattern if (p->error_indicator) { @@ -30444,7 +30294,7 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30530,14 +30380,8 @@ _loop0_53_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' type_param if (p->error_indicator) { @@ -30561,7 +30405,7 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30647,14 +30491,8 @@ _loop1_55_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' expression) if (p->error_indicator) { @@ -30669,7 +30507,7 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30719,14 +30557,8 @@ _loop1_56_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_expression) if (p->error_indicator) { @@ -30741,7 +30573,7 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30791,14 +30623,8 @@ _loop0_57_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' star_named_expression if (p->error_indicator) { @@ -30822,7 +30648,7 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30908,14 +30734,8 @@ _loop1_59_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('or' conjunction) if (p->error_indicator) { @@ -30930,7 +30750,7 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -30980,14 +30800,8 @@ _loop1_60_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('and' inversion) if (p->error_indicator) { @@ -31002,7 +30816,7 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31052,14 +30866,8 @@ _loop1_61_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // compare_op_bitwise_or_pair if (p->error_indicator) { @@ -31074,7 +30882,7 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31167,14 +30975,8 @@ _loop0_63_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (slice | starred_expression) if (p->error_indicator) { @@ -31198,7 +31000,7 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31615,14 +31417,8 @@ _loop0_70_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31637,7 +31433,7 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31682,14 +31478,8 @@ _loop0_71_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31704,7 +31494,7 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31749,14 +31539,8 @@ _loop1_72_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_no_default if (p->error_indicator) { @@ -31771,7 +31555,7 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31821,14 +31605,8 @@ _loop1_73_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_with_default if (p->error_indicator) { @@ -31843,7 +31621,7 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31893,14 +31671,8 @@ _loop0_74_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31915,7 +31687,7 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -31960,14 +31732,8 @@ _loop1_75_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // lambda_param_maybe_default if (p->error_indicator) { @@ -31982,7 +31748,7 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32032,14 +31798,8 @@ _loop0_76_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_format_spec if (p->error_indicator) { @@ -32054,7 +31814,7 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32099,14 +31859,8 @@ _loop0_77_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // fstring_middle if (p->error_indicator) { @@ -32121,7 +31875,7 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32166,14 +31920,8 @@ _loop0_78_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_format_spec if (p->error_indicator) { @@ -32188,7 +31936,7 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32233,14 +31981,8 @@ _loop0_79_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring_middle if (p->error_indicator) { @@ -32255,7 +31997,7 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32300,14 +32042,8 @@ _loop1_80_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (fstring | string) if (p->error_indicator) { @@ -32322,7 +32058,7 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32372,14 +32108,8 @@ _loop1_81_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // tstring if (p->error_indicator) { @@ -32394,7 +32124,7 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32444,14 +32174,8 @@ _loop0_82_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' display_item if (p->error_indicator) { @@ -32475,7 +32199,7 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32610,14 +32334,8 @@ _loop0_85_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' double_starred_kvpair if (p->error_indicator) { @@ -32641,7 +32359,7 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32727,14 +32445,8 @@ _loop1_87_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // for_if_clause if (p->error_indicator) { @@ -32749,7 +32461,7 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32799,14 +32511,8 @@ _loop0_88_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ('if' disjunction) if (p->error_indicator) { @@ -32821,7 +32527,7 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -32944,14 +32650,8 @@ _loop0_90_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression | (assignment_expression | expression !':=') !'=') if (p->error_indicator) { @@ -32975,7 +32675,7 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33108,14 +32808,8 @@ _loop0_93_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_starred if (p->error_indicator) { @@ -33139,7 +32833,7 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33271,14 +32965,8 @@ _loop0_96_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' kwarg_or_double_starred if (p->error_indicator) { @@ -33302,7 +32990,7 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33388,14 +33076,8 @@ _loop0_98_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' star_target) if (p->error_indicator) { @@ -33410,7 +33092,7 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33571,14 +33253,8 @@ _loop0_101_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' del_target if (p->error_indicator) { @@ -33602,7 +33278,7 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33688,14 +33364,8 @@ _loop0_103_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' expression if (p->error_indicator) { @@ -33719,7 +33389,7 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -33905,14 +33575,8 @@ _loop0_107_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (starred_expression !'=') if (p->error_indicator) { @@ -33936,7 +33600,7 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34221,14 +33885,8 @@ _loop1_113_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (!STRING expression_without_invalid) if (p->error_indicator) { @@ -34243,7 +33901,7 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34676,14 +34334,8 @@ _loop0_119_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // star_named_expressions if (p->error_indicator) { @@ -34698,7 +34350,7 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -34743,14 +34395,8 @@ _loop0_120_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (star_targets '=') if (p->error_indicator) { @@ -34765,7 +34411,7 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35288,14 +34934,8 @@ _loop0_129_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' lambda_param if (p->error_indicator) { @@ -35319,7 +34959,7 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35624,14 +35264,8 @@ _loop0_135_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' dotted_name if (p->error_indicator) { @@ -35655,7 +35289,7 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35782,14 +35416,8 @@ _loop0_138_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expression ['as' star_target]) if (p->error_indicator) { @@ -35813,7 +35441,7 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -35899,14 +35527,8 @@ _loop0_140_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // ',' (expressions ['as' star_target]) if (p->error_indicator) { @@ -35930,7 +35552,7 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -36073,14 +35695,8 @@ _loop0_143_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // block if (p->error_indicator) { @@ -36095,7 +35711,7 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); @@ -37726,14 +37342,8 @@ _loop0_172_rule(Parser *p) } void *_res = NULL; int _mark = p->mark; - void **_children = PyMem_Malloc(sizeof(void *)); - if (!_children) { - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _children_capacity = 1; + void **_children = NULL; + Py_ssize_t _children_capacity = 0; Py_ssize_t _n = 0; { // (',' bitwise_or) if (p->error_indicator) { @@ -37748,7 +37358,7 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity *= 2; + _children_capacity = _children_capacity ? _children_capacity * 2 : 1; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); if (!_new_children) { PyMem_Free(_children); diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 6dd598beb1b39c4..43685e6d211b87e 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -31,12 +31,6 @@ def emit(self, value: str) -> None: self.writer.print("p->level--;") self.writer.print(f"return {value};") - def check_memory(self, expr: str) -> None: - self.writer.print(f"if ({expr}) {{") - with self.writer.indent(): - self.no_memory() - self.writer.print("}") - def no_memory(self) -> None: self.writer.print("p->error_indicator = 1;") self.writer.print("PyErr_NoMemory();") @@ -55,15 +49,14 @@ def __init__(self, writer: CWriter, returns: _CReturnEmitter): self.error_returns = returns.with_cleanup(self._release) def initialize(self) -> None: - self._print("void **_children = PyMem_Malloc(sizeof(void *));") - self._returns.check_memory("!_children") - self._print("Py_ssize_t _children_capacity = 1;") + self._print("void **_children = NULL;") + self._print("Py_ssize_t _children_capacity = 0;") self._print("Py_ssize_t _n = 0;") def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity *= 2;") + self._print("_children_capacity = _children_capacity ? _children_capacity * 2 : 1;") self._print( "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" ) From bfa0b827b6c1f99d0d4403ee58ff912aae368892 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:20:09 +0100 Subject: [PATCH 18/29] Allocate type comment storage only when needed --- Parser/pegen.c | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 8a45435edaa7307..05f0c578bf230be 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -146,20 +146,10 @@ init_normalization(Parser *p) return 1; } -static int -growable_comment_array_init(growable_comment_array *arr, size_t initial_size) { - assert(initial_size > 0); - arr->items = PyMem_Malloc(initial_size * sizeof(*arr->items)); - arr->size = initial_size; - arr->num_items = 0; - - return arr->items != NULL; -} - static int growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { if (arr->num_items >= arr->size) { - size_t new_size = arr->size * 2; + size_t new_size = arr->size ? arr->size * 2 : 10; void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; @@ -323,6 +313,7 @@ _PyPegen_fill_token(Parser *p) tag[len] = '\0'; // Ownership of tag passes to the growable array if (!growable_comment_array_add(&p->type_ignore_comments, new_token.end_loc.lineno, tag)) { + PyMem_Free(tag); PyErr_NoMemory(); goto error; } @@ -946,12 +937,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, PyMem_Free(p); return (Parser *) PyErr_NoMemory(); } - if (!growable_comment_array_init(&p->type_ignore_comments, 10)) { - PyMem_Free(p->tokens[0]); - PyMem_Free(p->tokens); - PyMem_Free(p); - return (Parser *) PyErr_NoMemory(); - } + p->type_ignore_comments = (growable_comment_array){0}; p->mark = 0; p->fill = 0; From 96a07b20f975206e657a800ee1645887e06751f3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:22:19 +0100 Subject: [PATCH 19/29] Reuse decoded multiline text when counting token columns --- Python/Python-tokenize.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index f9b8a949ab5649e..c6f690e27724311 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -228,7 +228,8 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token, it->byte_col_offset_diff += token_end - token_start - token_col_offset; } else { - *end_col_offset = _PyPegen_byte_offset_to_character_offset_raw(end_line_start, end_byte_offset); + *end_col_offset = _PyPegen_byte_offset_to_character_offset_line( + line, end_line_start - line_start, token_end - line_start); if (*end_col_offset < 0) { return -1; } From 930bb191aa6cbb5ea1628e4e6a09ff30fa5e1ec5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:23:26 +0100 Subject: [PATCH 20/29] Skip empty decoded tokenizer chunks --- Parser/tokenizer/reader.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index e692ae8ac2a24fe..a49946eeed25ede 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -86,6 +86,9 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, static int append_decoded(_PyTok_Reader *reader, const char *data, Py_ssize_t len) { + if (len == 0) { + return 0; + } if (reader->decoded_pos > 0) { Py_ssize_t remaining = reader->decoded_len - reader->decoded_pos; memmove(reader->decoded, reader->decoded + reader->decoded_pos, From e8eda4278e4f87d65588f3eff61054c5466a9947 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:44:09 +0100 Subject: [PATCH 21/29] Check parser array growth before allocating --- Lib/test/test_type_comments.py | 10 + Parser/parser.c | 910 +++++++++++++++--- Parser/pegen.c | 29 +- Parser/pegen.h | 2 + .../peg_generator/pegen/c_generator_rules.py | 13 +- 5 files changed, 822 insertions(+), 142 deletions(-) diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index d827ac271085bdd..832a1e283e0c279 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -338,6 +338,16 @@ def test_ignores(self): tree = self.classic_parse(ignores) self.assertEqual(tree.type_ignores, []) + def test_many_ignores(self): + comment_count = 25 + tags = [f"[tag_{index}]" for index in range(comment_count)] + source = "".join(f"pass # type: ignore{tag}\n" for tag in tags) + for tree in self.parse_all(source): + self.assertEqual( + [(item.lineno, item.tag) for item in tree.type_ignores], + list(enumerate(tags, start=1)), + ) + def test_longargs(self): for tree in self.parse_all(longargs, minver=8): for t in tree.body: diff --git a/Parser/parser.c b/Parser/parser.c index c3b5c1592cee71f..d8efdd3e3a085cd 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27298,8 +27298,17 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27308,6 +27317,7 @@ _loop0_1_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27359,8 +27369,17 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27369,6 +27388,7 @@ _loop1_2_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27434,8 +27454,17 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27444,6 +27473,7 @@ _loop0_3_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27973,8 +28003,17 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27983,6 +28022,7 @@ _loop1_12_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28048,8 +28088,17 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28058,6 +28107,7 @@ _loop0_13_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28253,8 +28303,17 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28263,6 +28322,7 @@ _loop0_17_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28314,8 +28374,17 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28324,6 +28393,7 @@ _loop1_18_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28389,8 +28459,17 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28399,6 +28478,7 @@ _loop0_19_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28546,8 +28626,17 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28556,6 +28645,7 @@ _loop0_22_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28648,8 +28738,17 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28658,6 +28757,7 @@ _loop1_24_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28809,8 +28909,17 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28819,6 +28928,7 @@ _loop0_27_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28870,8 +28980,17 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28880,6 +28999,7 @@ _loop0_28_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28931,8 +29051,17 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28941,6 +29070,7 @@ _loop1_29_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28997,8 +29127,17 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29007,6 +29146,7 @@ _loop1_30_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29063,8 +29203,17 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29073,6 +29222,7 @@ _loop0_31_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29124,8 +29274,17 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29134,6 +29293,7 @@ _loop1_32_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29199,8 +29359,17 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29209,6 +29378,7 @@ _loop0_33_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29377,8 +29547,17 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29387,6 +29566,7 @@ _loop1_36_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29443,8 +29623,17 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29453,6 +29642,7 @@ _loop1_37_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29509,8 +29699,17 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29519,6 +29718,7 @@ _loop1_38_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29584,8 +29784,17 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29594,6 +29803,7 @@ _loop0_39_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29904,8 +30114,17 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29914,6 +30133,7 @@ _loop0_44_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30015,8 +30235,17 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30025,6 +30254,7 @@ _loop0_46_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30183,8 +30413,17 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30193,6 +30432,7 @@ _loop0_49_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30294,8 +30534,17 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30304,6 +30553,7 @@ _loop0_51_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30405,8 +30655,17 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30415,6 +30674,7 @@ _loop0_53_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30507,8 +30767,17 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30517,6 +30786,7 @@ _loop1_55_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30573,8 +30843,17 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30583,6 +30862,7 @@ _loop1_56_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30648,8 +30928,17 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30658,6 +30947,7 @@ _loop0_57_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30750,8 +31040,17 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30760,6 +31059,7 @@ _loop1_59_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30816,8 +31116,17 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30826,6 +31135,7 @@ _loop1_60_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30882,8 +31192,17 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30892,6 +31211,7 @@ _loop1_61_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31000,8 +31320,17 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31010,6 +31339,7 @@ _loop0_63_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31433,8 +31763,17 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31443,6 +31782,7 @@ _loop0_70_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31494,8 +31834,17 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31504,6 +31853,7 @@ _loop0_71_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31555,8 +31905,17 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31565,6 +31924,7 @@ _loop1_72_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31621,8 +31981,17 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31631,6 +32000,7 @@ _loop1_73_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31687,8 +32057,17 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31697,6 +32076,7 @@ _loop0_74_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31748,8 +32128,17 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31758,6 +32147,7 @@ _loop1_75_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31814,8 +32204,17 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31824,6 +32223,7 @@ _loop0_76_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31875,8 +32275,17 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31885,6 +32294,7 @@ _loop0_77_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31936,8 +32346,17 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31946,6 +32365,7 @@ _loop0_78_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31997,8 +32417,17 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32007,6 +32436,7 @@ _loop0_79_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32058,8 +32488,17 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32068,6 +32507,7 @@ _loop1_80_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32124,8 +32564,17 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32134,6 +32583,7 @@ _loop1_81_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32199,8 +32649,17 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32209,6 +32668,7 @@ _loop0_82_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32359,8 +32819,17 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32369,6 +32838,7 @@ _loop0_85_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32461,8 +32931,17 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32471,6 +32950,7 @@ _loop1_87_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32527,8 +33007,17 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32537,6 +33026,7 @@ _loop0_88_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32675,8 +33165,17 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32685,6 +33184,7 @@ _loop0_90_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32833,8 +33333,17 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32843,6 +33352,7 @@ _loop0_93_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32990,8 +33500,17 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33000,6 +33519,7 @@ _loop0_96_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33092,8 +33612,17 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33102,6 +33631,7 @@ _loop0_98_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33278,8 +33808,17 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33288,6 +33827,7 @@ _loop0_101_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33389,8 +33929,17 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33399,6 +33948,7 @@ _loop0_103_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33600,8 +34150,17 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33610,6 +34169,7 @@ _loop0_107_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33901,8 +34461,17 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33911,6 +34480,7 @@ _loop1_113_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34350,8 +34920,17 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34360,6 +34939,7 @@ _loop0_119_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34411,8 +34991,17 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34421,6 +35010,7 @@ _loop0_120_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34959,8 +35549,17 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34969,6 +35568,7 @@ _loop0_129_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35289,8 +35889,17 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35299,6 +35908,7 @@ _loop0_135_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35441,8 +36051,17 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35451,6 +36070,7 @@ _loop0_138_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35552,8 +36172,17 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35562,6 +36191,7 @@ _loop0_140_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35711,8 +36341,17 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35721,6 +36360,7 @@ _loop0_143_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -37358,8 +37998,17 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - _children_capacity = _children_capacity ? _children_capacity * 2 : 1; - void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _new_capacity = _children_capacity == 0 + ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **_new_children = PyMem_Realloc( + _children, _new_capacity * sizeof(*_children)); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -37368,6 +38017,7 @@ _loop0_172_rule(Parser *p) return NULL; } _children = _new_children; + _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; diff --git a/Parser/pegen.c b/Parser/pegen.c index 05f0c578bf230be..010de72a356d327 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -12,6 +12,7 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { const char *key; // Borrowed from arena-owned token bytes. @@ -147,10 +148,18 @@ init_normalization(Parser *p) } static int -growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { +growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) +{ if (arr->num_items >= arr->size) { - size_t new_size = arr->size ? arr->size * 2 : 10; - void *new_items_array = PyMem_Realloc(arr->items, new_size * sizeof(*arr->items)); + if (arr->size > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*arr->items))) { + return 0; + } + size_t new_size = arr->size == 0 + ? TYPE_IGNORE_INITIAL_CAPACITY + : arr->size * PEGEN_ARRAY_GROWTH_FACTOR; + void *new_items_array = PyMem_Realloc( + arr->items, new_size * sizeof(*arr->items)); if (!new_items_array) { return 0; } @@ -165,8 +174,9 @@ growable_comment_array_add(growable_comment_array *arr, int lineno, char *commen } static void -growable_comment_array_deallocate(growable_comment_array *arr) { - for (unsigned i = 0; i < arr->num_items; i++) { +growable_comment_array_deallocate(growable_comment_array *arr) +{ + for (size_t i = 0; i < arr->num_items; i++) { PyMem_Free(arr->items[i].comment); } PyMem_Free(arr->items); @@ -266,12 +276,13 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to static int _resize_tokens_array(Parser *p) { - if (p->size > INT_MAX / 2 || - (size_t)p->size > PY_SSIZE_T_MAX / (2 * sizeof(*p->tokens))) { + if (p->size > INT_MAX / PEGEN_ARRAY_GROWTH_FACTOR || + (size_t)p->size > PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*p->tokens))) { PyErr_NoMemory(); return -1; } - int newsize = p->size * 2; + int newsize = p->size * PEGEN_ARRAY_GROWTH_FACTOR; Token **new_tokens = PyMem_Realloc(p->tokens, (size_t)newsize * sizeof(Token *)); if (new_tokens == NULL) { PyErr_NoMemory(); @@ -984,7 +995,7 @@ _PyPegen_Parser_Free(Parser *p) Py_XDECREF(p->normalize); // Resizes allocate blocks starting at indices 1, 2, 4, and so on. PyMem_Free(p->tokens[0]); - for (int i = 1; i < p->size; i *= 2) { + for (int i = 1; i < p->size; i *= PEGEN_ARRAY_GROWTH_FACTOR) { PyMem_Free(p->tokens[i]); } PyMem_Free(p->tokens); diff --git a/Parser/pegen.h b/Parser/pegen.h index eb3f1d8a3041a7c..2c5332b4b7e8281 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -29,6 +29,8 @@ #define CURRENT_POS (-5) +#define PEGEN_ARRAY_GROWTH_FACTOR 2 + typedef struct _memo { int type; void *node; diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 43685e6d211b87e..86abea9c9a8c3a8 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -56,12 +56,19 @@ def initialize(self) -> None: def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._print("_children_capacity = _children_capacity ? _children_capacity * 2 : 1;") - self._print( - "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" + self._check_memory( + "(size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / " + "(PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))" ) + self._print("Py_ssize_t _new_capacity = _children_capacity == 0") + with self._indent(): + self._print("? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR;") + self._print("void **_new_children = PyMem_Realloc(") + with self._indent(): + self._print("_children, _new_capacity * sizeof(*_children));") self._check_memory("!_new_children") self._print("_children = _new_children;") + self._print("_children_capacity = _new_capacity;") self._print("}") self._print(f"_children[_n++] = {value};") From 4fe3bd0ff0c17c2946a2af2d27c5c6e317980732 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:46:14 +0100 Subject: [PATCH 22/29] Name tokenizer bounds and derive byte lengths from their data --- Modules/_testinternalcapi/tokenizer.c | 53 ++++++++++++------ Parser/tokenizer/decoder.c | 77 +++++++++++++++------------ Parser/tokenizer/reader.c | 6 ++- Parser/tokenizer/reader_internal.h | 6 ++- Parser/tokenizer/source.c | 9 ++-- 5 files changed, 95 insertions(+), 56 deletions(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index fd131abe985937c..94b5742d3825f6e 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -38,6 +38,12 @@ static PyObject * test_tokenizer_source(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + const char multiple_lines[] = "a\nb\n"; + const char first_line[] = "alpha\n"; + const char second_line[] = "\xce\xb2\n"; + const char expected[] = "alpha\n\xce\xb2\n"; + const char tail[] = "tail"; + const char terminated_line[] = "x\n"; _PyTok_SourceText source; _PyTok_SourceInit(&source); @@ -49,13 +55,15 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), _PyTok_SourceAppendLine(&source, "", 0) < 0, "accepted empty source line") < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "a\nb\n", 4) < 0, + _PyTok_SourceAppendLine( + &source, multiple_lines, sizeof(multiple_lines) - 1) < 0, "accepted multiple source lines") < 0 || check(_PyTok_SourceAppendLine( - &source, "alpha\n", 6) == 0, + &source, first_line, sizeof(first_line) - 1) == 0, "wrong first source offset") < 0 || check(_PyTok_SourceAppendLine( - &source, "\xce\xb2\n", 3) == 6, + &source, second_line, sizeof(second_line) - 1) == + (Py_ssize_t)sizeof(first_line) - 1, "wrong second source offset") < 0) { goto error; } @@ -68,16 +76,17 @@ test_tokenizer_source(PyObject *Py_UNUSED(module), goto error; } - if (check(source.len == 9 && - memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0, + if (check(source.len == (Py_ssize_t)sizeof(expected) - 1 && + memcmp(source.bytes, expected, sizeof(expected)) == 0, "wrong source contents") < 0) { goto error; } _PyTok_SourceClear(&source); - if (_PyTok_SourceAppendLine(&source, "tail", 4) < 0 || + if (_PyTok_SourceAppendLine(&source, tail, sizeof(tail) - 1) < 0 || check_system_error( - _PyTok_SourceAppendLine(&source, "x\n", 2) < 0, + _PyTok_SourceAppendLine( + &source, terminated_line, sizeof(terminated_line) - 1) < 0, "appended after unterminated source line") < 0) { goto error; } @@ -98,24 +107,34 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { + enum { LINE_COUNT = 260 }; + const char first_line[] = "x\n"; + const char second_line[] = "y\n"; + const char tail[] = "tail"; + const char final_line[] = "z\n"; + const _PyTok_Off first_batch_len = LINE_COUNT * (sizeof(first_line) - 1); + const _PyTok_Off second_batch_len = LINE_COUNT * (sizeof(second_line) - 1); + const _PyTok_Off discarded_len = first_batch_len + second_batch_len; _PyTok_SourceText source; _PyTok_SourceInit(&source); - for (int i = 0; i < 260; i++) { - if (_PyTok_SourceAppendLine(&source, "x\n", 2) < 0) { + for (int i = 0; i < LINE_COUNT; i++) { + if (_PyTok_SourceAppendLine(&source, first_line, sizeof(first_line) - 1) < 0) { goto error; } } char *bytes = source.bytes; _PyTok_Off capacity = source.cap; _PyTok_SourceDiscard(&source); - if (check(source.base_offset == 520 && source.len == 0 && + if (check(source.base_offset == first_batch_len && source.len == 0 && source.nlines == 0 && source.bytes == bytes && source.cap == capacity && source.bytes[0] == '\0', "discard did not preserve source allocation") < 0) { goto error; } - for (int i = 0; i < 260; i++) { - if (check(_PyTok_SourceAppendLine(&source, "y\n", 2) == 520 + 2 * i, + for (int i = 0; i < LINE_COUNT; i++) { + if (check(_PyTok_SourceAppendLine( + &source, second_line, sizeof(second_line) - 1) == + first_batch_len + ((Py_ssize_t)sizeof(second_line) - 1) * i, "wrong source offset after discard") < 0) { goto error; } @@ -125,18 +144,22 @@ test_tokenizer_source_discard(PyObject *Py_UNUSED(module), goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "tail", 4) == 1040, + if (check(_PyTok_SourceAppendLine( + &source, tail, sizeof(tail) - 1) == discarded_len, "wrong source offset after repeated discard") < 0) { goto error; } _PyTok_SourceDiscard(&source); - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) == 1044, + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) == + discarded_len + (Py_ssize_t)sizeof(tail) - 1, "cannot append after discarding unterminated line") < 0) { goto error; } _PyTok_SourceDiscard(&source); source.base_offset = PY_SSIZE_T_MAX - 1; - if (check(_PyTok_SourceAppendLine(&source, "z\n", 2) < 0 && + if (check(_PyTok_SourceAppendLine( + &source, final_line, sizeof(final_line) - 1) < 0 && PyErr_ExceptionMatches(PyExc_MemoryError), "accepted overflowing logical source offset") < 0) { goto error; diff --git a/Parser/tokenizer/decoder.c b/Parser/tokenizer/decoder.c index d30d5e7a0227c5e..0af8cc31a3b947f 100644 --- a/Parser/tokenizer/decoder.c +++ b/Parser/tokenizer/decoder.c @@ -8,6 +8,14 @@ #include "helpers.h" #include "../lexer/state.h" +#define UNICODE_BOM 0xFEFF +#define UTF16_LE_BOM "\xff\xfe" +#define UTF16_BE_BOM "\xfe\xff" +#define UTF16_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF16_LE_BOM) - 1)) +#define UTF8_BOM "\xef\xbb\xbf" +#define UTF8_BOM_LENGTH ((Py_ssize_t)(sizeof(UTF8_BOM) - 1)) +#define NORMALIZED_LINE_INITIAL_CAPACITY 256 + char * _PyTok_CopyBytes(const char *data, Py_ssize_t len) { @@ -60,9 +68,9 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return -1; } if (strip_bom && PyUnicode_GET_LENGTH(unicode) > 0 && - PyUnicode_ReadChar(unicode, 0) == 0xFEFF) { - utf8 += 3; - utf8_len -= 3; + PyUnicode_ReadChar(unicode, 0) == UNICODE_BOM) { + utf8 += UTF8_BOM_LENGTH; + utf8_len -= UTF8_BOM_LENGTH; } chunk_release_data(chunk); chunk->owner = unicode; @@ -72,8 +80,8 @@ chunk_set_unicode(struct tok_state *tok, _PyTok_Chunk *chunk, return 0; } -// The caller must provide len + 2 bytes: normalization can append a final -// newline and always writes a NUL terminator. +// The caller provides len + add_final_newline + 1 bytes for an optional +// final newline and the NUL terminator. static void normalize_newlines_into(char *result, const char *data, Py_ssize_t len, int preserve_crlf, int add_final_newline, @@ -108,11 +116,11 @@ char * _PyTok_NormalizeNewlines(const char *data, Py_ssize_t len, Py_ssize_t *out_len) { - if (len > PY_SSIZE_T_MAX - 2) { + if (len < 0 || len == PY_SSIZE_T_MAX) { PyErr_NoMemory(); return NULL; } - char *result = PyMem_Malloc((size_t)len + 2); + char *result = PyMem_Malloc((size_t)len + 1); if (result == NULL) { PyErr_NoMemory(); return NULL; @@ -152,11 +160,12 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) return 0; } } - for (; i + 6 < len; i++) { - if (memcmp(line + i, "coding", 6) != 0) { + const Py_ssize_t marker_len = sizeof("coding") - 1; + for (; i < len - marker_len; i++) { + if (memcmp(line + i, "coding", marker_len) != 0) { continue; } - const char *cursor = line + i + 6; + const char *cursor = line + i + marker_len; if (*cursor != ':' && *cursor != '=') { continue; } @@ -174,23 +183,26 @@ find_cookie(const char *line, Py_ssize_t len, char **encoding, int *scan_next) continue; } Py_ssize_t encoding_len = cursor - start; - char normalized[13]; - int n; - for (n = 0; n < 12 && n < encoding_len; n++) { + // Include the longest recognized prefix and its NUL terminator. + char normalized[sizeof("iso-latin-1-")]; + const Py_ssize_t prefix_len = Py_MIN( + encoding_len, (Py_ssize_t)sizeof(normalized) - 1); + Py_ssize_t n; + for (n = 0; n < prefix_len; n++) { normalized[n] = start[n] == '_' ? '-' : Py_TOLOWER(start[n]); } normalized[n] = '\0'; const char *canonical = start; if (strcmp(normalized, "utf-8") == 0 || - strncmp(normalized, "utf-8-", 6) == 0) { + strncmp(normalized, "utf-8-", sizeof("utf-8-") - 1) == 0) { canonical = "utf-8"; } else if (strcmp(normalized, "latin-1") == 0 || strcmp(normalized, "iso-8859-1") == 0 || strcmp(normalized, "iso-latin-1") == 0 || - strncmp(normalized, "latin-1-", 8) == 0 || - strncmp(normalized, "iso-8859-1-", 11) == 0 || - strncmp(normalized, "iso-latin-1-", 12) == 0) { + strncmp(normalized, "latin-1-", sizeof("latin-1-") - 1) == 0 || + strncmp(normalized, "iso-8859-1-", sizeof("iso-8859-1-") - 1) == 0 || + strncmp(normalized, "iso-latin-1-", sizeof("iso-latin-1-") - 1) == 0) { canonical = "iso-8859-1"; } if (canonical != start) { @@ -211,17 +223,15 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, const _PyTok_Chunk *second, int final, Py_ssize_t *bom_len) { - int bom = first->len >= 3 && - (unsigned char)first->data[0] == 0xEF && - (unsigned char)first->data[1] == 0xBB && - (unsigned char)first->data[2] == 0xBF; - *bom_len = bom ? 3 : 0; + int bom = first->len >= UTF8_BOM_LENGTH && + memcmp(first->data, UTF8_BOM, UTF8_BOM_LENGTH) == 0; + *bom_len = bom ? UTF8_BOM_LENGTH : 0; char *cookie = NULL; int scan_next = 0; int cookie_line = 1; - const char *first_data = first->data + (bom ? 3 : 0); - Py_ssize_t first_len = first->len - (bom ? 3 : 0); + const char *first_data = first->data + *bom_len; + Py_ssize_t first_len = first->len - *bom_len; if (find_cookie(first_data, first_len, &cookie, &scan_next) < 0) { return _PYTOK_ENCODING_ERROR; } @@ -246,8 +256,8 @@ _PyTok_DetectEncoding(struct tok_state *tok, const _PyTok_Chunk *first, } if (bom && strcmp(cookie, "utf-8") != 0) { const _PyTok_Chunk *line = cookie_line == 2 ? second : first; - const char *line_data = line->data + (cookie_line == 1 ? 3 : 0); - Py_ssize_t line_len = line->len - (cookie_line == 1 ? 3 : 0); + const char *line_data = line->data + (cookie_line == 1 ? *bom_len : 0); + Py_ssize_t line_len = line->len - (cookie_line == 1 ? *bom_len : 0); int end_col = (int)Py_MIN(line_len, INT_MAX); if (end_col > 0 && (line_data[end_col - 1] == '\n' || line_data[end_col - 1] == '\r')) { @@ -325,14 +335,15 @@ store_prepared_source(struct tok_state *tok, const char *data, Py_ssize_t len, (line[line_len - 1] == '\r' || (line_len > 1 && line[line_len - 2] == '\r'))); if (normalize) { - if (line_len > PY_SSIZE_T_MAX - 2) { + if (line_len > PY_SSIZE_T_MAX - add_newline - 1) { PyErr_NoMemory(); tok->done = E_NOMEM; goto error; } // Reserve space for an optional final '\n' and the NUL terminator. - Py_ssize_t needed = line_len + 2; - if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, 256) < 0) { + Py_ssize_t needed = line_len + add_newline + 1; + if (_PyTok_ReserveBuffer(&normalized, &capacity, needed, + NORMALIZED_LINE_INITIAL_CAPACITY) < 0) { tok->done = E_NOMEM; goto error; } @@ -461,11 +472,9 @@ _PyTok_DecodeChunk(struct tok_state *tok, _PyTok_Chunk *chunk, int final) return 0; } int strip_bom = reader->kind == _PYTOK_READER_READLINE && - chunk->len >= 2 && - (((unsigned char)chunk->data[0] == 0xFF && - (unsigned char)chunk->data[1] == 0xFE) || - ((unsigned char)chunk->data[0] == 0xFE && - (unsigned char)chunk->data[1] == 0xFF)); + chunk->len >= UTF16_BOM_LENGTH && + (memcmp(chunk->data, UTF16_LE_BOM, UTF16_BOM_LENGTH) == 0 || + memcmp(chunk->data, UTF16_BE_BOM, UTF16_BOM_LENGTH) == 0); PyObject *input; if (chunk->ownership == _PYTOK_CHUNK_PYOBJECT && PyBytes_Check(chunk->owner) && diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index a49946eeed25ede..99436e1604af777 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -9,6 +9,8 @@ #include "reader_internal.h" #include "../lexer/state.h" +#define READER_BUFFER_GROWTH_FACTOR 2 + #ifdef HAVE_UNISTD_H # include #endif @@ -67,11 +69,11 @@ _PyTok_ReserveBuffer(char **buffer, Py_ssize_t *capacity, Py_ssize_t needed, // Grow geometrically to avoid reallocating for every longer line. Py_ssize_t cap = *capacity > 0 ? *capacity : initial_capacity; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / READER_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= READER_BUFFER_GROWTH_FACTOR; } char *resized = PyMem_Realloc(*buffer, cap); if (resized == NULL) { diff --git a/Parser/tokenizer/reader_internal.h b/Parser/tokenizer/reader_internal.h index 26e5957b68af70c..f74063de8d653a4 100644 --- a/Parser/tokenizer/reader_internal.h +++ b/Parser/tokenizer/reader_internal.h @@ -3,6 +3,8 @@ #include "Python.h" +#define _PYTOK_ENCODING_COOKIE_MAX_LINES 2 + typedef enum { _PYTOK_READER_PREPARED, _PYTOK_READER_FILE, @@ -46,11 +48,11 @@ typedef struct _PyTok_Reader { char *file_buffer; Py_ssize_t file_buffer_cap; - _PyTok_Chunk prefetched_lines[2]; + _PyTok_Chunk prefetched_lines[_PYTOK_ENCODING_COOKIE_MAX_LINES]; char *decoded; Py_ssize_t decoded_pos; - Py_ssize_t decoded_scan; + Py_ssize_t decoded_scan; // First byte not yet scanned for a newline. Py_ssize_t decoded_len; Py_ssize_t decoded_cap; _PyTok_ReaderKind kind; diff --git a/Parser/tokenizer/source.c b/Parser/tokenizer/source.c index d08e09c597d7c68..ebb7b05507c56ec 100644 --- a/Parser/tokenizer/source.c +++ b/Parser/tokenizer/source.c @@ -1,7 +1,10 @@ #include "Python.h" +#include "pycore_pymem.h" // PYMEM_DEADBYTE #include "source.h" +#define SOURCE_BUFFER_GROWTH_FACTOR 2 + void _PyTok_SourceInit(_PyTok_SourceText *source) { @@ -35,11 +38,11 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } Py_ssize_t cap = source->cap > 0 ? source->cap : BUFSIZ; while (cap < needed) { - if (cap > PY_SSIZE_T_MAX / 2) { + if (cap > PY_SSIZE_T_MAX / SOURCE_BUFFER_GROWTH_FACTOR) { cap = needed; break; } - cap *= 2; + cap *= SOURCE_BUFFER_GROWTH_FACTOR; } char *bytes; #ifdef Py_DEBUG @@ -57,7 +60,7 @@ reserve_bytes(_PyTok_SourceText *source, Py_ssize_t needed) } #ifdef Py_DEBUG if (source->bytes != NULL) { - memset(source->bytes, 0xDD, source->cap); + memset(source->bytes, PYMEM_DEADBYTE, source->cap); PyMem_Free(source->bytes); } #endif From 1cb4394bfcde14f55f2fd5194d99935a0b32ac1e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 19:46:14 +0100 Subject: [PATCH 23/29] Set the tokenizer exception when file input size overflows --- Parser/tokenizer/reader.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Parser/tokenizer/reader.c b/Parser/tokenizer/reader.c index 99436e1604af777..6a0b7615f8df5de 100644 --- a/Parser/tokenizer/reader.c +++ b/Parser/tokenizer/reader.c @@ -192,9 +192,12 @@ read_file_line(struct tok_state *tok, _PyTok_Chunk *chunk) _PyTok_Reader *reader = tok->reader; Py_ssize_t len = 0; for (;;) { - if (len > PY_SSIZE_T_MAX - BUFSIZ || - _PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, - len + BUFSIZ, BUFSIZ) < 0) { + if (len > PY_SSIZE_T_MAX - BUFSIZ) { + PyErr_NoMemory(); + return _PYTOK_READ_ERROR; + } + if (_PyTok_ReserveBuffer(&reader->file_buffer, &reader->file_buffer_cap, + len + BUFSIZ, BUFSIZ) < 0) { return _PYTOK_READ_ERROR; } int available = (int)Py_MIN(reader->file_buffer_cap - len, INT_MAX); From 849c26b025b77d530af1db95c9e3a6f0b383cec2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 20:48:09 +0100 Subject: [PATCH 24/29] gh-153569: Send exact line endings in the REPL test --- Lib/test/test_repl.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index e1ce17b4dd4256c..516d9486c731b30 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -219,10 +219,12 @@ def test_long_non_ascii_input(self): for newline in ("\n", "\r\n"): with self.subTest(newline=newline): p = spawn_repl(encoding="utf-8") - p.stdin.write( + # Send the requested line ending without Windows text-mode + # translation turning CRLF into CRCRLF. + p.stdin.buffer.write(( f"value = {value!r}{newline}" f"print(len(value), value[:2]){newline}" - ) + ).encode("utf-8")) output = kill_python(p) self.assertEqual(p.returncode, 0) self.assertIn(">>> 10000 é漢\n>>> ", output) From 59a2c995cbd57123101b9f22c22ad0544a6aab8e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 23:45:30 +0100 Subject: [PATCH 25/29] Share parser repetition buffer growth --- Parser/parser.c | 845 ++---------------- Parser/pegen.c | 16 + Parser/pegen.h | 2 + .../peg_generator/pegen/c_generator_rules.py | 12 +- 4 files changed, 85 insertions(+), 790 deletions(-) diff --git a/Parser/parser.c b/Parser/parser.c index d8efdd3e3a085cd..09ed5b81eeb9105 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -27298,17 +27298,7 @@ _loop0_1_rule(Parser *p) { _res = newline_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27317,7 +27307,6 @@ _loop0_1_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27369,17 +27358,7 @@ _loop1_2_rule(Parser *p) { _res = statement_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27388,7 +27367,6 @@ _loop1_2_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -27454,17 +27432,7 @@ _loop0_3_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -27473,7 +27441,6 @@ _loop0_3_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28003,17 +27970,7 @@ _loop1_12_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28022,7 +27979,6 @@ _loop1_12_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28088,17 +28044,7 @@ _loop0_13_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28107,7 +28053,6 @@ _loop0_13_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28303,17 +28248,7 @@ _loop0_17_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28322,7 +28257,6 @@ _loop0_17_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28374,17 +28308,7 @@ _loop1_18_rule(Parser *p) { _res = _tmp_158_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28393,7 +28317,6 @@ _loop1_18_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28459,17 +28382,7 @@ _loop0_19_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28478,7 +28391,6 @@ _loop0_19_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28626,17 +28538,7 @@ _loop0_22_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28645,7 +28547,6 @@ _loop0_22_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28738,17 +28639,7 @@ _loop1_24_rule(Parser *p) { _res = _tmp_159_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28757,7 +28648,6 @@ _loop1_24_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28909,17 +28799,7 @@ _loop0_27_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28928,7 +28808,6 @@ _loop0_27_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -28980,17 +28859,7 @@ _loop0_28_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -28999,7 +28868,6 @@ _loop0_28_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29051,17 +28919,7 @@ _loop1_29_rule(Parser *p) { _res = param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29070,7 +28928,6 @@ _loop1_29_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29127,17 +28984,7 @@ _loop1_30_rule(Parser *p) { _res = param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29146,7 +28993,6 @@ _loop1_30_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29203,17 +29049,7 @@ _loop0_31_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29222,7 +29058,6 @@ _loop0_31_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29274,17 +29109,7 @@ _loop1_32_rule(Parser *p) { _res = param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29293,7 +29118,6 @@ _loop1_32_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29359,17 +29183,7 @@ _loop0_33_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29378,7 +29192,6 @@ _loop0_33_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29547,17 +29360,7 @@ _loop1_36_rule(Parser *p) { _res = except_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29566,7 +29369,6 @@ _loop1_36_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29623,17 +29425,7 @@ _loop1_37_rule(Parser *p) { _res = except_star_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29642,7 +29434,6 @@ _loop1_37_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29699,17 +29490,7 @@ _loop1_38_rule(Parser *p) { _res = case_block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29718,7 +29499,6 @@ _loop1_38_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -29784,17 +29564,7 @@ _loop0_39_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -29803,7 +29573,6 @@ _loop0_39_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30114,17 +29883,7 @@ _loop0_44_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30133,7 +29892,6 @@ _loop0_44_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30235,17 +29993,7 @@ _loop0_46_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30254,7 +30002,6 @@ _loop0_46_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30413,17 +30160,7 @@ _loop0_49_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30432,7 +30169,6 @@ _loop0_49_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30534,17 +30270,7 @@ _loop0_51_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30553,7 +30279,6 @@ _loop0_51_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30655,17 +30380,7 @@ _loop0_53_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30674,7 +30389,6 @@ _loop0_53_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30767,17 +30481,7 @@ _loop1_55_rule(Parser *p) { _res = _tmp_16_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30786,7 +30490,6 @@ _loop1_55_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30843,17 +30546,7 @@ _loop1_56_rule(Parser *p) { _res = _tmp_160_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30862,7 +30555,6 @@ _loop1_56_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -30928,17 +30620,7 @@ _loop0_57_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -30947,7 +30629,6 @@ _loop0_57_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31040,17 +30721,7 @@ _loop1_59_rule(Parser *p) { _res = _tmp_161_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31059,7 +30730,6 @@ _loop1_59_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31116,17 +30786,7 @@ _loop1_60_rule(Parser *p) { _res = _tmp_162_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31135,7 +30795,6 @@ _loop1_60_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31192,17 +30851,7 @@ _loop1_61_rule(Parser *p) { _res = compare_op_bitwise_or_pair_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31211,7 +30860,6 @@ _loop1_61_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31320,17 +30968,7 @@ _loop0_63_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31339,7 +30977,6 @@ _loop0_63_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31763,17 +31400,7 @@ _loop0_70_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31782,7 +31409,6 @@ _loop0_70_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31834,17 +31460,7 @@ _loop0_71_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31853,7 +31469,6 @@ _loop0_71_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31905,17 +31520,7 @@ _loop1_72_rule(Parser *p) { _res = lambda_param_no_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -31924,7 +31529,6 @@ _loop1_72_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -31981,17 +31585,7 @@ _loop1_73_rule(Parser *p) { _res = lambda_param_with_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32000,7 +31594,6 @@ _loop1_73_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32057,17 +31650,7 @@ _loop0_74_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32076,7 +31659,6 @@ _loop0_74_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32128,17 +31710,7 @@ _loop1_75_rule(Parser *p) { _res = lambda_param_maybe_default_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32147,7 +31719,6 @@ _loop1_75_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32204,17 +31775,7 @@ _loop0_76_rule(Parser *p) { _res = fstring_format_spec_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32223,7 +31784,6 @@ _loop0_76_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32275,17 +31835,7 @@ _loop0_77_rule(Parser *p) { _res = fstring_middle_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32294,7 +31844,6 @@ _loop0_77_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32346,17 +31895,7 @@ _loop0_78_rule(Parser *p) { _res = tstring_format_spec_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32365,7 +31904,6 @@ _loop0_78_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32417,17 +31955,7 @@ _loop0_79_rule(Parser *p) { _res = tstring_middle_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32436,7 +31964,6 @@ _loop0_79_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32488,17 +32015,7 @@ _loop1_80_rule(Parser *p) { _res = _tmp_154_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32507,7 +32024,6 @@ _loop1_80_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32564,17 +32080,7 @@ _loop1_81_rule(Parser *p) { _res = tstring_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32583,7 +32089,6 @@ _loop1_81_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32649,17 +32154,7 @@ _loop0_82_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32668,7 +32163,6 @@ _loop0_82_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32819,17 +32313,7 @@ _loop0_85_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32838,7 +32322,6 @@ _loop0_85_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -32931,17 +32414,7 @@ _loop1_87_rule(Parser *p) { _res = for_if_clause_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -32950,7 +32423,6 @@ _loop1_87_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33007,17 +32479,7 @@ _loop0_88_rule(Parser *p) { _res = _tmp_164_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33026,7 +32488,6 @@ _loop0_88_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33165,17 +32626,7 @@ _loop0_90_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33184,7 +32635,6 @@ _loop0_90_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33333,17 +32783,7 @@ _loop0_93_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33352,7 +32792,6 @@ _loop0_93_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33500,17 +32939,7 @@ _loop0_96_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33519,7 +32948,6 @@ _loop0_96_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33612,17 +33040,7 @@ _loop0_98_rule(Parser *p) { _res = _tmp_166_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33631,7 +33049,6 @@ _loop0_98_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33808,17 +33225,7 @@ _loop0_101_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33827,7 +33234,6 @@ _loop0_101_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -33929,17 +33335,7 @@ _loop0_103_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -33948,7 +33344,6 @@ _loop0_103_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34150,17 +33545,7 @@ _loop0_107_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34169,7 +33554,6 @@ _loop0_107_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34461,17 +33845,7 @@ _loop1_113_rule(Parser *p) { _res = _tmp_169_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34480,7 +33854,6 @@ _loop1_113_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34920,17 +34293,7 @@ _loop0_119_rule(Parser *p) { _res = star_named_expressions_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -34939,7 +34302,6 @@ _loop0_119_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -34991,17 +34353,7 @@ _loop0_120_rule(Parser *p) { _res = _tmp_157_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35010,7 +34362,6 @@ _loop0_120_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35549,17 +34900,7 @@ _loop0_129_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35568,7 +34909,6 @@ _loop0_129_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -35889,17 +35229,7 @@ _loop0_135_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -35908,7 +35238,6 @@ _loop0_135_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36051,17 +35380,7 @@ _loop0_138_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36070,7 +35389,6 @@ _loop0_138_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36172,17 +35490,7 @@ _loop0_140_rule(Parser *p) return NULL; } if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36191,7 +35499,6 @@ _loop0_140_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -36341,17 +35648,7 @@ _loop0_143_rule(Parser *p) { _res = block_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -36360,7 +35657,6 @@ _loop0_143_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; @@ -37998,17 +37294,7 @@ _loop0_172_rule(Parser *p) { _res = _tmp_177_var; if (_n == _children_capacity) { - if ((size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))) { - PyMem_Free(_children); - p->error_indicator = 1; - PyErr_NoMemory(); - p->level--; - return NULL; - } - Py_ssize_t _new_capacity = _children_capacity == 0 - ? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR; - void **_new_children = PyMem_Realloc( - _children, _new_capacity * sizeof(*_children)); + void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity); if (!_new_children) { PyMem_Free(_children); p->error_indicator = 1; @@ -38017,7 +37303,6 @@ _loop0_172_rule(Parser *p) return NULL; } _children = _new_children; - _children_capacity = _new_capacity; } _children[_n++] = _res; _mark = p->mark; diff --git a/Parser/pegen.c b/Parser/pegen.c index 010de72a356d327..770f1febccc53f7 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -147,6 +147,22 @@ init_normalization(Parser *p) return 1; } +void ** +_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity) +{ + if ((size_t)*capacity > (size_t)PY_SSIZE_T_MAX / + (PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*buffer))) { + return NULL; + } + Py_ssize_t new_capacity = *capacity == 0 + ? 1 : *capacity * PEGEN_ARRAY_GROWTH_FACTOR; + void **new_buffer = PyMem_Realloc(buffer, new_capacity * sizeof(*buffer)); + if (new_buffer != NULL) { + *capacity = new_capacity; + } + return new_buffer; +} + static int growable_comment_array_add(growable_comment_array *arr, int lineno, char *comment) { diff --git a/Parser/pegen.h b/Parser/pegen.h index 2c5332b4b7e8281..1a0a20290dc94d3 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -192,6 +192,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p); expr_ty _PyPegen_fstring_middle_token(Parser* p); Token *_PyPegen_get_last_nonnwhitespace_token(Parser *); int _PyPegen_fill_token(Parser *p); +// Return NULL on failure, leaving the buffer and capacity unchanged. +void **_PyPegen_grow_loop_buffer(void **buffer, Py_ssize_t *capacity); expr_ty _PyPegen_name_token(Parser *p); expr_ty _PyPegen_number_token(Parser *p); void *_PyPegen_string_token(Parser *p); diff --git a/Tools/peg_generator/pegen/c_generator_rules.py b/Tools/peg_generator/pegen/c_generator_rules.py index 86abea9c9a8c3a8..a5a07d02dbc029e 100644 --- a/Tools/peg_generator/pegen/c_generator_rules.py +++ b/Tools/peg_generator/pegen/c_generator_rules.py @@ -56,19 +56,11 @@ def initialize(self) -> None: def append(self, value: str) -> None: self._print("if (_n == _children_capacity) {") with self._indent(): - self._check_memory( - "(size_t)_children_capacity > (size_t)PY_SSIZE_T_MAX / " - "(PEGEN_ARRAY_GROWTH_FACTOR * sizeof(*_children))" + self._print( + "void **_new_children = _PyPegen_grow_loop_buffer(_children, &_children_capacity);" ) - self._print("Py_ssize_t _new_capacity = _children_capacity == 0") - with self._indent(): - self._print("? 1 : _children_capacity * PEGEN_ARRAY_GROWTH_FACTOR;") - self._print("void **_new_children = PyMem_Realloc(") - with self._indent(): - self._print("_children, _new_capacity * sizeof(*_children));") self._check_memory("!_new_children") self._print("_children = _new_children;") - self._print("_children_capacity = _new_capacity;") self._print("}") self._print(f"_children[_n++] = {value};") From 89806d73d277769dad4e417e7af29937e9ce242e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 24 Sep 2026 23:45:30 +0100 Subject: [PATCH 26/29] Fold Unicode input coverage into the existing REPL test --- Lib/test/test_repl.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 516d9486c731b30..3525dee84493acc 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -187,16 +187,19 @@ def read_until(marker, start=0): def test_lexer_buffer_realloc_with_null_start(self): # gh-144759: NULL pointer arithmetic when the lexer buffer grows # while parsing long input. - long_value = "a" * 2000 + long_value = "é漢" * 2000 user_input = dedent(f"""\ x = f'{{{long_value!r}}}' print(x) """) - p = spawn_repl() - p.stdin.write(user_input) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(long_value, output) + for newline in ("\n", "\r\n"): + with self.subTest(newline=newline): + p = spawn_repl(encoding="utf-8") + # Bypass Windows text-mode translation of CRLF to CRCRLF. + p.stdin.buffer.write(user_input.replace("\n", newline).encode("utf-8")) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) @cpython_only def test_multiline_fstring_source_reallocation(self): @@ -214,21 +217,6 @@ def test_multiline_fstring_source_reallocation(self): self.assertEqual(p.returncode, 0) self.assertIn(">>> 3\n>>> ", output) - def test_long_non_ascii_input(self): - value = "é漢" * 5000 - for newline in ("\n", "\r\n"): - with self.subTest(newline=newline): - p = spawn_repl(encoding="utf-8") - # Send the requested line ending without Windows text-mode - # translation turning CRLF into CRCRLF. - p.stdin.buffer.write(( - f"value = {value!r}{newline}" - f"print(len(value), value[:2]){newline}" - ).encode("utf-8")) - output = kill_python(p) - self.assertEqual(p.returncode, 0) - self.assertIn(">>> 10000 é漢\n>>> ", output) - def test_close_stdin(self): user_input = dedent(''' import os From 0eea3f1a4e9bed467870dc38a0877c9a214066e0 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 00:00:03 +0100 Subject: [PATCH 27/29] Keep parser array growth policy private --- Parser/pegen.c | 1 + Parser/pegen.h | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Parser/pegen.c b/Parser/pegen.c index 770f1febccc53f7..35c85d1a5a557a4 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -12,6 +12,7 @@ #define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. #define IDENTIFIER_CACHE_MAX_PROBES 8 +#define PEGEN_ARRAY_GROWTH_FACTOR 2 #define TYPE_IGNORE_INITIAL_CAPACITY 10 struct _identifier_cache_entry { diff --git a/Parser/pegen.h b/Parser/pegen.h index 1a0a20290dc94d3..825cc9197bfabf3 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -29,8 +29,6 @@ #define CURRENT_POS (-5) -#define PEGEN_ARRAY_GROWTH_FACTOR 2 - typedef struct _memo { int type; void *node; From e74e40b47de9ba58309fcbcc2855d1d3ba53ecaa Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 00:00:03 +0100 Subject: [PATCH 28/29] Drop obsolete bitmap stress from source discard test --- Modules/_testinternalcapi/tokenizer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testinternalcapi/tokenizer.c b/Modules/_testinternalcapi/tokenizer.c index 94b5742d3825f6e..0c2bdee2e803b22 100644 --- a/Modules/_testinternalcapi/tokenizer.c +++ b/Modules/_testinternalcapi/tokenizer.c @@ -107,7 +107,7 @@ static PyObject * test_tokenizer_source_discard(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) { - enum { LINE_COUNT = 260 }; + enum { LINE_COUNT = 2 }; const char first_line[] = "x\n"; const char second_line[] = "y\n"; const char tail[] = "tail"; From b6085cd05b1b91aa2bd6ba9b5a2685ada0a8cb88 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Fri, 25 Sep 2026 03:10:53 +0100 Subject: [PATCH 29/29] Preserve input failures while scanning string literals --- Lib/test/test_tokenize.py | 6 ++++++ .../2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst | 1 + Parser/lexer/string.c | 3 +++ 3 files changed, 10 insertions(+) diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 1602489f3d52db2..413fcfc41da6f3e 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -2409,6 +2409,12 @@ def test_multiline_readline_chunk_with_unterminated_tail(self): ) self.assertEqual(readline.call_count, 2) + def test_readline_memory_error_in_string(self): + readline = mock.Mock(side_effect=['"""first\n', MemoryError]) + iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) + with self.assertRaises(MemoryError): + next(iterator) + def test_readline_callback_is_not_read_ahead(self): readline = mock.Mock(side_effect=["x\n", "y\n", ""]) iterator = _tokenize.TokenizerIter(readline, extra_tokens=True) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst index a1b233bc758f951..7bd6b023fb32da5 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-09-24-18-20-00.gh-issue-153569.diagnostics.rst @@ -1,3 +1,4 @@ Correct missing-brace error messages for multiline f-strings and t-strings. Improve tokenization of UTF-8 input split across many readline calls. Preserve source order when reading files with stateful encodings. +Preserve input errors when scanning string literals. diff --git a/Parser/lexer/string.c b/Parser/lexer/string.c index 3c38e1ab7c32cec..b481ee42f7b33b2 100644 --- a/Parser/lexer/string.c +++ b/Parser/lexer/string.c @@ -335,6 +335,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c) break; } if (c == EOF || (quote_size == 1 && c == '\n')) { + if (tok_failed(tok)) { + return MAKE_TOKEN(ERRORTOKEN); + } int end_lineno = tok->lineno; _PyTok_Loc location = tok->start_loc; const char *line = _PyLexer_BufferPointer(tok, tok->start) - location.byte_col;