gh-153569: assert tokenizer state invariants

Check formatted-string stack transitions, decoder cursor bounds, and input chunks at the points where the tokenizer relies on them. These checks stay debug-only and make state corruption fail close to its source.

Author: pablogsal

Parser/lexer/state.c

@@ -9,6 +9,7 @@
 ftstring_state *
 _PyLexer_PushFTString(struct tok_state *tok)
 {
+    assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity);
     int next_depth = tok->ftstring_depth + 1;
     if (next_depth >= MAXFTSTRINGLEVEL) {
         _PyTokenizer_syntaxerror(

Parser/lexer/state.h

@@ -115,7 +115,7 @@ static inline ftstring_state *
 _PyLexer_CurrentFTString(struct tok_state *tok)
 {
     assert(tok->ftstring_stack != NULL);
-    assert(tok->ftstring_depth <= tok->ftstring_capacity);
+    assert(tok->ftstring_depth >= 0 && tok->ftstring_depth <= tok->ftstring_capacity);
     if (tok->ftstring_depth == 0) {
         return NULL;
     }

Parser/lexer/string.c

@@ -20,6 +20,7 @@ int
 _PyLexer_record_ftstring_comment(struct tok_state *tok, ftstring_state *state,
                                  const char *start, const char *end)
 {
+    assert(state == _PyLexer_CurrentFTString(tok) && state->mode == FTSTRING_MODE_EXPRESSION);
     if (state->expr_span.end >= 0) {
         return 0;
     }
@@ -57,7 +58,8 @@ int
 _PyLexer_finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
                               struct token *token)
 {
-    assert(token != NULL);
+    assert(token != NULL && state == _PyLexer_CurrentFTString(tok));
+    assert(state->mode == FTSTRING_MODE_EXPRESSION && tok->start != NULL);
 
     if (state->expr_span.end >= 0) {
         return 0;
@@ -338,6 +340,9 @@ _PyLexer_scan_string(struct tok_state *tok, struct token *token, int c)
 int
 _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct token *token)
 {
+    assert(current == _PyLexer_CurrentFTString(tok) && current->mode != FTSTRING_MODE_EXPRESSION);
+    assert((current->quote_size == 1 || current->quote_size == 3) &&
+           current->replacement_depth <= MAX_EXPR_NESTING);
     const char *p_start = NULL;
     const char *p_end = NULL;
     int end_quote_size = 0;

Parser/tokenizer/reader.c

@@ -169,6 +169,7 @@ append_implicit_newline(_PyTok_Reader *reader)
 static int
 pop_decoded_line(_PyTok_Reader *reader, _PyTok_Chunk *chunk)
 {
+    assert(reader->decoded_pos >= 0 && reader->decoded_pos <= reader->decoded_len);
     if (reader->decoded_pos == reader->decoded_len) {
         return 0;
     }
@@ -627,6 +628,8 @@ reset_streaming_buffer(struct tok_state *tok)
 int
 _PyTok_ReaderUnderflow(struct tok_state *tok)
 {
+    assert(tok->cur == tok->inp || (tok->buf != NULL &&
+           tok->cur >= tok->buf && tok->cur < tok->inp));
     _PyTok_ReaderKind kind = tok->reader->kind;
     int prepared = kind == _PYTOK_READER_PREPARED;
     int streaming = reader_is_streaming(kind);
@@ -660,6 +663,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         return 0;
     }
+    assert(chunk.data != NULL && chunk.len > 0);
     if (tok->lineno == INT_MAX) {
         PyErr_SetString(PyExc_OverflowError, "too many tokenizer source lines");
         tok->done = E_ERROR;