gh-153569: consolidate tokenizer reader ownership and source views

Lib/test/test_repl.py

@@ -184,9 +184,8 @@ def read_until(marker, start=0):
 
     @cpython_only
     def test_lexer_buffer_realloc_with_null_start(self):
-        # gh-144759: NULL pointer arithmetic in the lexer when start and
-        # multi_line_start are NULL (uninitialized in tok_mode_stack[0])
-        # and the lexer buffer is reallocated while parsing long input.
+        # gh-144759: NULL pointer arithmetic when the lexer buffer grows
+        # while parsing long input.
         long_value = "a" * 2000
         user_input = dedent(f"""\
         x = f'{{{long_value!r}}}'

Makefile.pre.in

@@ -394,7 +394,6 @@ PEGEN_OBJS=		\
 		Parser/peg_api.o
 
 TOKENIZER_OBJS=		\
-		Parser/lexer/buffer.o \
 		Parser/lexer/lexer.o \
 		Parser/lexer/number.o \
 		Parser/lexer/state.o \
@@ -411,7 +410,6 @@ PEGEN_HEADERS= \
 		$(srcdir)/Parser/string_parser.h
 
 TOKENIZER_HEADERS= \
-		Parser/lexer/buffer.h \
 		Parser/lexer/lexer.h \
 		Parser/lexer/lexer_internal.h \
 		Parser/lexer/state.h \

Modules/_testinternalcapi/tokenizer.c

@@ -23,6 +23,17 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
+static int
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                const char *expected)
+{
+    Py_ssize_t len;
+    const char *line = _PyTok_SourceLineView(source, lineno, &len);
+    return check(len == (Py_ssize_t)strlen(expected) &&
+                 memcmp(line, expected, len) == 0,
+                 "wrong source line view");
+}
+
 static int
 same_cursor(const _PyTok_Cursor *left, const _PyTok_Cursor *right)
 {
@@ -40,6 +51,10 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
+    if (check_line_view(&source, 1, "") < 0) {
+        goto error;
+    }
+
     _PyTok_Loc loc;
     _PyTok_Line line;
     if (check(_PyTok_SourceLocation(
@@ -67,10 +82,20 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
               "wrong first source offset") < 0 ||
             check(_PyTok_SourceAppendLine(
                       &source, "\xce\xb2\n", 3, 1) == 6,
-                  "wrong second source offset") < 0 ||
-            check(_PyTok_SourceAppendLine(
-                      &source, "nul\0x\n", 6, 0) == 9,
-                  "wrong third source offset") < 0) {
+                  "wrong second source offset") < 0) {
+        goto error;
+    }
+
+    if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+            check_line_view(&source, 1, "alpha") < 0 ||
+            check_line_view(&source, 2, "\xce\xb2") < 0 ||
+            check_line_view(&source, 3, "") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
+        goto error;
+    }
+
+    if (check(_PyTok_SourceAppendLine(&source, "nul\0x\n", 6, 0) == 9,
+              "wrong third source offset") < 0) {
         goto error;
     }
 
@@ -195,6 +220,11 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    if (check_line_view(&source, 1, "tail") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
+        goto error;
+    }
+
     _PyTok_SourceDiscard(&source);
     if (check(_PyTok_SourceAppendLine(&source, "a\n", 2, 0) == 4,
               "wrong retained source offset") < 0 ||

PCbuild/_freeze_module.vcxproj

@@ -181,7 +181,6 @@
     <ClCompile Include="..\Parser\action_helpers.c" />
     <ClCompile Include="..\Parser\string_parser.c" />
     <ClCompile Include="..\Parser\token.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\lexer\state.c" />
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -469,9 +469,6 @@
     <ClCompile Include="..\Parser\lexer\string.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -423,10 +423,9 @@
     <ClInclude Include="..\Parser\lexer\state.h" />
     <ClInclude Include="..\Parser\lexer\lexer.h" />
     <ClInclude Include="..\Parser\lexer\lexer_internal.h" />
-    <ClInclude Include="..\Parser\lexer\buffer.h" />
-    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\reader.h" />
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h" />
+    <ClInclude Include="..\Parser\tokenizer\cursor.h" />
     <ClInclude Include="..\Parser\tokenizer\source.h" />
     <ClInclude Include="..\Parser\tokenizer\helpers.h" />
     <ClInclude Include="..\Parser\tokenizer\tokenizer.h" />
@@ -593,7 +592,6 @@
     <ClCompile Include="..\Parser\lexer\lexer.c" />
     <ClCompile Include="..\Parser\lexer\number.c" />
     <ClCompile Include="..\Parser\lexer\string.c" />
-    <ClCompile Include="..\Parser\lexer\buffer.c" />
     <ClCompile Include="..\Parser\tokenizer\cursor.c" />
     <ClCompile Include="..\Parser\tokenizer\source.c" />
     <ClCompile Include="..\Parser\tokenizer\decoder.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -330,18 +330,15 @@
     <ClInclude Include="..\Parser\lexer\lexer_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
-    <ClInclude Include="..\Parser\lexer\buffer.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
-    <ClInclude Include="..\Parser\tokenizer\cursor.h">
-      <Filter>Parser</Filter>
-    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader.h">
       <Filter>Parser</Filter>
     </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\reader_internal.h">
       <Filter>Parser</Filter>
     </ClInclude>
+    <ClInclude Include="..\Parser\tokenizer\cursor.h">
+      <Filter>Parser</Filter>
+    </ClInclude>
     <ClInclude Include="..\Parser\tokenizer\source.h">
       <Filter>Parser</Filter>
     </ClInclude>
@@ -1361,9 +1358,6 @@
     <ClCompile Include="..\Parser\lexer\state.c">
       <Filter>Parser</Filter>
     </ClCompile>
-    <ClCompile Include="..\Parser\lexer\buffer.c">
-      <Filter>Parser</Filter>
-    </ClCompile>
     <ClCompile Include="..\Parser\tokenizer\cursor.c">
       <Filter>Parser</Filter>
     </ClCompile>

Parser/lexer/lexer.c

@@ -204,15 +204,16 @@ _PyLexer_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, str
         }
         tok_backup(tok, c);
         if (c == '#' || c == '\n' || c == '\r') {
+            int interactive = _PyTok_ReaderIsInteractive(tok);
             /* Lines with only whitespace and/or comments
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
                mode, which signal the end of a command group. */
-            if (col == 0 && c == '\n' && tok->prompt != NULL) {
+            if (col == 0 && c == '\n' && interactive) {
                 blankline = 0; /* Let it through */
             }
-            else if (tok->prompt != NULL && tok->lineno == 1) {
+            else if (interactive && tok->lineno == 1) {
                 /* In interactive mode, if the first line contains
                    only spaces and/or a comment, let it through. */
                 blankline = 0;

Parser/lexer/state.c

@@ -1,59 +1,10 @@
 #include "Python.h"
-#include "pycore_pystate.h"
 #include "pycore_token.h"
 #include "errcode.h"
 
 #include "state.h"
 #include "../tokenizer/reader.h"
 
-/* Create and initialize a new tok_state structure */
-struct tok_state *
-_PyTokenizer_tok_new(void)
-{
-    struct tok_state *tok = (struct tok_state *)PyMem_Calloc(
-                                            1,
-                                            sizeof(struct tok_state));
-    if (tok == NULL) {
-        PyErr_NoMemory();
-        return NULL;
-    }
-
-    tok->buf = tok->cur = tok->inp = NULL;
-    tok->fp_interactive = 0;
-    tok->interactive_src_start = NULL;
-    tok->interactive_src_end = NULL;
-    tok->start = NULL;
-    tok->done = E_OK;
-    tok->fp = NULL;
-    tok->indent = 0;
-    tok->indstack[0] = 0;
-    tok->atbol = 1;
-    tok->pendin = 0;
-    tok->prompt = NULL;
-    tok->lineno = 0;
-    tok->start_loc = (_PyTok_Loc){-1, -1};
-    tok->level = 0;
-    tok->altindstack[0] = 0;
-    tok->encoding = NULL;
-    tok->filename = NULL;
-    tok->module = NULL;
-    tok->type_comments = 0;
-    tok->interactive_underflow = IUNDERFLOW_NORMAL;
-    tok->str = NULL;
-    tok->report_warnings = 1;
-    tok->tok_extra_tokens = 0;
-    tok->comment_newline = 0;
-    tok->implicit_newline = 0;
-    _PyTok_SourceInit(&tok->source);
-    tok->reader = NULL;
-    tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .quote='\0', .quote_size = 0, .in_debug=0};
-    tok->tok_mode_stack_index = 0;
-#ifdef Py_DEBUG
-    tok->debug = _Py_GetConfig()->parser_debug;
-#endif
-    return tok;
-}
-
 /* Free a tok_state structure */
 void
 _PyTokenizer_Free(struct tok_state *tok)

Parser/lexer/state.h

@@ -13,14 +13,6 @@
 #define INSIDE_FSTRING_EXPR_AT_TOP(tok) \
     (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1)
 
-enum interactive_underflow_t {
-    /* Normal mode of operation: return a new token when asked in interactive mode */
-    IUNDERFLOW_NORMAL,
-    /* Forcefully return ENDMARKER when asked for a new token in interactive mode. This
-     * can be used to prevent the tokenizer to prompt the user for new tokens */
-    IUNDERFLOW_STOP,
-};
-
 struct token {
     int level;
     _PyTok_Span span;
@@ -76,9 +68,6 @@ struct tok_state {
     char *cur;          /* Next character in buffer */
     char *inp;          /* End of data in buffer */
     _PyTok_Off buf_offset; /* Logical offset of buf[0]. */
-    int fp_interactive; /* If the file descriptor is interactive */
-    char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
-    char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
     const char *start;  /* Start of current token if not NULL */
     int done;           /* E_OK normally, E_EOF at EOF, otherwise error code */
     /* NB If done != E_OK, cur must be == inp!!! */
@@ -87,7 +76,6 @@ struct tok_state {
     int indstack[MAXINDENT];            /* Stack of indents */
     int atbol;          /* Nonzero if at begin of new line */
     int pendin;         /* Pending indents (if > 0) or dedents (if < 0) */
-    const char *prompt;          /* For interactive prompting */
     int lineno;         /* Current line number */
     _PyTok_Loc start_loc;
     int level;          /* () [] {} Parentheses nesting level */
@@ -102,17 +90,11 @@ struct tok_state {
     /* Stuff for PEP 0263 */
     char *encoding;         /* Source encoding. */
     const char* line_start;     /* pointer to start of current line */
-    char* str;          /* Source string being tokenized (if tokenizing from a string)*/
-
     _PyTok_SourceText source;
     struct _PyTok_Reader *reader;
 
     int type_comments;      /* Whether to look for type comments */
 
-    /* How to proceed when asked for a new token in interactive mode */
-    enum interactive_underflow_t interactive_underflow;
-    int report_warnings;
-    // TODO: Factor this into its own thing
     tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL];
     int tok_mode_stack_index;
     int tok_extra_tokens;
@@ -182,7 +164,6 @@ _PyLexer_BufferSpan(const struct tok_state *tok, const char *start,
 
 int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end);
 
-struct tok_state *_PyTokenizer_tok_new(void);
 void _PyTokenizer_Free(struct tok_state *);
 void _PyToken_Free(struct token *);
 void _PyToken_Init(struct token *);