gh-153569: read diagnostic lines through the source API

Author: pablogsal

Modules/_testinternalcapi/tokenizer.c

@@ -23,13 +23,28 @@ check_system_error(int failed, const char *message)
     return 0;
 }
 
+static int
+check_line_view(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                const char *expected)
+{
+    Py_ssize_t len;
+    const char *line = _PyTok_SourceLineView(source, lineno, &len);
+    return check(len == (Py_ssize_t)strlen(expected) &&
+                 memcmp(line, expected, len) == 0,
+                 "wrong source line view");
+}
+
 static PyObject *
 test_tokenizer_source(PyObject *Py_UNUSED(module),
                       PyObject *Py_UNUSED(args))
 {
     _PyTok_SourceText source;
     _PyTok_SourceInit(&source);
 
+    if (check_line_view(&source, 1, "") < 0) {
+        goto error;
+    }
+
     if (check_system_error(
             _PyTok_SourceAppendLine(&source, "", 0, 0) < 0,
             "accepted empty source line") < 0 ||
@@ -51,6 +66,14 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
         goto error;
     }
 
+    if (check_line_view(&source, PY_SSIZE_T_MIN, "alpha") < 0 ||
+            check_line_view(&source, 1, "alpha") < 0 ||
+            check_line_view(&source, 2, "\xce\xb2") < 0 ||
+            check_line_view(&source, 3, "") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "") < 0) {
+        goto error;
+    }
+
     if (check(source.len == 9 &&
                   memcmp(source.bytes, "alpha\n\xce\xb2\n", 10) == 0,
               "wrong source contents") < 0) {
@@ -64,6 +87,10 @@ test_tokenizer_source(PyObject *Py_UNUSED(module),
                 "appended after unterminated source line") < 0) {
         goto error;
     }
+    if (check_line_view(&source, 1, "tail") < 0 ||
+            check_line_view(&source, PY_SSIZE_T_MAX, "tail") < 0) {
+        goto error;
+    }
 
     _PyTok_SourceClear(&source);
     Py_RETURN_NONE;

Parser/pegen_errors.c

@@ -226,22 +226,10 @@ _PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *err
 static PyObject *
 get_error_line_from_source(Parser *p, Py_ssize_t lineno)
 {
-    const char *cur_line = _PyTok_SourceData(&p->tok->source);
-
     Py_ssize_t relative_lineno = p->starting_lineno ? lineno - p->starting_lineno + 1 : lineno;
-    const char *buf_end = cur_line + p->tok->source.len;
-
-    for (int i = 0; i < relative_lineno - 1; i++) {
-        const char *new_line = memchr(cur_line, '\n', buf_end - cur_line);
-        if (new_line == NULL) {
-            break;
-        }
-        cur_line = new_line + 1;
-    }
-
-    const char *next_newline = memchr(cur_line, '\n', buf_end - cur_line);
-    next_newline = next_newline != NULL ? next_newline : buf_end;
-    return PyUnicode_DecodeUTF8(cur_line, next_newline - cur_line, "replace");
+    Py_ssize_t len;
+    const char *line = _PyTok_SourceLineView(&p->tok->source, relative_lineno, &len);
+    return PyUnicode_DecodeUTF8(line, len, "replace");
 }
 
 void *

Parser/tokenizer/source.c

@@ -134,6 +134,26 @@ _PyTok_SourceAppendLine(_PyTok_SourceText *source, const char *bytes,
     return start;
 }
 
+const char *
+_PyTok_SourceLineView(const _PyTok_SourceText *source, Py_ssize_t lineno,
+                      Py_ssize_t *len)
+{
+    assert(len != NULL);
+    const char *line = _PyTok_SourceData(source);
+    const char *end = line + source->len;
+    while (lineno > 1) {
+        const char *newline = memchr(line, '\n', end - line);
+        if (newline == NULL) {
+            break;
+        }
+        line = newline + 1;
+        lineno--;
+    }
+    const char *newline = memchr(line, '\n', end - line);
+    *len = (newline != NULL ? newline : end) - line;
+    return line;
+}
+
 int
 _PyTok_SourceLineIsImplicit(const _PyTok_SourceText *source, int lineno)
 {

Parser/tokenizer/source.h

@@ -43,6 +43,12 @@ PyAPI_FUNC(void) _PyTok_SourceClear(_PyTok_SourceText *);
 PyAPI_FUNC(_PyTok_Off) _PyTok_SourceAppendLine(
     _PyTok_SourceText *source, const char *bytes, Py_ssize_t len,
     int implicit_newline);
+/* Return borrowed bytes excluding '\n', writing the byte length to *len.
+   Line numbers are 1-based and clamp to the first or final line; a trailing
+   '\n' adds an empty final line. The view need not be NUL-terminated.
+   This does not set an exception. Append and clear invalidate the view. */
+PyAPI_FUNC(const char *) _PyTok_SourceLineView(
+    const _PyTok_SourceText *source, Py_ssize_t lineno, Py_ssize_t *len);
 /* Return false for invalid line numbers and the virtual EOF line. */
 PyAPI_FUNC(int) _PyTok_SourceLineIsImplicit(
     const _PyTok_SourceText *, int);