Skip to content

Commit bbc061f

Browse files
pablogsalpablogsal
committed
gh-153569: keep character scanning inline with source offsets
1 parent 2ff2f74 commit bbc061f

5 files changed

Lines changed: 53 additions & 40 deletions

File tree

‎Lib/test/test_tstring.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -225,6 +225,10 @@ def test_syntax_errors(self):
225225
("t'{x=!}'", "t-string: missing conversion character"),
226226
("t'{x!z}'", "t-string: invalid conversion character 'z': "
227227
"expected 's', 'r', or 'a'"),
228+
("f\"{t'{x!z}'}\"", "t-string: invalid conversion character 'z': "
229+
"expected 's', 'r', or 'a'"),
230+
("t'{f\"{x!z}\"}'", "f-string: invalid conversion character 'z': "
231+
"expected 's', 'r', or 'a'"),
228232
("t'{lambda:1}'", "t-string: lambda expressions are not allowed "
229233
"without parentheses"),
230234
("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"),

‎Makefile.pre.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3463,7 +3463,7 @@ MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.
34633463
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_openssl_mem.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
34643464
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_testcapi/util.h
34653465
MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h
3466-
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
3466+
MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h $(srcdir)/Parser/tokenizer/cursor.h $(srcdir)/Parser/tokenizer/source.h $(srcdir)/Parser/tokenizer/types.h $(srcdir)/Python/ceval.h $(srcdir)/Modules/_testinternalcapi/test_targets.h $(srcdir)/Modules/_testinternalcapi/test_cases.c.h
34673467
MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h
34683468
MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h
34693469

‎Parser/lexer/lexer.c‎

Lines changed: 22 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,44 +23,33 @@ contains_null_bytes(const char* str, size_t size)
2323
return memchr(str, 0, size) != NULL;
2424
}
2525

26-
/* Get next char, updating state; error code goes into tok->done */
2726
int
28-
_PyLexer_nextc(struct tok_state *tok)
27+
_PyLexer_refill(struct tok_state *tok)
2928
{
30-
int rc;
31-
for (;;) {
32-
if (tok->cur != tok->inp) {
33-
if ((unsigned int) tok->col_offset >= (unsigned int) INT_MAX) {
34-
tok->done = E_COLUMNOVERFLOW;
35-
return EOF;
36-
}
37-
tok->col_offset++;
38-
return Py_CHARMASK(tok->source.bytes[tok->cur++ - tok->source.base_offset]); /* Fast path */
39-
}
40-
if (tok->done != E_OK) {
41-
return EOF;
42-
}
43-
rc = _PyTok_ReaderUnderflow(tok);
29+
if (tok->done != E_OK) {
30+
return 0;
31+
}
32+
int rc = _PyTok_ReaderUnderflow(tok);
4433
#if defined(Py_DEBUG)
45-
if (tok->debug) {
46-
fprintf(stderr, "line[%d] = ", tok->lineno);
47-
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur), tok->inp - tok->cur);
48-
fprintf(stderr, " tok->done = %d\n", tok->done);
49-
}
34+
if (tok->debug) {
35+
fprintf(stderr, "line[%d] = ", tok->lineno);
36+
_PyTokenizer_print_escape(stderr, _PyLexer_BufferPointer(tok, tok->cur),
37+
tok->inp - tok->cur);
38+
fprintf(stderr, " tok->done = %d\n", tok->done);
39+
}
5040
#endif
51-
if (!rc) {
52-
tok->cur = tok->inp;
53-
return EOF;
54-
}
55-
tok->line_start = tok->cur;
56-
57-
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start), tok->inp - tok->line_start)) {
58-
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
59-
tok->cur = tok->inp;
60-
return EOF;
61-
}
41+
if (!rc) {
42+
tok->cur = tok->inp;
43+
return 0;
6244
}
63-
Py_UNREACHABLE();
45+
tok->line_start = tok->cur;
46+
if (contains_null_bytes(_PyLexer_BufferPointer(tok, tok->line_start),
47+
tok->inp - tok->line_start)) {
48+
_PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes");
49+
tok->cur = tok->inp;
50+
return 0;
51+
}
52+
return 1;
6453
}
6554

6655
/* Back-up one character */

‎Parser/lexer/lexer_internal.h‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#ifndef _PY_LEXER_INTERNAL_H_
22
#define _PY_LEXER_INTERNAL_H_
33

4+
#include "errcode.h"
45
#include "lexer.h"
56

67
#define is_potential_identifier_start(c) (\
@@ -41,10 +42,29 @@ TOK_NEXT_MODE(struct tok_state *tok)
4142
#define FTSTRING_END(tok_mode) ((tok_mode)->string_kind == TSTRING ? TSTRING_END : FSTRING_END)
4243
#define TOK_GET_STRING_PREFIX(tok) (TOK_GET_MODE(tok)->string_kind == TSTRING ? 't' : 'f')
4344

44-
#define tok_nextc _PyLexer_nextc
4545
#define tok_backup _PyLexer_backup
4646

47-
int _PyLexer_nextc(struct tok_state *);
47+
int _PyLexer_refill(struct tok_state *);
48+
49+
static inline int
50+
tok_nextc(struct tok_state *tok)
51+
{
52+
while (tok->cur == tok->inp) {
53+
if (!_PyLexer_refill(tok)) {
54+
return EOF;
55+
}
56+
}
57+
assert(tok->cur >= tok->source.base_offset);
58+
assert(tok->cur - tok->source.base_offset < tok->source.len);
59+
if ((unsigned int)tok->col_offset >= (unsigned int)INT_MAX) {
60+
tok->done = E_COLUMNOVERFLOW;
61+
return EOF;
62+
}
63+
tok->col_offset++;
64+
return Py_CHARMASK(
65+
tok->source.bytes[tok->cur++ - tok->source.base_offset]);
66+
}
67+
4868
void _PyLexer_backup(struct tok_state *, int);
4969
int _PyLexer_set_ftstring_expr(struct tok_state *, struct token *, char);
5070
int _PyLexer_check_string_prefixes(struct tok_state *, int, int, int, int, int);

‎Parser/lexer/state.h‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,15 @@ typedef struct _tokenizer_mode {
5959

6060
/* Tokenizer state */
6161
struct tok_state {
62+
_PyTok_Off buf_offset;
6263
_PyTok_Off cur;
6364
_PyTok_Off inp;
64-
_PyTok_Off buf_offset;
65+
_PyTok_Off start;
66+
_PyTok_Off line_start;
67+
_PyTok_SourceText source;
6568
int fp_interactive; /* If the file descriptor is interactive */
6669
char *interactive_src_start; /* The start of the source parsed so far in interactive mode */
6770
char *interactive_src_end; /* The end of the source parsed so far in interactive mode */
68-
_PyTok_Off start;
6971
int done; /* E_OK normally, E_EOF at EOF, otherwise error code */
7072
/* NB If done != E_OK, cur must be == inp!!! */
7173
FILE *fp; /* Rest of input; NULL if tokenizing a string */
@@ -92,11 +94,9 @@ struct tok_state {
9294
/* Stuff for PEP 0263 */
9395
int input_error;
9496
char *encoding; /* Source encoding. */
95-
_PyTok_Off line_start;
9697
_PyTok_Off multi_line_start;
9798
char* str; /* Source string being tokenized (if tokenizing from a string)*/
9899

99-
_PyTok_SourceText source;
100100
struct _PyTok_Reader *reader;
101101

102102
int type_comments; /* Whether to look for type comments */

0 commit comments

Comments
 (0)