gh-153569: Remove redundant tokenizer views and simplify reader branching

Author: pablogsal

Parser/lexer/string.c

@@ -79,7 +79,8 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
     tokenizer_comments *comments = state->comments;
     PyObject *res;
     if (comments != NULL && comments->count > 0) {
-        Py_ssize_t stripped_size = state->expr_span.end - state->expr_span.start;
+        Py_ssize_t stripped_size =
+            state->expr_span.end - state->expr_span.start;
         Py_ssize_t comment_count = 0;
         for (Py_ssize_t i = 0; i < comments->count; i++) {
             _PyTok_Span comment = comments->spans[i];
@@ -98,22 +99,22 @@ finish_ftstring_expr(struct tok_state *tok, ftstring_state *state,
             PyErr_NoMemory();
             return -1;
         }
-        _PyTok_Off copied_to = state->expr_span.start;
+        _PyTok_Span kept = {state->expr_span.start, state->expr_span.start};
         Py_ssize_t stripped_len = 0;
-        for (Py_ssize_t i = 0; i <= comment_count; i++) {
-            _PyTok_Span span = {
-                copied_to,
-                i < comment_count ? comments->spans[i].start : state->expr_span.end,
-            };
+        for (Py_ssize_t i = 0; i < comment_count; i++) {
+            kept.end = comments->spans[i].start;
             Py_ssize_t length;
-            const char *text = _PyTok_SourceSpanView(&tok->source, span, &length);
+            const char *text = _PyTok_SourceSpanView(
+                &tok->source, kept, &length);
             memcpy(stripped + stripped_len, text, (size_t)length);
             stripped_len += length;
-            if (i < comment_count) {
-                copied_to = comments->spans[i].end;
-            }
+            kept.start = comments->spans[i].end;
         }
-        res = PyUnicode_DecodeUTF8(stripped, stripped_len, NULL);
+        kept.end = state->expr_span.end;
+        Py_ssize_t length;
+        const char *text = _PyTok_SourceSpanView(&tok->source, kept, &length);
+        memcpy(stripped + stripped_len, text, (size_t)length);
+        res = PyUnicode_DecodeUTF8(stripped, stripped_size, NULL);
         PyMem_Free(stripped);
     }
     else {

Parser/tokenizer/api.c

@@ -68,7 +68,6 @@ _PyToken_GetView(const struct tok_state *tok, const struct token *token,
             ? token->span.start - token->start_loc.byte_col : tok->line_start,
         tok->inp,
     };
-    view->line = _PyTok_SourcePointer(&tok->source, view->line_span.start);
     view->implicit_newline = tok->implicit_newline;
     view->at_eof = tok->done == E_EOF;
 }

Parser/tokenizer/reader.c

@@ -177,7 +177,8 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
     _PyTok_Span tail = {tok->inp, tok->source.base_offset + tok->source.len};
     _PyTok_Off newline = _PyTok_SourceFindByte(&tok->source, tail, '\n');
     _PyTok_Span line = {tail.start, newline >= 0 ? newline + 1 : tail.end};
-    chunk->data = (char *)_PyTok_SourceSpanView(&tok->source, line, &chunk->len);
+    chunk->data = (char *)_PyTok_SourceSpanView(
+        &tok->source, line, &chunk->len);
     chunk->ownership = _PYTOK_CHUNK_BORROWED;
     chunk->implicit_newline = line.end == tail.end &&
         tok->reader->prepared_final_newline_is_implicit;
@@ -607,7 +608,7 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
     _PyTok_ReaderKind kind = reader->kind;
     int prepared = kind == _PYTOK_READER_PREPARED;
     int streaming = reader_is_streaming(kind);
-    int reset_buffer = !prepared && tok->start < 0 &&
+    int reset_buffer = tok->start < 0 &&
         _PyLexer_CurrentFTString(tok) == NULL;
 
     _PyTok_Chunk chunk;
@@ -660,12 +661,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
             tok->cur = source_start;
             tok->buf_offset = source_start;
             tok->line_start = tok->buf_offset;
-            tok->start = -1;
         }
         tok->inp = source_start + chunk.len;
     }
     else {
-        if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
+        if (reset_buffer) {
             tok->buf_offset = tok->inp;
         }
         tok->inp += chunk.len;
@@ -674,9 +674,11 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
 
     tok->lineno++;
     if (kind == _PYTOK_READER_FILE &&
-            (reader->encoding == NULL || strcmp(reader->encoding, "utf-8") == 0) &&
+            (reader->encoding == NULL ||
+             strcmp(reader->encoding, "utf-8") == 0) &&
             !_PyTokenizer_ensure_utf8(
-                _PyTok_SourcePointer(&tok->source, tok->cur), tok, tok->lineno)) {
+                _PyTok_SourcePointer(&tok->source, tok->cur),
+                tok, tok->lineno)) {
         _PyTok_ChunkClear(&chunk);
         return 0;
     }

Parser/tokenizer/source.h

@@ -43,14 +43,15 @@ _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
 {
     assert(length != NULL);
     assert(_PyTok_SpanIsValid(span));
+    assert(span.end - source->base_offset <= source->len);
     *length = span.end - span.start;
-    (void)_PyTok_SourcePointer(source, span.end);
     return _PyTok_SourcePointer(source, span.start);
 }
 
 /* Return the first matching offset within span, or -1 if absent. */
 static inline _PyTok_Off
-_PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span, int byte)
+_PyTok_SourceFindByte(const _PyTok_SourceText *source, _PyTok_Span span,
+                      int byte)
 {
     Py_ssize_t length;
     const char *data = _PyTok_SourceSpanView(source, span, &length);

Parser/tokenizer/tokenizer.h

@@ -21,7 +21,6 @@ struct token {
 typedef struct {
     const char *text;
     Py_ssize_t length;
-    const char *line;
     _PyTok_Span line_span;
     _PyTok_Off end_line_start;
     int implicit_newline;
@@ -74,8 +73,8 @@ _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
 const char *_PyToken_TextView(
     const struct tok_state *, const struct token *, Py_ssize_t *);
 /* Use the token from the most recent Get. An absent span has nonnull empty text.
-   line contains the bytes of line_span, the token's complete physical line
-   range. end_line_start is the logical offset of its final physical line. */
+   line_span covers the token's complete physical line range. end_line_start
+   is the logical offset of its final physical line. */
 void _PyToken_GetView(
     const struct tok_state *tok, const struct token *token,
     _PyToken_View *view);

Python/Python-tokenize.c

@@ -158,12 +158,16 @@ _tokenizer_error(tokenizeriterobject *it)
 
 static PyObject *
 _get_current_line(tokenizeriterobject *it, int current_lineno,
-                  const char *line_start, Py_ssize_t size, int *line_changed)
+                  const _PyToken_View *view, int *line_changed)
 {
     _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(it);
     if (current_lineno != it->last_lineno) {
-        // Line has changed since last token, so we fetch the new line and cache it
-        // in the iter object.
+        Py_ssize_t size;
+        const char *line_start = _PyTokenizer_SpanView(
+            it->tok, view->line_span, &size);
+        if (size > 0 && view->implicit_newline) {
+            size--;
+        }
         Py_XDECREF(it->last_line);
         it->last_line = PyUnicode_DecodeUTF8(line_start, size, "replace");
         it->byte_col_offset_diff = 0;
@@ -263,13 +267,8 @@ tokenizeriter_next(PyObject *op)
     if (it->extra_tokens && is_trailing_token) {
         line = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     } else {
-        Py_ssize_t size = view.line_span.end - view.line_span.start;
-        if (size >= 1 && view.implicit_newline) {
-            size -= 1;
-        }
-
         line = _get_current_line(
-            it, token.end_loc.lineno, view.line, size, &line_changed);
+            it, token.end_loc.lineno, &view, &line_changed);
     }
     if (line == NULL) {
         Py_DECREF(str);