gh-153569: Consolidate tokenizer source access and token views

Author: pablogsal

Parser/lexer/lexer_internal.h

@@ -40,14 +40,11 @@ tok_nextc(struct tok_state *tok)
             return EOF;
         }
     }
-    assert(tok->cur >= tok->source.base_offset);
-    assert(tok->cur - tok->source.base_offset < tok->source.len);
     if (tok->cur - tok->line_start >= INT_MAX) {
         tok->done = E_COLUMNOVERFLOW;
         return EOF;
     }
-    return Py_CHARMASK(
-        tok->source.bytes[tok->cur++ - tok->source.base_offset]);
+    return _PyTok_SourceByte(&tok->source, tok->cur++);
 }
 
 /* Return -1 on error, otherwise whether the line is blank. */

Parser/tokenizer/api.c

@@ -61,14 +61,7 @@ _PyToken_GetView(const struct tok_state *tok, const struct token *token,
                  _PyToken_View *view)
 {
     assert(view != NULL);
-    assert((token->span.start == -1 && token->span.end == -1) ||
-           _PyTok_SpanIsValid(token->span));
-    if (token->span.start >= 0) {
-        (void)_PyTok_SourcePointer(&tok->source, token->span.end);
-    }
-    view->text = token->span.start < 0
-        ? NULL : _PyTok_SourcePointer(&tok->source, token->span.start);
-    view->length = token->span.end - token->span.start;
+    view->text = _PyToken_TextView(tok, token, &view->length);
     view->end_line_start = tok->line_start;
     view->line_span = (_PyTok_Span){
         ISSTRINGLIT(token->type)

Parser/tokenizer/reader.c

@@ -174,15 +174,14 @@ next_prepared(struct tok_state *tok, _PyTok_Chunk *chunk)
     if (tok->lineno >= tok->source.nlines) {
         return _PYTOK_READ_EOF;
     }
-    const char *start = _PyTok_SourcePointer(&tok->source, tok->inp);
-    const char *newline = memchr(
-        start, '\n', tok->source.bytes + tok->source.len - start);
-    _PyTok_Off end = newline != NULL
-        ? newline - tok->source.bytes + 1 : tok->source.len;
+    _PyTok_Span tail = {tok->inp, tok->source.base_offset + tok->source.len};
+    Py_ssize_t remaining;
+    const char *start = _PyTok_SourceSpanView(&tok->source, tail, &remaining);
+    const char *newline = memchr(start, '\n', remaining);
     chunk->data = (char *)start;
-    chunk->len = tok->source.bytes + end - start;
+    chunk->len = newline != NULL ? newline - start + 1 : remaining;
     chunk->ownership = _PYTOK_CHUNK_BORROWED;
-    chunk->implicit_newline = end == tok->source.len &&
+    chunk->implicit_newline = chunk->len == remaining &&
         tok->reader->prepared_final_newline_is_implicit;
     return _PYTOK_READ_LINE;
 }
@@ -667,12 +666,12 @@ _PyTok_ReaderUnderflow(struct tok_state *tok)
         }
         tok->inp = source_start + chunk.len;
     }
-    if (prepared) {
+    else {
+        _PyTok_Off source_start = _PyTok_SourceOffset(&tok->source, chunk.data);
         if (tok->start < 0 && _PyLexer_CurrentFTString(tok) == NULL) {
-            tok->buf_offset = tok->source.base_offset +
-                (chunk.data - tok->source.bytes);
+            tok->buf_offset = source_start;
         }
-        tok->inp = _PyTok_SourceOffset(&tok->source, chunk.data) + chunk.len;
+        tok->inp = source_start + chunk.len;
     }
     tok->implicit_newline = chunk.implicit_newline;
 

Parser/tokenizer/source.h

@@ -37,6 +37,14 @@ _PyTok_SourcePointer(const _PyTok_SourceText *source, _PyTok_Off offset)
     return _PyTok_SourceData(source) + (offset - source->base_offset);
 }
 
+static inline unsigned char
+_PyTok_SourceByte(const _PyTok_SourceText *source, _PyTok_Off offset)
+{
+    assert(offset >= source->base_offset);
+    assert(offset - source->base_offset < source->len);
+    return (unsigned char)source->bytes[offset - source->base_offset];
+}
+
 static inline const char *
 _PyTok_SourceSpanView(const _PyTok_SourceText *source, _PyTok_Span span,
                       Py_ssize_t *length)

Parser/tokenizer/tokenizer.h

@@ -73,7 +73,7 @@ _PyTokenizer_Info _PyTokenizer_GetInfo(const struct tok_state *);
 /* An absent token span has a nonnull empty text view. */
 const char *_PyToken_TextView(
     const struct tok_state *, const struct token *, Py_ssize_t *);
-/* Use the token from the most recent Get. text is NULL for an absent span;
+/* Use the token from the most recent Get. An absent span has nonnull empty text.
    line contains the bytes of line_span, the token's complete physical line
    range. end_line_start is the logical offset of its final physical line. */
 void _PyToken_GetView(

Python/Python-tokenize.c

@@ -185,7 +185,7 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
     Py_ssize_t lineno = token->start_loc.lineno;
     Py_ssize_t end_lineno = token->end_loc.lineno;
     Py_ssize_t byte_offset = -1;
-    if (token_start >= 0 && token_start >= view->line_span.start) {
+    if (token_start >= view->line_span.start) {
         byte_offset = token_start - view->line_span.start;
         if (line_changed) {
             *col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
@@ -199,7 +199,7 @@ _get_col_offsets(tokenizeriterobject *it, const struct token *token,
         }
     }
 
-    if (token_end >= 0 && token_end >= view->end_line_start) {
+    if (token_end >= view->end_line_start) {
         Py_ssize_t end_byte_offset = token_end - view->end_line_start;
         if (lineno == end_lineno) {
             // Avoid rescanning the prefix of a very long line.
@@ -251,15 +251,7 @@ tokenizeriter_next(PyObject *op)
     }
     _PyToken_View view;
     _PyToken_GetView(it->tok, &token, &view);
-    const char *token_start = view.text;
-    PyObject *str;
-    if (token.span.start < 0) {
-        assert(token.span.start == -1 && token.span.end == -1);
-        str = Py_GetConstant(Py_CONSTANT_EMPTY_STR);
-    }
-    else {
-        str = PyUnicode_FromStringAndSize(token_start, view.length);
-    }
+    PyObject *str = PyUnicode_FromStringAndSize(view.text, view.length);
     if (str == NULL) {
         goto exit;
     }
@@ -307,8 +299,7 @@ tokenizeriter_next(PyObject *op)
         else if (type == NEWLINE) {
             if (!view.implicit_newline) {
                 Py_DECREF(str);
-                assert(token_start != NULL);
-                if (token_start[0] == '\r') {
+                if (view.text[0] == '\r') {
                     str = PyUnicode_FromString("\r\n");
                 } else {
                     str = PyUnicode_FromString("\n");