diff --git a/ide/languages.go/nbproject/project.xml b/ide/languages.go/nbproject/project.xml index 48c4f4a3336d..3c0bb29c6534 100644 --- a/ide/languages.go/nbproject/project.xml +++ b/ide/languages.go/nbproject/project.xml @@ -136,6 +136,15 @@ + + + unit + + org.netbeans.libs.junit4 + + + + diff --git a/ide/languages.go/test/unit/src/org/netbeans/modules/languages/go/GoLangLexerTest.java b/ide/languages.go/test/unit/src/org/netbeans/modules/languages/go/GoLangLexerTest.java new file mode 100644 index 000000000000..4ade164f7953 --- /dev/null +++ b/ide/languages.go/test/unit/src/org/netbeans/modules/languages/go/GoLangLexerTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.netbeans.modules.languages.go; + +import java.util.Collection; +import java.util.EnumSet; +import org.netbeans.api.lexer.Language; +import org.netbeans.api.lexer.Token; +import org.netbeans.api.lexer.TokenHierarchy; +import org.netbeans.api.lexer.TokenSequence; +import org.netbeans.spi.lexer.LanguageHierarchy; +import org.netbeans.spi.lexer.Lexer; +import org.netbeans.spi.lexer.LexerRestartInfo; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Tests for the Go lexer. + * + *

Contains regression tests for apache/netbeans#9654: typing a double + * quote as the last characters of a document made the lexer report end of + * input (a null token) while characters were still pending on the lexer + * input, which the lexer infrastructure rejects with an + * {@link IllegalStateException} ("returned null token but + * lexerInput.readLength()=..."). + */ +public class GoLangLexerTest { + + private static final Language LANGUAGE = new LanguageHierarchy() { + @Override + protected Collection createTokenIds() { + return EnumSet.allOf(GoTokenId.class); + } + + @Override + protected Lexer createLexer(LexerRestartInfo info) { + return new GoLangLexer(info); + } + + @Override + protected String mimeType() { + return "text/x-go"; + } + }.language(); + + /** + * The document shape from issue #9654: the typed quote leaves six + * characters, {@code ")}, LF, {@code }}, LF, LF, untokenized at the end + * of the document. + */ + private static final String ISSUE_9654_TEXT = "package main\n" + + "\n" + + "import (\n" + + " \"fmt\"\n" + + ")\n" + + "\n" + + "func main() {\n" + + " fmt.Println(\")\n" + + "}\n" + + "\n"; + + /** + * An unterminated interpreted string literal at the end of the text must + * still receive a token instead of an end-of-input signal. + */ + @Test + public void testUnterminatedStringAtEndOfText() { + Token last = lastToken(ISSUE_9654_TEXT); + assertEquals(GoTokenId.ERROR, last.id()); + assertEquals("\")\n}\n\n", last.text().toString()); + } + + /** + * Issue #9654: typing {@code "} to declare a string variable, before the + * literal is closed, used to throw while re-tokenizing the document. + */ + @Test + public void testTypedQuoteDoesNotThrow() { + String text = "package main\n" + + "\n" + + "func main() {\n" + + " var foo string = \"\n" + + "}\n"; + Token last = lastToken(text); + assertEquals(GoTokenId.ERROR, last.id()); + assertEquals("\"\n}\n", last.text().toString()); + } + + /** An unterminated rune literal at the end of the text. */ + @Test + public void testUnterminatedRuneAtEndOfText() { + Token last = lastToken("package main\nvar x = '"); + assertEquals(GoTokenId.ERROR, last.id()); + assertEquals("'", last.text().toString()); + } + + /** An unterminated raw string literal at the end of the text. */ + @Test + public void testUnterminatedRawStringAtEndOfText() { + Token last = lastToken("package main\nvar x = `raw"); + assertEquals(GoTokenId.ERROR, last.id()); + assertEquals("`raw", last.text().toString()); + } + + /** A closed string literal at the end of the text is still a STRING. */ + @Test + public void testClosedStringAtEndOfText() { + Token last = lastToken("package main\nvar foo = \"bar\""); + assertEquals(GoTokenId.STRING, last.id()); + assertEquals("\"bar\"", last.text().toString()); + } + + /** + * Well formed text must be tokenized completely and without any ERROR + * token: end of input without pending characters stays a null token. + */ + @Test + public void testWellFormedText() { + String text = "package main\n" + + "\n" + + "import (\n" + + " \"fmt\"\n" + + ")\n" + + "\n" + + "func main() {\n" + + " fmt.Println(\"hello world\")\n" + + "}\n"; + TokenSequence seq = TokenHierarchy.create(text, LANGUAGE) + .tokenSequence(LANGUAGE); + boolean foundString = false; + int count = 0; + while (seq.moveNext()) { + count++; + Token token = seq.token(); + assertNotEquals("Unexpected ERROR token in:\n" + text, + GoTokenId.ERROR, token.id()); + if (token.id() == GoTokenId.STRING + && "\"hello world\"".equals(token.text().toString())) { + foundString = true; + } + } + assertTrue("No tokens recognized", count > 0); + assertTrue("String literal was not tokenized", foundString); + } + + /** + * Lex the given text to its end. Fails with the IllegalStateException + * from issue #9654 if the lexer returns a null token while characters + * are still pending on the lexer input. + */ + private static Token lastToken(String text) { + TokenSequence seq = TokenHierarchy.create(text, LANGUAGE) + .tokenSequence(LANGUAGE); + Token last = null; + while (seq.moveNext()) { + last = seq.token(); + } + assertNotNull("No tokens recognized", last); + return last; + } +} diff --git a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java index 709d539f7cff..4d84d53df5f4 100644 --- a/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java +++ b/ide/languages.toml/src/org/netbeans/modules/languages/toml/TomlLexer.java @@ -41,8 +41,11 @@ public TomlLexer(LexerRestartInfo info) { @Override protected Token mapToken(org.antlr.v4.runtime.Token antlrToken) { switch (antlrToken.getType()) { + // EOF is only reported here while input is still pending when the + // text ended in something the lexer could not recognize, e.g. an + // unterminated multi-line string (#9654). case TOMLAntlrLexer.EOF: - return null; + return token(ERROR); // Strings case TOMLAntlrLexer.BASIC_STRING: diff --git a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java index 12e3fb41422d..9b4fc4c26f3f 100644 --- a/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java +++ b/ide/lexer.antlr4/src/org/netbeans/spi/lexer/antlr4/AbstractAntlrLexerBridge.java @@ -83,7 +83,20 @@ public final Token nextToken() { } else { nextToken = nextRealToken(); } - return nextToken.getType() != EOF ? mapToken(nextToken) : null; + if (nextToken.getType() != EOF) { + return mapToken(nextToken); + } + // The ANTLR lexer ran out of input. Normally that is the end of the + // text, but it also happens when the trailing characters could not be + // recognized: an ANTLR lexer throws LexerNoViableAltException and then + // emits EOF after consuming e.g. an unterminated string literal at the + // end of the document. Returning null in that state would leave the + // consumed characters without a NetBeans token and the lexer + // infrastructure would fail with "returned null token but + // lexerInput.readLength()=...". Let the implementation map the EOF + // token over those still unread characters instead (usually to an + // error token), so they receive a token. + return input.readLength() > 0 ? mapToken(nextToken) : null; } /** @@ -103,6 +116,16 @@ public final Token nextToken() { * return token(SomeTokenId.ERROR); * } * } + * + *

This method is also called with the {@code EOF} token when the ANTLR + * lexer reached the end of the input on characters it could not recognize, + * for example after an unterminated string literal at the end of the text. + * Those characters have already been read from the + * {@link org.netbeans.spi.lexer.LexerInput} (their count is reported by + * {@link org.netbeans.spi.lexer.LexerInput#readLength()}) and still need a + * token, so implementations shall return a token covering them, usually an + * error token, and shall not return {@code null} in that case. + * * @param antlrToken the token from the ANTLR Lexer * * @return a NetBeans lexer token.