| /* |
| * Copyright (c) 2012, the Dart project authors. |
| * |
| * Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except |
| * in compliance with the License. You may obtain a copy of the License at |
| * |
| * http://www.eclipse.org/legal/epl-v10.html |
| * |
| * Unless required by applicable law or agreed to in writing, software distributed under the License |
| * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express |
| * or implied. See the License for the specific language governing permissions and limitations under |
| * the License. |
| */ |
| package com.google.dart.engine.scanner; |
| |
| import com.google.dart.engine.error.AnalysisError; |
| import com.google.dart.engine.error.AnalysisErrorListener; |
| import com.google.dart.engine.error.GatheringErrorListener; |
| import com.google.dart.engine.source.TestSource; |
| import com.google.dart.engine.utilities.os.OSUtilities; |
| import com.google.dart.engine.utilities.source.LineInfo; |
| |
| import junit.framework.TestCase; |
| |
| public class ScannerTest extends TestCase { |
| /** |
| * Instances of the class {@code ExpectedLocation} encode information about the expected location |
| * of a given offset in source code. |
| */ |
| private static class ExpectedLocation { |
| private int offset; |
| private int lineNumber; |
| private int columnNumber; |
| |
| public ExpectedLocation(int offset, int lineNumber, int columnNumber) { |
| this.offset = offset; |
| this.lineNumber = lineNumber; |
| this.columnNumber = columnNumber; |
| } |
| } |
| |
| public void fail_incomplete_string_interpolation() throws Exception { |
| // https://code.google.com/p/dart/issues/detail?id=18073 |
| assertErrorAndTokens( |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 9, |
| "\"foo ${bar", |
| new StringToken(TokenType.STRING, "\"foo ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 5), |
| new StringToken(TokenType.IDENTIFIER, "bar", 7)); |
| } |
| |
| public void test_ampersand() throws Exception { |
| assertToken(TokenType.AMPERSAND, "&"); |
| } |
| |
| public void test_ampersand_ampersand() throws Exception { |
| assertToken(TokenType.AMPERSAND_AMPERSAND, "&&"); |
| } |
| |
| public void test_ampersand_eq() throws Exception { |
| assertToken(TokenType.AMPERSAND_EQ, "&="); |
| } |
| |
| public void test_at() throws Exception { |
| assertToken(TokenType.AT, "@"); |
| } |
| |
| public void test_backping() throws Exception { |
| assertToken(TokenType.BACKPING, "`"); |
| } |
| |
| public void test_backslash() throws Exception { |
| assertToken(TokenType.BACKSLASH, "\\"); |
| } |
| |
| public void test_bang() throws Exception { |
| assertToken(TokenType.BANG, "!"); |
| } |
| |
| public void test_bang_eq() throws Exception { |
| assertToken(TokenType.BANG_EQ, "!="); |
| } |
| |
| public void test_bar() throws Exception { |
| assertToken(TokenType.BAR, "|"); |
| } |
| |
| public void test_bar_bar() throws Exception { |
| assertToken(TokenType.BAR_BAR, "||"); |
| } |
| |
| public void test_bar_eq() throws Exception { |
| assertToken(TokenType.BAR_EQ, "|="); |
| } |
| |
| public void test_caret() throws Exception { |
| assertToken(TokenType.CARET, "^"); |
| } |
| |
| public void test_caret_eq() throws Exception { |
| assertToken(TokenType.CARET_EQ, "^="); |
| } |
| |
| public void test_close_curly_bracket() throws Exception { |
| assertToken(TokenType.CLOSE_CURLY_BRACKET, "}"); |
| } |
| |
| public void test_close_paren() throws Exception { |
| assertToken(TokenType.CLOSE_PAREN, ")"); |
| } |
| |
| public void test_close_quare_bracket() throws Exception { |
| assertToken(TokenType.CLOSE_SQUARE_BRACKET, "]"); |
| } |
| |
| public void test_colon() throws Exception { |
| assertToken(TokenType.COLON, ":"); |
| } |
| |
| public void test_comma() throws Exception { |
| assertToken(TokenType.COMMA, ","); |
| } |
| |
| public void test_comment_disabled_multi() throws Exception { |
| Scanner scanner = new Scanner( |
| null, |
| new CharSequenceReader("/* comment */ "), |
| AnalysisErrorListener.NULL_LISTENER); |
| scanner.setPreserveComments(false); |
| Token token = scanner.tokenize(); |
| assertNotNull(token); |
| assertNull(token.getPrecedingComments()); |
| } |
| |
| public void test_comment_multi() throws Exception { |
| assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment */"); |
| } |
| |
| public void test_comment_multi_unterminated() throws Exception { |
| assertError(ScannerErrorCode.UNTERMINATED_MULTI_LINE_COMMENT, 3, "/* x"); |
| } |
| |
| public void test_comment_nested() throws Exception { |
| assertComment(TokenType.MULTI_LINE_COMMENT, "/* comment /* within a */ comment */"); |
| } |
| |
| public void test_comment_single() throws Exception { |
| assertComment(TokenType.SINGLE_LINE_COMMENT, "// comment"); |
| } |
| |
| public void test_double_both_e() throws Exception { |
| assertToken(TokenType.DOUBLE, "0.123e4"); |
| } |
| |
| public void test_double_both_E() throws Exception { |
| assertToken(TokenType.DOUBLE, "0.123E4"); |
| } |
| |
| public void test_double_fraction() throws Exception { |
| assertToken(TokenType.DOUBLE, ".123"); |
| } |
| |
| public void test_double_fraction_e() throws Exception { |
| assertToken(TokenType.DOUBLE, ".123e4"); |
| } |
| |
| public void test_double_fraction_E() throws Exception { |
| assertToken(TokenType.DOUBLE, ".123E4"); |
| } |
| |
| public void test_double_missingDigitInExponent() throws Exception { |
| assertError(ScannerErrorCode.MISSING_DIGIT, 1, "1e"); |
| } |
| |
| public void test_double_whole_e() throws Exception { |
| assertToken(TokenType.DOUBLE, "12e4"); |
| } |
| |
| public void test_double_whole_E() throws Exception { |
| assertToken(TokenType.DOUBLE, "12E4"); |
| } |
| |
| public void test_eq() throws Exception { |
| assertToken(TokenType.EQ, "="); |
| } |
| |
| public void test_eq_eq() throws Exception { |
| assertToken(TokenType.EQ_EQ, "=="); |
| } |
| |
| public void test_gt() throws Exception { |
| assertToken(TokenType.GT, ">"); |
| } |
| |
| public void test_gt_eq() throws Exception { |
| assertToken(TokenType.GT_EQ, ">="); |
| } |
| |
| public void test_gt_gt() throws Exception { |
| assertToken(TokenType.GT_GT, ">>"); |
| } |
| |
| public void test_gt_gt_eq() throws Exception { |
| assertToken(TokenType.GT_GT_EQ, ">>="); |
| } |
| |
| public void test_hash() throws Exception { |
| assertToken(TokenType.HASH, "#"); |
| } |
| |
| public void test_hexidecimal() throws Exception { |
| assertToken(TokenType.HEXADECIMAL, "0x1A2B3C"); |
| } |
| |
| public void test_hexidecimal_missingDigit() throws Exception { |
| assertError(ScannerErrorCode.MISSING_HEX_DIGIT, 1, "0x"); |
| } |
| |
| public void test_identifier() throws Exception { |
| assertToken(TokenType.IDENTIFIER, "result"); |
| } |
| |
| public void test_illegalChar_cyrillicLetter_middle() throws Exception { |
| assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 5, "Shche\u0433lov"); |
| } |
| |
| public void test_illegalChar_cyrillicLetter_start() throws Exception { |
| assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0429"); |
| } |
| |
| public void test_illegalChar_nbsp() throws Exception { |
| assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u00A0"); |
| } |
| |
| public void test_illegalChar_notLetter() throws Exception { |
| assertError(ScannerErrorCode.ILLEGAL_CHARACTER, 0, "\u0312"); |
| } |
| |
| public void test_index() throws Exception { |
| assertToken(TokenType.INDEX, "[]"); |
| } |
| |
| public void test_index_eq() throws Exception { |
| assertToken(TokenType.INDEX_EQ, "[]="); |
| } |
| |
| public void test_int() throws Exception { |
| assertToken(TokenType.INT, "123"); |
| } |
| |
| public void test_int_initialZero() throws Exception { |
| assertToken(TokenType.INT, "0123"); |
| } |
| |
| public void test_keyword_abstract() throws Exception { |
| assertKeywordToken("abstract"); |
| } |
| |
| public void test_keyword_as() throws Exception { |
| assertKeywordToken("as"); |
| } |
| |
| public void test_keyword_assert() throws Exception { |
| assertKeywordToken("assert"); |
| } |
| |
| public void test_keyword_break() throws Exception { |
| assertKeywordToken("break"); |
| } |
| |
| public void test_keyword_case() throws Exception { |
| assertKeywordToken("case"); |
| } |
| |
| public void test_keyword_catch() throws Exception { |
| assertKeywordToken("catch"); |
| } |
| |
| public void test_keyword_class() throws Exception { |
| assertKeywordToken("class"); |
| } |
| |
| public void test_keyword_const() throws Exception { |
| assertKeywordToken("const"); |
| } |
| |
| public void test_keyword_continue() throws Exception { |
| assertKeywordToken("continue"); |
| } |
| |
| public void test_keyword_default() throws Exception { |
| assertKeywordToken("default"); |
| } |
| |
| public void test_keyword_deferred() throws Exception { |
| assertKeywordToken("deferred"); |
| } |
| |
| public void test_keyword_do() throws Exception { |
| assertKeywordToken("do"); |
| } |
| |
| public void test_keyword_dynamic() throws Exception { |
| assertKeywordToken("dynamic"); |
| } |
| |
| public void test_keyword_else() throws Exception { |
| assertKeywordToken("else"); |
| } |
| |
| public void test_keyword_enum() throws Exception { |
| assertKeywordToken("enum"); |
| } |
| |
| public void test_keyword_export() throws Exception { |
| assertKeywordToken("export"); |
| } |
| |
| public void test_keyword_extends() throws Exception { |
| assertKeywordToken("extends"); |
| } |
| |
| public void test_keyword_factory() throws Exception { |
| assertKeywordToken("factory"); |
| } |
| |
| public void test_keyword_false() throws Exception { |
| assertKeywordToken("false"); |
| } |
| |
| public void test_keyword_final() throws Exception { |
| assertKeywordToken("final"); |
| } |
| |
| public void test_keyword_finally() throws Exception { |
| assertKeywordToken("finally"); |
| } |
| |
| public void test_keyword_for() throws Exception { |
| assertKeywordToken("for"); |
| } |
| |
| public void test_keyword_get() throws Exception { |
| assertKeywordToken("get"); |
| } |
| |
| public void test_keyword_if() throws Exception { |
| assertKeywordToken("if"); |
| } |
| |
| public void test_keyword_implements() throws Exception { |
| assertKeywordToken("implements"); |
| } |
| |
| public void test_keyword_import() throws Exception { |
| assertKeywordToken("import"); |
| } |
| |
| public void test_keyword_in() throws Exception { |
| assertKeywordToken("in"); |
| } |
| |
| public void test_keyword_is() throws Exception { |
| assertKeywordToken("is"); |
| } |
| |
| public void test_keyword_library() throws Exception { |
| assertKeywordToken("library"); |
| } |
| |
| public void test_keyword_new() throws Exception { |
| assertKeywordToken("new"); |
| } |
| |
| public void test_keyword_null() throws Exception { |
| assertKeywordToken("null"); |
| } |
| |
| public void test_keyword_operator() throws Exception { |
| assertKeywordToken("operator"); |
| } |
| |
| public void test_keyword_part() throws Exception { |
| assertKeywordToken("part"); |
| } |
| |
| public void test_keyword_rethrow() throws Exception { |
| assertKeywordToken("rethrow"); |
| } |
| |
| public void test_keyword_return() throws Exception { |
| assertKeywordToken("return"); |
| } |
| |
| public void test_keyword_set() throws Exception { |
| assertKeywordToken("set"); |
| } |
| |
| public void test_keyword_static() throws Exception { |
| assertKeywordToken("static"); |
| } |
| |
| public void test_keyword_super() throws Exception { |
| assertKeywordToken("super"); |
| } |
| |
| public void test_keyword_switch() throws Exception { |
| assertKeywordToken("switch"); |
| } |
| |
| public void test_keyword_this() throws Exception { |
| assertKeywordToken("this"); |
| } |
| |
| public void test_keyword_throw() throws Exception { |
| assertKeywordToken("throw"); |
| } |
| |
| public void test_keyword_true() throws Exception { |
| assertKeywordToken("true"); |
| } |
| |
| public void test_keyword_try() throws Exception { |
| assertKeywordToken("try"); |
| } |
| |
| public void test_keyword_typedef() throws Exception { |
| assertKeywordToken("typedef"); |
| } |
| |
| public void test_keyword_var() throws Exception { |
| assertKeywordToken("var"); |
| } |
| |
| public void test_keyword_void() throws Exception { |
| assertKeywordToken("void"); |
| } |
| |
| public void test_keyword_while() throws Exception { |
| assertKeywordToken("while"); |
| } |
| |
| public void test_keyword_with() throws Exception { |
| assertKeywordToken("with"); |
| } |
| |
| public void test_lineInfo_multilineComment() throws Exception { |
| String source = "/*\r *\r */"; |
| assertLineInfo( |
| source, |
| new ExpectedLocation(0, 1, 1), |
| new ExpectedLocation(4, 2, 2), |
| new ExpectedLocation(source.length() - 1, 3, 3)); |
| } |
| |
| public void test_lineInfo_multilineString() throws Exception { |
| String source = "'''a\r\nbc\r\nd'''"; |
| assertLineInfo( |
| source, |
| new ExpectedLocation(0, 1, 1), |
| new ExpectedLocation(7, 2, 2), |
| new ExpectedLocation(source.length() - 1, 3, 4)); |
| } |
| |
| public void test_lineInfo_simpleClass() throws Exception { |
| String source = "class Test {\r\n String s = '...';\r\n int get x => s.MISSING_GETTER;\r\n}"; |
| assertLineInfo( |
| source, |
| new ExpectedLocation(0, 1, 1), |
| new ExpectedLocation(source.indexOf("MISSING_GETTER"), 3, 20), |
| new ExpectedLocation(source.length() - 1, 4, 1)); |
| } |
| |
| public void test_lineInfo_slashN() throws Exception { |
| String source = "class Test {\n}"; |
| assertLineInfo(source, new ExpectedLocation(0, 1, 1), new ExpectedLocation( |
| source.indexOf("}"), |
| 2, |
| 1)); |
| } |
| |
| public void test_lt() throws Exception { |
| assertToken(TokenType.LT, "<"); |
| } |
| |
| public void test_lt_eq() throws Exception { |
| assertToken(TokenType.LT_EQ, "<="); |
| } |
| |
| public void test_lt_lt() throws Exception { |
| assertToken(TokenType.LT_LT, "<<"); |
| } |
| |
| public void test_lt_lt_eq() throws Exception { |
| assertToken(TokenType.LT_LT_EQ, "<<="); |
| } |
| |
| public void test_minus() throws Exception { |
| assertToken(TokenType.MINUS, "-"); |
| } |
| |
| public void test_minus_eq() throws Exception { |
| assertToken(TokenType.MINUS_EQ, "-="); |
| } |
| |
| public void test_minus_minus() throws Exception { |
| assertToken(TokenType.MINUS_MINUS, "--"); |
| } |
| |
| public void test_open_curly_bracket() throws Exception { |
| assertToken(TokenType.OPEN_CURLY_BRACKET, "{"); |
| } |
| |
| public void test_open_paren() throws Exception { |
| assertToken(TokenType.OPEN_PAREN, "("); |
| } |
| |
| public void test_open_square_bracket() throws Exception { |
| assertToken(TokenType.OPEN_SQUARE_BRACKET, "["); |
| } |
| |
| public void test_openSquareBracket() throws Exception { |
| assertToken(TokenType.OPEN_SQUARE_BRACKET, "["); |
| } |
| |
| public void test_percent() throws Exception { |
| assertToken(TokenType.PERCENT, "%"); |
| } |
| |
| public void test_percent_eq() throws Exception { |
| assertToken(TokenType.PERCENT_EQ, "%="); |
| } |
| |
| public void test_period() throws Exception { |
| assertToken(TokenType.PERIOD, "."); |
| } |
| |
| public void test_period_period() throws Exception { |
| assertToken(TokenType.PERIOD_PERIOD, ".."); |
| } |
| |
| public void test_period_period_period() throws Exception { |
| assertToken(TokenType.PERIOD_PERIOD_PERIOD, "..."); |
| } |
| |
| public void test_periodAfterNumberNotIncluded_identifier() throws Exception { |
| assertTokens( |
| "42.isEven()", |
| new StringToken(TokenType.INT, "42", 0), |
| new Token(TokenType.PERIOD, 2), |
| new StringToken(TokenType.IDENTIFIER, "isEven", 3), |
| new Token(TokenType.OPEN_PAREN, 9), |
| new Token(TokenType.CLOSE_PAREN, 10)); |
| } |
| |
| public void test_periodAfterNumberNotIncluded_period() throws Exception { |
| assertTokens( |
| "42..isEven()", |
| new StringToken(TokenType.INT, "42", 0), |
| new Token(TokenType.PERIOD_PERIOD, 2), |
| new StringToken(TokenType.IDENTIFIER, "isEven", 4), |
| new Token(TokenType.OPEN_PAREN, 10), |
| new Token(TokenType.CLOSE_PAREN, 11)); |
| } |
| |
| public void test_plus() throws Exception { |
| assertToken(TokenType.PLUS, "+"); |
| } |
| |
| public void test_plus_eq() throws Exception { |
| assertToken(TokenType.PLUS_EQ, "+="); |
| } |
| |
| public void test_plus_plus() throws Exception { |
| assertToken(TokenType.PLUS_PLUS, "++"); |
| } |
| |
| public void test_question() throws Exception { |
| assertToken(TokenType.QUESTION, "?"); |
| } |
| |
| public void test_scriptTag_withArgs() throws Exception { |
| assertToken(TokenType.SCRIPT_TAG, "#!/bin/dart -debug"); |
| } |
| |
| public void test_scriptTag_withoutSpace() throws Exception { |
| assertToken(TokenType.SCRIPT_TAG, "#!/bin/dart"); |
| } |
| |
| public void test_scriptTag_withSpace() throws Exception { |
| assertToken(TokenType.SCRIPT_TAG, "#! /bin/dart"); |
| } |
| |
| public void test_semicolon() throws Exception { |
| assertToken(TokenType.SEMICOLON, ";"); |
| } |
| |
| public void test_setSourceStart() throws Exception { |
| int offsetDelta = 42; |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| Scanner scanner = new Scanner(null, new SubSequenceReader("a", offsetDelta), listener); |
| scanner.setSourceStart(3, 9); |
| scanner.tokenize(); |
| int[] lineStarts = scanner.getLineStarts(); |
| assertNotNull(lineStarts); |
| assertEquals(3, lineStarts.length); |
| assertEquals(33, lineStarts[2]); |
| } |
| |
| public void test_slash() throws Exception { |
| assertToken(TokenType.SLASH, "/"); |
| } |
| |
| public void test_slash_eq() throws Exception { |
| assertToken(TokenType.SLASH_EQ, "/="); |
| } |
| |
| public void test_star() throws Exception { |
| assertToken(TokenType.STAR, "*"); |
| } |
| |
| public void test_star_eq() throws Exception { |
| assertToken(TokenType.STAR_EQ, "*="); |
| } |
| |
| public void test_startAndEnd() { |
| Token token = scan("a"); |
| Token previous = token.getPrevious(); |
| assertEquals(token, previous.getNext()); |
| assertEquals(previous, previous.getPrevious()); |
| Token next = token.getNext(); |
| assertEquals(next, next.getNext()); |
| assertEquals(token, next.getPrevious()); |
| } |
| |
| public void test_string_multi_double() throws Exception { |
| assertToken(TokenType.STRING, "\"\"\"line1\nline2\"\"\""); |
| } |
| |
| public void test_string_multi_embeddedQuotes() throws Exception { |
| assertToken(TokenType.STRING, "\"\"\"line1\n\"\"\nline2\"\"\""); |
| } |
| |
| public void test_string_multi_embeddedQuotes_escapedChar() throws Exception { |
| assertToken(TokenType.STRING, "\"\"\"a\"\"\\tb\"\"\""); |
| } |
| |
| public void test_string_multi_interpolation_block() throws Exception { |
| assertTokens( |
| "\"Hello ${name}!\"", |
| new StringToken(TokenType.STRING, "\"Hello ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 7), |
| new StringToken(TokenType.IDENTIFIER, "name", 9), |
| new Token(TokenType.CLOSE_CURLY_BRACKET, 13), |
| new StringToken(TokenType.STRING, "!\"", 14)); |
| } |
| |
| public void test_string_multi_interpolation_identifier() throws Exception { |
| assertTokens( // |
| "\"Hello $name!\"", |
| new StringToken(TokenType.STRING, "\"Hello ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 7), |
| new StringToken(TokenType.IDENTIFIER, "name", 8), |
| new StringToken(TokenType.STRING, "!\"", 12)); |
| } |
| |
| public void test_string_multi_single() throws Exception { |
| assertToken(TokenType.STRING, "'''string'''"); |
| } |
| |
| public void test_string_multi_slashEnter() throws Exception { |
| assertToken(TokenType.STRING, "'''\\\n'''"); |
| } |
| |
| public void test_string_multi_unterminated() throws Exception { |
| assertErrorAndTokens( |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 8, |
| "'''string", |
| new StringToken(TokenType.STRING, "'''string", 0)); |
| } |
| |
| public void test_string_multi_unterminated_interpolation_block() throws Exception { |
| assertErrorAndTokens( // |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 8, |
| "'''${name", |
| new StringToken(TokenType.STRING, "'''", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 3), |
| new StringToken(TokenType.IDENTIFIER, "name", 5), |
| new StringToken(TokenType.STRING, "", 9)); |
| } |
| |
| public void test_string_multi_unterminated_interpolation_identifier() throws Exception { |
| assertErrorAndTokens( // |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 7, |
| "'''$name", |
| new StringToken(TokenType.STRING, "'''", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 3), |
| new StringToken(TokenType.IDENTIFIER, "name", 4), |
| new StringToken(TokenType.STRING, "", 8)); |
| } |
| |
| public void test_string_raw_multi_double() throws Exception { |
| assertToken(TokenType.STRING, "r\"\"\"line1\nline2\"\"\""); |
| } |
| |
| public void test_string_raw_multi_single() throws Exception { |
| assertToken(TokenType.STRING, "r'''string'''"); |
| } |
| |
| public void test_string_raw_multi_unterminated() throws Exception { |
| String source = "r'''string"; |
| assertErrorAndTokens( // |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 9, |
| source, |
| new StringToken(TokenType.STRING, source, 0)); |
| } |
| |
| public void test_string_raw_simple_double() throws Exception { |
| assertToken(TokenType.STRING, "r\"string\""); |
| } |
| |
| public void test_string_raw_simple_single() throws Exception { |
| assertToken(TokenType.STRING, "r'string'"); |
| } |
| |
| public void test_string_raw_simple_unterminated_eof() throws Exception { |
| String source = "r'string"; |
| assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 7, source, new StringToken( |
| TokenType.STRING, |
| source, |
| 0)); |
| } |
| |
| public void test_string_raw_simple_unterminated_eol() throws Exception { |
| String source = "r'string"; |
| assertErrorAndTokens( |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 8, |
| source + "\n", |
| new StringToken(TokenType.STRING, source, 0)); |
| } |
| |
| public void test_string_simple_double() throws Exception { |
| assertToken(TokenType.STRING, "\"string\""); |
| } |
| |
| public void test_string_simple_escapedDollar() throws Exception { |
| assertToken(TokenType.STRING, "'a\\$b'"); |
| } |
| |
| public void test_string_simple_interpolation_adjacentIdentifiers() throws Exception { |
| assertTokens( // |
| "'$a$b'", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 1), |
| new StringToken(TokenType.IDENTIFIER, "a", 2), |
| new StringToken(TokenType.STRING, "", 3), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 3), |
| new StringToken(TokenType.IDENTIFIER, "b", 4), |
| new StringToken(TokenType.STRING, "'", 5)); |
| } |
| |
| public void test_string_simple_interpolation_block() throws Exception { |
| assertTokens( |
| "'Hello ${name}!'", |
| new StringToken(TokenType.STRING, "'Hello ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 7), |
| new StringToken(TokenType.IDENTIFIER, "name", 9), |
| new Token(TokenType.CLOSE_CURLY_BRACKET, 13), |
| new StringToken(TokenType.STRING, "!'", 14)); |
| } |
| |
| public void test_string_simple_interpolation_blockWithNestedMap() throws Exception { |
| assertTokens( |
| "'a ${f({'b' : 'c'})} d'", |
| new StringToken(TokenType.STRING, "'a ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 3), |
| new StringToken(TokenType.IDENTIFIER, "f", 5), |
| new Token(TokenType.OPEN_PAREN, 6), |
| new Token(TokenType.OPEN_CURLY_BRACKET, 7), |
| new StringToken(TokenType.STRING, "'b'", 8), |
| new Token(TokenType.COLON, 12), |
| new StringToken(TokenType.STRING, "'c'", 14), |
| new Token(TokenType.CLOSE_CURLY_BRACKET, 17), |
| new Token(TokenType.CLOSE_PAREN, 18), |
| new Token(TokenType.CLOSE_CURLY_BRACKET, 19), |
| new StringToken(TokenType.STRING, " d'", 20)); |
| } |
| |
| public void test_string_simple_interpolation_firstAndLast() throws Exception { |
| assertTokens( // |
| "'$greeting $name'", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 1), |
| new StringToken(TokenType.IDENTIFIER, "greeting", 2), |
| new StringToken(TokenType.STRING, " ", 10), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 11), |
| new StringToken(TokenType.IDENTIFIER, "name", 12), |
| new StringToken(TokenType.STRING, "'", 16)); |
| } |
| |
| public void test_string_simple_interpolation_identifier() throws Exception { |
| assertTokens( // |
| "'Hello $name!'", |
| new StringToken(TokenType.STRING, "'Hello ", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 7), |
| new StringToken(TokenType.IDENTIFIER, "name", 8), |
| new StringToken(TokenType.STRING, "!'", 12)); |
| } |
| |
| public void test_string_simple_interpolation_missingIdentifier() throws Exception { |
| assertTokens( // |
| "'$x$'", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 1), |
| new StringToken(TokenType.IDENTIFIER, "x", 2), |
| new StringToken(TokenType.STRING, "", 3), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 3), |
| new StringToken(TokenType.STRING, "'", 4)); |
| } |
| |
| public void test_string_simple_interpolation_nonIdentifier() throws Exception { |
| assertTokens( // |
| "'$1'", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 1), |
| new StringToken(TokenType.STRING, "1'", 2)); |
| } |
| |
| public void test_string_simple_single() throws Exception { |
| assertToken(TokenType.STRING, "'string'"); |
| } |
| |
| public void test_string_simple_unterminated_eof() throws Exception { |
| String source = "'string"; |
| assertErrorAndTokens(ScannerErrorCode.UNTERMINATED_STRING_LITERAL, 6, source, new StringToken( |
| TokenType.STRING, |
| source, |
| 0)); |
| } |
| |
| public void test_string_simple_unterminated_eol() throws Exception { |
| String source = "'string"; |
| assertErrorAndTokens( |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 7, |
| source + "\r", |
| new StringToken(TokenType.STRING, source, 0)); |
| } |
| |
| public void test_string_simple_unterminated_interpolation_block() throws Exception { |
| assertErrorAndTokens( // |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 6, |
| "'${name", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_EXPRESSION, "${", 1), |
| new StringToken(TokenType.IDENTIFIER, "name", 3), |
| new StringToken(TokenType.STRING, "", 7)); |
| } |
| |
| public void test_string_simple_unterminated_interpolation_identifier() throws Exception { |
| assertErrorAndTokens( // |
| ScannerErrorCode.UNTERMINATED_STRING_LITERAL, |
| 5, |
| "'$name", |
| new StringToken(TokenType.STRING, "'", 0), |
| new StringToken(TokenType.STRING_INTERPOLATION_IDENTIFIER, "$", 1), |
| new StringToken(TokenType.IDENTIFIER, "name", 2), |
| new StringToken(TokenType.STRING, "", 6)); |
| } |
| |
| public void test_tilde() throws Exception { |
| assertToken(TokenType.TILDE, "~"); |
| } |
| |
| public void test_tilde_slash() throws Exception { |
| assertToken(TokenType.TILDE_SLASH, "~/"); |
| } |
| |
| public void test_tilde_slash_eq() throws Exception { |
| assertToken(TokenType.TILDE_SLASH_EQ, "~/="); |
| } |
| |
| public void test_unclosedPairInInterpolation() throws Exception { |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| scanWithListener("'${(}'", listener); |
| } |
| |
| private void assertComment(TokenType commentType, String source) throws Exception { |
| // |
| // Test without a trailing end-of-line marker |
| // |
| Token token = scan(source); |
| assertNotNull(token); |
| assertEquals(TokenType.EOF, token.getType()); |
| |
| Token comment = token.getPrecedingComments(); |
| assertNotNull(comment); |
| assertEquals(commentType, comment.getType()); |
| assertEquals(0, comment.getOffset()); |
| assertEquals(source.length(), comment.getLength()); |
| assertEquals(source, comment.getLexeme()); |
| // |
| // Test with a trailing end-of-line marker |
| // |
| token = scan(source + OSUtilities.LINE_SEPARATOR); |
| assertNotNull(token); |
| assertEquals(TokenType.EOF, token.getType()); |
| |
| comment = token.getPrecedingComments(); |
| assertNotNull(comment); |
| assertEquals(commentType, comment.getType()); |
| assertEquals(0, comment.getOffset()); |
| assertEquals(source.length(), comment.getLength()); |
| assertEquals(source, comment.getLexeme()); |
| } |
| |
| /** |
| * Assert that scanning the given source produces an error with the given code. |
| * |
| * @param expectedError the error that should be produced |
| * @param expectedOffset the string offset that should be associated with the error |
| * @param source the source to be scanned to produce the error |
| */ |
| private void assertError(ScannerErrorCode expectedError, int expectedOffset, String source) { |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| scanWithListener(source, listener); |
| listener.assertErrors(new AnalysisError( |
| null, |
| expectedOffset, |
| 1, |
| expectedError, |
| (int) source.charAt(expectedOffset))); |
| } |
| |
| /** |
| * Assert that scanning the given source produces an error with the given code, and also produces |
| * the given tokens. |
| * |
| * @param expectedError the error that should be produced |
| * @param expectedOffset the string offset that should be associated with the error |
| * @param source the source to be scanned to produce the error |
| * @param expectedTokens the tokens that are expected to be in the source |
| */ |
| private void assertErrorAndTokens(ScannerErrorCode expectedError, int expectedOffset, |
| String source, Token... expectedTokens) { |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| Token token = scanWithListener(source, listener); |
| listener.assertErrors(new AnalysisError( |
| null, |
| expectedOffset, |
| 1, |
| expectedError, |
| (int) source.charAt(expectedOffset))); |
| checkTokens(token, expectedTokens); |
| } |
| |
| /** |
| * Assert that when scanned the given source contains a single keyword token with the same lexeme |
| * as the original source. |
| * |
| * @param source the source to be scanned |
| */ |
| private void assertKeywordToken(String source) { |
| Token token = scan(source); |
| assertNotNull(token); |
| assertEquals(TokenType.KEYWORD, token.getType()); |
| assertEquals(0, token.getOffset()); |
| assertEquals(source.length(), token.getLength()); |
| assertEquals(source, token.getLexeme()); |
| Object value = token.value(); |
| assertTrue(value instanceof Keyword); |
| assertEquals(source, ((Keyword) value).getSyntax()); |
| |
| token = scan(" " + source + " "); |
| assertNotNull(token); |
| assertEquals(TokenType.KEYWORD, token.getType()); |
| assertEquals(1, token.getOffset()); |
| assertEquals(source.length(), token.getLength()); |
| assertEquals(source, token.getLexeme()); |
| value = token.value(); |
| assertTrue(value instanceof Keyword); |
| assertEquals(source, ((Keyword) value).getSyntax()); |
| |
| assertEquals(TokenType.EOF, token.getNext().getType()); |
| } |
| |
| private void assertLineInfo(String source, ExpectedLocation... expectedLocations) { |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| scanWithListener(source, listener); |
| listener.assertNoErrors(); |
| LineInfo info = listener.getLineInfo(new TestSource()); |
| assertNotNull(info); |
| for (ExpectedLocation expectedLocation : expectedLocations) { |
| LineInfo.Location location = info.getLocation(expectedLocation.offset); |
| assertEquals(expectedLocation.lineNumber, location.getLineNumber()); |
| assertEquals(expectedLocation.columnNumber, location.getColumnNumber()); |
| } |
| } |
| |
| /** |
| * Assert that the token scanned from the given source has the expected type. |
| * |
| * @param expectedType the expected type of the token |
| * @param source the source to be scanned to produce the actual token |
| */ |
| private Token assertToken(TokenType expectedType, String source) { |
| Token originalToken = scan(source); |
| assertNotNull(originalToken); |
| assertEquals(expectedType, originalToken.getType()); |
| assertEquals(0, originalToken.getOffset()); |
| assertEquals(source.length(), originalToken.getLength()); |
| assertEquals(source, originalToken.getLexeme()); |
| |
| if (expectedType == TokenType.SCRIPT_TAG) { |
| // Adding space before the script tag is not allowed, and adding text at the end changes nothing. |
| return originalToken; |
| } else if (expectedType == TokenType.SINGLE_LINE_COMMENT) { |
| // Adding space to an end-of-line comment changes the comment. |
| Token tokenWithSpaces = scan(" " + source); |
| assertNotNull(tokenWithSpaces); |
| assertEquals(expectedType, tokenWithSpaces.getType()); |
| assertEquals(1, tokenWithSpaces.getOffset()); |
| assertEquals(source.length(), tokenWithSpaces.getLength()); |
| assertEquals(source, tokenWithSpaces.getLexeme()); |
| return originalToken; |
| } else if (expectedType == TokenType.INT || expectedType == TokenType.DOUBLE) { |
| Token tokenWithLowerD = scan(source + "d"); |
| assertNotNull(tokenWithLowerD); |
| assertEquals(expectedType, tokenWithLowerD.getType()); |
| assertEquals(0, tokenWithLowerD.getOffset()); |
| assertEquals(source.length(), tokenWithLowerD.getLength()); |
| assertEquals(source, tokenWithLowerD.getLexeme()); |
| |
| Token tokenWithUpperD = scan(source + "D"); |
| assertNotNull(tokenWithUpperD); |
| assertEquals(expectedType, tokenWithUpperD.getType()); |
| assertEquals(0, tokenWithUpperD.getOffset()); |
| assertEquals(source.length(), tokenWithUpperD.getLength()); |
| assertEquals(source, tokenWithUpperD.getLexeme()); |
| } |
| |
| Token tokenWithSpaces = scan(" " + source + " "); |
| assertNotNull(tokenWithSpaces); |
| assertEquals(expectedType, tokenWithSpaces.getType()); |
| assertEquals(1, tokenWithSpaces.getOffset()); |
| assertEquals(source.length(), tokenWithSpaces.getLength()); |
| assertEquals(source, tokenWithSpaces.getLexeme()); |
| |
| assertEquals(TokenType.EOF, originalToken.getNext().getType()); |
| |
| return originalToken; |
| } |
| |
| /** |
| * Assert that when scanned the given source contains a sequence of tokens identical to the given |
| * tokens. |
| * |
| * @param source the source to be scanned |
| * @param expectedTokens the tokens that are expected to be in the source |
| */ |
| private void assertTokens(String source, Token... expectedTokens) { |
| Token token = scan(source); |
| checkTokens(token, expectedTokens); |
| } |
| |
| private void checkTokens(Token firstToken, Token... expectedTokens) { |
| assertNotNull(firstToken); |
| Token token = firstToken; |
| for (int i = 0; i < expectedTokens.length; i++) { |
| Token expectedToken = expectedTokens[i]; |
| assertEquals("Wrong type for token " + i, expectedToken.getType(), token.getType()); |
| assertEquals("Wrong offset for token " + i, expectedToken.getOffset(), token.getOffset()); |
| assertEquals("Wrong length for token " + i, expectedToken.getLength(), token.getLength()); |
| assertEquals("Wrong lexeme for token " + i, expectedToken.getLexeme(), token.getLexeme()); |
| token = token.getNext(); |
| assertNotNull(token); |
| } |
| assertEquals(TokenType.EOF, token.getType()); |
| } |
| |
| private Token scan(String source) { |
| GatheringErrorListener listener = new GatheringErrorListener(); |
| Token token = scanWithListener(source, listener); |
| listener.assertNoErrors(); |
| return token; |
| } |
| |
| private Token scanWithListener(String source, GatheringErrorListener listener) { |
| Scanner scanner = new Scanner(null, new CharSequenceReader(source), listener); |
| Token result = scanner.tokenize(); |
| listener.setLineInfo(new TestSource(), scanner.getLineStarts()); |
| return result; |
| } |
| } |