token,scanner: EOL is a delimiter, no literal for EOL
diff --git a/scanner/example_test.go b/scanner/example_test.go
index 31dd03e..05eadf5 100644
--- a/scanner/example_test.go
+++ b/scanner/example_test.go
@@ -30,17 +30,17 @@
if tok == token.EOF {
break
}
- fmt.Printf("%s\t%s\t%q\n", fset.Position(pos), tok, lit)
+ fmt.Printf("%s\t%q\t%q\n", fset.Position(pos), tok, lit)
}
// output:
- // 1:1 [ ""
- // 1:2 IDENT "profile"
- // 1:10 STRING "\"A\""
- // 1:13 ] ""
- // 1:14 EOL "\n"
- // 2:1 IDENT "color"
- // 2:7 = ""
- // 2:9 STRING "blue"
- // 2:14 COMMENT "; Comment"
+ // 1:1 "[" ""
+ // 1:2 "IDENT" "profile"
+ // 1:10 "STRING" "\"A\""
+ // 1:13 "]" ""
+ // 1:14 "\n" ""
+ // 2:1 "IDENT" "color"
+ // 2:7 "=" ""
+ // 2:9 "STRING" "blue"
+ // 2:14 "COMMENT" "; Comment"
}
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 47b5f35..bb16182 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -306,7 +306,7 @@
case -1:
tok = token.EOF
case '\n':
- return pos, token.EOL, "\n"
+ tok = token.EOL
case '"':
tok = token.STRING
lit = s.scanString()
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
index 28414b0..f1feeb3 100644
--- a/scanner/scanner_test.go
+++ b/scanner/scanner_test.go
@@ -42,15 +42,14 @@
var tokens = [...]elt{
// Special tokens
- {token.EOL, "\n", special, "", ""},
-
- {token.COMMENT, "; a comment \n", special, "", ""},
- {token.COMMENT, "# a comment \n", special, "", ""},
+ {token.COMMENT, "; a comment", special, "", "\n"},
+ {token.COMMENT, "# a comment", special, "", "\n"},
// Operators and delimiters
{token.ASSIGN, "=", operator, "", "value"},
{token.LBRACK, "[", operator, "", ""},
{token.RBRACK, "]", operator, "", ""},
+ {token.EOL, "\n", operator, "", ""},
// Identifiers
{token.IDENT, "foobar", literal, "", ""},
@@ -143,9 +142,8 @@
Line: 1,
Column: 1,
}
- pos, tok, lit := s.Scan()
-outer:
for {
+ pos, tok, lit := s.Scan()
if lit == "" {
// no literal value for non-literal tokens
lit = tok.String()
@@ -159,14 +157,19 @@
epos.Line = src_linecount
epos.Column = 2
}
- checkPos(t, lit, pos, epos)
if strings.ContainsRune(e.pre, '=') {
+ epos.Column = 1
+ checkPos(t, lit, pos, epos)
if tok != token.ASSIGN {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.ASSIGN)
}
- epos.Offset += len(e.pre)
pos, tok, lit = s.Scan()
}
+ epos.Offset += len(e.pre)
+ if tok != token.EOF {
+ epos.Column = 1 + len(e.pre)
+ }
+ checkPos(t, lit, pos, epos)
if tok != e.tok {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok)
}
@@ -184,21 +187,12 @@
if tokenclass(tok) != e.class {
t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
}
- epos.Offset += len(lit) + len(whitespace)
- epos.Line += newlineCount(lit) + whitespace_linecount
- if tok == token.COMMENT && strings.HasSuffix(e.lit, "\n") {
- epos.Offset++
- epos.Line++
- }
+ epos.Offset += len(lit) + len(e.suf) + len(whitespace)
+ epos.Line += newlineCount(lit) + newlineCount(e.suf) + whitespace_linecount
index++
if tok == token.EOF {
break
}
- // adjust for suffix
- epos.Offset += len(e.suf)
- if strings.ContainsRune(e.suf, '\n') {
- epos.Line++
- }
if e.suf == "value" {
pos, tok, lit = s.Scan()
if tok != token.STRING {
@@ -211,10 +205,10 @@
}
}
// skip EOLs
- for {
+ for i := 0; i < whitespace_linecount+newlineCount(e.suf); i++ {
pos, tok, lit = s.Scan()
if tok != token.EOL {
- continue outer
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.EOL)
}
}
}
diff --git a/token/token.go b/token/token.go
index c9528a1..ba7f770 100644
--- a/token/token.go
+++ b/token/token.go
@@ -16,7 +16,6 @@
const (
// Special tokens
ILLEGAL Token = iota
- EOL
EOF
COMMENT
@@ -32,13 +31,13 @@
ASSIGN // =
LBRACK // [
RBRACK // ]
+ EOL // \n
operator_end
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
- EOL: "EOL",
EOF: "EOF",
COMMENT: "COMMENT",
@@ -48,6 +47,7 @@
ASSIGN: "=",
LBRACK: "[",
RBRACK: "]",
+ EOL: "\n",
}
// String returns the string corresponding to the token tok.
