blob: 2188da0630d49bd87a82b6f2a1c41e4d676c5da6 [file] [log] [blame]
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package scanner
import (
"os"
"strings"
"testing"
)
import (
"code.google.com/p/gcfg/token"
)
var fset = token.NewFileSet()
const /* class */ (
special = iota
literal
operator
)
func tokenclass(tok token.Token) int {
switch {
case tok.IsLiteral():
return literal
case tok.IsOperator():
return operator
}
return special
}
type elt struct {
tok token.Token
lit string
class int
pre string
suf string
}
var tokens = [...]elt{
// Special tokens
{token.COMMENT, "; a comment", special, "", "\n"},
{token.COMMENT, "# a comment", special, "", "\n"},
// Operators and delimiters
{token.ASSIGN, "=", operator, "", "value"},
{token.LBRACK, "[", operator, "", ""},
{token.RBRACK, "]", operator, "", ""},
{token.EOL, "\n", operator, "", ""},
// Identifiers
{token.IDENT, "foobar", literal, "", ""},
{token.IDENT, "a۰۱۸", literal, "", ""},
{token.IDENT, "foo६४", literal, "", ""},
{token.IDENT, "bar9876", literal, "", ""},
{token.IDENT, "foo-bar", literal, "", ""},
// String literals (subsection names)
{token.STRING, `"foobar"`, literal, "", ""},
{token.STRING, `"\n"`, literal, "", ""},
{token.STRING, `"\""`, literal, "", ""},
// String literals (values)
{token.STRING, `"foobar"`, literal, "=", ""},
{token.STRING, `"foo\nbar"`, literal, "=", ""},
{token.STRING, `"foo\"bar"`, literal, "=", ""},
{token.STRING, `"foo\\bar"`, literal, "=", ""},
{token.STRING, `"foobar"`, literal, "=", ""},
{token.STRING, `"foobar"`, literal, "= ", ""},
{token.STRING, `"foobar"`, literal, "=", "\n"},
{token.STRING, `"foobar"`, literal, "=", ";"},
{token.STRING, `"foobar"`, literal, "=", " ;"},
{token.STRING, `"foobar"`, literal, "=", "#"},
{token.STRING, `"foobar"`, literal, "=", " #"},
{token.STRING, "foobar", literal, "=", ""},
{token.STRING, "foobar", literal, "= ", ""},
{token.STRING, "foobar", literal, "=", " "},
{token.STRING, `"foo" "bar"`, literal, "=", " "},
{token.STRING, "foo\\\nbar", literal, "=", ""},
{token.STRING, "foo\\\r\nbar", literal, "=", ""},
}
const whitespace = " \t \n\n\n" // to separate tokens
var source = func() []byte {
var src []byte
for _, t := range tokens {
src = append(src, t.pre...)
src = append(src, t.lit...)
src = append(src, t.suf...)
src = append(src, whitespace...)
}
return src
}()
func newlineCount(s string) int {
n := 0
for i := 0; i < len(s); i++ {
if s[i] == '\n' {
n++
}
}
return n
}
func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
pos := fset.Position(p)
if pos.Filename != expected.Filename {
t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
}
if pos.Offset != expected.Offset {
t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
}
if pos.Line != expected.Line {
t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
}
if pos.Column != expected.Column {
t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
}
}
// Verify that calling Scan() provides the correct results.
func TestScan(t *testing.T) {
// make source
src_linecount := newlineCount(string(source))
whitespace_linecount := newlineCount(whitespace)
// error handler
eh := func(_ token.Position, msg string) {
t.Errorf("error handler called (msg = %s)", msg)
}
// verify scan
var s Scanner
s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments)
index := 0
// epos is the expected position
epos := token.Position{
Filename: "",
Offset: 0,
Line: 1,
Column: 1,
}
for {
pos, tok, lit := s.Scan()
if lit == "" {
// no literal value for non-literal tokens
lit = tok.String()
}
e := elt{token.EOF, "", special, "", ""}
if index < len(tokens) {
e = tokens[index]
}
if tok == token.EOF {
lit = "<EOF>"
epos.Line = src_linecount
epos.Column = 2
}
if strings.ContainsRune(e.pre, '=') {
epos.Column = 1
checkPos(t, lit, pos, epos)
if tok != token.ASSIGN {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.ASSIGN)
}
pos, tok, lit = s.Scan()
}
epos.Offset += len(e.pre)
if tok != token.EOF {
epos.Column = 1 + len(e.pre)
}
checkPos(t, lit, pos, epos)
if tok != e.tok {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok)
}
if e.tok.IsLiteral() {
// no CRs in value string literals
elit := e.lit
if strings.ContainsRune(e.pre, '=') {
elit = string(stripCR([]byte(elit)))
epos.Offset += len(e.lit) - len(lit) // correct position
}
if lit != elit {
t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit)
}
}
if tokenclass(tok) != e.class {
t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
}
epos.Offset += len(lit) + len(e.suf) + len(whitespace)
epos.Line += newlineCount(lit) + newlineCount(e.suf) + whitespace_linecount
index++
if tok == token.EOF {
break
}
if e.suf == "value" {
pos, tok, lit = s.Scan()
if tok != token.STRING {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.STRING)
}
} else if strings.ContainsRune(e.suf, ';') || strings.ContainsRune(e.suf, '#') {
pos, tok, lit = s.Scan()
if tok != token.COMMENT {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.COMMENT)
}
}
// skip EOLs
for i := 0; i < whitespace_linecount+newlineCount(e.suf); i++ {
pos, tok, lit = s.Scan()
if tok != token.EOL {
t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.EOL)
}
}
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func TestScanValStringEOF(t *testing.T) {
var s Scanner
src := "= value"
f := fset.AddFile("src", fset.Base(), len(src))
s.Init(f, []byte(src), nil, 0)
s.Scan() // =
s.Scan() // value
_, tok, _ := s.Scan() // EOF
if tok != token.EOF {
t.Errorf("bad token: got %s, expected %s", tok, token.EOF)
}
if s.ErrorCount > 0 {
t.Error("scanning error")
}
}
// Verify that initializing the same scanner more then once works correctly.
func TestInit(t *testing.T) {
var s Scanner
// 1st init
src1 := "\nname = value"
f1 := fset.AddFile("src1", fset.Base(), len(src1))
s.Init(f1, []byte(src1), nil, 0)
if f1.Size() != len(src1) {
t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
}
s.Scan() // \n
s.Scan() // name
_, tok, _ := s.Scan() // =
if tok != token.ASSIGN {
t.Errorf("bad token: got %s, expected %s", tok, token.ASSIGN)
}
// 2nd init
src2 := "[section]"
f2 := fset.AddFile("src2", fset.Base(), len(src2))
s.Init(f2, []byte(src2), nil, 0)
if f2.Size() != len(src2) {
t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
}
_, tok, _ = s.Scan() // [
if tok != token.LBRACK {
t.Errorf("bad token: got %s, expected %s", tok, token.LBRACK)
}
if s.ErrorCount != 0 {
t.Errorf("found %d errors", s.ErrorCount)
}
}
func TestStdErrorHandler(t *testing.T) {
const src = "@\n" + // illegal character, cause an error
"@ @\n" // two errors on the same line
var list ErrorList
eh := func(pos token.Position, msg string) { list.Add(pos, msg) }
var s Scanner
s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, 0)
for {
if _, tok, _ := s.Scan(); tok == token.EOF {
break
}
}
if len(list) != s.ErrorCount {
t.Errorf("found %d errors, expected %d", len(list), s.ErrorCount)
}
if len(list) != 3 {
t.Errorf("found %d raw errors, expected 3", len(list))
PrintError(os.Stderr, list)
}
list.Sort()
if len(list) != 3 {
t.Errorf("found %d sorted errors, expected 3", len(list))
PrintError(os.Stderr, list)
}
list.RemoveMultiples()
if len(list) != 2 {
t.Errorf("found %d one-per-line errors, expected 2", len(list))
PrintError(os.Stderr, list)
}
}
type errorCollector struct {
cnt int // number of errors encountered
msg string // last error message encountered
pos token.Position // last error position encountered
}
func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s Scanner
var h errorCollector
eh := func(pos token.Position, msg string) {
h.cnt++
h.msg = msg
h.pos = pos
}
s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments)
if src[0] == '=' {
_, _, _ = s.Scan()
}
_, tok0, _ := s.Scan()
_, tok1, _ := s.Scan()
if tok0 != tok {
t.Errorf("%q: got %s, expected %s", src, tok0, tok)
}
if tok1 != token.EOF {
t.Errorf("%q: got %s, expected EOF", src, tok1)
}
cnt := 0
if err != "" {
cnt = 1
}
if h.cnt != cnt {
t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
}
if h.msg != err {
t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
}
if h.pos.Offset != pos {
t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
}
}
var errors = []struct {
src string
tok token.Token
pos int
err string
}{
{"\a", token.ILLEGAL, 0, "illegal character U+0007"},
{"/", token.ILLEGAL, 0, "illegal character U+002F '/'"},
{"_", token.ILLEGAL, 0, "illegal character U+005F '_'"},
{`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
{`""`, token.STRING, 0, ""},
{`"`, token.STRING, 0, "string not terminated"},
{"\"\n", token.STRING, 0, "string not terminated"},
{`="`, token.STRING, 1, "string not terminated"},
{"=\"\n", token.STRING, 1, "string not terminated"},
{"=\\", token.STRING, 1, "unquoted '\\' must be followed by new line"},
{"=\\\r", token.STRING, 1, "unquoted '\\' must be followed by new line"},
{`"\z"`, token.STRING, 2, "unknown escape sequence"},
{`"\a"`, token.STRING, 2, "unknown escape sequence"},
{`"\b"`, token.STRING, 2, "unknown escape sequence"},
{`"\f"`, token.STRING, 2, "unknown escape sequence"},
{`"\r"`, token.STRING, 2, "unknown escape sequence"},
{`"\t"`, token.STRING, 2, "unknown escape sequence"},
{`"\v"`, token.STRING, 2, "unknown escape sequence"},
{`"\0"`, token.STRING, 2, "unknown escape sequence"},
}
func TestScanErrors(t *testing.T) {
for _, e := range errors {
checkError(t, e.src, e.tok, e.pos, e.err)
}
}
func BenchmarkScan(b *testing.B) {
b.StopTimer()
fset := token.NewFileSet()
file := fset.AddFile("", fset.Base(), len(source))
var s Scanner
b.StartTimer()
for i := b.N - 1; i >= 0; i-- {
s.Init(file, source, nil, ScanComments)
for {
_, tok, _ := s.Scan()
if tok == token.EOF {
break
}
}
}
}