scanner: remove code for unused tokens, add EOL, revise TestInit
diff --git a/scanner/scanner.go b/scanner/scanner.go
index c0ef4e6..54a1930 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -45,7 +45,6 @@
 	offset     int  // character offset
 	rdOffset   int  // reading offset (position after current character)
 	lineOffset int  // current line offset
-	insertSemi bool // insert a semicolon before next newline
 
 	// public state - ok to modify
 	ErrorCount int // number of errors encountered
@@ -90,8 +89,7 @@
 type Mode uint
 
 const (
-	ScanComments    Mode = 1 << iota // return comments as COMMENT tokens
-	dontInsertSemis                  // do not automatically insert semicolons - for testing only
+	ScanComments Mode = 1 << iota // return comments as COMMENT tokens
 )
 
 // Init prepares the scanner s to tokenize the text src by setting the
@@ -124,7 +122,6 @@
 	s.offset = 0
 	s.rdOffset = 0
 	s.lineOffset = 0
-	s.insertSemi = false
 	s.ErrorCount = 0
 
 	s.next()
@@ -254,7 +251,6 @@
 func (s *Scanner) scanEscape(quote rune) {
 	offs := s.offset
 
-	var i, base, max uint32
 	switch s.ch {
 	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
 		s.next()
@@ -265,34 +261,6 @@
 	return
 }
 
-func (s *Scanner) scanChar() string {
-	// '\'' opening already consumed
-	offs := s.offset - 1
-
-	n := 0
-	for s.ch != '\'' {
-		ch := s.ch
-		n++
-		s.next()
-		if ch == '\n' || ch < 0 {
-			s.error(offs, "character literal not terminated")
-			n = 1
-			break
-		}
-		if ch == '\\' {
-			s.scanEscape('\'')
-		}
-	}
-
-	s.next()
-
-	if n != 1 {
-		s.error(offs, "illegal character literal")
-	}
-
-	return string(s.src[offs:s.offset])
-}
-
 func (s *Scanner) scanString() string {
 	// '"' opening already consumed
 	offs := s.offset - 1
@@ -354,53 +322,11 @@
 }
 
 func (s *Scanner) skipWhitespace() {
-	for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' && !s.insertSemi || s.ch == '\r' {
+	for s.ch == ' ' || s.ch == '\t' || s.ch == '\n' || s.ch == '\r' {
 		s.next()
 	}
 }
 
-// Helper functions for scanning multi-byte tokens such as >> += >>= .
-// Different routines recognize different length tok_i based on matches
-// of ch_i. If a token ends in '=', the result is tok1 or tok3
-// respectively. Otherwise, the result is tok0 if there was no other
-// matching character, or tok2 if the matching character was ch2.
-
-func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token {
-	if s.ch == '=' {
-		s.next()
-		return tok1
-	}
-	return tok0
-}
-
-func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token {
-	if s.ch == '=' {
-		s.next()
-		return tok1
-	}
-	if s.ch == ch2 {
-		s.next()
-		return tok2
-	}
-	return tok0
-}
-
-func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token {
-	if s.ch == '=' {
-		s.next()
-		return tok1
-	}
-	if s.ch == ch2 {
-		s.next()
-		if s.ch == '=' {
-			s.next()
-			return tok3
-		}
-		return tok2
-	}
-	return tok0
-}
-
 // Scan scans the next token and returns the token position, the token,
 // and its literal string if applicable. The source end is indicated by
 // token.EOF.
@@ -438,143 +364,43 @@
 	pos = s.file.Pos(s.offset)
 
 	// determine token value
-	insertSemi := false
 	switch ch := s.ch; {
 	case isLetter(ch):
 		lit = s.scanIdentifier()
-		tok = token.IDENT // TODO string
-		insertSemi = true
+		tok = token.IDENT
 	default:
 		s.next() // always make progress
 		switch ch {
 		case -1:
-			if s.insertSemi {
-				s.insertSemi = false // EOF consumed
-				return pos, token.SEMICOLON, "\n"
-			}
 			tok = token.EOF
 		case '\n':
-			// we only reach here if s.insertSemi was
-			// set in the first place and exited early
-			// from s.skipWhitespace()
-			s.insertSemi = false // newline consumed
-			return pos, token.SEMICOLON, "\n"
+			return pos, token.EOL, "\n"
 		case '"':
-			insertSemi = true
 			tok = token.STRING
 			lit = s.scanString()
-		case '\'':
-			insertSemi = true
-			tok = token.CHAR
-			lit = s.scanChar()
 		case '`':
-			insertSemi = true
 			tok = token.STRING
 			lit = s.scanRawString()
-		case ':':
-			tok = s.switch2(token.COLON, token.DEFINE)
-		case '.':
-			if digitVal(s.ch) < 10 {
-				insertSemi = true
-				tok, lit = s.scanNumber(true)
-			} else if s.ch == '.' {
-				s.next()
-				if s.ch == '.' {
-					s.next()
-					tok = token.ELLIPSIS
-				}
-			} else {
-				tok = token.PERIOD
-			}
-		case ',':
-			tok = token.COMMA
-		case ';':
-			tok = token.SEMICOLON
-			lit = ";"
-		case '(':
-			tok = token.LPAREN
-		case ')':
-			insertSemi = true
-			tok = token.RPAREN
 		case '[':
 			tok = token.LBRACK
 		case ']':
-			insertSemi = true
 			tok = token.RBRACK
-		case '{':
-			tok = token.LBRACE
-		case '}':
-			insertSemi = true
-			tok = token.RBRACE
-		case '+':
-			tok = s.switch3(token.ADD, token.ADD_ASSIGN, '+', token.INC)
-			if tok == token.INC {
-				insertSemi = true
+		case ';', '#':
+			// comment
+			lit = s.scanComment()
+			if s.mode&ScanComments == 0 {
+				// skip comment
+				goto scanAgain
 			}
-		case '-':
-			tok = s.switch3(token.SUB, token.SUB_ASSIGN, '-', token.DEC)
-			if tok == token.DEC {
-				insertSemi = true
-			}
-		case '*':
-			tok = s.switch2(token.MUL, token.MUL_ASSIGN)
-		case '/':
-			if s.ch == '/' || s.ch == '*' {
-				// comment
-				if s.insertSemi && s.findLineEnd() {
-					// reset position to the beginning of the comment
-					s.ch = '/'
-					s.offset = s.file.Offset(pos)
-					s.rdOffset = s.offset + 1
-					s.insertSemi = false // newline consumed
-					return pos, token.SEMICOLON, "\n"
-				}
-				lit = s.scanComment()
-				if s.mode&ScanComments == 0 {
-					// skip comment
-					s.insertSemi = false // newline consumed
-					goto scanAgain
-				}
-				tok = token.COMMENT
-			} else {
-				tok = s.switch2(token.QUO, token.QUO_ASSIGN)
-			}
-		case '%':
-			tok = s.switch2(token.REM, token.REM_ASSIGN)
-		case '^':
-			tok = s.switch2(token.XOR, token.XOR_ASSIGN)
-		case '<':
-			if s.ch == '-' {
-				s.next()
-				tok = token.ARROW
-			} else {
-				tok = s.switch4(token.LSS, token.LEQ, '<', token.SHL, token.SHL_ASSIGN)
-			}
-		case '>':
-			tok = s.switch4(token.GTR, token.GEQ, '>', token.SHR, token.SHR_ASSIGN)
+			tok = token.COMMENT
 		case '=':
-			tok = s.switch2(token.ASSIGN, token.EQL)
-		case '!':
-			tok = s.switch2(token.NOT, token.NEQ)
-		case '&':
-			if s.ch == '^' {
-				s.next()
-				tok = s.switch2(token.AND_NOT, token.AND_NOT_ASSIGN)
-			} else {
-				tok = s.switch3(token.AND, token.AND_ASSIGN, '&', token.LAND)
-			}
-		case '|':
-			tok = s.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
+			tok = token.ASSIGN
 		default:
 			s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
-			insertSemi = s.insertSemi // preserve insertSemi info
 			tok = token.ILLEGAL
 			lit = string(ch)
 		}
 	}
-	if s.mode&dontInsertSemis == 0 {
-		s.insertSemi = insertSemi
-	}
 
 	return
 }
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
index a903bad..ff38539 100644
--- a/scanner/scanner_test.go
+++ b/scanner/scanner_test.go
@@ -21,7 +21,6 @@
 	special = iota
 	literal
 	operator
-	keyword
 )
 
 func tokenclass(tok token.Token) int {
@@ -30,8 +29,6 @@
 		return literal
 	case tok.IsOperator():
 		return operator
-	case tok.IsKeyword():
-		return keyword
 	}
 	return special
 }
@@ -52,34 +49,6 @@
 	{token.IDENT, "a۰۱۸", literal},
 	{token.IDENT, "foo६४", literal},
 	{token.IDENT, "bar9876", literal},
-	{token.INT, "0", literal},
-	{token.INT, "1", literal},
-	{token.INT, "123456789012345678890", literal},
-	{token.INT, "01234567", literal},
-	{token.INT, "0xcafebabe", literal},
-	{token.FLOAT, "0.", literal},
-	{token.FLOAT, ".0", literal},
-	{token.FLOAT, "3.14159265", literal},
-	{token.FLOAT, "1e0", literal},
-	{token.FLOAT, "1e+100", literal},
-	{token.FLOAT, "1e-100", literal},
-	{token.FLOAT, "2.71828e-1000", literal},
-	{token.IMAG, "0i", literal},
-	{token.IMAG, "1i", literal},
-	{token.IMAG, "012345678901234567889i", literal},
-	{token.IMAG, "123456789012345678890i", literal},
-	{token.IMAG, "0.i", literal},
-	{token.IMAG, ".0i", literal},
-	{token.IMAG, "3.14159265i", literal},
-	{token.IMAG, "1e0i", literal},
-	{token.IMAG, "1e+100i", literal},
-	{token.IMAG, "1e-100i", literal},
-	{token.IMAG, "2.71828e-1000i", literal},
-	{token.CHAR, "'a'", literal},
-	{token.CHAR, "'\\000'", literal},
-	{token.CHAR, "'\\xFF'", literal},
-	{token.CHAR, "'\\uff16'", literal},
-	{token.CHAR, "'\\U0000ff16'", literal},
 	{token.STRING, "`foobar`", literal},
 	{token.STRING, "`" + `foo
 	                        bar` +
@@ -90,92 +59,9 @@
 	{token.STRING, "`foo\r\nbar`", literal},
 
 	// Operators and delimiters
-	{token.ADD, "+", operator},
-	{token.SUB, "-", operator},
-	{token.MUL, "*", operator},
-	{token.QUO, "/", operator},
-	{token.REM, "%", operator},
-
-	{token.AND, "&", operator},
-	{token.OR, "|", operator},
-	{token.XOR, "^", operator},
-	{token.SHL, "<<", operator},
-	{token.SHR, ">>", operator},
-	{token.AND_NOT, "&^", operator},
-
-	{token.ADD_ASSIGN, "+=", operator},
-	{token.SUB_ASSIGN, "-=", operator},
-	{token.MUL_ASSIGN, "*=", operator},
-	{token.QUO_ASSIGN, "/=", operator},
-	{token.REM_ASSIGN, "%=", operator},
-
-	{token.AND_ASSIGN, "&=", operator},
-	{token.OR_ASSIGN, "|=", operator},
-	{token.XOR_ASSIGN, "^=", operator},
-	{token.SHL_ASSIGN, "<<=", operator},
-	{token.SHR_ASSIGN, ">>=", operator},
-	{token.AND_NOT_ASSIGN, "&^=", operator},
-
-	{token.LAND, "&&", operator},
-	{token.LOR, "||", operator},
-	{token.ARROW, "<-", operator},
-	{token.INC, "++", operator},
-	{token.DEC, "--", operator},
-
-	{token.EQL, "==", operator},
-	{token.LSS, "<", operator},
-	{token.GTR, ">", operator},
 	{token.ASSIGN, "=", operator},
-	{token.NOT, "!", operator},
-
-	{token.NEQ, "!=", operator},
-	{token.LEQ, "<=", operator},
-	{token.GEQ, ">=", operator},
-	{token.DEFINE, ":=", operator},
-	{token.ELLIPSIS, "...", operator},
-
-	{token.LPAREN, "(", operator},
 	{token.LBRACK, "[", operator},
-	{token.LBRACE, "{", operator},
-	{token.COMMA, ",", operator},
-	{token.PERIOD, ".", operator},
-
-	{token.RPAREN, ")", operator},
 	{token.RBRACK, "]", operator},
-	{token.RBRACE, "}", operator},
-	{token.SEMICOLON, ";", operator},
-	{token.COLON, ":", operator},
-
-	// Keywords
-	{token.BREAK, "break", keyword},
-	{token.CASE, "case", keyword},
-	{token.CHAN, "chan", keyword},
-	{token.CONST, "const", keyword},
-	{token.CONTINUE, "continue", keyword},
-
-	{token.DEFAULT, "default", keyword},
-	{token.DEFER, "defer", keyword},
-	{token.ELSE, "else", keyword},
-	{token.FALLTHROUGH, "fallthrough", keyword},
-	{token.FOR, "for", keyword},
-
-	{token.FUNC, "func", keyword},
-	{token.GO, "go", keyword},
-	{token.GOTO, "goto", keyword},
-	{token.IF, "if", keyword},
-	{token.IMPORT, "import", keyword},
-
-	{token.INTERFACE, "interface", keyword},
-	{token.MAP, "map", keyword},
-	{token.PACKAGE, "package", keyword},
-	{token.RANGE, "range", keyword},
-	{token.RETURN, "return", keyword},
-
-	{token.SELECT, "select", keyword},
-	{token.STRUCT, "struct", keyword},
-	{token.SWITCH, "switch", keyword},
-	{token.TYPE, "type", keyword},
-	{token.VAR, "var", keyword},
 }
 
 const whitespace = "  \t  \n\n\n" // to separate tokens
@@ -228,7 +114,7 @@
 
 	// verify scan
 	var s Scanner
-	s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments|dontInsertSemis)
+	s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments)
 	index := 0
 	// epos is the expected position
 	epos := token.Position{
@@ -287,39 +173,6 @@
 	}
 }
 
-func checkSemi(t *testing.T, line string, mode Mode) {
-	var S Scanner
-	file := fset.AddFile("TestSemis", fset.Base(), len(line))
-	S.Init(file, []byte(line), nil, mode)
-	pos, tok, lit := S.Scan()
-	for tok != token.EOF {
-		if tok == token.ILLEGAL {
-			// the illegal token literal indicates what
-			// kind of semicolon literal to expect
-			semiLit := "\n"
-			if lit[0] == '#' {
-				semiLit = ";"
-			}
-			// next token must be a semicolon
-			semiPos := file.Position(pos)
-			semiPos.Offset++
-			semiPos.Column++
-			pos, tok, lit = S.Scan()
-			if tok == token.SEMICOLON {
-				if lit != semiLit {
-					t.Errorf(`bad literal for %q: got %q, expected %q`, line, lit, semiLit)
-				}
-				checkPos(t, line, pos, semiPos)
-			} else {
-				t.Errorf("bad token for %q: got %s, expected ;", line, tok)
-			}
-		} else if tok == token.SEMICOLON {
-			t.Errorf("bad token for %q: got ;, expected no ;", line)
-		}
-		pos, tok, lit = S.Scan()
-	}
-}
-
 var lines = []string{
 	// # indicates a semicolon present in the source
 	// $ indicates an automatically inserted semicolon
@@ -443,20 +296,6 @@
 	"package main$",
 }
 
-func TestSemis(t *testing.T) {
-	for _, line := range lines {
-		checkSemi(t, line, 0)
-		checkSemi(t, line, ScanComments)
-
-		// if the input ended in newlines, the input must tokenize the
-		// same with or without those newlines
-		for i := len(line) - 1; i >= 0 && line[i] == '\n'; i-- {
-			checkSemi(t, line[0:i], 0)
-			checkSemi(t, line[0:i], ScanComments)
-		}
-	}
-}
-
 type segment struct {
 	srcline  string // a line of source text
 	filename string // filename for current token
@@ -507,7 +346,7 @@
 	// verify scan
 	var S Scanner
 	file := fset.AddFile(filepath.Join("dir", "TestLineComments"), fset.Base(), len(src))
-	S.Init(file, []byte(src), nil, dontInsertSemis)
+	S.Init(file, []byte(src), nil, 0)
 	for _, s := range segs {
 		p, _, lit := S.Scan()
 		pos := file.Position(p)
@@ -529,29 +368,29 @@
 	var s Scanner
 
 	// 1st init
-	src1 := "if true { }"
+	src1 := "\nname = value"
 	f1 := fset.AddFile("src1", fset.Base(), len(src1))
-	s.Init(f1, []byte(src1), nil, dontInsertSemis)
+	s.Init(f1, []byte(src1), nil, 0)
 	if f1.Size() != len(src1) {
 		t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
 	}
-	s.Scan()              // if
-	s.Scan()              // true
-	_, tok, _ := s.Scan() // {
-	if tok != token.LBRACE {
-		t.Errorf("bad token: got %s, expected %s", tok, token.LBRACE)
+	s.Scan()              // \n
+	s.Scan()              // name
+	_, tok, _ := s.Scan() // =
+	if tok != token.ASSIGN {
+		t.Errorf("bad token: got %s, expected %s", tok, token.ASSIGN)
 	}
 
 	// 2nd init
-	src2 := "go true { ]"
+	src2 := "[section]"
 	f2 := fset.AddFile("src2", fset.Base(), len(src2))
-	s.Init(f2, []byte(src2), nil, dontInsertSemis)
+	s.Init(f2, []byte(src2), nil, 0)
 	if f2.Size() != len(src2) {
 		t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
 	}
-	_, tok, _ = s.Scan() // go
-	if tok != token.GO {
-		t.Errorf("bad token: got %s, expected %s", tok, token.GO)
+	_, tok, _ = s.Scan() // [
+	if tok != token.LBRACK {
+		t.Errorf("bad token: got %s, expected %s", tok, token.LBRACK)
 	}
 
 	if s.ErrorCount != 0 {
@@ -573,7 +412,7 @@
 	eh := func(pos token.Position, msg string) { list.Add(pos, msg) }
 
 	var s Scanner
-	s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, dontInsertSemis)
+	s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, 0)
 	for {
 		if _, tok, _ := s.Scan(); tok == token.EOF {
 			break
@@ -616,7 +455,7 @@
 		h.msg = msg
 		h.pos = pos
 	}
-	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments|dontInsertSemis)
+	s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments)
 	_, tok0, _ := s.Scan()
 	_, tok1, _ := s.Scan()
 	if tok0 != tok {
@@ -649,27 +488,12 @@
 	{"\a", token.ILLEGAL, 0, "illegal character U+0007"},
 	{`#`, token.ILLEGAL, 0, "illegal character U+0023 '#'"},
 	{`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
-	{`' '`, token.CHAR, 0, ""},
-	{`''`, token.CHAR, 0, "illegal character literal"},
-	{`'\8'`, token.CHAR, 2, "unknown escape sequence"},
-	{`'\08'`, token.CHAR, 3, "illegal character in escape sequence"},
-	{`'\x0g'`, token.CHAR, 4, "illegal character in escape sequence"},
-	{`'\Uffffffff'`, token.CHAR, 2, "escape sequence is invalid Unicode code point"},
-	{`'`, token.CHAR, 0, "character literal not terminated"},
 	{`""`, token.STRING, 0, ""},
 	{`"`, token.STRING, 0, "string not terminated"},
 	{"``", token.STRING, 0, ""},
 	{"`", token.STRING, 0, "string not terminated"},
 	{"/**/", token.COMMENT, 0, ""},
 	{"/*", token.COMMENT, 0, "comment not terminated"},
-	{"077", token.INT, 0, ""},
-	{"078.", token.FLOAT, 0, ""},
-	{"07801234567.", token.FLOAT, 0, ""},
-	{"078e0", token.FLOAT, 0, ""},
-	{"078", token.INT, 0, "illegal octal number"},
-	{"07800000009", token.INT, 0, "illegal octal number"},
-	{"0x", token.INT, 0, "illegal hexadecimal number"},
-	{"0X", token.INT, 0, "illegal hexadecimal number"},
 	{"\"abc\x00def\"", token.STRING, 4, "illegal character NUL"},
 	{"\"abc\x80def\"", token.STRING, 4, "illegal UTF-8 encoding"},
 }