Merge branch 'parser'
diff --git a/LICENSE b/LICENSE
index e5a449f..8fe3e90 100644
--- a/LICENSE
+++ b/LICENSE
@@ -22,3 +22,36 @@
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+Portions of gcfg's source code have been derived from Go, and are
+covered by the following license:
+----------------------------------------------------------------------
+
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+ * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/gcfg.go b/gcfg.go
index b1d1a2c..39ee5b8 100644
--- a/gcfg.go
+++ b/gcfg.go
@@ -35,7 +35,6 @@
// - reading
// - define internal representation structure
// - support multi-value variables
-// - non-regexp based parser
// - support partially quoted strings
// - support escaping in strings
// - support multiple inputs (readers, strings, files)
@@ -47,9 +46,7 @@
// - support matching on unique prefix (?)
// - writing gcfg files
// - error handling
-// - include error context
-// - more helpful error messages
-// - error types / codes?
+// - make error context accessible programmatically?
// - limit input size?
// - move TODOs to issue tracker (eventually)
//
diff --git a/read.go b/read.go
index a2b5220..780e492 100644
--- a/read.go
+++ b/read.go
@@ -1,99 +1,95 @@
package gcfg
import (
- "bufio"
"fmt"
"io"
+ "io/ioutil"
"os"
- "reflect"
- "regexp"
"strings"
)
-var (
- reCmnt = regexp.MustCompile(`^([^;#"]*)[;#].*$`)
- reCmntQ = regexp.MustCompile(`^([^;#"]*"[^"]*"[^;#"]*)[;#].*$`)
- reBlank = regexp.MustCompile(`^\s*$`)
- reSect = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*\]\s*$`)
- reSectSub = regexp.MustCompile(`^\s*\[\s*([^"\s]*)\s*"([^"]+)"\s*\]\s*$`)
- reVar = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*([^"\s]*)\s*$`)
- reVarQ = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*"([^"\n\\]*)"\s*$`)
- reVarDflt = regexp.MustCompile(`^\s*\b(.*)\b\s*$`)
+import (
+ "code.google.com/p/gcfg/scanner"
+ "code.google.com/p/gcfg/token"
)
-const (
- // Default value string in case a value for a variable isn't provided.
- defaultValue = "true"
-)
-
-func fieldFold(v reflect.Value, name string) reflect.Value {
- n := strings.Replace(name, "-", "_", -1)
- return v.FieldByNameFunc(func(fieldName string) bool {
- return strings.EqualFold(n, fieldName)
- })
+func unquote(s string) string {
+ if s != "" && s[0] == '"' {
+ return s[1 : len(s)-1] // FIXME
+ }
+ return s
}
-func set(cfg interface{}, sect, sub, name, value string) error {
- vPCfg := reflect.ValueOf(cfg)
- if vPCfg.Kind() != reflect.Ptr || vPCfg.Elem().Kind() != reflect.Struct {
- panic(fmt.Errorf("config must be a pointer to a struct"))
+func readInto(config interface{}, fset *token.FileSet, file *token.File, src []byte) error {
+ var s scanner.Scanner
+ s.Init(file, src, nil, 0)
+ sect, sectsub := "", ""
+ pos, tok, lit := s.Scan()
+ errfn := func(msg string) error {
+ return fmt.Errorf("%s: %s", fset.Position(pos), msg)
}
- vCfg := vPCfg.Elem()
- vSect := fieldFold(vCfg, sect)
- if !vSect.IsValid() {
- return fmt.Errorf("invalid section: section %q", sect)
- }
- if vSect.Kind() == reflect.Map {
- vst := vSect.Type()
- if vst.Key().Kind() != reflect.String ||
- vst.Elem().Kind() != reflect.Ptr ||
- vst.Elem().Elem().Kind() != reflect.Struct {
- panic(fmt.Errorf("map field for section must have string keys and "+
- " pointer-to-struct values: section %q", sect))
+ for {
+ switch tok {
+ case token.EOF:
+ return nil
+ case token.EOL, token.COMMENT:
+ pos, tok, lit = s.Scan()
+ continue
+ case token.LBRACK:
+ pos, tok, lit = s.Scan()
+ if tok != token.IDENT {
+ return errfn("expected section name")
+ }
+ sect, sectsub = lit, ""
+ pos, tok, lit = s.Scan()
+ if tok == token.STRING {
+ sectsub = unquote(lit)
+ if sectsub == "" {
+ return errfn("empty subsection name")
+ }
+ pos, tok, lit = s.Scan()
+ }
+ if tok != token.RBRACK {
+ if sectsub == "" {
+ return errfn("expected subsection name or right bracket")
+ }
+ return errfn("expected right bracket")
+ }
+ pos, tok, lit = s.Scan()
+ if tok != token.EOL && tok != token.EOF && tok != token.COMMENT {
+ return errfn("expected EOL, EOF, or comment")
+ }
+ case token.IDENT:
+ if sect == "" {
+ return errfn("expected section header")
+ }
+ n := lit
+ pos, tok, lit = s.Scan()
+ var v string
+ if tok == token.EOF || tok == token.EOL || tok == token.COMMENT {
+ v = defaultValue
+ } else {
+ if tok != token.ASSIGN {
+ return errfn("expected '='")
+ }
+ pos, tok, lit = s.Scan()
+ if tok != token.STRING {
+ return errfn("expected value")
+ }
+ v = unquote(lit)
+ pos, tok, lit = s.Scan()
+ if tok != token.EOL && tok != token.EOF && tok != token.COMMENT {
+ return errfn("expected EOL, EOF, or comment")
+ }
+ }
+ err := set(config, sect, sectsub, n, v)
+ if err != nil {
+ return err
+ }
+ default:
+ return fmt.Errorf("%s invalid token %s: %q", fset.Position(pos),
+ tok, lit)
}
- if vSect.IsNil() {
- vSect.Set(reflect.MakeMap(vst))
- }
- k := reflect.ValueOf(sub)
- pv := vSect.MapIndex(k)
- if !pv.IsValid() {
- vType := vSect.Type().Elem().Elem()
- pv = reflect.New(vType)
- vSect.SetMapIndex(k, pv)
- }
- vSect = pv.Elem()
- } else if vSect.Kind() != reflect.Struct {
- panic(fmt.Errorf("field for section must be a map or a struct: "+
- "section %q", sect))
- } else if sub != "" {
- return fmt.Errorf("invalid subsection: "+
- "section %q subsection %q", sect, sub)
- }
- vName := fieldFold(vSect, name)
- if !vName.IsValid() {
- return fmt.Errorf("invalid variable: "+
- "section %q subsection %q variable %q", sect, sub, name)
- }
- vAddr := vName.Addr().Interface()
- switch v := vAddr.(type) {
- case *string:
- *v = value
- return nil
- case *bool:
- vAddr = (*gbool)(v)
- }
- // attempt to read an extra rune to make sure the value is consumed
- var r rune
- n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
- switch {
- case n < 1 || n == 1 && err != io.EOF:
- return fmt.Errorf("failed to parse %q as %#v: parse error %v", value,
- vName.Type(), err)
- case n > 1:
- return fmt.Errorf("failed to parse %q as %#v: extra characters", value,
- vName.Type())
- case n == 1 && err == io.EOF:
- return nil
}
panic("never reached")
}
@@ -140,74 +136,13 @@
// See ReadStringInto for examples.
//
func ReadInto(config interface{}, reader io.Reader) error {
- r := bufio.NewReader(reader)
- sect, sectsub := "", ""
- lp := []byte{}
- for line := 1; true; line++ {
- l, pre, err := r.ReadLine()
- if err != nil && err != io.EOF {
- return err
- }
- if pre {
- lp = append(lp, l...)
- line--
- continue
- }
- if len(l) > 0 {
- l = append(lp, l...)
- lp = []byte{}
- }
- // exclude comments
- if c := reCmnt.FindSubmatch(l); c != nil {
- l = c[1]
- } else if c := reCmntQ.FindSubmatch(l); c != nil {
- l = c[1]
- }
- if !reBlank.Match(l) {
- // "switch" based on line contents
- if s, ss := reSect.FindSubmatch(l), reSectSub.FindSubmatch(l); //
- s != nil || ss != nil {
- // section
- if s != nil {
- sect, sectsub = string(s[1]), ""
- } else { // ss != nil
- sect, sectsub = string(ss[1]), string(ss[2])
- }
- if sect == "" {
- return fmt.Errorf("empty section name not allowed")
- }
- if ss != nil && sectsub == "" {
- return fmt.Errorf("subsection name \"\" not allowed; " +
- "use [section-name] for blank subsection name")
- }
- } else if v, vq, vd := reVar.FindSubmatch(l),
- reVarQ.FindSubmatch(l), reVarDflt.FindSubmatch(l); //
- v != nil || vq != nil || vd != nil {
- // variable
- if sect == "" {
- return fmt.Errorf("variable must be defined in a section")
- }
- var name, value string
- if v != nil {
- name, value = string(v[1]), string(v[2])
- } else if vq != nil {
- name, value = string(vq[1]), string(vq[2])
- } else { // vd != nil
- name, value = string(vd[1]), defaultValue
- }
- err := set(config, sect, sectsub, name, value)
- if err != nil {
- return err
- }
- } else {
- return fmt.Errorf("invalid line %q", string(l))
- }
- }
- if err == io.EOF {
- break
- }
+ src, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return err
}
- return nil
+ fset := token.NewFileSet()
+ file := fset.AddFile("", fset.Base(), len(src))
+ return readInto(config, fset, file, src)
}
// ReadStringInto reads gcfg formatted data from str and sets the values into
@@ -229,5 +164,11 @@
return err
}
defer f.Close()
- return ReadInto(config, f)
+ src, err := ioutil.ReadAll(f)
+ if err != nil {
+ return err
+ }
+ fset := token.NewFileSet()
+ file := fset.AddFile(filename, fset.Base(), len(src))
+ return readInto(config, fset, file, src)
}
diff --git a/read_test.go b/read_test.go
index 1e18b6e..6d46c5b 100644
--- a/read_test.go
+++ b/read_test.go
@@ -126,7 +126,7 @@
for _, tg := range readtests {
for i, tt := range tg.tests {
id := fmt.Sprintf("%s:%d", tg.group, i)
- // get the type of the expected result
+ // get the type of the expected result
restyp := reflect.TypeOf(tt.exp).Elem()
// create a new instance to hold the actual result
res := reflect.New(restyp).Interface()
diff --git a/scanner/errors.go b/scanner/errors.go
new file mode 100644
index 0000000..4ff920a
--- /dev/null
+++ b/scanner/errors.go
@@ -0,0 +1,121 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+ "fmt"
+ "io"
+ "sort"
+)
+
+import (
+ "code.google.com/p/gcfg/token"
+)
+
+// In an ErrorList, an error is represented by an *Error.
+// The position Pos, if valid, points to the beginning of
+// the offending token, and the error condition is described
+// by Msg.
+//
+type Error struct {
+ Pos token.Position
+ Msg string
+}
+
+// Error implements the error interface.
+func (e Error) Error() string {
+ if e.Pos.Filename != "" || e.Pos.IsValid() {
+ // don't print "<unknown position>"
+ // TODO(gri) reconsider the semantics of Position.IsValid
+ return e.Pos.String() + ": " + e.Msg
+ }
+ return e.Msg
+}
+
+// ErrorList is a list of *Errors.
+// The zero value for an ErrorList is an empty ErrorList ready to use.
+//
+type ErrorList []*Error
+
+// Add adds an Error with given position and error message to an ErrorList.
+func (p *ErrorList) Add(pos token.Position, msg string) {
+ *p = append(*p, &Error{pos, msg})
+}
+
+// Reset resets an ErrorList to no errors.
+func (p *ErrorList) Reset() { *p = (*p)[0:0] }
+
+// ErrorList implements the sort Interface.
+func (p ErrorList) Len() int { return len(p) }
+func (p ErrorList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p ErrorList) Less(i, j int) bool {
+ e := &p[i].Pos
+ f := &p[j].Pos
+ if e.Filename < f.Filename {
+ return true
+ }
+ if e.Filename == f.Filename {
+ return e.Offset < f.Offset
+ }
+ return false
+}
+
+// Sort sorts an ErrorList. *Error entries are sorted by position,
+// other errors are sorted by error message, and before any *Error
+// entry.
+//
+func (p ErrorList) Sort() {
+ sort.Sort(p)
+}
+
+// RemoveMultiples sorts an ErrorList and removes all but the first error per line.
+func (p *ErrorList) RemoveMultiples() {
+ sort.Sort(p)
+ var last token.Position // initial last.Line is != any legal error line
+ i := 0
+ for _, e := range *p {
+ if e.Pos.Filename != last.Filename || e.Pos.Line != last.Line {
+ last = e.Pos
+ (*p)[i] = e
+ i++
+ }
+ }
+ (*p) = (*p)[0:i]
+}
+
+// An ErrorList implements the error interface.
+func (p ErrorList) Error() string {
+ switch len(p) {
+ case 0:
+ return "no errors"
+ case 1:
+ return p[0].Error()
+ }
+ return fmt.Sprintf("%s (and %d more errors)", p[0], len(p)-1)
+}
+
+// Err returns an error equivalent to this error list.
+// If the list is empty, Err returns nil.
+func (p ErrorList) Err() error {
+ if len(p) == 0 {
+ return nil
+ }
+ return p
+}
+
+// PrintError is a utility function that prints a list of errors to w,
+// one error per line, if the err parameter is an ErrorList. Otherwise
+// it prints the err string.
+//
+func PrintError(w io.Writer, err error) {
+ if list, ok := err.(ErrorList); ok {
+ for _, e := range list {
+ fmt.Fprintf(w, "%s\n", e)
+ }
+ } else if err != nil {
+ fmt.Fprintf(w, "%s\n", err)
+ }
+}
diff --git a/scanner/example_test.go b/scanner/example_test.go
new file mode 100644
index 0000000..05eadf5
--- /dev/null
+++ b/scanner/example_test.go
@@ -0,0 +1,46 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner_test
+
+import (
+ "fmt"
+)
+
+import (
+ "code.google.com/p/gcfg/scanner"
+ "code.google.com/p/gcfg/token"
+)
+
+func ExampleScanner_Scan() {
+ // src is the input that we want to tokenize.
+ src := []byte(`[profile "A"]
+color = blue ; Comment`)
+
+ // Initialize the scanner.
+ var s scanner.Scanner
+ fset := token.NewFileSet() // positions are relative to fset
+ file := fset.AddFile("", fset.Base(), len(src)) // register input "file"
+ s.Init(file, src, nil /* no error handler */, scanner.ScanComments)
+
+ // Repeated calls to Scan yield the token sequence found in the input.
+ for {
+ pos, tok, lit := s.Scan()
+ if tok == token.EOF {
+ break
+ }
+ fmt.Printf("%s\t%q\t%q\n", fset.Position(pos), tok, lit)
+ }
+
+ // output:
+ // 1:1 "[" ""
+ // 1:2 "IDENT" "profile"
+ // 1:10 "STRING" "\"A\""
+ // 1:13 "]" ""
+ // 1:14 "\n" ""
+ // 2:1 "IDENT" "color"
+ // 2:7 "=" ""
+ // 2:9 "STRING" "blue"
+ // 2:14 "COMMENT" "; Comment"
+}
diff --git a/scanner/scanner.go b/scanner/scanner.go
new file mode 100644
index 0000000..099cc74
--- /dev/null
+++ b/scanner/scanner.go
@@ -0,0 +1,339 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package scanner implements a scanner for gcfg configuration text.
+// It takes a []byte as source which can then be tokenized
+// through repeated calls to the Scan method.
+//
+// Note that the API for the scanner package may change to accommodate new
+// features or implementation changes in gcfg.
+//
+package scanner
+
+import (
+ "fmt"
+ "path/filepath"
+ "unicode"
+ "unicode/utf8"
+)
+
+import (
+ "code.google.com/p/gcfg/token"
+)
+
+// An ErrorHandler may be provided to Scanner.Init. If a syntax error is
+// encountered and a handler was installed, the handler is called with a
+// position and an error message. The position points to the beginning of
+// the offending token.
+//
+type ErrorHandler func(pos token.Position, msg string)
+
+// A Scanner holds the scanner's internal state while processing
+// a given text. It can be allocated as part of another data
+// structure but must be initialized via Init before use.
+//
+type Scanner struct {
+ // immutable state
+ file *token.File // source file handle
+ dir string // directory portion of file.Name()
+ src []byte // source
+ err ErrorHandler // error reporting; or nil
+ mode Mode // scanning mode
+
+ // scanning state
+ ch rune // current character
+ offset int // character offset
+ rdOffset int // reading offset (position after current character)
+ lineOffset int // current line offset
+ nextVal bool // next token is expected to be a value
+
+ // public state - ok to modify
+ ErrorCount int // number of errors encountered
+}
+
+// Read the next Unicode char into s.ch.
+// s.ch < 0 means end-of-file.
+//
+func (s *Scanner) next() {
+ if s.rdOffset < len(s.src) {
+ s.offset = s.rdOffset
+ if s.ch == '\n' {
+ s.lineOffset = s.offset
+ s.file.AddLine(s.offset)
+ }
+ r, w := rune(s.src[s.rdOffset]), 1
+ switch {
+ case r == 0:
+ s.error(s.offset, "illegal character NUL")
+ case r >= 0x80:
+ // not ASCII
+ r, w = utf8.DecodeRune(s.src[s.rdOffset:])
+ if r == utf8.RuneError && w == 1 {
+ s.error(s.offset, "illegal UTF-8 encoding")
+ }
+ }
+ s.rdOffset += w
+ s.ch = r
+ } else {
+ s.offset = len(s.src)
+ if s.ch == '\n' {
+ s.lineOffset = s.offset
+ s.file.AddLine(s.offset)
+ }
+ s.ch = -1 // eof
+ }
+}
+
+// A mode value is a set of flags (or 0).
+// They control scanner behavior.
+//
+type Mode uint
+
+const (
+ ScanComments Mode = 1 << iota // return comments as COMMENT tokens
+)
+
+// Init prepares the scanner s to tokenize the text src by setting the
+// scanner at the beginning of src. The scanner uses the file set file
+// for position information and it adds line information for each line.
+// It is ok to re-use the same file when re-scanning the same file as
+// line information which is already present is ignored. Init causes a
+// panic if the file size does not match the src size.
+//
+// Calls to Scan will invoke the error handler err if they encounter a
+// syntax error and err is not nil. Also, for each error encountered,
+// the Scanner field ErrorCount is incremented by one. The mode parameter
+// determines how comments are handled.
+//
+// Note that Init may call err if there is an error in the first character
+// of the file.
+//
+func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) {
+ // Explicitly initialize all fields since a scanner may be reused.
+ if file.Size() != len(src) {
+ panic(fmt.Sprintf("file size (%d) does not match src len (%d)", file.Size(), len(src)))
+ }
+ s.file = file
+ s.dir, _ = filepath.Split(file.Name())
+ s.src = src
+ s.err = err
+ s.mode = mode
+
+ s.ch = ' '
+ s.offset = 0
+ s.rdOffset = 0
+ s.lineOffset = 0
+ s.ErrorCount = 0
+ s.nextVal = false
+
+ s.next()
+}
+
+func (s *Scanner) error(offs int, msg string) {
+ if s.err != nil {
+ s.err(s.file.Position(s.file.Pos(offs)), msg)
+ }
+ s.ErrorCount++
+}
+
+func (s *Scanner) scanComment() string {
+ // initial [;#] already consumed
+ offs := s.offset - 1 // position of initial [;#]
+
+ s.next()
+ for s.ch != '\n' && s.ch >= 0 {
+ s.next()
+ }
+ return string(s.src[offs:s.offset])
+}
+
+func isLetter(ch rune) bool {
+ return 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' || ch >= 0x80 && unicode.IsLetter(ch)
+}
+
+func isDigit(ch rune) bool {
+ return '0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch)
+}
+
+func (s *Scanner) scanIdentifier() string {
+ offs := s.offset
+ for isLetter(s.ch) || isDigit(s.ch) || s.ch == '-' {
+ s.next()
+ }
+ return string(s.src[offs:s.offset])
+}
+
+func (s *Scanner) scanEscape() {
+ offs := s.offset
+
+ switch s.ch {
+ case 'n', '\\', '"':
+ s.next()
+ return
+ }
+ s.next() // always make progress
+ s.error(offs, "unknown escape sequence")
+ return
+}
+
+func (s *Scanner) scanString() string {
+ // '"' opening already consumed
+ offs := s.offset - 1
+
+ for s.ch != '"' {
+ ch := s.ch
+ s.next()
+ if ch == '\n' || ch < 0 {
+ s.error(offs, "string not terminated")
+ break
+ }
+ if ch == '\\' {
+ s.scanEscape()
+ }
+ }
+
+ s.next()
+
+ return string(s.src[offs:s.offset])
+}
+
+func stripCR(b []byte) []byte {
+ c := make([]byte, len(b))
+ i := 0
+ for _, ch := range b {
+ if ch != '\r' {
+ c[i] = ch
+ i++
+ }
+ }
+ return c[:i]
+}
+
+func (s *Scanner) scanValString() string {
+ offs := s.offset
+
+ hasCR := false
+ end := offs
+ inQuote := false
+loop:
+ for inQuote || s.ch != '\n' && s.ch != ';' && s.ch != '#' {
+ ch := s.ch
+ s.next()
+ switch {
+ case inQuote && ch == '\\':
+ s.scanEscape()
+ case !inQuote && ch == '\\':
+ if s.ch == '\r' {
+ hasCR = true
+ s.next()
+ }
+ if s.ch != '\n' {
+ s.error(offs, "unquoted '\\' must be followed by new line")
+ break loop
+ }
+ s.next()
+ case ch == '"':
+ inQuote = !inQuote
+ case ch == '\r':
+ hasCR = true
+ case ch < 0 || inQuote && ch == '\n':
+ s.error(offs, "string not terminated")
+ break loop
+ }
+ if inQuote || !isWhiteSpace(ch) {
+ end = s.offset
+ }
+ }
+
+ lit := s.src[offs:end]
+ if hasCR {
+ lit = stripCR(lit)
+ }
+
+ return string(lit)
+}
+
+func isWhiteSpace(ch rune) bool {
+ return ch == ' ' || ch == '\t' || ch == '\r'
+}
+
+func (s *Scanner) skipWhitespace() {
+ for isWhiteSpace(s.ch) {
+ s.next()
+ }
+}
+
+// Scan scans the next token and returns the token position, the token,
+// and its literal string if applicable. The source end is indicated by
+// token.EOF.
+//
+// If the returned token is a literal (token.IDENT, token.STRING) or
+// token.COMMENT, the literal string has the corresponding value.
+//
+// If the returned token is token.ILLEGAL, the literal string is the
+// offending character.
+//
+// In all other cases, Scan returns an empty literal string.
+//
+// For more tolerant parsing, Scan will return a valid token if
+// possible even if a syntax error was encountered. Thus, even
+// if the resulting token sequence contains no illegal tokens,
+// a client may not assume that no error occurred. Instead it
+// must check the scanner's ErrorCount or the number of calls
+// of the error handler, if there was one installed.
+//
+// Scan adds line information to the file added to the file
+// set with Init. Token positions are relative to that file
+// and thus relative to the file set.
+//
+func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) {
+scanAgain:
+ s.skipWhitespace()
+
+ // current token start
+ pos = s.file.Pos(s.offset)
+
+ // determine token value
+ switch ch := s.ch; {
+ case s.nextVal:
+ lit = s.scanValString()
+ tok = token.STRING
+ s.nextVal = false
+ case isLetter(ch):
+ lit = s.scanIdentifier()
+ tok = token.IDENT
+ default:
+ s.next() // always make progress
+ switch ch {
+ case -1:
+ tok = token.EOF
+ case '\n':
+ tok = token.EOL
+ case '"':
+ tok = token.STRING
+ lit = s.scanString()
+ case '[':
+ tok = token.LBRACK
+ case ']':
+ tok = token.RBRACK
+ case ';', '#':
+ // comment
+ lit = s.scanComment()
+ if s.mode&ScanComments == 0 {
+ // skip comment
+ goto scanAgain
+ }
+ tok = token.COMMENT
+ case '=':
+ tok = token.ASSIGN
+ s.nextVal = true
+ default:
+ s.error(s.file.Offset(pos), fmt.Sprintf("illegal character %#U", ch))
+ tok = token.ILLEGAL
+ lit = string(ch)
+ }
+ }
+
+ return
+}
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
new file mode 100644
index 0000000..f1feeb3
--- /dev/null
+++ b/scanner/scanner_test.go
@@ -0,0 +1,381 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package scanner
+
+import (
+ "os"
+ "strings"
+ "testing"
+)
+
+import (
+ "code.google.com/p/gcfg/token"
+)
+
+var fset = token.NewFileSet()
+
+const /* class */ (
+ special = iota
+ literal
+ operator
+)
+
+func tokenclass(tok token.Token) int {
+ switch {
+ case tok.IsLiteral():
+ return literal
+ case tok.IsOperator():
+ return operator
+ }
+ return special
+}
+
+type elt struct {
+ tok token.Token
+ lit string
+ class int
+ pre string
+ suf string
+}
+
+var tokens = [...]elt{
+ // Special tokens
+ {token.COMMENT, "; a comment", special, "", "\n"},
+ {token.COMMENT, "# a comment", special, "", "\n"},
+
+ // Operators and delimiters
+ {token.ASSIGN, "=", operator, "", "value"},
+ {token.LBRACK, "[", operator, "", ""},
+ {token.RBRACK, "]", operator, "", ""},
+ {token.EOL, "\n", operator, "", ""},
+
+ // Identifiers
+ {token.IDENT, "foobar", literal, "", ""},
+ {token.IDENT, "a۰۱۸", literal, "", ""},
+ {token.IDENT, "foo६४", literal, "", ""},
+ {token.IDENT, "bar９８７６", literal, "", ""},
+ {token.IDENT, "foo-bar", literal, "", ""},
+ // String literals (subsection names)
+ {token.STRING, `"foobar"`, literal, "", ""},
+ {token.STRING, `"\n"`, literal, "", ""},
+ {token.STRING, `"\""`, literal, "", ""},
+ // String literals (values)
+ {token.STRING, `"foobar"`, literal, "=", ""},
+ {token.STRING, `"foo\nbar"`, literal, "=", ""},
+ {token.STRING, `"foo\"bar"`, literal, "=", ""},
+ {token.STRING, `"foo\\bar"`, literal, "=", ""},
+ {token.STRING, `"foobar"`, literal, "=", ""},
+ {token.STRING, `"foobar"`, literal, "= ", ""},
+ {token.STRING, `"foobar"`, literal, "=", "\n"},
+ {token.STRING, `"foobar"`, literal, "=", ";"},
+ {token.STRING, `"foobar"`, literal, "=", " ;"},
+ {token.STRING, `"foobar"`, literal, "=", "#"},
+ {token.STRING, `"foobar"`, literal, "=", " #"},
+ {token.STRING, "foobar", literal, "=", ""},
+ {token.STRING, "foobar", literal, "= ", ""},
+ {token.STRING, "foobar", literal, "=", " "},
+ {token.STRING, `"foo" "bar"`, literal, "=", " "},
+ {token.STRING, "foo\\\nbar", literal, "=", ""},
+ {token.STRING, "foo\\\r\nbar", literal, "=", ""},
+}
+
+const whitespace = " \t \n\n\n" // to separate tokens
+
+var source = func() []byte {
+ var src []byte
+ for _, t := range tokens {
+ src = append(src, t.pre...)
+ src = append(src, t.lit...)
+ src = append(src, t.suf...)
+ src = append(src, whitespace...)
+ }
+ return src
+}()
+
+func newlineCount(s string) int {
+ n := 0
+ for i := 0; i < len(s); i++ {
+ if s[i] == '\n' {
+ n++
+ }
+ }
+ return n
+}
+
+func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
+ pos := fset.Position(p)
+ if pos.Filename != expected.Filename {
+ t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
+ }
+ if pos.Offset != expected.Offset {
+ t.Errorf("bad position for %q: got %d, expected %d", lit, pos.Offset, expected.Offset)
+ }
+ if pos.Line != expected.Line {
+ t.Errorf("bad line for %q: got %d, expected %d", lit, pos.Line, expected.Line)
+ }
+ if pos.Column != expected.Column {
+ t.Errorf("bad column for %q: got %d, expected %d", lit, pos.Column, expected.Column)
+ }
+}
+
+// Verify that calling Scan() provides the correct results.
+func TestScan(t *testing.T) {
+ // make source
+ src_linecount := newlineCount(string(source))
+ whitespace_linecount := newlineCount(whitespace)
+
+ // error handler
+ eh := func(_ token.Position, msg string) {
+ t.Errorf("error handler called (msg = %s)", msg)
+ }
+
+ // verify scan
+ var s Scanner
+ s.Init(fset.AddFile("", fset.Base(), len(source)), source, eh, ScanComments)
+ index := 0
+ // epos is the expected position
+ epos := token.Position{
+ Filename: "",
+ Offset: 0,
+ Line: 1,
+ Column: 1,
+ }
+ for {
+ pos, tok, lit := s.Scan()
+ if lit == "" {
+ // no literal value for non-literal tokens
+ lit = tok.String()
+ }
+ e := elt{token.EOF, "", special, "", ""}
+ if index < len(tokens) {
+ e = tokens[index]
+ }
+ if tok == token.EOF {
+ lit = "<EOF>"
+ epos.Line = src_linecount
+ epos.Column = 2
+ }
+ if strings.ContainsRune(e.pre, '=') {
+ epos.Column = 1
+ checkPos(t, lit, pos, epos)
+ if tok != token.ASSIGN {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.ASSIGN)
+ }
+ pos, tok, lit = s.Scan()
+ }
+ epos.Offset += len(e.pre)
+ if tok != token.EOF {
+ epos.Column = 1 + len(e.pre)
+ }
+ checkPos(t, lit, pos, epos)
+ if tok != e.tok {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, e.tok)
+ }
+ if e.tok.IsLiteral() {
+ // no CRs in value string literals
+ elit := e.lit
+ if strings.ContainsRune(e.pre, '=') {
+ elit = string(stripCR([]byte(elit)))
+ epos.Offset += len(e.lit) - len(lit) // correct position
+ }
+ if lit != elit {
+ t.Errorf("bad literal for %q: got %q, expected %q", lit, lit, elit)
+ }
+ }
+ if tokenclass(tok) != e.class {
+ t.Errorf("bad class for %q: got %d, expected %d", lit, tokenclass(tok), e.class)
+ }
+ epos.Offset += len(lit) + len(e.suf) + len(whitespace)
+ epos.Line += newlineCount(lit) + newlineCount(e.suf) + whitespace_linecount
+ index++
+ if tok == token.EOF {
+ break
+ }
+ if e.suf == "value" {
+ pos, tok, lit = s.Scan()
+ if tok != token.STRING {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.STRING)
+ }
+ } else if strings.ContainsRune(e.suf, ';') || strings.ContainsRune(e.suf, '#') {
+ pos, tok, lit = s.Scan()
+ if tok != token.COMMENT {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.COMMENT)
+ }
+ }
+ // skip EOLs
+ for i := 0; i < whitespace_linecount+newlineCount(e.suf); i++ {
+ pos, tok, lit = s.Scan()
+ if tok != token.EOL {
+ t.Errorf("bad token for %q: got %s, expected %s", lit, tok, token.EOL)
+ }
+ }
+ }
+ if s.ErrorCount != 0 {
+ t.Errorf("found %d errors", s.ErrorCount)
+ }
+}
+
+// Verify that initializing the same scanner more then once works correctly.
+func TestInit(t *testing.T) {
+ var s Scanner
+
+ // 1st init
+ src1 := "\nname = value"
+ f1 := fset.AddFile("src1", fset.Base(), len(src1))
+ s.Init(f1, []byte(src1), nil, 0)
+ if f1.Size() != len(src1) {
+ t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
+ }
+ s.Scan() // \n
+ s.Scan() // name
+ _, tok, _ := s.Scan() // =
+ if tok != token.ASSIGN {
+ t.Errorf("bad token: got %s, expected %s", tok, token.ASSIGN)
+ }
+
+ // 2nd init
+ src2 := "[section]"
+ f2 := fset.AddFile("src2", fset.Base(), len(src2))
+ s.Init(f2, []byte(src2), nil, 0)
+ if f2.Size() != len(src2) {
+ t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
+ }
+ _, tok, _ = s.Scan() // [
+ if tok != token.LBRACK {
+ t.Errorf("bad token: got %s, expected %s", tok, token.LBRACK)
+ }
+
+ if s.ErrorCount != 0 {
+ t.Errorf("found %d errors", s.ErrorCount)
+ }
+}
+
+func TestStdErrorHandler(t *testing.T) {
+ const src = "@\n" + // illegal character, cause an error
+ "@ @\n" // two errors on the same line
+
+ var list ErrorList
+ eh := func(pos token.Position, msg string) { list.Add(pos, msg) }
+
+ var s Scanner
+ s.Init(fset.AddFile("File1", fset.Base(), len(src)), []byte(src), eh, 0)
+ for {
+ if _, tok, _ := s.Scan(); tok == token.EOF {
+ break
+ }
+ }
+
+ if len(list) != s.ErrorCount {
+ t.Errorf("found %d errors, expected %d", len(list), s.ErrorCount)
+ }
+
+ if len(list) != 3 {
+ t.Errorf("found %d raw errors, expected 3", len(list))
+ PrintError(os.Stderr, list)
+ }
+
+ list.Sort()
+ if len(list) != 3 {
+ t.Errorf("found %d sorted errors, expected 3", len(list))
+ PrintError(os.Stderr, list)
+ }
+
+ list.RemoveMultiples()
+ if len(list) != 2 {
+ t.Errorf("found %d one-per-line errors, expected 2", len(list))
+ PrintError(os.Stderr, list)
+ }
+}
+
+type errorCollector struct {
+ cnt int // number of errors encountered
+ msg string // last error message encountered
+ pos token.Position // last error position encountered
+}
+
+func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
+ var s Scanner
+ var h errorCollector
+ eh := func(pos token.Position, msg string) {
+ h.cnt++
+ h.msg = msg
+ h.pos = pos
+ }
+ s.Init(fset.AddFile("", fset.Base(), len(src)), []byte(src), eh, ScanComments)
+ if src[0] == '=' {
+ _, _, _ = s.Scan()
+ }
+ _, tok0, _ := s.Scan()
+ _, tok1, _ := s.Scan()
+ if tok0 != tok {
+ t.Errorf("%q: got %s, expected %s", src, tok0, tok)
+ }
+ if tok1 != token.EOF {
+ t.Errorf("%q: got %s, expected EOF", src, tok1)
+ }
+ cnt := 0
+ if err != "" {
+ cnt = 1
+ }
+ if h.cnt != cnt {
+ t.Errorf("%q: got cnt %d, expected %d", src, h.cnt, cnt)
+ }
+ if h.msg != err {
+ t.Errorf("%q: got msg %q, expected %q", src, h.msg, err)
+ }
+ if h.pos.Offset != pos {
+ t.Errorf("%q: got offset %d, expected %d", src, h.pos.Offset, pos)
+ }
+}
+
+var errors = []struct {
+ src string
+ tok token.Token
+ pos int
+ err string
+}{
+ {"\a", token.ILLEGAL, 0, "illegal character U+0007"},
+ {"/", token.ILLEGAL, 0, "illegal character U+002F '/'"},
+ {"_", token.ILLEGAL, 0, "illegal character U+005F '_'"},
+ {`…`, token.ILLEGAL, 0, "illegal character U+2026 '…'"},
+ {`""`, token.STRING, 0, ""},
+ {`"`, token.STRING, 0, "string not terminated"},
+ {"\"\n", token.STRING, 0, "string not terminated"},
+ {`="`, token.STRING, 1, "string not terminated"},
+ {"=\"\n", token.STRING, 1, "string not terminated"},
+ {"=\\", token.STRING, 1, "unquoted '\\' must be followed by new line"},
+ {"=\\\r", token.STRING, 1, "unquoted '\\' must be followed by new line"},
+ {`"\z"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\a"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\b"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\f"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\r"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\t"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\v"`, token.STRING, 2, "unknown escape sequence"},
+ {`"\0"`, token.STRING, 2, "unknown escape sequence"},
+}
+
+func TestScanErrors(t *testing.T) {
+ for _, e := range errors {
+ checkError(t, e.src, e.tok, e.pos, e.err)
+ }
+}
+
+func BenchmarkScan(b *testing.B) {
+ b.StopTimer()
+ fset := token.NewFileSet()
+ file := fset.AddFile("", fset.Base(), len(source))
+ var s Scanner
+ b.StartTimer()
+ for i := b.N - 1; i >= 0; i-- {
+ s.Init(file, source, nil, ScanComments)
+ for {
+ _, tok, _ := s.Scan()
+ if tok == token.EOF {
+ break
+ }
+ }
+ }
+}
diff --git a/set.go b/set.go
new file mode 100644
index 0000000..935a7a9
--- /dev/null
+++ b/set.go
@@ -0,0 +1,85 @@
+package gcfg
+
+import (
+ "fmt"
+ "io"
+ "reflect"
+ "strings"
+)
+
+const (
+ // Default value string in case a value for a variable isn't provided.
+ defaultValue = "true"
+)
+
+func fieldFold(v reflect.Value, name string) reflect.Value {
+ n := strings.Replace(name, "-", "_", -1)
+ return v.FieldByNameFunc(func(fieldName string) bool {
+ return strings.EqualFold(n, fieldName)
+ })
+}
+
+func set(cfg interface{}, sect, sub, name, value string) error {
+ vPCfg := reflect.ValueOf(cfg)
+ if vPCfg.Kind() != reflect.Ptr || vPCfg.Elem().Kind() != reflect.Struct {
+ panic(fmt.Errorf("config must be a pointer to a struct"))
+ }
+ vCfg := vPCfg.Elem()
+ vSect := fieldFold(vCfg, sect)
+ if !vSect.IsValid() {
+ return fmt.Errorf("invalid section: section %q", sect)
+ }
+ if vSect.Kind() == reflect.Map {
+ vst := vSect.Type()
+ if vst.Key().Kind() != reflect.String ||
+ vst.Elem().Kind() != reflect.Ptr ||
+ vst.Elem().Elem().Kind() != reflect.Struct {
+ panic(fmt.Errorf("map field for section must have string keys and "+
+ " pointer-to-struct values: section %q", sect))
+ }
+ if vSect.IsNil() {
+ vSect.Set(reflect.MakeMap(vst))
+ }
+ k := reflect.ValueOf(sub)
+ pv := vSect.MapIndex(k)
+ if !pv.IsValid() {
+ vType := vSect.Type().Elem().Elem()
+ pv = reflect.New(vType)
+ vSect.SetMapIndex(k, pv)
+ }
+ vSect = pv.Elem()
+ } else if vSect.Kind() != reflect.Struct {
+ panic(fmt.Errorf("field for section must be a map or a struct: "+
+ "section %q", sect))
+ } else if sub != "" {
+ return fmt.Errorf("invalid subsection: "+
+ "section %q subsection %q", sect, sub)
+ }
+ vName := fieldFold(vSect, name)
+ if !vName.IsValid() {
+ return fmt.Errorf("invalid variable: "+
+ "section %q subsection %q variable %q", sect, sub, name)
+ }
+ vAddr := vName.Addr().Interface()
+ switch v := vAddr.(type) {
+ case *string:
+ *v = value
+ return nil
+ case *bool:
+ vAddr = (*gbool)(v)
+ }
+ // attempt to read an extra rune to make sure the value is consumed
+ var r rune
+ n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
+ switch {
+ case n < 1 || n == 1 && err != io.EOF:
+ return fmt.Errorf("failed to parse %q as %#v: parse error %v", value,
+ vName.Type(), err)
+ case n > 1:
+ return fmt.Errorf("failed to parse %q as %#v: extra characters", value,
+ vName.Type())
+ case n == 1 && err == io.EOF:
+ return nil
+ }
+ panic("never reached")
+}
diff --git a/token/position.go b/token/position.go
new file mode 100644
index 0000000..fc45c1e
--- /dev/null
+++ b/token/position.go
@@ -0,0 +1,435 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO(gri) consider making this a separate package outside the go directory.
+
+package token
+
+import (
+ "fmt"
+ "sort"
+ "sync"
+)
+
+// -----------------------------------------------------------------------------
+// Positions
+
+// Position describes an arbitrary source position
+// including the file, line, and column location.
+// A Position is valid if the line number is > 0.
+//
+type Position struct {
+ Filename string // filename, if any
+ Offset int // offset, starting at 0
+ Line int // line number, starting at 1
+ Column int // column number, starting at 1 (character count)
+}
+
+// IsValid returns true if the position is valid.
+func (pos *Position) IsValid() bool { return pos.Line > 0 }
+
+// String returns a string in one of several forms:
+//
+// file:line:column valid position with file name
+// line:column valid position without file name
+// file invalid position with file name
+// - invalid position without file name
+//
+func (pos Position) String() string {
+ s := pos.Filename
+ if pos.IsValid() {
+ if s != "" {
+ s += ":"
+ }
+ s += fmt.Sprintf("%d:%d", pos.Line, pos.Column)
+ }
+ if s == "" {
+ s = "-"
+ }
+ return s
+}
+
+// Pos is a compact encoding of a source position within a file set.
+// It can be converted into a Position for a more convenient, but much
+// larger, representation.
+//
+// The Pos value for a given file is a number in the range [base, base+size],
+// where base and size are specified when adding the file to the file set via
+// AddFile.
+//
+// To create the Pos value for a specific source offset, first add
+// the respective file to the current file set (via FileSet.AddFile)
+// and then call File.Pos(offset) for that file. Given a Pos value p
+// for a specific file set fset, the corresponding Position value is
+// obtained by calling fset.Position(p).
+//
+// Pos values can be compared directly with the usual comparison operators:
+// If two Pos values p and q are in the same file, comparing p and q is
+// equivalent to comparing the respective source file offsets. If p and q
+// are in different files, p < q is true if the file implied by p was added
+// to the respective file set before the file implied by q.
+//
+type Pos int
+
+// The zero value for Pos is NoPos; there is no file and line information
+// associated with it, and NoPos().IsValid() is false. NoPos is always
+// smaller than any other Pos value. The corresponding Position value
+// for NoPos is the zero value for Position.
+//
+const NoPos Pos = 0
+
+// IsValid returns true if the position is valid.
+func (p Pos) IsValid() bool {
+ return p != NoPos
+}
+
+// -----------------------------------------------------------------------------
+// File
+
+// A File is a handle for a file belonging to a FileSet.
+// A File has a name, size, and line offset table.
+//
+type File struct {
+ set *FileSet
+ name string // file name as provided to AddFile
+ base int // Pos value range for this file is [base...base+size]
+ size int // file size as provided to AddFile
+
+ // lines and infos are protected by set.mutex
+ lines []int
+ infos []lineInfo
+}
+
+// Name returns the file name of file f as registered with AddFile.
+func (f *File) Name() string {
+ return f.name
+}
+
+// Base returns the base offset of file f as registered with AddFile.
+func (f *File) Base() int {
+ return f.base
+}
+
+// Size returns the size of file f as registered with AddFile.
+func (f *File) Size() int {
+ return f.size
+}
+
+// LineCount returns the number of lines in file f.
+func (f *File) LineCount() int {
+ f.set.mutex.RLock()
+ n := len(f.lines)
+ f.set.mutex.RUnlock()
+ return n
+}
+
+// AddLine adds the line offset for a new line.
+// The line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise the line offset is ignored.
+//
+func (f *File) AddLine(offset int) {
+ f.set.mutex.Lock()
+ if i := len(f.lines); (i == 0 || f.lines[i-1] < offset) && offset < f.size {
+ f.lines = append(f.lines, offset)
+ }
+ f.set.mutex.Unlock()
+}
+
+// SetLines sets the line offsets for a file and returns true if successful.
+// The line offsets are the offsets of the first character of each line;
+// for instance for the content "ab\nc\n" the line offsets are {0, 3}.
+// An empty file has an empty line offset table.
+// Each line offset must be larger than the offset for the previous line
+// and smaller than the file size; otherwise SetLines fails and returns
+// false.
+//
+func (f *File) SetLines(lines []int) bool {
+ // verify validity of lines table
+ size := f.size
+ for i, offset := range lines {
+ if i > 0 && offset <= lines[i-1] || size <= offset {
+ return false
+ }
+ }
+
+ // set lines table
+ f.set.mutex.Lock()
+ f.lines = lines
+ f.set.mutex.Unlock()
+ return true
+}
+
+// SetLinesForContent sets the line offsets for the given file content.
+func (f *File) SetLinesForContent(content []byte) {
+ var lines []int
+ line := 0
+ for offset, b := range content {
+ if line >= 0 {
+ lines = append(lines, line)
+ }
+ line = -1
+ if b == '\n' {
+ line = offset + 1
+ }
+ }
+
+ // set lines table
+ f.set.mutex.Lock()
+ f.lines = lines
+ f.set.mutex.Unlock()
+}
+
+// A lineInfo object describes alternative file and line number
+// information (such as provided via a //line comment in a .go
+// file) for a given file offset.
+type lineInfo struct {
+ // fields are exported to make them accessible to gob
+ Offset int
+ Filename string
+ Line int
+}
+
+// AddLineInfo adds alternative file and line number information for
+// a given file offset. The offset must be larger than the offset for
+// the previously added alternative line info and smaller than the
+// file size; otherwise the information is ignored.
+//
+// AddLineInfo is typically used to register alternative position
+// information for //line filename:line comments in source files.
+//
+func (f *File) AddLineInfo(offset int, filename string, line int) {
+ f.set.mutex.Lock()
+ if i := len(f.infos); i == 0 || f.infos[i-1].Offset < offset && offset < f.size {
+ f.infos = append(f.infos, lineInfo{offset, filename, line})
+ }
+ f.set.mutex.Unlock()
+}
+
+// Pos returns the Pos value for the given file offset;
+// the offset must be <= f.Size().
+// f.Pos(f.Offset(p)) == p.
+//
+func (f *File) Pos(offset int) Pos {
+ if offset > f.size {
+ panic("illegal file offset")
+ }
+ return Pos(f.base + offset)
+}
+
+// Offset returns the offset for the given file position p;
+// p must be a valid Pos value in that file.
+// f.Offset(f.Pos(offset)) == offset.
+//
+func (f *File) Offset(p Pos) int {
+ if int(p) < f.base || int(p) > f.base+f.size {
+ panic("illegal Pos value")
+ }
+ return int(p) - f.base
+}
+
+// Line returns the line number for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Line(p Pos) int {
+ // TODO(gri) this can be implemented much more efficiently
+ return f.Position(p).Line
+}
+
+func searchLineInfos(a []lineInfo, x int) int {
+ return sort.Search(len(a), func(i int) bool { return a[i].Offset > x }) - 1
+}
+
+// info returns the file name, line, and column number for a file offset.
+func (f *File) info(offset int) (filename string, line, column int) {
+ filename = f.name
+ if i := searchInts(f.lines, offset); i >= 0 {
+ line, column = i+1, offset-f.lines[i]+1
+ }
+ if len(f.infos) > 0 {
+ // almost no files have extra line infos
+ if i := searchLineInfos(f.infos, offset); i >= 0 {
+ alt := &f.infos[i]
+ filename = alt.Filename
+ if i := searchInts(f.lines, alt.Offset); i >= 0 {
+ line += alt.Line - i - 1
+ }
+ }
+ }
+ return
+}
+
+func (f *File) position(p Pos) (pos Position) {
+ offset := int(p) - f.base
+ pos.Offset = offset
+ pos.Filename, pos.Line, pos.Column = f.info(offset)
+ return
+}
+
+// Position returns the Position value for the given file position p;
+// p must be a Pos value in that file or NoPos.
+//
+func (f *File) Position(p Pos) (pos Position) {
+ if p != NoPos {
+ if int(p) < f.base || int(p) > f.base+f.size {
+ panic("illegal Pos value")
+ }
+ pos = f.position(p)
+ }
+ return
+}
+
+// -----------------------------------------------------------------------------
+// FileSet
+
+// A FileSet represents a set of source files.
+// Methods of file sets are synchronized; multiple goroutines
+// may invoke them concurrently.
+//
+type FileSet struct {
+ mutex sync.RWMutex // protects the file set
+ base int // base offset for the next file
+ files []*File // list of files in the order added to the set
+ last *File // cache of last file looked up
+}
+
+// NewFileSet creates a new file set.
+func NewFileSet() *FileSet {
+ s := new(FileSet)
+ s.base = 1 // 0 == NoPos
+ return s
+}
+
+// Base returns the minimum base offset that must be provided to
+// AddFile when adding the next file.
+//
+func (s *FileSet) Base() int {
+ s.mutex.RLock()
+ b := s.base
+ s.mutex.RUnlock()
+ return b
+
+}
+
+// AddFile adds a new file with a given filename, base offset, and file size
+// to the file set s and returns the file. Multiple files may have the same
+// name. The base offset must not be smaller than the FileSet's Base(), and
+// size must not be negative.
+//
+// Adding the file will set the file set's Base() value to base + size + 1
+// as the minimum base value for the next file. The following relationship
+// exists between a Pos value p for a given file offset offs:
+//
+// int(p) = base + offs
+//
+// with offs in the range [0, size] and thus p in the range [base, base+size].
+// For convenience, File.Pos may be used to create file-specific position
+// values from a file offset.
+//
+func (s *FileSet) AddFile(filename string, base, size int) *File {
+ s.mutex.Lock()
+ defer s.mutex.Unlock()
+ if base < s.base || size < 0 {
+ panic("illegal base or size")
+ }
+ // base >= s.base && size >= 0
+ f := &File{s, filename, base, size, []int{0}, nil}
+ base += size + 1 // +1 because EOF also has a position
+ if base < 0 {
+ panic("token.Pos offset overflow (> 2G of source code in file set)")
+ }
+ // add the file to the file set
+ s.base = base
+ s.files = append(s.files, f)
+ s.last = f
+ return f
+}
+
+// Iterate calls f for the files in the file set in the order they were added
+// until f returns false.
+//
+func (s *FileSet) Iterate(f func(*File) bool) {
+ for i := 0; ; i++ {
+ var file *File
+ s.mutex.RLock()
+ if i < len(s.files) {
+ file = s.files[i]
+ }
+ s.mutex.RUnlock()
+ if file == nil || !f(file) {
+ break
+ }
+ }
+}
+
+func searchFiles(a []*File, x int) int {
+ return sort.Search(len(a), func(i int) bool { return a[i].base > x }) - 1
+}
+
+func (s *FileSet) file(p Pos) *File {
+ // common case: p is in last file
+ if f := s.last; f != nil && f.base <= int(p) && int(p) <= f.base+f.size {
+ return f
+ }
+ // p is not in last file - search all files
+ if i := searchFiles(s.files, int(p)); i >= 0 {
+ f := s.files[i]
+ // f.base <= int(p) by definition of searchFiles
+ if int(p) <= f.base+f.size {
+ s.last = f
+ return f
+ }
+ }
+ return nil
+}
+
+// File returns the file that contains the position p.
+// If no such file is found (for instance for p == NoPos),
+// the result is nil.
+//
+func (s *FileSet) File(p Pos) (f *File) {
+ if p != NoPos {
+ s.mutex.RLock()
+ f = s.file(p)
+ s.mutex.RUnlock()
+ }
+ return
+}
+
+// Position converts a Pos in the fileset into a general Position.
+func (s *FileSet) Position(p Pos) (pos Position) {
+ if p != NoPos {
+ s.mutex.RLock()
+ if f := s.file(p); f != nil {
+ pos = f.position(p)
+ }
+ s.mutex.RUnlock()
+ }
+ return
+}
+
+// -----------------------------------------------------------------------------
+// Helper functions
+
+func searchInts(a []int, x int) int {
+ // This function body is a manually inlined version of:
+ //
+ // return sort.Search(len(a), func(i int) bool { return a[i] > x }) - 1
+ //
+ // With better compiler optimizations, this may not be needed in the
+ // future, but at the moment this change improves the go/printer
+ // benchmark performance by ~30%. This has a direct impact on the
+ // speed of gofmt and thus seems worthwhile (2011-04-29).
+ // TODO(gri): Remove this when compilers have caught up.
+ i, j := 0, len(a)
+ for i < j {
+ h := i + (j-i)/2 // avoid overflow when computing h
+ // i ≤ h < j
+ if a[h] <= x {
+ i = h + 1
+ } else {
+ j = h
+ }
+ }
+ return i - 1
+}
diff --git a/token/position_test.go b/token/position_test.go
new file mode 100644
index 0000000..160107d
--- /dev/null
+++ b/token/position_test.go
@@ -0,0 +1,181 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+import (
+ "fmt"
+ "testing"
+)
+
+func checkPos(t *testing.T, msg string, p, q Position) {
+ if p.Filename != q.Filename {
+ t.Errorf("%s: expected filename = %q; got %q", msg, q.Filename, p.Filename)
+ }
+ if p.Offset != q.Offset {
+ t.Errorf("%s: expected offset = %d; got %d", msg, q.Offset, p.Offset)
+ }
+ if p.Line != q.Line {
+ t.Errorf("%s: expected line = %d; got %d", msg, q.Line, p.Line)
+ }
+ if p.Column != q.Column {
+ t.Errorf("%s: expected column = %d; got %d", msg, q.Column, p.Column)
+ }
+}
+
+func TestNoPos(t *testing.T) {
+ if NoPos.IsValid() {
+ t.Errorf("NoPos should not be valid")
+ }
+ var fset *FileSet
+ checkPos(t, "nil NoPos", fset.Position(NoPos), Position{})
+ fset = NewFileSet()
+ checkPos(t, "fset NoPos", fset.Position(NoPos), Position{})
+}
+
+var tests = []struct {
+ filename string
+ source []byte // may be nil
+ size int
+ lines []int
+}{
+ {"a", []byte{}, 0, []int{}},
+ {"b", []byte("01234"), 5, []int{0}},
+ {"c", []byte("\n\n\n\n\n\n\n\n\n"), 9, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}},
+ {"d", nil, 100, []int{0, 5, 10, 20, 30, 70, 71, 72, 80, 85, 90, 99}},
+ {"e", nil, 777, []int{0, 80, 100, 120, 130, 180, 267, 455, 500, 567, 620}},
+ {"f", []byte("package p\n\nimport \"fmt\""), 23, []int{0, 10, 11}},
+ {"g", []byte("package p\n\nimport \"fmt\"\n"), 24, []int{0, 10, 11}},
+ {"h", []byte("package p\n\nimport \"fmt\"\n "), 25, []int{0, 10, 11, 24}},
+}
+
+func linecol(lines []int, offs int) (int, int) {
+ prevLineOffs := 0
+ for line, lineOffs := range lines {
+ if offs < lineOffs {
+ return line, offs - prevLineOffs + 1
+ }
+ prevLineOffs = lineOffs
+ }
+ return len(lines), offs - prevLineOffs + 1
+}
+
+func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
+ for offs := 0; offs < f.Size(); offs++ {
+ p := f.Pos(offs)
+ offs2 := f.Offset(p)
+ if offs2 != offs {
+ t.Errorf("%s, Offset: expected offset %d; got %d", f.Name(), offs, offs2)
+ }
+ line, col := linecol(lines, offs)
+ msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+ checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
+ checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
+ }
+}
+
+func makeTestSource(size int, lines []int) []byte {
+ src := make([]byte, size)
+ for _, offs := range lines {
+ if offs > 0 {
+ src[offs-1] = '\n'
+ }
+ }
+ return src
+}
+
+func TestPositions(t *testing.T) {
+ const delta = 7 // a non-zero base offset increment
+ fset := NewFileSet()
+ for _, test := range tests {
+ // verify consistency of test case
+ if test.source != nil && len(test.source) != test.size {
+ t.Errorf("%s: inconsistent test case: expected file size %d; got %d", test.filename, test.size, len(test.source))
+ }
+
+ // add file and verify name and size
+ f := fset.AddFile(test.filename, fset.Base()+delta, test.size)
+ if f.Name() != test.filename {
+ t.Errorf("expected filename %q; got %q", test.filename, f.Name())
+ }
+ if f.Size() != test.size {
+ t.Errorf("%s: expected file size %d; got %d", f.Name(), test.size, f.Size())
+ }
+ if fset.File(f.Pos(0)) != f {
+ t.Errorf("%s: f.Pos(0) was not found in f", f.Name())
+ }
+
+ // add lines individually and verify all positions
+ for i, offset := range test.lines {
+ f.AddLine(offset)
+ if f.LineCount() != i+1 {
+ t.Errorf("%s, AddLine: expected line count %d; got %d", f.Name(), i+1, f.LineCount())
+ }
+ // adding the same offset again should be ignored
+ f.AddLine(offset)
+ if f.LineCount() != i+1 {
+ t.Errorf("%s, AddLine: expected unchanged line count %d; got %d", f.Name(), i+1, f.LineCount())
+ }
+ verifyPositions(t, fset, f, test.lines[0:i+1])
+ }
+
+ // add lines with SetLines and verify all positions
+ if ok := f.SetLines(test.lines); !ok {
+ t.Errorf("%s: SetLines failed", f.Name())
+ }
+ if f.LineCount() != len(test.lines) {
+ t.Errorf("%s, SetLines: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+ }
+ verifyPositions(t, fset, f, test.lines)
+
+ // add lines with SetLinesForContent and verify all positions
+ src := test.source
+ if src == nil {
+ // no test source available - create one from scratch
+ src = makeTestSource(test.size, test.lines)
+ }
+ f.SetLinesForContent(src)
+ if f.LineCount() != len(test.lines) {
+ t.Errorf("%s, SetLinesForContent: expected line count %d; got %d", f.Name(), len(test.lines), f.LineCount())
+ }
+ verifyPositions(t, fset, f, test.lines)
+ }
+}
+
+func TestLineInfo(t *testing.T) {
+ fset := NewFileSet()
+ f := fset.AddFile("foo", fset.Base(), 500)
+ lines := []int{0, 42, 77, 100, 210, 220, 277, 300, 333, 401}
+ // add lines individually and provide alternative line information
+ for _, offs := range lines {
+ f.AddLine(offs)
+ f.AddLineInfo(offs, "bar", 42)
+ }
+ // verify positions for all offsets
+ for offs := 0; offs <= f.Size(); offs++ {
+ p := f.Pos(offs)
+ _, col := linecol(lines, offs)
+ msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
+ checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
+ checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
+ }
+}
+
+func TestFiles(t *testing.T) {
+ fset := NewFileSet()
+ for i, test := range tests {
+ fset.AddFile(test.filename, fset.Base(), test.size)
+ j := 0
+ fset.Iterate(func(f *File) bool {
+ if f.Name() != tests[j].filename {
+ t.Errorf("expected filename = %s; got %s", tests[j].filename, f.Name())
+ }
+ j++
+ return true
+ })
+ if j != i+1 {
+ t.Errorf("expected %d files; got %d", i+1, j)
+ }
+ }
+}
diff --git a/token/serialize.go b/token/serialize.go
new file mode 100644
index 0000000..4adc8f9
--- /dev/null
+++ b/token/serialize.go
@@ -0,0 +1,56 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+type serializedFile struct {
+ // fields correspond 1:1 to fields with same (lower-case) name in File
+ Name string
+ Base int
+ Size int
+ Lines []int
+ Infos []lineInfo
+}
+
+type serializedFileSet struct {
+ Base int
+ Files []serializedFile
+}
+
+// Read calls decode to deserialize a file set into s; s must not be nil.
+func (s *FileSet) Read(decode func(interface{}) error) error {
+ var ss serializedFileSet
+ if err := decode(&ss); err != nil {
+ return err
+ }
+
+ s.mutex.Lock()
+ s.base = ss.Base
+ files := make([]*File, len(ss.Files))
+ for i := 0; i < len(ss.Files); i++ {
+ f := &ss.Files[i]
+ files[i] = &File{s, f.Name, f.Base, f.Size, f.Lines, f.Infos}
+ }
+ s.files = files
+ s.last = nil
+ s.mutex.Unlock()
+
+ return nil
+}
+
+// Write calls encode to serialize the file set s.
+func (s *FileSet) Write(encode func(interface{}) error) error {
+ var ss serializedFileSet
+
+ s.mutex.Lock()
+ ss.Base = s.base
+ files := make([]serializedFile, len(s.files))
+ for i, f := range s.files {
+ files[i] = serializedFile{f.name, f.base, f.size, f.lines, f.infos}
+ }
+ ss.Files = files
+ s.mutex.Unlock()
+
+ return encode(ss)
+}
diff --git a/token/serialize_test.go b/token/serialize_test.go
new file mode 100644
index 0000000..4e925ad
--- /dev/null
+++ b/token/serialize_test.go
@@ -0,0 +1,111 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package token
+
+import (
+ "bytes"
+ "encoding/gob"
+ "fmt"
+ "testing"
+)
+
+// equal returns nil if p and q describe the same file set;
+// otherwise it returns an error describing the discrepancy.
+func equal(p, q *FileSet) error {
+ if p == q {
+ // avoid deadlock if p == q
+ return nil
+ }
+
+ // not strictly needed for the test
+ p.mutex.Lock()
+ q.mutex.Lock()
+ defer q.mutex.Unlock()
+ defer p.mutex.Unlock()
+
+ if p.base != q.base {
+ return fmt.Errorf("different bases: %d != %d", p.base, q.base)
+ }
+
+ if len(p.files) != len(q.files) {
+ return fmt.Errorf("different number of files: %d != %d", len(p.files), len(q.files))
+ }
+
+ for i, f := range p.files {
+ g := q.files[i]
+ if f.set != p {
+ return fmt.Errorf("wrong fileset for %q", f.name)
+ }
+ if g.set != q {
+ return fmt.Errorf("wrong fileset for %q", g.name)
+ }
+ if f.name != g.name {
+ return fmt.Errorf("different filenames: %q != %q", f.name, g.name)
+ }
+ if f.base != g.base {
+ return fmt.Errorf("different base for %q: %d != %d", f.name, f.base, g.base)
+ }
+ if f.size != g.size {
+ return fmt.Errorf("different size for %q: %d != %d", f.name, f.size, g.size)
+ }
+ for j, l := range f.lines {
+ m := g.lines[j]
+ if l != m {
+ return fmt.Errorf("different offsets for %q", f.name)
+ }
+ }
+ for j, l := range f.infos {
+ m := g.infos[j]
+ if l.Offset != m.Offset || l.Filename != m.Filename || l.Line != m.Line {
+ return fmt.Errorf("different infos for %q", f.name)
+ }
+ }
+ }
+
+ // we don't care about .last - it's just a cache
+ return nil
+}
+
+func checkSerialize(t *testing.T, p *FileSet) {
+ var buf bytes.Buffer
+ encode := func(x interface{}) error {
+ return gob.NewEncoder(&buf).Encode(x)
+ }
+ if err := p.Write(encode); err != nil {
+ t.Errorf("writing fileset failed: %s", err)
+ return
+ }
+ q := NewFileSet()
+ decode := func(x interface{}) error {
+ return gob.NewDecoder(&buf).Decode(x)
+ }
+ if err := q.Read(decode); err != nil {
+ t.Errorf("reading fileset failed: %s", err)
+ return
+ }
+ if err := equal(p, q); err != nil {
+ t.Errorf("filesets not identical: %s", err)
+ }
+}
+
+func TestSerialization(t *testing.T) {
+ p := NewFileSet()
+ checkSerialize(t, p)
+ // add some files
+ for i := 0; i < 10; i++ {
+ f := p.AddFile(fmt.Sprintf("file%d", i), p.Base()+i, i*100)
+ checkSerialize(t, p)
+ // add some lines and alternative file infos
+ line := 1000
+ for offs := 0; offs < f.Size(); offs += 40 + i {
+ f.AddLine(offs)
+ if offs%7 == 0 {
+ f.AddLineInfo(offs, fmt.Sprintf("file%d", offs), line)
+ line += 33
+ }
+ }
+ checkSerialize(t, p)
+ }
+}
diff --git a/token/token.go b/token/token.go
new file mode 100644
index 0000000..b3c7c83
--- /dev/null
+++ b/token/token.go
@@ -0,0 +1,83 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package token defines constants representing the lexical tokens of the gcfg
+// configuration syntax and basic operations on tokens (printing, predicates).
+//
+// Note that the API for the token package may change to accommodate new
+// features or implementation changes in gcfg.
+//
+package token
+
+import "strconv"
+
+// Token is the set of lexical tokens of the gcfg configuration syntax.
+type Token int
+
+// The list of tokens.
+const (
+ // Special tokens
+ ILLEGAL Token = iota
+ EOF
+ COMMENT
+
+ literal_beg
+ // Identifiers and basic type literals
+ // (these tokens stand for classes of literals)
+ IDENT // section-name, variable-name
+ STRING // "subsection-name", variable value
+ literal_end
+
+ operator_beg
+ // Operators and delimiters
+ ASSIGN // =
+ LBRACK // [
+ RBRACK // ]
+ EOL // \n
+ operator_end
+)
+
+var tokens = [...]string{
+ ILLEGAL: "ILLEGAL",
+
+ EOF: "EOF",
+ COMMENT: "COMMENT",
+
+ IDENT: "IDENT",
+ STRING: "STRING",
+
+ ASSIGN: "=",
+ LBRACK: "[",
+ RBRACK: "]",
+ EOL: "\n",
+}
+
+// String returns the string corresponding to the token tok.
+// For operators and delimiters, the string is the actual token character
+// sequence (e.g., for the token ASSIGN, the string is "="). For all other
+// tokens the string corresponds to the token constant name (e.g. for the
+// token IDENT, the string is "IDENT").
+//
+func (tok Token) String() string {
+ s := ""
+ if 0 <= tok && tok < Token(len(tokens)) {
+ s = tokens[tok]
+ }
+ if s == "" {
+ s = "token(" + strconv.Itoa(int(tok)) + ")"
+ }
+ return s
+}
+
+// Predicates
+
+// IsLiteral returns true for tokens corresponding to identifiers
+// and basic type literals; it returns false otherwise.
+//
+func (tok Token) IsLiteral() bool { return literal_beg < tok && tok < literal_end }
+
+// IsOperator returns true for tokens corresponding to operators and
+// delimiters; it returns false otherwise.
+//
+func (tok Token) IsOperator() bool { return operator_beg < tok && tok < operator_end }
