| // Copyright 2020 The LUCI Authors. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| // Package buildifier implements processing of Starlark files via buildifier. |
| // |
| // Buildifier is primarily intended for Bazel files. We try to disable as much |
| // of Bazel-specific logic as possible, keeping only generally useful |
| // Starlark rules. |
| package buildifier |
| |
| import ( |
| "bytes" |
| "fmt" |
| "runtime" |
| "strings" |
| "sync" |
| |
| "github.com/bazelbuild/buildtools/build" |
| "github.com/bazelbuild/buildtools/tables" |
| "github.com/bazelbuild/buildtools/warn" |
| |
| "go.chromium.org/luci/common/data/stringset" |
| "go.chromium.org/luci/common/errors" |
| "go.chromium.org/luci/common/sync/parallel" |
| "go.chromium.org/luci/starlark/interpreter" |
| ) |
| |
| var ( |
| // ErrActionableFindings is returned by Lint if there are actionable findings. |
| ErrActionableFindings = errors.New("some *.star files have linter warnings, please fix them") |
| ) |
| |
| // formattingCategory is linter check to represent `lucicfg fmt` checks. |
| // |
| // It's not a real buildifier category, we should be careful not to pass it to |
| // warn.FileWarnings. |
| const formattingCategory = "formatting" |
| |
| // Finding is information about one linting or formatting error. |
| // |
| // Implements error interface. Non-actionable findings are assumed to be |
| // non-blocking errors. |
| type Finding struct { |
| Path string `json:"path"` |
| Start *Position `json:"start,omitempty"` |
| End *Position `json:"end,omitempty"` |
| Category string `json:"string,omitempty"` |
| Message string `json:"message,omitempty"` |
| Actionable bool `json:"actionable,omitempty"` |
| } |
| |
| // Position indicates a position within a file. |
| type Position struct { |
| Line int `json:"line"` // starting from 1 |
| Column int `json:"column` // in runes, starting from 1 |
| Offset int `json:"offset"` // absolute offset in bytes |
| } |
| |
| // Error returns a short summary of the finding. |
| func (f *Finding) Error() string { |
| switch { |
| case f.Path == "": |
| return f.Category |
| case f.Start == nil: |
| return fmt.Sprintf("%s: %s", f.Path, f.Category) |
| default: |
| return fmt.Sprintf("%s:%d: %s", f.Path, f.Start.Line, f.Category) |
| } |
| } |
| |
| // Format returns a detailed reported that can be printed to stderr. |
| func (f *Finding) Format() string { |
| if strings.ContainsRune(f.Message, '\n') { |
| return fmt.Sprintf("%s: %s\n\n", f.Error(), f.Message) |
| } else { |
| return fmt.Sprintf("%s: %s\n", f.Error(), f.Message) |
| } |
| } |
| |
| // Lint appliers linting and formatting checks to the given files. |
| // |
| // Returns all findings and a non-nil error (usually a MultiError) if some |
| // findings are blocking. |
| func Lint(loader interpreter.Loader, paths []string, lintChecks []string) (findings []*Finding, err error) { |
| checks, err := normalizeLintChecks(lintChecks) |
| if err != nil { |
| return nil, err |
| } |
| |
| // Transform unrecognized linter checks into warning-level findings. |
| allPossible := allChecks() |
| buildifierWarns := make([]string, 0, checks.Len()) |
| checkFmt := false |
| for _, check := range checks.ToSortedSlice() { |
| switch { |
| case !allPossible.Has(check): |
| findings = append(findings, &Finding{ |
| Category: "linter", |
| Message: fmt.Sprintf("Unknown linter check %q", check), |
| }) |
| case check == formattingCategory: |
| checkFmt = true |
| default: |
| buildifierWarns = append(buildifierWarns, check) |
| } |
| } |
| |
| if len(paths) == 0 || (!checkFmt && len(buildifierWarns) == 0) { |
| return findings, nil |
| } |
| |
| errs := Visit(loader, paths, func(path string, body []byte, f *build.File) (merr errors.MultiError) { |
| if len(buildifierWarns) != 0 { |
| findings := warn.FileWarnings(f, buildifierWarns, nil, warn.ModeWarn, newFileReader(loader)) |
| for _, f := range findings { |
| merr = append(merr, &Finding{ |
| Path: path, |
| Start: &Position{ |
| Line: f.Start.Line, |
| Column: f.Start.LineRune, |
| Offset: f.Start.Byte, |
| }, |
| End: &Position{ |
| Line: f.End.Line, |
| Column: f.End.LineRune, |
| Offset: f.End.Byte, |
| }, |
| Category: f.Category, |
| Message: f.Message, |
| Actionable: f.Actionable, |
| }) |
| } |
| } |
| if checkFmt && !bytes.Equal(build.Format(f), body) { |
| merr = append(merr, &Finding{ |
| Path: path, |
| Category: formattingCategory, |
| Message: `The file is not properly formatted, use 'lucicfg fmt' to format it.`, |
| Actionable: true, |
| }) |
| } |
| return merr |
| }) |
| if len(errs) == 0 { |
| return findings, nil |
| } |
| |
| // Extract findings into a dedicated slice. Return an overall error if there |
| // are actionable findings. |
| filtered := errs[:0] |
| actionable := false |
| for _, err := range errs { |
| if f, ok := err.(*Finding); ok { |
| findings = append(findings, f) |
| if f.Actionable { |
| actionable = true |
| } |
| } else { |
| filtered = append(filtered, err) |
| } |
| } |
| if actionable { |
| filtered = append(filtered, ErrActionableFindings) |
| } |
| |
| if len(filtered) == 0 { |
| return findings, nil |
| } |
| return findings, filtered |
| } |
| |
| // Visitor processes a parsed Starlark file, returning all errors encountered |
| // when processing it. |
| type Visitor func(path string, body []byte, f *build.File) errors.MultiError |
| |
| // Visit parses Starlark files using Buildifier and calls the callback for each |
| // parsed file, in parallel. |
| // |
| // Collects all errors from all callbacks in a single joint multi-error. |
| func Visit(loader interpreter.Loader, paths []string, v Visitor) errors.MultiError { |
| initTables() |
| |
| m := sync.Mutex{} |
| perPath := make(map[string]errors.MultiError, len(paths)) |
| |
| parallel.WorkPool(runtime.NumCPU(), func(tasks chan<- func() error) { |
| for _, path := range paths { |
| path := path |
| tasks <- func() error { |
| var errs []error |
| switch body, f, err := parseFile(loader, path); { |
| case err != nil: |
| errs = []error{err} |
| case f != nil: |
| errs = v(path, body, f) |
| } |
| m.Lock() |
| perPath[path] = errs |
| m.Unlock() |
| return nil |
| } |
| } |
| }) |
| |
| // Assemble errors in original order. |
| var errs errors.MultiError |
| for _, path := range paths { |
| errs = append(errs, perPath[path]...) |
| } |
| return errs |
| } |
| |
| var tablesOnce sync.Once |
| |
| // initTables tweaks Buildifier to forget as much as possible about Bazel rules. |
| func initTables() { |
| tablesOnce.Do(func() { |
| tables.OverrideTables(nil, nil, nil, nil, nil, nil, nil, false, false) |
| }) |
| } |
| |
| // parseFile parses a Starlark module using the buildifier parser. |
| // |
| // Returns (nil, nil, nil) if the module is a native Go module. |
| func parseFile(loader interpreter.Loader, path string) ([]byte, *build.File, error) { |
| switch dict, src, err := loader(path); { |
| case err != nil: |
| return nil, nil, err |
| case dict != nil: |
| return nil, nil, nil |
| default: |
| body := []byte(src) |
| f, err := build.ParseDefault(path, body) |
| if f != nil { |
| f.Type = build.TypeDefault // always generic Starlark file, not a BUILD |
| f.Label = path // lucicfg loader paths ~= map to Bazel labels |
| } |
| return body, f, err |
| } |
| } |
| |
| // newFileReader returns a warn.FileReader based on the loader. |
| // |
| // Note: *warn.FileReader doesn't protect its caching guts with any locks so we |
| // can't share a single copy across multiple goroutines. |
| func newFileReader(loader interpreter.Loader) *warn.FileReader { |
| return warn.NewFileReader(func(path string) ([]byte, error) { |
| switch dict, src, err := loader(path); { |
| case err != nil: |
| return nil, err |
| case dict != nil: |
| return nil, nil // skip native modules |
| default: |
| return []byte(src), nil |
| } |
| }) |
| } |
| |
| // normalizeLintChecks replaces `all` with an explicit list of checks and does |
| // other similar transformations. |
| // |
| // Checks has a form ["<optional initial category>", "+warn", "-warn", ...]. |
| // Where <optional initial category> can be `none`, `default` or `all`. |
| // |
| // Doesn't check all added checks are actually defined. |
| func normalizeLintChecks(checks []string) (stringset.Set, error) { |
| if len(checks) == 0 { |
| checks = []string{"default"} |
| } |
| |
| var set stringset.Set |
| if cat := checks[0]; !strings.HasPrefix(cat, "+") && !strings.HasPrefix(cat, "-") { |
| switch cat { |
| case "none": |
| set = stringset.New(0) |
| case "all": |
| set = allChecks() |
| case "default": |
| set = defaultChecks() |
| default: |
| return nil, fmt.Errorf( |
| `unrecognized linter checks category %q: must be one of "none", "all", "default" `+ |
| `(if you want to enable individual checks, use "+name" syntax)`, cat) |
| } |
| checks = checks[1:] |
| } else { |
| set = defaultChecks() |
| } |
| |
| for _, check := range checks { |
| switch { |
| case strings.HasPrefix(check, "+"): |
| set.Add(check[1:]) |
| case strings.HasPrefix(check, "-"): |
| set.Del(check[1:]) |
| default: |
| return nil, fmt.Errorf(`use "+name" to enable a check or "-name" to disable it, got %q instead`, check) |
| } |
| } |
| |
| return set, nil |
| } |
| |
| func allChecks() stringset.Set { |
| s := stringset.NewFromSlice(warn.AllWarnings...) |
| s.Add(formattingCategory) |
| return s |
| } |
| |
| func defaultChecks() stringset.Set { |
| s := stringset.NewFromSlice(warn.DefaultWarnings...) |
| s.Add(formattingCategory) |
| s.Del("load-on-top") // order of loads may matter in lucicfg |
| s.Del("uninitialized") // this check doesn't work well with lambdas and inner functions |
| return s |
| } |