blob: b280459f66e335bd21416db8354e695b1238a64b [file] [log] [blame]
// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package lucicfg
import (
"bufio"
"bytes"
"context"
"fmt"
"io/ioutil"
"path/filepath"
"regexp"
"strings"
"testing"
"github.com/sergi/go-diff/diffmatchpatch"
"go.starlark.net/resolve"
"go.starlark.net/starlark"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/starlark/interpreter"
"go.chromium.org/luci/starlark/starlarktest"
)
func init() {
// Enable not-yet-standard features.
resolve.AllowLambda = true
resolve.AllowNestedDef = true
resolve.AllowFloat = true
resolve.AllowSet = true
}
// TestAllStarlark loads and executes all test scripts (testdata/*.star).
func TestAllStarlark(t *testing.T) {
t.Parallel()
starlarktest.RunTests(t, starlarktest.Options{
TestsDir: "testdata",
Skip: "support",
Executor: func(t *testing.T, path string, predeclared starlark.StringDict) error {
blob, err := ioutil.ReadFile(path)
if err != nil {
return err
}
body := string(blob)
expectErrExct := readCommentBlock(body, "Expect errors:")
expectErrLike := readCommentBlock(body, "Expect errors like:")
expectCfg := readCommentBlock(body, "Expect configs:")
if expectErrExct != "" && expectErrLike != "" {
t.Errorf("Cannot use 'Expect errors' and 'Expect errors like' at the same time")
return nil
}
// We treat tests that compare the generator output to some expected
// output as "integration tests", and everything else is a unit tests.
// See below for why this is important.
integrationTest := expectErrExct != "" || expectErrLike != "" || expectCfg != ""
state, err := Generate(context.Background(), Inputs{
// Use file system loader so test scripts can load supporting scripts
// (from '**/support/*' which is skipped by the test runner). This also
// makes error messages have the original scripts full name. Note that
// 'go test' executes tests with cwd set to corresponding package
// directories, regardless of what cwd was when 'go test' was called.
Code: interpreter.FileSystemLoader("."),
Entry: filepath.ToSlash(path),
// Expose 'assert' module, hook up error reporting to 't'.
testPredeclared: predeclared,
testThreadModifier: func(th *starlark.Thread) {
starlarktest.HookThread(th, t)
},
// Failure collector interferes with assert.fails() in a bad way.
// assert.fails() captures errors, but it doesn't clear the failure
// collector state, so we may end up in a situation when the script
// fails with one error (some native starlark error, e.g. invalid
// function call, not 'fail'), but the failure collector remembers
// another (stale!) error, emitted by 'fail' before and caught by
// assert.fails(). This results in invalid error message at the end
// of the script execution.
//
// Unfortunately, it is not easy to modify assert.fails() without
// forking it. So instead we do a cheesy thing and disable the failure
// collector if the file under test appears to be unit-testy (rather
// than integration-testy). We define integration tests to be tests
// that examine the output of the generator using "Expect ..." blocks
// (see above), and unit tests are tests that use asserts.
//
// Disabling the failure collector results in fail(..., trace=t)
// ignoring the custom stack trace 't'. But unit tests don't generally
// check the stack trace (only the error message), so it's not a big
// deal for them.
testDisableFailureCollector: !integrationTest,
})
// If test was expected to fail on Starlark side, make sure it did, in
// an expected way.
if expectErrExct != "" || expectErrLike != "" {
allErrs := strings.Builder{}
errors.WalkLeaves(err, func(err error) bool {
if bt, ok := err.(BacktracableError); ok {
allErrs.WriteString(bt.Backtrace())
} else {
allErrs.WriteString(err.Error())
}
allErrs.WriteString("\n\n")
return true
})
if expectErrExct != "" {
errorOnDiff(t, allErrs.String(), expectErrExct)
} else {
errorOnPatternMismatch(t, allErrs.String(), expectErrLike)
}
return nil
}
// Otherwise just report all errors to Mr. T.
errors.WalkLeaves(err, func(err error) bool {
if bt, ok := err.(BacktracableError); ok {
t.Errorf("%s\n", bt.Backtrace())
} else {
t.Errorf("%s\n", err)
}
return true
})
if err != nil {
return nil // the error has been reported already
}
// If was expecting to see some configs, assert we did see them.
if expectCfg != "" {
got := bytes.Buffer{}
for _, f := range state.Output.Files() {
fmt.Fprintf(&got, "=== %s\n", f)
fmt.Fprintf(&got, string(state.Output.Data[f]))
fmt.Fprintf(&got, "===\n\n")
}
errorOnDiff(t, got.String(), expectCfg)
}
return nil
},
})
}
// readCommentBlock reads a comment block that start with "# <hdr>\n".
//
// Return empty string if there's no such block.
func readCommentBlock(script, hdr string) string {
scanner := bufio.NewScanner(strings.NewReader(script))
for scanner.Scan() && scanner.Text() != "# "+hdr {
continue
}
sb := strings.Builder{}
for scanner.Scan() {
if line := scanner.Text(); strings.HasPrefix(line, "#") {
sb.WriteString(strings.TrimPrefix(line[1:], " "))
sb.WriteRune('\n')
}
}
return sb.String()
}
// errorOnDiff emits an error to T if got != exp.
func errorOnDiff(t *testing.T, got, exp string) {
t.Helper()
got = strings.TrimSpace(got)
exp = strings.TrimSpace(exp)
switch {
case got == "":
t.Errorf("Got nothing, but was expecting:\n\n%s\n", exp)
case got != exp:
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(exp, got, false)
t.Errorf(
"Got:\n\n%s\n\nWas expecting:\n\n%s\n\nDiff:\n\n%s\n",
got, exp, dmp.DiffPrettyText(diffs))
}
}
// errorOnMismatch emits an error to T if got doesn't match a pattern pat.
//
// The pattern is syntax is:
// * A line "[space]...[space]" matches zero or more arbitrary lines.
// * Trigram "???" matches [0-9a-zA-Z]+.
// * The rest should match as is.
func errorOnPatternMismatch(t *testing.T, got, pat string) {
t.Helper()
got = strings.TrimSpace(got)
pat = strings.TrimSpace(pat)
re := strings.Builder{}
re.WriteRune('^')
for _, line := range strings.Split(pat, "\n") {
if strings.TrimSpace(line) == "..." {
re.WriteString(`(.*\n)*`)
} else {
for line != "" {
idx := strings.Index(line, "???")
if idx == -1 {
re.WriteString(regexp.QuoteMeta(line))
break
}
re.WriteString(regexp.QuoteMeta(line[:idx]))
re.WriteString(`[0-9a-zA-Z]+`)
line = line[idx+3:]
}
re.WriteString(`\n`)
}
}
re.WriteRune('$')
if exp := regexp.MustCompile(re.String()); !exp.MatchString(got + "\n") {
t.Errorf("Got:\n\n%s\n\nWas expecting pattern:\n\n%s\n\n", got, pat)
t.Errorf("Regexp: %s", re.String())
}
}