lucicfg/starlark_test.go - infra/luci/luci-go - Git at Google

 // Copyright 2018 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package lucicfg

 import (
 	"bufio"
 	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"

 	"github.com/sergi/go-diff/diffmatchpatch"

 	"go.starlark.net/resolve"
 	"go.starlark.net/starlark"

 	"go.chromium.org/luci/common/errors"
 	"go.chromium.org/luci/starlark/builtins"
 	"go.chromium.org/luci/starlark/interpreter"
 	"go.chromium.org/luci/starlark/starlarktest"
 )

 // If this env var is 1, the test will regenerate the "Expect configs:" part of
 // test *.star files.
 const RegenEnvVar = "LUCICFG_TEST_REGEN"

 const (
 	expectConfigsHeader    = "Expect configs:"
 	expectErrorsHeader     = "Expect errors:"
 	expectErrorsLikeHeader = "Expect errors like:"
 )

 func init() {
 	// Enable not-yet-standard features.
 	resolve.AllowLambda = true
 	resolve.AllowNestedDef = true
 	resolve.AllowFloat = true
 	resolve.AllowSet = true
 }

 // TestAllStarlark loads and executes all test scripts (testdata/*.star).
 func TestAllStarlark(t *testing.T) {
 	t.Parallel()

 	gotExpectationErrors := false

 	starlarktest.RunTests(t, starlarktest.Options{
 		TestsDir: "testdata",
 		Skip:     "support",

 		Executor: func(t *testing.T, path string, predeclared starlark.StringDict) error {
 			blob, err := ioutil.ReadFile(path)
 			if err != nil {
 				return err
 			}
 			body := string(blob)

 			// Read "mocked" `-var name=value` assignments.
 			presetVars := map[string]string{}
 			presetVarsBlock := readCommentBlock(body, "Prepare CLI vars as:")
 			for _, line := range strings.Split(presetVarsBlock, "\n") {
 				if line = strings.TrimSpace(line); line != "" {
 					chunks := strings.SplitN(line, "=", 2)
 					if len(chunks) != 2 {
 						t.Errorf("Bad CLI var declaration %q", line)
 						return nil
 					}
 					presetVars[chunks[0]] = chunks[1]
 				}
 			}

 			expectErrExct := readCommentBlock(body, expectErrorsHeader)
 			expectErrLike := readCommentBlock(body, expectErrorsLikeHeader)
 			expectCfg := readCommentBlock(body, expectConfigsHeader)
 			if expectErrExct != "" && expectErrLike != "" {
 				t.Errorf("Cannot use %q and %q at the same time", expectErrorsHeader, expectErrorsLikeHeader)
 				return nil
 			}

 			// We treat tests that compare the generator output to some expected
 			// output as "integration tests", and everything else is a unit tests.
 			// See below for why this is important.
 			integrationTest := expectErrExct != "" || expectErrLike != "" || expectCfg != ""

 			state, err := Generate(context.Background(), Inputs{
 				// Use file system loader so test scripts can load supporting scripts
 				// (from '**/support/*' which is skipped by the test runner). This also
 				// makes error messages have the original scripts full name. Note that
 				// 'go test' executes tests with cwd set to corresponding package
 				// directories, regardless of what cwd was when 'go test' was called.
 				Code:  interpreter.FileSystemLoader("."),
 				Entry: filepath.ToSlash(path),
 				Vars:  presetVars,

 				// Expose 'assert' module, hook up error reporting to 't'.
 				testPredeclared: predeclared,
 				testThreadModifier: func(th *starlark.Thread) {
 					starlarktest.HookThread(th, t)
 				},

 				// Don't spit out "# This file is generated by lucicfg" headers.
 				testOmitHeader: true,

 				// Failure collector interferes with assert.fails() in a bad way.
 				// assert.fails() captures errors, but it doesn't clear the failure
 				// collector state, so we may end up in a situation when the script
 				// fails with one error (some native starlark error, e.g. invalid
 				// function call, not 'fail'), but the failure collector remembers
 				// another (stale!) error, emitted by 'fail' before and caught by
 				// assert.fails(). This results in invalid error message at the end
 				// of the script execution.
 				//
 				// Unfortunately, it is not easy to modify assert.fails() without
 				// forking it. So instead we do a cheesy thing and disable the failure
 				// collector if the file under test appears to be unit-testy (rather
 				// than integration-testy). We define integration tests to be tests
 				// that examine the output of the generator using "Expect ..." blocks
 				// (see above), and unit tests are tests that use asserts.
 				//
 				// Disabling the failure collector results in fail(..., trace=t)
 				// ignoring the custom stack trace 't'. But unit tests don't generally
 				// check the stack trace (only the error message), so it's not a big
 				// deal for them.
 				testDisableFailureCollector: !integrationTest,

 				// Do not put frequently changing version string into test outputs.
 				testVersion: "1.1.1",
 			})

 			// If test was expected to fail on Starlark side, make sure it did, in
 			// an expected way.
 			if expectErrExct != "" || expectErrLike != "" {
 				allErrs := strings.Builder{}
 				var skip bool
 				errors.Walk(err, func(err error) bool {
 					if skip {
 						skip = false
 						return true
 					}

 					if bt, ok := err.(BacktracableError); ok {
 						allErrs.WriteString(bt.Backtrace())
 						// We need to skip Unwrap from starlark.EvalError
 						_, skip = err.(*starlark.EvalError)
 					} else {
 						switch err.(type) {
 						case errors.MultiError, errors.Wrapped:
 							return true
 						}

 						allErrs.WriteString(err.Error())
 					}
 					allErrs.WriteString("\n\n")
 					return true
 				})

 				// Strip line and column numbers from backtraces.
 				normalized := builtins.NormalizeStacktrace(allErrs.String())

 				if expectErrExct != "" {
 					errorOnDiff(t, normalized, expectErrExct)
 				} else {
 					errorOnPatternMismatch(t, normalized, expectErrLike)
 				}
 				return nil
 			}

 			// Otherwise just report all errors to Mr. T.
 			errors.WalkLeaves(err, func(err error) bool {
 				if bt, ok := err.(BacktracableError); ok {
 					t.Errorf("%s\n", bt.Backtrace())
 				} else {
 					t.Errorf("%s\n", err)
 				}
 				return true
 			})
 			if err != nil {
 				return nil // the error has been reported already
 			}

 			// If was expecting to see some configs, assert we did see them.
 			if expectCfg != "" {
 				got := bytes.Buffer{}
 				for idx, f := range state.Output.Files() {
 					if idx != 0 {
 						fmt.Fprintf(&got, "\n\n")
 					}
 					fmt.Fprintf(&got, "=== %s\n", f)
 					if blob, err := state.Output.Data[f].Bytes(); err != nil {
 						t.Errorf("Serializing %s: %s", f, err)
 					} else {
 						fmt.Fprintf(&got, string(blob))
 					}
 					fmt.Fprintf(&got, "===")
 				}
 				if os.Getenv(RegenEnvVar) == "1" {
 					if err := updateExpected(path, got.String()); err != nil {
 						t.Errorf("Failed to updated %q: %s", path, err)
 					}
 				} else if errorOnDiff(t, got.String(), expectCfg) {
 					gotExpectationErrors = true
 				}
 			}

 			return nil
 		},
 	})

 	if gotExpectationErrors {
 		t.Errorf("\n\n"+
 			"========================================================\n"+
 			"If you want to update expectations stored in *.star run:\n"+
 			"$ %s=1 go test .\n"+
 			"========================================================", RegenEnvVar)
 	}
 }

 // readCommentBlock reads a comment block that start with "# <hdr>\n".
 //
 // Returns empty string if there's no such block.
 func readCommentBlock(script, hdr string) string {
 	scanner := bufio.NewScanner(strings.NewReader(script))
 	for scanner.Scan() && scanner.Text() != "# "+hdr {
 		continue
 	}
 	sb := strings.Builder{}
 	for scanner.Scan() {
 		if line := scanner.Text(); strings.HasPrefix(line, "#") {
 			sb.WriteString(strings.TrimPrefix(line[1:], " "))
 			sb.WriteRune('\n')
 		} else {
 			break // the comment block has ended
 		}
 	}
 	return sb.String()
 }

 // updateExpected updates the expected generated config stored in the comment
 // block at the end of the *.star file.
 func updateExpected(path, exp string) error {
 	blob, err := ioutil.ReadFile(path)
 	if err != nil {
 		return err
 	}

 	idx := bytes.Index(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader)))
 	if idx == -1 {
 		return errors.Reason("doesn't have `Expect configs` comment block").Err()
 	}
 	blob = blob[:idx]

 	blob = append(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader))...)
 	blob = append(blob, []byte("#\n")...)
 	for _, line := range strings.Split(exp, "\n") {
 		if len(line) == 0 {
 			blob = append(blob, '#')
 		} else {
 			blob = append(blob, []byte("# ")...)
 			blob = append(blob, []byte(line)...)
 		}
 		blob = append(blob, '\n')
 	}

 	return ioutil.WriteFile(path, blob, 0666)
 }

 // errorOnDiff emits an error to T and returns true if got != exp.
 func errorOnDiff(t *testing.T, got, exp string) bool {
 	t.Helper()

 	got = strings.TrimSpace(got)
 	exp = strings.TrimSpace(exp)

 	switch {
 	case got == "":
 		t.Errorf("Got nothing, but was expecting:\n\n%s\n", exp)
 		return true
 	case got != exp:
 		dmp := diffmatchpatch.New()
 		diffs := dmp.DiffMain(exp, got, false)
 		t.Errorf(
 			"Got:\n\n%s\n\nWas expecting:\n\n%s\n\nDiff:\n\n%s\n",
 			got, exp, dmp.DiffPrettyText(diffs))
 		return true
 	}

 	return false
 }

 // errorOnMismatch emits an error to T if got doesn't match a pattern pat.
 //
 // The pattern is syntax is:
 //   * A line "[space]...[space]" matches zero or more arbitrary lines.
 //   * Trigram "???" matches [0-9a-zA-Z]+.
 //   * The rest should match as is.
 func errorOnPatternMismatch(t *testing.T, got, pat string) {
 	t.Helper()

 	got = strings.TrimSpace(got)
 	pat = strings.TrimSpace(pat)

 	re := strings.Builder{}
 	re.WriteRune('^')
 	for _, line := range strings.Split(pat, "\n") {
 		if strings.TrimSpace(line) == "..." {
 			re.WriteString(`(.*\n)*`)
 		} else {
 			for line != "" {
 				idx := strings.Index(line, "???")
 				if idx == -1 {
 					re.WriteString(regexp.QuoteMeta(line))
 					break
 				}
 				re.WriteString(regexp.QuoteMeta(line[:idx]))
 				re.WriteString(`[0-9a-zA-Z]+`)
 				line = line[idx+3:]
 			}
 			re.WriteString(`\n`)
 		}
 	}
 	re.WriteRune('$')

 	if exp := regexp.MustCompile(re.String()); !exp.MatchString(got + "\n") {
 		t.Errorf("Got:\n\n%s\n\nWas expecting pattern:\n\n%s\n\n", got, pat)
 		t.Errorf("Regexp: %s", re.String())
 	}
 }
	// Copyright 2018 The LUCI Authors.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package lucicfg

	import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"

	"go.starlark.net/resolve"
	"go.starlark.net/starlark"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/starlark/builtins"
	"go.chromium.org/luci/starlark/interpreter"
	"go.chromium.org/luci/starlark/starlarktest"
	)

	// If this env var is 1, the test will regenerate the "Expect configs:" part of
	// test *.star files.
	const RegenEnvVar = "LUCICFG_TEST_REGEN"

	const (
	expectConfigsHeader = "Expect configs:"
	expectErrorsHeader = "Expect errors:"
	expectErrorsLikeHeader = "Expect errors like:"
	)

	func init() {
	// Enable not-yet-standard features.
	resolve.AllowLambda = true
	resolve.AllowNestedDef = true
	resolve.AllowFloat = true
	resolve.AllowSet = true
	}

	// TestAllStarlark loads and executes all test scripts (testdata/*.star).
	func TestAllStarlark(t *testing.T) {
	t.Parallel()

	gotExpectationErrors := false

	starlarktest.RunTests(t, starlarktest.Options{
	TestsDir: "testdata",
	Skip: "support",

	Executor: func(t *testing.T, path string, predeclared starlark.StringDict) error {
	blob, err := ioutil.ReadFile(path)
	if err != nil {
	return err
	}
	body := string(blob)

	// Read "mocked" `-var name=value` assignments.
	presetVars := map[string]string{}
	presetVarsBlock := readCommentBlock(body, "Prepare CLI vars as:")
	for _, line := range strings.Split(presetVarsBlock, "\n") {
	if line = strings.TrimSpace(line); line != "" {
	chunks := strings.SplitN(line, "=", 2)
	if len(chunks) != 2 {
	t.Errorf("Bad CLI var declaration %q", line)
	return nil
	}
	presetVars[chunks[0]] = chunks[1]
	}
	}

	expectErrExct := readCommentBlock(body, expectErrorsHeader)
	expectErrLike := readCommentBlock(body, expectErrorsLikeHeader)
	expectCfg := readCommentBlock(body, expectConfigsHeader)
	if expectErrExct != "" && expectErrLike != "" {
	t.Errorf("Cannot use %q and %q at the same time", expectErrorsHeader, expectErrorsLikeHeader)
	return nil
	}

	// We treat tests that compare the generator output to some expected
	// output as "integration tests", and everything else is a unit tests.
	// See below for why this is important.
	integrationTest := expectErrExct != "" \|\| expectErrLike != "" \|\| expectCfg != ""

	state, err := Generate(context.Background(), Inputs{
	// Use file system loader so test scripts can load supporting scripts
	// (from '*/support/' which is skipped by the test runner). This also
	// makes error messages have the original scripts full name. Note that
	// 'go test' executes tests with cwd set to corresponding package
	// directories, regardless of what cwd was when 'go test' was called.
	Code: interpreter.FileSystemLoader("."),
	Entry: filepath.ToSlash(path),
	Vars: presetVars,

	// Expose 'assert' module, hook up error reporting to 't'.
	testPredeclared: predeclared,
	testThreadModifier: func(th *starlark.Thread) {
	starlarktest.HookThread(th, t)
	},

	// Don't spit out "# This file is generated by lucicfg" headers.
	testOmitHeader: true,

	// Failure collector interferes with assert.fails() in a bad way.
	// assert.fails() captures errors, but it doesn't clear the failure
	// collector state, so we may end up in a situation when the script
	// fails with one error (some native starlark error, e.g. invalid
	// function call, not 'fail'), but the failure collector remembers
	// another (stale!) error, emitted by 'fail' before and caught by
	// assert.fails(). This results in invalid error message at the end
	// of the script execution.
	//
	// Unfortunately, it is not easy to modify assert.fails() without
	// forking it. So instead we do a cheesy thing and disable the failure
	// collector if the file under test appears to be unit-testy (rather
	// than integration-testy). We define integration tests to be tests
	// that examine the output of the generator using "Expect ..." blocks
	// (see above), and unit tests are tests that use asserts.
	//
	// Disabling the failure collector results in fail(..., trace=t)
	// ignoring the custom stack trace 't'. But unit tests don't generally
	// check the stack trace (only the error message), so it's not a big
	// deal for them.
	testDisableFailureCollector: !integrationTest,

	// Do not put frequently changing version string into test outputs.
	testVersion: "1.1.1",
	})

	// If test was expected to fail on Starlark side, make sure it did, in
	// an expected way.
	if expectErrExct != "" \|\| expectErrLike != "" {
	allErrs := strings.Builder{}
	var skip bool
	errors.Walk(err, func(err error) bool {
	if skip {
	skip = false
	return true
	}

	if bt, ok := err.(BacktracableError); ok {
	allErrs.WriteString(bt.Backtrace())
	// We need to skip Unwrap from starlark.EvalError
	_, skip = err.(*starlark.EvalError)
	} else {
	switch err.(type) {
	case errors.MultiError, errors.Wrapped:
	return true
	}

	allErrs.WriteString(err.Error())
	}
	allErrs.WriteString("\n\n")
	return true
	})

	// Strip line and column numbers from backtraces.
	normalized := builtins.NormalizeStacktrace(allErrs.String())

	if expectErrExct != "" {
	errorOnDiff(t, normalized, expectErrExct)
	} else {
	errorOnPatternMismatch(t, normalized, expectErrLike)
	}
	return nil
	}

	// Otherwise just report all errors to Mr. T.
	errors.WalkLeaves(err, func(err error) bool {
	if bt, ok := err.(BacktracableError); ok {
	t.Errorf("%s\n", bt.Backtrace())
	} else {
	t.Errorf("%s\n", err)
	}
	return true
	})
	if err != nil {
	return nil // the error has been reported already
	}

	// If was expecting to see some configs, assert we did see them.
	if expectCfg != "" {
	got := bytes.Buffer{}
	for idx, f := range state.Output.Files() {
	if idx != 0 {
	fmt.Fprintf(&got, "\n\n")
	}
	fmt.Fprintf(&got, "=== %s\n", f)
	if blob, err := state.Output.Data[f].Bytes(); err != nil {
	t.Errorf("Serializing %s: %s", f, err)
	} else {
	fmt.Fprintf(&got, string(blob))
	}
	fmt.Fprintf(&got, "===")
	}
	if os.Getenv(RegenEnvVar) == "1" {
	if err := updateExpected(path, got.String()); err != nil {
	t.Errorf("Failed to updated %q: %s", path, err)
	}
	} else if errorOnDiff(t, got.String(), expectCfg) {
	gotExpectationErrors = true
	}
	}

	return nil
	},
	})

	if gotExpectationErrors {
	t.Errorf("\n\n"+
	"========================================================\n"+
	"If you want to update expectations stored in *.star run:\n"+
	"$ %s=1 go test .\n"+
	"========================================================", RegenEnvVar)
	}
	}

	// readCommentBlock reads a comment block that start with "# <hdr>\n".
	//
	// Returns empty string if there's no such block.
	func readCommentBlock(script, hdr string) string {
	scanner := bufio.NewScanner(strings.NewReader(script))
	for scanner.Scan() && scanner.Text() != "# "+hdr {
	continue
	}
	sb := strings.Builder{}
	for scanner.Scan() {
	if line := scanner.Text(); strings.HasPrefix(line, "#") {
	sb.WriteString(strings.TrimPrefix(line[1:], " "))
	sb.WriteRune('\n')
	} else {
	break // the comment block has ended
	}
	}
	return sb.String()
	}

	// updateExpected updates the expected generated config stored in the comment
	// block at the end of the *.star file.
	func updateExpected(path, exp string) error {
	blob, err := ioutil.ReadFile(path)
	if err != nil {
	return err
	}

	idx := bytes.Index(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader)))
	if idx == -1 {
	return errors.Reason("doesn't have `Expect configs` comment block").Err()
	}
	blob = blob[:idx]

	blob = append(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader))...)
	blob = append(blob, []byte("#\n")...)
	for _, line := range strings.Split(exp, "\n") {
	if len(line) == 0 {
	blob = append(blob, '#')
	} else {
	blob = append(blob, []byte("# ")...)
	blob = append(blob, []byte(line)...)
	}
	blob = append(blob, '\n')
	}

	return ioutil.WriteFile(path, blob, 0666)
	}

	// errorOnDiff emits an error to T and returns true if got != exp.
	func errorOnDiff(t *testing.T, got, exp string) bool {
	t.Helper()

	got = strings.TrimSpace(got)
	exp = strings.TrimSpace(exp)

	switch {
	case got == "":
	t.Errorf("Got nothing, but was expecting:\n\n%s\n", exp)
	return true
	case got != exp:
	dmp := diffmatchpatch.New()
	diffs := dmp.DiffMain(exp, got, false)
	t.Errorf(
	"Got:\n\n%s\n\nWas expecting:\n\n%s\n\nDiff:\n\n%s\n",
	got, exp, dmp.DiffPrettyText(diffs))
	return true
	}

	return false
	}

	// errorOnMismatch emits an error to T if got doesn't match a pattern pat.
	//
	// The pattern is syntax is:
	// * A line "[space]...[space]" matches zero or more arbitrary lines.
	// * Trigram "???" matches [0-9a-zA-Z]+.
	// * The rest should match as is.
	func errorOnPatternMismatch(t *testing.T, got, pat string) {
	t.Helper()

	got = strings.TrimSpace(got)
	pat = strings.TrimSpace(pat)

	re := strings.Builder{}
	re.WriteRune('^')
	for _, line := range strings.Split(pat, "\n") {
	if strings.TrimSpace(line) == "..." {
	re.WriteString(`(.\n)`)
	} else {
	for line != "" {
	idx := strings.Index(line, "???")
	if idx == -1 {
	re.WriteString(regexp.QuoteMeta(line))
	break
	}
	re.WriteString(regexp.QuoteMeta(line[:idx]))
	re.WriteString(`[0-9a-zA-Z]+`)
	line = line[idx+3:]
	}
	re.WriteString(`\n`)
	}
	}
	re.WriteRune('$')

	if exp := regexp.MustCompile(re.String()); !exp.MatchString(got + "\n") {
	t.Errorf("Got:\n\n%s\n\nWas expecting pattern:\n\n%s\n\n", got, pat)
	t.Errorf("Regexp: %s", re.String())
	}
	}