pointless_build_checker: Use new build_irrelevance config

This removes the hardcoded paths that are irrelevant to Portage.

This also fixes some questionable logic I was using previously, which
just used prefix-matching on file paths as strings. Now file and
directory paths are handled properly (and "a/b/cccc" isn't regarded as
an irrelevant path just because "a/b/c" is an irrelevant path).

BUG=chromium:956577
TEST=Unit tests, local run

Change-Id: I7ea2beadcf7ce81b201a2531852dd1298269173f
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/infra/test_planner/+/1600986
Reviewed-by: Evan Hernandez <evanhernandez@chromium.org>
Tested-by: Sean Abraham <seanabraham@chromium.org>
Commit-Queue: Sean Abraham <seanabraham@chromium.org>
diff --git a/deploy_cipd.json b/deploy_cipd.json
index 12d63d9..e4da296 100644
--- a/deploy_cipd.json
+++ b/deploy_cipd.json
@@ -1,6 +1,6 @@
 {
   "result": {
     "package": "chromiumos/infra/test_planner",
-    "instance_id": "69DMKSgWF9MjpefsjD59RFRaMa6CE4IsOXeQC9h8aOEC"
+    "instance_id": "_5cJO3Qxf6n_5LyXE9JNkD6vPITTY1TwGEoJMdD0l1MC"
   }
 }
\ No newline at end of file
diff --git a/src/testplans/cmd/pointless_build_checker/pointless_build_checker.go b/src/testplans/cmd/pointless_build_checker/pointless_build_checker.go
index 631f7e0..47eab00 100644
--- a/src/testplans/cmd/pointless_build_checker/pointless_build_checker.go
+++ b/src/testplans/cmd/pointless_build_checker/pointless_build_checker.go
@@ -22,15 +22,14 @@
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"testplans/internal/git"
 	"testplans/internal/pointless"
 	"testplans/internal/repo"
 )
 
-var (
-	// TODO(crbug.com/956577): Define a config schema and move this list into Starlark, where it can
-	// be expanded upon.
-	buildIrrelevantPaths = []string{"chromite/config"}
+const (
+	buildIrrelevanceConfigPath = "testingconfig/generated/build_irrelevance_config.cfg"
 )
 
 func cmdCheckBuild(authOpts auth.Options) *subcommands.Command {
@@ -60,6 +59,12 @@
 		return 1
 	}
 
+	cfg, err := c.fetchConfigFromGitiles()
+	if err != nil {
+		log.Print(err)
+		return 2
+	}
+
 	build, err := readBuildbucketBuild(req.BuildbucketProto)
 	if err != nil {
 		log.Print(err)
@@ -77,7 +82,7 @@
 		return 5
 	}
 
-	resp, err := pointless.CheckBuilder(build, changeRevs, req.DepGraph, *repoToSrcRoot, buildIrrelevantPaths)
+	resp, err := pointless.CheckBuilder(build, changeRevs, req.DepGraph, *repoToSrcRoot, *cfg)
 	if err != nil {
 		log.Printf("Error checking if build is pointless:\n%v", err)
 		return 6
@@ -110,6 +115,33 @@
 	return req, nil
 }
 
+func (c *checkBuild) fetchConfigFromGitiles() (*testplans_pb.BuildIrrelevanceCfg, error) {
+	// Create an authenticated client for Gerrit RPCs, then fetch all required CL data from Gerrit.
+	ctx := context.Background()
+	authOpts, err := c.authFlags.Options()
+	if err != nil {
+		return nil, err
+	}
+	authedClient, err := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts).Client()
+	if err != nil {
+		return nil, err
+	}
+	m, err := git.FetchFilesFromGitiles(authedClient, ctx,
+		"chrome-internal.googlesource.com",
+		"chromeos/infra/config",
+		"master",
+		[]string{buildIrrelevanceConfigPath})
+	if err != nil {
+		return nil, err
+	}
+	buildIrrelevanceConfig := &testplans_pb.BuildIrrelevanceCfg{}
+	if err := jsonpb.Unmarshal(strings.NewReader((*m)[buildIrrelevanceConfigPath]), buildIrrelevanceConfig); err != nil {
+		return nil, fmt.Errorf("Couldn't decode %s as a BuildIrrelevanceCfg\n%v", (*m)[buildIrrelevanceConfigPath], err)
+	}
+	log.Printf("Fetched config from Gitiles: %s\n", proto.MarshalTextString(buildIrrelevanceConfig))
+	return buildIrrelevanceConfig, nil
+}
+
 func readBuildbucketBuild(bbBuildBytes *testplans_pb.ProtoBytes) (*bbproto.Build, error) {
 	bbBuild := &bbproto.Build{}
 	if err := proto.Unmarshal(bbBuildBytes.SerializedProto, bbBuild); err != nil {
diff --git a/src/testplans/go.sum b/src/testplans/go.sum
index c77ec73..f068f0a 100644
--- a/src/testplans/go.sum
+++ b/src/testplans/go.sum
@@ -117,34 +117,6 @@
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190412142951-855d82ed0e18 h1:Ut3OeEoubOLznQy8MrpkzFxhPdvaOd1KcU4tHpXLfmo=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190419204223-c1ccbfd39f21 h1:ykPFXJN2m+qMTwIRudnmsK9TIW7Ft5QQmiqDKN0wc88=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190419210954-9d7b9f8e6de7 h1:CNvQMd6PLbN4k/va9vVP1s4ylQZY/e9+BTdBaVXW9p4=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190425170743-fdde26dcee5b h1:ixkdr2Z8oZ25cIKblZ8eoR6OuYZPkTsOhzYELNPqFOc=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190425172812-229c600b0eb9 h1:WlZKIu5hi8G5cfJFNyEYMUnX+mUmG+dtCB8YdTYDwcI=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190425182316-29541c36daec h1:l79zRHYKfXMc+RZ72vsWH/W8y+RFR6KonUuf9bbwbWY=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190502191712-2c5f9aaf0ebe h1:7d4cYFQnJ2nLFNmTP8Fow/fgABVu42wX+RWiQLctaeU=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190506151252-fd4aa9823619 h1:8j1HqiAv0tczpKdzpKRNu2cQ62IQCOJtU8N1/tBlQdg=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190506153826-232eeaaa29a9 h1:B+BDapd5NonFHK9dNjT6adYkLdtq0lMLDyYu39pbKGY=
-go.chromium.org/chromiumos/infra/proto v0.0.0-20190506153826-232eeaaa29a9/go.mod h1:iVW1SALLZE5lx9/iMexcc242WJddrJ+ZkD22Inegxoo=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190409164829-d91e8295a7fb h1:cWqWZGdECKxs4YHX0be08U4ET35UfqjLK3jetupgwpY=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190409164829-d91e8295a7fb/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190412142951-855d82ed0e18 h1:GTcC8sHpBpklNIwS7fCkuOYnPK8RF38u4nIkl9DCi5I=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190412142951-855d82ed0e18/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190419204223-c1ccbfd39f21 h1:eB+1X8U1qXVHj4pP7P+D1qw79Pkn8YvZXKIExgI8rfM=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190419204223-c1ccbfd39f21/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190419210954-9d7b9f8e6de7 h1:l8/j5Z0mRUbKKXM/D4DbTyWxD6z2OYmxDvv6EvJJ28M=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190419210954-9d7b9f8e6de7/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425170743-fdde26dcee5b h1:eO0ZP/eUKmUMs3ksfDhNCuujtLqGfCFkzq/r86VbYHM=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425170743-fdde26dcee5b/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425172812-229c600b0eb9 h1:v/3LJK3Y/2ce+0usJFAnoY8P79fXF4wVb+E79yM5yi8=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425172812-229c600b0eb9/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425182316-29541c36daec h1:Ng0Rmj0ZRLCmoERZiZvT22Sp0Ho45EQtyzcSKtWQPSM=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190425182316-29541c36daec/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190502191712-2c5f9aaf0ebe h1:qeJS1l11SZGxShfu723zXm81VhRSkrW1fbDoLcLSWqI=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190502191712-2c5f9aaf0ebe/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190506151252-fd4aa9823619 h1:iUhJVEWTNhNhBq2gImy7aFSUPzV4H9jeVSnxGBVvgM8=
-go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190506151252-fd4aa9823619/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
 go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190506153826-232eeaaa29a9 h1:87ML72wybRl6dKe4qyvTH5Y6ihvrTxT13+xTsz7F6tE=
 go.chromium.org/chromiumos/infra/proto/go v0.0.0-20190506153826-232eeaaa29a9/go.mod h1:Y1AiM5bpNKKecKy3wgiBwF9q1PJ/f57z86DpkNrMehk=
 go.chromium.org/luci v0.0.0-20190404171609-14fb4fbce8ea h1:qf3Ri7sv3yJPL4E0iq4hBYn3AFSf2dNNJpuZDBXAT2E=
diff --git a/src/testplans/internal/pointless/build_checker.go b/src/testplans/internal/pointless/build_checker.go
index b0ba522..34c13b5 100644
--- a/src/testplans/internal/pointless/build_checker.go
+++ b/src/testplans/internal/pointless/build_checker.go
@@ -10,6 +10,7 @@
 	testplans_pb "go.chromium.org/chromiumos/infra/proto/go/testplans"
 	bbproto "go.chromium.org/luci/buildbucket/proto"
 	"log"
+	"path/filepath"
 	"sort"
 	"strings"
 	"testplans/internal/git"
@@ -30,7 +31,7 @@
 	changeRevs *git.ChangeRevData,
 	depGraph *chromite.DepGraph,
 	repoToSrcRoot map[string]string,
-	buildIrrelevantPaths []string) (*testplans_pb.PointlessBuildCheckResponse, error) {
+	cfg testplans_pb.BuildIrrelevanceCfg) (*testplans_pb.PointlessBuildCheckResponse, error) {
 
 	// Get all of the files referenced by each GerritCommit in the Build.
 	affectedFiles, err := extractAffectedFiles(build, changeRevs, repoToSrcRoot)
@@ -45,8 +46,8 @@
 		}, nil
 	}
 
-	// Filter out files that are irrelevant to Portage because of the BuildIrrelevantPaths.
-	affectedFiles = filterByBuildIrrelevantPaths(affectedFiles, buildIrrelevantPaths)
+	// Filter out files that are irrelevant to Portage because of the config.
+	affectedFiles = filterByBuildIrrelevantPaths(affectedFiles, cfg)
 	if len(affectedFiles) == 0 {
 		log.Printf("Builder %s: All files ruled out by build-irrelevant paths. This means that "+
 			"none of the Gerrit changes in the build input could affect the outcome of the build",
@@ -103,13 +104,24 @@
 	return allAffectedFiles, nil
 }
 
-func filterByBuildIrrelevantPaths(files, portageIrrelevantPaths []string) []string {
+func filterByBuildIrrelevantPaths(files []string, cfg testplans_pb.BuildIrrelevanceCfg) []string {
 	pipFilteredFiles := make([]string, 0)
 affectedFile:
 	for _, f := range files {
-		for _, pip := range portageIrrelevantPaths {
-			if strings.HasPrefix(f, pip) {
-				log.Printf("Ignoring file %s, since it's contained in Portage irrelevant path %s", f, pip)
+		for _, isp := range cfg.IrrelevantSourcePaths {
+			if f == isp.Path {
+				log.Printf("Ignoring file %s, since it matches Portage irrelevant path %s", f, isp.Path)
+				continue affectedFile
+			}
+			spAsDir := strings.TrimSuffix(isp.Path, "/") + "/"
+			if strings.HasPrefix(f, spAsDir) {
+				log.Printf("Ignoring file %s, since it's contained in Portage irrelevant dir %s", f, spAsDir)
+				continue affectedFile
+			}
+		}
+		for _, ifbn := range cfg.IrrelevantFileBaseNames {
+			if filepath.Base(f) == ifbn.Name {
+				log.Printf("Ignoring file %s, since it has Portage irrelevant file base name %s", f, ifbn)
 				continue affectedFile
 			}
 		}
@@ -134,11 +146,17 @@
 affectedFile:
 	for _, f := range files {
 		for _, pd := range portageDeps {
-			if strings.HasPrefix(f, pd) {
+			if f == pd {
 				log.Printf("Cannot ignore file %s due to Portage dependency %s", f, pd)
 				portageFilteredFiles = append(portageFilteredFiles, f)
 				continue affectedFile
 			}
+			pdAsDir := strings.TrimSuffix(pd, "/") + "/"
+			if strings.HasPrefix(f, pdAsDir) {
+				log.Printf("Cannot ignore file %s since it's in Portage dependency %s", f, pd)
+				portageFilteredFiles = append(portageFilteredFiles, f)
+				continue affectedFile
+			}
 		}
 		log.Printf("Ignoring file %s because no prefix of it is referenced in the dep graph", f)
 	}
diff --git a/src/testplans/internal/pointless/build_checker_test.go b/src/testplans/internal/pointless/build_checker_test.go
index 3020b35..f168719 100644
--- a/src/testplans/internal/pointless/build_checker_test.go
+++ b/src/testplans/internal/pointless/build_checker_test.go
@@ -13,7 +13,7 @@
 
 func makeBuildbucketBuild(changes []*bbproto.GerritChange) *bbproto.Build {
 	b := &bbproto.Build{
-		Input: &bbproto.Build_Input{},
+		Input:   &bbproto.Build_Input{},
 		Builder: &bbproto.BuilderID{Builder: "reef"},
 	}
 	for _, c := range changes {
@@ -35,7 +35,7 @@
 				Revision:  2,
 			},
 			Project: "chromiumos/public/example",
-			Files:   []string{"a/b/c"},
+			Files:   []string{"relevantfile", "irrelevantdir2"},
 		},
 	})
 	depGraph := &chromite.DepGraph{
@@ -46,9 +46,13 @@
 	repoToSrcRoot := map[string]string{
 		"chromiumos/public/example": "src/pub/ex",
 	}
-	buildIrrelevantPaths := []string{"src/internal/catpics"}
+	cfg := testplans_pb.BuildIrrelevanceCfg{
+		IrrelevantSourcePaths: []*testplans_pb.SourceTree{
+			{Path: "src/pub/ex/irrelevantdir"},
+		},
+	}
 
-	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, buildIrrelevantPaths)
+	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, cfg)
 	if err != nil {
 		t.Error(err)
 	}
@@ -96,9 +100,13 @@
 		"chromiumos/public/example":   "src/pub/ex",
 		"chromiumos/internal/example": "src/internal/ex",
 	}
-	buildIrrelevantPaths := []string{"src/internal/catpics"}
+	cfg := testplans_pb.BuildIrrelevanceCfg{
+		IrrelevantSourcePaths: []*testplans_pb.SourceTree{
+			{Path: "src/internal/catpics"},
+		},
+	}
 
-	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, buildIrrelevantPaths)
+	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, cfg)
 	if err != nil {
 		t.Error(err)
 	}
@@ -121,7 +129,11 @@
 				Revision:  2,
 			},
 			Project: "chromiumos/public/example",
-			Files:   []string{"chromite-maybe/config-thing/file1", "chromite-maybe/other-config/file2"},
+			Files: []string{
+				"chromite-maybe/config-thing/file1",
+				"chromite-maybe/other-config",
+				"chromite-maybe/somedir/img_123.jpg",
+			},
 		},
 	})
 	depGraph := &chromite.DepGraph{
@@ -132,9 +144,18 @@
 	repoToSrcRoot := map[string]string{
 		"chromiumos/public/example": "src/pub/ex",
 	}
-	buildIrrelevantPaths := []string{"src/pub/ex/chromite-maybe/config-thing", "src/pub/ex/chromite-maybe/other-config"}
 
-	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, buildIrrelevantPaths)
+	cfg := testplans_pb.BuildIrrelevanceCfg{
+		IrrelevantSourcePaths: []*testplans_pb.SourceTree{
+			{Path: "src/pub/ex/chromite-maybe/config-thing"},
+			{Path: "src/pub/ex/chromite-maybe/other-config"},
+		},
+		IrrelevantFileBaseNames: []*testplans_pb.FileBaseName{
+			{Name: "img_123.jpg"},
+		},
+	}
+
+	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, cfg)
 	if err != nil {
 		t.Error(err)
 	}
@@ -157,9 +178,9 @@
 	repoToSrcRoot := map[string]string{
 		"chromiumos/public/example": "src/pub/ex",
 	}
-	buildIrrelevantPaths := []string{""}
+	cfg := testplans_pb.BuildIrrelevanceCfg{}
 
-	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, buildIrrelevantPaths)
+	res, err := CheckBuilder(build, chRevData, depGraph, repoToSrcRoot, cfg)
 	if err != nil {
 		t.Error(err)
 	}