ctpv2: Grab filters from firestore Update filter container info building to grab from the firestore. Updated logic goes as follows: 1. Get metadata container info. 2. Get container info from firestore. 3. Get filter container info from input. 4. Build default and non-default containers using the combined info from 1-3. BUG=b:323374380 TEST=led Change-Id: Idc10016af29de2ba3a1d41afa6e1d3b5239c91e8 Reviewed-on: https://chromium-review.googlesource.com/c/infra/infra/+/5646720 Reviewed-by: Derek Beckett <dbeckett@chromium.org> Reviewed-by: Azizur Rahman <azrahman@google.com> Commit-Queue: Chris DeLaGarza <cdelagarza@google.com> Cr-Commit-Position: refs/heads/main@{#66346}
diff --git a/go/src/infra/cros/cmd/common_lib/common/constants.go b/go/src/infra/cros/cmd/common_lib/common/constants.go index 858af03..1523c9c 100644 --- a/go/src/infra/cros/cmd/common_lib/common/constants.go +++ b/go/src/infra/cros/cmd/common_lib/common/constants.go
@@ -31,6 +31,7 @@ TesthausURLPrefix = "https://tests.chromeos.goog/p/chromeos/logs/unified/" GcsURLPrefix = "https://pantheon.corp.google.com/storage/browser/" HwTestCtrInputPropertyName = "$chromeos/cros_tool_runner" + HwTestCtpv2InputPropertyName = "$chromeos/ctpv2" CftServiceMetadataFileName = ".cftmeta" CftServiceMetadataLineContentSeparator = "=" CftServiceMetadataServicePortKey = "SERVICE_PORT"
diff --git a/go/src/infra/cros/cmd/common_lib/common/filters.go b/go/src/infra/cros/cmd/common_lib/common/filters.go index 8ceef3d..91b46de 100644 --- a/go/src/infra/cros/cmd/common_lib/common/filters.go +++ b/go/src/infra/cros/cmd/common_lib/common/filters.go
@@ -10,7 +10,6 @@ "encoding/json" "fmt" - "cloud.google.com/go/firestore" "golang.org/x/exp/slices" buildapi "go.chromium.org/chromiumos/config/go/build/api" @@ -36,18 +35,15 @@ // Default shas for backwards compatibility defaultLegacyHWSha = "0d2d1b14940de10b4e1985065876357bf5bfe9d04f103892265d17e0b0b86350" defaultTTCPSha = "71916beca11e2454f4b17bde85ec2e9d2b93b9dca6fec0f2fd1ce3bb1be5acfa" - defaultProvisionSha = "e2265e32b9d42bda405212f35a2c318c9528174a67dcc629081017725e5d57a5" defaultUseFlagFilterSha = "9e422219baadf129f1b30bc69b17c93577e6633d8688ceed6eb797ee6d9a3b03" prodShas = map[string]string{ TtcpContainerName: defaultTTCPSha, LegacyHWContainerName: defaultLegacyHWSha, - ProvisionContainerName: defaultProvisionSha, UseFlagFilterContainerName: defaultUseFlagFilterSha} binaryLookup = map[string]string{ LegacyHWContainerName: "legacy_hw_filter", TtcpContainerName: "solver_service", - ProvisionContainerName: "provision-filter", TestFinderContainerName: "test_finder_filter", UseFlagFilterContainerName: "use_flag_filter", } @@ -80,6 +76,7 @@ defaultFilters := make([]*api.CTPFilter, 0) logging.Infof(ctx, "Inside Default Filters: %s", defaultFilterNames) for _, filterName := range defaultFilterNames { + logging.Infof(ctx, "Checking container metadata map for %s", filterName) // Attempt to map the filter from the known container metadata. ctpFilter, err := CreateCTPFilterWithContainerName(ctx, filterName, contMetadataMap, build, true) if err == nil { @@ -170,12 +167,23 @@ nonDefFilters := []*api.CTPFilter{} for _, filter := range filtersToAdd { - filterContainerName := filter.GetContainerInfo().GetContainer().GetName() + filterName := filter.GetContainerInfo().GetContainer().GetName() + ctpFilter, err := CreateCTPFilterWithContainerName(ctx, filterName, contMetadataMap, build, true) + if err != nil { + logging.Infof(ctx, "failed to create ctp filter for %s", filterName) + continue + } + // BinaryName is assumed to be same as FilterName. + // If this is not the case, it can be resolved by the input. + if filter.GetContainerInfo().GetBinaryName() != "" { + ctpFilter.ContainerInfo.BinaryName = filter.GetContainerInfo().GetBinaryName() + } + ctpFilter.ContainerInfo.BinaryArgs = filter.GetContainerInfo().GetBinaryArgs() // Overwrite the default filter with the user defined filter. - if slices.Contains(defaultFilterNames, filterContainerName) { - defFilters[defFiltersIndexMap[filterContainerName]] = filter + if slices.Contains(defaultFilterNames, filterName) { + defFilters[defFiltersIndexMap[filterName]] = ctpFilter } else { - nonDefFilters = append(nonDefFilters, filter) + nonDefFilters = append(nonDefFilters, ctpFilter) } } @@ -270,23 +278,3 @@ return retBytes } - -// FetchDigestFromFirestore grabs the digest and previous digest -// from the collection based on the container name. -func FetchDigestFromFirestore(ctx context.Context, collection *firestore.CollectionRef, containerName string) (digest string, prevDigest string) { - doc, err := collection.Doc(containerName).Get(ctx) - if err != nil { - return - } - data := doc.Data() - digest, ok := data["digest"].(string) - if !ok { - return "", "" - } - prevDigest, ok = data["prevDigest"].(string) - if !ok { - return digest, "" - } - - return -}
diff --git a/go/src/infra/cros/cmd/common_lib/common/filters_firestore.go b/go/src/infra/cros/cmd/common_lib/common/filters_firestore.go new file mode 100644 index 0000000..82c77bb --- /dev/null +++ b/go/src/infra/cros/cmd/common_lib/common/filters_firestore.go
@@ -0,0 +1,135 @@ +// Copyright 2024 The Chromium Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package common + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "cloud.google.com/go/firestore" + + buildapi "go.chromium.org/chromiumos/config/go/build/api" + "go.chromium.org/chromiumos/config/go/test/api" + "go.chromium.org/luci/common/errors" + "go.chromium.org/luci/common/logging" +) + +// ContainerInfoItem represents an Uprev record for +// containers built during this program's execution. +type ContainerInfoItem struct { + TimeRecord time.Time + RepositoryHostname string + RepositoryProject string + Digest string +} + +func NewContainerInfoItem(host, project, digest string) *ContainerInfoItem { + if host == "" { + host = DefaultDockerHost + } + if project == "" { + project = DefaultDockerProject + } + return &ContainerInfoItem{ + TimeRecord: time.Now().UTC(), + RepositoryHostname: host, + RepositoryProject: project, + Digest: digest, + } +} + +// FetchFiltersFromFirestore grabs every filter stored within the +// the firestore database. +func FetchFiltersFromFirestore(ctx context.Context, creds, tag string) (filters []*api.CTPFilter, err error) { + firestoreClient, err := EstablishFirestoreConnection(ctx, creds) + if err != nil { + err = errors.Annotate(err, "failed to initialize firestore client").Err() + return + } + defer func() { + closeErr := firestoreClient.Close() + if closeErr != nil { + logging.Infof(ctx, "failed to close firestore client, %w", closeErr) + } + }() + + collectionName := GetFirestoreCollection(tag) + collection := firestoreClient.Collection(collectionName) + filters, err = fetchFiltersFromFirestoreCollection(ctx, collection) + + return +} + +// FetchContainerInfoFromFirestoreDoc grabs the ContainerInfoItems +// from the provided firestore document reference. +func FetchContainerInfoFromFirestoreDoc(ctx context.Context, docRef *firestore.DocumentRef) []*ContainerInfoItem { + items := []*ContainerInfoItem{} + + doc, err := docRef.Get(ctx) + if err != nil { + logging.Infof(ctx, "failed to get document %s, %w", docRef.ID, err) + return items + } + data := doc.Data() + jsonStr, ok := data["info"].(string) + if !ok { + return items + } + err = json.Unmarshal([]byte(jsonStr), &items) + if err != nil { + logging.Infof(ctx, "failed to unmarshal %s, %w", docRef.ID, err) + return items + } + + return items +} + +// GetFirestoreCollection returns the collection name +// based on whether its the prod or staging environment. +func GetFirestoreCollection(tag string) string { + if tag == LabelProd { + return FireStoreContainersProdCollection + } + return FireStoreContainersStagingCollection +} + +// fetchFiltersFromFirestoreCollection grabs every filter from the +// provided firestored collection reference. +func fetchFiltersFromFirestoreCollection(ctx context.Context, collection *firestore.CollectionRef) (filters []*api.CTPFilter, err error) { + filters = []*api.CTPFilter{} + + documentRefs, err := collection.DocumentRefs(ctx).GetAll() + if err != nil { + err = errors.Annotate(err, "failed to get document refs from filter's firestore").Err() + return + } + + for _, documentRef := range documentRefs { + containerInfos := FetchContainerInfoFromFirestoreDoc(ctx, documentRef) + if len(containerInfos) == 0 { + err = fmt.Errorf("%s is missing container info", documentRef.ID) + return + } + // Grab most recent. + // Convert to CTPFilter. + containerInfo := containerInfos[0] + filters = append(filters, &api.CTPFilter{ + ContainerInfo: &api.ContainerInfo{ + Container: &buildapi.ContainerImageInfo{ + Name: documentRef.ID, + Digest: containerInfo.Digest, + Repository: &buildapi.GcrRepository{ + Hostname: containerInfo.RepositoryHostname, + Project: containerInfo.RepositoryProject, + }, + }, + }, + }) + } + + return +}
diff --git a/go/src/infra/cros/cmd/common_lib/common/firestore.go b/go/src/infra/cros/cmd/common_lib/common/firestore.go index 95ed38c..41a21b8 100644 --- a/go/src/infra/cros/cmd/common_lib/common/firestore.go +++ b/go/src/infra/cros/cmd/common_lib/common/firestore.go
@@ -7,9 +7,13 @@ import ( "context" + "time" "cloud.google.com/go/firestore" + "github.com/cenkalti/backoff/v4" "google.golang.org/api/option" + + "go.chromium.org/luci/common/logging" ) // InitClient returns a firestore client set to access the given project and @@ -24,6 +28,32 @@ return client, nil } +// EstablishFirestoreConnection uses provided credentials +// to establish a connection to the test platform firestore database. +func EstablishFirestoreConnection(ctx context.Context, creds string) (client *firestore.Client, err error) { + projectID := TestPlatformDataProjectID + firestoreDatabaseName := TestPlatformFireStore + + clientOpts := []option.ClientOption{} + + if creds != "" { + clientOpts = append(clientOpts, option.WithCredentialsFile(creds)) + } + + retryFunc := func() (*firestore.Client, error) { + return InitClient(ctx, projectID, firestoreDatabaseName, clientOpts...) + } + notifyFunc := func(e error, t time.Duration) { + logging.Infof(ctx, "failed to initialize client after %s with error: %s", t, e) + } + backer := backoff.NewExponentialBackOff( + backoff.WithInitialInterval(time.Second*2), + backoff.WithMaxInterval(time.Second*16), + backoff.WithMaxElapsedTime(time.Minute), + ) + return backoff.RetryNotifyWithData(retryFunc, backer, notifyFunc) +} + // FirestoreItem wraps the interface item with it's intended document name. type FirestoreItem struct { DocName string
diff --git a/go/src/infra/cros/cmd/container_uprev/executions/build_execution.go b/go/src/infra/cros/cmd/container_uprev/executions/build_execution.go index 387a3d4..a63d6ac 100644 --- a/go/src/infra/cros/cmd/container_uprev/executions/build_execution.go +++ b/go/src/infra/cros/cmd/container_uprev/executions/build_execution.go
@@ -64,7 +64,7 @@ if !runAsAdmin { // Do not update the sha storage on local execution. - UpdateShaStorage = func(ctx context.Context, containerSHAs map[string]*internal.ContainerInfoItem, creds, label string) (err error) { + UpdateShaStorage = func(ctx context.Context, containerSHAs map[string]*common.ContainerInfoItem, creds, label string) (err error) { logging.Infof(ctx, "Local execution, skipping sha storage update") return nil } @@ -84,7 +84,7 @@ // executeContainerUprev steps through the uprev configs, creates a new container, // and uploads its sha to the storage. func executeContainerUprev(ctx context.Context, dockerKeyFile, cipdLabel, imageTag string) (err error) { - containerInfos := map[string]*internal.ContainerInfoItem{} + containerInfos := map[string]*common.ContainerInfoItem{} configs := internal.GetConfigs() for _, config := range configs { if err = internal.GcloudAuth(ctx, config.RepositoryHostname, dockerKeyFile); err != nil { @@ -96,7 +96,7 @@ if uprevErr != nil { err = errors.Append(err, uprevErr) } else { - containerInfo := internal.NewContainerInfoItem(config.RepositoryHostname, config.RepositoryProject, sha) + containerInfo := common.NewContainerInfoItem(config.RepositoryHostname, config.RepositoryProject, sha) containerInfos[config.Name] = containerInfo } }
diff --git a/go/src/infra/cros/cmd/container_uprev/internal/firestore.go b/go/src/infra/cros/cmd/container_uprev/internal/firestore.go index 869c4a7..75e4754 100644 --- a/go/src/infra/cros/cmd/container_uprev/internal/firestore.go +++ b/go/src/infra/cros/cmd/container_uprev/internal/firestore.go
@@ -10,99 +10,15 @@ "time" "cloud.google.com/go/firestore" - "github.com/cenkalti/backoff/v4" - "google.golang.org/api/option" "go.chromium.org/luci/common/errors" - "go.chromium.org/luci/common/logging" "infra/cros/cmd/common_lib/common" ) -// ContainerInfoItem represents an Uprev record for -// containers built during this program's execution. -type ContainerInfoItem struct { - TimeRecord time.Time - RepositoryHostname string - RepositoryProject string - Digest string -} - -func NewContainerInfoItem(host, project, digest string) *ContainerInfoItem { - if host == "" { - host = common.DefaultDockerHost - } - if project == "" { - project = common.DefaultDockerProject - } - return &ContainerInfoItem{ - TimeRecord: time.Now().UTC(), - RepositoryHostname: host, - RepositoryProject: project, - Digest: digest, - } -} - -// establishFirestoreConnection connects to the firestore with -// the option to provide in a credentials file. -func establishFirestoreConnection(ctx context.Context, creds string) (client *firestore.Client, err error) { - projectID := common.TestPlatformDataProjectID - firestoreDatabaseName := common.TestPlatformFireStore - - clientOpts := []option.ClientOption{} - - if creds != "" { - clientOpts = append(clientOpts, option.WithCredentialsFile(creds)) - } - - retryFunc := func() (*firestore.Client, error) { - return common.InitClient(ctx, projectID, firestoreDatabaseName, clientOpts...) - } - notifyFunc := func(e error, t time.Duration) { - logging.Infof(ctx, "failed to initialize client after %s with error: %s", t, e) - } - backer := backoff.NewExponentialBackOff( - backoff.WithInitialInterval(time.Second*2), - backoff.WithMaxInterval(time.Second*16), - backoff.WithMaxElapsedTime(time.Minute), - ) - return backoff.RetryNotifyWithData(retryFunc, backer, notifyFunc) -} - -// getFirestoreCollection returns the collection name -// based on whether its the prod or staging environment. -func getFirestoreCollection(tag string) string { - if tag == common.LabelProd { - return common.FireStoreContainersProdCollection - } - return common.FireStoreContainersStagingCollection -} - -// fetchContainerInfoFromFirestore grabs the data from the collection -// and Unmarshals it into a list of ContainerInfoItems. -func fetchContainerInfoFromFirestore(ctx context.Context, collection *firestore.CollectionRef, docName string) []*ContainerInfoItem { - items := []*ContainerInfoItem{} - - doc, err := collection.Doc(docName).Get(ctx) - if err != nil { - return items - } - data := doc.Data() - jsonStr, ok := data["info"].(string) - if !ok { - return items - } - err = json.Unmarshal([]byte(jsonStr), &items) - if err != nil { - return items - } - - return items -} - // buildContainerInfosForFirestore converts the map of ContainerInfoItems // into the record stored within Firestore. -func buildContainerInfosForFirestore(infos map[string][]*ContainerInfoItem) []*common.FirestoreItem { +func buildContainerInfosForFirestore(infos map[string][]*common.ContainerInfoItem) []*common.FirestoreItem { items := []*common.FirestoreItem{} for containerName, info := range infos { @@ -124,12 +40,12 @@ } // filterContainerInfos filters out stale records. -func filterContainerInfos(infos map[string][]*ContainerInfoItem) map[string][]*ContainerInfoItem { - filtered := map[string][]*ContainerInfoItem{} +func filterContainerInfos(infos map[string][]*common.ContainerInfoItem) map[string][]*common.ContainerInfoItem { + filtered := map[string][]*common.ContainerInfoItem{} now := time.Now().UTC() for containerName, info := range infos { - filteredInfo := []*ContainerInfoItem{} + filteredInfo := []*common.ContainerInfoItem{} for i, infoItem := range info { // Keep at least 5 entries, filter out the rest. if i >= 5 { @@ -150,7 +66,7 @@ // pushContainerInfoToFirestore packages the map of ContainerInfoItems // and pushes them into to Firestore collection. -func pushContainerInfoToFirestore(ctx context.Context, client *firestore.Client, collectionName string, infos map[string][]*ContainerInfoItem) (err error) { +func pushContainerInfoToFirestore(ctx context.Context, client *firestore.Client, collectionName string, infos map[string][]*common.ContainerInfoItem) (err error) { collection := client.Collection(collectionName) filteredInfos := filterContainerInfos(infos)
diff --git a/go/src/infra/cros/cmd/container_uprev/internal/sha_updater.go b/go/src/infra/cros/cmd/container_uprev/internal/sha_updater.go index beb87ea..63007ae 100644 --- a/go/src/infra/cros/cmd/container_uprev/internal/sha_updater.go +++ b/go/src/infra/cros/cmd/container_uprev/internal/sha_updater.go
@@ -12,15 +12,17 @@ "go.chromium.org/luci/common/errors" "go.chromium.org/luci/common/logging" "go.chromium.org/luci/luciexe/build" + + "infra/cros/cmd/common_lib/common" ) // UpdateShaStorage connects the the firestore and uploads the SHAs produced // during the uprev service. -func UpdateShaStorage(ctx context.Context, containerInfo map[string]*ContainerInfoItem, creds, tag string) (err error) { +func UpdateShaStorage(ctx context.Context, containerInfo map[string]*common.ContainerInfoItem, creds, tag string) (err error) { step, ctx := build.StartStep(ctx, "Update SHAs") defer func() { step.End(err) }() - firestoreClient, err := establishFirestoreConnection(ctx, creds) + firestoreClient, err := common.EstablishFirestoreConnection(ctx, creds) if err != nil { err = errors.Annotate(err, "failed to initialize firestore client").Err() return @@ -32,7 +34,7 @@ } }() - collectionName := getFirestoreCollection(tag) + collectionName := common.GetFirestoreCollection(tag) err = addContainerInfoToStorage(ctx, firestoreClient, collectionName, containerInfo) if err != nil { err = errors.Annotate(err, "failed to upload container info").Err() @@ -45,7 +47,7 @@ // RevertShas swaps the previous sha with the current sha // and updates the firestore. func RevertShas(ctx context.Context, containerNames []string, creds, tag string) (err error) { - firestoreClient, err := establishFirestoreConnection(ctx, creds) + firestoreClient, err := common.EstablishFirestoreConnection(ctx, creds) if err != nil { err = errors.Annotate(err, "failed to initialize firestore client").Err() return @@ -57,12 +59,12 @@ } }() - collectionName := getFirestoreCollection(tag) + collectionName := common.GetFirestoreCollection(tag) containersCollection := firestoreClient.Collection(collectionName) - infosMap := map[string][]*ContainerInfoItem{} + infosMap := map[string][]*common.ContainerInfoItem{} for _, containerName := range containerNames { - currentInfos := fetchContainerInfoFromFirestore(ctx, containersCollection, containerName) + currentInfos := common.FetchContainerInfoFromFirestoreDoc(ctx, containersCollection.Doc(containerName)) // Can't revert the only record. if len(currentInfos) <= 1 { continue @@ -74,14 +76,14 @@ return } -func addContainerInfoToStorage(ctx context.Context, firestoreClient *firestore.Client, collectionName string, containerInfos map[string]*ContainerInfoItem) (err error) { +func addContainerInfoToStorage(ctx context.Context, firestoreClient *firestore.Client, collectionName string, containerInfos map[string]*common.ContainerInfoItem) (err error) { containersCollection := firestoreClient.Collection(collectionName) - infosMap := map[string][]*ContainerInfoItem{} + infosMap := map[string][]*common.ContainerInfoItem{} // Add new container info to storage record. for containerName, containerInfo := range containerInfos { - currentInfos := fetchContainerInfoFromFirestore(ctx, containersCollection, containerName) - infos := append([]*ContainerInfoItem{containerInfo}, currentInfos...) + currentInfos := common.FetchContainerInfoFromFirestoreDoc(ctx, containersCollection.Doc(containerName)) + infos := append([]*common.ContainerInfoItem{containerInfo}, currentInfos...) infosMap[containerName] = infos }
diff --git a/go/src/infra/cros/cmd/ctpv2/data/filter_statekeeper.go b/go/src/infra/cros/cmd/ctpv2/data/filter_statekeeper.go index 32b941b..8f9a908 100644 --- a/go/src/infra/cros/cmd/ctpv2/data/filter_statekeeper.go +++ b/go/src/infra/cros/cmd/ctpv2/data/filter_statekeeper.go
@@ -29,6 +29,8 @@ SuiteTestResults map[string]*TestResults BuildsMap map[string]*BuildRequest Config *config.Config + DockerKeyFile string + CTPversion string // For V1 req, it will be key provided with input, // for direct v2, this will be index of the request. // Cannot use suite here coz two different suites can be present with different
diff --git a/go/src/infra/cros/cmd/ctpv2/executions/build_execution.go b/go/src/infra/cros/cmd/ctpv2/executions/build_execution.go index f8bc4dc..a951226 100644 --- a/go/src/infra/cros/cmd/ctpv2/executions/build_execution.go +++ b/go/src/infra/cros/cmd/ctpv2/executions/build_execution.go
@@ -38,7 +38,9 @@ func LuciBuildExecution() { // Set input property reader functions var ctrCipdInfoReader func(context.Context) *protos.CipdVersionInfo + var ctpv2CipdInfoReader func(context.Context) *protos.CipdVersionInfo build.MakePropertyReader(common.HwTestCtrInputPropertyName, &ctrCipdInfoReader) + build.MakePropertyReader(common.HwTestCtpv2InputPropertyName, &ctpv2CipdInfoReader) input := &steps.CTPv2BinaryBuildInput{} // Set output props writer functions @@ -52,13 +54,15 @@ log.SetFlags(log.LstdFlags | log.Lshortfile | log.Lmsgprefix) logging.Infof(ctx, "have input %v", input) ctrCipdInfo := ctrCipdInfoReader(ctx) + ctpv2CipdInfo := ctpv2CipdInfoReader(ctx) + logging.Infof(ctx, "ctpv2 label: %s", ctpv2CipdInfo.GetVersion().GetCipdLabel()) bqClient := analytics.CtpAnalyticsBQClient(ctx) if bqClient != nil { defer bqClient.Close() } logging.Infof(ctx, "have ctr info: %v", ctrCipdInfo) logging.Infof(ctx, "ctr label: %s", ctrCipdInfo.GetVersion().GetCipdLabel()) - resp, err := executeRequests(ctx, input, ctrCipdInfo.GetVersion().GetCipdLabel(), st, bqClient) + resp, err := executeRequests(ctx, input, ctrCipdInfo.GetVersion().GetCipdLabel(), st, bqClient, ctpv2CipdInfo.GetVersion().GetCipdLabel()) if err != nil { logging.Infof(ctx, "error found: %s", err) st.SetSummaryMarkdown(err.Error()) @@ -76,7 +80,8 @@ input *steps.CTPv2BinaryBuildInput, ctrCipdVersion string, buildState *build.State, - BQClient *bigquery.Client) (*steps.CTPv2BinaryBuildOutput, error) { + BQClient *bigquery.Client, + ctpv2CipdVersion string) (*steps.CTPv2BinaryBuildOutput, error) { buildOutput := &steps.CTPv2BinaryBuildOutput{} // Validation @@ -139,7 +144,7 @@ } else { keyReqMap = sk.V1KeyToCTPv2Req } - resultsMap := executeCtpv2Reqs(ctx, keyReqMap, input.Config, buildState, ctr, BQClient) + resultsMap := executeCtpv2Reqs(ctx, keyReqMap, input.Config, buildState, ctr, BQClient, ctpv2CipdVersion) sk.AllTestResults = resultsMap // Execute post configs @@ -162,7 +167,7 @@ } func executeCtpv2Reqs(ctx context.Context, - keyRequestMap map[string]*api.CTPRequest, config *config.Config, buildState *build.State, ctr *crostoolrunner.CrosToolRunner, BQClient *bigquery.Client) map[string][]*data.TestResults { + keyRequestMap map[string]*api.CTPRequest, config *config.Config, buildState *build.State, ctr *crostoolrunner.CrosToolRunner, BQClient *bigquery.Client, ctpVersion string) map[string][]*data.TestResults { resultsMap := map[string][]*data.TestResults{} var err error step, ctx := build.StartStep(ctx, "Suite Executions (async)") @@ -184,7 +189,7 @@ suiteDisplayName = fmt.Sprintf("%s_%d", suiteName, suiteNum) } wg.Add(1) - go executeFiltersInLuciBuild(ctx, ctpReq, config, buildState, wg, ctr, contInfoMap, resultsChan, suiteDisplayName, BQClient, key) + go executeFiltersInLuciBuild(ctx, ctpReq, config, buildState, wg, ctr, contInfoMap, resultsChan, suiteDisplayName, BQClient, key, ctpVersion) } go func() { wg.Wait() @@ -209,12 +214,18 @@ req *api.CTPRequest, config *config.Config, buildState *build.State, - wg *sync.WaitGroup, ctr *crostoolrunner.CrosToolRunner, contInfoMap *data.ContainerInfoMap, results chan<- map[string][]*data.TestResults, suiteDisplayName string, BQClient *bigquery.Client, reqKey string) error { + wg *sync.WaitGroup, ctr *crostoolrunner.CrosToolRunner, contInfoMap *data.ContainerInfoMap, results chan<- map[string][]*data.TestResults, suiteDisplayName string, BQClient *bigquery.Client, reqKey, ctpVersion string) error { defer wg.Done() var err error step, ctx := build.StartStep(ctx, suiteDisplayName) defer func() { step.End(err) }() + dockerKeyFile, err := common.LocateFile([]string{common.LabDockerKeyFileLocation, common.VmLabDockerKeyFileLocation}) + if err != nil { + err = fmt.Errorf("unable to locate dockerKeyFile during initialization: %w", err) + return err + } + executorCfg := configs.NewExecutorConfig(ctr, nil) cmdCfg := configs.NewCommandConfig(executorCfg) @@ -228,6 +239,8 @@ BQClient: BQClient, Config: config, RequestKey: reqKey, + DockerKeyFile: dockerKeyFile, + CTPversion: ctpVersion, } nFilters := getTotalFilters(ctx, req, common.MakeDefaultFilters(ctx, req.GetSuiteRequest()), common.DefaultKoffeeFilterNames)
diff --git a/go/src/infra/cros/cmd/ctpv2/internal/commands/prepare_filter_containers.go b/go/src/infra/cros/cmd/ctpv2/internal/commands/prepare_filter_containers.go index fc317cf..6e8db58 100644 --- a/go/src/infra/cros/cmd/ctpv2/internal/commands/prepare_filter_containers.go +++ b/go/src/infra/cros/cmd/ctpv2/internal/commands/prepare_filter_containers.go
@@ -29,7 +29,9 @@ *interfaces.AbstractSingleCmdByNoExecutor // Deps - CtpReq *testapi.CTPRequest + CtpReq *testapi.CTPRequest + CredsFile string + CTPversion string // Updates ContainerInfoQueue *list.List @@ -83,6 +85,8 @@ return fmt.Errorf("Cmd %q missing dependency: CtpV2Req", cmd.GetCommandType()) } + cmd.CTPversion = sk.CTPversion + cmd.CredsFile = sk.DockerKeyFile cmd.CtpReq = sk.CtpReq return nil } @@ -138,7 +142,7 @@ return errors.Annotate(err, "failed to fetch container image data: ").Err() } logging.Infof(ctx, "ctpreq:", cmd.CtpReq) - finalMetadataMap := createContainerImagesInfoMap(ctx, cmd.CtpReq, buildContainerMetadata) + finalMetadataMap := createContainerImagesInfoMap(ctx, cmd.CtpReq, buildContainerMetadata, cmd.CredsFile, cmd.CTPversion) logging.Infof(ctx, "FINALMAP:", finalMetadataMap) cmd.ContainerMetadataMap = finalMetadataMap @@ -258,7 +262,7 @@ } } -func createContainerImagesInfoMap(ctx context.Context, req *testapi.CTPRequest, buildContMetadata map[string]*buildapi.ContainerImageInfo) map[string]*buildapi.ContainerImageInfo { +func createContainerImagesInfoMap(ctx context.Context, req *testapi.CTPRequest, buildContMetadata map[string]*buildapi.ContainerImageInfo, creds, ctpVersion string) map[string]*buildapi.ContainerImageInfo { // In case of any overlap of container metadata between input and build metadata, // the input metadata will be prioritized. bcm := make(map[string]*buildapi.ContainerImageInfo) @@ -266,17 +270,39 @@ bcm[k] = v } - for _, filter := range req.GetKarbonFilters() { + // Add staging/prod containers from the firestore. + firestoreFilters, err := common.FetchFiltersFromFirestore(ctx, creds, ctpVersion) + if err != nil { + logging.Infof(ctx, "%w", err) + } + for _, filter := range firestoreFilters { bcm[filter.GetContainerInfo().GetContainer().GetName()] = filter.GetContainerInfo().GetContainer() } - for _, filter := range req.GetKoffeeFilters() { - bcm[filter.GetContainerInfo().GetContainer().GetName()] = filter.GetContainerInfo().GetContainer() + // Combine karbon and koffee filters. + // They use identical logic. + for _, filter := range append(req.GetKarbonFilters(), req.GetKoffeeFilters()...) { + container := filter.GetContainerInfo().GetContainer() + // Overwrite container image info if present within filter input. + // Else, check if map already contains image info for the filter. + if hasValidDigest(container.Digest) || len(container.Tags) > 0 { + bcm[container.GetName()] = container + } else { + if _, ok := bcm[container.GetName()]; !ok { + logging.Infof(ctx, "container %s missing container info", container.GetName()) + } + } } return bcm } +// hasValidDigest ensures the digest starts with `sha256:` and +// the SHA value itself is 64 characters in length. +func hasValidDigest(digest string) bool { + return strings.HasPrefix(digest, "sha256:") && len(digest) == len("sha256:")+64 +} + // CtpFilterToContainerInfo creates container info from provided ctp filter. func CtpFilterToContainerInfo(ctpFilter *api.CTPFilter, build int) *data.ContainerInfo { contName := ctpFilter.GetContainerInfo().GetContainer().GetName()