finder - add flexible sources port from infra/infra When we move cros-test to infra/infra we can remove this forked code and use that as the true source BUG=None TEST=build Change-Id: I6f9afb6c2b87588a4e64e2c774c7111aa246039e Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/6272939 Reviewed-by: Derek Beckett <dbeckett@chromium.org> Reviewed-by: Billy Zhao <billyzhao@chromium.org> Tested-by: Derek Beckett <dbeckett@chromium.org> Commit-Queue: Derek Beckett <dbeckett@chromium.org>
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go index 8296882..250aed4 100644 --- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go +++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go
@@ -65,7 +65,20 @@ // runTests runs the requested tests. func runTests(ctx context.Context, logger *log.Logger, resultRootDir, tlwAddr string, metadataList *api.TestCaseMetadataList, req *api.CrosTestRequest) (*api.CrosTestResponse, error) { - matchedMdList, err := finder.MatchedTestsForSuites(metadataList.Values, req.TestSuites) + md := metadataList.GetValues() + shouldPullGcsMD := isMoblySuite(req) + + if shouldPullGcsMD { + src, err := finder.GetSourceData(context.Background(), "mobly_priv_artifacts/out") + if err != nil { + fmt.Printf("err %s", err) + } + metadata := finder.TranslateMoblySrcToMetadata(src) + md = append(md, metadata...) + + } + + matchedMdList, err := finder.MatchedTestsForSuites(md, req.TestSuites) if err != nil { return nil, statuserrors.NewStatusError(statuserrors.InvalidArgument, fmt.Errorf("failed to match test metadata: %v", err)) @@ -157,3 +170,25 @@ } return nil } + +func isMoblySuite(req *api.CrosTestRequest) bool { + for _, suite := range req.GetTestSuites() { + md := suite.GetExecutionMetadata() + if mdFlagPresent(md, "moblySuite") { + + return true + } + } + return false +} + +func mdFlagPresent(metadata *api.ExecutionMetadata, flagName string) bool { + if metadata != nil && len(metadata.Args) > 0 { + for _, arg := range metadata.Args { + if arg.Flag == flagName { + return true + } + } + } + return false +}
diff --git a/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go b/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go new file mode 100644 index 0000000..e30e8cf --- /dev/null +++ b/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go
@@ -0,0 +1,174 @@ +// Copyright 2025 The ChromiumOS Authors +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Package finder find all matched tests from test metadata based on test criteria. +package finder + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "sort" + "strings" + + "cloud.google.com/go/storage" + "go.chromium.org/chromiumos/config/go/test/api" +) + +func GetSourceData(ctx context.Context, gcsBasePath string) ([][]byte, error) { + client, err := storage.NewClient(ctx) + if err != nil { + return nil, fmt.Errorf("storage.NewClient: %w", err) + } + defer client.Close() + + bucketName, pf, err := ExtractBucketAndPrefixFromPath(gcsBasePath) + if err != nil { + return nil, err + } + + bucket := client.Bucket(bucketName) + + // Currently G3Mobly content is fetched off the latest. + // TODO, we probably should expose a way to pass in a desired path to allow repeatability on tests here. + newestDir, err := FindNewestDirInGcsBucket(ctx, gcsBasePath, bucket, pf) + if err != nil { + return nil, err + } + data, err := PullAllFilesFromGcsDir(ctx, bucket, newestDir, ".json") + if err != nil { + return nil, err + } + return data, nil +} + +func FindNewestDirInGcsBucket(ctx context.Context, gcsBasePath string, bucket *storage.BucketHandle, pf string) (string, error) { + // List subdirectories under the prefix directory. + it := bucket.Objects(ctx, &storage.Query{Prefix: pf, Delimiter: "/"}) + + var subdirs []string + for { + attrs, err := it.Next() + if errors.Is(err, io.EOF) { + fmt.Println("eof") + break + } + if err != nil { + break + } + if attrs.Prefix != "" && strings.HasPrefix(path.Base(attrs.Prefix), "cros") { + subdirs = append(subdirs, attrs.Prefix) + } + } + + if len(subdirs) == 0 { + return "", fmt.Errorf("no subdirectories found under %s", gcsBasePath) + } + + // Sort subdirectories to find the newest one (assuming lexicographical order corresponds to time). + sort.Strings(subdirs) + newestDir := subdirs[len(subdirs)-1] + return newestDir, nil +} + +// PullAllFilesFromGcsDir will grab all the files from a dir matching the postfix +func PullAllFilesFromGcsDir(ctx context.Context, bucket *storage.BucketHandle, dir string, postfix string) ([][]byte, error) { + it := bucket.Objects(ctx, &storage.Query{Prefix: dir}) + var data [][]byte // Accumulate data from all files + for { + attrs, err := it.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + break + } + fmt.Println(attrs.Name) + if strings.HasSuffix(attrs.Name, postfix) { + r, err := bucket.Object(attrs.Name).NewReader(ctx) + if err != nil { + return nil, fmt.Errorf("creating reader for %s: %w", attrs.Name, err) + } + defer r.Close() + + fileBytes, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("reading data from %s: %w", attrs.Name, err) + } + data = append(data, fileBytes) // Append data from current file + } + } + + if data == nil { + return nil, fmt.Errorf("no files found in newest directory %s", dir) + } + return data, nil +} + +func ExtractBucketAndPrefixFromPath(gcsPath string) (bucketName, prefix string, err error) { + parts := strings.SplitN(gcsPath, "/", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid GCS path: %s", gcsPath) + } + bucketName = parts[0] + prefix = fmt.Sprintf("%s/", parts[1]) + return +} + +type TestInfo struct { + Name string `json:"name"` + Tags []string `json:"tags"` + ExtraInfo []ExtraInfo `json:"extra_info"` +} + +type ExtraInfo struct { + ExecutableName string `json:"executable_name"` +} + +func TranslateMoblySrcToMetadata(src [][]byte) (metaData []*api.TestCaseMetadata) { + var testInfoList []TestInfo + + for _, srcInfo := range src { + var testInfoListLocal []TestInfo + err := json.Unmarshal(srcInfo, &testInfoListLocal) + testInfoList = append(testInfoList, testInfoListLocal...) + if err != nil { + return nil + } + } + + for _, testInfo := range testInfoList { + tc := createTestCaseFromTestInfo(testInfo) + metaData = append(metaData, tc) + } + return metaData +} + +func createTestCaseFromTestInfo(testInfo TestInfo) *api.TestCaseMetadata { + return &api.TestCaseMetadata{ + TestCase: &api.TestCase{ + Name: testInfo.Name, + Tags: formatTags(testInfo.Tags), + }, + TestCaseInfo: &api.TestCaseInfo{ + ExtraInfo: createExtraInfo(testInfo), + }, + } +} + +func formatTags(tags []string) (tagList []*api.TestCase_Tag) { + for _, tag := range tags { + tagList = append(tagList, &api.TestCase_Tag{Value: tag}) + } + return tagList +} + +func createExtraInfo(testInfo TestInfo) map[string]string { + extraInfo := make(map[string]string) + extraInfo["executable_name"] = testInfo.ExtraInfo[0].ExecutableName + return extraInfo +}