finder - add flexible sources port from infra/infra
When we move cros-test to infra/infra we can remove this forked code and use that as the true source
BUG=None
TEST=build
Change-Id: I6f9afb6c2b87588a4e64e2c774c7111aa246039e
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/6272939
Reviewed-by: Derek Beckett <dbeckett@chromium.org>
Reviewed-by: Billy Zhao <billyzhao@chromium.org>
Tested-by: Derek Beckett <dbeckett@chromium.org>
Commit-Queue: Derek Beckett <dbeckett@chromium.org>
diff --git a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go
index 8296882..250aed4 100644
--- a/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go
+++ b/src/go.chromium.org/chromiumos/test/execution/cmd/cros-test/cli/testexecserver.go
@@ -65,7 +65,20 @@
// runTests runs the requested tests.
func runTests(ctx context.Context, logger *log.Logger, resultRootDir, tlwAddr string, metadataList *api.TestCaseMetadataList, req *api.CrosTestRequest) (*api.CrosTestResponse, error) {
- matchedMdList, err := finder.MatchedTestsForSuites(metadataList.Values, req.TestSuites)
+ md := metadataList.GetValues()
+ shouldPullGcsMD := isMoblySuite(req)
+
+ if shouldPullGcsMD {
+ src, err := finder.GetSourceData(context.Background(), "mobly_priv_artifacts/out")
+ if err != nil {
+ fmt.Printf("err %s", err)
+ }
+ metadata := finder.TranslateMoblySrcToMetadata(src)
+ md = append(md, metadata...)
+
+ }
+
+ matchedMdList, err := finder.MatchedTestsForSuites(md, req.TestSuites)
if err != nil {
return nil, statuserrors.NewStatusError(statuserrors.InvalidArgument,
fmt.Errorf("failed to match test metadata: %v", err))
@@ -157,3 +170,25 @@
}
return nil
}
+
+func isMoblySuite(req *api.CrosTestRequest) bool {
+ for _, suite := range req.GetTestSuites() {
+ md := suite.GetExecutionMetadata()
+ if mdFlagPresent(md, "moblySuite") {
+
+ return true
+ }
+ }
+ return false
+}
+
+func mdFlagPresent(metadata *api.ExecutionMetadata, flagName string) bool {
+ if metadata != nil && len(metadata.Args) > 0 {
+ for _, arg := range metadata.Args {
+ if arg.Flag == flagName {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go b/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go
new file mode 100644
index 0000000..e30e8cf
--- /dev/null
+++ b/src/go.chromium.org/chromiumos/test/util/finder/flexible_sources.go
@@ -0,0 +1,174 @@
+// Copyright 2025 The ChromiumOS Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package finder find all matched tests from test metadata based on test criteria.
+package finder
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "path"
+ "sort"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "go.chromium.org/chromiumos/config/go/test/api"
+)
+
+func GetSourceData(ctx context.Context, gcsBasePath string) ([][]byte, error) {
+ client, err := storage.NewClient(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("storage.NewClient: %w", err)
+ }
+ defer client.Close()
+
+ bucketName, pf, err := ExtractBucketAndPrefixFromPath(gcsBasePath)
+ if err != nil {
+ return nil, err
+ }
+
+ bucket := client.Bucket(bucketName)
+
+ // Currently G3Mobly content is fetched off the latest.
+ // TODO, we probably should expose a way to pass in a desired path to allow repeatability on tests here.
+ newestDir, err := FindNewestDirInGcsBucket(ctx, gcsBasePath, bucket, pf)
+ if err != nil {
+ return nil, err
+ }
+ data, err := PullAllFilesFromGcsDir(ctx, bucket, newestDir, ".json")
+ if err != nil {
+ return nil, err
+ }
+ return data, nil
+}
+
+func FindNewestDirInGcsBucket(ctx context.Context, gcsBasePath string, bucket *storage.BucketHandle, pf string) (string, error) {
+ // List subdirectories under the prefix directory.
+ it := bucket.Objects(ctx, &storage.Query{Prefix: pf, Delimiter: "/"})
+
+ var subdirs []string
+ for {
+ attrs, err := it.Next()
+ if errors.Is(err, io.EOF) {
+ fmt.Println("eof")
+ break
+ }
+ if err != nil {
+ break
+ }
+ if attrs.Prefix != "" && strings.HasPrefix(path.Base(attrs.Prefix), "cros") {
+ subdirs = append(subdirs, attrs.Prefix)
+ }
+ }
+
+ if len(subdirs) == 0 {
+ return "", fmt.Errorf("no subdirectories found under %s", gcsBasePath)
+ }
+
+ // Sort subdirectories to find the newest one (assuming lexicographical order corresponds to time).
+ sort.Strings(subdirs)
+ newestDir := subdirs[len(subdirs)-1]
+ return newestDir, nil
+}
+
+// PullAllFilesFromGcsDir will grab all the files from a dir matching the postfix
+func PullAllFilesFromGcsDir(ctx context.Context, bucket *storage.BucketHandle, dir string, postfix string) ([][]byte, error) {
+ it := bucket.Objects(ctx, &storage.Query{Prefix: dir})
+ var data [][]byte // Accumulate data from all files
+ for {
+ attrs, err := it.Next()
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ break
+ }
+ fmt.Println(attrs.Name)
+ if strings.HasSuffix(attrs.Name, postfix) {
+ r, err := bucket.Object(attrs.Name).NewReader(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("creating reader for %s: %w", attrs.Name, err)
+ }
+ defer r.Close()
+
+ fileBytes, err := io.ReadAll(r)
+ if err != nil {
+ return nil, fmt.Errorf("reading data from %s: %w", attrs.Name, err)
+ }
+ data = append(data, fileBytes) // Append data from current file
+ }
+ }
+
+ if data == nil {
+ return nil, fmt.Errorf("no files found in newest directory %s", dir)
+ }
+ return data, nil
+}
+
+func ExtractBucketAndPrefixFromPath(gcsPath string) (bucketName, prefix string, err error) {
+ parts := strings.SplitN(gcsPath, "/", 2)
+ if len(parts) != 2 {
+ return "", "", fmt.Errorf("invalid GCS path: %s", gcsPath)
+ }
+ bucketName = parts[0]
+ prefix = fmt.Sprintf("%s/", parts[1])
+ return
+}
+
+type TestInfo struct {
+ Name string `json:"name"`
+ Tags []string `json:"tags"`
+ ExtraInfo []ExtraInfo `json:"extra_info"`
+}
+
+type ExtraInfo struct {
+ ExecutableName string `json:"executable_name"`
+}
+
+func TranslateMoblySrcToMetadata(src [][]byte) (metaData []*api.TestCaseMetadata) {
+ var testInfoList []TestInfo
+
+ for _, srcInfo := range src {
+ var testInfoListLocal []TestInfo
+ err := json.Unmarshal(srcInfo, &testInfoListLocal)
+ testInfoList = append(testInfoList, testInfoListLocal...)
+ if err != nil {
+ return nil
+ }
+ }
+
+ for _, testInfo := range testInfoList {
+ tc := createTestCaseFromTestInfo(testInfo)
+ metaData = append(metaData, tc)
+ }
+ return metaData
+}
+
+func createTestCaseFromTestInfo(testInfo TestInfo) *api.TestCaseMetadata {
+ return &api.TestCaseMetadata{
+ TestCase: &api.TestCase{
+ Name: testInfo.Name,
+ Tags: formatTags(testInfo.Tags),
+ },
+ TestCaseInfo: &api.TestCaseInfo{
+ ExtraInfo: createExtraInfo(testInfo),
+ },
+ }
+}
+
+func formatTags(tags []string) (tagList []*api.TestCase_Tag) {
+ for _, tag := range tags {
+ tagList = append(tagList, &api.TestCase_Tag{Value: tag})
+ }
+ return tagList
+}
+
+func createExtraInfo(testInfo TestInfo) map[string]string {
+ extraInfo := make(map[string]string)
+ extraInfo["executable_name"] = testInfo.ExtraInfo[0].ExecutableName
+ return extraInfo
+}
