rdb publish: upload the updated eqc_category_expression to BQ
Dependent CLs:
- https://crrev.com/c/6249144
- https://crrev.com/c/6248741
LED: http://ci.chromium.org/b/8723080997439984897/infra
BQ data: http://screen/7b43fS7W63v9JD7.png
Bug: b:395760082, b:379801095
Test: Unit tests and LED
Change-Id: Ie6f67ad7f707a3aa649ae4dc902bf6ccabe01325
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/dev-util/+/6261828
Reviewed-by: Jason Kusuma <jkusuma@google.com>
Tested-by: Allen Xie <zhihuixie@google.com>
Reviewed-by: Tom Handakas <thandakas@chromium.org>
Commit-Queue: Allen Xie <zhihuixie@google.com>
diff --git a/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service.go b/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service.go
index f0b545a..8f038b0 100644
--- a/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service.go
+++ b/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service.go
@@ -44,11 +44,10 @@
// EQCRow represents a row in the EqC info BigQuery table.
type EQCRow struct {
- EQCHash string
- EQCName string
- // TODO: b/395760082 - Add the EqcCategoryExpression back once its type
- // is updated to JSON from string.
- EQCDimensions map[string]string
+ EQCHash string
+ EQCName string
+ EQCCategoryExpression map[string]string
+ EQCDimensions map[string]string
}
// NewEQCPublishService create a new EqC publish service to interact with
@@ -112,11 +111,10 @@
// Inserts the EqC info because it doesn't exist in the BQ table.
eqcRow := &EQCRow{
- EQCHash: eqcInfo.EqcHash,
- EQCName: eqcInfo.EqcName,
- // TODO: b/395760082 - Add the EqcCategoryExpression back once its type
- // is updated to JSON from string.
- EQCDimensions: eqcInfo.EqcDimensions,
+ EQCHash: eqcInfo.EqcHash,
+ EQCName: eqcInfo.EqcName,
+ EQCCategoryExpression: eqcInfo.EqcCategoryExpression,
+ EQCDimensions: eqcInfo.EqcDimensions,
}
inserter := eps.bqClient.Dataset(TestInfoDataset).Table(EQCInfoTable).Inserter()
@@ -173,6 +171,15 @@
// Save implements the ValueSaver interface for the EQCRow struct.
func (e *EQCRow) Save() (map[string]bigquery.Value, string, error) {
+ if e.EQCCategoryExpression == nil {
+ e.EQCCategoryExpression = make(map[string]string)
+ }
+
+ categoryExpression, err := json.Marshal(e.EQCCategoryExpression)
+ if err != nil {
+ return nil, "", fmt.Errorf("marshalling the EqC category expression: %w", err)
+ }
+
if e.EQCDimensions == nil {
e.EQCDimensions = make(map[string]string)
}
@@ -183,11 +190,10 @@
}
return map[string]bigquery.Value{
- "eqc_hash": e.EQCHash,
- "eqc_name": e.EQCName,
- // TODO: b/395760082 - Add the EqcCategoryExpression back once its type
- // is updated to JSON from string.
- "eqc_dimensions": string(dimensions),
+ "eqc_hash": e.EQCHash,
+ "eqc_name": e.EQCName,
+ "eqc_category_expression": string(categoryExpression),
+ "eqc_dimensions": string(dimensions),
}, e.EQCHash, nil
}
@@ -213,20 +219,45 @@
}
eqcInfo := &artifact.EqcInfo{
- EqcHash: eqcInfoMap["eqcHash"],
- EqcName: eqcInfoMap["eqcName"],
- // TODO: b/395760082 - Add the EqcCategoryExpression back once its type
- // is updated to JSON from string.
+ EqcHash: eqcInfoMap["eqcHash"],
+ EqcName: eqcInfoMap["eqcName"],
+ EqcCategoryExpression: make(map[string]string),
+ EqcDimensions: make(map[string]string),
+ }
+
+ // The content of the "eqcCategoryExpression" field is aligned with the
+ // CategoryExpression proto in "/ttcp/protos/ttcp/syntax/syntax.proto".
+ if categoryExpressionJSON, ok := eqcInfoMap["eqcCategoryExpression"]; ok {
+ categoryExpressionMap := make(map[string]interface{})
+ if err := json.Unmarshal([]byte(categoryExpressionJSON), &categoryExpressionMap); err != nil {
+ return nil, fmt.Errorf("unmarshalling the EqC category expression: %w", err)
+ }
+
+ for k, v := range categoryExpressionMap {
+ switch value := v.(type) {
+ case string:
+ // Corresponds to the name field which indicates the name of the
+ // predefined category. Most of the time (> 99.99%) it will be
+ // mapped to this string type.
+ eqcInfo.EqcCategoryExpression[k] = fmt.Sprintf("%v", value)
+ default:
+ // Corresponds to the category proto field which explicitly
+ // specifies the category contents.
+ jsonString, err := json.Marshal(value)
+ if err != nil {
+ log.Printf("Failed to marshal the category expression key: %s", k)
+ continue
+ }
+ eqcInfo.EqcCategoryExpression[k] = string(jsonString)
+ }
+
+ }
}
if dimensionsJSON, ok := eqcInfoMap["eqcDimensions"]; ok {
- dimensionsMap := make(map[string]string)
- if err := json.Unmarshal([]byte(dimensionsJSON), &dimensionsMap); err != nil {
+ if err := json.Unmarshal([]byte(dimensionsJSON), &eqcInfo.EqcDimensions); err != nil {
return nil, fmt.Errorf("unmarshalling the EqC dimensions: %w", err)
}
- eqcInfo.EqcDimensions = dimensionsMap
- } else {
- eqcInfo.EqcDimensions = make(map[string]string)
}
log.Printf("Successfully extracted the EqC info: %#v", eqcInfo)
diff --git a/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service_test.go b/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service_test.go
index 0a43e12..d2c5c0f 100644
--- a/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service_test.go
+++ b/src/go.chromium.org/chromiumos/test/publish/cmd/rdb-publish/service/eqc_service_test.go
@@ -27,6 +27,9 @@
wantEQCInfo := &artifact.EqcInfo{
EqcHash: "9073744604696850342",
EqcName: "Cometlake-U__INTEL_HRP2_AX201__5.15",
+ EqcCategoryExpression: map[string]string{
+ "name": "WifiBtChipset_Soc_Kernel_Intel",
+ },
EqcDimensions: map[string]string{
"dlm:soc": "Cometlake-U",
"image:_kernel_version": "5.15",
@@ -34,9 +37,10 @@
},
}
eqcInfoMap := map[string]string{
- "eqcDimensions": "{\"dlm:soc\":\"Cometlake-U\",\"image:_kernel_version\":\"5.15\",\"wireless_field\":\"INTEL_HRP2_AX201\"}",
- "eqcHash": "9073744604696850342",
- "eqcName": "Cometlake-U__INTEL_HRP2_AX201__5.15",
+ "eqcCategoryExpression": "{\"name\": \"WifiBtChipset_Soc_Kernel_Intel\"}",
+ "eqcDimensions": "{\"dlm:soc\":\"Cometlake-U\",\"image:_kernel_version\":\"5.15\",\"wireless_field\":\"INTEL_HRP2_AX201\"}",
+ "eqcHash": "9073744604696850342",
+ "eqcName": "Cometlake-U__INTEL_HRP2_AX201__5.15",
}
rdbMetadata := &metadata.PublishRdbMetadata{
PublishKeys: []*api.PublishKey{
@@ -105,6 +109,9 @@
entry := &EQCRow{
EQCHash: eqcHash,
EQCName: eqcName,
+ EQCCategoryExpression: map[string]string{
+ "name": "WifiBtChipset_Soc_Kernel_Intel",
+ },
EQCDimensions: map[string]string{
"soc": "value1",
"wifiChip": "value2",
@@ -112,9 +119,10 @@
}
wantValue := map[string]bigquery.Value{
- "eqc_hash": eqcHash,
- "eqc_name": eqcName,
- "eqc_dimensions": `{"soc":"value1","wifiChip":"value2"}`, // Note: order may vary
+ "eqc_hash": eqcHash,
+ "eqc_name": eqcName,
+ "eqc_category_expression": `{"name": "WifiBtChipset_Soc_Kernel_Intel"}`,
+ "eqc_dimensions": `{"soc":"value1","wifiChip":"value2"}`, // Note: order may vary
}
gotValue, gotInsertID, err := entry.Save()
@@ -124,6 +132,17 @@
So(gotValue["eqc_hash"], ShouldEqual, wantValue["eqc_hash"])
So(gotValue["eqc_name"], ShouldEqual, eqcName)
+ // Compare maps, ignoring key order in eqc_category_expression.
+ gotCategoryExpression := gotValue["eqc_category_expression"]
+ wantCategoryExpression := wantValue["eqc_category_expression"]
+
+ var gotCategoryExprMap, wantCategoryExprMap map[string]string
+ err = json.Unmarshal([]byte(gotCategoryExpression.(string)), &gotCategoryExprMap)
+ So(err, ShouldBeNil)
+ err = json.Unmarshal([]byte(wantCategoryExpression.(string)), &wantCategoryExprMap)
+ So(err, ShouldBeNil)
+ So(gotCategoryExprMap, ShouldResemble, wantCategoryExprMap)
+
// Compare maps, ignoring key order in eqc_dimensions.
gotDimensions := gotValue["eqc_dimensions"]
wantDimensions := wantValue["eqc_dimensions"]
@@ -139,15 +158,17 @@
Convey("Empty dimensions", func() {
entry := &EQCRow{
- EQCHash: eqcHash,
- EQCName: eqcName,
- EQCDimensions: make(map[string]string), // Empty map
+ EQCHash: eqcHash,
+ EQCName: eqcName,
+ EQCCategoryExpression: make(map[string]string), // Empty map
+ EQCDimensions: make(map[string]string), // Empty map
}
wantValue := map[string]bigquery.Value{
- "eqc_hash": eqcHash,
- "eqc_name": eqcName,
- "eqc_dimensions": `{}`,
+ "eqc_hash": eqcHash,
+ "eqc_name": eqcName,
+ "eqc_category_expression": `{}`,
+ "eqc_dimensions": `{}`,
}
gotValue, _, err := entry.Save()
@@ -158,15 +179,17 @@
Convey("Nil dimensions", func() {
entry := &EQCRow{
- EQCHash: eqcHash,
- EQCName: eqcName,
- EQCDimensions: nil,
+ EQCHash: eqcHash,
+ EQCName: eqcName,
+ EQCCategoryExpression: nil,
+ EQCDimensions: nil,
}
wantValue := map[string]bigquery.Value{
- "eqc_hash": eqcHash,
- "eqc_name": eqcName,
- "eqc_dimensions": `{}`, // Should be treated as an empty map
+ "eqc_hash": eqcHash,
+ "eqc_name": eqcName,
+ "eqc_category_expression": `{}`,
+ "eqc_dimensions": `{}`, // Should be treated as an empty map
}
gotValue, _, err := entry.Save()
@@ -185,6 +208,9 @@
wantEQCInfo := &artifact.EqcInfo{
EqcHash: "9073744604696850342",
EqcName: "Cometlake-U__INTEL_HRP2_AX201__5.15",
+ EqcCategoryExpression: map[string]string{
+ "value": "{\"combinatorial\":{\"subcategories\":[{\"value\":{\"enumerated\":{\"classes\":[{\"value\":{\"expression\":{\"property\":{\"propertyPath\":\"swarming:label-wifi_state\",\"strEqual\":\"NORMAL\"}},\"name\":\"swarming:label-wifi_state:NORMAL\"}}]}}},{\"name\":\"WifiBtChipset_Soc_Kernel\"}]}}",
+ },
EqcDimensions: map[string]string{
"dlm:soc": "Cometlake-U",
"image:_kernel_version": "5.15",
@@ -192,9 +218,10 @@
},
}
eqcInfoMap := map[string]string{
- "eqcDimensions": "{\"dlm:soc\":\"Cometlake-U\",\"image:_kernel_version\":\"5.15\",\"wireless_field\":\"INTEL_HRP2_AX201\"}",
- "eqcHash": "9073744604696850342",
- "eqcName": "Cometlake-U__INTEL_HRP2_AX201__5.15",
+ "eqcCategoryExpression": "{\"value\":{\"combinatorial\":{\"subcategories\":[{\"value\":{\"enumerated\":{\"classes\":[{\"value\":{\"name\":\"swarming:label-wifi_state:NORMAL\",\"expression\":{\"property\":{\"propertyPath\":\"swarming:label-wifi_state\",\"strEqual\":\"NORMAL\"}}}}]}}},{\"name\":\"WifiBtChipset_Soc_Kernel\"}]}}}",
+ "eqcDimensions": "{\"dlm:soc\":\"Cometlake-U\",\"image:_kernel_version\":\"5.15\",\"wireless_field\":\"INTEL_HRP2_AX201\"}",
+ "eqcHash": "9073744604696850342",
+ "eqcName": "Cometlake-U__INTEL_HRP2_AX201__5.15",
}
rdbMetadata := &metadata.PublishRdbMetadata{
PublishKeys: []*api.PublishKey{
