blob: 2f1a7fb0052b7e7fd43befd35ae4248cbc4f4115 [file] [log] [blame]
// Copyright 2023 The ChromiumOS Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package main
import (
"fmt"
"log"
"os"
"strconv"
"strings"
"go.chromium.org/chromiumos/config/go/test/api"
dut_api "go.chromium.org/chromiumos/config/go/test/lab/api"
server "go.chromium.org/chromiumos/test/ctpv2/common/server_template"
conf "go.chromium.org/chromiumos/config/go"
)
func executor(req *api.InternalTestplan, log *log.Logger) (*api.InternalTestplan, error) {
// Will be the map of hardware which we can make provisionInfo for directly
foundHW := make(map[string]bool)
// Milestone which the task is being run from.
milestone := int64(0)
var swRequirements *api.LegacySW
var err error
for _, target := range req.GetSuiteInfo().GetSuiteMetadata().GetSchedulingUnits() {
swRequirements = target.GetPrimaryTarget().GetSwReq()
primaryHwTarget := target.GetPrimaryTarget().GetSwarmingDef()
addToFoundCache(primaryHwTarget, foundHW)
generateProvisionInfo(primaryHwTarget, target.GetPrimaryTarget().GetSwReq(), log)
for _, t := range target.GetCompanionTargets() {
generateProvisionInfo(t.GetSwarmingDef(), t.GetSwReq(), log)
addToFoundCache(t.GetSwarmingDef(), foundHW)
}
}
for _, legacyTarget := range req.GetSuiteInfo().GetSuiteMetadata().GetTargetRequirements() {
for _, innerTarget := range legacyTarget.GetHwRequirements().GetHwDefinition() {
swRequirements = legacyTarget.GetSwRequirement()
generateProvisionInfo(innerTarget, swRequirements, log)
addToFoundCache(innerTarget, foundHW)
}
}
if dddSuite(req) {
// For DDD suites, we need to create the targets for all found boards on the milestone.
milestone, err = milestoneFromGcs(swRequirements.GetGcsPath())
if err != nil {
return req, fmt.Errorf("unable to determine milestone for provision builder: %s", err)
}
req, err := createTargetsForBoards(log, req, swRequirements.GetBuild(), foundHW, milestone)
return req, err
}
return req, nil
}
func addToFoundCache(hwTarget *api.SwarmingDefinition, foundHW map[string]bool) {
switch hw := hwTarget.GetDutInfo().GetDutType().(type) {
case *dut_api.Dut_Chromeos:
board := hw.Chromeos.GetDutModel().GetBuildTarget()
variant := hwTarget.GetVariant()
if variant != "" {
board = fmt.Sprintf("%s-%s", board, variant)
}
foundHW[board] = true
}
}
// Extract the milestone # as an int64 from the standard gcs path.
func milestoneFromGcs(gcsPath string) (int64, error) {
// example: gs://chromeos-image-archive/snappy-kernelnext-release/R122-15753.34.0
parts := strings.Split(gcsPath, "/")
if len(parts) < 2 {
return 0, fmt.Errorf("improper or incomplete gcs path given: %s", gcsPath)
}
// Will now be R122-15753.34.0
fullBuildNum := parts[len(parts)-1]
// Will now be ['R122', '15753.34.0'] 0 index --> 'R122'
f := strings.Split(fullBuildNum, "-")[0]
// Replace R with "", then convert it to int64
// 122
milestone, err := strconv.ParseInt(strings.Replace(f, "R", "", -1), 10, 64)
if err != nil {
return -1, err
}
return milestone, nil
}
// Remove any variant info from the board and return the board, only
func boardOnly(targ string) (string, error) {
f := strings.Split(targ, "-")
if len(f) == 1 {
return targ, nil
}
if len(f) == 0 {
return "", fmt.Errorf("no board was provided: %s", targ)
}
return f[0], nil
}
func createTargetsForBoards(log *log.Logger, req *api.InternalTestplan, channel string, foundHW map[string]bool, milestone int64) (*api.InternalTestplan, error) {
build, targets, err := getBuildsForMilestone(log, milestone)
if err != nil {
return req, fmt.Errorf("unable to determine provision info for 3d suite")
}
for _, targ := range targets {
found, ok := foundHW[targ]
// If we already have provision info provided, don't overwrite that.
if found && ok {
log.Println(fmt.Sprintf("FOUND: %s", targ))
continue
}
log.Println(fmt.Sprintf("NOT FOUND: %s", targ))
board, err := boardOnly(targ)
if err != nil {
return nil, err
}
gcsPath := createGcsPath(targ, channel, build, milestone)
if len(req.GetSuiteInfo().GetSuiteMetadata().GetTargetRequirements()) > 0 {
hwTarget := &api.HWRequirements{
HwDefinition: buildHWTargetOld(board, getVariantFromBuildTarg(targ), generateProvisionInfoOld(nil, gcsPath)),
// the OPTIONAL mark indicates to HW filters this board does not need to be added to the test request
State: api.HWRequirements_OPTIONAL,
}
targRequirements := &api.TargetRequirements{
HwRequirements: hwTarget,
SwRequirement: &api.LegacySW{GcsPath: gcsPath},
}
req.GetSuiteInfo().GetSuiteMetadata().TargetRequirements = append(req.GetSuiteInfo().GetSuiteMetadata().TargetRequirements, targRequirements)
} else {
targ := &api.Target{
SwarmingDef: buildHWTarget(board, getVariantFromBuildTarg(targ)),
SwReq: &api.LegacySW{GcsPath: gcsPath},
}
generateProvisionInfo(targ.SwarmingDef, targ.SwReq, log)
SU := &api.SchedulingUnit{
PrimaryTarget: targ,
}
req.GetSuiteInfo().GetSuiteMetadata().SchedulingUnits = append(req.GetSuiteInfo().GetSuiteMetadata().SchedulingUnits, SU)
}
}
return req, nil
}
func getVariantFromBuildTarg(board string) string {
// foo-bar --> bar
parts := strings.Split(board, "-")
if len(parts) < 2 {
return ""
}
return strings.Join(parts[1:][:], "-")
}
// format a GCS path using the standard string base.
func createGcsPath(board string, channel string, build string, milestone int64) string {
return fmt.Sprintf("gs://chromeos-image-archive/%s-%s/R%v-%s", board, channel, milestone, build)
}
func dddSuite(req *api.InternalTestplan) bool {
return req.GetSuiteInfo().GetSuiteRequest().GetDddSuite()
}
func generateCrosImageProvisionInfo(target *api.SwarmingDefinition, swReq *api.LegacySW, log *log.Logger) {
current := target.GetProvisionInfo()
path := swReq.GetGcsPath()
log.Println(fmt.Sprintf("generateCrosImageProvisionInfo PATH: %s", path))
log.Println(fmt.Sprintf("generateCrosImageProvisionInfo sw: %s", swReq))
if path != "" {
installInfo := &api.InstallRequest{
ImagePath: &conf.StoragePath{
Path: path,
HostType: conf.StoragePath_GS,
},
}
info := &api.ProvisionInfo{
Type: api.ProvisionInfo_CROS,
InstallRequest: installInfo,
}
target.ProvisionInfo = append(current, info)
}
}
// Currently this is simple... just append the new provision info onto the existing.
// We are appending it in case another provsion filter has run and populated info.
func generateProvisionInfo(target *api.SwarmingDefinition, swReq *api.LegacySW, log *log.Logger) {
switch target.GetDutInfo().GetDutType().(type) {
case *dut_api.Dut_Chromeos:
log.Println("chromeos DUT")
generateCrosImageProvisionInfo(target, swReq, log)
}
}
func getBuildsForMilestone(log *log.Logger, milestone int64) (string, []string, error) {
log.Println("in parent check")
bqIter, err := queryForBuilds(log)
if err != nil {
return "", nil, fmt.Errorf("unable to determine stabily: %s", err)
}
data := iterThroughData(bqIter, log)
log.Println("past iter check")
return determineBuilds(data, milestone)
}
// determineBuilds will return the latest build from the given milestone, and boards which built on that build.
// Variants are considered a unique board.
// Latest, will be either the most recent, or most recent-1; to cover for the case where the most recent is still
// "mid build", and boards have not completed their builds yet.
func determineBuilds(data []resSchema, tMilestone int64) (string, []string, error) {
newestOnMilestone := ""
secondNewestnewestOnMilestone := ""
buildMap := make(map[string][]string)
for _, d := range data {
if d.Milestone != tMilestone {
continue
}
if newestOnMilestone == "" {
newestOnMilestone = d.Platform
} else {
if buildNewer(d.Platform, newestOnMilestone) {
newestOnMilestone = d.Platform
}
}
// If the build is not the newest on milestone, check if its the second newest
if d.Platform != newestOnMilestone {
if secondNewestnewestOnMilestone == "" {
secondNewestnewestOnMilestone = d.Platform
} else {
if buildNewer(d.Platform, secondNewestnewestOnMilestone) {
secondNewestnewestOnMilestone = d.Platform
}
}
}
_, ok := buildMap[d.Platform]
if !ok {
buildMap[d.Platform] = []string{d.Board}
} else {
buildMap[d.Platform] = append(buildMap[d.Platform], d.Board)
}
}
if len(buildMap[newestOnMilestone]) >= len(buildMap[secondNewestnewestOnMilestone]) {
return newestOnMilestone, buildMap[newestOnMilestone], nil
}
return secondNewestnewestOnMilestone, buildMap[secondNewestnewestOnMilestone], nil
}
func buildNewer(platform string, platform2 string) bool {
parts := strings.Split(platform, ".")
parts2 := strings.Split(platform2, ".")
if len(parts) != 3 || len(parts2) != 3 {
fmt.Println(platform, platform2)
panic("not good:")
}
p1, _ := strconv.Atoi(parts[0])
p2, _ := strconv.Atoi(parts2[0])
m1, _ := strconv.Atoi(parts[1])
m2, _ := strconv.Atoi(parts2[1])
f1, _ := strconv.Atoi(parts[2])
f2, _ := strconv.Atoi(parts2[2])
// If the major version > other major version, its newer
if p1 > p2 {
return true
// If the second ietems major version >, then its false
} else if p2 > p1 {
return false
// If its equal, we do the same check with the minor versions
} else if m1 > m2 {
return true
} else if m2 > m1 {
return false
// and finally the final version check
} else if f1 > f2 {
return true
} else if f2 > f1 {
return false
}
return false
}
func buildHWTarget(boardName string, variant string) *api.SwarmingDefinition {
dut := &dut_api.Dut{}
Cros := &dut_api.Dut_ChromeOS{DutModel: &dut_api.DutModel{
BuildTarget: boardName,
}}
dut.DutType = &dut_api.Dut_Chromeos{Chromeos: Cros}
def := &api.SwarmingDefinition{DutInfo: dut}
if variant != "" {
def.Variant = variant
}
return def
}
func buildHWTargetOld(boardName string, variant string, provInfo []*api.ProvisionInfo) []*api.SwarmingDefinition {
dut := &dut_api.Dut{}
Cros := &dut_api.Dut_ChromeOS{DutModel: &dut_api.DutModel{
BuildTarget: boardName,
}}
dut.DutType = &dut_api.Dut_Chromeos{Chromeos: Cros}
def := &api.SwarmingDefinition{DutInfo: dut,
ProvisionInfo: provInfo,
}
if variant != "" {
def.Variant = variant
}
f := []*api.SwarmingDefinition{def}
return f
}
func generateProvisionInfoOld(current []*api.ProvisionInfo, path string) []*api.ProvisionInfo {
installInfo := &api.InstallRequest{
ImagePath: &conf.StoragePath{
Path: path,
HostType: conf.StoragePath_GS,
},
}
info := &api.ProvisionInfo{
Type: api.ProvisionInfo_CROS,
InstallRequest: installInfo,
}
return append(current, info)
}
func main() {
err := server.Server(executor, "provision_filter")
if err != nil {
os.Exit(2)
}
os.Exit(0)
}