blob: 4ce12b5c63f0a206fd4ded6a3ad501c8032f222c [file] [log] [blame]
// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package job
import (
fmt "fmt"
"sort"
"strconv"
"strings"
"time"
"github.com/golang/protobuf/ptypes"
durpb "google.golang.org/protobuf/types/known/durationpb"
"go.chromium.org/luci/common/data/stringset"
"go.chromium.org/luci/common/errors"
)
// ExpiringValue represents a tuple of dimension value, plus an expiration time.
//
// If Expiration is zero, it counts as "no expiration".
type ExpiringValue struct {
Value string
Expiration time.Duration
}
// DimensionEditCommand is instruction on how to process the values in the task
// associated with a swarming dimension.
//
// The fields are processed in order:
// - if SetValues is non-nil, the dimension values are set to this set
// (including empty).
// - if RemoveValues is non-empty, these values will be removed from the
// dimension values.
// - if AddValues is non-empty, these values will ber added to the dimension
// values.
//
// If the set of values at the end of this process is empty, the dimension will
// be removed from the task. Otherwise the dimension will be set to the sorted
// remaining values.
type DimensionEditCommand struct {
SetValues []ExpiringValue
RemoveValues []string
AddValues []ExpiringValue
}
// DimensionEditCommands is a mapping of dimension name to a set of commands to
// apply to the values of that dimension.
type DimensionEditCommands map[string]*DimensionEditCommand
func split2(s, sep string) (a, b string, ok bool) {
idx := strings.Index(s, sep)
if idx == -1 {
return s, "", false
}
return s[:idx], s[idx+1:], true
}
func rsplit2(s, sep string) (a, b string) {
idx := strings.LastIndex(s, sep)
if idx == -1 {
return s, ""
}
return s[:idx], s[idx+1:]
}
func parseDimensionEditCmd(cmd string) (dim, op, val string, exp time.Duration, err error) {
dim, valueExp, ok := split2(cmd, "=")
if !ok {
err = errors.Reason("expected $key$op$value, but op was missing").Err()
return
}
switch dim[len(dim)-1] {
case '-':
op = "-="
dim = dim[:len(dim)-1]
case '+':
op = "+="
dim = dim[:len(dim)-1]
default:
op = "="
}
val, expStr := rsplit2(valueExp, "@")
if expStr != "" {
var expSec int
if expSec, err = strconv.Atoi(expStr); err != nil {
err = errors.Annotate(err, "parsing expiration %q", expStr).Err()
return
}
exp = time.Second * time.Duration(expSec)
}
if val == "" && op != "=" {
err = errors.Reason("empty value not allowed for operator %q: %q", op, cmd).Err()
}
if exp != 0 && op == "-=" {
err = errors.Reason("expiration seconds not allowed for operator %q: %q", op, cmd).Err()
}
return
}
// MakeDimensionEditCommands takes a slice of commands in the form of:
//
// dimension=
// dimension=value
// dimension=value@1234
//
// dimension-=value
//
// dimension+=value
// dimension+=value@1234
//
// Logically:
// - dimension_name - The name of the dimension to modify
// - operator
// - "=" - Add value to SetValues. If empty, ensures that SetValues is
// non-nil (i.e. clear all values for this dimension).
// - "-=" - Add value to RemoveValues.
// - "+=" - Add value to AddValues.
// - value - The dimension value for the operand
// - expiration seconds - The time at which this value should expire.
//
// All equivalent operations for the same dimension will be grouped into
// a single DimensionEditCommand in the order they appear in `commands`.
func MakeDimensionEditCommands(commands []string) (DimensionEditCommands, error) {
if len(commands) == 0 {
return nil, nil
}
ret := DimensionEditCommands{}
for _, command := range commands {
dimension, operator, value, expiration, err := parseDimensionEditCmd(command)
if err != nil {
return nil, errors.Annotate(err, "parsing %q", command).Err()
}
editCmd := ret[dimension]
if editCmd == nil {
editCmd = &DimensionEditCommand{}
ret[dimension] = editCmd
}
switch operator {
case "=":
// explicitly setting SetValues takes care of the 'dimension=' case.
if editCmd.SetValues == nil {
editCmd.SetValues = []ExpiringValue{}
}
if value != "" {
editCmd.SetValues = append(editCmd.SetValues, ExpiringValue{
Value: value, Expiration: expiration,
})
}
case "-=":
editCmd.RemoveValues = append(editCmd.RemoveValues, value)
case "+=":
editCmd.AddValues = append(editCmd.AddValues, ExpiringValue{
Value: value, Expiration: expiration,
})
}
}
return ret, nil
}
// Applies the DimensionEditCommands to the given logicalDimensions.
func (dimEdits DimensionEditCommands) apply(dimMap logicalDimensions, minExp time.Duration) {
if len(dimEdits) == 0 {
return
}
shouldApply := func(eVal ExpiringValue) bool {
return eVal.Expiration == 0 || minExp == 0 || eVal.Expiration >= minExp
}
for dim, edits := range dimEdits {
if edits.SetValues != nil {
dimMap[dim] = make(dimValueExpiration, len(edits.SetValues))
for _, expVal := range edits.SetValues {
if shouldApply(expVal) {
dimMap[dim][expVal.Value] = expVal.Expiration
}
}
}
for _, value := range edits.RemoveValues {
delete(dimMap[dim], value)
}
for _, expVal := range edits.AddValues {
if shouldApply(expVal) {
expValMap := dimMap[dim]
if expValMap == nil {
expValMap = dimValueExpiration{}
dimMap[dim] = expValMap
}
expValMap[expVal.Value] = expVal.Expiration
}
}
}
toRemove := stringset.New(len(dimMap))
for dim, valExps := range dimMap {
if len(valExps) == 0 {
toRemove.Add(dim)
}
}
toRemove.Iter(func(dim string) bool {
delete(dimMap, dim)
return true
})
}
// ExpiringDimensions is a map from dimension name to a list of values
// corresponding to that dimension.
//
// When retrieved from a led library, the values will be sorted by expiration
// time, followed by value. Expirations of 0 (i.e. "infinite") are sorted last.
type ExpiringDimensions map[string][]ExpiringValue
func (e ExpiringDimensions) String() string {
bits := []string{}
for key, values := range e {
for _, value := range values {
if value.Expiration == 0 {
bits = append(bits, fmt.Sprintf("%s=%s", key, value.Value))
} else {
bits = append(bits, fmt.Sprintf(
"%s=%s@%d", key, value.Value, value.Expiration/time.Second))
}
}
}
return strings.Join(bits, ", ")
}
func (e ExpiringDimensions) toLogical() logicalDimensions {
ret := logicalDimensions{}
for key, expVals := range e {
dve := ret[key]
if dve == nil {
dve = dimValueExpiration{}
ret[key] = dve
}
for _, expVal := range expVals {
dve[expVal.Value] = expVal.Expiration
}
}
return ret
}
type dimValueExpiration map[string]time.Duration
func (valExps dimValueExpiration) toSlice() []string {
ret := make([]string, 0, len(valExps))
for value := range valExps {
ret = append(ret, value)
}
sort.Strings(ret)
return ret
}
// A multimap of dimension to values.
type logicalDimensions map[string]dimValueExpiration
func (dims logicalDimensions) update(dim, value string, expiration *durpb.Duration) {
var exp time.Duration
if expiration != nil {
var err error
if exp, err = ptypes.Duration(expiration); err != nil {
panic(err)
}
}
dims.updateDuration(dim, value, exp)
}
func (dims logicalDimensions) updateDuration(dim, value string, exp time.Duration) {
if dims[dim] == nil {
dims[dim] = dimValueExpiration{}
}
dims[dim][value] = exp
}
func expLess(a, b time.Duration) bool {
if a == 0 { // b is either infinity (==a) or finite (<a)
return false
} else if b == 0 { // b is infinity, a is finite
return true
}
return a < b
}
func (dims logicalDimensions) toExpiringDimensions() ExpiringDimensions {
ret := ExpiringDimensions{}
for key, dve := range dims {
for dim, expiration := range dve {
ret[key] = append(ret[key], ExpiringValue{dim, expiration})
}
sort.Slice(ret[key], func(i, j int) bool {
a, b := ret[key][i], ret[key][j]
if a.Expiration == b.Expiration {
return a.Value < b.Value
}
return expLess(a.Expiration, b.Expiration)
})
}
return ret
}