cv/internal/tryjob/model.go - infra/luci/luci-go - Git at Google

 // Copyright 2021 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package tryjob

 import (
 	"fmt"
 	"strconv"
 	"strings"
 	"time"

 	"go.chromium.org/luci/common/errors"
 	"go.chromium.org/luci/gae/service/datastore"

 	"go.chromium.org/luci/cv/internal/common"
 )

 const (
 	// TryjobKind is the Datastore entity kind for Tryjob.
 	TryjobKind = "Tryjob"
 )

 // Tryjob is an entity tracking CV Tryjobs.
 type Tryjob struct {
 	// $kind must match TryjobKind.
 	_kind  string                `gae:"$kind,Tryjob"`
 	_extra datastore.PropertyMap `gae:"-,extra"`

 	// ID is the Tryjob ID, autogenerated by the Datastore.
 	ID common.TryjobID `gae:"$id"`
 	// ExternalID is a Tryjob ID in external system, e.g. Buildbucket.
 	//
 	// There can be at most one Tryjob with a given ExternalID.
 	//
 	// Indexed.
 	ExternalID ExternalID
 	// EVersion is the entity version.
 	//
 	// It increments by one upon every successful modification.
 	EVersion int64 `gae:",noindex"`
 	// EntityCreateTime is the timestamp when this entity was created.
 	//
 	// NOTE: this is not the backend's tryjob creation time, which is stored in
 	// .Result.CreateTime.
 	EntityCreateTime time.Time `gae:",noindex"`
 	// UpdateTime is the timestamp when this entity was last updated.
 	//
 	// NOTE: this is not the backend's tryjob update time, which is stored in
 	// .Result.UpdateTime.
 	EntityUpdateTime time.Time `gae:",noindex"`

 	// ReuseKey is used to quickly decide if this Tryjob can be reused by a run.
 	//
 	// Note that, even if reuse is allowed here, reuse is still subjected to
 	// other restrictions (for example, Tryjob is not fresh enough for the run).
 	//
 	// reusekey is currently computed in the following way:
 	//  base64(
 	//    sha256(
 	//      '\0'.join(sorted('%d/%d' % (cl.ID, cl.minEquiPatchSet) for cl in cls))
 	//    )
 	//  )
 	//
 	// Indexed
 	ReuseKey string

 	// Definition of the tryjob.
 	//
 	// Immutable.
 	Definition *Definition

 	// Status of the Tryjob.
 	//
 	// Indexed.
 	Status Status

 	// TriggeredByCV is true if it was triggered by CV.
 	//
 	// This is used for cancelation, since CV shouldn't cancel tryjobs it
 	// didn't trigger.
 	TriggeredByCV bool `gae:",noindex"`

 	// Result of the Tryjob.
 	//
 	// Must be set if Status is ENDED.
 	// May be set if Status is TRIGGERED.
 	//
 	// It's used by the Run Manager.
 	Result *Result

 	// TriggeredBy is the Run that triggered this Tryjob.
 	//
 	// May be unset if the Tryjob was not triggered by CV, in which case
 	// ReusedBy has at least one Run.
 	//
 	// Indexed.
 	TriggeredBy common.RunID

 	// ReusedBy are the Runs that are interested in the result of this Tryjob.
 	//
 	// Indexed.
 	ReusedBy common.RunIDs

 	// CLPatchsets is an array of CLPatchset that each identify a specific
 	// patchset in a specific CL.
 	//
 	// The values are to be computed by MakeCLPatchset().
 	// See its documentation for details.
 	//
 	// Sorted and Indexed.
 	CLPatchsets CLPatchsets
 }

 // tryjobMap is intended to quickly determine if a given ExternalID is
 // associated with a Tryjob entity in the datastore.
 //
 // This also ensures that at most one TryjobID will be associated with a given
 // ExternalID.
 type tryjobMap struct {
 	_kind string `gae:"$kind,TryjobMap"`

 	// ExternalID is an ID for the tryjob in the external backend.
 	//
 	// Making this the key of the map ensures uniqueness.
 	ExternalID ExternalID `gae:"$id"`

 	// InternalID is auto-generated by Datastore for Tryjob entity.
 	InternalID common.TryjobID `gae:",noindex"` // int64. Indexed in Tryjob entities.
 }

 // LUCIProject() returns the project in the context of which the Tryjob is
 // updated, and which is thus allowed to "read" the Tryjob.
 //
 // In the case of Buildbucket, this may be different from the LUCI project to
 // which the corresponding build belongs. For example, consider a "v8" project
 // with configuration saying to trigger "chromium/try/linux_rel" builder: when
 // CV triggers a new tryjob T for a "v8" Run, T.LUCIProject() will be "v8" even
 // though the build itself will be in the "chromium/try" Buildbucket bucket.
 //
 // In general, a Run of project P must not re-use tryjob T if
 // T.LUCIProject() != P, until it has been verified with the tryjob backend
 // that P has access to T.
 func (t *Tryjob) LUCIProject() string {
 	if t.TriggeredBy != "" {
 		return t.TriggeredBy.LUCIProject()
 	}
 	if len(t.ReusedBy) == 0 {
 		panic("tryjob is not associated with any runs")
 	}
 	return t.ReusedBy[0].LUCIProject()
 }

 // AllWatchingRuns returns the IDs for the Runs that care about this tryjob.
 //
 // This includes the triggerer (if the tryjob was triggered by CV) and all the
 // Runs reusing this tryjob (if any).
 func (t *Tryjob) AllWatchingRuns() common.RunIDs {
 	ret := make(common.RunIDs, 0, 1+len(t.ReusedBy))
 	if t.TriggeredBy != "" {
 		ret = append(ret, t.TriggeredBy)
 	}
 	return append(ret, t.ReusedBy...)
 }

 func (t *Tryjob) IsEnded() bool {
 	switch t.Status {
 	case Status_CANCELLED, Status_ENDED, Status_UNTRIGGERED:
 		return true
 	case Status_PENDING, Status_TRIGGERED:
 		return false
 	default:
 		panic(fmt.Errorf("unexpected tryjob status %s", t.Status.String()))
 	}
 }

 // CLPatchsets is a slice of `CLPatchset`s.
 //
 // Implements sort.Interface
 type CLPatchsets []CLPatchset

 // Len implements sort.Interface.
 func (c CLPatchsets) Len() int {
 	return len(c)
 }

 // Less implements sort.Interface.
 func (c CLPatchsets) Less(i int, j int) bool {
 	return c[i] < c[j]
 }

 // Swap implements sort.Interface.
 func (c CLPatchsets) Swap(i int, j int) {
 	c[i], c[j] = c[j], c[i]
 }

 // CLPatchset is a value computed combining a CL's ID and a patchset number.
 //
 // This is intended to efficiently query Tryjob entities associated with a
 // patchset.
 //
 // The values are hex string encoded and padded so that lexicographical sorting
 // will put the patchsets for a given CL together.
 type CLPatchset string

 const clPatchsetEncodingVersion = 1

 // MakeCLPatchset computes a new CLPatchset value.
 func MakeCLPatchset(cl common.CLID, patchset int32) CLPatchset {
 	return CLPatchset(fmt.Sprintf("%02x/%016x/%08x", clPatchsetEncodingVersion, cl, patchset))
 }

 // Parse extracts CLID and Patchset number from a valid CLPatchset value.
 //
 // Returns an error if the format is unexpected.
 func (cp CLPatchset) Parse() (common.CLID, int32, error) {
 	var clid, patchset int64
 	values := strings.Split(string(cp), "/")
 	// If any valid encoding versions require a different number of values,
 	// check it here.
 	switch len(values) {
 	case 3:
 		// Version 1 requires three slash-separated values.
 	default:
 		return 0, 0, errors.Reason("CLPatchset in unexpected format %q", cp).Err()
 	}

 	ver, err := strconv.ParseInt(values[0], 16, 32)
 	switch {
 	case err != nil:
 		return 0, 0, errors.Annotate(err, "version segment in unexpected format %q", values[0]).Err()
 	case ver == clPatchsetEncodingVersion:
 		if len(values) != 3 {
 			panic(fmt.Errorf("impossible: number of values is not 3"))
 		}
 		clid, err = strconv.ParseInt(values[1], 16, 64)
 		if err != nil {
 			return 0, 0, errors.Annotate(err, "clid segment in unexpected format %q", values[1]).Err()
 		}
 		patchset, err = strconv.ParseInt(values[2], 16, 32)
 		if err != nil {
 			return 0, 0, errors.Annotate(err, "patchset segment in unexpected format %q", values[2]).Err()
 		}
 		return common.CLID(clid), int32(patchset), nil
 	default:
 		return 0, 0, errors.Reason("unsupported version %d", ver).Err()
 	}
 }
	// Copyright 2021 The LUCI Authors.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package tryjob

	import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
	)

	const (
	// TryjobKind is the Datastore entity kind for Tryjob.
	TryjobKind = "Tryjob"
	)

	// Tryjob is an entity tracking CV Tryjobs.
	type Tryjob struct {
	// $kind must match TryjobKind.
	_kind string `gae:"$kind,Tryjob"`
	_extra datastore.PropertyMap `gae:"-,extra"`

	// ID is the Tryjob ID, autogenerated by the Datastore.
	ID common.TryjobID `gae:"$id"`
	// ExternalID is a Tryjob ID in external system, e.g. Buildbucket.
	//
	// There can be at most one Tryjob with a given ExternalID.
	//
	// Indexed.
	ExternalID ExternalID
	// EVersion is the entity version.
	//
	// It increments by one upon every successful modification.
	EVersion int64 `gae:",noindex"`
	// EntityCreateTime is the timestamp when this entity was created.
	//
	// NOTE: this is not the backend's tryjob creation time, which is stored in
	// .Result.CreateTime.
	EntityCreateTime time.Time `gae:",noindex"`
	// UpdateTime is the timestamp when this entity was last updated.
	//
	// NOTE: this is not the backend's tryjob update time, which is stored in
	// .Result.UpdateTime.
	EntityUpdateTime time.Time `gae:",noindex"`

	// ReuseKey is used to quickly decide if this Tryjob can be reused by a run.
	//
	// Note that, even if reuse is allowed here, reuse is still subjected to
	// other restrictions (for example, Tryjob is not fresh enough for the run).
	//
	// reusekey is currently computed in the following way:
	// base64(
	// sha256(
	// '\0'.join(sorted('%d/%d' % (cl.ID, cl.minEquiPatchSet) for cl in cls))
	// )
	// )
	//
	// Indexed
	ReuseKey string

	// Definition of the tryjob.
	//
	// Immutable.
	Definition *Definition

	// Status of the Tryjob.
	//
	// Indexed.
	Status Status

	// TriggeredByCV is true if it was triggered by CV.
	//
	// This is used for cancelation, since CV shouldn't cancel tryjobs it
	// didn't trigger.
	TriggeredByCV bool `gae:",noindex"`

	// Result of the Tryjob.
	//
	// Must be set if Status is ENDED.
	// May be set if Status is TRIGGERED.
	//
	// It's used by the Run Manager.
	Result *Result

	// TriggeredBy is the Run that triggered this Tryjob.
	//
	// May be unset if the Tryjob was not triggered by CV, in which case
	// ReusedBy has at least one Run.
	//
	// Indexed.
	TriggeredBy common.RunID

	// ReusedBy are the Runs that are interested in the result of this Tryjob.
	//
	// Indexed.
	ReusedBy common.RunIDs

	// CLPatchsets is an array of CLPatchset that each identify a specific
	// patchset in a specific CL.
	//
	// The values are to be computed by MakeCLPatchset().
	// See its documentation for details.
	//
	// Sorted and Indexed.
	CLPatchsets CLPatchsets
	}

	// tryjobMap is intended to quickly determine if a given ExternalID is
	// associated with a Tryjob entity in the datastore.
	//
	// This also ensures that at most one TryjobID will be associated with a given
	// ExternalID.
	type tryjobMap struct {
	_kind string `gae:"$kind,TryjobMap"`

	// ExternalID is an ID for the tryjob in the external backend.
	//
	// Making this the key of the map ensures uniqueness.
	ExternalID ExternalID `gae:"$id"`

	// InternalID is auto-generated by Datastore for Tryjob entity.
	InternalID common.TryjobID `gae:",noindex"` // int64. Indexed in Tryjob entities.
	}

	// LUCIProject() returns the project in the context of which the Tryjob is
	// updated, and which is thus allowed to "read" the Tryjob.
	//
	// In the case of Buildbucket, this may be different from the LUCI project to
	// which the corresponding build belongs. For example, consider a "v8" project
	// with configuration saying to trigger "chromium/try/linux_rel" builder: when
	// CV triggers a new tryjob T for a "v8" Run, T.LUCIProject() will be "v8" even
	// though the build itself will be in the "chromium/try" Buildbucket bucket.
	//
	// In general, a Run of project P must not re-use tryjob T if
	// T.LUCIProject() != P, until it has been verified with the tryjob backend
	// that P has access to T.
	func (t *Tryjob) LUCIProject() string {
	if t.TriggeredBy != "" {
	return t.TriggeredBy.LUCIProject()
	}
	if len(t.ReusedBy) == 0 {
	panic("tryjob is not associated with any runs")
	}
	return t.ReusedBy[0].LUCIProject()
	}

	// AllWatchingRuns returns the IDs for the Runs that care about this tryjob.
	//
	// This includes the triggerer (if the tryjob was triggered by CV) and all the
	// Runs reusing this tryjob (if any).
	func (t *Tryjob) AllWatchingRuns() common.RunIDs {
	ret := make(common.RunIDs, 0, 1+len(t.ReusedBy))
	if t.TriggeredBy != "" {
	ret = append(ret, t.TriggeredBy)
	}
	return append(ret, t.ReusedBy...)
	}

	func (t *Tryjob) IsEnded() bool {
	switch t.Status {
	case Status_CANCELLED, Status_ENDED, Status_UNTRIGGERED:
	return true
	case Status_PENDING, Status_TRIGGERED:
	return false
	default:
	panic(fmt.Errorf("unexpected tryjob status %s", t.Status.String()))
	}
	}

	// CLPatchsets is a slice of `CLPatchset`s.
	//
	// Implements sort.Interface
	type CLPatchsets []CLPatchset

	// Len implements sort.Interface.
	func (c CLPatchsets) Len() int {
	return len(c)
	}

	// Less implements sort.Interface.
	func (c CLPatchsets) Less(i int, j int) bool {
	return c[i] < c[j]
	}

	// Swap implements sort.Interface.
	func (c CLPatchsets) Swap(i int, j int) {
	c[i], c[j] = c[j], c[i]
	}

	// CLPatchset is a value computed combining a CL's ID and a patchset number.
	//
	// This is intended to efficiently query Tryjob entities associated with a
	// patchset.
	//
	// The values are hex string encoded and padded so that lexicographical sorting
	// will put the patchsets for a given CL together.
	type CLPatchset string

	const clPatchsetEncodingVersion = 1

	// MakeCLPatchset computes a new CLPatchset value.
	func MakeCLPatchset(cl common.CLID, patchset int32) CLPatchset {
	return CLPatchset(fmt.Sprintf("%02x/%016x/%08x", clPatchsetEncodingVersion, cl, patchset))
	}

	// Parse extracts CLID and Patchset number from a valid CLPatchset value.
	//
	// Returns an error if the format is unexpected.
	func (cp CLPatchset) Parse() (common.CLID, int32, error) {
	var clid, patchset int64
	values := strings.Split(string(cp), "/")
	// If any valid encoding versions require a different number of values,
	// check it here.
	switch len(values) {
	case 3:
	// Version 1 requires three slash-separated values.
	default:
	return 0, 0, errors.Reason("CLPatchset in unexpected format %q", cp).Err()
	}

	ver, err := strconv.ParseInt(values[0], 16, 32)
	switch {
	case err != nil:
	return 0, 0, errors.Annotate(err, "version segment in unexpected format %q", values[0]).Err()
	case ver == clPatchsetEncodingVersion:
	if len(values) != 3 {
	panic(fmt.Errorf("impossible: number of values is not 3"))
	}
	clid, err = strconv.ParseInt(values[1], 16, 64)
	if err != nil {
	return 0, 0, errors.Annotate(err, "clid segment in unexpected format %q", values[1]).Err()
	}
	patchset, err = strconv.ParseInt(values[2], 16, 32)
	if err != nil {
	return 0, 0, errors.Annotate(err, "patchset segment in unexpected format %q", values[2]).Err()
	}
	return common.CLID(clid), int32(patchset), nil
	default:
	return 0, 0, errors.Reason("unsupported version %d", ver).Err()
	}
	}