buildbucket/appengine/model/build.go

// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package model

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"google.golang.org/protobuf/proto"

	"go.chromium.org/luci/auth/identity"
	"go.chromium.org/luci/common/data/strpair"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/gae/service/datastore"

	bb "go.chromium.org/luci/buildbucket"
	pb "go.chromium.org/luci/buildbucket/proto"
	"go.chromium.org/luci/buildbucket/protoutil"
)

const (
	// BuildKind is a Build entity's kind in the datastore.
	BuildKind = "Build"
)

// isHiddenTag returns whether the given tag should be hidden by ToProto.
func isHiddenTag(key string) bool {
	// build_address is reserved by the server so that the TagIndex infrastructure
	// can be reused to fetch builds by builder + number (see tagindex.go and
	// rpc/get_build.go).
	// TODO(crbug/1042991): Unhide builder and gitiles_ref.
	// builder and gitiles_ref are allowed to be specified, are not internal,
	// and are only hidden here to match Python behavior.
	return key == "build_address" || key == "builder" || key == "gitiles_ref"
}

// PubSubCallback encapsulates parameters for a Pub/Sub callback.
type PubSubCallback struct {
	AuthToken string `gae:"auth_token,noindex"`
	Topic     string `gae:"topic,noindex"`
	UserData  []byte `gae:"user_data,noindex"`
}

// Build is a representation of a build in the datastore.
// Implements datastore.PropertyLoadSaver.
type Build struct {
	_     datastore.PropertyMap `gae:"-,extra"`
	_kind string                `gae:"$kind,Build"`
	ID    int64                 `gae:"$id"`

	// LegacyProperties are properties set for v1 legacy builds.
	LegacyProperties
	// UnusedProperties are properties set previously but currently unused.
	UnusedProperties

	// Proto is the pb.Build proto representation of the build.
	//
	// infra, input.properties, output.properties, and steps
	// are zeroed and stored in separate datastore entities
	// due to their potentially large size (see details.go).
	// tags are given their own field so they can be indexed.
	//
	// noindex is not respected here, it's set in pb.Build.ToProperty.
	Proto *pb.Build `gae:"proto,legacy"`

	Project string `gae:"project"`
	// <project>/<bucket>. Bucket is in v2 format.
	// e.g. chromium/try (never chromium/luci.chromium.try).
	BucketID string `gae:"bucket_id"`
	// <project>/<bucket>/<builder>. Bucket is in v2 format.
	// e.g. chromium/try/linux-rel.
	BuilderID string `gae:"builder_id"`

	Canary bool `gae:"canary"`

	CreatedBy identity.Identity `gae:"created_by"`
	// TODO(nodir): Replace reliance on create_time indices with id.
	CreateTime time.Time `gae:"create_time"`
	// Experimental, if true, means to exclude from monitoring and search results
	// (unless specifically requested in search results).
	Experimental bool `gae:"experimental"`
	// Experiments is a slice of experiments enabled or disabled on this build.
	// Each element should look like "[-+]$experiment_name".
	//
	// Special case:
	//   "-luci.non_production" is not kept here as a storage/index
	//   optimization.
	//
	//   Notably, all search/query implementations on the Build model
	//   apply this filter in post by checking that
	//   `b.ExperimentStatus("luci.non_production") == pb.Trinary_YES`.
	//
	//   This is because directly including this value in the datastore query
	//   results in bad performance due to excessive zig-zag join overhead
	//   in the datastore, since 99%+ of the builds in Buildbucket are production
	//   builds.
	Experiments []string `gae:"experiments"`
	Incomplete  bool     `gae:"incomplete"`

	// Deprecated; remove after v1 api turndown
	IsLuci bool `gae:"is_luci"`

	ResultDBUpdateToken string    `gae:"resultdb_update_token,noindex"`
	Status              pb.Status `gae:"status_v2"`
	StatusChangedTime   time.Time `gae:"status_changed_time"`
	// Tags is a slice of "<key>:<value>" strings taken from Proto.Tags.
	// Stored separately in order to index.
	Tags []string `gae:"tags"`

	// UpdateToken is set at the build creation time, and UpdateBuild requests are required
	// to have it in the header.
	UpdateToken string `gae:"update_token,noindex"`

	// PubSubCallback, if set, creates notifications for build status changes.
	PubSubCallback PubSubCallback `gae:"pubsub_callback,noindex"`
}

// Realm returns this build's auth realm, or an empty string if not opted into the
// realms experiment.
func (b *Build) Realm() string {
	return fmt.Sprintf("%s:%s", b.Proto.Builder.Project, b.Proto.Builder.Bucket)
}

// ExperimentStatus scans the experiments attached to this Build and returns:
//   * YES - The experiment was known at schedule time and enabled.
//   * NO - The experiment was known at schedule time and disabled.
//   * UNSET - The experiment was unknown at schedule time.
//
// Malformed Experiment filters are treated as UNSET.
func (b *Build) ExperimentStatus(expname string) (ret pb.Trinary) {
	b.IterExperiments(func(enabled bool, exp string) bool {
		if exp == expname {
			if enabled {
				ret = pb.Trinary_YES
			} else {
				ret = pb.Trinary_NO
			}
			return false
		}
		return true
	})
	return
}

// IterExperiments parses all experiments and calls `cb` for each.
//
// This will always include a call with bb.ExperimentNonProduction, even
// if '-'+bb.ExperimentNonProduction isn't recorded in the underlying
// Experiments field.
func (b *Build) IterExperiments(cb func(enabled bool, exp string) bool) {
	var hadNonProd bool

	for _, expFilter := range b.Experiments {
		if len(expFilter) == 0 {
			continue
		}
		plusMinus, exp := expFilter[0], expFilter[1:]
		hadNonProd = hadNonProd || exp == bb.ExperimentNonProduction

		keepGoing := true
		if plusMinus == '+' {
			keepGoing = cb(true, exp)
		} else if plusMinus == '-' {
			keepGoing = cb(false, exp)
		}
		if !keepGoing {
			return
		}
	}
	if !hadNonProd {
		cb(false, bb.ExperimentNonProduction)
	}
}

// ExperimentsString sorts, joins, and returns the enabled experiments with "|".
//
// Returns "None" if no experiments were enabled in the build.
func (b *Build) ExperimentsString() string {
	if len(b.Experiments) == 0 {
		return "None"
	}

	enables := make([]string, 0, len(b.Experiments))
	b.IterExperiments(func(isEnabled bool, name string) bool {
		if isEnabled {
			enables = append(enables, name)
		}
		return true
	})
	if len(enables) > 0 {
		sort.Strings(enables)
		return strings.Join(enables, "|")
	}
	return "None"
}

// Load overwrites this representation of a build by reading the given
// datastore.PropertyMap. Mutates this entity.
func (b *Build) Load(p datastore.PropertyMap) error {
	return datastore.GetPLS(b).Load(p)
}

// Save returns the datastore.PropertyMap representation of this build. Mutates
// this entity to reflect computed datastore fields in the returned PropertyMap.
func (b *Build) Save(withMeta bool) (datastore.PropertyMap, error) {
	b.BucketID = protoutil.FormatBucketID(b.Proto.Builder.Project, b.Proto.Builder.Bucket)
	b.BuilderID = protoutil.FormatBuilderID(b.Proto.Builder)
	b.Canary = b.Proto.Canary
	b.Experimental = b.Proto.Input.GetExperimental()
	b.Incomplete = !protoutil.IsEnded(b.Proto.Status)
	b.Project = b.Proto.Builder.Project

	oldStatus := b.Status
	b.Status = b.Proto.Status
	if b.Status != oldStatus {
		b.StatusChangedTime = b.Proto.UpdateTime.AsTime()
	}
	b.CreateTime = b.Proto.CreateTime.AsTime()

	// Set legacy values used by Python.
	switch b.Status {
	case pb.Status_SCHEDULED:
		b.LegacyProperties.Result = 0
		b.LegacyProperties.Status = Scheduled
	case pb.Status_STARTED:
		b.LegacyProperties.Result = 0
		b.LegacyProperties.Status = Started
	case pb.Status_SUCCESS:
		b.LegacyProperties.Result = Success
		b.LegacyProperties.Status = Completed
	case pb.Status_FAILURE:
		b.LegacyProperties.FailureReason = BuildFailure
		b.LegacyProperties.Result = Failure
		b.LegacyProperties.Status = Completed
	case pb.Status_INFRA_FAILURE:
		if b.Proto.StatusDetails.GetTimeout() != nil {
			b.LegacyProperties.CancelationReason = TimeoutCanceled
			b.LegacyProperties.Result = Canceled
		} else {
			b.LegacyProperties.FailureReason = InfraFailure
			b.LegacyProperties.Result = Failure
		}
		b.LegacyProperties.Status = Completed
	case pb.Status_CANCELED:
		b.LegacyProperties.CancelationReason = ExplicitlyCanceled
		b.LegacyProperties.Result = Canceled
		b.LegacyProperties.Status = Completed
	}

	p, err := datastore.GetPLS(b).Save(withMeta)
	if err != nil {
		return nil, err
	}

	// Parameters and ResultDetails are only set via v1 API which is unsupported in
	// Go. In order to preserve the value of these fields without having to interpret
	// them, the type is set to []byte. But if no values for these fields are set,
	// the []byte type causes an empty-type specific value (i.e. empty string) to be
	// written to the datastore. Since Python interprets these fields as JSON, and
	// and an empty string is not a valid JSON object, convert empty strings to nil.
	// TODO(crbug/1042991): Remove Properties default once v1 API is removed.
	if len(b.LegacyProperties.Parameters) == 0 {
		p["parameters"] = datastore.MkProperty(nil)
	}
	// TODO(crbug/1042991): Remove ResultDetails default once v1 API is removed.
	if len(b.LegacyProperties.ResultDetails) == 0 {
		p["result_details"] = datastore.MkProperty(nil)
	}

	// Writing a value for PubSubCallback confuses the Python implementation which
	// expects PubSubCallback to be a LocalStructuredProperty. See also unused.go.
	delete(p, "pubsub_callback")
	return p, nil
}

// ToProto returns the *pb.Build representation of this build.
func (b *Build) ToProto(ctx context.Context, m *BuildMask) (*pb.Build, error) {
	build := b.ToSimpleBuildProto(ctx)
	if err := LoadBuildDetails(ctx, m, build); err != nil {
		return nil, err
	}
	return build, nil
}

// ToSimpleBuildProto returns the *pb.Build without loading steps, infra,
// input/output properties.
func (b *Build) ToSimpleBuildProto(ctx context.Context) *pb.Build {
	p := proto.Clone(b.Proto).(*pb.Build)
	p.Tags = make([]*pb.StringPair, 0, len(b.Tags))
	for _, t := range b.Tags {
		k, v := strpair.Parse(t)
		if !isHiddenTag(k) {
			p.Tags = append(p.Tags, &pb.StringPair{
				Key:   k,
				Value: v,
			})
		}
	}
	return p
}

// LoadBuildDetails loads the details of the given builds, trimming them
// according to the mask.
func LoadBuildDetails(ctx context.Context, m *BuildMask, builds ...*pb.Build) error {
	l := len(builds)
	inf := make([]*BuildInfra, 0, l)
	inp := make([]*BuildInputProperties, 0, l)
	out := make([]*BuildOutputProperties, 0, l)
	stp := make([]*BuildSteps, 0, l)
	var dets []interface{}

	included := map[string]bool{
		"infra":             m.Includes("infra"),
		"input.properties":  m.Includes("input.properties"),
		"output.properties": m.Includes("output.properties"),
		"steps":             m.Includes("steps"),
	}

	for i, p := range builds {
		if p.GetId() <= 0 {
			return errors.Reason("invalid build for %q", p).Err()
		}
		key := datastore.KeyForObj(ctx, &Build{ID: p.Id})
		inf = append(inf, &BuildInfra{Build: key})
		inp = append(inp, &BuildInputProperties{Build: key})
		out = append(out, &BuildOutputProperties{Build: key})
		stp = append(stp, &BuildSteps{Build: key})
		appendIfIncluded := func(path string, det interface{}) {
			if included[path] {
				dets = append(dets, det)
			}
		}
		appendIfIncluded("infra", inf[i])
		appendIfIncluded("input.properties", inp[i])
		appendIfIncluded("output.properties", out[i])
		appendIfIncluded("steps", stp[i])
	}

	if err := GetIgnoreMissing(ctx, dets); err != nil {
		return errors.Annotate(err, "error fetching build details").Err()
	}

	var err error
	for i, p := range builds {
		p.Infra = inf[i].Proto
		if p.Input == nil {
			p.Input = &pb.Build_Input{}
		}
		p.Input.Properties = &inp[i].Proto.Struct
		if p.Output == nil {
			p.Output = &pb.Build_Output{}
		}
		p.Output.Properties = &out[i].Proto.Struct
		p.Steps, err = stp[i].ToProto(ctx)
		if err != nil {
			return errors.Annotate(err, "error fetching steps for build %q", p).Err()
		}
		if err = m.Trim(p); err != nil {
			return errors.Annotate(err, "error trimming fields for %q", p).Err()
		}
	}
	return nil
}