cv/internal/changelist/model.go

// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package changelist

import (
	"context"
	"fmt"
	"sort"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
)

// CL is a CL entity in Datastore.
type CL struct {
	_kind  string                `gae:"$kind,CL"`
	_extra datastore.PropertyMap `gae:"-,extra"`

	// ID is auto-generated by Datastore.
	ID common.CLID `gae:"$id"` // int64
	// ExternalID must not be modified once entity is created.
	ExternalID ExternalID `gae:",noindex"` // string. Indexed in CLMap entities.

	// EVersion is entity version. Every update should increment it by 1.
	// See Update() function.
	EVersion int64 `gae:",noindex"`

	// UpdateTime is exact time of when this entity was last updated.
	//
	// It's not indexed to avoid hot areas in the index.
	UpdateTime time.Time `gae:",noindex"`

	// TODO(tandrii): implement deletion of the oldest entities via additional
	// indexed field based on UpdateTime but with entropy in the lowest bits to
	// avoid hotspots.

	// Snapshot is the latest known state of a CL. It may be and often is
	// behind the source of truth, which is the code review site (e.g. Gerrit).
	Snapshot *Snapshot

	// ApplicableConfig keeps track of configs applicable to the CL.
	//
	// TODO(tandrii): merge into .Access.
	ApplicableConfig *ApplicableConfig

	// Access records per-LUCI project visibility of a CL.
	//
	// See description in protobuf type with the same name.
	//
	// TODO(tandrii): rename GAE field to `Access`.
	Access *Access `gae:"DependentMeta"`

	// IncompleteRuns tracks not yet finalized Runs working on this CL. Sorted.
	//
	// It's updated transactionally with the Run being modified.
	IncompleteRuns common.RunIDs `gae:",noindex"`

	// TriggerNewPatchsetRunAfterPS indicates the patchset number after which
	// new patchset runs can be triggered.
	//
	// E.g. if this value is set to 2, do not trigger new patchset runs for
	// patchsets 1 or 2. Presumably this is because those runs were already
	// completed/failed/otherwise purged.
	//
	// This is needed because unlike label votes which CV can remove, triggering
	// new patchset upload runs relies on the presence of the patchset in the CL
	// snapshot, which cannot be removed.
	TriggerNewPatchsetRunAfterPS int32 `gae:",noindex"`
}

// clMap is CLMap entity in Datastore which ensures strict 1:1 mapping
// between internal and external IDs.
type clMap struct {
	_kind string `gae:"$kind,CLMap"`

	// ExternalID as entity ID ensures uniqueness.
	ExternalID ExternalID `gae:"$id"` // string
	// InternalID is auto-generated by Datastore for CL entity.
	InternalID common.CLID `gae:",noindex"` // int64. Indexed in CL entities.
}

// URL returns URL of the CL.
func (cl *CL) URL() (string, error) { return cl.ExternalID.URL() }

// ToUpdatedEvent returns CLUpdatedEvent corresponding to the current CL
// version.
func (cl *CL) ToUpdatedEvent() *CLUpdatedEvent {
	return &CLUpdatedEvent{
		Clid:     int64(cl.ID),
		Eversion: cl.EVersion,
	}
}

// ToUpdatedEvents returns CLUpdatedEvents from a slice of CLs.
func ToUpdatedEvents(cls ...*CL) *CLUpdatedEvents {
	events := make([]*CLUpdatedEvent, len(cls))
	for i, cl := range cls {
		if cl.ID == 0 || cl.EVersion == 0 {
			panic(fmt.Errorf("ID %d and EVersion %d must not be 0", cl.ID, cl.EVersion))
		}
		events[i] = &CLUpdatedEvent{
			Clid:     int64(cl.ID),
			Eversion: cl.EVersion,
		}
	}
	sort.Slice(events, func(i, j int) bool {
		// Assume unique CLIDs.
		return events[i].GetClid() < events[j].GetClid()
	})
	return &CLUpdatedEvents{Events: events}
}

// Load reads a CL from Datastore.
//
// Returns nil, nil if it doesn't exist.
func (eid ExternalID) Load(ctx context.Context) (*CL, error) {
	m := clMap{ExternalID: eid}
	switch err := datastore.Get(ctx, &m); {
	case err == datastore.ErrNoSuchEntity:
		return nil, nil
	case err != nil:
		return nil, errors.Annotate(err, "failed to get CLMap").Tag(transient.Tag).Err()
	}
	cl := &CL{ID: m.InternalID}
	switch err := datastore.Get(ctx, cl); {
	case err == datastore.ErrNoSuchEntity:
		// This should not happen in practice except in the case of a very old CL
		// which is being deleted due to retention policy. Log error but return it
		// as transient as it's expected that CLMap entity would be removed soon,
		// and so a retry would be produce proper datastore.ErrNoSuchEntity error.
		msg := fmt.Sprintf("unexpectedly failed to get CL#%d given existing CLMap%q", m.InternalID, eid)
		logging.Errorf(ctx, msg)
		return nil, errors.Reason(msg).Tag(transient.Tag).Err()
	case err != nil:
		return nil, errors.Annotate(err, "failed to get CL").Tag(transient.Tag).Err()
	}
	return cl, nil
}

// MustCreateIfNotExists is for use in tests to ensure CL exists.
//
// Panicks on errors.
func (eid ExternalID) MustCreateIfNotExists(ctx context.Context) *CL {
	// Fast path without transaction.
	if cl, err := eid.Load(ctx); err == nil && cl != nil {
		return cl
	}
	var cl *CL
	err := datastore.RunInTransaction(ctx, func(ctx context.Context) (err error) {
		cl, err = eid.Load(ctx)
		switch {
		case err != nil:
			return err
		case cl != nil:
			return nil
		}
		cl = &CL{
			ExternalID: eid,
			EVersion:   1,
			UpdateTime: datastore.RoundTime(clock.Now(ctx).UTC()),
		}
		if err := datastore.AllocateIDs(ctx, cl); err != nil {
			return err
		}
		m := clMap{ExternalID: eid, InternalID: cl.ID}
		return datastore.Put(ctx, &m, cl)
	}, nil)

	if err != nil {
		panic(err)
	}
	return cl
}

// Delete deletes CL and its CLMap entities transactionally.
//
// Thus, deletion and insertion (part of ExternalID.getOrInsert) are atomic with
// respect to one another.
//
// However, ExternalID.get and fast path of ExternalID.getOrInsert if called
// concurrently with Delete may return a temporary error, but on retry they would
// return ErrNoSuchEntity.
func Delete(ctx context.Context, id common.CLID) error {
	cl := CL{ID: id}
	switch err := datastore.Get(ctx, &cl); {
	case err == datastore.ErrNoSuchEntity:
		return nil // Nothing to do.
	case err != nil:
		return errors.Annotate(err, "failed to get a CL").Tag(transient.Tag).Err()
	}

	err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
		m := clMap{ExternalID: cl.ExternalID}
		return datastore.Delete(ctx, &cl, &m)
	}, nil)
	if err != nil {
		return errors.Annotate(err, "failed to delete a CL").Tag(transient.Tag).Err()
	}
	return nil
}

// Lookup loads CLID for each given ExternalID.
//
// CLID is 0 if ExternalID is not yet known.
// Returns a single error (not MultiError) if there were multiple errors.
func Lookup(ctx context.Context, eids []ExternalID) ([]common.CLID, error) {
	out := make([]common.CLID, len(eids))
	entities := make([]clMap, len(eids))
	for i, eid := range eids {
		entities[i].ExternalID = eid
	}
	err := datastore.Get(ctx, entities)
	merrs, _ := err.(errors.MultiError)
	switch {
	case err == nil:
		for i, e := range entities {
			out[i] = e.InternalID
		}
		return out, nil
	case merrs == nil:
		return nil, errors.Annotate(err, "failed to load clMap").Tag(transient.Tag).Err()
	default:
		for i, err := range merrs {
			switch {
			case err == nil:
				out[i] = entities[i].InternalID
			case err != datastore.ErrNoSuchEntity:
				return nil, errors.Annotate(common.MostSevereError(merrs), "failed to load clMap").Tag(transient.Tag).Err()
			}
		}
		return out, nil
	}
}