blob: 3ac66111cf1ad2aefe6d71e123df824f5403fd8c [file] [log] [blame]
// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package changelist
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"go.chromium.org/luci/common/clock"
"go.chromium.org/luci/common/errors"
"go.chromium.org/luci/common/logging"
"go.chromium.org/luci/common/retry/transient"
"go.chromium.org/luci/gae/service/datastore"
)
// CLID is a unique ID of a CL used internally in CV.
//
// It's just 8 bytes long and is thus much shorter than ExternalID,
// which reduces CPU & RAM & storage costs of CL graphs for multi-CL Runs.
type CLID int64
// ExternalID is a unique CL ID deterministically constructed based on CL data.
//
// Currently, only Gerrit is supported.
type ExternalID string
// GobID makes an ExternalID for a Gerrit CL.
//
// Host is typically "something-review.googlesource.com".
// Change is a number, e.g. 2515619 for
// https://chromium-review.googlesource.com/c/infra/luci/luci-go/+/2515619
func GobID(host string, change int64) (ExternalID, error) {
if strings.ContainsRune(host, '/') {
return "", errors.Reason("invalid host %q: must not contain /", host).Err()
}
return ExternalID(fmt.Sprintf("gerrit/%s/%d", host, change)), nil
}
// ParseGobID returns Gerrit host and change if this is a GobID.
func (e ExternalID) ParseGobID() (host string, change int64, err error) {
parts := strings.Split(string(e), "/")
if len(parts) != 3 || parts[0] != "gerrit" {
err = errors.Reason("%q is not a valid GobID", e).Err()
return
}
host = parts[1]
change, err = strconv.ParseInt(parts[2], 10, 63)
if err != nil {
err = errors.Annotate(err, "%q is not a valid GobID", e).Err()
}
return
}
// CL is a CL entity in Datastore.
type CL struct {
_kind string `gae:"$kind,CL"`
_extra datastore.PropertyMap `gae:"-,extra"`
// ID is auto-generated by Datastore.
ID CLID `gae:"$id"` // int64
// ExternalID must not be modified once entity is created.
ExternalID ExternalID `gae:",noindex"` // string. Indexed in CLMap entities.
// EVersion is entity version. Every update should increment it by 1.
// See Update() function.
EVersion int `gae:",noindex"`
// Snapshot is latest known state of a CL.
// It may and often is behind the source of truth -- the code reveview site
// (e.g. Gerrit).
Snapshot *Snapshot
// ApplicableConfig keeps track of configs applicable to the CL.
ApplicableConfig *ApplicableConfig
// UpdateTime is exact time of when this entity was last updated.
//
// It's not indexed to avoid hot areas in the index.
UpdateTime time.Time `gae:",noindex"`
// TODO(tandrii): implement deletion of the oldest entities via additional
// indexed field based on UpdateTime but with entropy in the lowest bits to
// avoid hotspots.
}
// clMap is CLMap entity in Datastore which ensures strict 1:1 mapping
// between internal and external IDs.
type clMap struct {
_kind string `gae:"$kind,CLMap"`
// ExternalID as entity ID ensures uniqueness.
ExternalID ExternalID `gae:"$id"` // string
// InternalID is auto-generated by Datastore for CL entity.
InternalID CLID `gae:",noindex"` // int64. Indexed in CL entities.
}
// Get reads a CL from datastore.
//
// Returns datastore.ErrNoSuchEntity if it doesn't exist.
func (eid ExternalID) Get(ctx context.Context) (*CL, error) {
m := clMap{ExternalID: eid}
switch err := datastore.Get(ctx, &m); {
case err == datastore.ErrNoSuchEntity:
return nil, err
case err != nil:
return nil, errors.Annotate(err, "failed to get CLMap").Tag(transient.Tag).Err()
}
return getExisting(ctx, m.InternalID, eid)
}
// GetOrInsert reads a CL from datastore, creating a new one if not exists yet.
//
// populate is called within a transaction to populate fields of a new entity.
// It should be a fast function.
//
// Warning:
// * populate may be called several times since transaction can be retried.
// * cl.ExternalID and cl.ID must not be changed by populate.
func (eid ExternalID) GetOrInsert(ctx context.Context, populate func(cl *CL)) (*CL, error) {
// Fast path without transaction.
if cl, err := eid.Get(ctx); err != datastore.ErrNoSuchEntity {
return cl, err
}
var cl *CL
m := clMap{ExternalID: eid}
err := datastore.RunInTransaction(ctx, func(ctx context.Context) (err error) {
cl = nil
switch err = datastore.Get(ctx, &m); {
case err == nil:
// Has just been created by someone else.
return nil
case err != datastore.ErrNoSuchEntity:
return err
}
cl, err = insert(ctx, eid, populate)
return
}, nil)
switch {
case err != nil:
return nil, errors.Annotate(err, "failed to getOrInsert a CL").Tag(transient.Tag).Err()
case cl == nil:
return getExisting(ctx, m.InternalID, eid)
}
return cl, nil
}
// Delete deletes CL and its CLMap entities trasactionally.
//
// Thus, Delete and insertion (part of ExternalID.getOrInsert)
// are atomic with respect to one another.
//
// However, ExternalID.get and fast path of ExternalID.getOrInsert if called
// concurrently with Delete may return temporary error, but on retry they would
// return ErrNoSuchEntity.
func Delete(ctx context.Context, id CLID) error {
cl := CL{ID: id}
switch err := datastore.Get(ctx, &cl); {
case err == datastore.ErrNoSuchEntity:
return nil // Nothing to do.
case err != nil:
return errors.Annotate(err, "failed to get a CL").Tag(transient.Tag).Err()
}
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
m := clMap{ExternalID: cl.ExternalID}
return datastore.Delete(ctx, &cl, &m)
}, nil)
if err != nil {
return errors.Annotate(err, "failed to delete a CL").Tag(transient.Tag).Err()
}
return nil
}
// Update updates CL entity with Snapshot and ApplicableConfig.
//
// Either ExternalID or a known CLID must be provided.
// Either new Snapshot or ApplicableConfig must be provided.
//
// If CLID is not known and CL for provided ExternalID doesn't exist,
// then a new CL is created with the given Snapshot & ApplicableConfig.
//
// Otherwise, an existing CL entity will be updated iff either:
// * if snapshot is given and it is more recent than what is already stored,
// as measured by ExternalUpdateTime.
// * same but for ApplicableConfig and ApplicableConfig.UpdateTime,
// respectively.
//
// TODO(tandrii): emit notification events.
func Update(ctx context.Context, eid ExternalID, knownCLID CLID, snapshot *Snapshot, acfg *ApplicableConfig) error {
if eid == "" && knownCLID == 0 {
panic("either ExternalID or known CLID must be provided")
}
if snapshot == nil && acfg == nil {
panic("either new snapshot or new ApplicableConfig must be provided")
}
err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
if knownCLID == 0 {
m := clMap{ExternalID: eid}
switch err := datastore.Get(ctx, &m); {
case err == datastore.ErrNoSuchEntity:
// Insert new entity.
_, err = insert(ctx, eid, func(cl *CL) {
cl.Snapshot = snapshot
cl.ApplicableConfig = acfg
})
return err
case err != nil:
return errors.Annotate(err, "failed to get CLMap entity").Tag(transient.Tag).Err()
}
knownCLID = m.InternalID
}
cl, err := getExisting(ctx, knownCLID, eid)
if err != nil {
return err
}
// Update exsting entity.
return update(ctx, cl, func(cl *CL) (changed bool) {
if snapshot != nil && !cl.Snapshot.IsUpToDate(snapshot.GetLuciProject(), snapshot.GetExternalUpdateTime().AsTime()) {
cl.Snapshot = snapshot
changed = true
}
if acfg != nil && !cl.ApplicableConfig.IsUpToDate(acfg.GetUpdateTime().AsTime()) {
cl.ApplicableConfig = acfg
changed = true
}
return
})
}, nil)
return errors.Annotate(err, "failed to update CL").Tag(transient.Tag).Err()
}
func getExisting(ctx context.Context, clid CLID, eid ExternalID) (*CL, error) {
cl := &CL{ID: clid}
switch err := datastore.Get(ctx, cl); {
case err == datastore.ErrNoSuchEntity:
// This should not happen in practice except in the case of a very old CL
// which is being deleted due to retention policy. Log error but return it
// as transient as it's expected that CLMap entity would be removed soon,
// and so a retry would be produce proper datastore.ErrNoSuchEntity error.
msg := fmt.Sprintf("unexpectedly failed to get CL#%d given existing CLMap%q", clid, eid)
logging.Errorf(ctx, msg)
return nil, errors.Reason(msg).Tag(transient.Tag).Err()
case err != nil:
return nil, errors.Annotate(err, "failed to get CL").Tag(transient.Tag).Err()
}
return cl, nil
}
// insert creates new CL entity for given external ID.
//
// Must be called after verifying that such CLMap record doesn't exist.
func insert(ctx context.Context, eid ExternalID, populate func(*CL)) (*CL, error) {
if datastore.CurrentTransaction(ctx) == nil {
panic("must be called in transaction context")
}
// Create new CL and CLMap entry atomically.
cl := &CL{
ID: 0, // autogenerate by Datastore
ExternalID: eid,
EVersion: 1,
}
populate(cl)
if cl.ID != 0 || cl.ExternalID != eid || cl.EVersion != 1 {
panic(errors.New("populate changed ID or ExternalID or EVersion, but must not do this."))
}
cl.UpdateTime = datastore.RoundTime(clock.Now(ctx).UTC())
if err := datastore.Put(ctx, cl); err != nil {
return nil, errors.Annotate(err, "failed to save CL entity").Tag(transient.Tag).Err()
}
if err := datastore.Put(ctx, &clMap{ExternalID: eid, InternalID: cl.ID}); err != nil {
return nil, errors.Annotate(err, "failed to save CLMap entity").Tag(transient.Tag).Err()
}
return cl, nil
}
func update(ctx context.Context, justRead *CL, mut func(*CL) (update bool)) error {
if datastore.CurrentTransaction(ctx) == nil {
panic("must be called in transaction context")
}
before := *justRead // shallow copy, avoiding cloning Snapshot.
if !mut(justRead) {
return nil
}
justRead.EVersion = before.EVersion + 1
justRead.UpdateTime = clock.Now(ctx).UTC()
if err := datastore.Put(ctx, justRead); err != nil {
return errors.Annotate(err, "failed to put CL entity").Tag(transient.Tag).Err()
}
return nil
}