cv/internal/changelist/model.go - infra/luci/luci-go - Git at Google

 // Copyright 2020 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package changelist

 import (
 	"context"
 	"fmt"
 	"sort"
 	"time"

 	"go.chromium.org/luci/common/clock"
 	"go.chromium.org/luci/common/errors"
 	"go.chromium.org/luci/common/logging"
 	"go.chromium.org/luci/common/retry/transient"
 	"go.chromium.org/luci/gae/service/datastore"

 	"go.chromium.org/luci/cv/internal/common"
 )

 // CL is a CL entity in Datastore.
 type CL struct {
 	_kind  string                `gae:"$kind,CL"`
 	_extra datastore.PropertyMap `gae:"-,extra"`

 	// ID is auto-generated by Datastore.
 	ID common.CLID `gae:"$id"` // int64
 	// ExternalID must not be modified once entity is created.
 	ExternalID ExternalID `gae:",noindex"` // string. Indexed in CLMap entities.

 	// EVersion is entity version. Every update should increment it by 1.
 	// See Update() function.
 	EVersion int64 `gae:",noindex"`

 	// UpdateTime is exact time of when this entity was last updated.
 	//
 	// It's not indexed to avoid hot areas in the index.
 	UpdateTime time.Time `gae:",noindex"`

 	// RetentionKey is for data retention purpose.
 	//
 	// It is indexed and tries to avoid hot areas in the index. The format is
 	// `{shard_key}/{unix_time_of_UpdateTime}`. Shard key is the last 2
 	// digit of CLID with left padded zero. Unix timestamp is a 10 digit integer
 	// with left padded zero if necessary.
 	RetentionKey string

 	// Snapshot is the latest known state of a CL. It may be and often is
 	// behind the source of truth, which is the code review site (e.g. Gerrit).
 	Snapshot *Snapshot

 	// ApplicableConfig keeps track of configs applicable to the CL.
 	//
 	// TODO(tandrii): merge into .Access.
 	ApplicableConfig *ApplicableConfig

 	// Access records per-LUCI project visibility of a CL.
 	//
 	// See description in protobuf type with the same name.
 	//
 	// TODO(tandrii): rename GAE field to `Access`.
 	Access *Access `gae:"DependentMeta"`

 	// IncompleteRuns tracks not yet finalized Runs working on this CL. Sorted.
 	//
 	// It's updated transactionally with the Run being modified.
 	IncompleteRuns common.RunIDs `gae:",noindex"`

 	// TriggerNewPatchsetRunAfterPS indicates the patchset number after which
 	// new patchset runs can be triggered.
 	//
 	// E.g. if this value is set to 2, do not trigger new patchset runs for
 	// patchsets 1 or 2. Presumably this is because those runs were already
 	// completed/failed/otherwise purged.
 	//
 	// This is needed because unlike label votes which CV can remove, triggering
 	// new patchset upload runs relies on the presence of the patchset in the CL
 	// snapshot, which cannot be removed.
 	TriggerNewPatchsetRunAfterPS int32 `gae:",noindex"`
 }

 // clMap is CLMap entity in Datastore which ensures strict 1:1 mapping
 // between internal and external IDs.
 type clMap struct {
 	_kind string `gae:"$kind,CLMap"`

 	// ExternalID as entity ID ensures uniqueness.
 	ExternalID ExternalID `gae:"$id"` // string
 	// InternalID is auto-generated by Datastore for CL entity.
 	InternalID common.CLID `gae:",noindex"` // int64. Indexed in CL entities.
 }

 // URL returns URL of the CL.
 func (cl *CL) URL() (string, error) { return cl.ExternalID.URL() }

 // ToUpdatedEvent returns CLUpdatedEvent corresponding to the current CL
 // version.
 func (cl *CL) ToUpdatedEvent() *CLUpdatedEvent {
 	return &CLUpdatedEvent{
 		Clid:     int64(cl.ID),
 		Eversion: cl.EVersion,
 	}
 }

 // DO NOT decrease the shard. It will cause olds CLs that are out of retention
 // period in the shard not getting wiped out.
 const retentionKeyShards = 100

 // UpdateRetentionKey updates the RetentionKey of the CL.
 //
 // Panics if the CL.ID and/or CL.UpdateTime is absent.
 func (cl *CL) UpdateRetentionKey() {
 	switch {
 	case cl.ID == 0:
 		panic(errors.New("clid is not set"))
 	case cl.UpdateTime.IsZero():
 		panic(errors.New("cl.UpdateTime is not set"))
 	}
 	cl.RetentionKey = fmt.Sprintf("%02d/%010d", cl.ID%retentionKeyShards, cl.UpdateTime.Unix())
 }

 // ToUpdatedEvents returns CLUpdatedEvents from a slice of CLs.
 func ToUpdatedEvents(cls ...*CL) *CLUpdatedEvents {
 	events := make([]*CLUpdatedEvent, len(cls))
 	for i, cl := range cls {
 		if cl.ID == 0 || cl.EVersion == 0 {
 			panic(fmt.Errorf("ID %d and EVersion %d must not be 0", cl.ID, cl.EVersion))
 		}
 		events[i] = &CLUpdatedEvent{
 			Clid:     int64(cl.ID),
 			Eversion: cl.EVersion,
 		}
 	}
 	sort.Slice(events, func(i, j int) bool {
 		// Assume unique CLIDs.
 		return events[i].GetClid() < events[j].GetClid()
 	})
 	return &CLUpdatedEvents{Events: events}
 }

 // Load reads a CL from Datastore.
 //
 // Returns nil, nil if it doesn't exist.
 func (eid ExternalID) Load(ctx context.Context) (*CL, error) {
 	m := clMap{ExternalID: eid}
 	switch err := datastore.Get(ctx, &m); {
 	case err == datastore.ErrNoSuchEntity:
 		return nil, nil
 	case err != nil:
 		return nil, errors.Annotate(err, "failed to get CLMap").Tag(transient.Tag).Err()
 	}
 	cl := &CL{ID: m.InternalID}
 	switch err := datastore.Get(ctx, cl); {
 	case err == datastore.ErrNoSuchEntity:
 		// This should not happen in practice except in the case of a very old CL
 		// which is being deleted due to retention policy. Log error but return it
 		// as transient as it's expected that CLMap entity would be removed soon,
 		// and so a retry would be produce proper datastore.ErrNoSuchEntity error.
 		msg := fmt.Sprintf("unexpectedly failed to get CL#%d given existing CLMap%q", m.InternalID, eid)
 		logging.Errorf(ctx, msg)
 		return nil, errors.Reason(msg).Tag(transient.Tag).Err()
 	case err != nil:
 		return nil, errors.Annotate(err, "failed to get CL").Tag(transient.Tag).Err()
 	}
 	return cl, nil
 }

 // MustCreateIfNotExists is for use in tests to ensure CL exists.
 //
 // Panicks on errors.
 func (eid ExternalID) MustCreateIfNotExists(ctx context.Context) *CL {
 	// Fast path without transaction.
 	if cl, err := eid.Load(ctx); err == nil && cl != nil {
 		return cl
 	}
 	var cl *CL
 	err := datastore.RunInTransaction(ctx, func(ctx context.Context) (err error) {
 		cl, err = eid.Load(ctx)
 		switch {
 		case err != nil:
 			return err
 		case cl != nil:
 			return nil
 		}
 		cl = &CL{
 			ExternalID: eid,
 			EVersion:   1,
 			UpdateTime: datastore.RoundTime(clock.Now(ctx).UTC()),
 		}
 		if err := datastore.AllocateIDs(ctx, cl); err != nil {
 			return err
 		}
 		cl.UpdateRetentionKey()
 		m := clMap{ExternalID: eid, InternalID: cl.ID}
 		return datastore.Put(ctx, &m, cl)
 	}, nil)

 	if err != nil {
 		panic(err)
 	}
 	return cl
 }

 // Delete deletes CL and its CLMap entities transactionally.
 //
 // Thus, deletion and insertion (part of ExternalID.getOrInsert) are atomic with
 // respect to one another.
 //
 // However, ExternalID.get and fast path of ExternalID.getOrInsert if called
 // concurrently with Delete may return a temporary error, but on retry they would
 // return ErrNoSuchEntity.
 func Delete(ctx context.Context, id common.CLID) error {
 	cl := CL{ID: id}
 	switch err := datastore.Get(ctx, &cl); {
 	case err == datastore.ErrNoSuchEntity:
 		return nil // Nothing to do.
 	case err != nil:
 		return errors.Annotate(err, "failed to get a CL").Tag(transient.Tag).Err()
 	}

 	err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
 		m := clMap{ExternalID: cl.ExternalID}
 		return datastore.Delete(ctx, &cl, &m)
 	}, nil)
 	if err != nil {
 		return errors.Annotate(err, "failed to delete a CL").Tag(transient.Tag).Err()
 	}
 	return nil
 }

 // Lookup loads CLID for each given ExternalID.
 //
 // CLID is 0 if ExternalID is not yet known.
 // Returns a single error (not MultiError) if there were multiple errors.
 func Lookup(ctx context.Context, eids []ExternalID) ([]common.CLID, error) {
 	out := make([]common.CLID, len(eids))
 	entities := make([]clMap, len(eids))
 	for i, eid := range eids {
 		entities[i].ExternalID = eid
 	}
 	err := datastore.Get(ctx, entities)
 	merrs, _ := err.(errors.MultiError)
 	switch {
 	case err == nil:
 		for i, e := range entities {
 			out[i] = e.InternalID
 		}
 		return out, nil
 	case merrs == nil:
 		return nil, errors.Annotate(err, "failed to load clMap").Tag(transient.Tag).Err()
 	default:
 		for i, err := range merrs {
 			switch {
 			case err == nil:
 				out[i] = entities[i].InternalID
 			case err != datastore.ErrNoSuchEntity:
 				return nil, errors.Annotate(common.MostSevereError(merrs), "failed to load clMap").Tag(transient.Tag).Err()
 			}
 		}
 		return out, nil
 	}
 }
	// Copyright 2020 The LUCI Authors.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package changelist

	import (
	"context"
	"fmt"
	"sort"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/gae/service/datastore"

	"go.chromium.org/luci/cv/internal/common"
	)

	// CL is a CL entity in Datastore.
	type CL struct {
	_kind string `gae:"$kind,CL"`
	_extra datastore.PropertyMap `gae:"-,extra"`

	// ID is auto-generated by Datastore.
	ID common.CLID `gae:"$id"` // int64
	// ExternalID must not be modified once entity is created.
	ExternalID ExternalID `gae:",noindex"` // string. Indexed in CLMap entities.

	// EVersion is entity version. Every update should increment it by 1.
	// See Update() function.
	EVersion int64 `gae:",noindex"`

	// UpdateTime is exact time of when this entity was last updated.
	//
	// It's not indexed to avoid hot areas in the index.
	UpdateTime time.Time `gae:",noindex"`

	// RetentionKey is for data retention purpose.
	//
	// It is indexed and tries to avoid hot areas in the index. The format is
	// `{shard_key}/{unix_time_of_UpdateTime}`. Shard key is the last 2
	// digit of CLID with left padded zero. Unix timestamp is a 10 digit integer
	// with left padded zero if necessary.
	RetentionKey string

	// Snapshot is the latest known state of a CL. It may be and often is
	// behind the source of truth, which is the code review site (e.g. Gerrit).
	Snapshot *Snapshot

	// ApplicableConfig keeps track of configs applicable to the CL.
	//
	// TODO(tandrii): merge into .Access.
	ApplicableConfig *ApplicableConfig

	// Access records per-LUCI project visibility of a CL.
	//
	// See description in protobuf type with the same name.
	//
	// TODO(tandrii): rename GAE field to `Access`.
	Access *Access `gae:"DependentMeta"`

	// IncompleteRuns tracks not yet finalized Runs working on this CL. Sorted.
	//
	// It's updated transactionally with the Run being modified.
	IncompleteRuns common.RunIDs `gae:",noindex"`

	// TriggerNewPatchsetRunAfterPS indicates the patchset number after which
	// new patchset runs can be triggered.
	//
	// E.g. if this value is set to 2, do not trigger new patchset runs for
	// patchsets 1 or 2. Presumably this is because those runs were already
	// completed/failed/otherwise purged.
	//
	// This is needed because unlike label votes which CV can remove, triggering
	// new patchset upload runs relies on the presence of the patchset in the CL
	// snapshot, which cannot be removed.
	TriggerNewPatchsetRunAfterPS int32 `gae:",noindex"`
	}

	// clMap is CLMap entity in Datastore which ensures strict 1:1 mapping
	// between internal and external IDs.
	type clMap struct {
	_kind string `gae:"$kind,CLMap"`

	// ExternalID as entity ID ensures uniqueness.
	ExternalID ExternalID `gae:"$id"` // string
	// InternalID is auto-generated by Datastore for CL entity.
	InternalID common.CLID `gae:",noindex"` // int64. Indexed in CL entities.
	}

	// URL returns URL of the CL.
	func (cl *CL) URL() (string, error) { return cl.ExternalID.URL() }

	// ToUpdatedEvent returns CLUpdatedEvent corresponding to the current CL
	// version.
	func (cl CL) ToUpdatedEvent() CLUpdatedEvent {
	return &CLUpdatedEvent{
	Clid: int64(cl.ID),
	Eversion: cl.EVersion,
	}
	}

	// DO NOT decrease the shard. It will cause olds CLs that are out of retention
	// period in the shard not getting wiped out.
	const retentionKeyShards = 100

	// UpdateRetentionKey updates the RetentionKey of the CL.
	//
	// Panics if the CL.ID and/or CL.UpdateTime is absent.
	func (cl *CL) UpdateRetentionKey() {
	switch {
	case cl.ID == 0:
	panic(errors.New("clid is not set"))
	case cl.UpdateTime.IsZero():
	panic(errors.New("cl.UpdateTime is not set"))
	}
	cl.RetentionKey = fmt.Sprintf("%02d/%010d", cl.ID%retentionKeyShards, cl.UpdateTime.Unix())
	}

	// ToUpdatedEvents returns CLUpdatedEvents from a slice of CLs.
	func ToUpdatedEvents(cls ...CL) CLUpdatedEvents {
	events := make([]*CLUpdatedEvent, len(cls))
	for i, cl := range cls {
	if cl.ID == 0 \|\| cl.EVersion == 0 {
	panic(fmt.Errorf("ID %d and EVersion %d must not be 0", cl.ID, cl.EVersion))
	}
	events[i] = &CLUpdatedEvent{
	Clid: int64(cl.ID),
	Eversion: cl.EVersion,
	}
	}
	sort.Slice(events, func(i, j int) bool {
	// Assume unique CLIDs.
	return events[i].GetClid() < events[j].GetClid()
	})
	return &CLUpdatedEvents{Events: events}
	}

	// Load reads a CL from Datastore.
	//
	// Returns nil, nil if it doesn't exist.
	func (eid ExternalID) Load(ctx context.Context) (*CL, error) {
	m := clMap{ExternalID: eid}
	switch err := datastore.Get(ctx, &m); {
	case err == datastore.ErrNoSuchEntity:
	return nil, nil
	case err != nil:
	return nil, errors.Annotate(err, "failed to get CLMap").Tag(transient.Tag).Err()
	}
	cl := &CL{ID: m.InternalID}
	switch err := datastore.Get(ctx, cl); {
	case err == datastore.ErrNoSuchEntity:
	// This should not happen in practice except in the case of a very old CL
	// which is being deleted due to retention policy. Log error but return it
	// as transient as it's expected that CLMap entity would be removed soon,
	// and so a retry would be produce proper datastore.ErrNoSuchEntity error.
	msg := fmt.Sprintf("unexpectedly failed to get CL#%d given existing CLMap%q", m.InternalID, eid)
	logging.Errorf(ctx, msg)
	return nil, errors.Reason(msg).Tag(transient.Tag).Err()
	case err != nil:
	return nil, errors.Annotate(err, "failed to get CL").Tag(transient.Tag).Err()
	}
	return cl, nil
	}

	// MustCreateIfNotExists is for use in tests to ensure CL exists.
	//
	// Panicks on errors.
	func (eid ExternalID) MustCreateIfNotExists(ctx context.Context) *CL {
	// Fast path without transaction.
	if cl, err := eid.Load(ctx); err == nil && cl != nil {
	return cl
	}
	var cl *CL
	err := datastore.RunInTransaction(ctx, func(ctx context.Context) (err error) {
	cl, err = eid.Load(ctx)
	switch {
	case err != nil:
	return err
	case cl != nil:
	return nil
	}
	cl = &CL{
	ExternalID: eid,
	EVersion: 1,
	UpdateTime: datastore.RoundTime(clock.Now(ctx).UTC()),
	}
	if err := datastore.AllocateIDs(ctx, cl); err != nil {
	return err
	}
	cl.UpdateRetentionKey()
	m := clMap{ExternalID: eid, InternalID: cl.ID}
	return datastore.Put(ctx, &m, cl)
	}, nil)

	if err != nil {
	panic(err)
	}
	return cl
	}

	// Delete deletes CL and its CLMap entities transactionally.
	//
	// Thus, deletion and insertion (part of ExternalID.getOrInsert) are atomic with
	// respect to one another.
	//
	// However, ExternalID.get and fast path of ExternalID.getOrInsert if called
	// concurrently with Delete may return a temporary error, but on retry they would
	// return ErrNoSuchEntity.
	func Delete(ctx context.Context, id common.CLID) error {
	cl := CL{ID: id}
	switch err := datastore.Get(ctx, &cl); {
	case err == datastore.ErrNoSuchEntity:
	return nil // Nothing to do.
	case err != nil:
	return errors.Annotate(err, "failed to get a CL").Tag(transient.Tag).Err()
	}

	err := datastore.RunInTransaction(ctx, func(ctx context.Context) error {
	m := clMap{ExternalID: cl.ExternalID}
	return datastore.Delete(ctx, &cl, &m)
	}, nil)
	if err != nil {
	return errors.Annotate(err, "failed to delete a CL").Tag(transient.Tag).Err()
	}
	return nil
	}

	// Lookup loads CLID for each given ExternalID.
	//
	// CLID is 0 if ExternalID is not yet known.
	// Returns a single error (not MultiError) if there were multiple errors.
	func Lookup(ctx context.Context, eids []ExternalID) ([]common.CLID, error) {
	out := make([]common.CLID, len(eids))
	entities := make([]clMap, len(eids))
	for i, eid := range eids {
	entities[i].ExternalID = eid
	}
	err := datastore.Get(ctx, entities)
	merrs, _ := err.(errors.MultiError)
	switch {
	case err == nil:
	for i, e := range entities {
	out[i] = e.InternalID
	}
	return out, nil
	case merrs == nil:
	return nil, errors.Annotate(err, "failed to load clMap").Tag(transient.Tag).Err()
	default:
	for i, err := range merrs {
	switch {
	case err == nil:
	out[i] = entities[i].InternalID
	case err != datastore.ErrNoSuchEntity:
	return nil, errors.Annotate(common.MostSevereError(merrs), "failed to load clMap").Tag(transient.Tag).Err()
	}
	}
	return out, nil
	}
	}