blob: 43b170721d281ccdac4bc24dddef8bb310a15cd2 [file] [log] [blame]
// Copyright 2015 The LUCI Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package dscache
import (
ds ""
mc ""
type facts struct {
getKeys []*ds.Key
getMeta ds.MultiMetaGetter
lockItems []mc.Item
nonce []byte
type plan struct {
keepMeta bool
// idxMap maps from the original list of keys to the reduced set of keys in
// toGet. E.g. given the index `j` while looping over toGet, idxMap[j] will
// equal the index in the original facts.getKeys list.
idxMap []int
// toGet is a list of datstore keys to retrieve in the call down to the
// underlying datastore. It will have length >= facts.getKeys. All the keys
// in this will be valid (not nil).
toGet []*ds.Key
// toGetMeta is a MultiMetaGetter to be passed in the call down to the
// underlying datastore.GetMulti. It has the same length as toGet.
toGetMeta ds.MultiMetaGetter
// toSave is the list of memcache items to save the results from the
// underlying datastore.GetMulti. It MAY contain nils, which is an indicator
// that this entry SHOULD NOT be saved to memcache.
toSave []mc.Item
// decoded is a list of all the decoded property maps. Its length always ==
// len(facts.getKeys). After the plan is formed, it may contain nils. These
// nils will be filled in from the underlying datastore.GetMulti call in
// ds.go.
decoded []ds.PropertyMap
// lme is a LazyMultiError whose target Size == len(facts.getKeys). The errors
// will eventually bubble back to the layer above this filter in callbacks.
lme errors.LazyMultiError
// add adds a new entry to be retrieved from the actual underlying datastore
// (via GetMulti).
// - idx is the index into the original facts.getKeys
// - get and m are the pair of values that will be passed to datastore.GetMulti
// - save is the memcache item to save the result back to. If it's nil, then
// it will not be saved back to memcache.
func (p *plan) add(idx int, get *ds.Key, m ds.MetaGetter, save mc.Item) {
p.idxMap = append(p.idxMap, idx)
p.toGet = append(p.toGet, get)
p.toSave = append(p.toSave, save)
if p.keepMeta {
p.toGetMeta = append(p.toGetMeta, m)
func (p *plan) empty() bool {
return len(p.idxMap) == 0
// makeFetchPlan takes the input facts and makes a plan about what to do with them.
// Possible scenarios:
// * all entries we got from memcache are valid data, and so we don't need
// to call through to the underlying datastore at all.
// * some entries are 'lock' entries, owned by us, and so we should get them
// from datastore and then attempt to save them back to memcache.
// * some entries are 'lock' entries, owned by something else, so we should
// get them from datastore and then NOT save them to memcache.
// Or some combination thereof. This also handles memcache enries with invalid
// data in them, cases where items have caching disabled entirely, etc.
func (d *dsCache) makeFetchPlan(f *facts) *plan {
p := plan{
keepMeta: f.getMeta != nil,
decoded: make([]ds.PropertyMap, len(f.lockItems)),
lme: errors.NewLazyMultiError(len(f.lockItems)),
for i, lockItm := range f.lockItems {
m := f.getMeta.GetSingle(i)
getKey := f.getKeys[i]
if lockItm == nil {
// this item wasn't cacheable (e.g. the model had caching disabled,
// shardsForKey returned 0, etc.)
p.add(i, getKey, m, nil)
switch FlagValue(lockItm.Flags()) {
case ItemHasLock:
if bytes.Equal(f.nonce, lockItm.Value()) {
// we have the lock
p.add(i, getKey, m, lockItm)
} else {
// someone else has the lock, don't save
p.add(i, getKey, m, nil)
case ItemHasData:
pmap, err := decodeItemValue(lockItm.Value(), d.KeyContext)
switch err {
case nil:
p.decoded[i] = pmap
case ds.ErrNoSuchEntity:
p.lme.Assign(i, ds.ErrNoSuchEntity)
(logging.Fields{"error": err}).Warningf(d.c,
"dscache: error decoding %s, %s", lockItm.Key(), getKey)
p.add(i, getKey, m, nil)
// have some other sort of object, or our AddMulti failed to add this item.
p.add(i, getKey, m, nil)
return &p