blob: b561c19dbbc8552ef5b28cd39a006096807113d9 [file] [log] [blame]
// Copyright 2015 The LUCI Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package dscache
import (
ds ""
type facts struct {
getKeys []*ds.Key
getMeta ds.MultiMetaGetter
lockItems []CacheItem
nonce []byte
type plan struct {
keepMeta bool
// idxMap maps from the original list of keys to the reduced set of keys in
// toGet. E.g. given the index `j` while looping over toGet, idxMap[j] will
// equal the index in the original facts.getKeys list.
idxMap []int
// toGet is a list of datstore keys to retrieve in the call down to the
// underlying datastore. It will have length >= facts.getKeys. All the keys
// in this will be valid (not nil).
toGet []*ds.Key
// toGetMeta is a MultiMetaGetter to be passed in the call down to the
// underlying datastore.GetMulti. It has the same length as toGet.
toGetMeta ds.MultiMetaGetter
// toSave is the list of memcache items to save the results from the
// underlying datastore.GetMulti. It MAY contain nils, which is an indicator
// that this entry SHOULD NOT be saved to memcache. Items that are not nils
// are necessarily locks owned by us (since we can save only stuff we promised
// to save by locking it).
toSave []CacheItem
// decoded is a list of all the decoded property maps. Its length always ==
// len(facts.getKeys). After the plan is formed, it may contain nils. These
// nils will be filled in from the underlying datastore.GetMulti call in
// ds.go.
decoded []ds.PropertyMap
// lme is a LazyMultiError whose target Size == len(facts.getKeys). The errors
// will eventually bubble back to the layer above this filter in callbacks.
lme errors.LazyMultiError
// add adds a new entry to be retrieved from the actual underlying datastore
// (via GetMulti).
// - idx is the index into the original facts.getKeys
// - get and m are the pair of values that will be passed to datastore.GetMulti
// - save is the memcache item to save the result back to. If it's nil, then
// it will not be saved back to memcache.
func (p *plan) add(idx int, get *ds.Key, m ds.MetaGetter, save CacheItem) {
p.idxMap = append(p.idxMap, idx)
p.toGet = append(p.toGet, get)
p.toSave = append(p.toSave, save)
if p.keepMeta {
p.toGetMeta = append(p.toGetMeta, m)
func (p *plan) empty() bool {
return len(p.idxMap) == 0
// makeFetchPlan takes the input facts and makes a plan about what to do with them.
// Possible scenarios:
// * all entries we got from memcache are valid data, and so we don't need
// to call through to the underlying datastore at all.
// * some entries are 'lock' entries, owned by us, and so we should get them
// from datastore and then attempt to save them back to memcache.
// * some entries are 'lock' entries, owned by something else, so we should
// get them from datastore and then NOT save them to memcache.
// Or some combination thereof. This also handles memcache entries with invalid
// data in them, cases where items have caching disabled entirely, etc.
func (d *dsCache) makeFetchPlan(f facts) *plan {
p := plan{
keepMeta: f.getMeta != nil,
decoded: make([]ds.PropertyMap, len(f.lockItems)),
lme: errors.NewLazyMultiError(len(f.lockItems)),
for i, lockItm := range f.lockItems {
m := f.getMeta.GetSingle(i)
getKey := f.getKeys[i]
if lockItm == nil {
// this item wasn't cacheable (e.g. the model had caching disabled,
// shardsForKey returned 0, etc.)
p.add(i, getKey, m, nil)
switch nonce := lockItm.Nonce(); {
case bytes.Equal(f.nonce, nonce):
// we have the lock, need to fetch and store the items in the cache
p.add(i, getKey, m, lockItm)
case nonce != nil:
// someone else has the lock, don't save
p.add(i, getKey, m, nil)
// this is a data item, just load it, don't touch datastore
pmap, err := decodeItemValue(lockItm.Data(), d.KeyContext)
switch err {
case nil:
p.decoded[i] = pmap
case ds.ErrNoSuchEntity:
p.lme.Assign(i, ds.ErrNoSuchEntity)
logging.WithError(err).Warningf(d.c, "dscache: error decoding %s, %s", lockItm.Key(), getKey)
p.add(i, getKey, m, nil)
return &p