tokenserver/appengine/impl/certconfig/crl.go - external/github.com/luci/luci-go - Git at Google

 // Copyright 2016 The LUCI Authors. All rights reserved.
 // Use of this source code is governed under the Apache License, Version 2.0
 // that can be found in the LICENSE file.

 package certconfig

 import (
 	"crypto/sha1"
 	"crypto/x509/pkix"
 	"encoding/hex"
 	"fmt"
 	"math/big"
 	"sync"
 	"time"

 	"golang.org/x/net/context"

 	ds "github.com/luci/gae/service/datastore"
 	"github.com/luci/luci-go/common/clock"
 	"github.com/luci/luci-go/common/data/caching/lazyslot"
 	"github.com/luci/luci-go/common/errors"
 	"github.com/luci/luci-go/common/logging"
 	"github.com/luci/luci-go/common/proto/google"

 	"github.com/luci/luci-go/tokenserver/api/admin/v1"

 	"github.com/luci/luci-go/tokenserver/appengine/impl/utils"
 	"github.com/luci/luci-go/tokenserver/appengine/impl/utils/shards"
 )

 // CRLShardCount is a number of shards to use for storing CRL in the datastore.
 //
 // Each shard can hold ~2 MB of data (taking into account zlib compression),
 // so 16 shards ~= 32 MB. Good enough for a foreseeable future.
 //
 // Changing this value requires rerunning of Admin.FetchCRL RPC to rebuild
 // the entities.
 const CRLShardCount = 16

 // CRL represents a parsed Certificate Revocation List of some CA.
 //
 // ID is always "crl", the parent entity is corresponding CA.
 type CRL struct {
 	_id string `gae:"$id,crl"`

 	// Parent is pointing to parent CA entity.
 	Parent *ds.Key `gae:"$parent"`

 	// EntityVersion is used for simple concurrency control.
 	//
 	// Increase on each update of this entity.
 	EntityVersion int `gae:",noindex"`

 	// LastUpdateTime is extracted from corresponding field of CRL.
 	//
 	// It indicates a time when CRL was generated by the CA.
 	LastUpdateTime time.Time `gae:",noindex"`

 	// LastFetchTime is when this CRL was fetched the last time.
 	//
 	// Updated only when newer CRL version is fetched.
 	LastFetchTime time.Time `gae:",noindex"`

 	// LastFetchETag is ETag header of last downloaded CRL file.
 	//
 	// If CRL's etag doesn't change, we can skip reparsing CRL.
 	LastFetchETag string `gae:",noindex"`

 	// RevokedCertsCount is a number of revoked certificates in CRL. FYI only.
 	RevokedCertsCount int `gae:",noindex"`
 }

 // GetStatusProto returns populated CRLStatus proto message.
 func (crl *CRL) GetStatusProto() *admin.CRLStatus {
 	return &admin.CRLStatus{
 		LastUpdateTime:    google.NewTimestamp(crl.LastUpdateTime),
 		LastFetchTime:     google.NewTimestamp(crl.LastFetchTime),
 		LastFetchEtag:     crl.LastFetchETag,
 		RevokedCertsCount: int64(crl.RevokedCertsCount),
 	}
 }

 ////////////////////////////////////////////////////////////////////////////////

 // CRLShardHeader represents a hash of a shard of a CRL sharded set.
 //
 // We split CRL into a bunch of shards to avoid hitting datastore entity size
 // limits. Each shard lives in its own entity group, where root entity
 // (CRLShardHeader) contains a hash of the shard data (CRLShardBody).
 //
 // It is used to skip fetches of fat shard entities if we already have the same
 // data locally (based on matching hash).
 //
 // ID is "<cn name>|<total number of shards>|<shard index>" (see shardEntityID).
 type CRLShardHeader struct {
 	ID   string `gae:"$id"`
 	SHA1 string `gae:",noindex"` // SHA1 of serialized shard data (before compression)
 }

 // CRLShardBody is a fat entity that contains serialized CRL shard.
 //
 // See CRLShardHeader for more info.
 //
 // Parent entity is CRLShardHeader. ID is always "1".
 type CRLShardBody struct {
 	_id string `gae:"$id,1"`

 	Parent     *ds.Key `gae:"$parent"`  // key of CRLShardHeader
 	SHA1       string  `gae:",noindex"` // SHA1 of serialized shard data (before compression)
 	ZippedData []byte  `gae:",noindex"` // zlib-compressed serialized shards.Shard.
 }

 // UpdateCRLSet splits a set of revoked certificate serial numbers into shards,
 // storing each shard in a separate entity (CRLShardBody).
 //
 // It effectively overwrites the entire set.
 func UpdateCRLSet(c context.Context, cn string, shardCount int, crl *pkix.CertificateList) error {
 	// Split CRL into shards.
 	set := make(shards.Set, shardCount)
 	for _, cert := range crl.TBSCertList.RevokedCertificates {
 		sn, err := utils.SerializeSN(cert.SerialNumber)
 		if err != nil {
 			return err
 		}
 		set.Insert(sn)
 	}
 	// Update shards in parallel via a bunch of independent transactions.
 	wg := sync.WaitGroup{}
 	er := errors.NewLazyMultiError(len(set))
 	for idx, shard := range set {
 		wg.Add(1)
 		go func(idx int, shard shards.Shard) {
 			defer wg.Done()
 			er.Assign(idx, updateCRLShard(c, cn, shard, shardCount, idx))
 		}(idx, shard)
 	}
 	wg.Wait()
 	return er.Get()
 }

 // updateCRLShard updates entities that holds a single shard of a CRL set.
 func updateCRLShard(c context.Context, cn string, shard shards.Shard, count, idx int) error {
 	blob := shard.Serialize()
 	hash := sha1.Sum(blob)
 	digest := hex.EncodeToString(hash[:])

 	// Have it already?
 	header := CRLShardHeader{ID: shardEntityID(cn, count, idx)}
 	switch err := ds.Get(c, &header); {
 	case err != nil && err != ds.ErrNoSuchEntity:
 		return err
 	case err == nil && header.SHA1 == digest:
 		logging.Infof(c, "CRL for %q: shard %d/%d is up-to-date", cn, idx, count)
 		return nil
 	}

 	// Zip before uploading.
 	zipped, err := utils.ZlibCompress(blob)
 	if err != nil {
 		return err
 	}
 	logging.Infof(
 		c, "CRL for %q: shard %d/%d updated (%d bytes zipped, %d%% compression)",
 		cn, idx, count, len(zipped), 100*len(zipped)/len(blob))

 	// Upload, updating the header and the body at once.
 	return ds.RunInTransaction(c, func(c context.Context) error {
 		header.SHA1 = digest
 		body := CRLShardBody{
 			Parent:     ds.KeyForObj(c, &header),
 			SHA1:       digest,
 			ZippedData: zipped,
 		}
 		return ds.Put(c, &header, &body)
 	}, nil)
 }

 // shardEntityID returns an ID of CRLShardHeader entity for given shard.
 //
 // 'cn' is Common Name of the CRL. 'total' is total number of shards expected,
 // and 'index' is an index of some particular shard.
 func shardEntityID(cn string, total, index int) string {
 	return fmt.Sprintf("%s|%d|%d", cn, total, index)
 }

 ////////////////////////////////////////////////////////////////////////////////

 // CRLChecker knows how to check presence of a certificate serial number in CRL.
 //
 // Uses entities prepared by UpdateCRLSet.
 //
 // It is a stateful object that caches CRL shards in memory (occasionally
 // refetching them from the datastore), thus providing an eventually consistent
 // view of the CRL set.
 //
 // Safe for concurrent use. Should be reused between requests.
 type CRLChecker struct {
 	cn            string          // name of CA to check a CRL of
 	shardCount    int             // a total number of shards
 	shards        []lazyslot.Slot // per-shard local state, len(shards) == shardCount
 	cacheDuration time.Duration   // how often to refetch shards from datastore
 }

 // shardCache is kept inside 'shards' slots in CRLChecker.
 type shardCache struct {
 	shard shards.Shard // shard data as a map[]
 	sha1  string       // shard hash, to skip unnecessary refetches
 }

 // NewCRLChecker initializes new CRLChecker that knows how to examine CRL of
 // a CA (identifies by its Common Name).
 //
 // It must know number of shards in advance. Usually is it just CRLShardCount.
 //
 // It will cache shards in local memory, refetching them if necessary after
 // 'cacheDuration' interval.
 func NewCRLChecker(cn string, shardCount int, cacheDuration time.Duration) *CRLChecker {
 	checker := &CRLChecker{
 		cn:            cn,
 		shardCount:    shardCount,
 		shards:        make([]lazyslot.Slot, 0, shardCount),
 		cacheDuration: cacheDuration,
 	}
 	for idx := 0; idx < shardCount; idx++ {
 		idx := idx
 		checker.shards = append(checker.shards, lazyslot.Slot{
 			Fetcher: func(c context.Context, prev lazyslot.Value) (lazyslot.Value, error) {
 				return checker.refetchShard(c, idx, prev)
 			},
 		})
 	}
 	return checker
 }

 // IsRevokedSN returns true if given serial number is in the CRL.
 func (ch *CRLChecker) IsRevokedSN(c context.Context, sn *big.Int) (bool, error) {
 	snBlob, err := utils.SerializeSN(sn)
 	if err != nil {
 		return false, err
 	}
 	shard, err := ch.shard(c, shards.ShardIndex(snBlob, ch.shardCount))
 	if err != nil {
 		return false, err
 	}
 	_, revoked := shard[string(snBlob)]
 	return revoked, nil
 }

 // shard returns a shard given its index.
 func (ch *CRLChecker) shard(c context.Context, idx int) (shards.Shard, error) {
 	val, err := ch.shards[idx].Get(c) // this will lazy-load the shard
 	if err != nil {
 		return nil, err
 	}
 	// lazyslot.Get always returns non-nil val.Value on success. It is safe to
 	// cast it to whatever we put in the Value (which is always shardCache, see
 	// refetchShard).
 	return val.Value.(shardCache).shard, nil
 }

 // refetchShard is called by lazyslot.Slot to fetch a new version of a shard.
 func (ch *CRLChecker) refetchShard(c context.Context, idx int, prev lazyslot.Value) (lazyslot.Value, error) {
 	prevState := shardCache{}
 	if prev.Value != nil {
 		prevState = prev.Value.(shardCache)
 	}

 	// Have something locally already? Quickly fetch CRLShardHeader to check
 	// whether we need to pull a heavy CRLShardBody.
 	hdr := CRLShardHeader{ID: shardEntityID(ch.cn, ch.shardCount, idx)}
 	if prevState.sha1 != "" {
 		switch err := ds.Get(c, &hdr); {
 		case err == ds.ErrNoSuchEntity:
 			return lazyslot.Value{}, fmt.Errorf("shard header %q is missing", hdr.ID)
 		case err != nil:
 			return lazyslot.Value{}, errors.WrapTransient(err)
 		}
 		// The currently cached copy is still good enough?
 		if hdr.SHA1 == prevState.sha1 {
 			return lazyslot.Value{
 				Value:      prevState,
 				Expiration: clock.Now(c).Add(ch.cacheDuration),
 			}, nil
 		}
 	}

 	// Nothing is cached, or the datastore copy is fresher than what we have in
 	// the cache. Need to fetch a new copy, unzip and deserialize it. This entity
 	// is prepared by updateCRLShard.
 	body := CRLShardBody{Parent: ds.KeyForObj(c, &hdr)}
 	switch err := ds.Get(c, &body); {
 	case err == ds.ErrNoSuchEntity:
 		return lazyslot.Value{}, fmt.Errorf("shard body %q is missing", hdr.ID)
 	case err != nil:
 		return lazyslot.Value{}, errors.WrapTransient(err)
 	}

 	// Unzip and deserialize.
 	blob, err := utils.ZlibDecompress(body.ZippedData)
 	if err != nil {
 		return lazyslot.Value{}, err
 	}
 	shard, err := shards.ParseShard(blob)
 	if err != nil {
 		return lazyslot.Value{}, err
 	}

 	return lazyslot.Value{
 		Value: shardCache{
 			shard: shard,
 			sha1:  body.SHA1,
 		},
 		Expiration: clock.Now(c).Add(ch.cacheDuration),
 	}, nil
 }
	// Copyright 2016 The LUCI Authors. All rights reserved.
	// Use of this source code is governed under the Apache License, Version 2.0
	// that can be found in the LICENSE file.

	package certconfig

	import (
	"crypto/sha1"
	"crypto/x509/pkix"
	"encoding/hex"
	"fmt"
	"math/big"
	"sync"
	"time"

	"golang.org/x/net/context"

	ds "github.com/luci/gae/service/datastore"
	"github.com/luci/luci-go/common/clock"
	"github.com/luci/luci-go/common/data/caching/lazyslot"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/proto/google"

	"github.com/luci/luci-go/tokenserver/api/admin/v1"

	"github.com/luci/luci-go/tokenserver/appengine/impl/utils"
	"github.com/luci/luci-go/tokenserver/appengine/impl/utils/shards"
	)

	// CRLShardCount is a number of shards to use for storing CRL in the datastore.
	//
	// Each shard can hold ~2 MB of data (taking into account zlib compression),
	// so 16 shards ~= 32 MB. Good enough for a foreseeable future.
	//
	// Changing this value requires rerunning of Admin.FetchCRL RPC to rebuild
	// the entities.
	const CRLShardCount = 16

	// CRL represents a parsed Certificate Revocation List of some CA.
	//
	// ID is always "crl", the parent entity is corresponding CA.
	type CRL struct {
	_id string `gae:"$id,crl"`

	// Parent is pointing to parent CA entity.
	Parent *ds.Key `gae:"$parent"`

	// EntityVersion is used for simple concurrency control.
	//
	// Increase on each update of this entity.
	EntityVersion int `gae:",noindex"`

	// LastUpdateTime is extracted from corresponding field of CRL.
	//
	// It indicates a time when CRL was generated by the CA.
	LastUpdateTime time.Time `gae:",noindex"`

	// LastFetchTime is when this CRL was fetched the last time.
	//
	// Updated only when newer CRL version is fetched.
	LastFetchTime time.Time `gae:",noindex"`

	// LastFetchETag is ETag header of last downloaded CRL file.
	//
	// If CRL's etag doesn't change, we can skip reparsing CRL.
	LastFetchETag string `gae:",noindex"`

	// RevokedCertsCount is a number of revoked certificates in CRL. FYI only.
	RevokedCertsCount int `gae:",noindex"`
	}

	// GetStatusProto returns populated CRLStatus proto message.
	func (crl CRL) GetStatusProto() admin.CRLStatus {
	return &admin.CRLStatus{
	LastUpdateTime: google.NewTimestamp(crl.LastUpdateTime),
	LastFetchTime: google.NewTimestamp(crl.LastFetchTime),
	LastFetchEtag: crl.LastFetchETag,
	RevokedCertsCount: int64(crl.RevokedCertsCount),
	}
	}

	////////////////////////////////////////////////////////////////////////////////

	// CRLShardHeader represents a hash of a shard of a CRL sharded set.
	//
	// We split CRL into a bunch of shards to avoid hitting datastore entity size
	// limits. Each shard lives in its own entity group, where root entity
	// (CRLShardHeader) contains a hash of the shard data (CRLShardBody).
	//
	// It is used to skip fetches of fat shard entities if we already have the same
	// data locally (based on matching hash).
	//
	// ID is "<cn name>\|<total number of shards>\|<shard index>" (see shardEntityID).
	type CRLShardHeader struct {
	ID string `gae:"$id"`
	SHA1 string `gae:",noindex"` // SHA1 of serialized shard data (before compression)
	}

	// CRLShardBody is a fat entity that contains serialized CRL shard.
	//
	// See CRLShardHeader for more info.
	//
	// Parent entity is CRLShardHeader. ID is always "1".
	type CRLShardBody struct {
	_id string `gae:"$id,1"`

	Parent *ds.Key `gae:"$parent"` // key of CRLShardHeader
	SHA1 string `gae:",noindex"` // SHA1 of serialized shard data (before compression)
	ZippedData []byte `gae:",noindex"` // zlib-compressed serialized shards.Shard.
	}

	// UpdateCRLSet splits a set of revoked certificate serial numbers into shards,
	// storing each shard in a separate entity (CRLShardBody).
	//
	// It effectively overwrites the entire set.
	func UpdateCRLSet(c context.Context, cn string, shardCount int, crl *pkix.CertificateList) error {
	// Split CRL into shards.
	set := make(shards.Set, shardCount)
	for _, cert := range crl.TBSCertList.RevokedCertificates {
	sn, err := utils.SerializeSN(cert.SerialNumber)
	if err != nil {
	return err
	}
	set.Insert(sn)
	}
	// Update shards in parallel via a bunch of independent transactions.
	wg := sync.WaitGroup{}
	er := errors.NewLazyMultiError(len(set))
	for idx, shard := range set {
	wg.Add(1)
	go func(idx int, shard shards.Shard) {
	defer wg.Done()
	er.Assign(idx, updateCRLShard(c, cn, shard, shardCount, idx))
	}(idx, shard)
	}
	wg.Wait()
	return er.Get()
	}

	// updateCRLShard updates entities that holds a single shard of a CRL set.
	func updateCRLShard(c context.Context, cn string, shard shards.Shard, count, idx int) error {
	blob := shard.Serialize()
	hash := sha1.Sum(blob)
	digest := hex.EncodeToString(hash[:])

	// Have it already?
	header := CRLShardHeader{ID: shardEntityID(cn, count, idx)}
	switch err := ds.Get(c, &header); {
	case err != nil && err != ds.ErrNoSuchEntity:
	return err
	case err == nil && header.SHA1 == digest:
	logging.Infof(c, "CRL for %q: shard %d/%d is up-to-date", cn, idx, count)
	return nil
	}

	// Zip before uploading.
	zipped, err := utils.ZlibCompress(blob)
	if err != nil {
	return err
	}
	logging.Infof(
	c, "CRL for %q: shard %d/%d updated (%d bytes zipped, %d%% compression)",
	cn, idx, count, len(zipped), 100*len(zipped)/len(blob))

	// Upload, updating the header and the body at once.
	return ds.RunInTransaction(c, func(c context.Context) error {
	header.SHA1 = digest
	body := CRLShardBody{
	Parent: ds.KeyForObj(c, &header),
	SHA1: digest,
	ZippedData: zipped,
	}
	return ds.Put(c, &header, &body)
	}, nil)
	}

	// shardEntityID returns an ID of CRLShardHeader entity for given shard.
	//
	// 'cn' is Common Name of the CRL. 'total' is total number of shards expected,
	// and 'index' is an index of some particular shard.
	func shardEntityID(cn string, total, index int) string {
	return fmt.Sprintf("%s\|%d\|%d", cn, total, index)
	}

	////////////////////////////////////////////////////////////////////////////////

	// CRLChecker knows how to check presence of a certificate serial number in CRL.
	//
	// Uses entities prepared by UpdateCRLSet.
	//
	// It is a stateful object that caches CRL shards in memory (occasionally
	// refetching them from the datastore), thus providing an eventually consistent
	// view of the CRL set.
	//
	// Safe for concurrent use. Should be reused between requests.
	type CRLChecker struct {
	cn string // name of CA to check a CRL of
	shardCount int // a total number of shards
	shards []lazyslot.Slot // per-shard local state, len(shards) == shardCount
	cacheDuration time.Duration // how often to refetch shards from datastore
	}

	// shardCache is kept inside 'shards' slots in CRLChecker.
	type shardCache struct {
	shard shards.Shard // shard data as a map[]
	sha1 string // shard hash, to skip unnecessary refetches
	}

	// NewCRLChecker initializes new CRLChecker that knows how to examine CRL of
	// a CA (identifies by its Common Name).
	//
	// It must know number of shards in advance. Usually is it just CRLShardCount.
	//
	// It will cache shards in local memory, refetching them if necessary after
	// 'cacheDuration' interval.
	func NewCRLChecker(cn string, shardCount int, cacheDuration time.Duration) *CRLChecker {
	checker := &CRLChecker{
	cn: cn,
	shardCount: shardCount,
	shards: make([]lazyslot.Slot, 0, shardCount),
	cacheDuration: cacheDuration,
	}
	for idx := 0; idx < shardCount; idx++ {
	idx := idx
	checker.shards = append(checker.shards, lazyslot.Slot{
	Fetcher: func(c context.Context, prev lazyslot.Value) (lazyslot.Value, error) {
	return checker.refetchShard(c, idx, prev)
	},
	})
	}
	return checker
	}

	// IsRevokedSN returns true if given serial number is in the CRL.
	func (ch CRLChecker) IsRevokedSN(c context.Context, sn big.Int) (bool, error) {
	snBlob, err := utils.SerializeSN(sn)
	if err != nil {
	return false, err
	}
	shard, err := ch.shard(c, shards.ShardIndex(snBlob, ch.shardCount))
	if err != nil {
	return false, err
	}
	_, revoked := shard[string(snBlob)]
	return revoked, nil
	}

	// shard returns a shard given its index.
	func (ch *CRLChecker) shard(c context.Context, idx int) (shards.Shard, error) {
	val, err := ch.shards[idx].Get(c) // this will lazy-load the shard
	if err != nil {
	return nil, err
	}
	// lazyslot.Get always returns non-nil val.Value on success. It is safe to
	// cast it to whatever we put in the Value (which is always shardCache, see
	// refetchShard).
	return val.Value.(shardCache).shard, nil
	}

	// refetchShard is called by lazyslot.Slot to fetch a new version of a shard.
	func (ch *CRLChecker) refetchShard(c context.Context, idx int, prev lazyslot.Value) (lazyslot.Value, error) {
	prevState := shardCache{}
	if prev.Value != nil {
	prevState = prev.Value.(shardCache)
	}

	// Have something locally already? Quickly fetch CRLShardHeader to check
	// whether we need to pull a heavy CRLShardBody.
	hdr := CRLShardHeader{ID: shardEntityID(ch.cn, ch.shardCount, idx)}
	if prevState.sha1 != "" {
	switch err := ds.Get(c, &hdr); {
	case err == ds.ErrNoSuchEntity:
	return lazyslot.Value{}, fmt.Errorf("shard header %q is missing", hdr.ID)
	case err != nil:
	return lazyslot.Value{}, errors.WrapTransient(err)
	}
	// The currently cached copy is still good enough?
	if hdr.SHA1 == prevState.sha1 {
	return lazyslot.Value{
	Value: prevState,
	Expiration: clock.Now(c).Add(ch.cacheDuration),
	}, nil
	}
	}

	// Nothing is cached, or the datastore copy is fresher than what we have in
	// the cache. Need to fetch a new copy, unzip and deserialize it. This entity
	// is prepared by updateCRLShard.
	body := CRLShardBody{Parent: ds.KeyForObj(c, &hdr)}
	switch err := ds.Get(c, &body); {
	case err == ds.ErrNoSuchEntity:
	return lazyslot.Value{}, fmt.Errorf("shard body %q is missing", hdr.ID)
	case err != nil:
	return lazyslot.Value{}, errors.WrapTransient(err)
	}

	// Unzip and deserialize.
	blob, err := utils.ZlibDecompress(body.ZippedData)
	if err != nil {
	return lazyslot.Value{}, err
	}
	shard, err := shards.ParseShard(blob)
	if err != nil {
	return lazyslot.Value{}, err
	}

	return lazyslot.Value{
	Value: shardCache{
	shard: shard,
	sha1: body.SHA1,
	},
	Expiration: clock.Now(c).Add(ch.cacheDuration),
	}, nil
	}