common/isolated/large.go

// Copyright 2019 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package isolated

import (
	"bytes"
	"compress/zlib"
	"encoding/json"
	"io/ioutil"
	"sort"

	"go.chromium.org/luci/common/errors"
)

// Pack returns a deflate'd buffer of delta encoded varints.
func Pack(values []int64) ([]byte, error) {
	if len(values) == 0 {
		return nil, nil
	}
	if values[0] < 0 {
		return nil, errors.Reason("values must be between 0 and 2**63").Err()
	}
	if values[len(values)-1] < 0 {
		return nil, errors.Reason("values must be between 0 and 2**63").Err()
	}

	var b bytes.Buffer
	w := zlib.NewWriter(&b)
	var last int64
	for _, value := range values {
		v := value
		value -= last
		if value < 0 {
			return nil, errors.Reason("list must be sorted ascending").Err()
		}
		last = v
		for value > 127 {
			if _, err := w.Write([]byte{byte(1<<7 | value&0x7f)}); err != nil {
				return nil, errors.Annotate(err, "failed to write").Err()
			}
			value >>= 7
		}
		if _, err := w.Write([]byte{byte(value)}); err != nil {
			return nil, errors.Annotate(err, "failed to write").Err()
		}

	}
	if err := w.Close(); err != nil {
		return nil, errors.Annotate(err, "failed to close zlib writer").Err()
	}

	return b.Bytes(), nil
}

// Unpack decompresses a deflate'd delta encoded list of varints.
func Unpack(data []byte) ([]int64, error) {
	if len(data) == 0 {
		return nil, nil
	}

	var ret []int64
	var value int64
	var base int64 = 1
	var last int64

	r, err := zlib.NewReader(bytes.NewReader(data))
	if err != nil {
		return nil, errors.Annotate(err, "failed to get zlib reader").Err()
	}
	defer r.Close()

	data, err = ioutil.ReadAll(r)
	if err != nil {
		return nil, errors.Annotate(err, "failed to read all").Err()
	}

	for _, valByte := range data {
		value += int64(valByte&0x7f) * base
		if valByte&0x80 > 0 {
			base <<= 7
			continue
		}
		ret = append(ret, value+last)
		last += value
		value = 0
		base = 1
	}

	return ret, nil
}

// WriteStats writes cache stats in packed format.
func WriteStats(path string, hot, cold []int64) error {
	// Copy before sort.
	cold = append([]int64{}, cold...)
	hot = append([]int64{}, hot...)

	sort.Slice(cold, func(i, j int) bool { return cold[i] < cold[j] })
	sort.Slice(hot, func(i, j int) bool { return hot[i] < hot[j] })

	packedCold, err := Pack(cold)
	if err != nil {
		return errors.Annotate(err, "failed to pack uploaded items").Err()
	}

	packedHot, err := Pack(hot)
	if err != nil {
		return errors.Annotate(err, "failed to pack not uploaded items").Err()
	}

	statsJSON, err := json.Marshal(struct {
		ItemsCold []byte `json:"items_cold"`
		ItemsHot  []byte `json:"items_hot"`
	}{
		ItemsCold: packedCold,
		ItemsHot:  packedHot,
	})
	if err != nil {
		return errors.Annotate(err, "failed to marshal stats json").Err()
	}
	if err := ioutil.WriteFile(path, statsJSON, 0600); err != nil {
		return errors.Annotate(err, "failed to write stats json").Err()
	}

	return nil
}