cipd/client/cipd/reader/reader_test.go

// Copyright 2014 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reader

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"runtime"
	"sync"
	"testing"
	"time"

	api "go.chromium.org/luci/cipd/api/cipd/v1"
	"go.chromium.org/luci/cipd/client/cipd/builder"
	"go.chromium.org/luci/cipd/client/cipd/fs"
	"go.chromium.org/luci/cipd/client/cipd/pkg"

	. "github.com/smartystreets/goconvey/convey"
	. "go.chromium.org/luci/cipd/common"
)

func stringCounts(values []string) map[string]int {
	counts := make(map[string]int)
	for _, v := range values {
		counts[v]++
	}
	return counts
}

// shouldContainSameStrings checks if the left and right side are slices that
// contain the same strings, regardless of the ordering.
func shouldContainSameStrings(actual any, expected ...any) string {
	if len(expected) != 1 {
		return "Too many arguments for shouldContainSameStrings"
	}
	return ShouldResemble(stringCounts(actual.([]string)), stringCounts(expected[0].([]string)))
}

func normalizeJSON(s string) (string, error) {
	// Round trip through default json marshaller to normalize indentation.
	var x map[string]any
	if err := json.Unmarshal([]byte(s), &x); err != nil {
		return "", err
	}
	blob, err := json.Marshal(x)
	if err != nil {
		return "", err
	}
	return string(blob), nil
}

func shouldBeSameJSONDict(actual any, expected ...any) string {
	if len(expected) != 1 {
		return "Too many arguments for shouldBeSameJSONDict"
	}
	actualNorm, err := normalizeJSON(actual.(string))
	if err != nil {
		return err.Error()
	}
	expectedNorm, err := normalizeJSON(expected[0].(string))
	if err != nil {
		return err.Error()
	}
	return ShouldEqual(actualNorm, expectedNorm)
}

type bytesSource struct {
	*bytes.Reader
}

func (bytesSource) Close(context.Context, bool) error { return nil }

func bytesFile(buf *bytes.Buffer) pkg.Source {
	return bytesSource{bytes.NewReader(buf.Bytes())}
}

func TestPackageReading(t *testing.T) {
	ctx := context.Background()

	Convey("Open empty package works", t, func() {
		// Build an empty package.
		out := bytes.Buffer{}
		pin, err := builder.BuildInstance(ctx, builder.Options{
			Output:           &out,
			PackageName:      "testing",
			CompressionLevel: 5,
		})
		So(err, ShouldBeNil)
		So(pin, ShouldResemble, Pin{
			PackageName: "testing",
			InstanceID:  "PPM180-5i-V1q5554ewKGO4jq4cWB-cOwTuyhoCv3joC",
		})

		// Open it.
		inst, err := OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: CalculateHash,
			HashAlgo:         api.HashAlgo_SHA256,
		})
		So(inst, ShouldNotBeNil)
		So(err, ShouldBeNil)
		defer inst.Close(ctx, false)

		So(inst.Pin(), ShouldResemble, pin)
		So(len(inst.Files()), ShouldEqual, 1)

		// CalculatePin also agrees with the value of the pin.
		calcedPin, err := CalculatePin(ctx, pkg.NewBytesSource(out.Bytes()), api.HashAlgo_SHA256)
		So(err, ShouldBeNil)
		So(calcedPin, ShouldResemble, pin)

		// Contains single manifest file.
		f := inst.Files()[0]
		So(f.Name(), ShouldEqual, ".cipdpkg/manifest.json")
		So(f.Executable(), ShouldBeFalse)
		r, err := f.Open()
		if r != nil {
			defer r.Close()
		}
		So(err, ShouldBeNil)
		manifest, err := io.ReadAll(r)
		So(err, ShouldBeNil)

		goodManifest := `{
			"format_version": "1.1",
			"package_name": "testing"
		}`
		So(string(manifest), shouldBeSameJSONDict, goodManifest)
	})

	Convey("Open empty package with unexpected instance ID", t, func() {
		// Build an empty package.
		out := bytes.Buffer{}
		_, err := builder.BuildInstance(ctx, builder.Options{
			Output:           &out,
			PackageName:      "testing",
			CompressionLevel: 5,
		})
		So(err, ShouldBeNil)

		// Attempt to open it, providing correct instance ID, should work.
		inst, err := OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: VerifyHash,
			InstanceID:       "PPM180-5i-V1q5554ewKGO4jq4cWB-cOwTuyhoCv3joC",
		})
		So(err, ShouldBeNil)
		So(inst, ShouldNotBeNil)
		defer inst.Close(ctx, false)

		So(inst.Pin(), ShouldResemble, Pin{
			PackageName: "testing",
			InstanceID:  "PPM180-5i-V1q5554ewKGO4jq4cWB-cOwTuyhoCv3joC",
		})

		// Attempt to open it, providing incorrect instance ID.
		inst, err = OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: VerifyHash,
			InstanceID:       "ZZZZZZZZ_LlIHZUsZlTzpmiCs8AqvAhz9TZzN96Qpx4C",
		})
		So(err, ShouldEqual, ErrHashMismatch)
		So(inst, ShouldBeNil)

		// Open with incorrect instance ID, but skipping the verification..
		inst, err = OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: SkipHashVerification,
			InstanceID:       "ZZZZZZZZ_LlIHZUsZlTzpmiCs8AqvAhz9TZzN96Qpx4C",
		})
		So(err, ShouldBeNil)
		So(inst, ShouldNotBeNil)
		defer inst.Close(ctx, false)

		So(inst.Pin(), ShouldResemble, Pin{
			PackageName: "testing",
			InstanceID:  "ZZZZZZZZ_LlIHZUsZlTzpmiCs8AqvAhz9TZzN96Qpx4C",
		})
	})

	Convey("OpenInstanceFile works", t, func() {
		// Open temp file.
		tempFile, err := ioutil.TempFile("", "cipdtest")
		So(err, ShouldBeNil)
		tempFilePath := tempFile.Name()
		defer os.Remove(tempFilePath)

		// Write empty package to it.
		_, err = builder.BuildInstance(ctx, builder.Options{
			Output:           tempFile,
			PackageName:      "testing",
			CompressionLevel: 5,
		})
		So(err, ShouldBeNil)
		tempFile.Close()

		// Read the package.
		inst, err := OpenInstanceFile(ctx, tempFilePath, OpenInstanceOpts{
			VerificationMode: CalculateHash,
			HashAlgo:         api.HashAlgo_SHA256,
		})
		So(err, ShouldBeNil)
		So(inst, ShouldNotBeNil)
		So(inst.Close(ctx, false), ShouldBeNil)
	})

	Convey("ExtractFiles works", t, func() {
		testMTime := time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC)

		inFiles := []fs.File{
			fs.NewTestFile("testing/qwerty", "12345", fs.TestFileOpts{}),
			fs.NewTestFile("abc", "duh", fs.TestFileOpts{Executable: true}),
			fs.NewTestFile("writable", "write me", fs.TestFileOpts{Writable: true}),
			fs.NewTestFile("timestamped", "I'm old", fs.TestFileOpts{ModTime: testMTime}),
			fs.NewTestSymlink("rel_symlink", "abc"),
			fs.NewTestSymlink("abs_symlink", "/abc/def"),
		}
		if runtime.GOOS == "windows" {
			inFiles = append(inFiles,
				fs.NewWinTestFile("secret", "ninja", fs.WinAttrHidden),
				fs.NewWinTestFile("system", "machine", fs.WinAttrSystem),
			)
		}

		out := bytes.Buffer{}
		_, err := builder.BuildInstance(ctx, builder.Options{
			Input:            inFiles,
			Output:           &out,
			PackageName:      "testing",
			VersionFile:      "subpath/version.json",
			CompressionLevel: 5,
		})
		So(err, ShouldBeNil)

		// Extract files.
		inst, err := OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: CalculateHash,
			HashAlgo:         api.HashAlgo_SHA256,
		})
		So(err, ShouldBeNil)
		defer inst.Close(ctx, false)

		dest := &testDestination{}
		_, err = ExtractFilesTxn(ctx, inst.Files(), dest, 16, pkg.WithManifest, "")
		So(err, ShouldBeNil)
		So(dest.beginCalls, ShouldEqual, 1)
		So(dest.endCalls, ShouldEqual, 1)

		// Verify file list, file data and flags are correct.
		names := make([]string, len(dest.files))
		for i, f := range dest.files {
			names[i] = f.name
		}
		if runtime.GOOS != "windows" {
			So(names, shouldContainSameStrings, []string{
				"testing/qwerty",
				"abc",
				"writable",
				"timestamped",
				"rel_symlink",
				"abs_symlink",
				"subpath/version.json",
				".cipdpkg/manifest.json",
			})
		} else {
			So(names, shouldContainSameStrings, []string{
				"testing/qwerty",
				"abc",
				"writable",
				"timestamped",
				"rel_symlink",
				"abs_symlink",
				"secret",
				"system",
				"subpath/version.json",
				".cipdpkg/manifest.json",
			})
		}

		So(string(dest.fileByName("testing/qwerty").Bytes()), ShouldEqual, "12345")
		So(dest.fileByName("abc").executable, ShouldBeTrue)
		So(dest.fileByName("abc").writable, ShouldBeFalse)
		So(dest.fileByName("writable").writable, ShouldBeTrue)
		So(dest.fileByName("writable").modtime.IsZero(), ShouldBeTrue)
		So(dest.fileByName("timestamped").modtime, ShouldEqual, testMTime)
		So(dest.fileByName("rel_symlink").symlinkTarget, ShouldEqual, "abc")
		So(dest.fileByName("abs_symlink").symlinkTarget, ShouldEqual, "/abc/def")

		// Verify version file is correct.
		goodVersionFile := `{
			"instance_id": "OvNF-MsVw1eXYJtjkiq7pXCm6mLYYSaN9qSqsMT3DEAC",
			"package_name": "testing"
		}`
		if runtime.GOOS == "windows" {
			goodVersionFile = `{
				"instance_id": "ZM2WksvyI1lQiHnYcuLLXQSNquBPexuH-t57CkJPVDoC",
				"package_name": "testing"
			}`
		}
		So(string(dest.fileByName("subpath/version.json").Bytes()),
			shouldBeSameJSONDict, goodVersionFile)

		// Verify manifest file is correct.
		goodManifest := `{
			"format_version": "1.1",
			"package_name": "testing",
			"version_file": "subpath/version.json",
			"files": [
				{
					"name": "testing/qwerty",
					"size": 5,
					"hash": "WZRHGrsBESr8wYFZ9sx0tPURuZgG2lmzyvWpwXPKz8UC"
				},
				{
					"name": "abc",
					"size": 3,
					"executable": true,
					"hash": "i_jQPvLtCYwT3iPForJuG9tFWRu9c3ndgjxk7nXjY2kC"
				},
				{
					"name": "writable",
					"size": 8,
					"writable": true,
					"hash": "QeUFaPVoXLyp7lPHwnWGgBD5Wo-buja_bBGTx4s3jkkC"
				},
				{
					"modtime":1514764800,
					"name": "timestamped",
					"size": 7,
					"hash": "M2fO8ZiWvyqNVmp_Nu5QZo80JXSjkqHz60zRlhqNHzgC"
				},
				{
					"name": "rel_symlink",
					"size": 0,
					"symlink": "abc"
				},
				{
					"name": "abs_symlink",
					"size": 0,
					"symlink": "/abc/def"
				},%s
			]
		}`
		if runtime.GOOS == "windows" {
			goodManifest = fmt.Sprintf(goodManifest, `{
				"name": "secret",
				"size": 5,
				"win_attrs": "H",
				"hash": "VEgllRdxFuYQOwdtvzBkjl0FN90e2c9a5FYvqKcA1HsC"
			},
			{
				"name": "system",
				"size": 7,
				"win_attrs": "S",
				"hash": "vAIKNbf5yxOC57U0xo48Ux2EmxGb8U913erWzEXDzMEC"
			},
			{
				"name": "subpath/version.json",
				"size": 96,
				"hash": "LQ25MuK852zm7GjAUcBIzWpAzhcXV7L4pEVl2LCUz8YC"
			}`)
		} else {
			goodManifest = fmt.Sprintf(goodManifest, `{
				"name": "subpath/version.json",
				"size": 96,
				"hash": "XD6QlRyLX4Cj09wtPLAEQGacygrySk317U38Ku2d9zIC"
			}`)
		}
		So(string(dest.fileByName(".cipdpkg/manifest.json").Bytes()),
			shouldBeSameJSONDict, goodManifest)
	})

	Convey("ExtractFiles handles v1 packages correctly", t, func() {
		// ZipInfos in packages with format_version "1" always have the writable bit
		// set, and always have 0 timestamp. During the extraction of such package,
		// the writable bit should be cleared, and the timestamp should not be reset
		// to 0 (it will be set to whatever the current time is).
		inFiles := []fs.File{
			fs.NewTestFile("testing/qwerty", "12345", fs.TestFileOpts{Writable: true}),
			fs.NewTestFile("abc", "duh", fs.TestFileOpts{Executable: true, Writable: true}),
			fs.NewTestSymlink("rel_symlink", "abc"),
			fs.NewTestSymlink("abs_symlink", "/abc/def"),
		}
		if runtime.GOOS == "windows" {
			inFiles = append(inFiles,
				fs.NewWinTestFile("secret", "ninja", fs.WinAttrHidden),
				fs.NewWinTestFile("system", "machine", fs.WinAttrSystem),
			)
		}

		out := bytes.Buffer{}
		_, err := builder.BuildInstance(ctx, builder.Options{
			Input:                 inFiles,
			Output:                &out,
			PackageName:           "testing",
			VersionFile:           "subpath/version.json",
			CompressionLevel:      5,
			OverrideFormatVersion: "1",
		})
		So(err, ShouldBeNil)

		// Extract files.
		inst, err := OpenInstance(ctx, bytesFile(&out), OpenInstanceOpts{
			VerificationMode: CalculateHash,
			HashAlgo:         api.HashAlgo_SHA256,
		})
		So(err, ShouldBeNil)
		defer inst.Close(ctx, false)

		dest := &testDestination{}
		_, err = ExtractFilesTxn(ctx, inst.Files(), dest, 16, pkg.WithManifest, "")
		So(err, ShouldBeNil)
		So(dest.beginCalls, ShouldEqual, 1)
		So(dest.endCalls, ShouldEqual, 1)

		// Verify file list, file data and flags are correct.
		names := make([]string, len(dest.files))
		for i, f := range dest.files {
			names[i] = f.name
		}
		if runtime.GOOS != "windows" {
			So(names, shouldContainSameStrings, []string{
				"testing/qwerty",
				"abc",
				"rel_symlink",
				"abs_symlink",
				"subpath/version.json",
				".cipdpkg/manifest.json",
			})
		} else {
			So(names, shouldContainSameStrings, []string{
				"testing/qwerty",
				"abc",
				"rel_symlink",
				"abs_symlink",
				"secret",
				"system",
				"subpath/version.json",
				".cipdpkg/manifest.json",
			})
		}

		So(string(dest.fileByName("testing/qwerty").Bytes()), ShouldEqual, "12345")
		So(dest.fileByName("abc").executable, ShouldBeTrue)
		So(dest.fileByName("abc").writable, ShouldBeFalse)
		So(dest.fileByName("rel_symlink").symlinkTarget, ShouldEqual, "abc")
		So(dest.fileByName("abs_symlink").symlinkTarget, ShouldEqual, "/abc/def")

		// Verify version file is correct.
		goodVersionFile := `{
			"instance_id": "TFCuWXMWQSAoTRwjHXIu9ZTRTNrkwxuXXNKg_HTFQn0C",
			"package_name": "testing"
		}`
		if runtime.GOOS == "windows" {
			goodVersionFile = `{
				"instance_id": "JtKHZQLjvjsDbo2bRjfkCXCdlE4PZ-mRA7c2fAx09hkC",
				"package_name": "testing"
			}`
		}
		So(string(dest.fileByName("subpath/version.json").Bytes()),
			shouldBeSameJSONDict, goodVersionFile)

		// Verify manifest file is correct.
		goodManifest := `{
			"format_version": "1",
			"package_name": "testing",
			"version_file": "subpath/version.json",
			"files": [
				{
					"name": "testing/qwerty",
					"size": 5,
					"hash": "WZRHGrsBESr8wYFZ9sx0tPURuZgG2lmzyvWpwXPKz8UC"
				},
				{
					"name": "abc",
					"size": 3,
					"executable": true,
					"hash": "i_jQPvLtCYwT3iPForJuG9tFWRu9c3ndgjxk7nXjY2kC"
				},
				{
					"name": "rel_symlink",
					"size": 0,
					"symlink": "abc"
				},
				{
					"name": "abs_symlink",
					"size": 0,
					"symlink": "/abc/def"
				},%s
			]
		}`
		if runtime.GOOS == "windows" {
			goodManifest = fmt.Sprintf(goodManifest, `{
				"name": "secret",
				"size": 5,
				"win_attrs": "H",
				"hash": "VEgllRdxFuYQOwdtvzBkjl0FN90e2c9a5FYvqKcA1HsC"
			},
			{
				"name": "system",
				"size": 7,
				"win_attrs": "S",
				"hash": "vAIKNbf5yxOC57U0xo48Ux2EmxGb8U913erWzEXDzMEC"
			},
			{
				"name": "subpath/version.json",
				"size": 96,
				"hash": "NktdGybrepm6oPSqjFG_qrNZDeaY_KbP4elWDz4ww48C"
			}`)
		} else {
			goodManifest = fmt.Sprintf(goodManifest, `{
				"name": "subpath/version.json",
				"size": 96,
				"hash": "JlgQS4Xa4D7f94PYzpQcvgPsDfQqySYVlUBBYfF6x8sC"
			}`)
		}
		So(string(dest.fileByName(".cipdpkg/manifest.json").Bytes()),
			shouldBeSameJSONDict, goodManifest)
	})
}

////////////////////////////////////////////////////////////////////////////////

type testDestination struct {
	beginCalls       int
	endCalls         int
	files            []*testDestinationFile
	registerFileLock sync.Mutex
}

type testDestinationFile struct {
	bytes.Buffer
	name          string
	executable    bool
	writable      bool
	modtime       time.Time
	symlinkTarget string
	winAttrs      fs.WinAttrs
}

func (d *testDestinationFile) Close() error { return nil }

func (d *testDestination) Begin(context.Context) error {
	d.beginCalls++
	return nil
}

func (d *testDestination) CreateFile(ctx context.Context, name string, opts fs.CreateFileOptions) (io.WriteCloser, error) {
	f := &testDestinationFile{
		name:       name,
		executable: opts.Executable,
		writable:   opts.Writable,
		modtime:    opts.ModTime,
		winAttrs:   opts.WinAttrs,
	}
	d.registerFile(f)
	return f, nil
}

func (d *testDestination) CreateSymlink(ctx context.Context, name string, target string) error {
	f := &testDestinationFile{
		name:          name,
		symlinkTarget: target,
	}
	d.registerFile(f)
	return nil
}

func (d *testDestination) End(ctx context.Context, success bool) error {
	d.endCalls++
	return nil
}

func (d *testDestination) fileByName(name string) *testDestinationFile {
	for _, f := range d.files {
		if f.name == name {
			return f
		}
	}
	return nil
}

func (d *testDestination) registerFile(f *testDestinationFile) {
	d.registerFileLock.Lock()
	d.files = append(d.files, f)
	d.registerFileLock.Unlock()
}