common/system/filesystem/filesystem.go - infra/luci/luci-go - Git at Google

 // Copyright 2017 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package filesystem

 import (
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
 	"sort"
 	"strings"
 	"syscall"
 	"time"

 	"go.chromium.org/luci/common/data/sortby"
 	"go.chromium.org/luci/common/errors"
 )

 // IsNotExist calls os.IsNotExist on the unwrapped err.
 func IsNotExist(err error) bool { return os.IsNotExist(errors.Unwrap(err)) }

 // MakeDirs is a convenience wrapper around os.MkdirAll that applies a 0755
 // mask to all created directories.
 func MakeDirs(path string) error {
 	if err := os.MkdirAll(path, 0755); err != nil {
 		return errors.Annotate(err, "").Err()
 	}
 	return nil
 }

 // AbsPath is a convenience wrapper around filepath.Abs that accepts a string
 // pointer, base, and updates it on successful resolution.
 func AbsPath(base *string) error {
 	v, err := filepath.Abs(*base)
 	if err != nil {
 		return errors.Annotate(err, "unable to resolve absolute path").
 			InternalReason("base(%q)", *base).Err()
 	}
 	*base = v
 	return nil
 }

 // Touch creates a new, empty file at the specified path.
 //
 // If when is zero-value, time.Now will be used.
 func Touch(path string, when time.Time, mode os.FileMode) error {
 	// Try and create a file at the target path.
 	fd, err := os.OpenFile(path, (os.O_CREATE | os.O_RDWR | os.O_EXCL), mode)
 	if err == nil {
 		if err := fd.Close(); err != nil {
 			return errors.Annotate(err, "failed to close new file").Err()
 		}
 		if when.IsZero() {
 			// If "now" was specified, and we created a new file, then its times will
 			// be now by default.
 			return nil
 		}
 	}

 	// Couldn't create a new file. Either it exists already, it is a directory,
 	// or there was an OS-level failure. Since we can't really distinguish
 	// between these cases, try Chtimes (update timestamp) and error
 	// if this fails.
 	if when.IsZero() {
 		when = time.Now()
 	}
 	if err := os.Chtimes(path, when, when); err != nil {
 		return errors.Annotate(err, "failed to Chtimes").InternalReason("path(%q)", path).Err()
 	}

 	return nil
 }

 // RemoveAll is a fork of os.RemoveAll that attempts to deal with read only
 // files and directories by modifying permissions as necessary.
 //
 // If the specified path does not exist, RemoveAll will return nil.
 //
 // Note that RemoveAll will not modify permissions on parent directory of the
 // provided path, even if it is read only and preventing deletion of the path on
 // POSIX system.
 //
 // Copied from
 // https://go.googlesource.com/go/+/b86e76681366447798c94abb959bb60875bcc856/src/os/path.go#63
 func RemoveAll(path string) error {
 	const isWin = runtime.GOOS == "windows"
 	// Simple case: try removing as if it was a file or empty directory.
 	var err error
 	if isWin {
 		// In theory this call should not be necessary. os.Remove() already
 		// tries to remove the FILE_ATTRIBUTE_READONLY attribute at
 		// https://go.googlesource.com/go/+blame/go1.13/src/os/file_windows.go#296.
 		// In practice this doesn't work in one case, when it is a symlink that
 		// points to a missing file. In this case, ErrNotExist is returned, but
 		// the function call is still needed for the os.Remove() to work below.
 		err = MakePathUserWritable(path, nil)
 	}
 	if err == nil || IsNotExist(err) {
 		// On Windows, invalid symlink is treated as not exist error, but need to
 		// remove that.
 		err = os.Remove(path)
 	}
 	if err == nil || IsNotExist(err) {
 		return nil
 	}

 	// Otherwise, is this a directory we need to recurse into?
 	dir, serr := os.Lstat(path)
 	if serr != nil {
 		if serr, ok := serr.(*os.PathError); ok && (IsNotExist(serr.Err) || serr.Err == syscall.ENOTDIR) {
 			return nil
 		}
 		return serr
 	}
 	if !dir.IsDir() {
 		// Not a directory; return the error from Remove.
 		return err
 	}
 	// Directory.
 	if !isWin {
 		// On POSIX systems, the directory must have write access for its files to
 		// be deleted. Best effort attempt to make it writable.
 		_ = MakePathUserWritable(path, dir)
 	}
 	fd, err := os.Open(path)
 	if err != nil {
 		if IsNotExist(err) {
 			// Race. It was deleted between the Lstat and Open.
 			// Return nil per RemoveAll's docs.
 			return nil
 		}
 		return err
 	}
 	// Remove contents & return first error.
 	err = nil
 	for {
 		if err == nil && (runtime.GOOS == "plan9" || runtime.GOOS == "nacl") {
 			// Reset read offset after removing directory entries.
 			// See golang.org/issue/22572.
 			fd.Seek(0, 0)
 		}
 		names, err1 := fd.Readdirnames(100)
 		for _, name := range names {
 			err1 := RemoveAll(path + string(os.PathSeparator) + name)
 			if err == nil {
 				err = err1
 			}
 		}
 		if err1 == io.EOF {
 			break
 		}
 		// If Readdirnames returned an error, use it.
 		if err == nil {
 			err = err1
 		}
 		if len(names) == 0 {
 			break
 		}
 	}
 	// Close directory, because windows won't remove opened directory.
 	fd.Close()
 	// Remove directory.
 	err1 := os.Remove(path)
 	if err1 == nil || IsNotExist(err1) {
 		return nil
 	}
 	if err == nil {
 		err = err1
 	}
 	return err
 }

 // RenamingRemoveAll opportunistically renames a path first, and then removes it.
 //
 // The advantage over RemoveAll is, if renaming succeeds, lower chance of
 // interference from other writers/readers of the filesystem.
 // If renaming fails, removes the original path via RemoveAll.
 //
 // If renameToDir is given, a new temp directory will be created in it.
 // Else, a new temp directory is placed within the path's parent dir.
 // After this, a file/dir represented by the path is moved into the temp dir.
 //
 // In case of any failures during the temp dir creation or the move,
 // default to RemoveAll of path in place.
 //
 // Returned renamedToPath is the renamed path if renaming succeeded and ""
 // otherwise.
 // Returned error is the one from RemoveAll call.
 func RenamingRemoveAll(path, renameToDir string) (renamedToPath string, err error) {
 	pathParentDir, pathFileOrDir := filepath.Split(filepath.Clean(path))
 	if renameToDir == "" {
 		renameToDir = pathParentDir
 	}
 	renameToDir, err = ioutil.TempDir(renameToDir, ".trash-")
 	if err != nil {
 		err = RemoveAll(path)
 		return
 	}

 	renamedToPath = filepath.Join(renameToDir, pathFileOrDir)
 	if err = os.Rename(path, renamedToPath); err != nil {
 		// delete temp dir we just created and ignore errors -- there is not much we can do.
 		_ = os.Remove(renameToDir)
 		renamedToPath = ""
 		err = RemoveAll(path)
 		return
 	}
 	err = RemoveAll(renamedToPath)
 	return
 }

 // MakeReadOnly recursively iterates through all of the files and directories
 // starting at path and marks them read-only.
 func MakeReadOnly(path string, filter func(string) bool) error {
 	return recursiveChmod(path, filter, func(mode os.FileMode) os.FileMode {
 		return mode & (^os.FileMode(0222))
 	})
 }

 // MakePathUserWritable updates the filesystem metadata on a single file or
 // directory to make it user-writable.
 //
 // fi is optional. If nil, os.Stat will be called on path. Otherwise, fi will
 // be regarded as the results of calling os.Stat on path. This is provided as
 // an optimization, since some filesystem operations automatically yield a
 // FileInfo.
 func MakePathUserWritable(path string, fi os.FileInfo) error {
 	if fi == nil {
 		var err error
 		if fi, err = os.Stat(path); err != nil {
 			return errors.Annotate(err, "failed to Stat path").InternalReason("path(%q)", path).Err()
 		}
 	}

 	// Make user-writable, if it's not already.
 	mode := fi.Mode()
 	if (mode & 0200) == 0 {
 		mode |= 0200
 		if err := os.Chmod(path, mode); err != nil {
 			return errors.Annotate(err, "could not Chmod path").InternalReason("mode(%#o)/path(%q)", mode, path).Err()
 		}
 	}
 	return nil
 }

 func recursiveChmod(path string, filter func(string) bool, chmod func(mode os.FileMode) os.FileMode) error {
 	if filter == nil {
 		filter = func(string) bool { return true }
 	}

 	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return errors.Annotate(err, "").Err()
 		}

 		mode := info.Mode()
 		if (mode.IsRegular() || mode.IsDir()) && filter(path) {
 			if newMode := chmod(mode); newMode != mode {
 				if err := os.Chmod(path, newMode); err != nil {
 					return errors.Annotate(err, "failed to Chmod").InternalReason("path(%q)", path).Err()
 				}
 			}
 		}
 		return nil
 	})
 	if err != nil {
 		return errors.Annotate(err, "").Err()
 	}
 	return nil
 }

 // Copy makes a copy of the file.
 func Copy(outfile, infile string, mode os.FileMode) (err error) {
 	in, err := os.Open(infile)
 	if err != nil {
 		return err
 	}
 	defer func() {
 		if cerr := in.Close(); err == nil {
 			err = cerr
 		}
 	}()

 	out, err := os.OpenFile(outfile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
 	if err != nil {
 		return err
 	}
 	defer func() {
 		if cerr := out.Close(); err == nil {
 			err = cerr
 		}
 	}()

 	_, err = io.Copy(out, in)
 	return err
 }

 // ReadableCopy makes a copy of the file that is readable by everyone.
 func ReadableCopy(outfile, infile string) error {
 	istat, err := os.Stat(infile)
 	if err != nil {
 		return err
 	}

 	return Copy(outfile, infile, addReadMode(istat.Mode()))
 }

 func hardlinkWithFallback(outfile, infile string) error {
 	if err := os.Link(infile, outfile); err == nil {
 		return nil
 	}

 	return ReadableCopy(outfile, infile)
 }

 // HardlinkRecursively efficiently copies a file or directory from src to dst.
 //
 // `src` may be a file, directory, or a symlink to a file or directory.
 // All symlinks are replaced with their targets, so the resulting
 // directory structure in `dst` will never have any symlinks.
 //
 // To increase speed, HardlinkRecursively hardlinks individual files into the
 // (newly created) directory structure if possible.
 func HardlinkRecursively(src, dst string) error {
 	src, stat, err := ResolveSymlink(src)
 	if err != nil {
 		return errors.Annotate(err, "failed to call ResolveSymlink(%s)", src).Err()
 	}

 	if stat.Mode().IsRegular() {
 		return hardlinkWithFallback(dst, src)
 	}

 	if !stat.Mode().IsDir() {
 		return errors.Reason("%s is not a directory: %v", src, stat).Err()
 	}

 	if err := os.MkdirAll(dst, 0775); err != nil {
 		return errors.Annotate(err, "failed to call MkdirAll for %s", dst).Err()
 	}

 	file, err := os.Open(src)
 	if err != nil {
 		return errors.Annotate(err, "failed to Open %s", src).Err()
 	}
 	defer file.Close()

 	for {
 		names, err := file.Readdirnames(100)
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
 			return errors.Annotate(err, "failed to call Readdirnames for %s", src).Err()
 		}

 		for _, name := range names {
 			if err := HardlinkRecursively(filepath.Join(src, name), filepath.Join(dst, name)); err != nil {
 				return errors.Annotate(err, "failed to call HardlinkRecursively(%s, %s)", filepath.Join(src, name), filepath.Join(dst, name)).Err()
 			}

 		}
 	}

 	return nil
 }

 // CreateDirectories creates the directory structure needed by the given list of files.
 func CreateDirectories(baseDirectory string, files []string) error {
 	dirs := make([]string, len(files))
 	for i, file := range files {
 		if filepath.IsAbs(file) {
 			return errors.Reason("file should be relative path: %s", file).Err()
 		}
 		dirs[i] = filepath.Dir(file)
 	}

 	sort.Strings(dirs)

 	for i, dir := range dirs {
 		if dir == "" {
 			continue
 		}
 		if i+1 < len(dirs) && filepath.HasPrefix(dirs[i+1], dir) {
 			continue
 		}
 		dir = filepath.Join(baseDirectory, dir)

 		if err := os.MkdirAll(dir, 0755); err != nil {
 			return errors.Annotate(err, "failed to create directory for %s", dir).Err()
 		}
 	}

 	return nil
 }

 // IsEmptyDir returns whether |dir| is empty or not.
 // This returns error if |dir| is not directory, or find some error during checking.
 func IsEmptyDir(dir string) (bool, error) {
 	d, err := os.Open(dir)
 	if err != nil {
 		return false, errors.Annotate(err, "failed to Open(%s)", dir).Err()
 	}
 	defer d.Close()

 	names, err := d.Readdirnames(1)
 	if len(names) > 0 || err == io.EOF {
 		return len(names) == 0, nil
 	}

 	return false, errors.Annotate(err, "failed to call Readdirnames(1) for %s", dir).Err()
 }

 // IsDir to see whether |path| is a directory.
 // This is just a thin wrapper around os.Stat(...).
 // If this returns True, |path| is a directory.
 // If this returns False with nil err, |path| is not a directory.
 // If this returns non-nil error, failed to determine |path| is a drectory.
 func IsDir(path string) (bool, error) {
 	stat, err := os.Stat(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
 		}
 		return false, err
 	}
 	return stat.IsDir(), nil
 }

 // GetFreeSpace returns the number of free bytes.
 //
 // On POSIX platforms, this returns the free space as visible by the current
 // user. The returned value is what is usable, and it can be lower than the
 // actual free disk space. For example on linux there's by default a 5% that is
 // reserved to the root user.
 func GetFreeSpace(path string) (uint64, error) {
 	return getFreeSpace(path)
 }

 // findPathSeparators finds the index of all PathSeparators in `path` which
 // don't split the Volume.
 //
 // This function is only defined for clean, absolute, paths which end with
 // a path separator.
 //
 // For unix paths, the first returned index will always be 0.
 func findPathSeparators(path string) []int {
 	offset := len(filepath.VolumeName(path))
 	path = path[offset:]

 	ret := make([]int, 0, 10) // 10 is a guess, could be more or less.
 	for {
 		idx := strings.IndexByte(path, os.PathSeparator)
 		if idx == -1 {
 			break
 		}
 		ret = append(ret, offset+idx)
 		offset += idx + 1
 		path = path[idx+1:]
 	}

 	return ret
 }

 // ErrRootSentinel is wrapped and then returned from GetCommonAncestor when it
 // encounters one of the provided rootSentinels.
 var ErrRootSentinel = errors.New("hit root sentinel")

 // GetCommonAncestor returns the smallest path which is the ancestor of all
 // provided paths (which must actually exist on the filesystem).
 //
 // All paths here are converted to absolute paths before calculating the
 // ancestor, and the returned path will also be an absolute path. Note that this
 // doesn't do anything special with symlinks; the caller can resolve them if
 // necessary.
 //
 // This function works correctly on case-insensitive filesystems, or on
 // filesystems with a mix of case-sensitive and case-insensitive paths, but you
 // can get some wild filesystems out there, so this will probably break on
 // exotic setups. Note that the case of the path you get back will be derived
 // from the shortest input path (after making them absolute); this function
 // makes no attempt to "canonicalize" the case of any paths (but may do so
 // accidentally, depending on the operating system). This function does not
 // attempt to resolve symlinks.
 //
 // If a given path points to a file, the file's containing directory will be
 // considered instead (i.e. GetCommonAncestor("a/b.ext") will return the
 // absolute path of "a").
 //
 // `rootSentinels` is a list of sub paths to look for to stop walking up the
 // directory hierarchy. A typical value would be something like
 // []string{".git"}. If one of these is found, this function returns "" with
 // a wrapped ErrRootSentinel. Use errors.Is to identify this.
 //
 // Returns an error if any of the provided paths does not exist.
 // If successful, will return a path ending with PathSeparator.
 //
 // If no paths are prodvided, returns ("", nil)
 func GetCommonAncestor(paths []string, rootSentinels []string) (string, error) {
 	if len(paths) == 0 {
 		return "", nil
 	}

 	const sep = string(os.PathSeparator)

 	type cleanPath struct {
 		// The cleaned, absolute path to a directory which exists.
 		path string
 		// Indexes in `path` for each os.PathSeparator which is a valid split point.
 		// Note that UNC roots on windows may 'skip' PathSeparators (i.e. slashes[0]
 		// may contain multiple PathSeparators).
 		slashes []int
 	}
 	cleanPaths := make([]cleanPath, len(paths))

 	var commonVolume *string

 	// Clean all the paths, make them absolute.
 	// Find all the slashes in the paths.
 	//
 	// Note that a UNC path like '\\host\share\something' would have its first
 	// slash at index 12.
 	// A Unix path like '/host/share/something' would have its first
 	// slash at index 0.
 	for i, path := range paths {
 		if err := AbsPath(&path); err != nil {
 			return "", err
 		}
 		vol := filepath.VolumeName(path)

 		if commonVolume != nil {
 			// note: we check this first to allow testing; otherwise we would need to
 			// run tests on a machine with multiple volumes.
 			if vol != *commonVolume {
 				return "", errors.Reason("provided paths originate on different volumes: path[0]:%q, path[%d]:%q", *commonVolume, i, vol).Err()
 			}
 		} else {
 			commonVolume = &vol
 		}

 		fi, err := os.Lstat(path)
 		if err != nil {
 			return "", errors.Annotate(err, "reading path[%d]: %q", i, path).Err()
 		}
 		if !fi.IsDir() {
 			path = filepath.Dir(path)
 			fi, err := os.Lstat(path)
 			if err != nil {
 				// given that we know that the original `path` exists, this SHOULD be
 				// impossible, but FUSE exists so... idk.
 				return "", errors.Annotate(err, "reading Dir(path[%d]): %q", i, path).Err()
 			}
 			if !fi.IsDir() {
 				// this SHOULD ALSO be impossible...
 				return "", errors.Annotate(err, "path %q could not be resolved to parent dir", path).Err()
 			}
 		}

 		if !strings.HasSuffix(path, sep) {
 			path = path + sep
 		}
 		cleanPaths[i] = cleanPath{
 			path:    path,
 			slashes: findPathSeparators(path),
 		}
 	}

 	// sort by length and then alphabetically
 	sort.Slice(cleanPaths, sortby.Chain{
 		func(i, j int) bool { return len(cleanPaths[i].path) < len(cleanPaths[j].path) },
 		func(i, j int) bool { return cleanPaths[i].path < cleanPaths[j].path },
 	}.Use)

 	candidate := cleanPaths[0]

 	// We want to see if all the slashes in all other candidates line up with the
 	// slashes in `candidate`.
 	//
 	// We are already making some lexical assumptions about the paths here; If we
 	// wanted to discard lexical assumptions we would need to do all permutations
 	// of SameFile checks for every directory combination in all paths, and pick
 	// the lowest one which matched (due to the possibility of e.g. bind mounts).
 	//
 	// However, ain't nobody got time for that.
 	slashesMatch := func(whichSlash int) bool {
 		for _, other := range cleanPaths[1:] {
 			// whichSlash+1 because we want to include everything up to, and
 			// including, whichSlash.
 			for i, slashIdx := range candidate.slashes[:whichSlash+1] {
 				if other.slashes[i] != slashIdx {
 					return false
 				}
 			}
 		}
 		return true
 	}

 	// for each slash in the candidate, see if all cleanPaths at this slash return
 	// true from os.SameFile vs candidate.
 	//
 	// Calling this function implies that all other paths have already been
 	// verified with slashesMatch.
 	//
 	// The first check would avoid comparing "/long/f" vs "/s/long"; Although they
 	// both have a slash at 7, the prefix leading to that doesn't match. See
 	// comment on slashesMatch.
 	trySlash := func(curPath string) (bool, error) {
 		var curFi os.FileInfo
 		for _, other := range cleanPaths[1:] {
 			otherPath := other.path[:len(curPath)]
 			if otherPath == curPath {
 				// exact match, try the next one
 				continue
 			}

 			// ok, try SameFile
 			if curFi == nil {
 				var err error
 				if curFi, err = os.Lstat(curPath); err != nil {
 					return false, err
 				}
 			}
 			otherFi, err := os.Lstat(otherPath)
 			if err != nil {
 				return false, err
 			}
 			if !os.SameFile(curFi, otherFi) {
 				return false, nil
 			}
 		}
 		return true, nil
 	}

 	for whichSlash := len(candidate.slashes) - 1; whichSlash >= 0; whichSlash-- {
 		// curPath includes trailing slash
 		curPath := candidate.path[:candidate.slashes[whichSlash]+1]

 		// check if it's out of bounds
 		for _, sentinel := range rootSentinels {
 			sentinelPath := filepath.Join(curPath, sentinel)
 			if _, err := os.Lstat(sentinelPath); err == nil {
 				return "", errors.Annotate(ErrRootSentinel, "%q", sentinelPath).Err()
 			} else if !os.IsNotExist(err) {
 				return "", errors.Annotate(err, "failed to read root sentinel %q", sentinelPath).Err()
 			}
 		}

 		// Check to see if all other candidates have the same slash structure; i.e.
 		// the first `whichSlash` number of slashes line up across all paths. This
 		// avoids doing stats to see if "/a/path/" and "/path/a/" are the same file.
 		if !slashesMatch(whichSlash) {
 			continue
 		}

 		// all slashes match, let's see if the targeted files are the same (either
 		// lexically equivalent or are os.SameFile)
 		ok, err := trySlash(curPath)
 		if err != nil {
 			return "", err
 		}
 		if ok {
 			return curPath, nil
 		}
 	}

 	// Should never get here:
 	//   * We are on unix and so whichSlash==0 is always the path "/". Either we
 	//     found the root sentinel above, or returned "/" successfully.
 	//   * We are on !unix and so whichSlash==0 is always a Volume (e.g. "C:\\").
 	//     However we already checked when resolving `paths` at the top that all
 	//     the cleanPaths share the SAME volume. Thus we should have either found
 	//     the root sentinel or returned this same volume (i.e. `commonVolume`).
 	panic("impossible")
 }
	// Copyright 2017 The LUCI Authors.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package filesystem

	import (
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"syscall"
	"time"

	"go.chromium.org/luci/common/data/sortby"
	"go.chromium.org/luci/common/errors"
	)

	// IsNotExist calls os.IsNotExist on the unwrapped err.
	func IsNotExist(err error) bool { return os.IsNotExist(errors.Unwrap(err)) }

	// MakeDirs is a convenience wrapper around os.MkdirAll that applies a 0755
	// mask to all created directories.
	func MakeDirs(path string) error {
	if err := os.MkdirAll(path, 0755); err != nil {
	return errors.Annotate(err, "").Err()
	}
	return nil
	}

	// AbsPath is a convenience wrapper around filepath.Abs that accepts a string
	// pointer, base, and updates it on successful resolution.
	func AbsPath(base *string) error {
	v, err := filepath.Abs(*base)
	if err != nil {
	return errors.Annotate(err, "unable to resolve absolute path").
	InternalReason("base(%q)", *base).Err()
	}
	*base = v
	return nil
	}

	// Touch creates a new, empty file at the specified path.
	//
	// If when is zero-value, time.Now will be used.
	func Touch(path string, when time.Time, mode os.FileMode) error {
	// Try and create a file at the target path.
	fd, err := os.OpenFile(path, (os.O_CREATE \| os.O_RDWR \| os.O_EXCL), mode)
	if err == nil {
	if err := fd.Close(); err != nil {
	return errors.Annotate(err, "failed to close new file").Err()
	}
	if when.IsZero() {
	// If "now" was specified, and we created a new file, then its times will
	// be now by default.
	return nil
	}
	}

	// Couldn't create a new file. Either it exists already, it is a directory,
	// or there was an OS-level failure. Since we can't really distinguish
	// between these cases, try Chtimes (update timestamp) and error
	// if this fails.
	if when.IsZero() {
	when = time.Now()
	}
	if err := os.Chtimes(path, when, when); err != nil {
	return errors.Annotate(err, "failed to Chtimes").InternalReason("path(%q)", path).Err()
	}

	return nil
	}

	// RemoveAll is a fork of os.RemoveAll that attempts to deal with read only
	// files and directories by modifying permissions as necessary.
	//
	// If the specified path does not exist, RemoveAll will return nil.
	//
	// Note that RemoveAll will not modify permissions on parent directory of the
	// provided path, even if it is read only and preventing deletion of the path on
	// POSIX system.
	//
	// Copied from
	// https://go.googlesource.com/go/+/b86e76681366447798c94abb959bb60875bcc856/src/os/path.go#63
	func RemoveAll(path string) error {
	const isWin = runtime.GOOS == "windows"
	// Simple case: try removing as if it was a file or empty directory.
	var err error
	if isWin {
	// In theory this call should not be necessary. os.Remove() already
	// tries to remove the FILE_ATTRIBUTE_READONLY attribute at
	// https://go.googlesource.com/go/+blame/go1.13/src/os/file_windows.go#296.
	// In practice this doesn't work in one case, when it is a symlink that
	// points to a missing file. In this case, ErrNotExist is returned, but
	// the function call is still needed for the os.Remove() to work below.
	err = MakePathUserWritable(path, nil)
	}
	if err == nil \|\| IsNotExist(err) {
	// On Windows, invalid symlink is treated as not exist error, but need to
	// remove that.
	err = os.Remove(path)
	}
	if err == nil \|\| IsNotExist(err) {
	return nil
	}

	// Otherwise, is this a directory we need to recurse into?
	dir, serr := os.Lstat(path)
	if serr != nil {
	if serr, ok := serr.(*os.PathError); ok && (IsNotExist(serr.Err) \|\| serr.Err == syscall.ENOTDIR) {
	return nil
	}
	return serr
	}
	if !dir.IsDir() {
	// Not a directory; return the error from Remove.
	return err
	}
	// Directory.
	if !isWin {
	// On POSIX systems, the directory must have write access for its files to
	// be deleted. Best effort attempt to make it writable.
	_ = MakePathUserWritable(path, dir)
	}
	fd, err := os.Open(path)
	if err != nil {
	if IsNotExist(err) {
	// Race. It was deleted between the Lstat and Open.
	// Return nil per RemoveAll's docs.
	return nil
	}
	return err
	}
	// Remove contents & return first error.
	err = nil
	for {
	if err == nil && (runtime.GOOS == "plan9" \|\| runtime.GOOS == "nacl") {
	// Reset read offset after removing directory entries.
	// See golang.org/issue/22572.
	fd.Seek(0, 0)
	}
	names, err1 := fd.Readdirnames(100)
	for _, name := range names {
	err1 := RemoveAll(path + string(os.PathSeparator) + name)
	if err == nil {
	err = err1
	}
	}
	if err1 == io.EOF {
	break
	}
	// If Readdirnames returned an error, use it.
	if err == nil {
	err = err1
	}
	if len(names) == 0 {
	break
	}
	}
	// Close directory, because windows won't remove opened directory.
	fd.Close()
	// Remove directory.
	err1 := os.Remove(path)
	if err1 == nil \|\| IsNotExist(err1) {
	return nil
	}
	if err == nil {
	err = err1
	}
	return err
	}

	// RenamingRemoveAll opportunistically renames a path first, and then removes it.
	//
	// The advantage over RemoveAll is, if renaming succeeds, lower chance of
	// interference from other writers/readers of the filesystem.
	// If renaming fails, removes the original path via RemoveAll.
	//
	// If renameToDir is given, a new temp directory will be created in it.
	// Else, a new temp directory is placed within the path's parent dir.
	// After this, a file/dir represented by the path is moved into the temp dir.
	//
	// In case of any failures during the temp dir creation or the move,
	// default to RemoveAll of path in place.
	//
	// Returned renamedToPath is the renamed path if renaming succeeded and ""
	// otherwise.
	// Returned error is the one from RemoveAll call.
	func RenamingRemoveAll(path, renameToDir string) (renamedToPath string, err error) {
	pathParentDir, pathFileOrDir := filepath.Split(filepath.Clean(path))
	if renameToDir == "" {
	renameToDir = pathParentDir
	}
	renameToDir, err = ioutil.TempDir(renameToDir, ".trash-")
	if err != nil {
	err = RemoveAll(path)
	return
	}

	renamedToPath = filepath.Join(renameToDir, pathFileOrDir)
	if err = os.Rename(path, renamedToPath); err != nil {
	// delete temp dir we just created and ignore errors -- there is not much we can do.
	_ = os.Remove(renameToDir)
	renamedToPath = ""
	err = RemoveAll(path)
	return
	}
	err = RemoveAll(renamedToPath)
	return
	}

	// MakeReadOnly recursively iterates through all of the files and directories
	// starting at path and marks them read-only.
	func MakeReadOnly(path string, filter func(string) bool) error {
	return recursiveChmod(path, filter, func(mode os.FileMode) os.FileMode {
	return mode & (^os.FileMode(0222))
	})
	}

	// MakePathUserWritable updates the filesystem metadata on a single file or
	// directory to make it user-writable.
	//
	// fi is optional. If nil, os.Stat will be called on path. Otherwise, fi will
	// be regarded as the results of calling os.Stat on path. This is provided as
	// an optimization, since some filesystem operations automatically yield a
	// FileInfo.
	func MakePathUserWritable(path string, fi os.FileInfo) error {
	if fi == nil {
	var err error
	if fi, err = os.Stat(path); err != nil {
	return errors.Annotate(err, "failed to Stat path").InternalReason("path(%q)", path).Err()
	}
	}

	// Make user-writable, if it's not already.
	mode := fi.Mode()
	if (mode & 0200) == 0 {
	mode \|= 0200
	if err := os.Chmod(path, mode); err != nil {
	return errors.Annotate(err, "could not Chmod path").InternalReason("mode(%#o)/path(%q)", mode, path).Err()
	}
	}
	return nil
	}

	func recursiveChmod(path string, filter func(string) bool, chmod func(mode os.FileMode) os.FileMode) error {
	if filter == nil {
	filter = func(string) bool { return true }
	}

	err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
	if err != nil {
	return errors.Annotate(err, "").Err()
	}

	mode := info.Mode()
	if (mode.IsRegular() \|\| mode.IsDir()) && filter(path) {
	if newMode := chmod(mode); newMode != mode {
	if err := os.Chmod(path, newMode); err != nil {
	return errors.Annotate(err, "failed to Chmod").InternalReason("path(%q)", path).Err()
	}
	}
	}
	return nil
	})
	if err != nil {
	return errors.Annotate(err, "").Err()
	}
	return nil
	}

	// Copy makes a copy of the file.
	func Copy(outfile, infile string, mode os.FileMode) (err error) {
	in, err := os.Open(infile)
	if err != nil {
	return err
	}
	defer func() {
	if cerr := in.Close(); err == nil {
	err = cerr
	}
	}()

	out, err := os.OpenFile(outfile, os.O_CREATE\|os.O_EXCL\|os.O_WRONLY, mode)
	if err != nil {
	return err
	}
	defer func() {
	if cerr := out.Close(); err == nil {
	err = cerr
	}
	}()

	_, err = io.Copy(out, in)
	return err
	}

	// ReadableCopy makes a copy of the file that is readable by everyone.
	func ReadableCopy(outfile, infile string) error {
	istat, err := os.Stat(infile)
	if err != nil {
	return err
	}

	return Copy(outfile, infile, addReadMode(istat.Mode()))
	}

	func hardlinkWithFallback(outfile, infile string) error {
	if err := os.Link(infile, outfile); err == nil {
	return nil
	}

	return ReadableCopy(outfile, infile)
	}

	// HardlinkRecursively efficiently copies a file or directory from src to dst.
	//
	// `src` may be a file, directory, or a symlink to a file or directory.
	// All symlinks are replaced with their targets, so the resulting
	// directory structure in `dst` will never have any symlinks.
	//
	// To increase speed, HardlinkRecursively hardlinks individual files into the
	// (newly created) directory structure if possible.
	func HardlinkRecursively(src, dst string) error {
	src, stat, err := ResolveSymlink(src)
	if err != nil {
	return errors.Annotate(err, "failed to call ResolveSymlink(%s)", src).Err()
	}

	if stat.Mode().IsRegular() {
	return hardlinkWithFallback(dst, src)
	}

	if !stat.Mode().IsDir() {
	return errors.Reason("%s is not a directory: %v", src, stat).Err()
	}

	if err := os.MkdirAll(dst, 0775); err != nil {
	return errors.Annotate(err, "failed to call MkdirAll for %s", dst).Err()
	}

	file, err := os.Open(src)
	if err != nil {
	return errors.Annotate(err, "failed to Open %s", src).Err()
	}
	defer file.Close()

	for {
	names, err := file.Readdirnames(100)
	if err == io.EOF {
	break
	}
	if err != nil {
	return errors.Annotate(err, "failed to call Readdirnames for %s", src).Err()
	}

	for _, name := range names {
	if err := HardlinkRecursively(filepath.Join(src, name), filepath.Join(dst, name)); err != nil {
	return errors.Annotate(err, "failed to call HardlinkRecursively(%s, %s)", filepath.Join(src, name), filepath.Join(dst, name)).Err()
	}

	}
	}

	return nil
	}

	// CreateDirectories creates the directory structure needed by the given list of files.
	func CreateDirectories(baseDirectory string, files []string) error {
	dirs := make([]string, len(files))
	for i, file := range files {
	if filepath.IsAbs(file) {
	return errors.Reason("file should be relative path: %s", file).Err()
	}
	dirs[i] = filepath.Dir(file)
	}

	sort.Strings(dirs)

	for i, dir := range dirs {
	if dir == "" {
	continue
	}
	if i+1 < len(dirs) && filepath.HasPrefix(dirs[i+1], dir) {
	continue
	}
	dir = filepath.Join(baseDirectory, dir)

	if err := os.MkdirAll(dir, 0755); err != nil {
	return errors.Annotate(err, "failed to create directory for %s", dir).Err()
	}
	}

	return nil
	}

	// IsEmptyDir returns whether \|dir\| is empty or not.
	// This returns error if \|dir\| is not directory, or find some error during checking.
	func IsEmptyDir(dir string) (bool, error) {
	d, err := os.Open(dir)
	if err != nil {
	return false, errors.Annotate(err, "failed to Open(%s)", dir).Err()
	}
	defer d.Close()

	names, err := d.Readdirnames(1)
	if len(names) > 0 \|\| err == io.EOF {
	return len(names) == 0, nil
	}

	return false, errors.Annotate(err, "failed to call Readdirnames(1) for %s", dir).Err()
	}

	// IsDir to see whether \|path\| is a directory.
	// This is just a thin wrapper around os.Stat(...).
	// If this returns True, \|path\| is a directory.
	// If this returns False with nil err, \|path\| is not a directory.
	// If this returns non-nil error, failed to determine \|path\| is a drectory.
	func IsDir(path string) (bool, error) {
	stat, err := os.Stat(path)
	if err != nil {
	if os.IsNotExist(err) {
	return false, nil
	}
	return false, err
	}
	return stat.IsDir(), nil
	}

	// GetFreeSpace returns the number of free bytes.
	//
	// On POSIX platforms, this returns the free space as visible by the current
	// user. The returned value is what is usable, and it can be lower than the
	// actual free disk space. For example on linux there's by default a 5% that is
	// reserved to the root user.
	func GetFreeSpace(path string) (uint64, error) {
	return getFreeSpace(path)
	}

	// findPathSeparators finds the index of all PathSeparators in `path` which
	// don't split the Volume.
	//
	// This function is only defined for clean, absolute, paths which end with
	// a path separator.
	//
	// For unix paths, the first returned index will always be 0.
	func findPathSeparators(path string) []int {
	offset := len(filepath.VolumeName(path))
	path = path[offset:]

	ret := make([]int, 0, 10) // 10 is a guess, could be more or less.
	for {
	idx := strings.IndexByte(path, os.PathSeparator)
	if idx == -1 {
	break
	}
	ret = append(ret, offset+idx)
	offset += idx + 1
	path = path[idx+1:]
	}

	return ret
	}

	// ErrRootSentinel is wrapped and then returned from GetCommonAncestor when it
	// encounters one of the provided rootSentinels.
	var ErrRootSentinel = errors.New("hit root sentinel")

	// GetCommonAncestor returns the smallest path which is the ancestor of all
	// provided paths (which must actually exist on the filesystem).
	//
	// All paths here are converted to absolute paths before calculating the
	// ancestor, and the returned path will also be an absolute path. Note that this
	// doesn't do anything special with symlinks; the caller can resolve them if
	// necessary.
	//
	// This function works correctly on case-insensitive filesystems, or on
	// filesystems with a mix of case-sensitive and case-insensitive paths, but you
	// can get some wild filesystems out there, so this will probably break on
	// exotic setups. Note that the case of the path you get back will be derived
	// from the shortest input path (after making them absolute); this function
	// makes no attempt to "canonicalize" the case of any paths (but may do so
	// accidentally, depending on the operating system). This function does not
	// attempt to resolve symlinks.
	//
	// If a given path points to a file, the file's containing directory will be
	// considered instead (i.e. GetCommonAncestor("a/b.ext") will return the
	// absolute path of "a").
	//
	// `rootSentinels` is a list of sub paths to look for to stop walking up the
	// directory hierarchy. A typical value would be something like
	// []string{".git"}. If one of these is found, this function returns "" with
	// a wrapped ErrRootSentinel. Use errors.Is to identify this.
	//
	// Returns an error if any of the provided paths does not exist.
	// If successful, will return a path ending with PathSeparator.
	//
	// If no paths are prodvided, returns ("", nil)
	func GetCommonAncestor(paths []string, rootSentinels []string) (string, error) {
	if len(paths) == 0 {
	return "", nil
	}

	const sep = string(os.PathSeparator)

	type cleanPath struct {
	// The cleaned, absolute path to a directory which exists.
	path string
	// Indexes in `path` for each os.PathSeparator which is a valid split point.
	// Note that UNC roots on windows may 'skip' PathSeparators (i.e. slashes[0]
	// may contain multiple PathSeparators).
	slashes []int
	}
	cleanPaths := make([]cleanPath, len(paths))

	var commonVolume *string

	// Clean all the paths, make them absolute.
	// Find all the slashes in the paths.
	//
	// Note that a UNC path like '\\host\share\something' would have its first
	// slash at index 12.
	// A Unix path like '/host/share/something' would have its first
	// slash at index 0.
	for i, path := range paths {
	if err := AbsPath(&path); err != nil {
	return "", err
	}
	vol := filepath.VolumeName(path)

	if commonVolume != nil {
	// note: we check this first to allow testing; otherwise we would need to
	// run tests on a machine with multiple volumes.
	if vol != *commonVolume {
	return "", errors.Reason("provided paths originate on different volumes: path[0]:%q, path[%d]:%q", *commonVolume, i, vol).Err()
	}
	} else {
	commonVolume = &vol
	}

	fi, err := os.Lstat(path)
	if err != nil {
	return "", errors.Annotate(err, "reading path[%d]: %q", i, path).Err()
	}
	if !fi.IsDir() {
	path = filepath.Dir(path)
	fi, err := os.Lstat(path)
	if err != nil {
	// given that we know that the original `path` exists, this SHOULD be
	// impossible, but FUSE exists so... idk.
	return "", errors.Annotate(err, "reading Dir(path[%d]): %q", i, path).Err()
	}
	if !fi.IsDir() {
	// this SHOULD ALSO be impossible...
	return "", errors.Annotate(err, "path %q could not be resolved to parent dir", path).Err()
	}
	}

	if !strings.HasSuffix(path, sep) {
	path = path + sep
	}
	cleanPaths[i] = cleanPath{
	path: path,
	slashes: findPathSeparators(path),
	}
	}

	// sort by length and then alphabetically
	sort.Slice(cleanPaths, sortby.Chain{
	func(i, j int) bool { return len(cleanPaths[i].path) < len(cleanPaths[j].path) },
	func(i, j int) bool { return cleanPaths[i].path < cleanPaths[j].path },
	}.Use)

	candidate := cleanPaths[0]

	// We want to see if all the slashes in all other candidates line up with the
	// slashes in `candidate`.
	//
	// We are already making some lexical assumptions about the paths here; If we
	// wanted to discard lexical assumptions we would need to do all permutations
	// of SameFile checks for every directory combination in all paths, and pick
	// the lowest one which matched (due to the possibility of e.g. bind mounts).
	//
	// However, ain't nobody got time for that.
	slashesMatch := func(whichSlash int) bool {
	for _, other := range cleanPaths[1:] {
	// whichSlash+1 because we want to include everything up to, and
	// including, whichSlash.
	for i, slashIdx := range candidate.slashes[:whichSlash+1] {
	if other.slashes[i] != slashIdx {
	return false
	}
	}
	}
	return true
	}

	// for each slash in the candidate, see if all cleanPaths at this slash return
	// true from os.SameFile vs candidate.
	//
	// Calling this function implies that all other paths have already been
	// verified with slashesMatch.
	//
	// The first check would avoid comparing "/long/f" vs "/s/long"; Although they
	// both have a slash at 7, the prefix leading to that doesn't match. See
	// comment on slashesMatch.
	trySlash := func(curPath string) (bool, error) {
	var curFi os.FileInfo
	for _, other := range cleanPaths[1:] {
	otherPath := other.path[:len(curPath)]
	if otherPath == curPath {
	// exact match, try the next one
	continue
	}

	// ok, try SameFile
	if curFi == nil {
	var err error
	if curFi, err = os.Lstat(curPath); err != nil {
	return false, err
	}
	}
	otherFi, err := os.Lstat(otherPath)
	if err != nil {
	return false, err
	}
	if !os.SameFile(curFi, otherFi) {
	return false, nil
	}
	}
	return true, nil
	}

	for whichSlash := len(candidate.slashes) - 1; whichSlash >= 0; whichSlash-- {
	// curPath includes trailing slash
	curPath := candidate.path[:candidate.slashes[whichSlash]+1]

	// check if it's out of bounds
	for _, sentinel := range rootSentinels {
	sentinelPath := filepath.Join(curPath, sentinel)
	if _, err := os.Lstat(sentinelPath); err == nil {
	return "", errors.Annotate(ErrRootSentinel, "%q", sentinelPath).Err()
	} else if !os.IsNotExist(err) {
	return "", errors.Annotate(err, "failed to read root sentinel %q", sentinelPath).Err()
	}
	}

	// Check to see if all other candidates have the same slash structure; i.e.
	// the first `whichSlash` number of slashes line up across all paths. This
	// avoids doing stats to see if "/a/path/" and "/path/a/" are the same file.
	if !slashesMatch(whichSlash) {
	continue
	}

	// all slashes match, let's see if the targeted files are the same (either
	// lexically equivalent or are os.SameFile)
	ok, err := trySlash(curPath)
	if err != nil {
	return "", err
	}
	if ok {
	return curPath, nil
	}
	}

	// Should never get here:
	// * We are on unix and so whichSlash==0 is always the path "/". Either we
	// found the root sentinel above, or returned "/" successfully.
	// * We are on !unix and so whichSlash==0 is always a Volume (e.g. "C:\\").
	// However we already checked when resolving `paths` at the top that all
	// the cleanPaths share the SAME volume. Thus we should have either found
	// the root sentinel or returned this same volume (i.e. `commonVolume`).
	panic("impossible")
	}