remoteexec/inputroot.go - infra/goma/server - Git at Google

 // Copyright 2018 The Goma Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package remoteexec

 import (
 	"errors"
 	"fmt"
 	"strings"

 	"go.chromium.org/goma/server/command/descriptor/posixpath"
 	"go.chromium.org/goma/server/command/descriptor/winpath"
 	gomapb "go.chromium.org/goma/server/proto/api"
 )

 func samePathElement(filepath clientFilePath, a, b []string) []string {
 	for i, p := range a {
 		if i >= len(b) {
 			return a[:i]
 		}
 		switch filepath.(type) {
 		case winpath.FilePath:
 			if strings.ToLower(p) != strings.ToLower(b[i]) {
 				return a[:i]
 			}
 		default:
 			if p != b[i] {
 				return a[:i]
 			}
 		}
 	}
 	return a
 }

 func commonDir(filepath clientFilePath, paths []string) string {
 	if len(paths) == 0 {
 		return ""
 	}
 	path := filepath.SplitElem(paths[0])
 	for _, p := range paths[1:] {
 		path = samePathElement(filepath, path, filepath.SplitElem(p))
 	}
 	return filepath.Join(path...)
 }

 // argv0InInputRoot checks argv0 should be in input root or not.
 // when argv0 is /usr/bin/*, it might use
 // the command in platform container image, so not
 // considered it as part of input root.
 // TODO: non-linux platforms?
 func argv0InInputRoot(argv0 string) bool {
 	for _, dir := range []string{"/bin/", "/sbin/", "/usr/bin/", "/usr/sbin/"} {
 		if strings.HasPrefix(argv0, dir) {
 			return false
 		}
 	}
 	return true
 }

 func sysrootDir(args []string) (string, bool) {
 	// https://github.com/llvm/llvm-project/blob/1d99472875100b230bac2d9ea70b5cd4b45e788b/clang/include/clang/Driver/Options.td
 	//  def isysroot : JoinedOrSeparate<["-"], "isysroot">, Group<clang_i_Group>, Flags<[CC1Option]>,
 	//     HelpText<"Set the system root directory (usually /)">, MetaVarName<"<dir>">;
 	//  def _sysroot_EQ : Joined<["--"], "sysroot=">;
 	//  def _sysroot : Separate<["--"], "sysroot">, Alias<_sysroot_EQ>;
 	var sysrootDir, isysrootDir string
 	var sysrootFlag, isysrootFlag bool
 	// https://releases.llvm.org/10.0.0/tools/clang/docs/DiagnosticsReference.html#wmissing-sysroot
 	// enabled by default
 	need := true
 	for _, arg := range args {
 		if sysrootFlag {
 			// separated, next arg of --sysroot
 			sysrootDir = arg
 			sysrootFlag = false
 			continue
 		}
 		if isysrootFlag {
 			// separated, next arg of -isysroot
 			isysrootDir = arg
 			isysrootFlag = false
 			continue
 		}
 		switch arg {
 		case "-Wall", "-Wmissing-sysroot":
 			need = true
 		case "-Wno-missing-sysroot":
 			need = false
 		case "-isysroot":
 			isysrootFlag = true
 		case "--sysroot":
 			sysrootFlag = true
 		default:
 			// handled joined.
 			switch {
 			case strings.HasPrefix(arg, "-isysroot"):
 				isysrootDir = strings.TrimPrefix(arg, "-isysroot")
 			case strings.HasPrefix(arg, "--sysroot="):
 				sysrootDir = strings.TrimPrefix(arg, "--sysroot=")
 			}
 		}
 	}
 	// https://gcc.gnu.org/onlinedocs/gcc/Directory-Options.html
 	// If you use both this option and the -isysroot option, then
 	// the --sysroot option applies to libraries, but the
 	// -isysroot option applies to header files.
 	if isysrootDir != "" {
 		return isysrootDir, need
 	}
 	return sysrootDir, need
 }

 func execPaths(filepath clientFilePath, req *gomapb.ExecReq, argv0 string) ([]string, error) {
 	cwd := filepath.Clean(req.GetCwd())
 	if !filepath.IsAbs(cwd) {
 		return nil, fmt.Errorf("cwd is not abs path: %s", cwd)
 	}
 	paths := []string{cwd}
 	for _, input := range req.Input {
 		filename := input.GetFilename()
 		if !filepath.IsAbs(filename) {
 			filename = filepath.Join(cwd, filename)
 		}
 		paths = append(paths, filepath.Clean(filename))
 	}
 	if argv0InInputRoot(argv0) {
 		if !filepath.IsAbs(argv0) {
 			argv0 = filepath.Join(cwd, argv0)
 		}
 		paths = append(paths, filepath.Clean(argv0))
 	}
 	sysrootDir, need := sysrootDir(req.Arg)
 	// if missing sysroot is allowed, no need to ensure it
 	if need && sysrootDir != "" {
 		// if request not allows missing sysroot, we
 		// need to create the directory even if no input files
 		// in dir are used. b/150421328

 		// allow relative path only.
 		// if dir is absolute, request would require
 		// InputRootAbsolutePath.
 		// TODO: cwdAgnosticReq should check input file path too.
 		// http://b/167342635
 		// http://crbug.com/1123938
 		if !filepath.IsAbs(sysrootDir) {
 			dir := filepath.Join(cwd, sysrootDir)
 			paths = append(paths, filepath.Clean(dir))
 		}
 	}
 	for _, output := range req.ExpectedOutputFiles {
 		if !filepath.IsAbs(output) {
 			output = filepath.Join(cwd, output)
 		}
 		paths = append(paths, filepath.Clean(output))
 	}
 	for _, output := range req.ExpectedOutputDirs {
 		if !filepath.IsAbs(output) {
 			output = filepath.Join(cwd, output)
 		}
 		paths = append(paths, filepath.Clean(output))
 	}
 	return paths, nil
 }

 // validCommonDir takes a path `dir` generated by commonDir() and returns
 // true if it represents a valid common path.
 func validCommonDir(filepath clientFilePath, dir string) bool {
 	if dir == "" {
 		return false
 	}
 	switch filepath.(type) {
 	case posixpath.FilePath:
 		if dir == "/" {
 			return false
 		}
 	case winpath.FilePath:
 		// The drive should be considered as root on Win.
 		// e.g. c:\ will return false.
 		if len(dir) == 3 && dir[1:] == `:\` {
 			return false
 		}

 	default:
 		// unknown filepath?
 		return true
 	}

 	return true
 }

 // needChroot returns true if chroot is needed to run the task with
 // given common `dir`
 func needChroot(filepath clientFilePath, dir string) bool {
 	return !validCommonDir(filepath, dir)
 }

 // getPathsWithNoCommonDir returns two or more paths from the input paths that
 // don't have a common dir above the root directory. If there is a common dir,
 // then returns an empty array. Does not guarantee any kind of order.
 func getPathsWithNoCommonDir(filepath clientFilePath, paths []string) []string {
 	if len(paths) < 2 {
 		return nil
 	}

 	p0 := paths[0]
 	// If there are any pairs of paths in `paths` that have no common directories,
 	// then there should be a mismatch between paths[0] and paths[i] for some i > 0.
 	for _, p1 := range paths[1:] {
 		pair := []string{p0, p1}
 		dir := commonDir(filepath, pair)
 		if !validCommonDir(filepath, dir) {
 			return pair
 		}
 	}
 	return nil
 }

 func checkExecRoot(filepath clientFilePath, dir string) error {
 	switch filepath.(type) {
 	case posixpath.FilePath:
 		// if dir covers these paths, command (e.g. clang) won't
 		// work because required *.so etc would not be accessible.
 		for _, p := range []string{
 			"/lib/x86_64-linux-gnu/",
 			"/usr/lib/x86_64-linux-gnu/",
 			"/lib64/",
 		} {
 			if strings.HasPrefix(p, dir+"/") {
 				return fmt.Errorf("bad input root: %s", dir)
 			}
 		}
 		return nil

 	case winpath.FilePath:
 		// TODO: check for win path.
 		return nil

 	default:
 		// unknown filepath?
 		return nil
 	}
 }

 // deriveExecRoot returns common root of paths.
 // if execRootDir is not empty, use it as root of paths.
 // If second return value is true, chroot must be used.  It become true only
 // if `allowChroot` is true and common input root is "/".
 func deriveExecRoot(filepath clientFilePath, paths []string, allowChroot bool, execRootDir string) (string, bool, error) {
 	if execRootDir != "" {
 		return execRootDir, execRootDir == "/" && allowChroot, nil
 	}
 	root := commonDir(filepath, paths)
 	if needChroot(filepath, root) && allowChroot {
 		switch filepath.(type) {
 		// TODO: support non-posix platform
 		case posixpath.FilePath:
 			return "/", true, nil
 		}
 	}
 	if !validCommonDir(filepath, root) {
 		pair := getPathsWithNoCommonDir(filepath, paths)
 		return "", false, fmt.Errorf("no common paths in inputs: %v", pair)
 	}
 	err := checkExecRoot(filepath, root)
 	if err != nil {
 		return "", false, err
 	}
 	return root, false, nil
 }

 var errOutOfRoot = errors.New("out of root")

 // rootRel returns relative path from rootDir for fname,
 // which is relative path from cwd, or absolute path.
 // cwd and rootDir should be clean path.
 func rootRel(filepath clientFilePath, fname, cwd, rootDir string) (string, error) {
 	if filepath == nil {
 		return "", errors.New("rootRel: client filepath unknown")
 	}
 	if !filepath.IsAbs(fname) {
 		fname = filepath.Join(cwd, fname)
 	}
 	// filepath.Rel cleans paths, so we can't use it here.
 	// suppose rootdir and cwd are clean path, and
 	// cwd is under rootdir.
 	rootElems := filepath.SplitElem(rootDir)
 	fileElems := filepath.SplitElem(fname)

 	if len(fileElems) < len(rootElems) {
 		return "", errOutOfRoot
 	}
 	commonElems := samePathElement(filepath, rootElems, fileElems)
 	if len(commonElems) != len(rootElems) {
 		return "", errOutOfRoot
 	}
 	fileElems = fileElems[len(rootElems):]
 	relname := filepath.Join(fileElems...)
 	fileElems = filepath.SplitElem(filepath.Clean(relname))
 	if len(fileElems) > 0 && fileElems[0] == ".." {
 		return "", errOutOfRoot
 	}
 	return relname, nil
 }

 // hasPrefixDir returns true if p has prefix as a directory name.
 // Note: it is case sensitive.
 // TODO: support non-posix platforms.
 func hasPrefixDir(p, prefix string) bool {
 	prefix = strings.TrimRight(prefix, "/")
 	if !strings.HasPrefix(p, prefix) {
 		return false
 	}
 	if len(p) == len(prefix) {
 		return true
 	}
 	return p[len(prefix)] == '/'
 }
	// Copyright 2018 The Goma Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package remoteexec

	import (
	"errors"
	"fmt"
	"strings"

	"go.chromium.org/goma/server/command/descriptor/posixpath"
	"go.chromium.org/goma/server/command/descriptor/winpath"
	gomapb "go.chromium.org/goma/server/proto/api"
	)

	func samePathElement(filepath clientFilePath, a, b []string) []string {
	for i, p := range a {
	if i >= len(b) {
	return a[:i]
	}
	switch filepath.(type) {
	case winpath.FilePath:
	if strings.ToLower(p) != strings.ToLower(b[i]) {
	return a[:i]
	}
	default:
	if p != b[i] {
	return a[:i]
	}
	}
	}
	return a
	}

	func commonDir(filepath clientFilePath, paths []string) string {
	if len(paths) == 0 {
	return ""
	}
	path := filepath.SplitElem(paths[0])
	for _, p := range paths[1:] {
	path = samePathElement(filepath, path, filepath.SplitElem(p))
	}
	return filepath.Join(path...)
	}

	// argv0InInputRoot checks argv0 should be in input root or not.
	// when argv0 is /usr/bin/*, it might use
	// the command in platform container image, so not
	// considered it as part of input root.
	// TODO: non-linux platforms?
	func argv0InInputRoot(argv0 string) bool {
	for _, dir := range []string{"/bin/", "/sbin/", "/usr/bin/", "/usr/sbin/"} {
	if strings.HasPrefix(argv0, dir) {
	return false
	}
	}
	return true
	}

	func sysrootDir(args []string) (string, bool) {
	// https://github.com/llvm/llvm-project/blob/1d99472875100b230bac2d9ea70b5cd4b45e788b/clang/include/clang/Driver/Options.td
	// def isysroot : JoinedOrSeparate<["-"], "isysroot">, Group<clang_i_Group>, Flags<[CC1Option]>,
	// HelpText<"Set the system root directory (usually /)">, MetaVarName<"<dir>">;
	// def _sysroot_EQ : Joined<["--"], "sysroot=">;
	// def _sysroot : Separate<["--"], "sysroot">, Alias<_sysroot_EQ>;
	var sysrootDir, isysrootDir string
	var sysrootFlag, isysrootFlag bool
	// https://releases.llvm.org/10.0.0/tools/clang/docs/DiagnosticsReference.html#wmissing-sysroot
	// enabled by default
	need := true
	for _, arg := range args {
	if sysrootFlag {
	// separated, next arg of --sysroot
	sysrootDir = arg
	sysrootFlag = false
	continue
	}
	if isysrootFlag {
	// separated, next arg of -isysroot
	isysrootDir = arg
	isysrootFlag = false
	continue
	}
	switch arg {
	case "-Wall", "-Wmissing-sysroot":
	need = true
	case "-Wno-missing-sysroot":
	need = false
	case "-isysroot":
	isysrootFlag = true
	case "--sysroot":
	sysrootFlag = true
	default:
	// handled joined.
	switch {
	case strings.HasPrefix(arg, "-isysroot"):
	isysrootDir = strings.TrimPrefix(arg, "-isysroot")
	case strings.HasPrefix(arg, "--sysroot="):
	sysrootDir = strings.TrimPrefix(arg, "--sysroot=")
	}
	}
	}
	// https://gcc.gnu.org/onlinedocs/gcc/Directory-Options.html
	// If you use both this option and the -isysroot option, then
	// the --sysroot option applies to libraries, but the
	// -isysroot option applies to header files.
	if isysrootDir != "" {
	return isysrootDir, need
	}
	return sysrootDir, need
	}

	func execPaths(filepath clientFilePath, req *gomapb.ExecReq, argv0 string) ([]string, error) {
	cwd := filepath.Clean(req.GetCwd())
	if !filepath.IsAbs(cwd) {
	return nil, fmt.Errorf("cwd is not abs path: %s", cwd)
	}
	paths := []string{cwd}
	for _, input := range req.Input {
	filename := input.GetFilename()
	if !filepath.IsAbs(filename) {
	filename = filepath.Join(cwd, filename)
	}
	paths = append(paths, filepath.Clean(filename))
	}
	if argv0InInputRoot(argv0) {
	if !filepath.IsAbs(argv0) {
	argv0 = filepath.Join(cwd, argv0)
	}
	paths = append(paths, filepath.Clean(argv0))
	}
	sysrootDir, need := sysrootDir(req.Arg)
	// if missing sysroot is allowed, no need to ensure it
	if need && sysrootDir != "" {
	// if request not allows missing sysroot, we
	// need to create the directory even if no input files
	// in dir are used. b/150421328

	// allow relative path only.
	// if dir is absolute, request would require
	// InputRootAbsolutePath.
	// TODO: cwdAgnosticReq should check input file path too.
	// http://b/167342635
	// http://crbug.com/1123938
	if !filepath.IsAbs(sysrootDir) {
	dir := filepath.Join(cwd, sysrootDir)
	paths = append(paths, filepath.Clean(dir))
	}
	}
	for _, output := range req.ExpectedOutputFiles {
	if !filepath.IsAbs(output) {
	output = filepath.Join(cwd, output)
	}
	paths = append(paths, filepath.Clean(output))
	}
	for _, output := range req.ExpectedOutputDirs {
	if !filepath.IsAbs(output) {
	output = filepath.Join(cwd, output)
	}
	paths = append(paths, filepath.Clean(output))
	}
	return paths, nil
	}

	// validCommonDir takes a path `dir` generated by commonDir() and returns
	// true if it represents a valid common path.
	func validCommonDir(filepath clientFilePath, dir string) bool {
	if dir == "" {
	return false
	}
	switch filepath.(type) {
	case posixpath.FilePath:
	if dir == "/" {
	return false
	}
	case winpath.FilePath:
	// The drive should be considered as root on Win.
	// e.g. c:\ will return false.
	if len(dir) == 3 && dir[1:] == `:\` {
	return false
	}

	default:
	// unknown filepath?
	return true
	}

	return true
	}

	// needChroot returns true if chroot is needed to run the task with
	// given common `dir`
	func needChroot(filepath clientFilePath, dir string) bool {
	return !validCommonDir(filepath, dir)
	}

	// getPathsWithNoCommonDir returns two or more paths from the input paths that
	// don't have a common dir above the root directory. If there is a common dir,
	// then returns an empty array. Does not guarantee any kind of order.
	func getPathsWithNoCommonDir(filepath clientFilePath, paths []string) []string {
	if len(paths) < 2 {
	return nil
	}

	p0 := paths[0]
	// If there are any pairs of paths in `paths` that have no common directories,
	// then there should be a mismatch between paths[0] and paths[i] for some i > 0.
	for _, p1 := range paths[1:] {
	pair := []string{p0, p1}
	dir := commonDir(filepath, pair)
	if !validCommonDir(filepath, dir) {
	return pair
	}
	}
	return nil
	}

	func checkExecRoot(filepath clientFilePath, dir string) error {
	switch filepath.(type) {
	case posixpath.FilePath:
	// if dir covers these paths, command (e.g. clang) won't
	// work because required *.so etc would not be accessible.
	for _, p := range []string{
	"/lib/x86_64-linux-gnu/",
	"/usr/lib/x86_64-linux-gnu/",
	"/lib64/",
	} {
	if strings.HasPrefix(p, dir+"/") {
	return fmt.Errorf("bad input root: %s", dir)
	}
	}
	return nil

	case winpath.FilePath:
	// TODO: check for win path.
	return nil

	default:
	// unknown filepath?
	return nil
	}
	}

	// deriveExecRoot returns common root of paths.
	// if execRootDir is not empty, use it as root of paths.
	// If second return value is true, chroot must be used. It become true only
	// if `allowChroot` is true and common input root is "/".
	func deriveExecRoot(filepath clientFilePath, paths []string, allowChroot bool, execRootDir string) (string, bool, error) {
	if execRootDir != "" {
	return execRootDir, execRootDir == "/" && allowChroot, nil
	}
	root := commonDir(filepath, paths)
	if needChroot(filepath, root) && allowChroot {
	switch filepath.(type) {
	// TODO: support non-posix platform
	case posixpath.FilePath:
	return "/", true, nil
	}
	}
	if !validCommonDir(filepath, root) {
	pair := getPathsWithNoCommonDir(filepath, paths)
	return "", false, fmt.Errorf("no common paths in inputs: %v", pair)
	}
	err := checkExecRoot(filepath, root)
	if err != nil {
	return "", false, err
	}
	return root, false, nil
	}

	var errOutOfRoot = errors.New("out of root")

	// rootRel returns relative path from rootDir for fname,
	// which is relative path from cwd, or absolute path.
	// cwd and rootDir should be clean path.
	func rootRel(filepath clientFilePath, fname, cwd, rootDir string) (string, error) {
	if filepath == nil {
	return "", errors.New("rootRel: client filepath unknown")
	}
	if !filepath.IsAbs(fname) {
	fname = filepath.Join(cwd, fname)
	}
	// filepath.Rel cleans paths, so we can't use it here.
	// suppose rootdir and cwd are clean path, and
	// cwd is under rootdir.
	rootElems := filepath.SplitElem(rootDir)
	fileElems := filepath.SplitElem(fname)

	if len(fileElems) < len(rootElems) {
	return "", errOutOfRoot
	}
	commonElems := samePathElement(filepath, rootElems, fileElems)
	if len(commonElems) != len(rootElems) {
	return "", errOutOfRoot
	}
	fileElems = fileElems[len(rootElems):]
	relname := filepath.Join(fileElems...)
	fileElems = filepath.SplitElem(filepath.Clean(relname))
	if len(fileElems) > 0 && fileElems[0] == ".." {
	return "", errOutOfRoot
	}
	return relname, nil
	}

	// hasPrefixDir returns true if p has prefix as a directory name.
	// Note: it is case sensitive.
	// TODO: support non-posix platforms.
	func hasPrefixDir(p, prefix string) bool {
	prefix = strings.TrimRight(prefix, "/")
	if !strings.HasPrefix(p, prefix) {
	return false
	}
	if len(p) == len(prefix) {
	return true
	}
	return p[len(prefix)] == '/'
	}