command/descriptor/descriptor.go - infra/goma/server - Git at Google

 // Copyright 2017 The Goma Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 package descriptor

 import (
 	"bufio"
 	"bytes"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"regexp"
 	"strings"

 	"go.chromium.org/goma/server/hash"
 	pb "go.chromium.org/goma/server/proto/command"
 )

 var (
 	// subprogs is subprograms for compiler.
 	// CompilerInfoBuilder::LookupAndUpdate.
 	subprogs = []string{
 		"cpp", "cc1", "cc1plus", "as", "objcopy",
 		"ld", "nm", "strip", "cc1obj", "collect2",
 		"lto1", "lto-wrapper",
 	}

 	clexeVersionRegexp = regexp.MustCompile(`Compiler Version ([0-9\.]+) for`)
 	clexeTargetRegexp  = regexp.MustCompile(`Compiler Version [0-9\.]+ for (x\d+)`)

 	clangclExpectedVersionRegexp = regexp.MustCompile(`clang\s+version\s+\d+\.\d+\.\d+\s+\([^)]*\)`)
 	clangclExpectedTargetRegexp  = regexp.MustCompile(`Target:\s+\.*`)
 )

 // Descriptor represents a command descriptor.
 type Descriptor struct {
 	// fname is command filename in server path. relative to cwd (prefix).
 	fname string
 	*pb.CmdDescriptor

 	Runner       Runner
 	ToClientPath func(string) (string, error)
 	PathType     pb.CmdDescriptor_PathType

 	cwd    string // current working directory in server path.
 	cmddir string // abs command directory in server path.
 	seen   map[string]bool
 }

 // Config used for setting up cmd descriptor.
 type Config struct {
 	Key                    string // Key used for selection.
 	Filename               string // relative to pwd (prefix) in server path
 	AbsoluteBinaryHashFrom string // use Filename for binary hash if emtpy
 	Target                 string // cross target if specified

 	Runner       Runner
 	ToClientPath func(string) (string, error)
 	PathType     pb.CmdDescriptor_PathType

 	// compiler specific
 	// https://clang.llvm.org/docs/CrossCompilation.html#target-triple
 	ClangNeedTarget bool // true if -target is needed.
 	WindowsCross    bool
 }

 // Runner runs command and get combined output.
 type Runner func(cmds ...string) ([]byte, error)

 func (c Config) binaryHashFrom() string {
 	if c.AbsoluteBinaryHashFrom != "" {
 		return c.AbsoluteBinaryHashFrom
 	}
 	return c.Filename
 }

 // canGetTargetFromFilename returns true if target can be get from c.Filename.
 // if c.AbsoluteBinaryHashFrom is given, we suppose that target cannot be get
 // from either of Filename or AbsoluteBinaryHashFrom, the function returns
 // false.
 func (c Config) canGetTargetFromFilename() bool {
 	return c.AbsoluteBinaryHashFrom == ""
 }

 func (c Config) isValid() bool {
 	if c.Filename == "" {
 		return false
 	}
 	// if Target is set, we expect the config is for cross compiling.
 	// then, we require the config have AbsoluteBinaryHashFrom.
 	// subprogram/plugin may have AbsoluteBinaryHashFrom but they do not
 	// have Target.
 	return c.Target == "" || c.AbsoluteBinaryHashFrom != ""
 }

 func cmdDescriptor(c Config) (*pb.CmdDescriptor, error) {
 	sel, err := selector(c)
 	if err != nil {
 		return nil, err
 	}
 	cd := &pb.CmdDescriptor{
 		Selector:      sel,
 		Cross:         &pb.CmdDescriptor_Cross{},
 		EmulationOpts: &pb.CmdDescriptor_EmulationOpts{},
 	}

 	if c.ClangNeedTarget {
 		switch sel.Name {
 		case "clang", "clang++", "clang-cl":
 			cd.Cross.ClangNeedTarget = true
 		default:
 			return nil, fmt.Errorf("need_target=true for non clang: %s", sel.Name)
 		}
 	}
 	if c.WindowsCross {
 		switch sel.Name {
 		case "clang-cl", "clang", "clang++":
 			// clang, clang++ for pnacl-clang
 			cd.Cross.WindowsCross = true
 		default:
 			// gcc, g++ for nacl-gcc but we disable them in client side due to ELF-32
 			return nil, fmt.Errorf("windows_cross=true is not supported for %s", sel.Name)
 		}
 	}

 	// When filename starts with "/", it means the binary is specified with absolute path.
 	// It implies the binary is not relocatable.
 	// In this case, we need to respect include paths sent from the goma client,
 	// since the system include paths can be different (it's often based on compiler path).
 	if strings.HasPrefix(c.Filename, "/") {
 		cd.EmulationOpts.RespectClientIncludePaths = true
 	}

 	return cd, nil
 }

 // New creates cmd descriptor for c.
 func New(c Config) (*Descriptor, error) {
 	if !c.isValid() {
 		return nil, fmt.Errorf("invalid config %v", c)
 	}
 	cd, err := cmdDescriptor(c)
 	if err != nil {
 		return nil, err
 	}

 	return &Descriptor{
 		fname:         c.Filename,
 		CmdDescriptor: cd,
 		Runner:        c.Runner,
 		ToClientPath:  c.ToClientPath,
 		PathType:      c.PathType,
 		seen:          make(map[string]bool),
 	}, nil
 }

 // filespec is file spec represetned in server path.
 type filespec struct {
 	Path         string // in server path.
 	Symlink      string
 	Hash         string
 	Size         int64
 	IsExecutable bool
 }

 // newFilespec creates filespec from fname in server path.
 func newFilespec(fname string) (filespec, error) {
 	fi, err := os.Lstat(fname)
 	if err != nil {
 		return filespec{}, err
 	}
 	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 		linkname, err := os.Readlink(fname)
 		if err != nil {
 			return filespec{}, err
 		}
 		return filespec{
 			Path:    fname,
 			Symlink: linkname,
 		}, nil
 	}
 	sha256, err := hash.SHA256File(fname)
 	if err != nil {
 		return filespec{}, err
 	}
 	return filespec{
 		Path:         fname,
 		Hash:         sha256,
 		Size:         fi.Size(),
 		IsExecutable: fi.Mode()&0111 != 0,
 	}, nil
 }

 func (d *Descriptor) filespecProto(fs filespec) (*pb.FileSpec, error) {
 	cpath, err := d.ToClientPath(fs.Path)
 	if err != nil {
 		return nil, err
 	}
 	return &pb.FileSpec{
 		Path:         cpath,
 		Symlink:      fs.Symlink,
 		Hash:         fs.Hash,
 		Size:         fs.Size,
 		IsExecutable: fs.IsExecutable,
 	}, nil
 }

 // appendFileSpec appends filespec to Setup.Files.
 func (d *Descriptor) appendFileSpec(fs filespec) error {
 	p := fs.Path
 	if !filepath.IsAbs(p) {
 		p = filepath.Join(d.cmddir, p)
 	}
 	if d.seen[p] {
 		return nil
 	}
 	cfs, err := d.filespecProto(fs)
 	if err != nil {
 		return err
 	}
 	d.Setup.Files = append(d.Setup.Files, cfs)
 	d.seen[p] = true
 	return nil
 }

 // CmdSetup sets up command descriptor.
 func (d *Descriptor) CmdSetup() error {
 	if !filepath.IsAbs(d.fname) {
 		return d.relocCmdSetup()
 	}
 	fs, err := newFilespec(d.fname)
 	if err != nil {
 		return err
 	}
 	if fs.Symlink == "" && !fs.IsExecutable {
 		return fmt.Errorf("not executable: %s", d.fname)
 	}
 	cfs, err := d.filespecProto(fs)
 	if err != nil {
 		return err
 	}
 	d.Setup = &pb.CmdDescriptor_Setup{
 		CmdFile:  cfs,
 		PathType: d.PathType,
 	}
 	d.seen[fs.Path] = true
 	if fs.Symlink != "" {
 		err = d.resolveSymlinks("", fs)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }

 func (d *Descriptor) relocCmdSetup() error {
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
 	d.cwd = cwd
 	fs, err := newFilespec(d.fname)
 	if err != nil {
 		return err
 	}
 	if fs.Symlink == "" && !fs.IsExecutable {
 		return fmt.Errorf("not executable: %s", d.fname)
 	}
 	cfs, err := d.filespecProto(fs)
 	if err != nil {
 		return err
 	}
 	ccwd, err := d.ToClientPath(cwd)
 	if err != nil {
 		return err
 	}
 	d.Setup = &pb.CmdDescriptor_Setup{
 		CmdFile:  cfs,
 		CmdDir:   ccwd,
 		PathType: d.PathType,
 	}
 	fname := filepath.Join(cwd, d.fname)
 	d.seen[fname] = true
 	d.cmddir = filepath.Dir(fname)

 	fs.Path = fname
 	err = d.resolveSymlinks(d.cmddir, fs)
 	if err != nil {
 		return err
 	}
 	// ignore error, since fname may not be ELF executable.
 	_ = d.relocatableLibraries(fname)

 	if d.Setup.PathType == pb.CmdDescriptor_WINDOWS {
 		// don't detect subprograms for windows clang
 		// (e.g. pnacl-clang), because compilePath doesn't work.
 		// set them in files explicitly.
 		// TODO: fix?
 		return nil
 	}

 	// Find subprograms.
 	var langopt string
 	switch d.CmdDescriptor.Selector.Name {
 	case "gcc", "clang":
 		langopt = "-xc"
 	case "g++", "clang++":
 		langopt = "-xc++"
 	default:
 		return nil
 	}
 	cpaths, err := compilerPath(d.fname, langopt, d.Runner)
 	if err != nil {
 		return err
 	}
 	for _, sp := range subprogs {
 		spath, err := lookpath(sp, cpaths)
 		if err == nil {
 			// relative to cmd
 			fs, err := newFilespec(spath)
 			if err != nil {
 				return err
 			}
 			rel, err := filepath.Rel(d.cmddir, fs.Path)
 			if err != nil {
 				return err
 			}
 			fs.Path = rel
 			err = d.appendFileSpec(fs)
 			if err != nil {
 				return err
 			}
 			err = d.resolveSymlinks(d.cmddir, fs)
 			if err != nil {
 				return err
 			}
 			err = d.relocatableLibraries(spath)
 			if err != nil {
 				return err
 			}
 		}

 		// TODO: don't put system default subprograms
 		// in cmd descriptor, but put in runtime container image?
 		spath, err = lookpath(sp, []string{"/usr/bin", "/bin"})
 		if err == nil {
 			// abspath
 			fs, err := newFilespec(spath)
 			if err != nil {
 				return err
 			}
 			err = d.appendFileSpec(fs)
 			if err != nil {
 				return err
 			}
 			err = d.resolveSymlinks("", fs)
 			if err != nil {
 				return err
 			}
 			// assume libraries are in system library dir
 			// so no need to setup at each compilation.
 		}
 	}
 	return nil
 }

 func (d *Descriptor) resolveSymlinks(basedir string, fs filespec) error {
 	if fs.Symlink == "" {
 		return nil
 	}
 	if basedir != "" && !filepath.IsAbs(fs.Path) {
 		fs.Path = filepath.Join(basedir, fs.Path)
 	}
 	_, err := os.Stat(fs.Path)
 	if err != nil {
 		return err
 	}
 	for fs.Symlink != "" {
 		sname := fs.Symlink
 		if !filepath.IsAbs(fs.Symlink) {
 			sname = filepath.Join(filepath.Dir(fs.Path), fs.Symlink)
 		}
 		fs, err = newFilespec(sname)
 		if err != nil {
 			return err
 		}
 		if basedir != "" {
 			rel, err := filepath.Rel(basedir, fs.Path)
 			if err != nil {
 				return err
 			}
 			fs.Path = rel
 		}
 		err = d.appendFileSpec(fs)
 		if err != nil {
 			return err
 		}
 		fs.Path = filepath.Join(basedir, fs.Path)
 	}
 	return nil
 }

 func (d *Descriptor) relocatableLibraries(cmdpath string) error {
 	lpaths, err := relocatableLibraries(cmdpath)
 	if err != nil {
 		return err
 	}
 	for _, lpath := range lpaths {
 		fs, err := newFilespec(lpath)
 		if err != nil {
 			return err
 		}
 		rel, err := filepath.Rel(d.cmddir, fs.Path)
 		if err != nil {
 			return err
 		}
 		fs.Path = rel
 		err = d.appendFileSpec(fs)
 		if err != nil {
 			return err
 		}
 		err = d.resolveSymlinks(d.cmddir, fs)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }

 // Add adds fname for command.
 // fname should be absolute path, or relative to cwd.
 // It will emit relative path to command if cmd is relocatable.
 // It just adds fname as is. i.e. not resolve symlink, resolve shared obj etc.
 func (d *Descriptor) Add(fname string) error {
 	if !filepath.IsAbs(fname) {
 		fname = filepath.Join(d.cwd, fname)
 	}
 	fs, err := newFilespec(fname)
 	if err != nil {
 		return err
 	}
 	if !filepath.IsAbs(d.fname) {
 		// relocatable. fname should be relative to cmd dir.
 		rel, err := filepath.Rel(d.cmddir, fs.Path)
 		if err != nil {
 			return err
 		}
 		fs.Path = rel
 	}
 	return d.appendFileSpec(fs)
 }

 func lookpath(fname string, paths []string) (string, error) {
 	for _, path := range paths {
 		name := filepath.Join(path, fname)
 		_, err := os.Stat(name)
 		if err != nil {
 			continue
 		}
 		return name, nil
 	}
 	return "", fmt.Errorf("not found %s in %v", fname, paths)
 }

 func compilerPath(fname, langopt string, runner Runner) ([]string, error) {
 	cmds := []string{fname, "-E", "-v", langopt,
 		"/dev/null", "-o", "/dev/null"}
 	out, err := runner(cmds...)
 	if err != nil {
 		return nil, fmt.Errorf("compilerPath: %v: %v", cmds, err)
 	}
 	s := bufio.NewScanner(bytes.NewReader(out))
 	for s.Scan() {
 		line := s.Text()
 		if strings.HasPrefix(line, "COMPILER_PATH=") {
 			line = strings.TrimPrefix(line, "COMPILER_PATH=")
 			return strings.Split(line, ":"), nil
 		}
 	}
 	if err := s.Err(); err != nil {
 		return nil, err
 	}
 	return nil, err
 }

 // selector returns selector for c.
 func selector(c Config) (*pb.Selector, error) {
 	sha256, err := hash.SHA256File(c.binaryHashFrom())
 	if err != nil {
 		return nil, fmt.Errorf("binary hash for %s: %v", c.binaryHashFrom(), err)
 	}
 	var v, t string
 	switch c.Key {
 	case "gcc", "g++", "clang", "clang++":
 		v, err = version(c.Filename, c.Runner)
 		if err != nil {
 			return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
 		}
 		if !c.canGetTargetFromFilename() {
 			if c.Target == "" {
 				return nil, fmt.Errorf("target not given %v", c)
 			}
 			t = c.Target
 		} else {
 			t, err = target(c.Filename, c.Runner)
 			if err != nil {
 				return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
 			}
 		}
 	case "javac": // TODO: also java?
 		v, err = javaVersion(c.Filename, c.Runner)
 		if err != nil {
 			return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
 		}
 		// javac's target is set to java in goma client.
 		t = "java"
 	case "cl.exe":
 		v, err = clexeVersion(c.Filename, c.Runner)
 		if err != nil {
 			return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
 		}
 		t, err = clexeTarget(c.Filename, c.Runner)
 		if err != nil {
 			return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
 		}
 	case "clang-cl":
 		v, err = clangclVersion(c.Filename, c.Runner)
 		if err != nil {
 			return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
 		}
 		if !c.canGetTargetFromFilename() {
 			if c.Target == "" {
 				return nil, fmt.Errorf("target not given %v", c)
 			}
 			t = c.Target
 		} else {
 			t, err = clangclTarget(c.Filename, c.Runner)
 			if err != nil {
 				return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
 			}
 		}
 	case "dartanalyzer":
 		v, err = dartAnalyzerVersion(c.Filename, c.Runner)
 		if err != nil {
 			return nil, err
 		}
 		if c.Target == "" {
 			return nil, errors.New("missing target configuration for dartanalyzer")
 		}
 		t = c.Target

 	default:
 		// subprogram would not have version,target (?)
 	}
 	return &pb.Selector{
 		Name:       c.Key,
 		Version:    v,
 		Target:     t,
 		BinaryHash: sha256,
 	}, nil
 }

 func version(cmd string, runner Runner) (string, error) {
 	dout, err := runner(cmd, "-dumpversion")
 	if err != nil {
 		return "", fmt.Errorf("failed to take %s dump version: %v", cmd, err)
 	}
 	vout, err := runner(cmd, "--version")
 	if err != nil {
 		return "", fmt.Errorf("failed to take %s version: %v", cmd, err)
 	}
 	return Version(dout, vout), nil
 }

 // Version returns version field for command spec/selector
 // from -dumpversion output and --version output.
 func Version(dout, vout []byte) string {
 	dout = firstLine(dout)
 	vout = normalize(firstLine(vout))
 	return fmt.Sprintf("%s[%s]", dout, vout)
 }

 func normalize(v []byte) []byte {
 	i := bytes.IndexByte(v, '(')
 	if i < 0 {
 		return v
 	}
 	progname := v[:i]
 	if bytes.Contains(progname, []byte("clang")) {
 		return v
 	}
 	if !bytes.Contains(progname, []byte("g++")) && !bytes.Contains(progname, []byte("gcc")) {
 		return v
 	}
 	return v[i:]
 }

 func target(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd, "-dumpmachine")
 	if err != nil {
 		return "", fmt.Errorf("failed to take %s dump machine: %v", cmd, err)
 	}
 	return Target(out), nil
 }

 // Target returns target field of command spec/selector
 // from -dumpmachine output.
 func Target(out []byte) string {
 	return string(firstLine(out))
 }

 func firstLine(b []byte) []byte {
 	i := bytes.IndexAny(b, "\r\n")
 	if i >= 0 {
 		b = b[:i]
 	}
 	return b
 }

 func javaVersion(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd, "-version")
 	if err != nil {
 		return "", fmt.Errorf("failed to get version: %s: %s %v", cmd, out, err)
 	}
 	return JavaVersion(out)
 }

 // JavaVersion returns version string of javac.
 func JavaVersion(out []byte) (string, error) {
 	javac := []byte("javac ")
 	if !bytes.HasPrefix(out, javac) {
 		return "", fmt.Errorf("not starts with javac: %s", out)
 	}
 	return string(bytes.TrimPrefix(firstLine(out), javac)), nil
 }

 func clexeVersion(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd)
 	if err != nil {
 		return "", fmt.Errorf("failed to take cl.exe version: %v", err)
 	}
 	return ClexeVersion(out)
 }

 // ClexeVersion returns version string of CL.exe.
 func ClexeVersion(out []byte) (string, error) {
 	match := clexeVersionRegexp.FindStringSubmatch(string(out))
 	if match == nil {
 		return "", fmt.Errorf("could not find version string: %s", out)
 	}
 	if len(match) < 2 {
 		return "", fmt.Errorf("unexpected version string match: %s", out)
 	}
 	return match[1], nil
 }

 func clexeTarget(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd)
 	if err != nil {
 		return "", fmt.Errorf("failed to take cl.exe target: %v", err)
 	}
 	return ClexeTarget(out)
 }

 // ClexeTarget returns target string of CL.exe.
 func ClexeTarget(out []byte) (string, error) {
 	match := clexeTargetRegexp.FindStringSubmatch(string(out))
 	if match == nil {
 		return "", fmt.Errorf("could not find target string: %s", out)
 	}
 	if len(match) < 2 {
 		return "", fmt.Errorf("unexpected target string match: %s", out)
 	}
 	return match[1], nil
 }

 // ClangClVersion returns clang version from output of `clang-cl.exe -###`.
 //
 // `clang-cl -###` output is like the following:
 //
 //	clang version 3.5.0 (trunk 225621)
 //	Target: i686-pc-windows-msvc
 //
 //	clang version 6.0.0 (trunk 308728)
 //	Target: x86_64-pc-windows-msvc
 //
 //	clang version 9.0.0 (https://github.com/llvm/llvm-project/ 67510fac36d27b2e22c7cd955fc167136b737b93)
 //	Target: x86_64-pc-windows-msvc
 //	Thread model: posix
 //	InstalledDir: /var/tmp/clang/third_party/llvm-build/Release+Asserts/bin
 //
 // Note that endline might be CRLF.
 func ClangClVersion(out []byte) (string, error) {
 	lines := bytes.SplitN(out, []byte("\n"), 2)
 	if len(lines) >= 1 && clangclExpectedVersionRegexp.Match(lines[0]) {
 		return string(bytes.TrimSpace(lines[0])), nil
 	}

 	return "", fmt.Errorf("unexpected clang-cl output: %s", out)
 }

 func clangclVersion(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd, "-###")
 	if err != nil {
 		return "", fmt.Errorf("failed to take clang-cl version: %v", err)
 	}
 	return ClangClVersion(out)
 }

 // ClangClTarget returns clang target from output of `clang-cl.exe -###`
 // See ClangClVersion about the expected input.
 func ClangClTarget(out []byte) (string, error) {
 	lines := bytes.SplitN(out, []byte("\n"), 3)
 	if len(lines) >= 2 && clangclExpectedTargetRegexp.Match(lines[1]) {
 		s := lines[1][len("Target: "):]
 		return string(bytes.TrimSpace(s)), nil
 	}

 	return "", fmt.Errorf("unexpected clang-cl output: %s", out)
 }

 func clangclTarget(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd, "-###")
 	if err != nil {
 		return "", fmt.Errorf("failed to take clang-cl target: %v", err)
 	}
 	return ClangClTarget(out)
 }

 // DartAnalyzerVersion returns the dartanalyzer's version from
 // output of `dartanalyzer --version`
 func DartAnalyzerVersion(out []byte) (string, error) {
 	// output should be like
 	// `dartanalyzer version 2.1.1-dev.1.0`.
 	const dartanalyzerPrefix = "dartanalyzer version "
 	output := strings.TrimSpace(string(out))
 	if !strings.HasPrefix(output, dartanalyzerPrefix) {
 		return "", fmt.Errorf("failed to parse dartanalyzer version: %q", output)
 	}
 	return output[len(dartanalyzerPrefix):], nil
 }

 func dartAnalyzerVersion(cmd string, runner Runner) (string, error) {
 	out, err := runner(cmd, "--version")
 	if err != nil {
 		return "", fmt.Errorf("failed to take dartanalyzer version: %v", err)
 	}
 	return DartAnalyzerVersion(out)
 }
	// Copyright 2017 The Goma Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	package descriptor

	import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"go.chromium.org/goma/server/hash"
	pb "go.chromium.org/goma/server/proto/command"
	)

	var (
	// subprogs is subprograms for compiler.
	// CompilerInfoBuilder::LookupAndUpdate.
	subprogs = []string{
	"cpp", "cc1", "cc1plus", "as", "objcopy",
	"ld", "nm", "strip", "cc1obj", "collect2",
	"lto1", "lto-wrapper",
	}

	clexeVersionRegexp = regexp.MustCompile(`Compiler Version ([0-9\.]+) for`)
	clexeTargetRegexp = regexp.MustCompile(`Compiler Version [0-9\.]+ for (x\d+)`)

	clangclExpectedVersionRegexp = regexp.MustCompile(`clang\s+version\s+\d+\.\d+\.\d+\s+\([^)]*\)`)
	clangclExpectedTargetRegexp = regexp.MustCompile(`Target:\s+\.*`)
	)

	// Descriptor represents a command descriptor.
	type Descriptor struct {
	// fname is command filename in server path. relative to cwd (prefix).
	fname string
	*pb.CmdDescriptor

	Runner Runner
	ToClientPath func(string) (string, error)
	PathType pb.CmdDescriptor_PathType

	cwd string // current working directory in server path.
	cmddir string // abs command directory in server path.
	seen map[string]bool
	}

	// Config used for setting up cmd descriptor.
	type Config struct {
	Key string // Key used for selection.
	Filename string // relative to pwd (prefix) in server path
	AbsoluteBinaryHashFrom string // use Filename for binary hash if emtpy
	Target string // cross target if specified

	Runner Runner
	ToClientPath func(string) (string, error)
	PathType pb.CmdDescriptor_PathType

	// compiler specific
	// https://clang.llvm.org/docs/CrossCompilation.html#target-triple
	ClangNeedTarget bool // true if -target is needed.
	WindowsCross bool
	}

	// Runner runs command and get combined output.
	type Runner func(cmds ...string) ([]byte, error)

	func (c Config) binaryHashFrom() string {
	if c.AbsoluteBinaryHashFrom != "" {
	return c.AbsoluteBinaryHashFrom
	}
	return c.Filename
	}

	// canGetTargetFromFilename returns true if target can be get from c.Filename.
	// if c.AbsoluteBinaryHashFrom is given, we suppose that target cannot be get
	// from either of Filename or AbsoluteBinaryHashFrom, the function returns
	// false.
	func (c Config) canGetTargetFromFilename() bool {
	return c.AbsoluteBinaryHashFrom == ""
	}

	func (c Config) isValid() bool {
	if c.Filename == "" {
	return false
	}
	// if Target is set, we expect the config is for cross compiling.
	// then, we require the config have AbsoluteBinaryHashFrom.
	// subprogram/plugin may have AbsoluteBinaryHashFrom but they do not
	// have Target.
	return c.Target == "" \|\| c.AbsoluteBinaryHashFrom != ""
	}

	func cmdDescriptor(c Config) (*pb.CmdDescriptor, error) {
	sel, err := selector(c)
	if err != nil {
	return nil, err
	}
	cd := &pb.CmdDescriptor{
	Selector: sel,
	Cross: &pb.CmdDescriptor_Cross{},
	EmulationOpts: &pb.CmdDescriptor_EmulationOpts{},
	}

	if c.ClangNeedTarget {
	switch sel.Name {
	case "clang", "clang++", "clang-cl":
	cd.Cross.ClangNeedTarget = true
	default:
	return nil, fmt.Errorf("need_target=true for non clang: %s", sel.Name)
	}
	}
	if c.WindowsCross {
	switch sel.Name {
	case "clang-cl", "clang", "clang++":
	// clang, clang++ for pnacl-clang
	cd.Cross.WindowsCross = true
	default:
	// gcc, g++ for nacl-gcc but we disable them in client side due to ELF-32
	return nil, fmt.Errorf("windows_cross=true is not supported for %s", sel.Name)
	}
	}

	// When filename starts with "/", it means the binary is specified with absolute path.
	// It implies the binary is not relocatable.
	// In this case, we need to respect include paths sent from the goma client,
	// since the system include paths can be different (it's often based on compiler path).
	if strings.HasPrefix(c.Filename, "/") {
	cd.EmulationOpts.RespectClientIncludePaths = true
	}

	return cd, nil
	}

	// New creates cmd descriptor for c.
	func New(c Config) (*Descriptor, error) {
	if !c.isValid() {
	return nil, fmt.Errorf("invalid config %v", c)
	}
	cd, err := cmdDescriptor(c)
	if err != nil {
	return nil, err
	}

	return &Descriptor{
	fname: c.Filename,
	CmdDescriptor: cd,
	Runner: c.Runner,
	ToClientPath: c.ToClientPath,
	PathType: c.PathType,
	seen: make(map[string]bool),
	}, nil
	}

	// filespec is file spec represetned in server path.
	type filespec struct {
	Path string // in server path.
	Symlink string
	Hash string
	Size int64
	IsExecutable bool
	}

	// newFilespec creates filespec from fname in server path.
	func newFilespec(fname string) (filespec, error) {
	fi, err := os.Lstat(fname)
	if err != nil {
	return filespec{}, err
	}
	if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
	linkname, err := os.Readlink(fname)
	if err != nil {
	return filespec{}, err
	}
	return filespec{
	Path: fname,
	Symlink: linkname,
	}, nil
	}
	sha256, err := hash.SHA256File(fname)
	if err != nil {
	return filespec{}, err
	}
	return filespec{
	Path: fname,
	Hash: sha256,
	Size: fi.Size(),
	IsExecutable: fi.Mode()&0111 != 0,
	}, nil
	}

	func (d Descriptor) filespecProto(fs filespec) (pb.FileSpec, error) {
	cpath, err := d.ToClientPath(fs.Path)
	if err != nil {
	return nil, err
	}
	return &pb.FileSpec{
	Path: cpath,
	Symlink: fs.Symlink,
	Hash: fs.Hash,
	Size: fs.Size,
	IsExecutable: fs.IsExecutable,
	}, nil
	}

	// appendFileSpec appends filespec to Setup.Files.
	func (d *Descriptor) appendFileSpec(fs filespec) error {
	p := fs.Path
	if !filepath.IsAbs(p) {
	p = filepath.Join(d.cmddir, p)
	}
	if d.seen[p] {
	return nil
	}
	cfs, err := d.filespecProto(fs)
	if err != nil {
	return err
	}
	d.Setup.Files = append(d.Setup.Files, cfs)
	d.seen[p] = true
	return nil
	}

	// CmdSetup sets up command descriptor.
	func (d *Descriptor) CmdSetup() error {
	if !filepath.IsAbs(d.fname) {
	return d.relocCmdSetup()
	}
	fs, err := newFilespec(d.fname)
	if err != nil {
	return err
	}
	if fs.Symlink == "" && !fs.IsExecutable {
	return fmt.Errorf("not executable: %s", d.fname)
	}
	cfs, err := d.filespecProto(fs)
	if err != nil {
	return err
	}
	d.Setup = &pb.CmdDescriptor_Setup{
	CmdFile: cfs,
	PathType: d.PathType,
	}
	d.seen[fs.Path] = true
	if fs.Symlink != "" {
	err = d.resolveSymlinks("", fs)
	if err != nil {
	return err
	}
	}
	return nil
	}

	func (d *Descriptor) relocCmdSetup() error {
	cwd, err := os.Getwd()
	if err != nil {
	return err
	}
	d.cwd = cwd
	fs, err := newFilespec(d.fname)
	if err != nil {
	return err
	}
	if fs.Symlink == "" && !fs.IsExecutable {
	return fmt.Errorf("not executable: %s", d.fname)
	}
	cfs, err := d.filespecProto(fs)
	if err != nil {
	return err
	}
	ccwd, err := d.ToClientPath(cwd)
	if err != nil {
	return err
	}
	d.Setup = &pb.CmdDescriptor_Setup{
	CmdFile: cfs,
	CmdDir: ccwd,
	PathType: d.PathType,
	}
	fname := filepath.Join(cwd, d.fname)
	d.seen[fname] = true
	d.cmddir = filepath.Dir(fname)

	fs.Path = fname
	err = d.resolveSymlinks(d.cmddir, fs)
	if err != nil {
	return err
	}
	// ignore error, since fname may not be ELF executable.
	_ = d.relocatableLibraries(fname)

	if d.Setup.PathType == pb.CmdDescriptor_WINDOWS {
	// don't detect subprograms for windows clang
	// (e.g. pnacl-clang), because compilePath doesn't work.
	// set them in files explicitly.
	// TODO: fix?
	return nil
	}

	// Find subprograms.
	var langopt string
	switch d.CmdDescriptor.Selector.Name {
	case "gcc", "clang":
	langopt = "-xc"
	case "g++", "clang++":
	langopt = "-xc++"
	default:
	return nil
	}
	cpaths, err := compilerPath(d.fname, langopt, d.Runner)
	if err != nil {
	return err
	}
	for _, sp := range subprogs {
	spath, err := lookpath(sp, cpaths)
	if err == nil {
	// relative to cmd
	fs, err := newFilespec(spath)
	if err != nil {
	return err
	}
	rel, err := filepath.Rel(d.cmddir, fs.Path)
	if err != nil {
	return err
	}
	fs.Path = rel
	err = d.appendFileSpec(fs)
	if err != nil {
	return err
	}
	err = d.resolveSymlinks(d.cmddir, fs)
	if err != nil {
	return err
	}
	err = d.relocatableLibraries(spath)
	if err != nil {
	return err
	}
	}

	// TODO: don't put system default subprograms
	// in cmd descriptor, but put in runtime container image?
	spath, err = lookpath(sp, []string{"/usr/bin", "/bin"})
	if err == nil {
	// abspath
	fs, err := newFilespec(spath)
	if err != nil {
	return err
	}
	err = d.appendFileSpec(fs)
	if err != nil {
	return err
	}
	err = d.resolveSymlinks("", fs)
	if err != nil {
	return err
	}
	// assume libraries are in system library dir
	// so no need to setup at each compilation.
	}
	}
	return nil
	}

	func (d *Descriptor) resolveSymlinks(basedir string, fs filespec) error {
	if fs.Symlink == "" {
	return nil
	}
	if basedir != "" && !filepath.IsAbs(fs.Path) {
	fs.Path = filepath.Join(basedir, fs.Path)
	}
	_, err := os.Stat(fs.Path)
	if err != nil {
	return err
	}
	for fs.Symlink != "" {
	sname := fs.Symlink
	if !filepath.IsAbs(fs.Symlink) {
	sname = filepath.Join(filepath.Dir(fs.Path), fs.Symlink)
	}
	fs, err = newFilespec(sname)
	if err != nil {
	return err
	}
	if basedir != "" {
	rel, err := filepath.Rel(basedir, fs.Path)
	if err != nil {
	return err
	}
	fs.Path = rel
	}
	err = d.appendFileSpec(fs)
	if err != nil {
	return err
	}
	fs.Path = filepath.Join(basedir, fs.Path)
	}
	return nil
	}

	func (d *Descriptor) relocatableLibraries(cmdpath string) error {
	lpaths, err := relocatableLibraries(cmdpath)
	if err != nil {
	return err
	}
	for _, lpath := range lpaths {
	fs, err := newFilespec(lpath)
	if err != nil {
	return err
	}
	rel, err := filepath.Rel(d.cmddir, fs.Path)
	if err != nil {
	return err
	}
	fs.Path = rel
	err = d.appendFileSpec(fs)
	if err != nil {
	return err
	}
	err = d.resolveSymlinks(d.cmddir, fs)
	if err != nil {
	return err
	}
	}
	return nil
	}

	// Add adds fname for command.
	// fname should be absolute path, or relative to cwd.
	// It will emit relative path to command if cmd is relocatable.
	// It just adds fname as is. i.e. not resolve symlink, resolve shared obj etc.
	func (d *Descriptor) Add(fname string) error {
	if !filepath.IsAbs(fname) {
	fname = filepath.Join(d.cwd, fname)
	}
	fs, err := newFilespec(fname)
	if err != nil {
	return err
	}
	if !filepath.IsAbs(d.fname) {
	// relocatable. fname should be relative to cmd dir.
	rel, err := filepath.Rel(d.cmddir, fs.Path)
	if err != nil {
	return err
	}
	fs.Path = rel
	}
	return d.appendFileSpec(fs)
	}

	func lookpath(fname string, paths []string) (string, error) {
	for _, path := range paths {
	name := filepath.Join(path, fname)
	_, err := os.Stat(name)
	if err != nil {
	continue
	}
	return name, nil
	}
	return "", fmt.Errorf("not found %s in %v", fname, paths)
	}

	func compilerPath(fname, langopt string, runner Runner) ([]string, error) {
	cmds := []string{fname, "-E", "-v", langopt,
	"/dev/null", "-o", "/dev/null"}
	out, err := runner(cmds...)
	if err != nil {
	return nil, fmt.Errorf("compilerPath: %v: %v", cmds, err)
	}
	s := bufio.NewScanner(bytes.NewReader(out))
	for s.Scan() {
	line := s.Text()
	if strings.HasPrefix(line, "COMPILER_PATH=") {
	line = strings.TrimPrefix(line, "COMPILER_PATH=")
	return strings.Split(line, ":"), nil
	}
	}
	if err := s.Err(); err != nil {
	return nil, err
	}
	return nil, err
	}

	// selector returns selector for c.
	func selector(c Config) (*pb.Selector, error) {
	sha256, err := hash.SHA256File(c.binaryHashFrom())
	if err != nil {
	return nil, fmt.Errorf("binary hash for %s: %v", c.binaryHashFrom(), err)
	}
	var v, t string
	switch c.Key {
	case "gcc", "g++", "clang", "clang++":
	v, err = version(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
	}
	if !c.canGetTargetFromFilename() {
	if c.Target == "" {
	return nil, fmt.Errorf("target not given %v", c)
	}
	t = c.Target
	} else {
	t, err = target(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
	}
	}
	case "javac": // TODO: also java?
	v, err = javaVersion(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
	}
	// javac's target is set to java in goma client.
	t = "java"
	case "cl.exe":
	v, err = clexeVersion(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
	}
	t, err = clexeTarget(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
	}
	case "clang-cl":
	v, err = clangclVersion(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("version %s [%s]: %v", c.Filename, sha256, err)
	}
	if !c.canGetTargetFromFilename() {
	if c.Target == "" {
	return nil, fmt.Errorf("target not given %v", c)
	}
	t = c.Target
	} else {
	t, err = clangclTarget(c.Filename, c.Runner)
	if err != nil {
	return nil, fmt.Errorf("target %s [%s]: %v", c.Filename, sha256, err)
	}
	}
	case "dartanalyzer":
	v, err = dartAnalyzerVersion(c.Filename, c.Runner)
	if err != nil {
	return nil, err
	}
	if c.Target == "" {
	return nil, errors.New("missing target configuration for dartanalyzer")
	}
	t = c.Target

	default:
	// subprogram would not have version,target (?)
	}
	return &pb.Selector{
	Name: c.Key,
	Version: v,
	Target: t,
	BinaryHash: sha256,
	}, nil
	}

	func version(cmd string, runner Runner) (string, error) {
	dout, err := runner(cmd, "-dumpversion")
	if err != nil {
	return "", fmt.Errorf("failed to take %s dump version: %v", cmd, err)
	}
	vout, err := runner(cmd, "--version")
	if err != nil {
	return "", fmt.Errorf("failed to take %s version: %v", cmd, err)
	}
	return Version(dout, vout), nil
	}

	// Version returns version field for command spec/selector
	// from -dumpversion output and --version output.
	func Version(dout, vout []byte) string {
	dout = firstLine(dout)
	vout = normalize(firstLine(vout))
	return fmt.Sprintf("%s[%s]", dout, vout)
	}

	func normalize(v []byte) []byte {
	i := bytes.IndexByte(v, '(')
	if i < 0 {
	return v
	}
	progname := v[:i]
	if bytes.Contains(progname, []byte("clang")) {
	return v
	}
	if !bytes.Contains(progname, []byte("g++")) && !bytes.Contains(progname, []byte("gcc")) {
	return v
	}
	return v[i:]
	}

	func target(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd, "-dumpmachine")
	if err != nil {
	return "", fmt.Errorf("failed to take %s dump machine: %v", cmd, err)
	}
	return Target(out), nil
	}

	// Target returns target field of command spec/selector
	// from -dumpmachine output.
	func Target(out []byte) string {
	return string(firstLine(out))
	}

	func firstLine(b []byte) []byte {
	i := bytes.IndexAny(b, "\r\n")
	if i >= 0 {
	b = b[:i]
	}
	return b
	}

	func javaVersion(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd, "-version")
	if err != nil {
	return "", fmt.Errorf("failed to get version: %s: %s %v", cmd, out, err)
	}
	return JavaVersion(out)
	}

	// JavaVersion returns version string of javac.
	func JavaVersion(out []byte) (string, error) {
	javac := []byte("javac ")
	if !bytes.HasPrefix(out, javac) {
	return "", fmt.Errorf("not starts with javac: %s", out)
	}
	return string(bytes.TrimPrefix(firstLine(out), javac)), nil
	}

	func clexeVersion(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd)
	if err != nil {
	return "", fmt.Errorf("failed to take cl.exe version: %v", err)
	}
	return ClexeVersion(out)
	}

	// ClexeVersion returns version string of CL.exe.
	func ClexeVersion(out []byte) (string, error) {
	match := clexeVersionRegexp.FindStringSubmatch(string(out))
	if match == nil {
	return "", fmt.Errorf("could not find version string: %s", out)
	}
	if len(match) < 2 {
	return "", fmt.Errorf("unexpected version string match: %s", out)
	}
	return match[1], nil
	}

	func clexeTarget(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd)
	if err != nil {
	return "", fmt.Errorf("failed to take cl.exe target: %v", err)
	}
	return ClexeTarget(out)
	}

	// ClexeTarget returns target string of CL.exe.
	func ClexeTarget(out []byte) (string, error) {
	match := clexeTargetRegexp.FindStringSubmatch(string(out))
	if match == nil {
	return "", fmt.Errorf("could not find target string: %s", out)
	}
	if len(match) < 2 {
	return "", fmt.Errorf("unexpected target string match: %s", out)
	}
	return match[1], nil
	}

	// ClangClVersion returns clang version from output of `clang-cl.exe -###`.
	//
	// `clang-cl -###` output is like the following:
	//
	// clang version 3.5.0 (trunk 225621)
	// Target: i686-pc-windows-msvc
	//
	// clang version 6.0.0 (trunk 308728)
	// Target: x86_64-pc-windows-msvc
	//
	// clang version 9.0.0 (https://github.com/llvm/llvm-project/ 67510fac36d27b2e22c7cd955fc167136b737b93)
	// Target: x86_64-pc-windows-msvc
	// Thread model: posix
	// InstalledDir: /var/tmp/clang/third_party/llvm-build/Release+Asserts/bin
	//
	// Note that endline might be CRLF.
	func ClangClVersion(out []byte) (string, error) {
	lines := bytes.SplitN(out, []byte("\n"), 2)
	if len(lines) >= 1 && clangclExpectedVersionRegexp.Match(lines[0]) {
	return string(bytes.TrimSpace(lines[0])), nil
	}

	return "", fmt.Errorf("unexpected clang-cl output: %s", out)
	}

	func clangclVersion(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd, "-###")
	if err != nil {
	return "", fmt.Errorf("failed to take clang-cl version: %v", err)
	}
	return ClangClVersion(out)
	}

	// ClangClTarget returns clang target from output of `clang-cl.exe -###`
	// See ClangClVersion about the expected input.
	func ClangClTarget(out []byte) (string, error) {
	lines := bytes.SplitN(out, []byte("\n"), 3)
	if len(lines) >= 2 && clangclExpectedTargetRegexp.Match(lines[1]) {
	s := lines[1][len("Target: "):]
	return string(bytes.TrimSpace(s)), nil
	}

	return "", fmt.Errorf("unexpected clang-cl output: %s", out)
	}

	func clangclTarget(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd, "-###")
	if err != nil {
	return "", fmt.Errorf("failed to take clang-cl target: %v", err)
	}
	return ClangClTarget(out)
	}

	// DartAnalyzerVersion returns the dartanalyzer's version from
	// output of `dartanalyzer --version`
	func DartAnalyzerVersion(out []byte) (string, error) {
	// output should be like
	// `dartanalyzer version 2.1.1-dev.1.0`.
	const dartanalyzerPrefix = "dartanalyzer version "
	output := strings.TrimSpace(string(out))
	if !strings.HasPrefix(output, dartanalyzerPrefix) {
	return "", fmt.Errorf("failed to parse dartanalyzer version: %q", output)
	}
	return output[len(dartanalyzerPrefix):], nil
	}

	func dartAnalyzerVersion(cmd string, runner Runner) (string, error) {
	out, err := runner(cmd, "--version")
	if err != nil {
	return "", fmt.Errorf("failed to take dartanalyzer version: %v", err)
	}
	return DartAnalyzerVersion(out)
	}