commit	48e3e9026e60a470e440e865ff27b02dc9604410	[log] [tgz]
author	Goma Authors <noreply@google.com>	Fri Feb 26 04:42:32 2021
committer	Fumitoshi Ukai <ukai@google.com>	Fri Feb 26 05:48:16 2021
tree	b6b9840330eee8c93727a4a67d88f6c5327d7ea1
parent	95fbdbf9e9a153c96b88ea941eedfbeca00c7417 [diff]

Import changes for goma server

  - 789ec76a29be461d2540cfffd5e7c8fd328dc5fb fix comment in client_test.go
  - ebdafab25a0aa2e3288c4bac9b999dd59d2290b9 roll protobuf to 3.15.3
  - 68344beac18e53d32cc3b88796183d7c93bb7ce8 roll google.golang.org/grpc v1.35.0 -> v1.36.0
  - 6ce702757773f450ba4d385ba941965e256c63a9 roll go.opencensus.io v0.22.6 -> v0.23.0
  - 5ecf072d5be9f06cf5058a9bd7059259fddcee7f goma: run `go mod tidy` after `go mod download`
  - 167b599f0ca341f4a10cc6cf0bae01004880fb93 remoteexec: report unknown flag
  - dba9fd3ea0f9246412be2a54b1fa4f5a8563f552 roll protobuf to 3.15.2
  - 108b036bc2feb66c7855fd0fd5f9eea52c5b9077 roll cloud.google.com/go v0.77.0 -> v0.78.0
  - 8f3dcc059e2b0130ac89d44a5e900b4ec2d58af2 remoteexec: reply bad request, not server error
  - d28559ecc519b2edf9e8530af90ab0ee447ad71a removeexec: handle /winsysroot
  - 2813d2f262a81a2682ad10660b2cc24095a97ba1 roll protobuf to 3.15.1
  - 9c0dadc70b9b7f85a9c728734560d697e043a15a roll protobuf to 3.15.0
  - d69b3fa8e8dd235627958ba368140e77ad95166b roll protobuf to 3.14.0
  - db97f563d224ed1d7fb6bacb86ef112dc3e19bb7 roll github.com/gomodule/redigo v1.8.3 -> v1.8.4
  - 15fef80788328268a2183510b236fbecb3bef639 use go 1.16
  - 9cd4a0bf106f6d300dc4acdc5210722b701dcd49 roll cloud.google.com/go v0.76.0 -> v0.77.0
  - ea1fc6c5698322bad81fde5a35ffa23ab4bcd6da remoteexec: don't disable hardening for symlink
  - 6749dca7c476758e3f03bb347354b1674764405a fix nsjail seccomp filter for pnacl-clang++
  - bf23cfeffe67b132c66e6114956dbf01acf67131 fix nsjail seccomp filter for x86_64-nacl-clang++
  - 5d407616252431bdaa96e73cfda90db4d2df592f remoteexec: disable hardening for known commands
  - e22615c908911ff94f9a035ddf9444ca78f225ba execlog: record local_*_time
  - d7698c3c7b318d8a57b0defb31bb7d21114931bf roll google.golang.org/api v0.39.0 -> v0.40.0
  - d71ba3ca4549a1d818c4023028869c8874ff64c6 export more stats from execlog
  - 059fd8890f7b913fcb5ec38b5a4d723708e941ed roll contrib.go.opencensus.io/exporter/stackdriver v0.13....
  - 81e31abf0182addfa9f518466538a12843339f5e roll cloud.google.com/go/pubsub v1.9.1 -> v1.10.0
  - 2595d5118ab57cd9736c0791245507f9f1453e20 roll google.golang.org/api v0.38.0 -> v0.39.0
  - f715447946a7d37f11e0ee7815e77e64429afc2f roll go.opencensus.io v0.22.5 -> v0.22.6
  - 11b6fb88d1b628df1158a70060d5ff10ea988306 roll github.com/google/uuid v1.1.2 -> v1.2.0
  - 1f6f030412262a5d04544a0117c5a8a850316b63 roll github.com/gomodule/redigo v1.8.2 -> v1.8.3
  - 3d7be7d8969a96958db64a50b2d3dd38df3709b0 roll cloud.google.com/go/storage v1.12.0 -> v1.13.0
  - 80cfbd8e8c9db4aa727c475707b6df661a652a8c roll cloud.google.com/go/pubsub v1.8.2 -> v1.9.1
  - 4dfded4abf00d22dd9387140c13152126477c044 roll cloud.google.com/go v0.71.0 -> v0.76.0
  - 569662a8d1f2fb97f88f36c3e36c2e69af7538cc use go 1.15.8
  - 83b0d00fd1766e0302257634c3c4a6fb5bc7fa12 Allow 'cache-silo' as the platform property key
  - 4954de2bcc4b9978219240e8d337b4d50fc79c1b fix nsjail seccomp filter
  - 2c7855a0c8b51b0d40ef87b1615d3cfe9e9c556c distinguish runtime: nsjail-chroot and nsjail
  - 7605ba494acbba5000045e36b228ddb5ce0caca3 fix nsjail.cfg
  - 6b51c40324bd82420b036dd69964538f17f9ed94 Reland "inexpensive hardening by nsjail (seccomp-bpf)"
  - bdfc81a2b9f88989526c29b271b346a4cc54d2bd Revert "inexpensive hardening by nsjail (seccomp-bpf)"
  - d4fece1673c00514b5a8a657a68b8686e780d4d3 inexpensive hardening by nsjail (seccomp-bpf)
  - c2aae6e4eb3a2b0d85d7e8fe15d9bcba7cdce271 execlog: export cache_hit tag
  - 8aa77b857aca1af3280541ee4b737bbceed03b6f remoteexec: return unauthenticated of outputs
  - 3b83700213039f28413bdbbc505ce0f2d2e9c774 use go 1.15.7
  - ace47ad447b37d8421146bf3ed5d65b9b5699394 exec: fix api-error log
  - 43a9d54d55ff9736c132f12e33ba2e9bc9df61ce exec: log api-error
  - da01aab603dae90305a5a3a911ef96553d3937fa use go 1.15.6
  - d98544c80efde58fde08bc311ffd96534a88602c backend: fix log message
  - 5cb00c7d11a219d62716ca400563b208befd434e remoteexec: use user specified exec_root and platform pro...
  - 05e41f7e583be586446b58d6893f93b905fb1e81 Expose cache hit/miss metric
  - b97de93783107c45e7a682246a4621b22f888544 auth: don't cache context error of check group
  - 81db8d99fff4f8a66ac35725187b8dc7cd4bde3e Add dashboard setting files
  - 8e76e461279083d0a1d8bd692c529e1064b645c8 server: fix keepalive
  - 9897ca742639b0bdc5734438f61ea45dc5f5600f remoteexec: ignore environment variables for clang-cl rel...
  - 80bf8c20a7159c89f3654f2c202782bbf954f5a6 backend: don't specify max_concurrent_streams_low_watermark
  - a0ddb057e63f9694b81df5256cc3800537388e95 Mark -Wa,--fatal-warnings as relocatable
  - 49d5d5b8cf566b14fac126fec4686e6efec6c8cc backend: remote uses remote-apis-sdks grpc balancer.
  - ce907cb9278c4b89a0d844d76837733a5ce90fef httprpc: increase timeout
  - c46a4f8b8696944eaee23157d3fef6c13d31c196 redis: make max number of idle/active connections configu...
  - 29a6b80deac9a48446dcb070f6eb00297cbf7c2f Revert "enable grpc compression"
  - 2289ee211830cc1908d607f4098bb5891efdf33e cas: when failed to upload blobs, set it as missing inputs
  - ae6a505cce8431662bdf44a7bfe44798450c2544 remoteexec: don't retry with FailedPrecondition
  - 0679e405156fbfbeb360dc279763e3a2f61a8bb1 remoteexec: raise timeout of input
  - 04552c369a1d9280998a4914d57c6d2f00b8c70b execlog: export os_family key for execlog/requests, handl...
  - dcbb5c8e1b822739ed04d7abaed115968d98819b use go 1.15.5
  - 1e7e0220191c19617a993ad98a53c9d3546e6f5a server: enable grpc compression inside GKE
  - fdb4ebda473020b0b1f01d1d330131fadd70ff0b execlog_server emits metrics from ExecLog
  - 61d2a8150c4aecc64e4a54b2cb268f3f41aa2608 fix typo in default instance name
  - 1ed0a4fb213b53700deaca985dc6323e0d7470f0 enable grpc compression
  - 365db0d525c4e2bc1085d40c9aefa6419d0f13a0 k8s: use east_instance in us-east1-1
  - 5b9f5e2e1e2aa60a1ad9c1fa867f82a06a1e22e2 add acl in toolchain
  - d74bf121a0508e242afcf5bde664e528bda6fa68 remoteexec: assume 4MB/s for output
  - 50a55c467ff9014ad2d42d4e4818d8ff7ad4c449 roll google.golang.org/grpc v1.33.1 -> v1.33.2
  - d507cfbfb0e40c7707f7e152a1987ca2a6842926 Add -additional-tls-certificate to add a root cert
  - 788934cfc14369368a1f43114487c8c286188665 use go 1.15.4
  - 3b55f37dd68a14012b3eab6a767f782368679537 roll google.golang.org/api v0.34.0 -> v0.35.0
  - 4de9f623236bb1b4ca5c82e1ec8c5d04856aab2b roll cloud.google.com/go/pubsub v1.8.1 -> v1.8.2
  - c6f49d0725405322534c99aa6145efdfcbfb67b4 roll cloud.google.com/go v0.69.1 -> v0.71.0
  - 6d7627995fd166d0007f9a69e9330871d4ad6d2e auth: set 3 sec timeout to access authdb
  - e16437249f1b50d9c1bdd93c12beaffa65247cc1 Reland "roll cloud.google.com/go v0.68.0 -> v0.69.1"
  - 9e8abe8ebb7d8b91107d3b41ca3fed827831f31d Reland "roll go.opencensus.io v0.22.4 -> v0.22.5"
  - c8bf7db863321ae200676ea6ec36b15fbcbef163 Revert "roll cloud.google.com/go v0.68.0 -> v0.69.1"
  - 7be533cb19c2a4a98bbc2517b68f2e65380ce9b8 Revert "roll go.opencensus.io v0.22.4 -> v0.22.5"
  - 6ac746102c046530c289a9a029a597d994ed3765 Revert "use one redis conn per ExecReq"
  - d680a1b9184ca6542a9db4e53a173898dc4743bb use go 1.15.3
  - 318622046bce4819de177f1bbd8f158a61e0c8ba use one redis conn per ExecReq
  - 27837a44f763190a6205c3e14f9938a534636b66 roll github.com/golang/protobuf v1.4.2 -> v1.4.3
  - 46388efb73fa6240e86d384675cb508ae6ba1455 roll cloud.google.com/go v0.68.0 -> v0.69.1
  - 280e25a5ff68c6cdd507e025c6c5fcfd0e4c381c remoteexec: record client retries
  - a74eb41e7ab82c27955f440e22063314270d8d16 redis stress test
  - bffab0ef448876cac055d01a55c4d315dbb5eea2 roll google.golang.org/api v0.32.0 -> v0.33.0
  - 2c5891031f0d77449acbf22c0d08ebb0b87793cd remoteexec: make span timeout configurable
  - a0f2f1cdf4d894fec6360b15837c12be46ef2569 remoteexec: dynamically sets timeout to fetch output file
  - 86130a48f3de1ae6c21d7b2ed1c3d8147c03ce2b Mark -mllvm -enable-dse-memoryssa={true,false} as relocat...
  - a09ac0418d373bc48bf228c9359fc6e94f521fda roll go.opencensus.io v0.22.4 -> v0.22.5
  - bd2bdb37b41b5cec97a91fc9d2f81740392eee0e roll cloud.google.com/go/pubsub v1.6.2 -> v1.8.1
  - 8e2369cd47db09a74cf16d6e7862e715d2581680 roll cloud.google.com/go v0.66.0 -> v0.68.0
  - 94e8e5d88b4c6b42d0dc85cff3ec86a32208d928 redis: log for long wait for pool
  - e509e210bcb4eb2fdb5a5dc8cb29689bb9ac840b redis: Pool wait in client.
  - d7e22538e48d83a98c342b6013a8ef1b8363b47f remotexec: log Adapter.Exec error
  - b917d702a7901f5f15920f1ce3c5663f1039e950 redis: don't use Pool.Wait=true
  (And 21 more changes)

TBR=yyanagisawa@google.com, yekuang@google.com
GitOrigin-RevId: 789ec76a29be461d2540cfffd5e7c8fd328dc5fb
Change-Id: Id8b1860d617100a593460e6d624598c4bbada41b

63 files changed

tree: b6b9840330eee8c93727a4a67d88f6c5327d7ea1

README.md

Goma server

Goma is a distributed compiler service for open-source project such as Chromium and Android. It's some kind of replacement of distcc+ccache.

This is reference implementation of server code to be used with Goma client.

Dependencies

The Goma server uses a backend service that implements the Remote Execution API to distribute compile requests across a collection of worker machines and to cache the results of compilations. The Remote Execution API is an open-source standard, with multiple service implementations. The Goma server has been tested with Google's internal Remote Build Execution service, but could use other service implementations with some minor tweaks to the service code.

How to build

Goma server can be built on Linux.

$ GO111MODULE=on go get go.chromium.org/goma/server/cmd/remoteexec_proxy

You will get the binary in $(go env GOPATH)/bin.

How to run

remoteexec_proxy is a single server that acts as proxy server between Goma client and Remote Execution API.

$ remoteexec_proxy --port $PORT \
   --platform-container-image "docker://...@sha256:..." \
   --remoteexec-addr $REMOTEEXEC_ADDR \
   --remote-instance-name $REMOTE_INSTANCE_NAME

for chromium, platform container image should be created with install-build-dep.sh.

Example Dockerfile

FROM ubuntu:16.04
ENV DEBIAN_FRONTEND noninteractive

ADD https://chromium.googlesource.com/chromium/src/+/master/build/install-build-deps.sh?format=TEXT /tmp/install-build-deps.sh.base64
RUN base64 -d /tmp/install-build-deps.sh.base64 > /tmp/install-build-deps.sh \
  && \
  apt-get -y update \
  && \
  apt-get install -f -y build-essential gcc-4.8 g++-4.8 docker.io \
  && \
  bash /tmp/install-build-deps.sh --lib32 --no-prompt \
     --no-chromeos-fonts \
  && \
  rm -rf /var/lib/apt/lists/*

If Remote Execution API requires service account, specify service account JSON file for Remote Execution API by --service-account-json.

Running user is granted by default. If you need to allow other users, you need to specify them by --allowed-users.

Log messages will be output to stderr.

How to use

Install goma client. We provide prebuilt binary with cipd, which is available in depot_tools.

$ cipd install infra/goma/client/linux-amd64 -root ${HOME}/goma

or follow the build instructions to build your own local version of Goma client before running the server code, and install it in $HOME/goma.

Need to authenticate Goma client before use.

$ $HOME/goma/goma_auth.py login

Specify hostname in $GOMA_SERVER_HOST and port in $GOMA_SERVER_PORT, along with a few other environment flags.

$ export GOMA_SERVER_HOST='host-of-remoteexec_proxy-running'
$ export GOMA_SERVER_PORT='port-of-remoteexec_proxy-running'
$ export GOMA_USE_SSL=false
$ export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true

For example, if you are running remoteexec_proxy locally with --port 5050, use:

$ export GOMA_SERVER_HOST=localhost
$ export GOMA_SERVER_PORT=5050

Finally, start Goma client:

$ $HOME/goma/goma_ctl.py ensure_start

and in chromium tree.

$ rm -f out/Release/obj/base/base/base64.o
$ GOMA_USE_LOCAL=false autoninja -C out/Release obj/base/base/base64.o