commit	49e8b0076afc7aa4adaabbf0fa71827ca833d03a	[log] [tgz]
author	Goma Authors <noreply@google.com>	Thu Jan 11 06:15:37 2024
committer	Takuto Ikuta <tikuta@chromium.org>	Thu Jan 11 06:47:07 2024
tree	23efa40ddbf3f97155e259bfda5cd6e45012c2b8
parent	c03f906835565c4a1518a9405a9780fd953e7760 [diff]

Import changes for goma server

  - 3010b0321289e01cf8b7d4c66d1fd1854c432927 Add deprecation notice in README.md
  - 9adbb797bc7c1eecf47279be96ee409335aeda7e use go 1.21.6
  - a5e8638ab14bbcee788a929e26b99354ae94fb19 fix clangclExpectedVersionRegexp
  - 9354f2ff837318b90d7eb151ed52123d89907a65 use go 1.21.5
  - 9d62ca89129e0b85ef4fde0bb5c33dcb8022322a use go 1.21.4
  - c22433fa91e41ce4a75fc483f6984dcdb9757a65 Drop code for NaCl container configs
  - 43639153538b139d38fc258ff6b14c929a244294 use go 1.21.3
  - 44e29334b61a649262c3da1dd9d513a5212f65cc roll golang.org/x/net v0.14.0 -> v0.17.0
  - 01afb9ed40acaed45a7b1c467800dec491359276 use protoc 24.2
  - acb69c43a8a54851db2ba8b1db4113ebb0e185b3 use protoc 24.0
  - 29f87fd2efb16a3332757a88eb25a74f88d78cc1 use go 1.21.0
  - 843295bd52ed951d3487d8e9e6af4b2d68890856 use go 1.20.7
  - a740ca8230c5922bee57ee9ac0e5f05757b1eeac roll google.golang.org/grpc v1.56.2 -> v1.57.0
  - 338a99b6009eb12196e149db890d5471e6ff113b roll google.golang.org/api v0.128.0 -> v0.134.0
  - c6952002dea2dc7183b793783040f0ecafec4c2d roll golang.org/x/oauth2 v0.9.0 -> v0.11.0
  - 0d9a4fa19f4fed0d42680ab81d9be98f1fd1920c roll golang.org/x/net v0.11.0 -> v0.14.0
  - d6b7b0fe53c030c99c2b9de6fd46991042889bc1 roll go.uber.org/zap v1.24.0 -> v1.25.0
  - 905e15a85f8ba04708aea707863e922c0717f72b roll github.com/googleapis/gax-go/v2 v2.11.0 -> v2.12.0
  - fc140c4b80d24c0dfa2aa8ccb65c43b6b73908b0 roll github.com/aws/aws-sdk-go v1.44.284 -> v1.44.317
  - 2a25418d002f6a0bc3fa463b0119b8fb0b2aa27b roll cloud.google.com/go/storage v1.30.1 -> v1.31.0
  - 1de9d484c2b9108312f40d84f87f73524d3842a9 roll cloud.google.com/go/pubsub v1.31.0 -> v1.33.0
  - 05c393ff284704bcbbdf6210e1cc121facc595f8 roll cloud.google.com/go/monitoring v1.15.0 -> v1.15.1
  - 002f3337d7f3172711c2f5680040024cabdac79c auth: record group of googlers
  - 32797e2056e0a2a6326ba93fa3f4c27d3d5c2ef4 Reland "auth: log access from Googlers with email"
  - 1435eaa5419f7c079e93275d62e4cf580b6324db Revert "auth: log access from Googlers with email"
  - b209b55808c213f900cd55dad3eda59846014e43 auth: log access from Googlers with email
  - 6823b2f8094950958b9f1b23711b47cd8d197cbf roll google.golang.org/grpc v1.55.0 -> v1.56.0
  - 19408edae87807e2060eb153d2be82f03f2806b7 roll google.golang.org/api v0.126.0 -> v0.128.0
  - ef1c46d744839dd8605d5ea14313a1c98b022f3c roll golang.org/x/sync v0.2.0 -> v0.3.0
  - 5da7d56ce8d997b0f915d33c5a7a9a17b6254edf roll golang.org/x/oauth2 v0.8.0 -> v0.9.0
  - b02c28ffb677e4f4a6b3fe3acb2e19425213b908 roll golang.org/x/net v0.10.0 -> v0.11.0
  - 04c40436796322285535e8e835a87afd2ec99ac3 roll github.com/googleapis/gax-go/v2 v2.10.0 -> v2.11.0
  - db8941d1458155765bb91fbb76eb7176184c2240 roll github.com/aws/aws-sdk-go v1.44.280 -> v1.44.284
  - e6e4dfadab4f10f1866f97539a570a4da6e84d12 use go 1.20.5
  - 336986f655e8a82408e8723435bc7683ed1a7370 roll google.golang.org/api v0.125.0 -> v0.126.0
  - 59cb71066209e1c129463d94d82f112bd4297201 rol github.com/aws/aws-sdk-go v1.44.271 -> v1.44.280
  - 7786f1a8a4ef525fb8dc68a4e228701aee6a666e roll cloud.google.com/go/monitoring v1.14.0 -> v1.15.0
  - b0c16f5cfb72c1390df7436905cd5a87bfbe3d58 use protoc 23.2
  - e1a2605a40cee8b93733523dc022a1af3cd912ab roll github.com/aws/aws-sdk-go v1.44.266 -> v1.44.271
  - 2da18410abad88b4d4e3d92b25708a0c726a08c0 roll cloud.google.com/go/pubsub v1.30.1 -> v1.31.0
  - 15e7d6d7b216b70eefa0ec05a6191f6adee0010a roll google.golang.org/api v0.122.0 -> v0.123.0
  - 61bc560b90b4c31fd4933e1023f4e5c7cea67781 roll github.com/aws/aws-sdk-go v1.44.262 -> v1.44.266
  - ca45e9b6bb7825d65d020c8536f7846b16897949 use protoc 23.1
  - f166443496adfb8f6df9ce5bbb20f46ad33784f1 use go 1.20.4
  - 2d45c65f989ce93006bd8d3b49156c32f36156bb roll google.golang.org/api v0.121.0 -> v0.122.0
  - 2a3dd9d055141c3b35b7943a3b3d2b5a105d4995 roll github.com/aws/aws-sdk-go v1.44.259 -> v1.44.262
  - b4a0aadd79598ef0e68728df10c4527a1995e5da roll cloud.google.com/go/monitoring v1.13.1 -> v1.14.0
  - 311111bf910eb04cb9298c942ff5670535d527ef roll google.golang.org/api v0.120.0 -> v0.121.0
  - ada756948f7af80d20ca7dfb91107169cec65688 roll golang.org/x/sync v0.1.0 -> v0.2.0
  - c44033f4051396ef986fbd14fb3910ac7920dbd3 roll golang.org/x/oauth2 v0.7.0 -> v0.8.0
  - e3e41c9e00833787aa16aab38d3fad5fdd59ba36 roll golang.org/x/net v0.9.0 -> v0.10.0
  - 4e25d3e25f5b0a1c23d8c8d8ca593e8cc59c73d7 roll go.uber.org/atomic v1.10.0 -> v1.11.0
  - 2af1b98ff087aad7975ca5819c034d10a6e6e041 roll github.com/aws/aws-sdk-go v1.44.253 -> v1.44.259
  - 8ae426a4167ae15a88f5830d341c5d7cbd7ca4d1 roll cloud.google.com/go/monitoring v1.13.0 -> v1.13.1
  - 11a730e2e76e556d42432716a4f11ce28aa35b04 roll cloud.google.com/go/pubsub v1.30.0 -> v1.30.1
  - 0f71656060d65bc80afd815b075266b2adc61a2c roll google.golang.org/api v0.119.0 -> v0.120.0
  - 0b1b6e236f3f6d0907c85ad88b4f7ab5517987f6 roll github.com/aws/aws-sdk-go v1.44.248 -> v1.44.253
  - 29a6236eb0c59de0129494d6feb519dd27b46967 roll google.golang.org/api v0.118.0 -> v0.119.0
  - ee1b38d4949501abf804da327657256b0f141a53 roll github.com/aws/aws-sdk-go v1.44.244 -> v1.44.248
  - 0e277c0062719e4203734403fbe56f5affb70b3b roll google.golang.org/api v0.116.0 -> v0.118.0
  - 338902a02687f3197b327ab46d6406603246dadc roll github.com/aws/aws-sdk-go v1.44.239 -> v1.44.244
  - 82c4762d300e3e10ea88907a6fd09e4b72affde0 roll google.golang.org/api v0.114.0 -> v0.116.0
  - e31ee6c4159c9c5e0c5a787b238ea115da93643c roll golang.org/x/oauth2 v0.6.0 -> v0.7.0
  - 9c6b0900df477116580d4f56515312c3db7b1438 roll golang.org/x/net v0.8.0 -> v0.9.0
  - 26f98bebc943c9412f0d46cf43960e36a3a33ce8 roll github.com/aws/aws-sdk-go v1.44.235 -> v1.44.239
  - eb73c65b8c3425959bf7d58db7ad97103e126902 roll github.com/aws/aws-sdk-go v1.44.229 -> v1.44.235
  - be65a94c1fd8e10ace27079f4eeffaa50c30657a roll google.golang.org/grpc v1.53.0 -> v1.54.0
  - 8c27ed00ad63b10cfdc4cc5ec530eb7c1df27260 roll github.com/aws/aws-sdk-go v1.44.224 -> v1.44.229
  - f6cf25cc6e8bde159bbca7d0dc23d18522dc77a6 roll cloud.google.com/go/storage v1.30.0 -> v1.30.1
  - 6a828c33fb711d16b2b64bfdb4e3aafcf6a98a66 roll cloud.google.com/go/pubsub v1.29.0 -> v1.30.0
  - e4e7e52a696ceb246751daec1a2149f2350fc93a roll google.golang.org/protobuf v1.29.1 -> v1.30.0
  - 1fa36668af2416dd5bbeb5e40147ea615f3b21a8 roll google.golang.org/api v0.112.0 -> v0.114.0
  - d35f0d331b525284b22bfb7df8e44415611c35c1 roll github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 ...
  - b52d2ed81e3119d221d2df0715e973eb55115629 roll github.com/googleapis/gax-go/v2 v2.7.1 -> v2.8.0
  - ed62ef4185b6d0993134bb2d5c5df1ddcd33f4ec roll github.com/aws/aws-sdk-go v1.44.219 -> v1.44.224
  - 374bdf23a0590d17bd46eba82ce325f689aabae3 roll cloud.google.com/go/storage v1.29.0 -> v1.30.0
  - cf5a3be1630cc49f1ccb24cc294c2c63e70948e1 roll cloud.google.com/go/pubsub v1.28.0 -> v1.29.0
  - 33c762aa6a85ba6ed57afaf4f78ce228a5fb0ae3 roll cloud.google.com/go/monitoring v1.12.0 -> v1.13.0
  - a248c025dd2c62ffd0f7b2e8712d106e58b6e0fb use protoc 22.2
  - be33c5bdb976cc8ba9a47f4e71146b417db4dff8 roll google.golang.org/protobuf v1.28.1 -> v1.29.0
  - fe02dee498b6875278097cc6c8e648c1d3ac7138 roll google.golang.org/api v0.111.0 -> v0.112.0
  - 016181a4c2549fcb93b292ab9834cbd5bddb7899 roll github.com/googleapis/gax-go/v2 v2.7.0 -> v2.7.1
  - 8a65e681494d32fbf661fff162ba14a137444daa roll github.com/aws/aws-sdk-go v1.44.214 -> v1.44.219
  - e0fbe7f8fe13a7f686569b215a373d8542d84404 use go 1.20.2
  - e9b574523f9b9fb91349fca7299376a3404c5443 use protoc 22.1
  - 97af030c8cbc005129fae06f89bae05293a111fe roll google.golang.org/api v0.110.0 -> v0.111.0
  - 3de747158e4e03527eceacd5adc3f62db2f95864 roll golang.org/x/oauth2 v0.5.0 -> v0.6.0
  - a3f9b3f5ea276815882b9063aca0a738175bd941 roll golang.org/x/net v0.7.0 -> v0.8.0
  - d32eff4f26d6d3515be19a2146805d6bc9978cbe roll github.com/aws/aws-sdk-go v1.44.209 -> v1.44.214
  - eebd8b5038d5507688285ab332681e4f5c9e97e2 roll github.com/aws/aws-sdk-go v1.44.204 -> v1.44.209
  - 41c70b33449016ac189b4627dd82fff32f93bb11 use go 1.20.1
  - 31afe7832c33bb15b10d84e118a11cea2e7f48e3 roll google.golang.org/api v0.109.0 -> v0.110.0
  - f9444282d0daff999866cfe24c30f7a4a4f20d50 roll golang.org/x/net v0.6.0 -> v0.7.0
  - 62f45d9ed720cc7af1893ebd754413306850df10 roll go.uber.org/goleak v1.2.0 -> v1.2.1
  - 7d8600aa3012cdfe78e79cd8ca4d78dccd04cf10 roll github.com/aws/aws-sdk-go v1.44.199 -> v1.44.204
  - 266ad54f586a323c2cf198eda2aa65b558b1d1c4 roll protoc from 21.5 to 22.0
  - aa687026ad17417c1db7a5d819dc34adbd171f96 roll golang.org/x/oauth2 v0.4.0 -> v0.5.0
  - a987630d465ea4ff3bdb76ef28040e5aaa3477c7 roll google.golang.org/grpc v1.52.3 -> v1.53.0
  - 0f2d9c10bf83587f6094a5067f1aa1c9edd3f578 roll golang.org/x/net v0.5.0 -> v0.6.0
  - c164c4a950135eda6c63ea81242a395853ef4c66 roll github.com/aws/aws-sdk-go v1.44.194 -> v1.44.199
  (And 43 more changes)

GitOrigin-RevId: 3010b0321289e01cf8b7d4c66d1fd1854c432927
Change-Id: I5a45167d78d80deee673155e88eea32ff80dbc82

README.md[diff]
auth/client.go[diff]
auth/token_test.go[diff]
bytestreamio/bytestreamio.go[diff]
bytestreamio/bytestreamio_test.go[diff]
cipd_manifest.txt[diff]
cipd_manifest.versions[diff]
command/descriptor/descriptor.go[diff]
go.mod[diff]
go.sum[diff]
httprpc/bytestream/bytestream_test.go[diff]
proto/api/goma_data.pb.go[Deleted - diff]
proto/api/goma_log.pb.go[Deleted - diff]
proto/auth/acl.pb.go[Deleted - diff]
proto/auth/auth.pb.go[Deleted - diff]
proto/auth/auth_service.pb.go[Deleted - diff]
proto/auth/auth_service_grpc.pb.go[Deleted - diff]
proto/auth/authdb.pb.go[Deleted - diff]
proto/auth/authdb_service.pb.go[Deleted - diff]
proto/auth/authdb_service_grpc.pb.go[Deleted - diff]
proto/backend/backend.pb.go[Deleted - diff]
proto/cache/cache.pb.go[Deleted - diff]
proto/cache/cache_service.pb.go[Deleted - diff]
proto/cache/cache_service_grpc.pb.go[Deleted - diff]
proto/command/command.pb.go[Deleted - diff]
proto/command/command_service.pb.go[Deleted - diff]
proto/command/package_opts.pb.go[Deleted - diff]
proto/command/setup.pb.go[Deleted - diff]
proto/exec/exec_service.pb.go[Deleted - diff]
proto/exec/exec_service_grpc.pb.go[Deleted - diff]
proto/execlog/log_service.pb.go[Deleted - diff]
proto/execlog/log_service_grpc.pb.go[Deleted - diff]
proto/file/file_service.pb.go[Deleted - diff]
proto/file/file_service_grpc.pb.go[Deleted - diff]
proto/google/protobuf/timestamp.proto[diff]
proto/nsjail/config.pb.go[Deleted - diff]
proto/settings/settings.pb.go[Deleted - diff]
proto/settings/settings_service.pb.go[Deleted - diff]
proto/settings/settings_service_grpc.pb.go[Deleted - diff]

39 files changed

tree: 23efa40ddbf3f97155e259bfda5cd6e45012c2b8

README.md

Goma server

Goma is not maintained any more, please use reclient instead.

Goma is a distributed compiler service for open-source project such as Chromium and Android. It's some kind of replacement of distcc+ccache.

This is reference implementation of server code to be used with Goma client.

Dependencies

The Goma server uses a backend service that implements the Remote Execution API to distribute compile requests across a collection of worker machines and to cache the results of compilations. The Remote Execution API is an open-source standard, with multiple service implementations. The Goma server has been tested with Google's internal Remote Build Execution service, but could use other service implementations with some minor tweaks to the service code.

How to build

Goma server can be built on Linux.

$ GO111MODULE=on go get go.chromium.org/goma/server/cmd/remoteexec_proxy

You will get the binary in $(go env GOPATH)/bin.

How to run

remoteexec_proxy is a single server that acts as proxy server between Goma client and Remote Execution API.

$ remoteexec_proxy --port $PORT \
   --platform-container-image "docker://...@sha256:..." \
   --remoteexec-addr $REMOTEEXEC_ADDR \
   --remote-instance-name $REMOTE_INSTANCE_NAME

for chromium, platform container image would be something like

FROM marketing.gcr.io/google/ubuntu1804:latest
ENV DEBIAN_FRONTEND noninteractive

RUN apt-get -y update \
  && \
  apt-get install -f -y build-essential lsb-release python \
  && \
  rm -rf /var/lib/apt/lists/*

If Remote Execution API requires service account, specify service account JSON file for Remote Execution API by --service-account-json.

Running user is granted by default. If you need to allow other users, you need to specify them by --allowed-users.

Log messages will be output to stderr.

How to use

Install goma client. We provide prebuilt binary with cipd, which is available in depot_tools.

$ cipd install infra/goma/client/linux-amd64 -root ${HOME}/goma

or follow the build instructions to build your own local version of Goma client before running the server code, and install it in $HOME/goma.

Need to authenticate Goma client before use.

$ $HOME/goma/goma_auth.py login

Specify hostname in $GOMA_SERVER_HOST and port in $GOMA_SERVER_PORT, along with a few other environment flags.

$ export GOMA_SERVER_HOST='host-of-remoteexec_proxy-running'
$ export GOMA_SERVER_PORT='port-of-remoteexec_proxy-running'
$ export GOMA_USE_SSL=false
$ export GOMA_ARBITRARY_TOOLCHAIN_SUPPORT=true

For example, if you are running remoteexec_proxy locally with --port 5050, use:

$ export GOMA_SERVER_HOST=localhost
$ export GOMA_SERVER_PORT=5050

Finally, start Goma client:

$ $HOME/goma/goma_ctl.py ensure_start

and in chromium tree.

$ rm -f out/Release/obj/base/base/base64.o
$ GOMA_USE_LOCAL=false autoninja -C out/Release obj/base/base/base64.o