siso: warn if resource limit of file descriptors too low
Bug: b/348987499
Change-Id: Ic6e8aa97980fec740e612fd213c4f13cdcd75df2
Reviewed-on: https://chromium-review.googlesource.com/c/infra/infra/+/5645496
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Auto-Submit: Fumitoshi Ukai <ukai@google.com>
Reviewed-by: Junji Watanabe <jwata@google.com>
Commit-Queue: Fumitoshi Ukai <ukai@google.com>
Cr-Commit-Position: refs/heads/main@{#66369}
diff --git a/go/src/infra/build/siso/subcmd/ninja/limits_unix.go b/go/src/infra/build/siso/subcmd/ninja/limits_unix.go
new file mode 100644
index 0000000..ae37b6f
--- /dev/null
+++ b/go/src/infra/build/siso/subcmd/ninja/limits_unix.go
@@ -0,0 +1,41 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+//go:build unix
+
+package ninja
+
+import (
+ "context"
+ "fmt"
+
+ "golang.org/x/sys/unix"
+
+ "infra/build/siso/build"
+ "infra/build/siso/o11y/clog"
+ "infra/build/siso/ui"
+)
+
+func (c *ninjaCmdRun) checkResourceLimits(ctx context.Context) {
+ var lim unix.Rlimit
+ err := unix.Getrlimit(unix.RLIMIT_NOFILE, &lim)
+ if err != nil {
+ clog.Warningf(ctx, "failed to get rlimit: %v", err)
+ return
+ }
+ limits := build.DefaultLimits(ctx)
+ nfile := uint64(limits.Local) * 8 // 8 fds per proc?
+ switch {
+ case c.offline:
+ case c.remoteJobs > 0:
+ // reproxy grpc client+server, scandeps server client+server
+ nfile += uint64(c.remoteJobs) * 4
+ default:
+ nfile += uint64(limits.Remote) * 4
+ }
+ clog.Infof(ctx, "rlimit.nofile=%d,%d required=%d?", lim.Cur, lim.Max, nfile)
+ if lim.Cur < nfile {
+ ui.Default.PrintLines(ui.SGR(ui.Yellow, fmt.Sprintf("WARNING: too low file limit=%d. would fail with too many open files\n", lim.Cur)))
+ }
+}
diff --git a/go/src/infra/build/siso/subcmd/ninja/limits_windows.go b/go/src/infra/build/siso/subcmd/ninja/limits_windows.go
new file mode 100644
index 0000000..ecb887c
--- /dev/null
+++ b/go/src/infra/build/siso/subcmd/ninja/limits_windows.go
@@ -0,0 +1,12 @@
+// Copyright 2024 The Chromium Authors
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+//go:build windows
+
+package ninja
+
+import "context"
+
+func (c *ninjaCmdRun) checkResourceLimits(ctx context.Context) {
+}
diff --git a/go/src/infra/build/siso/subcmd/ninja/ninja.go b/go/src/infra/build/siso/subcmd/ninja/ninja.go
index ed89bbc..647e7b8 100644
--- a/go/src/infra/build/siso/subcmd/ninja/ninja.go
+++ b/go/src/infra/build/siso/subcmd/ninja/ninja.go
@@ -495,8 +495,9 @@
}
}
}
- properties.Add("job_id", c.jobID)
+ c.checkResourceLimits(ctx)
+ properties.Add("job_id", c.jobID)
clog.Infof(ctx, "job id: %q", c.jobID)
clog.Infof(ctx, "build id: %q", buildID)
clog.Infof(ctx, "project id: %q", projectID)
