[server] Add helpers for running background goroutines.

Such background goroutines will be used to do various maintenance jobs like
rereading settings or flushing metrics.

R=tandrii@chromium.org, jchinlee@chromium.org, nodir@chromium.org
BUG=959427

Change-Id: If8c794c118122cbd2e83145e7167d7a179e77fcc
Reviewed-on: https://chromium-review.googlesource.com/c/infra/luci/luci-go/+/1616741
Reviewed-by: Andrii Shyshkalov <tandrii@chromium.org>
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
diff --git a/server/server.go b/server/server.go
index ad76bc3..e775d4a 100644
--- a/server/server.go
+++ b/server/server.go
@@ -71,6 +71,10 @@
 	started bool            // true inside and after ListenAndServe
 	stopped bool            // true inside and after Shutdown
 	done    chan struct{}   // closed after Shutdown returns
+
+	bgrCtx    context.Context    // root context for background work, canceled in Shutdown
+	bgrCancel context.CancelFunc // cancels bgrCtx
+	bgrWg     sync.WaitGroup     // waits for runInBackground goroutines to stop
 }
 
 // New constructs a new server instance.
@@ -97,6 +101,10 @@
 	// TODO(vadimsh): Populate admin routes (admin portal, pprof).
 	srv.RegisterHTTP(opts.AdminAddr)
 
+	// Prepare the context used for background work. It is canceled as soon as we
+	// enter the shutdown sequence.
+	srv.bgrCtx, srv.bgrCancel = context.WithCancel(srv.ctx)
+
 	return srv
 }
 
@@ -195,7 +203,10 @@
 
 	logging.Infof(s.ctx, "Shutting down the server...")
 
-	// Stop them all in parallel. Each Shutdown call blocks until the
+	// Tell all runInBackground goroutines to stop.
+	s.bgrCancel()
+
+	// Stop all http.Servers in parallel. Each Shutdown call blocks until the
 	// corresponding server is stopped.
 	wg := sync.WaitGroup{}
 	wg.Add(len(s.httpSrv))
@@ -208,6 +219,9 @@
 	}
 	wg.Wait()
 
+	// Wait for all background goroutines to stop.
+	s.bgrWg.Wait()
+
 	// Notify ListenAndServe that it can exit now.
 	s.stopped = true
 	close(s.done)
@@ -228,6 +242,18 @@
 	return fmt.Errorf("test listener is not set")
 }
 
+// runInBackground starts a goroutine that does some background work.
+//
+// It is expected to exit soon after its context is canceled.
+func (s *Server) runInBackground(activity string, f func(context.Context)) {
+	ctx := logging.SetField(s.bgrCtx, "activity", activity)
+	s.bgrWg.Add(1)
+	go func() {
+		defer s.bgrWg.Done()
+		f(ctx)
+	}()
+}
+
 // rootMiddleware prepares the per-request context.
 func (s *Server) rootMiddleware(c *router.Context, next router.Handler) {
 	ctx, cancel := context.WithTimeout(s.ctx, time.Minute)