Add counting the number of CPU's in goma module

This will be used to set -j option when running build system (e.g. make, ninja, scons).
After landing this, I will set -j of chromeoffice buildbots using this function.

BUG=544330

Review-Url: https://codereview.chromium.org/2291273005
diff --git a/scripts/slave/recipe_modules/goma/api.py b/scripts/slave/recipe_modules/goma/api.py
index a570da8..59768de 100644
--- a/scripts/slave/recipe_modules/goma/api.py
+++ b/scripts/slave/recipe_modules/goma/api.py
@@ -14,6 +14,7 @@
 
     self._goma_ctl_env = {}
     self._cloudtail_pid = None
+    self._goma_jobs = None
 
   @property
   def service_account_json_path(self):
@@ -31,6 +32,46 @@
     assert self._goma_dir
     return self.m.path.join(self._goma_dir, 'jsonstatus')
 
+  @property
+  def recommended_goma_jobs(self):
+    """
+    Return the recommended number of jobs for parallel build using Goma.
+
+    This function caches the _goma_jobs.
+    """
+    if self._goma_jobs:
+      return self._goma_jobs
+
+    # In presubmit, a buildbot generates recipe json file and
+    # another buildbot checks generated recipe,
+    # so we need to use python.inline not to change
+    # behavior of recipes.
+    step_result = self.m.python.inline(
+      'calculate the number of recommended jobs',
+      """
+import multiprocessing
+import sys
+
+job_limit = 200
+if sys.platform.startswith('linux'):
+  # Use 80 for linux not to load goma backend.
+  job_limit = 80
+
+try:
+  jobs = min(job_limit, multiprocessing.cpu_count() * 10)
+except NotImplementedError:
+  jobs = 50
+
+print jobs
+      """,
+      stdout=self.m.raw_io.output(),
+      step_test_data=(
+          lambda: self.m.raw_io.test_api.stream_output('50\n'))
+    )
+    self._goma_jobs = int(step_result.stdout)
+
+    return self._goma_jobs
+
   def ensure_goma(self, canary=False):
     with self.m.step.nest('ensure_goma'):
       with self.m.step.context({'infra_step': True}):
diff --git a/scripts/slave/recipe_modules/goma/example.expected/linux.json b/scripts/slave/recipe_modules/goma/example.expected/linux.json
index 1523b2e..44fb9e0 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/linux.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/linux.json
@@ -88,6 +88,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]/cipd/goma/goma_ctl.py",
       "jsonstatus",
       "[CACHE]/cipd/goma/jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.expected/linux_upload_logs.json b/scripts/slave/recipe_modules/goma/example.expected/linux_upload_logs.json
index 8e295e9..3fd4938 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/linux_upload_logs.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/linux_upload_logs.json
@@ -92,6 +92,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]/cipd/goma/goma_ctl.py",
       "jsonstatus",
       "[CACHE]/cipd/goma/jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.expected/mac.json b/scripts/slave/recipe_modules/goma/example.expected/mac.json
index 2fc017d..fe5fc94 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/mac.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/mac.json
@@ -88,6 +88,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]/cipd/goma/goma_ctl.py",
       "jsonstatus",
       "[CACHE]/cipd/goma/jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.expected/mac_upload_logs.json b/scripts/slave/recipe_modules/goma/example.expected/mac_upload_logs.json
index d613a58..dcdb26e 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/mac_upload_logs.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/mac_upload_logs.json
@@ -92,6 +92,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]/cipd/goma/goma_ctl.py",
       "jsonstatus",
       "[CACHE]/cipd/goma/jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.expected/win.json b/scripts/slave/recipe_modules/goma/example.expected/win.json
index 97495e6..3ec6b79 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/win.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/win.json
@@ -88,6 +88,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]\\cipd\\goma\\goma_ctl.py",
       "jsonstatus",
       "[CACHE]\\cipd\\goma\\jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.expected/win_upload_logs.json b/scripts/slave/recipe_modules/goma/example.expected/win_upload_logs.json
index b4a457b..c7355e5 100644
--- a/scripts/slave/recipe_modules/goma/example.expected/win_upload_logs.json
+++ b/scripts/slave/recipe_modules/goma/example.expected/win_upload_logs.json
@@ -92,6 +92,47 @@
     "cmd": [
       "python",
       "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs"
+  },
+  {
+    "cmd": [
+      "echo",
+      "50"
+    ],
+    "name": "echo goma jobs second"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[CACHE]\\cipd\\goma\\goma_ctl.py",
       "jsonstatus",
       "[CACHE]\\cipd\\goma\\jsonstatus"
diff --git a/scripts/slave/recipe_modules/goma/example.py b/scripts/slave/recipe_modules/goma/example.py
index b4278d2..a7d7203 100644
--- a/scripts/slave/recipe_modules/goma/example.py
+++ b/scripts/slave/recipe_modules/goma/example.py
@@ -6,13 +6,18 @@
   'goma',
   'recipe_engine/platform',
   'recipe_engine/properties',
-  'recipe_engine/raw_io',
+  'recipe_engine/step',
 ]
 
 def RunSteps(api):
   api.goma.ensure_goma()
   api.goma.start(env={})
   # build something using goma.
+  api.step('echo goma jobs',
+           ['echo', str(api.goma.recommended_goma_jobs)])
+  api.step('echo goma jobs second',
+           ['echo', str(api.goma.recommended_goma_jobs)])
+
   api.goma.stop(
       ninja_log_outdir=api.properties.get('ninja_log_outdir'),
       ninja_log_compiler=api.properties.get('ninja_log_compiler'),