scripts/fetch_node_modules.py

#!/usr/bin/env python3
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Simple script to download the pinned Node modules from GCS.

This script exists because the node_modules archive currently contains
a node_modules/.bin directory with a bunch of symlinked files in it,
and download_from_google_storage.py won't let you have archives with
symlinks.

In theory we should probably rebuild the node_modules distro without
the ./bin directory (using `npm install --no-bin-lnks`) but that would
cause the build scripts to fail and we'd have to replace `npmw` with
something else.
"""

import argparse
import hashlib
import os
import subprocess
import sys
import tarfile

SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))

def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.parse_args()

  with open(os.path.join(SRC_ROOT, 'node_modules.tar.gz.sha1')) as fp:
    expected_sha1 = fp.read().strip()

  actual_sha1 = None
  tgz = os.path.join(SRC_ROOT, 'node_modules.tar.gz')
  if os.path.exists(tgz):
    with open(tgz, 'rb') as fp:
      s = hashlib.sha1()
      s.update(fp.read())
      actual_sha1 = s.hexdigest()

  # TODO(dpranke): Consider whether we should validate that node_modules/
  # and all of the expected files exist as well.
  if actual_sha1 == expected_sha1:
    return 0

  retcode = subprocess.call([
      sys.executable,
      os.path.join(SRC_ROOT, 'third_party', 'depot_tools', 'gsutil.py'),
      'cp',
      'gs://dpranke-chromium-website-exp-storage/%s' % expected_sha1,
      tgz
  ])
  if retcode:
    return retcode

  try:
    # TODO(dpranke): download_from_google_storage puts in a fair amount
    # of effort to not clobber an existing directory until it is sure it
    # can extract the archive completely. Consider whether we should do
    # the same.
    with tarfile.open(tgz, 'r:gz') as tar:
      tar.extractall(path=SRC_ROOT)
    return 0
  except Exception as e:
    print(e)
    return 1

if __name__ == '__main__':
  sys.exit(main())