scripts/fetch_node_modules.py

#!/usr/bin/env python3
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Simple script to download the pinned Node modules from GCS.

This script exists because the node_modules archive currently contains
a node_modules/.bin directory with a bunch of symlinked files in it,
and download_from_google_storage.py won't let you have archives with
symlinks.

In theory we should probably rebuild the node_modules distro without
the ./bin directory (using `npm install --no-bin-lnks`) but that would
cause the build scripts to fail and we'd have to replace `npmw` with
something else.
"""

import argparse
import hashlib
import os
import sys
import tarfile
from urllib.request import urlopen

import common


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.parse_args()

    with open(os.path.join(common.REPO_DIR, 'node_modules.tar.gz.sha1')) as fp:
        expected_sha1 = fp.read().strip()

    actual_sha1 = None
    tgz = os.path.join(common.REPO_DIR, 'node_modules.tar.gz')
    if os.path.exists(tgz):
        with open(tgz, 'rb') as fp:
            s = hashlib.sha1()
            s.update(fp.read())
            actual_sha1 = s.hexdigest()

    # TODO(crbug.com/1457681): Consider whether we should validate that
    # node_modules/ and all of the expected files exist as well.
    if actual_sha1 == expected_sha1:
        return 0

    url = 'https://storage.googleapis.com/%s/%s' % (
        'chromium-website-lob-storage', expected_sha1)
    with urlopen(url) as url_fp, open(tgz, 'wb') as tgz_fp:
        tgz_fp.write(url_fp.read())

    try:
        # TODO(crbug.com/1457681): download_from_google_storage puts in a fair
        # amount of effort to not clobber an existing directory until it is sure
        # it can extract the archive completely. Consider whether we should do
        # the same.
        with tarfile.open(tgz, 'r:gz') as tar:
            tar.extractall(path=common.REPO_DIR)
        return 0
    except Exception as e:
        print(e)
        return 1


if __name__ == '__main__':
    sys.exit(main())