scripts/build/compress_files.js

// Copyright 2021 The Chromium Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import crypto from 'node:crypto';
import fs from 'node:fs';
import {pipeline, Readable} from 'node:stream';
import zlib from 'node:zlib';

const {promises: pfs} = fs;

function sha1(data) {
  return crypto.createHash('sha1').update(data, 'binary').digest('hex');
}

async function readTextFile(filename) {
  return pfs.readFile(filename, 'utf8');
}

async function fileExists(filename) {
  try {
    await pfs.access(filename);
    return true;
  } catch {
    return false;
  }
}

async function writeTextFile(filename, data) {
  return pfs.writeFile(filename, data, 'utf8');
}

async function readBinaryFile(filename) {
  return pfs.readFile(filename);
}

async function brotli(sourceData, compressedFilename) {
  const sizeBytes = sourceData.length;

  // This replicates the following compression logic:
  // https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/grit/node/base.py;l=649;drc=84ef659584d3beb83b44cc168d02244dbd6b8f87
  const array = new BigUint64Array(1);
  // The length of the uncompressed data as 8 bytes little-endian.
  new DataView(array.buffer).setBigUint64(0, BigInt(sizeBytes), true);

  // BROTLI_CONST is prepended to brotli compressed data in order to
  // easily check if a resource has been brotli compressed.
  // It should be kept in sync with https://source.chromium.org/chromium/chromium/src/+/main:tools/grit/grit/constants.py;l=25;drc=84ef659584d3beb83b44cc168d02244dbd6b8f87.
  const brotliConst = new Uint8Array(2);
  brotliConst[0] = 0x1e;
  brotliConst[1] = 0x9b;

  // The length of the uncompressed data is also appended to the start,
  // truncated to 6 bytes, little-endian.
  const sizeHeader = new Uint8Array(array.buffer).slice(0, 6).buffer;
  const output = fs.createWriteStream(compressedFilename);
  output.write(Buffer.from(brotliConst));
  output.write(Buffer.from(sizeHeader));
  return new Promise((resolve, reject) => {
    pipeline(
        Readable.from(sourceData),
        zlib.createBrotliCompress(),
        output,
        err => {
          return err ? reject(err) : resolve();
        },
    );
  });
}

async function compressFile(filename) {
  const compressedFilename = filename + '.compressed';
  const hashFilename = filename + '.hash';

  let prevHash = '';
  if (await fileExists(hashFilename)) {
    prevHash = await readTextFile(hashFilename);
  }

  const sourceData = await readBinaryFile(filename);
  const currHash = sha1(sourceData);
  if (prevHash !== currHash || !(await fileExists(compressedFilename))) {
    await writeTextFile(hashFilename, currHash);
    await brotli(sourceData, compressedFilename);
  }
}

async function main(argv) {
  const fileListPosition = argv.indexOf('--file_list');
  const fileList = argv[fileListPosition + 1];
  const fileListContents = await readTextFile(fileList);
  const files = fileListContents.split(' ');
  await Promise.all(files.map(filename => filename.trim()).map(compressFile));
}

main(process.argv).catch(err => {
  console.log('compress_files.js failure', err);
  process.exit(1);
});