gce/uploader_iteration.sh

#!/bin/bash
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This enables bash to treat any unhandled non-zero exit codes as "exceptions"
set -e
source $(dirname $0)/uploader_exit_codes.sh

mark_step() {
  echo "$*"
  echo '====================='
}

unlock_git_repo() {
  local git_index_lock=".git/index.lock"
  if [ -e "$git_index_lock" ]; then
    # Can be left over from an interrupted iteration.
    # Safe to remove unless someone is trying to run two iterations in parallel.
    echo "!!! Warning: removing Git index lock in $1 repo !!!"
    rm -f "$git_index_lock"
  fi
}

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

mark_step 'Moving to the source code directory'
SOURCE_DIR=/uploader/src/chromium/src
cd "$SOURCE_DIR"

mark_step 'Updating chromium'
unlock_git_repo Chromium || exit $EXIT_CHROMIUM_UPDATE
git bisect reset || exit $EXIT_CHROMIUM_UPDATE
git reset --hard || exit $EXIT_CHROMIUM_UPDATE
git clean -df || exit $EXIT_CHROMIUM_UPDATE
git checkout -f main || exit $EXIT_CHROMIUM_UPDATE
git pull -f origin || exit $EXIT_CHROMIUM_UPDATE

mark_step 'Figuring out the list of branches'
# Special branches.
BRANCHES_LIST="main branch-heads/2454 branch-heads/2490"
# Get a list of branches that were branched after the "first good commit"
# (post-merge) on main, this is its Cr-Commit-Position:
AFTER_MERGE_COMMIT_POSITION=350370
ALL_BRANCHES=$(git branch -r --list 'branch-heads/*')
for BRANCH in $ALL_BRANCHES; do
    CR_BRANCHED_FROM=$(git log -1 --format=medium "$BRANCH" | grep "^\s*Cr-Branched-From:.*@{\#[0-9]\+}" | tail -1)
    if [ -z "$CR_BRANCHED_FROM" ]; then
        continue
    fi
    BRANCHED_POS=$(echo "$CR_BRANCHED_FROM" | grep -Po "@{#\d+}" | grep -Po "\d+")
    if [ "$BRANCHED_POS" -ge "$AFTER_MERGE_COMMIT_POSITION" ]; then
        BRANCHES_LIST="$BRANCHES_LIST $BRANCH"
    fi
done;

mark_step 'Getting Blink revisions blocklist'
GS_PATH=gs://chrome-devtools-frontend
BLINK_REVISIONS_BLACKLIST_FILE=$GS_PATH/blink_revisions_blacklist
BLINK_REVISIONS_BLACKLIST=""
if gsutil ls "$BLINK_REVISIONS_BLACKLIST_FILE"; then
    BLINK_REVISIONS_BLACKLIST=$(gsutil cat "$BLINK_REVISIONS_BLACKLIST_FILE")
    echo $BLINK_REVISIONS_BLACKLIST > "$SCRIPT_DIR/blink_revisions_blacklist"
fi

mark_step 'Getting list of uploaded revisions (takes ~2.5m)'
KNOWN_REVISIONS_DIR="$GS_PATH/revs"
UPLOADED_REVS_FILE="$SCRIPT_DIR/uploaded_revs.txt"
gsutil ls "$KNOWN_REVISIONS_DIR" > "$UPLOADED_REVS_FILE"

function log_duration() {
    local start=$1
    local dur_name=$2
    local end=$(date +%s.%N)
    local process_duration=$(echo "($end - $start) * 1000" | bc | sed 's/^\./0./')
    echo "Processed $dur_name duration: $process_duration"
}

# Keep track of revs uploaded in this iteration, for skipping
declare -A uploaded_revs

process_revision() {
    declare REVISION="$1"
    local start_time=$(date +%s.%N)

    LOCAL_STORAGE_DIR=/uploader/storage
    if [ -d $LOCAL_STORAGE_DIR ]; then
        rm -rf $LOCAL_STORAGE_DIR
    fi
    ZIPS_DIR="$LOCAL_STORAGE_DIR/zips"
    REVS_DIR="$LOCAL_STORAGE_DIR/revs"
    META_DIR="$LOCAL_STORAGE_DIR/meta"
    HASH_DIR="$LOCAL_STORAGE_DIR/hash"
    mkdir -p $ZIPS_DIR $REVS_DIR $META_DIR $HASH_DIR

    # Skip if uploaded in this iteration
    if [[ -v uploaded_revs[$REVISION] ]]; then
      echo "Skip upload. Already uploaded $REVISION in this iteration."
      log_duration "$start_time" "revision"
      return;
    fi

    if grep -w "@$REVISION" "$UPLOADED_REVS_FILE"; then
        echo "Skip upload. Revision $REVISION was uploaded previously."
        log_duration "$start_time" "revision"
        return
    fi

    if echo "$BLINK_REVISIONS_BLACKLIST" | grep -w $REVISION; then
        echo "Skip upload. Revision $REVISION is in the blacklist"
        log_duration "$start_time" "revision"
        return
    fi

    echo "Processing revision: $REVISION"

    local start_checkout_time=$(date +%s.%N)
    set -x
    git reset --hard || exit $EXIT_BLINK_UPDATE
    git clean -df || exit $EXIT_BLINK_UPDATE
    git checkout -f $REVISION || exit $EXIT_BLINK_UPDATE
    set +x
    log_duration "$start_checkout_time" "checkout"

    # vulkan-deps really effed up some things. this bad workaround seeems to be ok
    GYP_CHROMIUM_NO_ACTION=1 gclient sync --delete_unversioned_trees --reset --with_branch_heads --force || \
      ((rm -rf /uploader/src/chromium/src/third_party/vulkan-deps && GYP_CHROMIUM_NO_ACTION=1 gclient sync -D -R --with_branch_heads) || \
      exit $EXIT_BLINK_UPDATE)

    echo "Running GN"
    if [ -d out/Default ]; then
        rm -rf out/Default
    fi
    mkdir -p out/Default
    echo "is_debug=false" > out/Default/args.gn
    gn gen out/Default || exit $EXIT_GYP_DEVTOOLS

    echo "Generating LASTCHANGE files"
    LASTCHANGE_FILE="build/util/LASTCHANGE"
    python build/util/lastchange.py > $LASTCHANGE_FILE  || exit $EXIT_GYP_DEVTOOLS

    LASTCHANGE_BLINK_FILE="build/util/LASTCHANGE.blink"
    echo "LASTCHANGE=$REVISION" > $LASTCHANGE_BLINK_FILE

    echo "Running ninja"
    ninja -C out/Default devtools_frontend_resources || exit $EXIT_BUILD_FRONTEND
    ZIP_FILE_NAME=devtools_frontend.zip
    ZIP_FILE=$(pwd)/out/$ZIP_FILE_NAME

    # crbug/1174013 (march 2021)
    if [ ! -d out/Default/resources/inspector ]; then
      pushd out/Default/gen/third_party/devtools-frontend/src/front_end
    else
      pushd out/Default/resources/inspector
    fi

    # Appcache is dead.. no more of that!

    echo "Creating a zip file"
    zip -9 --recurse-paths --quiet --display-globaldots --dot-size 1m $ZIP_FILE * || exit $EXIT_ZIP_FRONTEND

    echo "Creating meta and hash files"
    # Copy each file to hash/<sha1>, write <sha1>:<path> to meta file.
    META_FILE_NAME="$META_DIR/@$REVISION"
    # 2021 tweak...  old resources/inspector had ~900 files.. gen/ has >4000. im filtering a few things out but could filter out much more if i wanted.
    # apparently can list all mandatory files with:
    #     gn desc out/Default front_end runtime_deps
    # see crbug.com/1174013#c46
    for FILE_NAME in $(find -type f \! -name "*.ts" \! -name "*tsconfig*"); do
        FILE_SHA1=$(sha1sum $FILE_NAME | awk '{print $1}')
        cp $FILE_NAME "$HASH_DIR/$FILE_SHA1"
        echo "$FILE_SHA1:${FILE_NAME##./}" >> $META_FILE_NAME
    done;

    popd

    echo "Creating revs file"
    ZIP_SHA1=$(sha1sum $ZIP_FILE | awk '{print $1}')
    mv $ZIP_FILE "$ZIPS_DIR/$ZIP_SHA1.zip"
    echo "$ZIP_SHA1" > "$REVS_DIR/@$REVISION"

    echo "Uploading to storage for revision $REVISION"
    # Comment out the line below for testing.
    gsutil -m cp -n -R $LOCAL_STORAGE_DIR/* $GS_PATH || exit $EXIT_CS_UPLOAD
    # It's been uploaded, officially
    uploaded_revs[$REVISION]=1
    echo ""; log_duration "$start_time" "revision"
}

function process_rls_revisions() {
    echo "Checking for new release bot commits…"

    git fetch -f origin || exit $EXIT_CHROMIUM_UPDATE
    RLS_BOT_REVS=$(git log origin/main --all --author "\(chromium-release-autoroll\)\|\(chrome-release-bot\)\|\(chrome-official-brancher\)" --since 100.days --format=format:%H)
    RLS_BOT_REVS_ARRAY=(${RLS_BOT_REVS})
    RLS_BOT_REVS_COUNT="${#RLS_BOT_REVS_ARRAY[@]}"

    for RLS_REVISION in $RLS_BOT_REVS; do
        echo "Processing a release bot revision (of $RLS_BOT_REVS_COUNT total): $RLS_REVISION"
        process_revision $RLS_REVISION
    done

    echo "Done handling release bot commits"
}

echo "Uploading revisions"

# Optional, add any priority revisions. Example:
#     REVISION_LIST="86b75a734eebad710e89bff301516c9f6ba3c8a 43e38a35389de98fa0b7d5cf66a550d4ffa5615 2a7440aadcdb4ac0da5c0a0b4341b74bbf84453"  # these revs are 404ing but chromium doesnt know them.
REVISION_LIST=""
mark_step "Attempting any hardcoded revisions"
for REVISION in $REVISION_LIST; do
    process_revision $REVISION
done

mark_step "Attempting release revisions"
process_rls_revisions

echo "Completed uploader cycle."