gce/uploader_iteration.sh

#!/bin/bash
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This enables bash to treat any unhandled non-zero exit codes as "exceptions"
set -e
source $(dirname $0)/uploader_exit_codes.sh

mark_step() {
  echo "$*"
  echo '====================='
}

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

mark_step 'Moving to the source code directory'
SOURCE_DIR=/uploader/src
cd "$SOURCE_DIR"

mark_step 'Updating gyp'
pushd gyp
CHROMIUM_DEPS=$(curl -fsS http://src.chromium.org/chrome/trunk/src/DEPS) || exit $EXIT_GYP_UPDATE
CHROMIUM_GYP_REVISION="$(echo "$CHROMIUM_DEPS" | $SCRIPT_DIR/uploader_helper.py --find_project_revision='src/tools/gyp')"
svn cleanup || exit $EXIT_GYP_UPDATE
svn up -r $CHROMIUM_GYP_REVISION || exit $EXIT_GYP_UPDATE
popd

mark_step 'Getting the last Blink revision for which the frontend has been uploaded'
GS_PATH=gs://chrome-devtools-frontend
KNOWN_REVISIONS_DIR="$GS_PATH/revs"
ALL_UPLOADED_REVS=$(gsutil ls "$KNOWN_REVISIONS_DIR") || exit $EXIT_LAST_UPLOADED_BLINK_REV
LAST_BLINK_REVISION=$(echo "$ALL_UPLOADED_REVS" | sed -e "s#^$KNOWN_REVISIONS_DIR/@##" | sort -nr | head -n 1)
if [ -z "$LAST_BLINK_REVISION" ]; then
    exit $EXIT_LAST_UPLOADED_BLINK_REV
fi
echo "Last uploaded was for Blink revision $LAST_BLINK_REVISION"

mark_step 'Getting Blink revisions blacklist'
BLINK_REVISIONS_BLACKLIST_FILE=$GS_PATH/blink_revisions_blacklist
BLINK_REVISIONS_BLACKLIST=""
if gsutil ls "$BLINK_REVISIONS_BLACKLIST_FILE"; then
    BLINK_REVISIONS_BLACKLIST=$(gsutil cat "$BLINK_REVISIONS_BLACKLIST_FILE")
fi

mark_step 'Working out SVN revisions list'
pushd blink
git fetch || exit $EXIT_BLINK_UPDATE
HEAD_BLINK_REVISION=$(git log -n 1 --all --grep="git-svn-id:" --pretty=medium | grep -Po "git-svn-id:[^@]*@\d+" | grep -Po "@\d+" | grep -Po "\d+")
popd
echo "Last committed was Blink revision $HEAD_BLINK_REVISION"
REVISION_LIST=$(seq $[LAST_BLINK_REVISION+1] $HEAD_BLINK_REVISION)
if [ -z "$REVISION_LIST" ]; then
    exit $EXIT_NO_UPDATED_BRANCHES
fi

mark_step 'Building DevTools frontend for revisions and uploading to Cloud Storage'
for REVISION in $REVISION_LIST; do
    LOCAL_STORAGE_DIR=/uploader/storage
    if [ -d $LOCAL_STORAGE_DIR ]; then
        rm -rf $LOCAL_STORAGE_DIR
    fi
    ZIPS_DIR="$LOCAL_STORAGE_DIR/zips"
    REVS_DIR="$LOCAL_STORAGE_DIR/revs"
    META_DIR="$LOCAL_STORAGE_DIR/meta"
    HASH_DIR="$LOCAL_STORAGE_DIR/hash"
    mkdir -p $ZIPS_DIR $REVS_DIR $META_DIR $HASH_DIR

    if echo "$BLINK_REVISIONS_BLACKLIST" | grep -w $REVISION; then
        echo "Revision $REVISION is in the blacklist, skipping over"
        continue
    fi

    pushd blink
    REVISION_COMMIT=$(git log -n 1 --all --grep="git-svn-id:[^@]*@$REVISION" --pretty=oneline | awk '{ print $1; }')
    if [ -z "$REVISION_COMMIT" ]; then
        echo "@$REVISION ==> missing!"
        popd
        continue
    fi
    echo "@$REVISION ==> $REVISION_COMMIT"
    GIT_INDEX_LOCK=".git/index.lock"
    if [ -e "$GIT_INDEX_LOCK" ]; then
        # Can be left over from an interrupted iteration.
        # Safe to remove unless someone is trying to run two iterations in parallel.
        echo "!!! Warning: removing Git index lock !!!"
        rm -f "$GIT_INDEX_LOCK" || exit $EXIT_BLINK_UPDATE
    fi
    git reset --hard $REVISION_COMMIT || exit $EXIT_BLINK_UPDATE

    # Exclude "trunk" to work around branches that accidentally have a second
    # copy of Blink sources inside it. See http://crbug.com/450975
    DEVTOOLS_GYP=$(find . -type f -name devtools.gyp -not -path "*/trunk/*")
    if [ -z "$DEVTOOLS_GYP" ]; then
        exit $EXIT_DEVTOOLS_GYP_FILE
    fi
    if [ -d ../devtools-frontend ]; then
        rm -rf ../devtools-frontend
    fi
    DEVTOOLS_DIR=$(dirname "$DEVTOOLS_GYP")
    GYP_GENERATORS=ninja ../gyp/gyp --toplevel-dir="$DEVTOOLS_DIR" --depth=../../../../devtools-frontend --generator-output=../../../../devtools-frontend "$DEVTOOLS_GYP" || exit $EXIT_GYP_DEVTOOLS
    popd

    ninja -C devtools-frontend/out/Default devtools_frontend_resources || exit $EXIT_BUILD_FRONTEND
    ZIP_FILE_NAME=devtools_frontend.zip
    ZIP_FILE=$(pwd)/devtools-frontend/$ZIP_FILE_NAME

    pushd devtools-frontend/out/Default/resources/inspector
    # Create an AppCache manifest and patch devtools.html and / or inspector.html to point to it
    MANIFEST_FILE_NAME=$REVISION.manifest
    FALLBACK_ENTRIES=
    for DEVTOOLS_MAIN_HTML in devtools.html inspector.html; do
        if [ ! -f "$DEVTOOLS_MAIN_HTML" ]; then
            continue
        fi
        sed -i -e 's/^<html>/<html manifest="'$MANIFEST_FILE_NAME'">/' $DEVTOOLS_MAIN_HTML
        FALLBACK_ENTRIES="$FALLBACK_ENTRIES $DEVTOOLS_MAIN_HTML"
    done
    if [ -z "$FALLBACK_ENTRIES" ]; then
        exit $EXIT_MANIFEST_CREATION
    fi

    cat <<EOF > $MANIFEST_FILE_NAME
CACHE MANIFEST
FALLBACK:
EOF
    for FALLBACK_ENTRY in $FALLBACK_ENTRIES; do
        echo "$FALLBACK_ENTRY $FALLBACK_ENTRY" >> $MANIFEST_FILE_NAME
    done
    echo "CACHE:" >> $MANIFEST_FILE_NAME
    find -type f -not -name $MANIFEST_FILE_NAME -print | sed -e 's#^\./##' >> $MANIFEST_FILE_NAME

    zip -9r $ZIP_FILE * || exit $EXIT_ZIP_FRONTEND

    # Copy each file to hash/<sha1>, write <sha1>:<path> to meta file.
    META_FILE_NAME="$META_DIR/@$REVISION"
    for FILE_NAME in $(find -type f); do
        FILE_SHA1=$(sha1sum $FILE_NAME | awk '{print $1}')
        cp $FILE_NAME "$HASH_DIR/$FILE_SHA1"
        echo "$FILE_SHA1:${FILE_NAME##./}" >> $META_FILE_NAME
    done;

    popd

    ZIP_SHA1=$(sha1sum $ZIP_FILE | awk '{print $1}')
    mv $ZIP_FILE "$ZIPS_DIR/$ZIP_SHA1.zip"
    echo "$ZIP_SHA1" > "$REVS_DIR/@$REVISION"

    # Comment out the line below for testing.
    gsutil -m cp -R $LOCAL_STORAGE_DIR/* $GS_PATH || exit $EXIT_CS_UPLOAD
done