buildbot/file_up_down_load.sh

#!/bin/bash
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
#@ This script for up/downloading native client toolchains, etc..
#@ To manually inspect what is on the store servers point your
#@ browser at:
#@ http://gsdview.appspot.com/nativeclient-archive2/

#set -o xtrace
set -o nounset
set -o errexit

######################################################################
# Helper
######################################################################

Banner() {
  echo "######################################################################"
  echo $*
  echo "######################################################################"
}


Usage() {
  egrep "^#@" $0 | cut --bytes=3-
}

SanityCheck() {
  Banner "Sanity Checks"
  if [[ $(basename $(pwd)) != "native_client" ]] ; then
    echo "ERROR: run this script from the native_client/ dir"
    exit -1
  fi
}

######################################################################
# Config
######################################################################

readonly GS_UTIL=${GS_UTIL:-buildbot/gsutil.sh}

readonly DIR_ARCHIVE=nativeclient-archive2
readonly DIR_TRYBOT=nativeclient-trybot

readonly GS_PREFIX_ARCHIVE="gs://${DIR_ARCHIVE}"
readonly GS_PREFIX_TRYBOT="gs://${DIR_TRYBOT}"

readonly URL_PREFIX_UI="http://gsdview.appspot.com"
######################################################################
# UTIL
######################################################################
GetFileSizeK() {
  # Note: this is tricky to make work on win/linux/mac
  du -k $1 | egrep -o "^[0-9]+"
}


Upload() {
  local size_kb=$(GetFileSizeK $1)
  echo "uploading: $2 (${size_kb}kB)"
  local path=${2:5}
  echo "@@@STEP_LINK@download (${size_kb}kB)@${URL_PREFIX_UI}/${path}@@@"
  ${GS_UTIL} cp -a public-read $1 $2
}

CheckPath() {
  if [[ $1 != toolchain/* &&
       $1 != between_builders/* &&
       $1 != canned_nexe/* ]] ; then
      echo "ERROR: Bad component name: $1"
      exit -1
  fi
}

UploadArchive() {
  local path=$1
  local tarball=$2

  CheckPath ${path}
  Upload ${tarball} ${GS_PREFIX_ARCHIVE}/${path}
}

DownloadArchive() {
  local path=$1
  local tarball=$2

  echo "@@@STEP_LINK@download@${URL_PREFIX_UI}/${path}@@@"
  ${GS_UTIL} cp ${GS_PREFIX_ARCHIVE}/${path} ${tarball}
}

UploadTrybot() {
  local path=$1
  local tarball=$2

  CheckPath ${path}
  Upload ${tarball} ${GS_PREFIX_TRYBOT}/${path}
}

DownloadTrybot() {
  local path=$1
  local tarball=$2

  echo "@@@STEP_LINK@download@${URL_PREFIX_UI}/${path}@@@"
  ${GS_UTIL} cp ${GS_PREFIX_TRYBOT}/${path} ${tarball}
}

ComputeSha1() {
  # on mac we do not have sha1sum so we fall back to openssl
  if which sha1sum >/dev/null ; then
    echo "$(SHA1=$(sha1sum -b $1) ; echo ${SHA1:0:40})"
  elif which openssl >/dev/null ; then
    echo "$(SHA1=$(openssl sha1 $1) ; echo ${SHA1/* /})"

  else
    echo "ERROR: do not know how to compute SHA1"
    exit 1
  fi
}

######################################################################
# ARM TRUSTED
######################################################################

UploadArmTrustedToolchain() {
  local rev=$1
  local tarball=$2

  UploadArchive toolchain/${rev}/naclsdk_linux_arm-trusted.tgz ${tarball}
}

DownloadArmTrustedToolchain() {
  local rev=$1
  local tarball=$2
  DownloadArchive toolchain/${rev}/naclsdk_linux_arm-trusted.tgz ${tarball}
}

ShowRecentArmTrustedToolchains() {
   local url=${GS_PREFIX_ARCHIVE}/toolchain/*/naclsdk_linux_arm-trusted.tgz
   local recent=$(${GS_UTIL} ls ${url} | tail -5)
   for url in ${recent} ; do
     if ${GS_UTIL} ls -L "${url}" ; then
       echo "====="
     fi
   done
}

######################################################################
# ARM UN-TRUSTED
######################################################################

#@ label should be in :
#@
#@ pnacl_linux_x86
#@ pnacl_mac_x86
#@ pnacl_win_x86

UploadToolchainTarball() {
  local rev=$1
  local label=$2
  local tarball=$3

  ComputeSha1 ${tarball} > ${tarball}.sha1hash
  UploadArchive toolchain/${rev}/naclsdk_${label}.tgz.sha1hash ${tarball}.sha1hash

  # NOTE: only the last link is shown on the waterfall so this should come last
  UploadArchive toolchain/${rev}/naclsdk_${label}.tgz ${tarball}
}

DownloadPnaclToolchains() {
  local rev=$1
  local label=$2
  local tarball=$3

  DownloadArchive toolchain/${rev}/naclsdk_${label}.tgz ${tarball}
}

ShowRecentPnaclToolchains() {
  local label=$1
  local url="${GS_PREFIX_ARCHIVE}/toolchain/*/naclsdk_${label}.tgz"

  local recent=$(${GS_UTIL} ls ${url} | tail -5)
  for url in ${recent} ; do
    if ${GS_UTIL} ls -L "${url}" ; then
      echo "====="
    fi
  done
}

######################################################################
# Nexes for regression/speed tests
######################################################################

UploadArchivedNexes() {
  local rev=$1
  local label="archived_nexes_$2.tar.bz2"
  local tarball=$3

  # TODO(robertm,bradn): find another place to store this and
  #                      negotiate long term storage guarantees
  UploadArchive canned_nexe/${rev}/${label} ${tarball}
}

DownloadArchivedNexes() {
  local rev=$1
  local label="archived_nexes_$2.tar.bz2"
  local tarball=$3

  DownloadArchive canned_nexe/${rev}/${label} ${tarball}
}

######################################################################
# Pexes for bitcode stability testing
######################################################################

UploadArchivedPexes() {
  local rev=$1
  local label="archived_pexes_$2.tar.bz2"
  local tarball=$3

  # TODO(robertm,bradn): find another place to store this and
  #                      negotiate long term storage guarantees
  # Note, we store the pexes with the toolchain rev for now
  UploadArchive toolchain/${rev}/${label} ${tarball}
}

DownloadArchivedPexes() {
  local rev=$1
  local label="archived_pexes_$2.tar.bz2"
  local tarball=$3
  DownloadArchive toolchain/${rev}/${label} ${tarball}
}

UploadArchivedPexesSpec2k() {
    UploadArchivedPexes $1 "spec2k" $2
}

DownloadArchivedPexesSpec2k() {
    DownloadArchivedPexes $1 "spec2k" $2
}
######################################################################
# ARM BETWEEN BOTS
######################################################################

UploadArmBinariesForHWBots() {
  local name=$1
  local tarball=$2
  UploadArchive between_builders/${name}/$(basename ${tarball}) ${tarball}
}


DownloadArmBinariesForHWBots() {
  local name=$1
  local tarball=$2
  DownloadArchive between_builders/${name}/$(basename ${tarball}) ${tarball}
}

######################################################################
# ARM BETWEEN BOTS TRY
######################################################################

UploadArmBinariesForHWBotsTry() {
  local name=$1
  local tarball=$2
  UploadTrybot between_builders/${name}/$(basename ${tarball}) ${tarball}
}


DownloadArmBinariesForHWBotsTry() {
  local name=$1
  local tarball=$2
  DownloadTrybot between_builders/${name}/$(basename ${tarball}) ${tarball}
}

######################################################################
# DISPATCH
######################################################################
SanityCheck

if [[ $# -eq 0 ]] ; then
  echo "you must specify a mode on the commandline:"
  echo
  Usage
  exit -1
elif [[ "$(type -t $1)" != "function" ]]; then
  echo "ERROR: unknown function '$1'." >&2
  echo "For help, try:"
  echo "    $0 help"
  exit 1
else
  "$@"
fi