pnacl-build/toolchain_pnacl_bionic.py

#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import fnmatch
import os
import stat
import shutil
import StringIO
import sys

BUILD_SCRIPT = os.path.abspath(__file__)
PNACL_BUILD_DIR = os.path.dirname(BUILD_SCRIPT)
BIONIC_DIR = os.path.dirname(PNACL_BUILD_DIR)
TOOLCHAIN_BUILD_SRC_DIR = os.path.dirname(BIONIC_DIR)
TOOLCHAIN_BUILD_DIR = os.path.dirname(TOOLCHAIN_BUILD_SRC_DIR)
NATIVE_CLIENT_DIR = os.path.dirname(TOOLCHAIN_BUILD_DIR)
SCONS_OUT_DIR = os.path.join(NATIVE_CLIENT_DIR, 'scons-out')
TOOLCHAIN_DIR = os.path.join(NATIVE_CLIENT_DIR, 'toolchain')

TOOLCHAIN_BUILD_OUT_DIR = os.path.join(TOOLCHAIN_BUILD_DIR, 'out')
LINUX_PNACL_BIONIC_DIR = os.path.join(TOOLCHAIN_BUILD_OUT_DIR, 'linux_pnacl_bionic')

sys.path.append(TOOLCHAIN_BUILD_DIR)

import process
from file_update import Mkdir, Rmdir, Symlink
from file_update import NeedsUpdate, UpdateFromTo, UpdateText


def ReplaceText(text, maplist):
  for m in maplist:
    for key in m:
      text = text.replace(key, m[key])
  return text


def ReplaceArch(text, arch, subarch=None):
  ARCH_ARCHES = {
    'arm': 'arm',
    'i686': 'x86',
    'x86_64': 'amd64',
    'pnacl': 'pnacl'
  }
  MARCH_ARCHES = {
    'arm': 'arm',
    'i686': 'x86',
    'x86_64': 'x86',
    'pnacl': 'pnacl'
  }
  CPU_ARCHES = {
    'arm': 'arm',
    'i686': 'i686',
    'x86_64': 'x86_64',
    'pnacl': 'pnacl'
  }
  MPU_ARCHES = {
    'arm': 'arm',
    'i686': 'x86_64',
    'x86_64': 'x86_64',
    'pnacl': 'pnacl'
  }
  NACL_ARCHES = {
    'arm': 'arm',
    'i686': 'x86_64',
    'x86_64': 'x86_64',
    'pnacl': 'pnacl'
  }
  TARGET_ARCHES = {
    'arm': 'armv7-unknown-nacl-gnueabihf',
    'i686': 'i686-unknown-nacl',
    'x86_64': 'x86_64-unknwon-nacl',
    'pnacl': 'pnacl',
  }
  REPLACE_MAP = {
    '$ARCH' : ARCH_ARCHES[arch],
    '$MARCH' : MARCH_ARCHES[arch],
    '$MPU': MPU_ARCHES[arch],
    '$CPU': CPU_ARCHES[arch],
    '$NACL': NACL_ARCHES[arch],
    '$TARG': TARGET_ARCHES[arch]
  }
  return ReplaceText(text, [REPLACE_MAP])


# Bionic uses the following include paths
BIONIC_PAIRS = [
  ('libc/arch-nacl/syscalls/irt_poll.h',
      '$CPU-nacl/include/irt_poll.h'),
  ('libc/arch-nacl/syscalls/irt_socket.h',
      '$CPU-nacl/include/irt_socket.h'),
  ('libc/include', '$CPU-nacl/include'),
  ('libc/arch-nacl/syscalls/nacl_socket.h',
      '$CPU-nacl/include/nacl_socket.h'),
  ('libc/arch-nacl/syscalls/nacl_stat.h',
      '$CPU-nacl/include/nacl_stat.h'),
  ('libc/arch-pnacl/include/machine',
      '$CPU-nacl/include/machine'),
  ('libc/kernel/common', '$CPU-nacl/include'),
  ('libc/kernel/arch-$MARCH/asm', '$CPU-nacl/include/asm'),
  ('libm/include', '$CPU-nacl/include'),
  ('libm/$CPU', '$CPU-nacl/include'),
#  ('safe-iop/include', '$CPU-nacl/include'),
#  ('bionic/libstdc++/nacl',
#      '$NACL-nacl/include/c++/$VER/$NACL-nacl'),
  ('nacl/$CPU-nacl', '$CPU-nacl'),
]

MAKEFILE_TEMPLATE = """
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# GNU Makefile based on shared rules provided by the Native Client SDK.
# See README.Makefiles for more details.

NATIVE_CLIENT_PATH?=$(nacl_path)
TOOLCHAIN_PATH?=$(tc_path)
TOOLCHAIN_PREFIX:=$(TOOLCHAIN_PATH)/host_x86_32/bin/

TARGET_COMPILE_ARGS:=--target=$TARG -integrated-as
TARGET_COMPILE_ARGS+=-nostdinc
TARGET_COMPILE_ARGS+=-I$(TOOLCHAIN_PATH)/$CPU-nacl/include
TARGET_COMPILE_ARGS+=-I$(TOOLCHAIN_PATH)/$CPU-nacl/include/linux
TARGET_COMPILE_ARGS+=-I$(TOOLCHAIN_PREFIX)/../lib/clang/3.4/include

TARGET_LIB_ARGS:=

TARGET_LINK_ARGS:=-nostdlib -L$(TOOLCHAIN_PATH)/$CPU-nacl/lib

CC:=$(TOOLCHAIN_PREFIX)clang
CXX:=$(TOOLCHAIN_PREFIX)clang++
AR:=$(TOOLCHAIN_PREFIX)le32-nacl-ar
AS:=$(TOOLCHAIN_PREFIX)le32-nacl-as
LD:=$(TOOLCHAIN_PREFIX)le32-nacl-ld

SRC_ROOT=$(src_path)
DST_ROOT=$(dst_path)
INS_ROOT=$(ins_path)

NACL_ARCH=$CPU

MAKEFILE_DEPS:=$(build_tc_path)/src/bionic/pnacl-build/tc_bionic.mk
MAKEFILE_DEPS+=$(src_path)/Makefile

include $(build_tc_path)/src/bionic/pnacl-build/tc_bionic.mk
include $(src_path)/Makefile
"""


def GetBionicBuildPath(target_arch, *extra_paths):
  os_name = 'linux'
  return os.path.join(TOOLCHAIN_BUILD_OUT_DIR,
                      "%s_pnacl_bionic" % (os_name),
                      *extra_paths)


def GetProjectPaths(arch, project):
  srcpath = os.path.join(BIONIC_DIR, 'v2', project)
  workpath = os.path.join(TOOLCHAIN_BUILD_OUT_DIR, 'bionic_pnacl_$CPU_work')

  toolpath = GetBionicBuildPath(arch)
  workpath = ReplaceArch(os.path.join(workpath, 'bionic', project), arch)
  instpath = ReplaceArch(os.path.join(toolpath, '$CPU-nacl', 'lib'), arch)
  gccpath = os.path.join(toolpath, 'lib', 'gcc', '$CPU-nacl', '$VER')
  gccpath = ReplaceArch(gccpath, arch)
  out = {
    'src': srcpath,
    'work': workpath,
    'ins': instpath,
    'gcc': gccpath,
    'tc': toolpath,
  }
  return out


def CreateProject(arch, project, clobber=False):
  paths = GetProjectPaths(arch, project)
  remap = {
    '$(src_path)': paths['src'],
    '$(dst_path)': paths['work'],
    '$(ins_path)': paths['ins'],
    '$(tc_path)':  GetBionicBuildPath(arch),
    '$(build_tc_path)': TOOLCHAIN_BUILD_DIR,
    '$(nacl_path)': NATIVE_CLIENT_DIR,
    '$(gcc_path)': paths['gcc'],
  }
  text = ReplaceText(MAKEFILE_TEMPLATE, [remap])
  text = ReplaceArch(text, arch)

  if clobber:
    print 'Clobbering Bionic project directory: ' + paths['work']
    Rmdir(paths['work'])

  Mkdir(paths['work'])
  Mkdir(paths['ins'])
  UpdateText(os.path.join(paths['work'], 'Makefile'), text)

  
def MungeIRT(src, dst):
  replace_map = {
    'off_t': 'int64_t',
    'native_client/src/untrusted/irt/' : '',
  }

  with open(src, 'r') as srcf:
    text = srcf.read()
    text = ReplaceText(text, [replace_map])
    current = open(dst,'r').read()
    if current != text:
      with open(dst, 'w') as dstf:
        dstf.write(text)


LIBGCC = {
  'arm' : [
    'nacl_arm_newlib/lib/gcc/arm-nacl/4.8.3/libgcc.a'
  ],
  'i686' : [
    'nacl_x86_newlib/lib/gcc/x86_64-nacl/4.4.3/32/libgcc.a'
  ],
  'x86_64' : [
    'nacl_x86_newlib/lib/gcc/x86_64-nacl/4.4.3/libgcc.a'
  ]
}

def UpdateToolchain(arches):
  DRIVER_TOOLS = [
    '$CPU-nacl/bin/$CPU-nacl-clang',
    '$CPU-nacl/bin/$CPU-nacl-clang++',
  ]
  UpdateFromTo(os.path.join(TOOLCHAIN_DIR, 'linux_x86', 'pnacl_newlib'),
               LINUX_PNACL_BIONIC_DIR)

  for arch in arches:
    for name in ['irt.h', 'irt_dev.h']:
      src = os.path.join(NATIVE_CLIENT_DIR, 'src', 'untrusted', 'irt', name)
      dst = GetBionicBuildPath(arch, arch + '-nacl', 'include')
      dst = ReplaceArch(dst, arch)
      Mkdir(dst)
      MungeIRT(src, os.path.join(dst, name))

    # Copy libgcc support libraries
    for src, dst in BIONIC_PAIRS:
      srcpath = ReplaceArch(os.path.join(TOOLCHAIN_BUILD_SRC_DIR, 'bionic', 'v2', src), arch)
      dstpath = ReplaceArch(os.path.join(LINUX_PNACL_BIONIC_DIR, dst), arch)
      UpdateFromTo(srcpath, dstpath)

    # Copy libgcc support libraries
    for tool in DRIVER_TOOLS:
      srcpath = 'pnacl-driver.py'
      dstpath = ReplaceArch(os.path.join(LINUX_PNACL_BIONIC_DIR, tool), arch)
      UpdateFromTo(srcpath, dstpath)

    # Copy libgcc support libraries
    for src in LIBGCC[arch]:
      srcpath = os.path.join(TOOLCHAIN_DIR, 'linux_x86', src)
      dstpath = GetBionicBuildPath(arch, arch + '-nacl', 'lib', os.path.basename(src))
      UpdateFromTo(srcpath, dstpath)


def SconsBuild(arch, run_scons = False):
  SCONS_ARCH = {
    'arm' : 'arm',
    'i686' : 'x86-32',
    'x86_64' : 'x86-64'
  }
  arch_name = SCONS_ARCH[arch]
  if (run_scons):
    process.Run(['./scons', 'platform=' + arch_name, '--mode=nacl,dbg-linux',
                 '-j20', 'sel_ldr', 'irt_core'], cwd=NATIVE_CLIENT_DIR)

  paths = GetProjectPaths(arch, 'tests')
  src = os.path.join(SCONS_OUT_DIR, 'nacl_irt-' + arch_name, 'staging', 'irt_core.nexe')
  UpdateFromTo(src, paths['work'])

  for filename in ['sel_ldr', 'nacl_helper_bootstrap']:
    src = os.path.join(SCONS_OUT_DIR, 'dbg-linux-' + arch_name, 'staging', filename)
    UpdateFromTo(src, os.path.join(paths['work'], os.path.basename(src)))

  for filename in ['irt_core.nexe']:
    src = os.path.join(SCONS_OUT_DIR, 'nacl_irt-' + arch_name, 'staging', filename)
    UpdateFromTo(src, os.path.join(paths['work'], os.path.basename(src)))


def MakeProject(arch, project, targets=None, clobber=False, verbose=False):
  paths = GetProjectPaths(arch, project)
  workpath = paths['work']
  targets = targets or []

  targetlist = ' '.join(targets)
  print 'Building %s for %s at %s %s.' % (project, arch, workpath, targetlist)
  if clobber:
    args = ['make', '-j12', 'clean']
    if verbose:
      args.append('V=1')
    if process.Run(args, cwd=workpath, outfile=sys.stdout):
      raise RuntimeError('Failed to clean %s for %s.\n' % (project, arch))

  args = ['make', '-j12'] + targets
  if verbose:
    args.append('V=1')
  if process.Run(args, cwd=workpath, outfile=sys.stdout):
    raise RuntimeError('Failed to build %s for %s.\n' % (project, arch))
  print 'Done with %s for %s.\n' % (project, arch)


def main(argv):
  parser = argparse.ArgumentParser(add_help=False)
  parser.add_argument(
      '-v', '--verbose', dest='verbose',
      default=False, action='store_true',
      help='Produce more output.')

  parser.add_argument(
      '-a', '--arch', dest='arches',
      default=[], action='append',
      help='Produce more output.')

  parser.add_argument(
      '-c', '--clobber', dest='clobber',
      default=False, action='store_true',
      help='Clobber working directories before building.')

  parser.add_argument(
      '-f', '--fast-clobber', dest='fast_clobber',
      default=False, action='store_true',
      help='Clobber bionic working directories before building.')

  parser.add_argument(
      '-u', '--upload', dest='upload',
      default=False, action='store_true',
      help='Upload build artifacts.')

  parser.add_argument(
      '--run-scons', dest='run_scons',
      default=False, action='store_true',
      help='Run scons to build sel_ldr and irt_core.')

  parser.add_argument(
      '--skip-build', dest='skip_build',
      default=False, action='store_true',
      help='Skip building projects.')

  options, leftover_args = parser.parse_known_args(argv)
  arches = options.arches or ['arm', 'i686', 'x86_64']

  UpdateToolchain(arches)

  for arch in arches:
    CreateProject(arch, 'libc')
    CreateProject(arch, 'tests')

  for arch in arches:
    SconsBuild(arch, options.run_scons)

  if not options.skip_build:
    for arch in arches:
      MakeProject(arch, 'libc', ['irt'])
      MakeProject(arch, 'libc', ['crt', 'static'])
      MakeProject(arch, 'tests')

    for arch in arches:
      MakeProject(arch, 'tests', ['static_tests'])

if __name__ == '__main__':
  sys.exit(main(sys.argv))