sysroot-creator.sh revision cedac228d2dd51db4b79ea1e72c7f249408ee061 
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4#
5# This script should not be run directly but sourced by the other
6# scripts (e.g. sysroot-creator-trusty.sh).  Its up to the parent scripts
7# to define certain environment variables: e.g.
8#  DIST=trusty
9#  APT_REPO=http://archive.ubuntu.com/ubuntu
10#  KEYRING_FILE=/usr/share/keyrings/ubuntu-archive-keyring.gpg
11#  DEBIAN_PACKAGES="gcc libz libssl"
12
13#@ This script builds a Debian sysroot images for building Google Chrome.
14#@
15#@  Generally this script is invoked as:
16#@  sysroot-creator-<flavour>.sh <mode> <args>*
17#@  Available modes are shown below.
18#@
19#@ List of modes:
20
21######################################################################
22# Config
23######################################################################
24
25set -o nounset
26set -o errexit
27
28SCRIPT_DIR=$(cd $(dirname $0) && pwd)
29
30if [ -z "${DIST:-}" ]; then
31  echo "error: DIST not defined"
32  exit 1
33fi
34
35if [ -z "${APT_REPO:-}" ]; then
36  echo "error: APT_REPO not defined"
37  exit 1
38fi
39
40if [ -z "${KEYRING_FILE:-}" ]; then
41  echo "error: KEYRING_FILE not defined"
42  exit 1
43fi
44
45if [ -z "${DEBIAN_PACKAGES:-}" ]; then
46  echo "error: DEBIAN_PACKAGES not defined"
47  exit 1
48fi
49
50readonly REPO_BASEDIR="${APT_REPO}/dists/${DIST}"
51
52# This is where the staging sysroot is.
53readonly INSTALL_ROOT_AMD64=$(pwd)/${DIST}_amd64_staging
54readonly INSTALL_ROOT_I386=$(pwd)/${DIST}_i386_staging
55readonly INSTALL_ROOT_ARM=$(pwd)/${DIST}_arm_staging
56
57readonly REQUIRED_TOOLS="wget"
58
59######################################################################
60# Package Config
61######################################################################
62
63readonly RELEASE_FILE="Release"
64readonly RELEASE_FILE_GPG="Release.gpg"
65readonly RELEASE_LIST="${REPO_BASEDIR}/${RELEASE_FILE}"
66readonly RELEASE_LIST_GPG="${REPO_BASEDIR}/${RELEASE_FILE_GPG}"
67readonly PACKAGE_FILE_AMD64="main/binary-amd64/Packages.bz2"
68readonly PACKAGE_FILE_I386="main/binary-i386/Packages.bz2"
69readonly PACKAGE_FILE_ARM="main/binary-armhf/Packages.bz2"
70readonly PACKAGE_LIST_AMD64="${REPO_BASEDIR}/${PACKAGE_FILE_AMD64}"
71readonly PACKAGE_LIST_I386="${REPO_BASEDIR}/${PACKAGE_FILE_I386}"
72readonly PACKAGE_LIST_ARM="${REPO_BASEDIR}/${PACKAGE_FILE_ARM}"
73
74readonly DEBIAN_DEP_LIST_AMD64="packagelist.${DIST}.amd64"
75readonly DEBIAN_DEP_LIST_I386="packagelist.${DIST}.i386"
76readonly DEBIAN_DEP_LIST_ARM="packagelist.${DIST}.arm"
77
78######################################################################
79# Helper
80######################################################################
81
82Banner() {
83  echo "######################################################################"
84  echo $*
85  echo "######################################################################"
86}
87
88
89SubBanner() {
90  echo "----------------------------------------------------------------------"
91  echo $*
92  echo "----------------------------------------------------------------------"
93}
94
95
96Usage() {
97  egrep "^#@" "${BASH_SOURCE[0]}" | cut --bytes=3-
98}
99
100
101DownloadOrCopy() {
102  if [ -f "$2" ] ; then
103    echo "$2 already in place"
104    return
105  fi
106
107  HTTP=0
108  echo "$1" | grep -qs ^http:// && HTTP=1
109  if [ "$HTTP" = "1" ]; then
110    SubBanner "downloading from $1 -> $2"
111    wget "$1" -O "${2}.partial"
112    mv "${2}.partial" $2
113  else
114    SubBanner "copying from $1"
115    cp "$1" "$2"
116  fi
117}
118
119
120SetEnvironmentVariables() {
121  ARCH=""
122  echo $1 | grep -qs Amd64$ && ARCH=AMD64
123  if [ -z "$ARCH" ]; then
124    echo $1 | grep -qs I386$ && ARCH=I386
125  fi
126  if [ -z "$ARCH" ]; then
127    echo $1 | grep -qs ARM$ && ARCH=ARM
128  fi
129  case "$ARCH" in
130    ARM)
131      INSTALL_ROOT="$INSTALL_ROOT_ARM";
132      ;;
133    AMD64)
134      INSTALL_ROOT="$INSTALL_ROOT_AMD64";
135      ;;
136    I386)
137      INSTALL_ROOT="$INSTALL_ROOT_I386";
138      ;;
139    *)
140      echo "ERROR: Unexpected bad architecture."
141      exit 1
142      ;;
143  esac
144}
145
146
147# some sanity checks to make sure this script is run from the right place
148# with the right tools
149SanityCheck() {
150  Banner "Sanity Checks"
151
152  if ! mkdir -p "${INSTALL_ROOT}" ; then
153     echo "ERROR: ${INSTALL_ROOT} can't be created."
154    exit 1
155  fi
156
157  CHROME_DIR=$(cd "${SCRIPT_DIR}/../../.." && pwd)
158  BUILD_DIR=${CHROME_DIR}/out/sysroot-build/${DIST}
159  mkdir -p ${BUILD_DIR}
160  echo "Using build directory: ${BUILD_DIR}"
161
162  for tool in ${REQUIRED_TOOLS} ; do
163    if ! which ${tool} > /dev/null ; then
164      echo "Required binary $tool not found."
165      echo "Exiting."
166      exit 1
167    fi
168  done
169}
170
171
172ChangeDirectory() {
173  # Change directory to where this script is.
174  cd ${SCRIPT_DIR}
175}
176
177
178ClearInstallDir() {
179  Banner "Clearing dirs in ${INSTALL_ROOT}"
180  rm -rf ${INSTALL_ROOT}/*
181}
182
183
184CreateTarBall() {
185  local tarball=$1
186  Banner "Creating tar ball ${tarball}"
187  tar zcf ${tarball} -C ${INSTALL_ROOT} .
188}
189
190CheckBuildSysrootArgs() {
191  if [ "$#" -ne "1" ]; then
192    echo "ERROR: BuildSysroot commands only take 1 argument"
193    exit 1
194  fi
195
196  if [ -z "$1" ]; then
197    echo "ERROR: tarball name required"
198    exit 1
199  fi
200}
201
202ExtractPackageBz2() {
203  bzcat "$1" | egrep '^(Package:|Filename:|SHA256:) ' > "$2"
204}
205
206GeneratePackageListAmd64() {
207  local output_file="$1"
208  local package_list="${BUILD_DIR}/Packages.${DIST}_amd64.bz2"
209  local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
210  DownloadOrCopy "${PACKAGE_LIST_AMD64}" "${package_list}"
211  VerifyPackageListing "${PACKAGE_FILE_AMD64}" "${package_list}"
212  ExtractPackageBz2 "$package_list" "$tmp_package_list"
213  GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
214    ${DEBIAN_PACKAGES_X86}"
215}
216
217GeneratePackageListI386() {
218  local output_file="$1"
219  local package_list="${BUILD_DIR}/Packages.${DIST}_i386.bz2"
220  local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_amd64"
221  DownloadOrCopy "${PACKAGE_LIST_I386}" "${package_list}"
222  VerifyPackageListing "${PACKAGE_FILE_I386}" "${package_list}"
223  ExtractPackageBz2 "$package_list" "$tmp_package_list"
224  GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}
225    ${DEBIAN_PACKAGES_X86}"
226}
227
228GeneratePackageListARM() {
229  local output_file="$1"
230  local package_list="${BUILD_DIR}/Packages.${DIST}_arm.bz2"
231  local tmp_package_list="${BUILD_DIR}/Packages.${DIST}_arm"
232  DownloadOrCopy "${PACKAGE_LIST_ARM}" "${package_list}"
233  VerifyPackageListing "${PACKAGE_FILE_ARM}" "${package_list}"
234  ExtractPackageBz2 "$package_list" "$tmp_package_list"
235  GeneratePackageList "$tmp_package_list" "$output_file" "${DEBIAN_PACKAGES}"
236}
237
238StripChecksumsFromPackageList() {
239  local package_file="$1"
240  sed -i 's/ [a-f0-9]\{64\}$//' "$package_file"
241}
242
243VerifyPackageFilesMatch() {
244  local downloaded_package_file="$1"
245  local stored_package_file="$2"
246  diff -u "$downloaded_package_file" "$stored_package_file"
247  if [ "$?" -ne "0" ]; then
248    echo "ERROR: downloaded package files does not match $2."
249    echo "You may need to run UpdatePackageLists."
250    exit 1
251  fi
252}
253
254######################################################################
255#
256######################################################################
257
258HacksAndPatchesAmd64() {
259  Banner "Misc Hacks & Patches"
260  # these are linker scripts with absolute pathnames in them
261  # which we rewrite here
262  lscripts="${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libpthread.so \
263            ${INSTALL_ROOT}/usr/lib/x86_64-linux-gnu/libc.so"
264
265  #SubBanner "Rewriting Linker Scripts"
266  sed -i -e 's|/usr/lib/x86_64-linux-gnu/||g'  ${lscripts}
267  sed -i -e 's|/lib/x86_64-linux-gnu/||g' ${lscripts}
268
269  # This is for chrome's ./build/linux/pkg-config-wrapper
270  # which overwrites PKG_CONFIG_PATH internally
271  SubBanner "Package Configs Symlink"
272  mkdir -p ${INSTALL_ROOT}/usr/share
273  ln -s ../lib/x86_64-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
274
275  SubBanner "Adding an additional ld.conf include"
276  LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
277  echo /usr/lib/gcc/x86_64-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
278  echo /usr/lib >> "$LD_SO_HACK_CONF"
279}
280
281
282HacksAndPatchesI386() {
283  Banner "Misc Hacks & Patches"
284  # these are linker scripts with absolute pathnames in them
285  # which we rewrite here
286  lscripts="${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libpthread.so \
287            ${INSTALL_ROOT}/usr/lib/i386-linux-gnu/libc.so"
288
289  #SubBanner "Rewriting Linker Scripts"
290  sed -i -e 's|/usr/lib/i386-linux-gnu/||g'  ${lscripts}
291  sed -i -e 's|/lib/i386-linux-gnu/||g' ${lscripts}
292
293  # This is for chrome's ./build/linux/pkg-config-wrapper
294  # which overwrites PKG_CONFIG_PATH internally
295  SubBanner "Package Configs Symlink"
296  mkdir -p ${INSTALL_ROOT}/usr/share
297  ln -s ../lib/i386-linux-gnu/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
298
299  SubBanner "Adding an additional ld.conf include"
300  LD_SO_HACK_CONF="${INSTALL_ROOT}/etc/ld.so.conf.d/zz_hack.conf"
301  echo /usr/lib/gcc/i486-linux-gnu/4.6 > "$LD_SO_HACK_CONF"
302  echo /usr/lib >> "$LD_SO_HACK_CONF"
303}
304
305
306HacksAndPatchesARM() {
307  Banner "Misc Hacks & Patches"
308  # these are linker scripts with absolute pathnames in them
309  # which we rewrite here
310  lscripts="${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libpthread.so \
311            ${INSTALL_ROOT}/usr/lib/arm-linux-gnueabihf/libc.so"
312
313  #SubBanner "Rewriting Linker Scripts"
314  sed -i -e 's|/usr/lib/arm-linux-gnueabihf/||g' ${lscripts}
315  sed -i -e 's|/lib/arm-linux-gnueabihf/||g' ${lscripts}
316
317  # This is for chrome's ./build/linux/pkg-config-wrapper
318  # which overwrites PKG_CONFIG_PATH internally
319  SubBanner "Package Configs Symlink"
320  mkdir -p ${INSTALL_ROOT}/usr/share
321  ln -s ../lib/arm-linux-gnueabihf/pkgconfig ${INSTALL_ROOT}/usr/share/pkgconfig
322}
323
324
325InstallIntoSysroot() {
326  Banner "Install Libs And Headers Into Jail"
327
328  mkdir -p ${BUILD_DIR}/debian-packages
329  mkdir -p ${INSTALL_ROOT}
330  while (( "$#" )); do
331    local file="$1"
332    local package="${BUILD_DIR}/debian-packages/${file##*/}"
333    shift
334    local sha256sum="$1"
335    shift
336    if [ "${#sha256sum}" -ne "64" ]; then
337      echo "Bad sha256sum from package list"
338      exit 1
339    fi
340
341    Banner "Installing ${file}"
342    DownloadOrCopy ${APT_REPO}/pool/${file} ${package}
343    if [ ! -s "${package}" ] ; then
344      echo
345      echo "ERROR: bad package ${package}"
346      exit 1
347    fi
348    echo "${sha256sum}  ${package}" | sha256sum --quiet -c
349
350    SubBanner "Extracting to ${INSTALL_ROOT}"
351    dpkg --fsys-tarfile ${package}\
352      | tar -xf - --exclude=./usr/share -C ${INSTALL_ROOT}
353  done
354}
355
356
357CleanupJailSymlinks() {
358  Banner "Jail symlink cleanup"
359
360  SAVEDPWD=$(pwd)
361  cd ${INSTALL_ROOT}
362  find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
363    # skip links with non-absolute paths
364    echo "${target}" | grep -qs ^/ || continue
365    echo "${link}: ${target}"
366    case "${link}" in
367      usr/lib/gcc/x86_64-linux-gnu/4.*/* | usr/lib/gcc/i486-linux-gnu/4.*/* | \
368      usr/lib/gcc/arm-linux-gnueabihf/4.*/*)
369        # Relativize the symlink.
370        ln -snfv "../../../../..${target}" "${link}"
371        ;;
372      usr/lib/x86_64-linux-gnu/* | usr/lib/i386-linux-gnu/* | \
373      usr/lib/arm-linux-gnueabihf/*)
374        # Relativize the symlink.
375        ln -snfv "../../..${target}" "${link}"
376        ;;
377      usr/lib/*)
378        # Relativize the symlink.
379        ln -snfv "../..${target}" "${link}"
380        ;;
381      lib64/* | lib/*)
382        # Relativize the symlink.
383        ln -snfv "..${target}" "${link}"
384        ;;
385    esac
386  done
387
388  find lib lib64 usr/lib -type l -printf '%p %l\n' | while read link target; do
389    # Make sure we catch new bad links.
390    if [ ! -r "${link}" ]; then
391      echo "ERROR: FOUND BAD LINK ${link}"
392      ls -l ${link}
393      exit 1
394    fi
395  done
396  cd "$SAVEDPWD"
397}
398
399#@
400#@ BuildSysrootAmd64 <tarball-name>
401#@
402#@    Build everything and package it
403BuildSysrootAmd64() {
404  CheckBuildSysrootArgs $@
405  ClearInstallDir
406  local package_file="$BUILD_DIR/package_with_sha256sum_amd64"
407  GeneratePackageListAmd64 "$package_file"
408  local files_and_sha256sums="$(cat ${package_file})"
409  StripChecksumsFromPackageList "$package_file"
410  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_AMD64"
411  InstallIntoSysroot ${files_and_sha256sums}
412  CleanupJailSymlinks
413  HacksAndPatchesAmd64
414  CreateTarBall "$1"
415}
416
417#@
418#@ BuildSysrootI386 <tarball-name>
419#@
420#@    Build everything and package it
421BuildSysrootI386() {
422  CheckBuildSysrootArgs $@
423  ClearInstallDir
424  local package_file="$BUILD_DIR/package_with_sha256sum_i386"
425  GeneratePackageListI386 "$package_file"
426  local files_and_sha256sums="$(cat ${package_file})"
427  StripChecksumsFromPackageList "$package_file"
428  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_I386"
429  InstallIntoSysroot ${files_and_sha256sums}
430  CleanupJailSymlinks
431  HacksAndPatchesI386
432  CreateTarBall "$1"
433}
434
435#@
436#@ BuildSysrootARM <tarball-name>
437#@
438#@    Build everything and package it
439BuildSysrootARM() {
440  CheckBuildSysrootArgs $@
441  ClearInstallDir
442  local package_file="$BUILD_DIR/package_with_sha256sum_arm"
443  GeneratePackageListARM "$package_file"
444  local files_and_sha256sums="$(cat ${package_file})"
445  StripChecksumsFromPackageList "$package_file"
446  VerifyPackageFilesMatch "$package_file" "$DEBIAN_DEP_LIST_ARM"
447  APT_REPO=${APR_REPO_ARM:=$APT_REPO}
448  InstallIntoSysroot ${files_and_sha256sums}
449  CleanupJailSymlinks
450  HacksAndPatchesARM
451  CreateTarBall "$1"
452}
453
454#
455# CheckForDebianGPGKeyring
456#
457#     Make sure the Debian GPG keys exist. Otherwise print a helpful message.
458#
459CheckForDebianGPGKeyring() {
460  if [ ! -e "$KEYRING_FILE" ]; then
461    echo "Debian GPG keys missing. Install the debian-archive-keyring package."
462    exit 1
463  fi
464}
465
466#
467# VerifyPackageListing
468#
469#     Verifies the downloaded Packages.bz2 file has the right checksums.
470#
471VerifyPackageListing() {
472  local file_path=$1
473  local output_file=$2
474  local release_file="${BUILD_DIR}/${RELEASE_FILE}"
475  local release_file_gpg="${BUILD_DIR}/${RELEASE_FILE_GPG}"
476
477  CheckForDebianGPGKeyring
478
479  DownloadOrCopy ${RELEASE_LIST} ${release_file}
480  DownloadOrCopy ${RELEASE_LIST_GPG} ${release_file_gpg}
481  echo "Verifying: ${release_file} with ${release_file_gpg}"
482  gpgv --keyring $KEYRING_FILE ${release_file_gpg} ${release_file}
483
484  echo "Verifying: ${output_file}"
485  local checksums=$(grep ${file_path} ${release_file} | cut -d " " -f 2)
486  local sha256sum=$(echo ${checksums} | cut -d " " -f 3)
487
488  if [ "${#sha256sum}" -ne "64" ]; then
489    echo "Bad sha256sum from ${RELEASE_LIST}"
490    exit 1
491  fi
492
493  echo "${sha256sum}  ${output_file}" | sha256sum --quiet -c
494}
495
496#
497# GeneratePackageList
498#
499#     Looks up package names in ${BUILD_DIR}/Packages and write list of URLs
500#     to output file.
501#
502GeneratePackageList() {
503  local input_file="$1"
504  local output_file="$2"
505  echo "Updating: ${output_file} from ${input_file}"
506  /bin/rm -f "${output_file}"
507  shift
508  shift
509  for pkg in $@ ; do
510    local pkg_full=$(grep -A 1 " ${pkg}\$" "$input_file" | \
511      egrep -o "pool/.*")
512    if [ -z "${pkg_full}" ]; then
513        echo "ERROR: missing package: $pkg"
514        exit 1
515    fi
516    local pkg_nopool=$(echo "$pkg_full" | sed "s/^pool\///")
517    local sha256sum=$(grep -A 4 " ${pkg}\$" "$input_file" | \
518      grep ^SHA256: | sed 's/^SHA256: //')
519    if [ "${#sha256sum}" -ne "64" ]; then
520      echo "Bad sha256sum from Packages"
521      exit 1
522    fi
523    echo $pkg_nopool $sha256sum >> "$output_file"
524  done
525  # sort -o does an in-place sort of this file
526  sort "$output_file" -o "$output_file"
527}
528
529#@
530#@ UpdatePackageListsAmd64
531#@
532#@     Regenerate the package lists such that they contain an up-to-date
533#@     list of URLs within the Debian archive. (For amd64)
534UpdatePackageListsAmd64() {
535  GeneratePackageListAmd64 "$DEBIAN_DEP_LIST_AMD64"
536  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_AMD64"
537}
538
539#@
540#@ UpdatePackageListsI386
541#@
542#@     Regenerate the package lists such that they contain an up-to-date
543#@     list of URLs within the Debian archive. (For i386)
544UpdatePackageListsI386() {
545  GeneratePackageListI386 "$DEBIAN_DEP_LIST_I386"
546  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_I386"
547}
548
549#@
550#@ UpdatePackageListsARM
551#@
552#@     Regenerate the package lists such that they contain an up-to-date
553#@     list of URLs within the Debian archive. (For arm)
554UpdatePackageListsARM() {
555  GeneratePackageListARM "$DEBIAN_DEP_LIST_ARM"
556  StripChecksumsFromPackageList "$DEBIAN_DEP_LIST_ARM"
557}
558
559if [ $# -eq 0 ] ; then
560  echo "ERROR: you must specify a mode on the commandline"
561  echo
562  Usage
563  exit 1
564elif [ "$(type -t $1)" != "function" ]; then
565  echo "ERROR: unknown function '$1'." >&2
566  echo "For help, try:"
567  echo "    $0 help"
568  exit 1
569else
570  ChangeDirectory
571  SetEnvironmentVariables "$1"
572  SanityCheck
573  "$@"
574fi
575