#!/bin/bash -e set -e # This script must be invoked from the root of the repository (e.g., as # bootstrap/shared/shared_build_bins.sh). It expects to run INSIDE the # bootstrap VM, so it does not have access to environment variables or bootstrap # functions. This script is invoked by shared/shared_configure_chef.sh. # FILECACHE_MOUNT_POINT is exported in shared/shared_configure_chef.sh when invoking # this script. if [[ -z $FILECACHE_MOUNT_POINT ]] { echo "FILECACHE_MOUNT_POINT must be set to proceed! Exiting." > !2 exit 1 } if [[ -z $BUILD_DEST ]] { setglobal BUILD_DEST = 'cookbooks/bcpc-binary-files/files/default'; } # directory used for storing build cache products setglobal BUILD_CACHE_DIR = "$FILECACHE_MOUNT_POINT/build_bins_cache" # Binary versions to grab/build source bootstrap/config/build_bins_versions.sh pushd $BUILD_DEST # copy existing build products out of the cache if dir exists # (will not exist if this is the first time run with the new script) if test -d $BUILD_CACHE_DIR { echo "Copying cached build products..." rsync -avxSH $BUILD_CACHE_DIR/* $[pwd -P] } # Install tools needed for packaging apt-get -y install git ruby-dev make pbuilder python-mock python-configobj python-support cdbs python-all-dev python-stdeb libmysqlclient-dev libldap2-dev libxml2-dev libxslt1-dev libpq-dev build-essential libssl-dev libffi-dev python-dev python-pip # install fpm and support gems if test -z $[gem list --local fpm | grep fpm | cut -f1 -d" ] { pushd $FILECACHE_MOUNT_POINT/fpm_gems/ gem install -l --no-ri --no-rdoc arr-pm-0.0.10.gem backports-3.6.4.gem cabin-0.7.1.gem childprocess-0.5.6.gem clamp-0.6.5.gem ffi-1.9.8.gem fpm-1.3.3.gem json-1.8.2.gem popd } # Delete old kibana 4 deb if test -f kibana_$(VER_KIBANA)_amd64.deb { rm -f kibana_$(VER_KIBANA)_amd64.deb kibana_$(VER_KIBANA).tar.gz } # fluentd plugins and dependencies are fetched by shared_prereqs.sh, just copy them # in from the local cache and add them to $FILES rsync -avxSH $FILECACHE_MOUNT_POINT/fluentd_gems/* $[pwd -P] setglobal FILES = ""$[ls -1 $FILECACHE_MOUNT_POINT/fluentd_gems/*.gem | xargs] $FILES"" # Fetch the cirros image for testing if test ! -f cirros-0.3.4-x86_64-disk.img { cp -v $FILECACHE_MOUNT_POINT/cirros-0.3.4-x86_64-disk.img . } setglobal FILES = ""cirros-0.3.4-x86_64-disk.img $FILES"" # Grab the Ubuntu 14.04 installer image if test ! -f ubuntu-14.04-mini.iso { cp -v $FILECACHE_MOUNT_POINT/ubuntu-14.04-mini.iso ubuntu-14.04-mini.iso } setglobal FILES = ""ubuntu-14.04-mini.iso $FILES"" # Test if diamond package version is <= 3.x, which implies a BrightCoveOS source if test -f diamond.deb { if test $[dpkg-deb -f diamond.deb Version | cut -b1] -le 3 { rm -f diamond.deb } } # Make the diamond package if test ! -f diamond.deb { git clone $FILECACHE_MOUNT_POINT/python-diamond Diamond cd Diamond git checkout $VER_DIAMOND make builddeb setglobal VERSION = $[cat version.txt] cd .. mv Diamond/build/diamond_$(VERSION)_all.deb diamond.deb rm -rf Diamond } setglobal FILES = ""diamond.deb $FILES"" if test ! -f elasticsearch-plugins.tgz { cp -r $FILECACHE_MOUNT_POINT/elasticsearch-head . cd elasticsearch-head git archive --output ../elasticsearch-plugins.tgz --prefix head/_site/ $VER_ESPLUGIN cd .. rm -rf elasticsearch-head } setglobal FILES = ""elasticsearch-plugins.tgz $FILES"" # Fetch pyrabbit if test ! -f pyrabbit-1.0.1.tar.gz { cp -v $FILECACHE_MOUNT_POINT/pyrabbit-1.0.1.tar.gz . } setglobal FILES = ""pyrabbit-1.0.1.tar.gz $FILES"" # Build requests-aws package if test ! -f python-requests-aws_$(VER_REQUESTS_AWS)_all.deb { cp -v $FILECACHE_MOUNT_POINT/requests-aws-$(VER_REQUESTS_AWS).tar.gz . tar zxf requests-aws-$(VER_REQUESTS_AWS).tar.gz fpm -s python -t deb -f requests-aws-$(VER_REQUESTS_AWS)/setup.py rm -rf requests-aws-$(VER_REQUESTS_AWS).tar.gz requests-aws-$(VER_REQUESTS_AWS) } setglobal FILES = ""python-requests-aws_$(VER_REQUESTS_AWS)_all.deb $FILES"" # Build pyzabbix package if test ! -f python-pyzabbix_$(VER_PYZABBIX)_all.deb { cp -v $FILECACHE_MOUNT_POINT/pyzabbix-$(VER_PYZABBIX).tar.gz . tar zxf pyzabbix-$(VER_PYZABBIX).tar.gz fpm -s python -t deb -f pyzabbix-$(VER_PYZABBIX)/setup.py rm -rf pyzabbix-$(VER_PYZABBIX).tar.gz pyzabbix-$(VER_PYZABBIX) } setglobal FILES = ""python-pyzabbix_$(VER_PYZABBIX)_all.deb $FILES"" # Grab Zabbix-Pagerduty notification script if test ! -f pagerduty-zabbix-proxy.py { cp -v $FILECACHE_MOUNT_POINT/pagerduty-zabbix-proxy.py . } setglobal FILES = ""pagerduty-zabbix-proxy.py $FILES"" # Build graphite packages if test ! -f python-carbon_$(VER_GRAPHITE_CARBON)_all.deb { cp -v $FILECACHE_MOUNT_POINT/carbon-$(VER_GRAPHITE_CARBON).tar.gz . tar zxf carbon-$(VER_GRAPHITE_CARBON).tar.gz fpm --python-install-bin /opt/graphite/bin -s python -t deb -f carbon-$(VER_GRAPHITE_CARBON)/setup.py rm -rf carbon-$(VER_GRAPHITE_CARBON) carbon-$(VER_GRAPHITE_CARBON).tar.gz } setglobal FILES = ""python-carbon_$(VER_GRAPHITE_CARBON)_all.deb $FILES"" if test ! -f python-whisper_$(VER_GRAPHITE_WHISPER)_all.deb { cp -v $FILECACHE_MOUNT_POINT/whisper-$(VER_GRAPHITE_WHISPER).tar.gz . tar zxf whisper-$(VER_GRAPHITE_WHISPER).tar.gz fpm --python-install-bin /opt/graphite/bin -s python -t deb -f whisper-$(VER_GRAPHITE_WHISPER)/setup.py rm -rf whisper-$(VER_GRAPHITE_WHISPER) whisper-$(VER_GRAPHITE_WHISPER).tar.gz } setglobal FILES = ""python-whisper_$(VER_GRAPHITE_WHISPER)_all.deb $FILES"" if test ! -f python-graphite-web_$(VER_GRAPHITE_WEB)_all.deb { cp -v $FILECACHE_MOUNT_POINT/graphite-web-$(VER_GRAPHITE_WEB).tar.gz . tar zxf graphite-web-$(VER_GRAPHITE_WEB).tar.gz fpm --python-install-lib /opt/graphite/webapp -s python -t deb -f graphite-web-$(VER_GRAPHITE_WEB)/setup.py rm -rf graphite-web-$(VER_GRAPHITE_WEB) graphite-web-$(VER_GRAPHITE_WEB).tar.gz } setglobal FILES = ""python-graphite-web_$(VER_GRAPHITE_WEB)_all.deb $FILES"" # Rally has a number of dependencies. Some of the dependencies are in apt by default but some are not. Those that # are not are built here. # We build a package for rally here but we also get the tar file of the source because it includes the samples # directory that we want and we need a good place to run our tests from. if test ! -f rally.tar.gz { cp $FILECACHE_MOUNT_POINT/rally/rally-$(VER_RALLY).tar.gz . tar xvf rally-$(VER_RALLY).tar.gz tar zcf rally.tar.gz -C rally-$(VER_RALLY)/ . rm -rf rally-$(VER_RALLY).tar.gz rally-$(VER_RALLY) } # TODO FOR erhudy: fix up these Rally packages to be built like the Graphite stuff # Also test for deb if migrating from 5.1.x if test ! -f rally-pip.tar.gz || test ! -f rally-bin.tar.gz || test ! -f python-pip_$(VER_PIP)_all.deb { # Rally has a very large number of version specific dependencies!! # The latest version of PIP is installed instead of the distro version. We don't want this to block to exit on error # so it is changed here and reset at the end. Several apt packages must be present since easy_install builds # some of the dependencies. # Note: Once we fully switch to trusty/kilo then we should not have to patch this (hopefully). echo "Processing Rally setup..." # Create a deb for pip to replace really old upstream pip if [[ ! -f python-pip_${VER_PIP}_all.deb ]] { cp $FILECACHE_MOUNT_POINT/rally/pip-$(VER_PIP).tar.gz . tar xvzf pip-$(VER_PIP).tar.gz fpm -s python -t deb pip-$(VER_PIP)/setup.py dpkg -i python-pip_$(VER_PIP)_all.deb rm -rf pip-$(VER_PIP) pip-$(VER_PIP).tar.gz } # We install rally and a few other items here. Since fpm does not resolve dependencies but only lists them, we # have to force an install and then tar up the dist-packages and local/bin setglobal PIP_INSTALL = ""pip install --no-cache-dir --disable-pip-version-check --no-index -f $FILECACHE_MOUNT_POINT/rally"" # this kludge is to prevent easy_install from trying to go out to PyPI: # rally calls setuptools.setup_requires(), which uses easy_install to # install any packages listed there; this forces easy_install to use # the same mechanism as we are telling pip to use in $PIP_INSTALL echo -e "[easy_install]\nallow_hosts = ''\nfind_links = file://$FILECACHE_MOUNT_POINT/rally/" > $HOME/.pydistutils.cfg $PIP_INSTALL --default-timeout 60 -I rally $PIP_INSTALL --default-timeout 60 python-openstackclient $PIP_INSTALL -U argparse $PIP_INSTALL -U setuptools tar zcf rally-pip.tar.gz -C /usr/local/lib/python2.7/dist-packages . tar zcf rally-bin.tar.gz --exclude="fpm" --exclude="ruby*" -C /usr/local/bin . } setglobal FILES = ""rally.tar.gz rally-pip.tar.gz rally-bin.tar.gz python-pip_$(VER_PIP)_all.deb $FILES"" # End of Rally # rsync build products with cache directory mkdir -p $BUILD_CACHE_DIR && rsync -avxSH $[pwd -P]/* $BUILD_CACHE_DIR popd # $BUILD_DEST