#!/bin/bash # # Build and runs tests for the protobuf project. The tests as written here are # used by both Jenkins and Travis, though some specialized logic is required to # handle the differences between them. on_travis() { if [ "$TRAVIS" == "true" ]; then "$@" fi } # For when some other test needs the C++ main build, including protoc and # libprotobuf. internal_build_cpp() { if [ -f src/protoc ]; then # Already built. return fi if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more # decent C++ 11 support in order to compile conformance tests. sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y sudo apt-get update -qq sudo apt-get install -qq g++-4.8 export CXX="g++-4.8" CC="gcc-4.8" fi ./autogen.sh ./configure CXXFLAGS="-fPIC" # -fPIC is needed for python cpp test. # See python/setup.py for more details make -j2 } build_cpp() { internal_build_cpp make check -j2 cd conformance && make test_cpp && cd .. # Verify benchmarking code can build successfully. git submodule init git submodule update cd third_party/benchmark && cmake -DCMAKE_BUILD_TYPE=Release && make && cd ../.. cd benchmarks && make && ./generate-datasets && cd .. } build_cpp_distcheck() { ./autogen.sh ./configure make dist # List all files that should be included in the distribution package. git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|cmake\|examples\)" |\ grep -v ".gitignore" | grep -v "java/compatibility_tests" > dist.lst # Unzip the dist tar file. DIST=`ls *.tar.gz` tar -xf $DIST cd ${DIST//.tar.gz} # Check if every file exists in the dist tar file. FILES_MISSING="" for FILE in $(<../dist.lst); do if ! file $FILE &>/dev/null; then echo "$FILE is not found!" FILES_MISSING="$FILE $FILES_MISSING" fi done cd .. if [ ! -z "$FILES_MISSING" ]; then echo "Missing files in EXTRA_DIST: $FILES_MISSING" exit 1 fi # Do the regular dist-check for C++. make distcheck -j2 } build_csharp() { # Just for the conformance tests. We don't currently # need to really build protoc, but it's simplest to keep with the # conventions of the other builds. internal_build_cpp NUGET=/usr/local/bin/nuget.exe if [ "$TRAVIS" == "true" ]; then # Install latest version of Mono sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1397BC53640DB551 echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list sudo apt-get update -qq sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit # Then install the dotnet SDK as per Ubuntu 14.04 instructions on dot.net. sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list' sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 sudo apt-get update -qq sudo apt-get install -qq dotnet-dev-1.0.0-preview2-003121 fi # Perform "dotnet new" once to get the setup preprocessing out of the # way. That spews a lot of output (including backspaces) into logs # otherwise, and can cause problems. It doesn't matter if this step # is performed multiple times; it's cheap after the first time anyway. mkdir dotnettmp (cd dotnettmp; dotnet new > /dev/null) rm -rf dotnettmp (cd csharp/src; dotnet restore) csharp/buildall.sh cd conformance && make test_csharp && cd .. } build_golang() { # Go build needs `protoc`. internal_build_cpp # Add protoc to the path so that the examples build finds it. export PATH="`pwd`/src:$PATH" # Install Go and the Go protobuf compiler plugin. on_travis sudo apt-get update -qq on_travis sudo apt-get install -qq golang export GOPATH="$HOME/gocode" mkdir -p "$GOPATH/src/github.com/google" rm -f "$GOPATH/src/github.com/google/protobuf" ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf" export PATH="$GOPATH/bin:$PATH" go get github.com/golang/protobuf/protoc-gen-go cd examples && make gotest && cd .. } use_java() { version=$1 case "$version" in jdk7) on_travis sudo apt-get install openjdk-7-jdk export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; oracle7) if [ "$TRAVIS" == "true" ]; then sudo apt-get install python-software-properties # for apt-add-repository echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \ sudo debconf-set-selections yes | sudo apt-add-repository ppa:webupd8team/java yes | sudo apt-get install oracle-java7-installer fi; export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 ;; esac if [ "$TRAVIS" != "true" ]; then MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY" fi; which java java -version $MVN -version } # --batch-mode supresses download progress output that spams the logs. MVN="mvn --batch-mode" build_java() { version=$1 dir=java_$version # Java build needs `protoc`. internal_build_cpp cp -r java $dir cd $dir && $MVN clean && $MVN test cd ../.. } # The conformance tests are hard-coded to work with the $ROOT/java directory. # So this can't run in parallel with two different sets of tests. build_java_with_conformance_tests() { # Java build needs `protoc`. internal_build_cpp cd java && $MVN test && $MVN install cd util && $MVN package assembly:single cd ../.. cd conformance && make test_java && cd .. } build_javanano() { # Java build needs `protoc`. internal_build_cpp cd javanano && $MVN test && cd .. } build_java_jdk7() { use_java jdk7 build_java_with_conformance_tests } build_java_oracle7() { use_java oracle7 build_java oracle7 } build_java_compatibility() { use_java jdk7 internal_build_cpp # Use the unit-tests extraced from 2.5.0 to test the compatibilty between # 3.0.0-beta-4 and the current version. cd java/compatibility_tests/v2.5.0 ./test.sh 3.0.0-beta-4 } build_javanano_jdk7() { use_java jdk7 build_javanano } build_javanano_oracle7() { use_java oracle7 build_javanano } internal_install_python_deps() { if [ "$TRAVIS" != "true" ]; then return; fi # Install tox (OS X doesn't have pip). if [ $(uname -s) == "Darwin" ]; then sudo easy_install tox else sudo pip install tox fi # Only install Python2.6/3.x on Linux. if [ $(uname -s) == "Linux" ]; then sudo apt-get install -y python-software-properties # for apt-add-repository sudo apt-add-repository -y ppa:fkrull/deadsnakes sudo apt-get update -qq sudo apt-get install -y python2.6 python2.6-dev sudo apt-get install -y python3.3 python3.3-dev sudo apt-get install -y python3.4 python3.4-dev fi } build_objectivec_ios() { # Reused the build script that takes care of configuring and ensuring things # are up to date. The OS X test runs the objc conformance test, so skip it # here. # Note: travis has xctool installed, and we've looked at using it in the past # but it has ended up proving unreliable (bugs), an they are removing build # support in favor of xcbuild (or just xcodebuild). objectivec/DevTools/full_mac_build.sh \ --core-only --skip-xcode-osx --skip-objc-conformance "$@" } build_objectivec_ios_debug() { build_objectivec_ios --skip-xcode-release } build_objectivec_ios_release() { build_objectivec_ios --skip-xcode-debug } build_objectivec_osx() { # Reused the build script that takes care of configuring and ensuring things # are up to date. objectivec/DevTools/full_mac_build.sh \ --core-only --skip-xcode-ios } build_objectivec_cocoapods_integration() { # First, load the RVM environment in bash, needed to update ruby. source ~/.rvm/scripts/rvm # Update rvm to the latest version. This is needed to solve # https://github.com/google/protobuf/issues/1786 and may not be needed in the # future when Travis updates the default version of rvm. rvm get head # Update ruby to 2.2.3 as the default one crashes with segmentation faults # when using pod. rvm use 2.2.3 --install --binary --fuzzy # Update pod to the latest version. gem install cocoapods --no-ri --no-rdoc objectivec/Tests/CocoaPods/run_tests.sh } build_python() { internal_build_cpp internal_install_python_deps cd python # Only test Python 2.6/3.x on Linux if [ $(uname -s) == "Linux" ]; then envlist=py\{26,27,33,34\}-python else envlist=py27-python fi tox -e $envlist cd .. } build_python_cpp() { internal_build_cpp internal_install_python_deps export LD_LIBRARY_PATH=../src/.libs # for Linux export DYLD_LIBRARY_PATH=../src/.libs # for OS X cd python # Only test Python 2.6/3.x on Linux if [ $(uname -s) == "Linux" ]; then # py26 is currently disabled due to json_format envlist=py\{27,33,34\}-cpp else envlist=py27-cpp fi tox -e $envlist cd .. } build_ruby21() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.1 && cd .. } build_ruby22() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-2.2 && cd .. } build_jruby() { internal_build_cpp # For conformance tests. # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be # fixed. cd ruby && bash travis-test.sh jruby-1.7 && cd .. } build_ruby_all() { build_ruby21 build_ruby22 build_jruby } build_javascript() { internal_build_cpp cd js && npm install && npm test && cd .. } use_php() { VERSION=$1 PHP=`which php` PHP_CONFIG=`which php-config` PHPIZE=`which phpize` rm $PHP rm $PHP_CONFIG rm $PHPIZE cp "/usr/bin/php$VERSION" $PHP cp "/usr/bin/php-config$VERSION" $PHP_CONFIG cp "/usr/bin/phpize$VERSION" $PHPIZE } use_php_zts() { VERSION=$1 PHP=`which php` PHP_CONFIG=`which php-config` PHPIZE=`which phpize` ln -sfn "/usr/local/php-${VERSION}-zts/bin/php" $PHP ln -sfn "/usr/local/php-${VERSION}-zts/bin/php-config" $PHP_CONFIG ln -sfn "/usr/local/php-${VERSION}-zts/bin/phpize" $PHPIZE } use_php_bc() { VERSION=$1 PHP=`which php` PHP_CONFIG=`which php-config` PHPIZE=`which phpize` ln -sfn "/usr/local/php-${VERSION}-bc/bin/php" $PHP ln -sfn "/usr/local/php-${VERSION}-bc/bin/php-config" $PHP_CONFIG ln -sfn "/usr/local/php-${VERSION}-bc/bin/phpize" $PHPIZE } build_php5.5() { use_php 5.5 rm -rf vendor cp -r /usr/local/vendor-5.5 vendor ./vendor/bin/phpunit } build_php5.5_c() { use_php 5.5 cd php/tests && /bin/bash ./test.sh && cd ../.. } build_php5.5_zts_c() { use_php_zts 5.5 wget https://phar.phpunit.de/phpunit-old.phar -O /usr/bin/phpunit cd php/tests && /bin/bash ./test.sh && cd ../.. } build_php5.5_32() { use_php_bc 5.5 rm -rf vendor cp -r /usr/local/vendor-5.5 vendor ./vendor/bin/phpunit } build_php5.6() { use_php 5.6 rm -rf vendor cp -r /usr/local/vendor-5.6 vendor ./vendor/bin/phpunit } build_php5.6_c() { use_php 5.6 cd php/tests && /bin/bash ./test.sh && cd ../.. } build_php5.6_mac() { # Install PHP curl -s https://php-osx.liip.ch/install.sh | bash -s 5.6 export PATH="/usr/local/php5-5.6.25-20160831-101628/bin:$PATH" # Install phpunit curl https://phar.phpunit.de/phpunit.phar -L -o phpunit.phar chmod +x phpunit.phar sudo mv phpunit.phar /usr/local/bin/phpunit # Install valgrind echo "#! /bin/bash" > valgrind chmod ug+x valgrind sudo mv valgrind /usr/local/bin/valgrind # Test cd php/tests && /bin/bash ./test.sh && cd ../.. } build_php7.0() { use_php 7.0 rm -rf vendor cp -r /usr/local/vendor-7.0 vendor ./vendor/bin/phpunit } build_php7.0_c() { use_php 7.0 cd php/tests && /bin/bash ./test.sh && cd ../.. } build_php_all() { build_php5.5 build_php5.6 build_php7.0 build_php5.5_c build_php5.6_c # build_php7.0_c build_php5.5_zts_c } build_php_all_32() { build_php5.5_32 } # Note: travis currently does not support testing more than one language so the # .travis.yml cheats and claims to only be cpp. If they add multiple language # support, this should probably get updated to install steps and/or # rvm/gemfile/jdk/etc. entries rather than manually doing the work. # .travis.yml uses matrix.exclude to block the cases where app-get can't be # use to install things. # -------- main -------- if [ "$#" -ne 1 ]; then echo " Usage: $0 { cpp | cpp_distcheck | csharp | java_jdk7 | java_oracle7 | java_compatibility | javanano_jdk7 | javanano_oracle7 | objectivec_ios | objectivec_ios_debug | objectivec_ios_release | objectivec_osx | objectivec_cocoapods_integration | python | python_cpp | ruby21 | ruby22 | jruby | ruby_all | php5.5 | php5.5_c | php5.6 | php5.6_c | php7.0 | php7.0_c | php_all) " exit 1 fi set -e # exit immediately on error set -x # display all commands eval "build_$1"