1#!/bin/bash
2#
3# Build and runs tests for the protobuf project.  The tests as written here are
4# used by both Jenkins and Travis, though some specialized logic is required to
5# handle the differences between them.
6
7on_travis() {
8  if [ "$TRAVIS" == "true" ]; then
9    "$@"
10  fi
11}
12
13# For when some other test needs the C++ main build, including protoc and
14# libprotobuf.
15internal_build_cpp() {
16  if [ -f src/protoc ]; then
17    # Already built.
18    return
19  fi
20
21  if [[ $(uname -s) == "Linux" && "$TRAVIS" == "true" ]]; then
22    # Install GCC 4.8 to replace the default GCC 4.6. We need 4.8 for more
23    # decent C++ 11 support in order to compile conformance tests.
24    sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
25    sudo apt-get update -qq
26    sudo apt-get install -qq g++-4.8
27    export CXX="g++-4.8" CC="gcc-4.8"
28  fi
29
30  ./autogen.sh
31  ./configure
32  make -j2
33}
34
35build_cpp() {
36  internal_build_cpp
37  make check -j2
38  cd conformance && make test_cpp && cd ..
39
40  # Verify benchmarking code can build successfully.
41  cd benchmarks && make && ./generate-datasets && cd ..
42}
43
44build_cpp_distcheck() {
45  ./autogen.sh
46  ./configure
47  make dist
48
49  # List all files that should be included in the distribution package.
50  git ls-files | grep "^\(java\|python\|objectivec\|csharp\|js\|ruby\|cmake\|examples\)" |\
51      grep -v ".gitignore" | grep -v "java/compatibility_tests" > dist.lst
52  # Unzip the dist tar file.
53  DIST=`ls *.tar.gz`
54  tar -xf $DIST
55  cd ${DIST//.tar.gz}
56  # Check if every file exists in the dist tar file.
57  FILES_MISSING=""
58  for FILE in $(<../dist.lst); do
59    if ! file $FILE &>/dev/null; then
60      echo "$FILE is not found!"
61      FILES_MISSING="$FILE $FILES_MISSING"
62    fi
63  done
64  cd ..
65  if [ ! -z "$FILES_MISSING" ]; then
66    echo "Missing files in EXTRA_DIST: $FILES_MISSING"
67    exit 1
68  fi
69
70  # Do the regular dist-check for C++.
71  make distcheck -j2
72}
73
74build_csharp() {
75  # Just for the conformance tests. We don't currently
76  # need to really build protoc, but it's simplest to keep with the
77  # conventions of the other builds.
78  internal_build_cpp
79  NUGET=/usr/local/bin/nuget.exe
80
81  if [ "$TRAVIS" == "true" ]; then
82    # Install latest version of Mono
83    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
84    sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1397BC53640DB551
85    echo "deb http://download.mono-project.com/repo/debian wheezy main" | sudo tee /etc/apt/sources.list.d/mono-xamarin.list
86    sudo apt-get update -qq
87    sudo apt-get install -qq mono-devel referenceassemblies-pcl nunit
88
89    # Then install the dotnet SDK as per Ubuntu 14.04 instructions on dot.net.
90    sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ trusty main" > /etc/apt/sources.list.d/dotnetdev.list'
91    sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893
92    sudo apt-get update -qq
93    sudo apt-get install -qq dotnet-dev-1.0.0-preview2-003121
94  fi
95
96  # Perform "dotnet new" once to get the setup preprocessing out of the
97  # way. That spews a lot of output (including backspaces) into logs
98  # otherwise, and can cause problems. It doesn't matter if this step
99  # is performed multiple times; it's cheap after the first time anyway.
100  mkdir dotnettmp
101  (cd dotnettmp; dotnet new > /dev/null)
102  rm -rf dotnettmp
103
104  (cd csharp/src; dotnet restore)
105  csharp/buildall.sh
106  cd conformance && make test_csharp && cd ..
107}
108
109build_golang() {
110  # Go build needs `protoc`.
111  internal_build_cpp
112  # Add protoc to the path so that the examples build finds it.
113  export PATH="`pwd`/src:$PATH"
114
115  # Install Go and the Go protobuf compiler plugin.
116  on_travis sudo apt-get update -qq
117  on_travis sudo apt-get install -qq golang
118
119  export GOPATH="$HOME/gocode"
120  mkdir -p "$GOPATH/src/github.com/google"
121  rm -f "$GOPATH/src/github.com/google/protobuf"
122  ln -s "`pwd`" "$GOPATH/src/github.com/google/protobuf"
123  export PATH="$GOPATH/bin:$PATH"
124  go get github.com/golang/protobuf/protoc-gen-go
125
126  cd examples && make gotest && cd ..
127}
128
129use_java() {
130  version=$1
131  case "$version" in
132    jdk7)
133      on_travis sudo apt-get install openjdk-7-jdk
134      export PATH=/usr/lib/jvm/java-7-openjdk-amd64/bin:$PATH
135      export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
136      ;;
137    oracle7)
138      if [ "$TRAVIS" == "true" ]; then
139        sudo apt-get install python-software-properties # for apt-add-repository
140        echo "oracle-java7-installer shared/accepted-oracle-license-v1-1 select true" | \
141          sudo debconf-set-selections
142        yes | sudo apt-add-repository ppa:webupd8team/java
143        yes | sudo apt-get install oracle-java7-installer
144      fi;
145      export PATH=/usr/lib/jvm/java-7-oracle/bin:$PATH
146      export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
147      ;;
148  esac
149
150  if [ "$TRAVIS" != "true" ]; then
151    MAVEN_LOCAL_REPOSITORY=/var/maven_local_repository
152    MVN="$MVN -e -X --offline -Dmaven.repo.local=$MAVEN_LOCAL_REPOSITORY"
153  fi;
154
155  which java
156  java -version
157  $MVN -version
158}
159
160# --batch-mode supresses download progress output that spams the logs.
161MVN="mvn --batch-mode"
162
163build_java() {
164  version=$1
165  dir=java_$version
166  # Java build needs `protoc`.
167  internal_build_cpp
168  cp -r java $dir
169  cd $dir && $MVN clean && $MVN test
170  cd ../..
171}
172
173# The conformance tests are hard-coded to work with the $ROOT/java directory.
174# So this can't run in parallel with two different sets of tests.
175build_java_with_conformance_tests() {
176  # Java build needs `protoc`.
177  internal_build_cpp
178  cd java && $MVN test && $MVN install
179  cd util && $MVN package assembly:single
180  cd ../..
181  cd conformance && make test_java && cd ..
182}
183
184build_javanano() {
185  # Java build needs `protoc`.
186  internal_build_cpp
187  cd javanano && $MVN test && cd ..
188}
189
190build_java_jdk7() {
191  use_java jdk7
192  build_java_with_conformance_tests
193}
194build_java_oracle7() {
195  use_java oracle7
196  build_java oracle7
197}
198
199build_javanano_jdk7() {
200  use_java jdk7
201  build_javanano
202}
203build_javanano_oracle7() {
204  use_java oracle7
205  build_javanano
206}
207
208internal_install_python_deps() {
209  if [ "$TRAVIS" != "true" ]; then
210    return;
211  fi
212  # Install tox (OS X doesn't have pip).
213  if [ $(uname -s) == "Darwin" ]; then
214    sudo easy_install tox
215  else
216    sudo pip install tox
217  fi
218  # Only install Python2.6/3.x on Linux.
219  if [ $(uname -s) == "Linux" ]; then
220    sudo apt-get install -y python-software-properties # for apt-add-repository
221    sudo apt-add-repository -y ppa:fkrull/deadsnakes
222    sudo apt-get update -qq
223    sudo apt-get install -y python2.6 python2.6-dev
224    sudo apt-get install -y python3.3 python3.3-dev
225    sudo apt-get install -y python3.4 python3.4-dev
226  fi
227}
228
229build_objectivec_ios() {
230  # Reused the build script that takes care of configuring and ensuring things
231  # are up to date.  The OS X test runs the objc conformance test, so skip it
232  # here.
233  # Note: travis has xctool installed, and we've looked at using it in the past
234  # but it has ended up proving unreliable (bugs), an they are removing build
235  # support in favor of xcbuild (or just xcodebuild).
236  objectivec/DevTools/full_mac_build.sh \
237      --core-only --skip-xcode-osx --skip-objc-conformance "$@"
238}
239
240build_objectivec_ios_debug() {
241  build_objectivec_ios --skip-xcode-release
242}
243
244build_objectivec_ios_release() {
245  build_objectivec_ios --skip-xcode-debug
246}
247
248build_objectivec_osx() {
249  # Reused the build script that takes care of configuring and ensuring things
250  # are up to date.
251  objectivec/DevTools/full_mac_build.sh \
252      --core-only --skip-xcode-ios
253}
254
255build_objectivec_cocoapods_integration() {
256  # First, load the RVM environment in bash, needed to update ruby.
257  source ~/.rvm/scripts/rvm
258  # Update rvm to the latest version. This is needed to solve
259  # https://github.com/google/protobuf/issues/1786 and may not be needed in the
260  # future when Travis updates the default version of rvm.
261  rvm get head
262  # Update ruby to 2.2.3 as the default one crashes with segmentation faults
263  # when using pod.
264  rvm use 2.2.3 --install --binary --fuzzy
265  # Update pod to the latest version.
266  gem install cocoapods --no-ri --no-rdoc
267  objectivec/Tests/CocoaPods/run_tests.sh
268}
269
270build_python() {
271  internal_build_cpp
272  internal_install_python_deps
273  cd python
274  # Only test Python 2.6/3.x on Linux
275  if [ $(uname -s) == "Linux" ]; then
276    envlist=py\{26,27,33,34\}-python
277  else
278    envlist=py27-python
279  fi
280  tox -e $envlist
281  cd ..
282}
283
284build_python_cpp() {
285  internal_build_cpp
286  internal_install_python_deps
287  export LD_LIBRARY_PATH=../src/.libs # for Linux
288  export DYLD_LIBRARY_PATH=../src/.libs # for OS X
289  cd python
290  # Only test Python 2.6/3.x on Linux
291  if [ $(uname -s) == "Linux" ]; then
292    # py26 is currently disabled due to json_format
293    envlist=py\{27,33,34\}-cpp
294  else
295    envlist=py27-cpp
296  fi
297  tox -e $envlist
298  cd ..
299}
300
301build_ruby21() {
302  internal_build_cpp  # For conformance tests.
303  cd ruby && bash travis-test.sh ruby-2.1 && cd ..
304}
305build_ruby22() {
306  internal_build_cpp  # For conformance tests.
307  cd ruby && bash travis-test.sh ruby-2.2 && cd ..
308}
309build_jruby() {
310  internal_build_cpp  # For conformance tests.
311  # TODO(xiaofeng): Upgrade to jruby-9.x. There are some broken jests to be
312  # fixed.
313  cd ruby && bash travis-test.sh jruby-1.7 && cd ..
314}
315build_ruby_all() {
316  build_ruby21
317  build_ruby22
318  build_jruby
319}
320
321build_javascript() {
322  internal_build_cpp
323  cd js && npm install && npm test && cd ..
324}
325
326# Note: travis currently does not support testing more than one language so the
327# .travis.yml cheats and claims to only be cpp.  If they add multiple language
328# support, this should probably get updated to install steps and/or
329# rvm/gemfile/jdk/etc. entries rather than manually doing the work.
330
331# .travis.yml uses matrix.exclude to block the cases where app-get can't be
332# use to install things.
333
334# -------- main --------
335
336if [ "$#" -ne 1 ]; then
337  echo "
338Usage: $0 { cpp |
339            cpp_distcheck |
340            csharp |
341            java_jdk7 |
342            java_oracle7 |
343            javanano_jdk7 |
344            javanano_oracle7 |
345            objectivec_ios |
346            objectivec_ios_debug |
347            objectivec_ios_release |
348            objectivec_osx |
349            objectivec_cocoapods_integration |
350            python |
351            python_cpp |
352            ruby21 |
353            ruby22 |
354            jruby |
355            ruby_all)
356"
357  exit 1
358fi
359
360set -e  # exit immediately on error
361set -x  # display all commands
362eval "build_$1"
363