Merge pull request #1541 from michael-berlin/fix_grpc_python

Fix gRPC Python issue since switch to 0.13.

Author: michael-berlin

bootstrap.sh

@@ -35,7 +35,8 @@ function zk_patch_mac() {
 
 go version 2>&1 >/dev/null || fail "Go is not installed or is not on \$PATH"
 
-. ./dev.env
+# Set up the proper GOPATH for go get below.
+source ./dev.env
 
 mkdir -p $VTROOT/dist
 mkdir -p $VTROOT/bin
@@ -75,22 +76,13 @@ else
   if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then
     brew unlink grpc/grpc/google-protobuf
   fi
-  # protobuf used to be a separate package, now we use the gRPC one
+
+  # protobuf used to be a separate package, now we use the gRPC one.
   rm -rf $VTROOT/dist/protobuf
-  rm -rf $grpc_dist
-  mkdir -p $grpc_dist/usr/local/bin
-  mkdir -p $grpc_dist/usr/local/lib/python2.7/dist-packages
-  # The directory may not have existed yet, so it may not have been
-  # picked up by dev.env yet, but the install needs it to be in
-  # PYTHONPATH.
-  export PYTHONPATH=$(prepend_path $PYTHONPATH $grpc_dist/usr/local/lib/python2.7/dist-packages)
-  export PATH=$(prepend_path $PATH $grpc_dist/usr/local/bin)
-  export LD_LIBRARY_PATH=$(prepend_path $LD_LIBRARY_PATH $grpc_dist/usr/local/lib)
 
-  if [ `uname -s` == "Darwin" ]; then
-    # on OSX tox is installed in the following path
-    export PATH=$(prepend_path $PATH /usr/local/Cellar/python/2.7.11/Frameworks/Python.framework/Versions/2.7/bin)
-  fi
+  # Cleanup any existing data and re-create the directory.
+  rm -rf $grpc_dist
+  mkdir -p $grpc_dist
 
   ./travis/install_grpc.sh $grpc_dist || fail "gRPC build failed"
   echo "$grpc_ver" > $grpc_dist/.build_finished
@@ -99,6 +91,11 @@ else
   if [[ `uname -s` == "Darwin" && "$(brew list -1 | grep google-protobuf)" ]]; then
     brew link grpc/grpc/google-protobuf
   fi
+
+  # Add newly installed Python code to PYTHONPATH such that other Python module
+  # installations can reuse it. (Once bootstrap.sh has finished, run
+  # source dev.env instead to set the correct PYTHONPATH.)
+  export PYTHONPATH=$(prepend_path $PYTHONPATH $grpc_dist/usr/local/lib/python2.7/dist-packages)
 fi
 
 ln -nfs $VTTOP/third_party/go/launchpad.net $VTROOT/src

dev.env

@@ -2,6 +2,15 @@
 # Use of this source code is governed by a BSD-style license that can
 # be found in the LICENSE file.
 
+# Import prepend_path function.
+dir="$(dirname "${BASH_SOURCE[0]}")"
+source "${dir}/tools/shell_functions.inc"
+if [ $? -ne 0 ]; then
+  echo "failed to load tools/shell_functions.inc"
+  return 1
+fi
+
+
 export VTTOP=$(pwd)
 export VTROOT="${VTROOT:-${VTTOP/\/src\/github.com\/youtube\/vitess/}}"
 # VTTOP sanity check
@@ -16,19 +25,6 @@ mkdir -p $VTDATAROOT
 
 export VTPORTSTART=15000
 
-function prepend_path()
-{
-  # $1 path variable
-  # $2 path to add
-  if [ -d "$2" ] && [[ ":$1:" != *":$2:"* ]]; then
-    echo "$2:$1"
-  else
-    echo "$1"
-  fi
-  if [ -d "$1" ] && [[ ":$PATH:" != *":$1:"* ]]; then
-    export PATH="$1:$PATH"
-  fi
-}
 
 for pypath in $(find $VTROOT/dist -name site-packages -or -name dist-packages | grep -v src/python/grpcio/.tox/py27/lib/python2.7/site-packages)
 do

docker/bootstrap/Dockerfile.common

@@ -33,7 +33,8 @@ RUN curl -sL --connect-timeout 10 --retry 3 \
     https://bootstrap.pypa.io/get-pip.py | python
 
 # Install PHP modules for running tests
-RUN mkdir -p /vt/bin && \
+RUN export MAKEFLAGS="-j$(nproc)" && \
+    mkdir -p /vt/bin && \
     curl -sL --connect-timeout 10 --retry 3 \
         https://phar.phpunit.de/phpunit-4.8.9.phar > /vt/bin/phpunit && \
     chmod +x /vt/bin/phpunit && \
@@ -46,7 +47,7 @@ RUN mkdir -p /vt/bin && \
 RUN mkdir -p /vt/dist && \
     cd /vt/dist && \
     curl -sL --connect-timeout 10 --retry 3 \
-        http://www.carfab.com/apachesoftware/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz | tar -xz && \
+        http://www-us.apache.org/dist/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz | tar -xz && \
     mv apache-maven-3.3.3 maven
 
 # Install etcd v2.0
@@ -84,7 +85,8 @@ COPY travis /vt/src/github.com/youtube/vitess/travis
 COPY composer.json composer.lock /vt/src/github.com/youtube/vitess/
 
 # Install gRPC (and protobuf) as root
-RUN cd /vt/dist && \
+RUN export MAKEFLAGS="-j$(nproc)" && \
+    cd /vt/dist && \
     INSTALL_GRPC_PHP=`php-config --extension-dir` $VTTOP/travis/install_grpc.sh && \
     echo 'extension=grpc.so' > /etc/php5/cli/conf.d/20-grpc.ini && \
     rm -rf /vt/dist/grpc

tools/shell_functions.inc

@@ -17,3 +17,19 @@ function goversion_min() {
   [ "$gotminor" -lt "$wantminor" ] && return 1
   return 0
 }
+
+# prepend_path returns $2 prepended the colon separated path $1.
+# If it's already part of the path, it won't be added again.
+# If $1 is a directory, it will also be prepended to $PATH.
+function prepend_path() {
+  # $1 path variable
+  # $2 path to add
+  if [ -d "$2" ] && [[ ":$1:" != *":$2:"* ]]; then
+    echo "$2:$1"
+  else
+    echo "$1"
+  fi
+  if [ -d "$1" ] && [[ ":$PATH:" != *":$1:"* ]]; then
+    export PATH="$1:$PATH"
+  fi
+}

travis/install_grpc.sh

@@ -5,20 +5,29 @@
 # as root in the image.
 set -ex
 
+# Import prepend_path function.
+dir="$(dirname "${BASH_SOURCE[0]}")"
+source "${dir}/../tools/shell_functions.inc"
+if [ $? -ne 0 ]; then
+  echo "failed to load ../tools/shell_functions.inc"
+  return 1
+fi
+
 # grpc_dist can be empty, in which case we just install to the default paths
 grpc_dist="$1"
 if [ -n "$grpc_dist" ]; then
   cd $grpc_dist
 fi
 
-# for python, we'll need the latest virtualenv and tox.
-# running gRPC requires the six package, version >=1.10.
+# Python requires a very recent version of virtualenv.
 if [ -n "$grpc_dist" ]; then
   # Create a virtualenv, which also creates a virualenv-boxed pip.
   virtualenv $grpc_dist/usr/local
-  $grpc_dist/usr/local/bin/pip install --upgrade --ignore-installed virtualenv tox six
+  $grpc_dist/usr/local/bin/pip install --upgrade --ignore-installed virtualenv
 else
-  pip install --upgrade --ignore-installed virtualenv tox six
+  # system wide installations require an explicit upgrade of
+  # certain gRPC Python dependencies e.g. "six" on Debian Jessie.
+  pip install --upgrade --ignore-installed six
 fi
 
 # clone the repository, setup the submodules
@@ -48,25 +57,29 @@ else
   make install
 fi
 
-# build and install python protobuf side
-cd python
-if [ -n "$grpc_dist" ]; then
-  python setup.py build --cpp_implementation
-  python setup.py install --cpp_implementation --prefix=$grpc_dist/usr/local
-else
-  python setup.py build --cpp_implementation
-  python setup.py install --cpp_implementation
-fi
-
 # now install grpc itself
-cd ../../..
+cd ../..
 if [ -n "$grpc_dist" ]; then
   make install prefix=$grpc_dist/usr/local
+
+  # Add bin directory to the path such that gRPC python won't complain that
+  # it cannot find "grpc_python_plugin".
+  export PATH=$(prepend_path $PATH $grpc_dist/usr/local/bin)
 else
   make install
 fi
 
+# Pin the protobuf python dependency to a specific version which is >=3.0.0a3.
+#
+# This prevents us from running into the bug that the protobuf package with the
+# version "3.0.0-alpha-1" is treated as newer than "3.0.0a3" (alpha-3).
+# See: https://github.com/google/protobuf/issues/855
+# Also discussed here: https://github.com/grpc/grpc/issues/5534
+# In particular, we hit this issue on Travis.
+sed -i -e 's/protobuf>=3.0.0a3/protobuf==3.0.0a3/' setup.py
+
 # and now build and install gRPC python libraries
+# (Dependencies like protobuf python will be installed automatically.)
 if [ -n "$grpc_dist" ]; then
   $grpc_dist/usr/local/bin/pip install .
 else