travis tweaks to make sure the correct versions of python are install…

…ed (onnx#584)

* Revert "Revert "osx travis support (onnx#566)""

This reverts commit 9621872.

* travis tweaks to make sure the correct versions of python are installed

.travis.yml

@@ -1,32 +1,62 @@
-os: linux
-dist: trusty
-sudo: required
-language: python
-python:
-  - "2.7"
-  - "3.6"
-
-addons:
-  apt:
-    packages:
-      - dos2unix
+language: generic
+
+matrix:
+  include:
+    - os: linux
+      sudo: required
+      env: PYTHON_VERSION=python2
+      language: python
+      python: "2.7"
+    - os: linux
+      sudo: required
+      env: PYTHON_VERSION=python3
+      language: python
+      python: "3.6"
+    - os: osx
+      osx_image: xcode9.3beta
+      env: PYTHON_VERSION=python2
+    - os: osx
+      osx_image: xcode9.3beta
+      env: PYTHON_VERSION=python3
+    - os: linux
+      sudo: required
+      env: PYTHON_VERSION=python2 ONNX_ML=1
+      language: python
+      python: "2.7"
+    - os: linux
+      sudo: required
+      env: PYTHON_VERSION=python3 ONNX_ML=1
+      language: python
+      python: "3.6"
+    - os: osx
+      osx_image: xcode9.3beta
+      env: PYTHON_VERSION=python2 ONNX_ML=1
+    - os: osx
+      osx_image: xcode9.3beta
+      env: PYTHON_VERSION=python3 ONNX_ML=1
+
+env:
+  global:
+    - PB_VERSION=2.6.1
+
+before_install:
+  - ./.travis/before_install.sh
 
 install:
   - ./.travis/install.sh
 
 script:
-  - ./.travis/build.sh
+  - ./.travis/script.sh
+
+after_success:
+  - ./.travis/after_success.sh
+
+after_failure:
+  - ./.travis/after_failure.sh
 
 cache:
+  - timeout: 300
   - directories:
-    - $HOME/.build_cache
+    - $BUILD_CCACHE_DIR
     - $HOME/.ccache
-
-env:
-  global:
-    - PB_VERSION=2.6.1
-  matrix:
-    - USE_NINJA=false
-    - USE_NINJA=true
-    - ONNX_ML=1 USE_NINJA=false
-    - ONNX_ML=1 USE_NINJA=true
+    - $HOME/.cache/pb

.travis/after_failure.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
+source "${script_path%/*}/setup.sh"

.travis/after_success.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
+source "${script_path%/*}/setup.sh"

.travis/before_install.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Don't source setup.sh here, because the virtualenv might not be set up yet
+
+export NUMCORES=`grep -c ^processor /proc/cpuinfo`
+if [ ! -n "$NUMCORES" ]; then
+  export NUMCORES=`sysctl -n hw.ncpu`
+fi
+echo Using $NUMCORES cores
+
+# Install dependencies
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  sudo apt-get update
+  APT_INSTALL_CMD='sudo apt-get install -y --no-install-recommends'
+  $APT_INSTALL_CMD dos2unix
+
+  function install_protobuf() {
+      # Install protobuf
+      pb_dir="~/.cache/pb"
+      mkdir -p "$pb_dir"
+      wget -qO- "https://github.com/google/protobuf/releases/download/v${PB_VERSION}/protobuf-${PB_VERSION}.tar.gz" | tar -xz -C "$pb_dir" --strip-components 1
+      ccache -z
+      cd "$pb_dir" && ./configure && make -j${NUMCORES} && make check && sudo make install && sudo ldconfig
+      ccache -s
+  }
+
+  install_protobuf
+
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  brew install ccache protobuf
+  if [ "${PYTHON_VERSION}" == "python3" ]; then
+    brew upgrade python
+  fi
+else
+  echo Unknown OS: $TRAVIS_OS_NAME
+  exit 1
+fi
+
+# TODO consider using "python3.6 -m venv"
+# which is recommended by python3.6 and may make some difference
+if [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  pip install virtualenv
+  virtualenv -p "${PYTHON_VERSION}" "${HOME}/virtualenv"
+  source "${HOME}/virtualenv/bin/activate"
+fi
+
+# Update all existing python packages
+pip list --outdated --format=freeze | grep -v '^\-e' | cut -d = -f 1  | xargs -n1 pip install -U
+
+pip install pytest-cov nbval

.travis/install.sh

@@ -1,17 +1,6 @@
 #!/bin/bash
 
-scripts_dir=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))
-source "$scripts_dir/common"
+script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
+source "${script_path%/*}/setup.sh"
 
-# install protobuf
-pb_dir="$build_cache_dir/pb"
-mkdir -p $pb_dir
-wget -qO- "https://github.com/google/protobuf/releases/download/v$PB_VERSION/protobuf-$PB_VERSION.tar.gz" | tar -xvz -C "$pb_dir" --strip-components 1
-ccache -z
-cd "$pb_dir" && ./configure && make && make check && sudo make install && sudo ldconfig
-ccache -s
-
-
-if [[ $USE_NINJA == true ]]; then
-    pip install ninja
-fi
+time ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI pip install -v .

.travis/build.sh → .travis/script.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
 
-scripts_dir=$(dirname $(readlink -e "${BASH_SOURCE[0]}"))
-source "$scripts_dir/common";
+script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
+source "${script_path%/*}/setup.sh"
 
 onnx_dir="$PWD"
 
-# install onnx
-cd $onnx_dir
-ccache -z
-ONNX_NAMESPACE=ONNX_NAMESPACE_FOO_BAR_FOR_CI pip install -v .
-ccache -s
-
 # onnx tests
 cd $onnx_dir
 pip install pytest-cov nbval

.travis/setup.sh

@@ -0,0 +1,18 @@
+set -ex
+
+export top_dir=$(dirname ${0%/*})
+
+if  [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  source "${HOME}/virtualenv/bin/activate"
+fi
+
+# setup ccache
+if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+  export PATH="/usr/lib/ccache:$PATH"
+elif [ "$TRAVIS_OS_NAME" == "osx" ]; then
+  export PATH="/usr/local/opt/ccache/libexec:$PATH"
+else
+  echo Unknown OS: $TRAVIS_OS_NAME
+  exit 1
+fi
+ccache --max-size 1G