[cyb-381] binary packages auto build

.travis.yml

@@ -1,41 +1,63 @@
-language: c
+language: minimal
+
 git:
   depth: false
+
 services:
   - docker
+
 before_install:
   - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+
 cache:
   directories:
     - $HOME/.ccache
   timeout: 1000
+
+before_script:
+  - docker pull mixbytes/haya-build-tools-$DISTRO
+  - docker tag mixbytes/haya-build-tools-$DISTRO haya-build-tools-$DISTRO
+  - |
+    if [[ "$TRAVIS_BRANCH" =~ ^debug- ]]; then
+      BUILD_TYPE="Debug"
+    else
+      BUILD_TYPE="Release"
+    fi
+  - |
+    if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ $BUILD_TYPE != "Debug" ]; then
+      TESTS="simulator|unit_test_wabt"
+    elif [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ $BUILD_TYPE != "Debug" ]; then
+      TESTS="simulator|unit_test_wabt|nodeos_sanity|nodeos_run"
+    fi
+
+script:
+  - ./cicd/run build -d $DISTRO -p --build-type=${BUILD_TYPE} --tests=$TESTS haya
+  - |
+    if [ $DISTRO == "ubuntu" ]; then
+      ./cicd/run build -d $DISTRO -i mixbytes/haya:$TRAVIS_BRANCH haya
+      docker push mixbytes/haya:$TRAVIS_BRANCH
+      if [ $TRAVIS_BRANCH == "master" ]; then
+        docker tag mixbytes/haya:master mixbytes/haya:latest
+        docker push mixbytes/haya:latest
+      fi
+    fi
+
+deploy:
+  provider: releases
+  api_key:
+    secure: "jBbGdRNt1t8yYdvdYS/kDpzAZe32Xr4AbwJjnxyGKw3S3gsY64IV2GAEcpWnQA5+wPNprGc7YhcQZv8OfzlVoSA6EfEsFOe8ZIG1Qspq1dA+HqcrwxyOS2t0FcmOqdpmdsv+NXBpxO+RE0ukT5OrZO+2HM8dw9XPO8aEuAfKOAhSdW30blRzu/Yo2bBBIh/WA//TFKiOz/UrAv25Y0Jws6BXY/A8eJkP5KiAVnEsrSKpWOYZfXwUlVLxZ8edm1NCm/QIflO2K8CKTFYIFJ2YbdvCx1xE7soWIY4X3UxUea3svjax4/E/HI+qYmKoJPQzSLgOk21DzLH96MetO9TicPX9Bd5VhMHEcp76dLpEpcu1pS67v3u4nrFX8fw12xmZ40pvwAzUsng/e6QtFvudcQMNS/cSb+YsP4/akJkqg6vA0A66yMT8OY8AiPNfJZbiZA9a9waLmKGwjwYDdKkYDdssNdHmnUuzFpEDmESjo+VDIbTnglKwX5bq4BiGrDsdqgk93Qz4xY1VC3ItSX72iGIB74INlUiNQ9TDRVU66OTSqC9/XLSWZjwYWupXq2UpML0I47wtduIyEIdc/HcbQjqZRO/+L4/IduG0XBx5vlbTfLz+OpVPM9FxiJBvuIiOPNz1/O10WomMx5I0rNZOCDmeROR8OQgKs5siWA3ceoU="
+  file_glob: true
+  file: build/pkgs/*
+  skip_cleanup: true
+  overwrite: true
+  on:
+    tags: true
+
+env:
+  matrix:
+    - "DISTRO=ubuntu"
+    - "DISTRO=centos"
+
 matrix:
   fast_finish: true
-jobs:
-  include:
-    - stage: build
-      script:
-        - set -e
-        - BUILD_TYPE=Release
-
-        - |
-          if [[ $TRAVIS_BRANCH =~ ^debug- ]]; then
-            echo "Setting build type to Debug"
-            BUILD_TYPE=Debug
-          fi
-
-        - cp -r $HOME/.ccache .ccache #copy cache into build context
-        - docker build --build-arg registry=mixbytes --build-arg build_type=$BUILD_TYPE -t tmp-builder -f Docker/Dockerfile --target builder .
-        - docker run --rm -v $HOME/.ccache:/out tmp-builder /bin/bash -c "cp -r /haya/.ccache/* /out" #save updated cache
-        - docker build --build-arg registry=mixbytes --build-arg build_type=$BUILD_TYPE  -t mixbytes/haya:$TRAVIS_BRANCH -f Docker/Dockerfile . #build main image
-
-        - if [ "$TRAVIS_PULL_REQUEST" = "false" ] && [ $BUILD_TYPE != "Debug" ]; then docker run tmp-builder /bin/bash -c 'cd /tmp/build && ctest -R "simulator|unit_test"'; fi
-        - if [ "$TRAVIS_PULL_REQUEST" != "false" ] && [ $BUILD_TYPE != "Debug" ]; then docker run tmp-builder /bin/bash -c 'cd /tmp/build && ctest -R "simulator|unit_test|nodeos_sanity|nodeos_run" -E "mongo"'; fi
-
-        - docker push mixbytes/haya:$TRAVIS_BRANCH #push main image to registry
-        - |
-          if [ $TRAVIS_BRANCH == "master" ]; then
-            docker tag mixbytes/haya:master mixbytes/haya:latest
-            docker push mixbytes/haya:latest
-          fi
 

cicd/Dockerfile-centos-tools

@@ -23,4 +23,5 @@ RUN yum -y update && \
       gettext-devel file sudo libusbx-devel libcurl-devel && \
       git submodule update --init --recursive && \
       cd scripts && \
-      bash ${project_name}_build.sh -y -t
+      bash ${project_name}_build.sh -y -t && \
+      rm -rf /var/src && rm -rf /var/cache/*

cicd/Dockerfile-ubuntu-tools

@@ -19,4 +19,5 @@ RUN apt-get update && \
       autoconf libtool curl zlib1g-dev sudo ruby libusb-1.0-0-dev libcurl4-gnutls-dev pkg-config && \
       git submodule update --init --recursive && \
       cd scripts && \
-      bash ${project_name}_build.sh -y -t
+      bash ${project_name}_build.sh -y -t && \
+      rm -rf /var/src && rm -rf /root/src && rm -rf /var/cache

cicd/README.md

@@ -50,6 +50,8 @@ Options:
     -i   build Docker image
     -s   start Docker container and run shell in specific platform (-d option required)
     -d   target platform {ubuntu,centos}
+    --tests         unit tests regex pattern
+    --build-type    build type {Debug,Release,RelWithDebInfo}
 
 Example:
 # Build toolchain and project package for Ubuntu

cicd/build-in-docker.sh

@@ -7,6 +7,8 @@ SRC_DIR="/mnt/src"
 BUILD_DIR="/var/src"
 
 DIST=$2
+BUILD_TYPE=${3:-Release}
+TESTS=$4
 
 if [[ -d ${BUILD_DIR} ]]; then
   rm -rf ${BUILD_DIR}
@@ -19,8 +21,14 @@ cp -arf ${SRC_DIR}/${PROJECT_NAME} ${BUILD_DIR}/${PROJECT_NAME}
 cd "${BUILD_DIR}/${PROJECT_NAME}"
 #git submodule update --init --recursive
 cd scripts
-./${PROJECT_NAME}_build.sh -y -o Release
-cd "/var/src/${PROJECT_NAME}/build/packages/"
+./${PROJECT_NAME}_build.sh -y -f -o ${BUILD_TYPE}
+
+if [ $TESTS ]; then
+    cd "${BUILD_DIR}/${PROJECT_NAME}/build"
+    /root/bin/ctest -j $(nproc) -E mongo -R "$TESTS"
+fi
+
+cd "${BUILD_DIR}/${PROJECT_NAME}/build/packages/"
 chmod +x generate_package.sh
 mkdir -p ${SRC_DIR}/${PROJECT_NAME}/build/pkgs
 
@@ -35,4 +43,5 @@ case $DIST in
     cd "/var/src/${PROJECT_NAME}/build/packages/"
     bash "generate_package.sh" rpm
     cp -av *.rpm ${SRC_DIR}/${PROJECT_NAME}/build/pkgs
+  ;;
 esac

cicd/run

@@ -11,9 +11,9 @@ set -e
 ProgName="./cicd/run"
 GIT_TAG=$(git describe --tags --abbrev=1)
 BUILD_DIR=build
+CACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+_BUILD_TYPE="Release"
 
-PROJECT_NAME=$1
-DIST=$2
 #SHELL=$3
 
 # DOCKER
@@ -27,7 +27,7 @@ function priv_build-in-docker(){
   if [[ $3 == "bash" ]]; then _command="bash"; fi
   docker run \
     -v "$(pwd):/mnt/src/${_name}" \
-    -v "$(pwd)/${BUILD_DIR}/${_dist}/.ccache:/root/.ccache" \
+    -v "${CACHE_DIR}/${_dist}:/root/.ccache" \
     --rm --name=${_name}-${_dist}-build \
     -ti ${_name}-${_dist}-build \
     ${_command}
@@ -44,6 +44,8 @@ function sub_help(){
       echo "    -i   build Docker image"
       echo "    -s   start Docker container and run shell in specific platform (-d option required)"
       echo "    -d   target platform {ubuntu,centos}"
+      echo "    --build-type    build type {Debug,Release,RelWithDebInfo}"
+      echo "    --tests         unit tests regex pattern"
       echo ""
       echo "Example:"
       echo "# Build toolchain and project package for Ubuntu"
@@ -75,36 +77,55 @@ function sub_help(){
 }
 
 function sub_build(){
-  while getopts ":tpishd:" opt; do
+  while getopts ":tpi:shd:o:u:-:" opt; do
     case ${opt} in
       h)
         sub_help build
         ;;
-      t )
+      t)
         build_tools=1
         ;;
-      p )
+      p)
         build_package=1
         ;;
-      i )
+      i)
         build_docker_image=1
+        _DOCKER_IMAGE_NAME=$OPTARG
         ;;
-      d )
+      d)
         _DIST=$OPTARG
         ;;
-      s )
+      s)
         login_to_container=1
         if [[ -z $OPTARG ]]; then
           _SHELL="bash"
         else
           _SHELL=$OPTARG
         fi
         ;;
-      \? )
+      -)
+        LONG_OPTARG="${OPTARG#*=}"
+        case $OPTARG in
+          tests*)
+            _TESTS=$LONG_OPTARG
+          ;;
+          build-type*)
+            _BUILD_TYPE=$LONG_OPTARG
+          ;;
+          '')
+            break
+          ;;
+          *)
+            echo "Invalid option --$OPTARG" 1>&2;
+            exit 1
+            ;;
+        esac
+        ;;
+      \?)
         echo "Invalid Option: -$OPTARG" 1>&2
         exit 1
         ;;
-      : )
+      :)
         echo "Invalid Option: -$OPTARG requires an argument" 1>&2
         exit 1
         ;;
@@ -117,25 +138,26 @@ function sub_build(){
     docker build \
       --build-arg project_name=${_PROJECT_NAME} \
       -t ${_PROJECT_NAME}-build-tools-${_DIST}:latest \
-      -f cicd/Dockerfile-${_DIST}-tools .
+      -f cicd/Dockerfile-${_DIST}-tools --squash .
   fi
   if [[ build_package -eq 1 ]]; then
     echo "Building ${_PROJECT_NAME} and create package for ${_DIST}"
-    local _command="/mnt/src/${_PROJECT_NAME}/cicd/build-in-docker.sh ${_PROJECT_NAME} ${_DIST}"
+    local _command="/mnt/src/${_PROJECT_NAME}/cicd/build-in-docker.sh ${_PROJECT_NAME} ${_DIST} ${_BUILD_TYPE} ${_TESTS}"
     local docker_image="${_PROJECT_NAME}-build-tools-${_DIST}:latest"
     if [[ login_to_container -eq 1 ]]; then
       _command=${_SHELL}
       docker_image="ubuntu:18.04"
     fi
     docker run \
       -v "$(pwd):/mnt/src/${_PROJECT_NAME}" \
-      -v "$(pwd)/${BUILD_DIR}/${_DIST}/.ccache:/root/.ccache" \
+      -v "${CACHE_DIR}/${_DIST}:/root/.ccache" \
       --rm --name=${_PROJECT_NAME}-${_DIST}-build \
       -ti ${docker_image} \
       ${_command}
-    fi
+  fi
   if [[ build_docker_image -eq 1 ]]; then
-    docker build --no-cache -t haya-test -f cicd/Dockerfile build/pkgs/
+    echo "Building docker image with name ${_DOCKER_IMAGE_NAME}"
+    docker build --no-cache -t ${_DOCKER_IMAGE_NAME} -f cicd/Dockerfile build/pkgs/
   fi
 }
 

scripts/haya_build.sh

@@ -109,7 +109,7 @@ NONINTERACTIVE=0
 BUILDTOOLSONLY=0
 
 if [ $# -ne 0 ]; then
-   while getopts ":cdo:s:ahyt" opt; do
+   while getopts ":cdo:s:ahytf" opt; do
       case "${opt}" in
          o )
             options=( "Debug" "Release" "RelWithDebInfo" "MinSizeRel" )
@@ -149,6 +149,9 @@ if [ $# -ne 0 ]; then
          t)
             BUILDTOOLSONLY=1
          ;;
+         f)
+            SKIP_DEPS_CHECK=1
+         ;;
          \? )
             printf "\\nInvalid Option: %s\\n" "-${OPTARG}" 1>&2
             usage
@@ -220,6 +223,7 @@ if [ "$ARCH" == "Linux" ]; then
          FILE="${REPO_ROOT}/scripts/haya_build_centos.sh"
          CXX_COMPILER=g++
          C_COMPILER=gcc
+         export PATH=$PATH:/opt/rh/devtoolset-7/root/usr/bin
       ;;
       "elementary OS")
          FILE="${REPO_ROOT}/scripts/haya_build_ubuntu.sh"

scripts/haya_build_centos.sh

@@ -43,6 +43,15 @@ if [ "${DISK_AVAIL%.*}" -lt "${DISK_MIN}" ]; then
 	exit 1;
 fi
 
+if [ -d /opt/rh/devtoolset-7 ]; then
+	printf "Enabling Centos devtoolset-7 so we can use GCC 7...\\n"
+	source /opt/rh/devtoolset-7/enable || exit 1
+	printf " - Centos devtoolset-7 successfully enabled!\\n"
+fi
+
+
+if [ -z $SKIP_DEPS_CHECK ]; then
+
 printf "\\n"
 
 printf "Checking Yum installation...\\n"
@@ -203,11 +212,13 @@ if [ "${BOOSTVERSION}" != "${BOOST_VERSION_MAJOR}0${BOOST_VERSION_MINOR}0${BOOST
 	&& cd .. \
 	&& rm -f boost_$BOOST_VERSION.tar.bz2 \
 	&& rm -rf $BOOST_LINK_LOCATION \
-	&& ln -s $BOOST_ROOT $BOOST_LINK_LOCATION \
+  && mkdir $BOOST_LINK_LOCATION \
+  && cp -r $BOOST_ROOT/lib $BOOST_LINK_LOCATION/ \
+  && cp -r $BOOST_ROOT/include $BOOST_LINK_LOCATION/ \
 	|| exit 1
-	printf " - Boost library successfully installed @ ${BOOST_ROOT} (Symlinked to ${BOOST_LINK_LOCATION}).\\n"
+	printf " - Boost library successfully installed @ ${BOOST_ROOT} (Copied to ${BOOST_LINK_LOCATION}).\\n"
 else
-	printf " - Boost library found with correct version @ ${BOOST_ROOT} (Symlinked to ${BOOST_LINK_LOCATION}).\\n"
+	printf " - Boost library found with correct version @ ${BOOST_ROOT} (Copied to ${BOOST_LINK_LOCATION}).\\n"
 fi
 if [ $? -ne 0 ]; then exit -1; fi
 
@@ -303,3 +314,4 @@ function print_instructions() {
 	printf "source /opt/rh/devtoolset-7/enable\\n"
 	return 0
 }
+fi

scripts/haya_build_ubuntu.sh

@@ -67,6 +67,8 @@ if [ "${DISK_AVAIL%.*}" -lt "${DISK_MIN}" ]; then
 	exit 1
 fi
 
+if [ -z $SKIP_DEPS_CHECK ]; then
+
 # llvm-4.0 is installed into /usr/lib/llvm-4.0
 # clang is necessary for building on ubuntu
 DEP_ARRAY=(
@@ -167,11 +169,13 @@ if [ "${BOOSTVERSION}" != "${BOOST_VERSION_MAJOR}0${BOOST_VERSION_MINOR}0${BOOST
 	&& cd .. \
 	&& rm -f boost_$BOOST_VERSION.tar.bz2 \
 	&& rm -rf $BOOST_LINK_LOCATION \
-	&& ln -s $BOOST_ROOT $BOOST_LINK_LOCATION \
+  && mkdir $BOOST_LINK_LOCATION \
+  && cp -r $BOOST_ROOT/lib $BOOST_LINK_LOCATION/ \
+  && cp -r $BOOST_ROOT/include $BOOST_LINK_LOCATION/ \
 	|| exit 1
-	printf " - Boost library successfully installed @ ${BOOST_ROOT} (Symlinked to ${BOOST_LINK_LOCATION}).\\n"
+	printf " - Boost library successfully installed @ ${BOOST_ROOT} (Copied to ${BOOST_LINK_LOCATION}).\\n"
 else
-	printf " - Boost library found with correct version @ ${BOOST_ROOT} (Symlinked to ${BOOST_LINK_LOCATION}).\\n"
+	printf " - Boost library found with correct version @ ${BOOST_ROOT} (Copied to ${BOOST_LINK_LOCATION}).\\n"
 fi
 if [ $? -ne 0 ]; then exit -1; fi
 
@@ -257,3 +261,4 @@ printf "\\n"
 function print_instructions() {
 	return 0
 }
+fi