Support prebuilt binaries

...to make Travis CI builds reasonable.

Progress towards #16

README.md

@@ -24,7 +24,7 @@ Subsequent builds will only build the parquet virtual table extension.
 
 Run `./make-linx-pgo` to build an instrumented binary, run tests to collect real-life usage samples, then build an optimized binary. PGO seems to give a 5-10% reduction in query times.
 
-#### Tests
+### Tests
 
 Run:
 

make-linux

@@ -1,41 +1,101 @@
 #!/bin/bash
 set -euo pipefail
 
-mkdir -p build/linux
-cp -f build/Makefile.linux build/linux/Makefile
+here=$(dirname "${BASH_SOURCE[0]}")
+here=$(readlink -f "$here")
+prebuilt="$here"/build/linux/prebuilt
+ubuntu="$(lsb_release -s -r)"
+libs=(libarrow.a libboost_filesystem.a libboost_regex.a libboost_system.a libbrotlicommon.a libbrotlidec.a \
+  libbrotlienc.a libicudata.a libicui18n.a libicuuc.a liblz4.a libparquet.a libsnappy.a libthrift.a libzstd.a)
+lib_locs=()
 
-cd build/linux
+setup_directories() {
+  cd "$here"
+  mkdir -p build/linux
+  mkdir -p "$prebuilt"
+  cp -f build/Makefile.linux build/linux/Makefile
+  cd build/linux
+}
 
-# Install prereqs based on https://github.com/apache/parquet-cpp#linux
-sudo apt-get install libboost-dev g++ libboost-filesystem-dev \
-  libboost-program-options-dev libboost-regex-dev \
-  libboost-system-dev libboost-test-dev \
-  libssl-dev libtool bison flex pkg-config
+install_prerequisites() {
+  # Install prereqs based on https://github.com/apache/parquet-cpp#linux
+  sudo apt-get install libboost-dev g++ libboost-filesystem-dev \
+    libboost-program-options-dev libboost-regex-dev \
+    libboost-system-dev libboost-test-dev \
+    libssl-dev libtool bison flex pkg-config
 
-# Install prereqs based on https://github.com/apache/arrow/tree/master/cpp
-sudo apt-get install cmake \
-  libboost-dev \
-  libboost-filesystem-dev \
-  libboost-system-dev
+  # Install prereqs based on https://github.com/apache/arrow/tree/master/cpp
+  sudo apt-get install cmake \
+    libboost-dev \
+    libboost-filesystem-dev \
+    libboost-system-dev
+}
 
-if [ ! -e ../../sqlite/sqlite3 ]; then
-  make sqlite
-fi
+build_sqlite() {
+  if [ ! -e ../../sqlite/sqlite3 ]; then
+    make sqlite
+  fi
+}
 
-case "$(lsb_release -s -r)" in
-  14.04)
-    export ICU_VERSION=52.1
-    ;;
-  16.04)
-    export ICU_VERSION=55.1
-    ;;
-  18.04)
-    export ICU_VERSION=60.2
-    ;;
-  *)
-  echo "unsure what libicu version to use" >&2
-  exit 1
-esac
+set_icu_version() {
+  case "$ubuntu" in
+    14.04)
+      export ICU_VERSION=52.1
+      ;;
+    16.04)
+      export ICU_VERSION=55.1
+      ;;
+    18.04)
+      export ICU_VERSION=60.2
+      ;;
+    *)
+    echo "unsure what libicu version to use" >&2
+    exit 1
+  esac
 
-export ICU_VERSION_U=${ICU_VERSION//./_}
-make "$@"
+  export ICU_VERSION_U=${ICU_VERSION//./_}
+}
+
+add_prebuilt_lib() {
+  lib_locs+=("$1=$prebuilt/$2.a")
+}
+
+fetch_prebuilt_libs() {
+  for lib in "${libs[@]}"; do
+    if [ ! -e "$prebuilt/$lib" ]; then
+      echo "Fetching prebuilt $lib for Ubuntu $ubuntu"
+      curl "https://s3.amazonaws.com/cldellow/public/libparquet/$ubuntu/$lib" > tmp.a
+      mv tmp.a "$prebuilt"/$lib
+    fi
+  done
+
+  add_prebuilt_lib "PARQUET_CPP_LIB" libparquet
+  add_prebuilt_lib "LZ4_LIB" liblz4
+  add_prebuilt_lib "ZSTD_LIB" libzstd
+  add_prebuilt_lib "THRIFT_LIB"  libthrift
+  add_prebuilt_lib "SNAPPY_LIB" libsnappy
+  add_prebuilt_lib "ARROW_LIB" libarrow
+  add_prebuilt_lib "ICU_I18N_LIB" libicui18n
+  add_prebuilt_lib "ICU_UC_LIB" libicuuc
+  add_prebuilt_lib "ICU_DATA_LIB" libicudata
+  add_prebuilt_lib "BROTLI_ENC_LIB" libbrotlienc
+  add_prebuilt_lib "BROTLI_COMMON_LIB" libbrotlicommon
+  add_prebuilt_lib "BROTLI_DEC_LIB" libbrotlidec
+  add_prebuilt_lib "BOOST_REGEX_LIB" libboost_regex
+  add_prebuilt_lib "BOOST_SYSTEM_LIB" libboost_system
+  add_prebuilt_lib "BOOST_FILESYSTEM_LIB" libboost_filesystem
+}
+
+main() {
+  setup_directories
+  install_prerequisites
+  build_sqlite
+  set_icu_version
+
+  if [ -v PREBUILT ]; then
+    fetch_prebuilt_libs
+  fi
+  make ${lib_locs[@]-} "$@"
+}
+
+main "$@"