From df5b6337ae4845c0a37cc86567b1744bca3604c3 Mon Sep 17 00:00:00 2001 From: Addie Morrison Date: Sun, 8 Dec 2019 19:16:18 -0600 Subject: [PATCH] Remove old build tools --- build-sqlite | 33 ------------ build/Makefile.linux | 121 ------------------------------------------- make-linux | 113 ---------------------------------------- make-linux-pgo | 9 ---- travis-linux | 8 --- 5 files changed, 284 deletions(-) delete mode 100755 build-sqlite delete mode 100644 build/Makefile.linux delete mode 100755 make-linux delete mode 100755 make-linux-pgo delete mode 100755 travis-linux diff --git a/build-sqlite b/build-sqlite deleted file mode 100755 index 77e686d..0000000 --- a/build-sqlite +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -euo pipefail - -VERSION=3240000 - -fetch_if_needed() { - if [ ! -e sqlite ]; then - curl --fail "https://sqlite.org/2018/sqlite-autoconf-${VERSION}.tar.gz" > sqlite.tar.gz - tar xf sqlite.tar.gz - rm sqlite.tar.gz - mv sqlite-autoconf-${VERSION} sqlite - fi -} - -build() { - cd sqlite - ./configure - - if [ ! -v PREBUILT ]; then - make -j$(nproc) - fi -} - -main() { - here=$(dirname "${BASH_SOURCE[0]}") - here=$(readlink -f "$here") - cd "$here" - - fetch_if_needed - build -} - -main "$@" diff --git a/build/Makefile.linux b/build/Makefile.linux deleted file mode 100644 index eeeb4d7..0000000 --- a/build/Makefile.linux +++ /dev/null @@ -1,121 +0,0 @@ -HERE:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -ROOT:=$(HERE)/../.. -VTABLE:=$(ROOT)/parquet -SQLITE:=$(ROOT)/sqlite - -# Directories -ARROW=$(HERE)/arrow -ARROW_RELEASE=$(ARROW)/cpp/release -BOOST_ROOT=$(ARROW_RELEASE)/boost_ep-prefix/src/boost_ep -BOOST=$(BOOST_ROOT)/stage/lib -BROTLI=$(ARROW_RELEASE)/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu -ICU=$(HERE)/icu -LZ4=$(ARROW_RELEASE)/lz4_ep-prefix/src/lz4_ep/lib -PARQUET_CPP=$(HERE)/parquet-cpp -SNAPPY=$(ARROW_RELEASE)/snappy_ep/src/snappy_ep-install/lib -ZLIB=$(ARROW_RELEASE)/zlib_ep/src/zlib_ep-install/lib -ZSTD=$(ARROW_RELEASE)/zstd_ep-prefix/src/zstd_ep/lib - -# Libraries -# profile_gen, profile_build for PGO -APACHE_BUILD=release - -ARROW_LIB = $(ARROW_RELEASE)/$(APACHE_BUILD)/libarrow.a -BOOST_FILESYSTEM_LIB = $(BOOST)/libboost_filesystem.a -BOOST_REGEX_LIB = $(BOOST)/libboost_regex.a -BOOST_SYSTEM_LIB = $(BOOST)/libboost_system.a -BROTLI_COMMON_LIB = $(BROTLI)/libbrotlicommon.a -BROTLI_DEC_LIB = $(BROTLI)/libbrotlidec.a -BROTLI_ENC_LIB = $(BROTLI)/libbrotlienc.a -ICU_I18N_LIB=$(ICU)/source/lib/libicui18n.a -ICU_UC_LIB=$(ICU)/source/lib/libicuuc.a -ICU_DATA_LIB=$(ICU)/source/lib/libicudata.a -LZ4_LIB = $(LZ4)/liblz4.a -PARQUET_CPP_LIB = $(PARQUET_CPP)/build/$(APACHE_BUILD)/libparquet.a -SNAPPY_LIB = $(SNAPPY)/libsnappy.a -THRIFT_LIB = $(PARQUET_CPP)/thrift_ep/src/thrift_ep-install/lib/libthrift.a -ZLIB_LIB = $(ZLIB)/libz.a -ZSTD_LIB = $(ZSTD)/libzstd.a - -# Flags -CC = gcc -CXX = g++ -OPTIMIZATIONS = -O3 -CPUS:=$(shell nproc) -CFLAGS = -I $(SQLITE) -I $(PARQUET_CPP)/src -I $(ARROW)/cpp/src $(OPTIMIZATIONS) -std=c++11 -Wall -fPIC -g - -ALL_LIBS = $(PARQUET_CPP_LIB) $(LZ4_LIB) $(ZSTD_LIB) $(THRIFT_LIB) $(SNAPPY_LIB) $(ARROW_LIB) \ - $(ICU_I18N_LIB) $(ICU_UC_LIB) $(ICU_DATA_LIB) \ - $(BROTLI_ENC_LIB) $(BROTLI_COMMON_LIB) $(BROTLI_DEC_LIB) $(BOOST_REGEX_LIB) $(BOOST_SYSTEM_LIB) $(BOOST_FILESYSTEM_LIB) - -LDFLAGS = $(OPTIMIZATIONS) \ - -Wl,--whole-archive $(ALL_LIBS) \ - -Wl,--no-whole-archive -lz -lcrypto -lssl -OBJ = parquet.o parquet_filter.o parquet_table.o parquet_cursor.o -LIBS = $(ARROW_LIB) $(PARQUET_CPP_LIB) $(ICU_I18N_LIB) - -PROF = - -libparquet.so: $(LIBS) $(OBJ) - $(CXX) $(PROF) -shared -o $@ $(OBJ) $(LDFLAGS) - -parquet_filter.o: $(VTABLE)/parquet_filter.cc $(VTABLE)/parquet_filter.h $(ARROW) $(PARQUET_CPP) - $(CXX) $(PROF) -c -o $@ $< $(CFLAGS) - -parquet_cursor.o: $(VTABLE)/parquet_cursor.cc $(VTABLE)/parquet_cursor.h $(VTABLE)/parquet_table.h $(VTABLE)/parquet_filter.h $(ARROW) $(PARQUET_CPP) - $(CXX) $(PROF) -c -o $@ $< $(CFLAGS) - -parquet_table.o: $(VTABLE)/parquet_table.cc $(VTABLE)/parquet_table.h $(ARROW) $(PARQUET_CPP) - $(CXX) $(PROF) -c -o $@ $< $(CFLAGS) - -parquet.o: $(VTABLE)/parquet.cc $(VTABLE)/parquet_cursor.h $(VTABLE)/parquet_table.h $(VTABLE)/parquet_filter.h $(ARROW) $(PARQUET_CPP) - $(CXX) $(PROF) -c -o $@ $< $(CFLAGS) - -$(ARROW): - rm -rf $(ARROW) - git clone https://github.com/apache/arrow.git $(ARROW) - cd $(ARROW) && git checkout apache-arrow-0.9.0 - mkdir $(ARROW)/cpp/release - cd $(ARROW)/cpp/release && cmake -DCMAKE_BUILD_TYPE=$(APACHE_BUILD) -DARROW_BOOST_VENDORED=ON -DARROW_BOOST_USE_SHARED=OFF -DPARQUET_BUILD_SHARED=OFF .. - touch -d @0 $(ARROW) - -$(ARROW_LIB): $(ARROW) - cd $(ARROW)/cpp/release && make -j$(CPUS) - -# This is pretty gross. I'm sure someone who knows what they're doing could do this more cleanly. -$(ICU_I18N_LIB): - rm -rf $(ICU) - mkdir $(ICU) - cd $(ICU) && wget http://download.icu-project.org/files/icu4c/$(ICU_VERSION)/icu4c-$(ICU_VERSION_U)-src.tgz - cd $(ICU) && tar xf icu4c-$(ICU_VERSION_U)-src.tgz --strip-components=1 - cd $(ICU)/source && ./configure --enable-static - cd $(ICU)/source && make -j$(CPUS) LIBCFLAGS='-fPIC' LIBCXXFLAGS='-fPIC' - -$(PARQUET_CPP): - rm -rf $(PARQUET_CPP) - git clone https://github.com/apache/parquet-cpp.git $(PARQUET_CPP) - cd $(PARQUET_CPP) && git checkout apache-parquet-cpp-1.4.0 - cd $(PARQUET_CPP) && BOOST_ROOT=$(BOOST_ROOT) BOOST_STATIC_REGEX_LIBRARY=$(BOOST_REGEX_LIB) SNAPPY_STATIC_LIB=$(SNAPPY_LIB) BROTLI_STATIC_LIB_ENC=$(BROTLI_ENC_LIB) BROTLI_STATIC_LIB_DEC=$(BROTLI_DEC_LIB) BROTLI_STATIC_LIB_COMMON=$(BROTLI_COMMON_LIB) ZLIB_STATIC_LIB=$(ZLIB_LIB) LZ4_STATIC_LIB=$(LZ4_LIB) ZSTD_STATIC_LIB=$(ZSTD_LIB) cmake -DCMAKE_BUILD_TYPE=$(APACHE_BUILD) -DPARQUET_MINIMAL_DEPENDENCY=ON -DPARQUET_ARROW_LINKAGE=static -DPARQUET_BOOST_USE_SHARED=OFF -DPARQUET_BUILD_SHARED=OFF . - touch -d @0 $(PARQUET_CPP) - -$(PARQUET_CPP_LIB): $(PARQUET_CPP) $(ARROW_LIB) - cd $(PARQUET_CPP) && make -j$(CPUS) - -.PHONY: clean arrow icu parquet publish_libs - -clean: - rm -f *.o *.so - -distclean: - rm -rf $(SQLITE) $(HERE) - - -arrow: $(ARROW_LIB) - -icu: $(ICU_I18N_LIB) - -parquet: $(PARQUET_CPP_LIB) - -publish_libs: - tar -cJf libs.tar.xz $(ALL_LIBS) $(SQLITE)/sqlite3 - s3cmd put libs.tar.xz s3://cldellow/public/libparquet/$$(lsb_release -s -r)/libs.tar.xz diff --git a/make-linux b/make-linux deleted file mode 100755 index d58ea37..0000000 --- a/make-linux +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -set -euo pipefail - -here=$(dirname "${BASH_SOURCE[0]}") -here=$(readlink -f "$here") -prebuilt="$here"/build/linux/prebuilt -ubuntu="$(lsb_release -s -r)" -libs=(libarrow.a libboost_filesystem.a libboost_regex.a libboost_system.a libbrotlicommon.a libbrotlidec.a \ - libbrotlienc.a libicudata.a libicui18n.a libicuuc.a liblz4.a libparquet.a libsnappy.a libthrift.a libzstd.a) -lib_locs=() - -setup_directories() { - cd "$here" - mkdir -p build/linux - mkdir -p "$prebuilt" - cp -f build/Makefile.linux build/linux/Makefile - cd build/linux -} - -install_prerequisites() { - # Install prereqs based on https://github.com/apache/parquet-cpp#linux - sudo apt-get install libboost-dev g++ libboost-filesystem-dev \ - libboost-program-options-dev libboost-regex-dev \ - libboost-system-dev libboost-test-dev \ - libssl-dev libtool bison flex pkg-config - - # Install prereqs based on https://github.com/apache/arrow/tree/master/cpp - sudo apt-get install cmake \ - libboost-dev \ - libboost-filesystem-dev \ - libboost-system-dev -} - -build_sqlite() { - if [ ! -e ../../sqlite/sqlite3 ]; then - ( cd "../.." && ./build-sqlite ) - fi -} - -set_icu_version() { - case "$ubuntu" in - 14.04) - export ICU_VERSION=52.1 - ;; - 16.04) - export ICU_VERSION=55.1 - ;; - 18.04) - export ICU_VERSION=60.2 - ;; - *) - echo "unsure what libicu version to use" >&2 - exit 1 - esac - - export ICU_VERSION_U=${ICU_VERSION//./_} -} - -add_prebuilt_lib() { - lib_locs+=("$1=$prebuilt/$2.a") -} - -fetch_prebuilt_libs() { - if [ ! -e "$prebuilt"/complete ]; then - ( - cd "$prebuilt" - curl "https://s3.amazonaws.com/cldellow/public/libparquet/$ubuntu/libs.tar.xz" > libs.tar.xz - tar xf libs.tar.xz --xform 's#.*/##' - touch "$prebuilt"/complete - ) - fi - - if [ ! -e "$here"/sqlite/sqlite3 ]; then - ln -s "$prebuilt"/sqlite3 "$here"/sqlite/sqlite3 - fi - - add_prebuilt_lib "PARQUET_CPP_LIB" libparquet - add_prebuilt_lib "LZ4_LIB" liblz4 - add_prebuilt_lib "ZSTD_LIB" libzstd - add_prebuilt_lib "THRIFT_LIB" libthrift - add_prebuilt_lib "SNAPPY_LIB" libsnappy - add_prebuilt_lib "ARROW_LIB" libarrow - add_prebuilt_lib "ICU_I18N_LIB" libicui18n - add_prebuilt_lib "ICU_UC_LIB" libicuuc - add_prebuilt_lib "ICU_DATA_LIB" libicudata - add_prebuilt_lib "BROTLI_ENC_LIB" libbrotlienc - add_prebuilt_lib "BROTLI_COMMON_LIB" libbrotlicommon - add_prebuilt_lib "BROTLI_DEC_LIB" libbrotlidec - add_prebuilt_lib "BOOST_REGEX_LIB" libboost_regex - add_prebuilt_lib "BOOST_SYSTEM_LIB" libboost_system - add_prebuilt_lib "BOOST_FILESYSTEM_LIB" libboost_filesystem - -} - -main() { - setup_directories - install_prerequisites - build_sqlite - set_icu_version - - if [ -v PREBUILT ]; then - fetch_prebuilt_libs - fi - - optimizations="-O3" - if [ -v COVERAGE ]; then - optimizations="-fprofile-arcs -ftest-coverage" - fi - - make ${lib_locs[@]-} "$@" OPTIMIZATIONS="$optimizations" -} - -main "$@" diff --git a/make-linux-pgo b/make-linux-pgo deleted file mode 100755 index 1c0b06d..0000000 --- a/make-linux-pgo +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail - -cd "$(dirname "${BASH_SOURCE[0]}")" -./make-linux distclean -./make-linux PROF=-fprofile-generate -./tests/test-all -./make-linux clean -./make-linux PROF=-fprofile-use diff --git a/travis-linux b/travis-linux deleted file mode 100755 index 0258ab9..0000000 --- a/travis-linux +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -set -euo pipefail - -sudo apt-get install software-properties-common -sudo add-apt-repository --yes ppa:george-edison55/cmake-3.x -sudo apt-get update -COVERAGE=1 PREBUILT=1 ./make-linux -COVERAGE=1 ./tests/test-all