From 263a6af7ec675dcc1ccae68fd83e934ea722d853 Mon Sep 17 00:00:00 2001 From: Colin Dellow Date: Tue, 26 Jun 2018 08:17:18 -0400 Subject: [PATCH] Use Arrow's compression libraries Fixes #27 --- build/Makefile.linux | 82 +++++++++----------------------------------- 1 file changed, 16 insertions(+), 66 deletions(-) diff --git a/build/Makefile.linux b/build/Makefile.linux index 14a60e7..86ca140 100644 --- a/build/Makefile.linux +++ b/build/Makefile.linux @@ -5,28 +5,29 @@ SQLITE:=$(ROOT)/sqlite # Directories ARROW=$(HERE)/arrow -BROTLI=$(HERE)/brotli -LZ4=$(HERE)/lz4 +ARROW_RELEASE=$(ARROW)/cpp/release +BROTLI=$(ARROW_RELEASE)/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu +LZ4=$(ARROW_RELEASE)/lz4_ep-prefix/src/lz4_ep/lib PARQUET_CPP=$(HERE)/parquet-cpp -SNAPPY=$(HERE)/snappy -ZLIB=$(HERE)/zlib -ZSTD=$(HERE)/zstd +SNAPPY=$(ARROW_RELEASE)/snappy_ep/src/snappy_ep-install/lib +ZLIB=$(ARROW_RELEASE)/zlib_ep/src/zlib_ep-install/lib +ZSTD=$(ARROW_RELEASE)/zstd_ep-prefix/src/zstd_ep/lib # Libraries -ARROW_LIB = $(ARROW)/cpp/release/release/libarrow.a +ARROW_LIB = $(ARROW_RELEASE)/release/libarrow.a BOOST_FILESYSTEM_LIB = /usr/lib/x86_64-linux-gnu/libboost_filesystem.so BOOST_LIB = /usr/lib/x86_64-linux-gnu/libboost_regex.so BOOST_SYSTEM_LIB = /usr/lib/x86_64-linux-gnu/libboost_system.so -BROTLI_COMMON_LIB = $(BROTLI)/out/libbrotlicommon-static.a -BROTLI_DEC_LIB = $(BROTLI)/out/libbrotlidec-static.a -BROTLI_ENC_LIB = $(BROTLI)/out/libbrotlienc-static.a -LZ4_LIB = $(LZ4)/lib/liblz4.a +BROTLI_COMMON_LIB = $(BROTLI)/libbrotlicommon.a +BROTLI_DEC_LIB = $(BROTLI)/libbrotlidec.a +BROTLI_ENC_LIB = $(BROTLI)/libbrotlienc.a +LZ4_LIB = $(LZ4)/liblz4.a PARQUET_CPP_LIB = $(PARQUET_CPP)/build/release/libparquet.a -SNAPPY_LIB = $(SNAPPY)/build/libsnappy.a +SNAPPY_LIB = $(SNAPPY)/libsnappy.a SQLITE3_LIB = $(SQLITE)/libsqlite3.a THRIFT_LIB = $(PARQUET_CPP)/thrift_ep/src/thrift_ep-install/lib/libthrift.a ZLIB_LIB = $(ZLIB)/libz.a -ZSTD_LIB = $(ZSTD)/lib/libzstd.a +ZSTD_LIB = $(ZSTD)/libzstd.a # Flags CC = g++ @@ -39,14 +40,7 @@ LDFLAGS = -O3 \ -Wl,--no-whole-archive -lz -lcrypto -lssl $(BOOST_LIB) $(BOOST_SYSTEM_LIB) $(BOOST_FILESYSTEM_LIB) OBJ = parquet.o parquet_filter.o parquet_table.o parquet_cursor.o LIBS = $(ARROW_LIB) \ - $(BROTLI_COMMON_LIB) \ - $(BROTLI_DEC_LIB) \ - $(BROTLI_ENC_LIB) \ - $(LZ4_LIB) \ - $(PARQUET_CPP_LIB) \ - $(SNAPPY_LIB) \ - $(ZLIB_LIB) \ - $(ZSTD_LIB) + $(PARQUET_CPP_LIB) libparquet.so: $(LIBS) $(OBJ) $(CC) -shared -o $@ $(OBJ) $(LDFLAGS) @@ -71,50 +65,17 @@ $(ARROW_LIB): cd $(ARROW)/cpp/release && cmake .. -DCMAKE_BUILD_TYPE=Release cd $(ARROW)/cpp/release && make -j$(CPUS) unittest -$(BROTLI_COMMON_LIB): - rm -rf $(BROTLI) - git clone https://github.com/google/brotli.git $(BROTLI) - mkdir $(BROTLI)/out - cd $(BROTLI)/out && ../configure-cmake && make -j$(CPUS) - -$(LZ4_LIB): - rm -rf $(LZ4) - git clone https://github.com/lz4/lz4.git $(LZ4) - sed -i 's/^CFLAGS *+=/CFLAGS += -fPIC /' $(LZ4)/lib/Makefile - cd $(LZ4) && make -j$(CPUS) - -$(PARQUET_CPP_LIB): $(SNAPPY_LIB) $(BROTLI_COMMON_LIB) $(ZLIB_LIB) $(LZ4_LIB) $(ZSTD_LIB) +$(PARQUET_CPP_LIB): $(ARROW_LIB) rm -rf $(PARQUET_CPP) git clone https://github.com/apache/parquet-cpp.git $(PARQUET_CPP) cd $(PARQUET_CPP) && git checkout apache-parquet-cpp-1.4.0 cd $(PARQUET_CPP) && SNAPPY_STATIC_LIB=$(SNAPPY_LIB) BROTLI_STATIC_LIB_ENC=$(BROTLI_ENC_LIB) BROTLI_STATIC_LIB_DEC=$(BROTLI_DEC_LIB) BROTLI_STATIC_LIB_COMMON=$(BROTLI_COMMON_LIB) ZLIB_STATIC_LIB=$(ZLIB_LIB) LZ4_STATIC_LIB=$(LZ4_LIB) ZSTD_STATIC_LIB=$(ZSTD_LIB) cmake -DCMAKE_BUILD_TYPE=Release -DPARQUET_MINIMAL_DEPENDENCY=ON -DPARQUET_ARROW_LINKAGE=static . cd $(PARQUET_CPP) && make -j$(CPUS) -$(SNAPPY_LIB): - rm -rf $(SNAPPY) - git clone https://github.com/google/snappy.git $(SNAPPY) - mkdir $(SNAPPY)/build - cd $(SNAPPY)/build && cmake .. - sed -i '3iset(CMAKE_POSITION_INDEPENDENT_CODE ON)' $(SNAPPY)/CMakeLists.txt - cd $(SNAPPY)/build && make -j$(CPUS) - $(SQLITE3_LIB): cd $(ROOT) && ./build-sqlite -$(ZLIB_LIB): - rm -rf $(ZLIB) - git clone https://github.com/madler/zlib.git $(ZLIB) - cd $(ZLIB) && ./configure - sed -i 's/^CFLAGS=-O3/CFLAGS=-fPIC -O3/' $(ZLIB)/Makefile - cd $(ZLIB) && make -j$(CPUS) - -$(ZSTD_LIB): - rm -rf $(ZSTD) - git clone https://github.com/facebook/zstd.git $(ZSTD) - sed -i 's/^CFLAGS *+=/CFLAGS += -fPIC /' $(ZSTD)/lib/Makefile - cd $(ZSTD) && make -j$(CPUS) - -.PHONY: clean parquet snappy brotli zlib lz4 zstd arrow +.PHONY: clean arrow parquet sqlite clean: rm -f *.o *.so @@ -125,17 +86,6 @@ distclean: arrow: $(ARROW_LIB) -brotli: $(BROTLI_COMMON_LIB) - -lz4: $(LZ4_LIB) - parquet: $(PARQUET_CPP_LIB) -snappy: $(SNAPPY_LIB) - sqlite: $(SQLITE3_LIB) - -zlib: $(ZLIB_LIB) - -zstd: $(ZSTD_LIB) -