Skip to content

Commit

Permalink
Part 2 of Bears-R-Us#3957: simplify offline builds (Bears-R-Us#3959)
Browse files Browse the repository at this point in the history 
Co-authored-by: Amanda Potts <[email protected]>
  • Loading branch information
@ajpotts
ajpotts and ajpotts authored Jan 8, 2025
1 parent 8edf364 commit 7869b4a
Showing 1 changed file with 60 additions and 30 deletions.
90 changes: 60 additions & 30 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,9 @@ ARROW_UTIL_O += $(ARKOUDA_SOURCE_DIR)/UtilParquet.o
.PHONY: install-deps
install-deps: install-zmq install-hdf5 install-arrow install-iconv install-idn2

.PHONY: deps-download-source
deps-download-source: zmq-download-source hdf5-download-source arrow-download-source iconv-download-source idn2-download-source

DEP_DIR := dep
DEP_INSTALL_DIR := $(ARKOUDA_PROJECT_DIR)/$(DEP_DIR)
DEP_BUILD_DIR := $(ARKOUDA_PROJECT_DIR)$(DEP_DIR)/build
Expand All @@ -118,11 +121,9 @@ ZMQ_NAME_VER := zeromq-$(ZMQ_VER)
ZMQ_BUILD_DIR := $(DEP_BUILD_DIR)/$(ZMQ_NAME_VER)
ZMQ_INSTALL_DIR := $(DEP_INSTALL_DIR)/zeromq-install
ZMQ_LINK := https://github.com/zeromq/libzmq/releases/download/v$(ZMQ_VER)/$(ZMQ_NAME_VER).tar.gz
install-zmq:
@echo "Installing ZeroMQ"
rm -rf $(ZMQ_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

zmq-download-source:
mkdir -p $(DEP_BUILD_DIR)
#If the build directory does not exist, create it
ifeq (,$(wildcard ${ZMQ_BUILD_DIR}*/.*))
# If the tar.gz not found, download it
Expand All @@ -134,6 +135,13 @@ install-zmq:
endif
endif

install-zmq:
@echo "Installing ZeroMQ"
rm -rf $(ZMQ_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

$(MAKE) zmq-download-source

cd $(ZMQ_BUILD_DIR) && ./configure --prefix=$(ZMQ_INSTALL_DIR) CFLAGS=-O3 CXXFLAGS=-O3 && make && make install
echo '$$(eval $$(call add-path,$(ZMQ_INSTALL_DIR)))' >> Makefile.paths

Expand All @@ -154,11 +162,7 @@ HDF5_INSTALL_DIR := $(DEP_INSTALL_DIR)/hdf5-install
# I think this seems good, but I don't love the hardcoded "-3" I'd like some input on that
HDF5_LINK := https://support.hdfgroup.org/releases/hdf5/$(UNDERSCORED_LINK_HDF5_MAJ_MIN_VER)/$(UNDERSCORED_LINK_HDF5_VER)/downloads/$(HDF5_NAME_VER)-3.tar.gz

install-hdf5:
@echo "Installing HDF5"
rm -rf $(HDF5_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

hdf5-download-source:
#If the build directory does not exist, create it
ifeq (,$(wildcard ${HDF5_BUILD_DIR}*/.*))
# If the tar.gz not found, download it
Expand All @@ -168,7 +172,14 @@ install-hdf5:
else
cd $(DEP_BUILD_DIR) && tar -xzf $(HDF5_NAME_VER)*.tar.gz
endif
endif
endif

install-hdf5:
@echo "Installing HDF5"
rm -rf $(HDF5_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

$(MAKE) hdf5-download-source

cd $(HDF5_BUILD_DIR)* && ./configure --prefix=$(HDF5_INSTALL_DIR) --enable-optimization=high --enable-hl && make && make install
echo '$$(eval $$(call add-path,$(HDF5_INSTALL_DIR)))' >> Makefile.paths
Expand All @@ -194,29 +205,36 @@ NUM_CORES := $(shell nproc --all)

ARROW_DEPENDENCY_SOURCE := BUNDLED


install-arrow:
@echo "Installing Apache Arrow/Parquet"
@echo "from build directory: ${DEP_BUILD_DIR}"
rm -rf $(ARROW_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)
touch $(DEP_BUILD_DIR)/arrow_exports.sh
arrow-download-source:

# If the tar.gz file does not exist, fetch it
ifeq (,$(wildcard ${DEP_BUILD_DIR}/$(ARROW_NAME_VER).tar.gz))
cd $(DEP_BUILD_DIR) && wget $(ARROW_SOURCE_LINK)
endif

cd $(DEP_BUILD_DIR) && tar -xvf $(ARROW_NAME_VER).tar.gz
mkdir -p $(ARROW_BUILD_DIR)/cpp/build-release

# if the arrow dependency directory is empty of tar.gz, download the dependencies
ifeq (,$(wildcard $(ARROW_DEP_DIR)/*.tar.gz))
rm -fr $(DEP_BUILD_DIR)/arrow_exports.sh
mkdir -p $(ARROW_DEP_DIR)
cd $(ARROW_BUILD_DIR)/cpp/thirdparty/ && ./download_dependencies.sh $(ARROW_DEP_DIR) > $(DEP_BUILD_DIR)/arrow_exports.sh
endif

rm -fr $(ARROW_BUILD_DIR)


install-arrow:
@echo "Installing Apache Arrow/Parquet"
@echo "from build directory: ${DEP_BUILD_DIR}"
rm -rf $(ARROW_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

$(MAKE) arrow-download-source

cd $(DEP_BUILD_DIR) && tar -xvf $(ARROW_NAME_VER).tar.gz
mkdir -p $(ARROW_BUILD_DIR)/cpp/build-release

cd $(DEP_BUILD_DIR) && . ./arrow_exports.sh && cd $(ARROW_BUILD_DIR)/cpp/build-release && cmake -S $(ARROW_BUILD_DIR)/cpp .. -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_INSTALL_PREFIX=$(ARROW_INSTALL_DIR) -DCMAKE_BUILD_TYPE=Release -DARROW_PARQUET=ON -DARROW_WITH_SNAPPY=ON -DARROW_WITH_BROTLI=ON -DARROW_WITH_BZ2=ON -DARROW_WITH_LZ4=ON -DARROW_WITH_ZLIB=ON -DARROW_WITH_ZSTD=ON -DARROW_DEPENDENCY_SOURCE=$(ARROW_DEPENDENCY_SOURCE) $(ARROW_OPTIONS) && make -j$(NUM_CORES)

cd $(ARROW_BUILD_DIR)/cpp/build-release && make install
Expand All @@ -231,16 +249,14 @@ arrow-clean:
rm -fr $(DEP_BUILD_DIR)/arrow_exports.sh



ICONV_VER := 1.17
ICONV_NAME_VER := libiconv-$(ICONV_VER)
ICONV_BUILD_DIR := $(DEP_BUILD_DIR)/$(ICONV_NAME_VER)
ICONV_INSTALL_DIR := $(DEP_INSTALL_DIR)/libiconv-install
ICONV_LINK := https://ftp.gnu.org/pub/gnu/libiconv/libiconv-$(ICONV_VER).tar.gz
install-iconv:
@echo "Installing iconv"
rm -rf $(ICONV_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

iconv-download-source:
#If the build directory does not exist, create it
ifeq (,$(wildcard ${ICONV_BUILD_DIR}*/.*))
# If the tar.gz not found, download it
Expand All @@ -251,6 +267,13 @@ install-iconv:
cd $(DEP_BUILD_DIR) && tar -xzf libiconv-$(ICONV_VER).tar.gz
endif
endif

install-iconv:
@echo "Installing iconv"
rm -rf $(ICONV_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

$(MAKE) iconv-download-source

cd $(ICONV_BUILD_DIR) && ./configure --prefix=$(ICONV_INSTALL_DIR) && make && make install
echo '$$(eval $$(call add-path,$(ICONV_INSTALL_DIR)))' >> Makefile.paths
Expand All @@ -263,11 +286,8 @@ LIBIDN_NAME_VER := libidn2-$(LIBIDN_VER)
LIBIDN_BUILD_DIR := $(DEP_BUILD_DIR)/$(LIBIDN_NAME_VER)
LIBIDN_INSTALL_DIR := $(DEP_INSTALL_DIR)/libidn2-install
LIBIDN_LINK := https://ftp.gnu.org/gnu/libidn/libidn2-$(LIBIDN_VER).tar.gz
install-idn2:
@echo "Installing libidn2"
rm -rf $(LIBIDN_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

idn2-download-source:
#If the build directory does not exist, create it
ifeq (,$(wildcard $(LIBIDN_BUILD_DIR)*/.*))
# If the tar.gz is not found, download it
Expand All @@ -278,6 +298,13 @@ install-idn2:
cd $(DEP_BUILD_DIR) && tar -xzf libidn2-$(LIBIDN_VER)*.tar.gz
endif
endif

install-idn2:
@echo "Installing libidn2"
rm -rf $(LIBIDN_INSTALL_DIR)
mkdir -p $(DEP_INSTALL_DIR) $(DEP_BUILD_DIR)

$(MAKE) idn2-download-source

cd $(LIBIDN_BUILD_DIR) && ./configure --prefix=$(LIBIDN_INSTALL_DIR) && make && make install
echo '$$(eval $$(call add-path,$(LIBIDN_INSTALL_DIR)))' >> Makefile.paths
Expand All @@ -288,15 +315,18 @@ idn2-clean:
BLOSC_BUILD_DIR := $(DEP_BUILD_DIR)/c-blosc
BLOSC_INSTALL_DIR := $(DEP_INSTALL_DIR)/c-blosc-install

blosc-download-source:
#If the build directory does not exist, create it
ifeq (,$(wildcard $(BLOSC_BUILD_DIR)/.*))
cd $(DEP_BUILD_DIR) && git clone https://github.com/Blosc/c-blosc.git
endif

install-blosc:
@echo "Installing blosc"
rm -rf $(BLOSC_INSTALL_DIR)
mkdir -p $(BLOSC_INSTALL_DIR)

#If the build directory does not exist, create it
ifeq (,$(wildcard $(BLOSC_BUILD_DIR)/.*))
cd $(DEP_BUILD_DIR) && git clone https://github.com/Blosc/c-blosc.git
endif
$(MAKE) blosc-download-source

cd $(BLOSC_BUILD_DIR) && cmake -DCMAKE_INSTALL_PREFIX=$(BLOSC_INSTALL_DIR) && make && make install
echo '$$(eval $$(call add-path,$(BLOSC_INSTALL_DIR)))' >> Makefile.paths
Expand Down

0 comments on commit 7869b4a

Please sign in to comment.