Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support for DPKG local caching #3292

Closed
wants to merge 3 commits into from
Closed

Support for DPKG local caching #3292

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b943612
Support for DPKG local caching
Kalimuthu-Velappan Aug 5, 2019
e7e3891
Merge remote-tracking branch 'upstream/master'
Kalimuthu-Velappan Oct 4, 2019
40b4d47
Merge remote-tracking branch 'upstream/master'
Kalimuthu-Velappan Oct 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions Makefile.work
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@
# * Default: yes
# * Values: yes, no
# * KERNEL_PROCURE_METHOD: Specifying method of obtaining kernel Debian package: download or build
# * SONIC_DPKG_CACHE_METHOD: Specifying method of obtaining the Debian packages from cache: none or cache
# * SONIC_DPKG_CACHE_SOURCE: Debian package cache location when cache enabled for debian packages
#
###############################################################################

Expand Down Expand Up @@ -137,6 +139,12 @@ ifneq (,$(filter $(CONFIGURED_ARCH), armhf arm64))

endif

ifneq ($(SONIC_DPKG_CACHE_SOURCE),)
$(shell test -d $(SONIC_DPKG_CACHE_SOURCE) || mkdir -p $(SONIC_DPKG_CACHE_SOURCE) )
DOCKER_RUN += -v "$(SONIC_DPKG_CACHE_SOURCE):/dpkg_cache"
endif


DOCKER_BASE_BUILD = docker build --no-cache \
-t $(SLAVE_BASE_IMAGE):$(SLAVE_BASE_TAG) \
--build-arg http_proxy=$(http_proxy) \
Expand Down Expand Up @@ -171,6 +179,8 @@ SONIC_BUILD_INSTRUCTION := make \
SONIC_USE_DOCKER_BUILDKIT=$(SONIC_USE_DOCKER_BUILDKIT) \
VS_PREPARE_MEM=$(VS_PREPARE_MEM) \
KERNEL_PROCURE_METHOD=$(KERNEL_PROCURE_METHOD) \
SONIC_DPKG_CACHE_METHOD=$(SONIC_DPKG_CACHE_METHOD) \
SONIC_DPKG_CACHE_SOURCE=$(SONIC_DPKG_CACHE_SOURCE) \
HTTP_PROXY=$(http_proxy) \
HTTPS_PROXY=$(https_proxy) \
SONIC_ENABLE_SYSTEM_TELEMETRY=$(ENABLE_SYSTEM_TELEMETRY) \
Expand Down
12 changes: 12 additions & 0 deletions rules/config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,18 @@ DEFAULT_KERNEL_PROCURE_METHOD = build
FRR_USER_UID = 300
FRR_USER_GID = 300

# DPKG cache allows the .deb files to be stored in the cache path. This allows the submodules
# package to be cached and restored back if its commit hash is not modified and its dependencies are not modified.
# SONIC_DPKG_CACHE_METHOD - Default method of deb package caching
# none: no caching
# cache: cache from local directory
# #url: cache over remote http server (not supported)
# #registry: cache on the docker local/remote registry (not supported)
# SONIC_DPKG_CACHE_SOURCE - provides the cache location detail
SONIC_DPKG_CACHE_METHOD=cache
SONIC_DPKG_CACHE_SOURCE=${PWD}/target/cache/


# Default VS build memory preparation
DEFAULT_VS_PREPARE_MEM = yes

Expand Down
11 changes: 11 additions & 0 deletions rules/linux-kernel.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,19 @@ KERNEL_SUBVERSION = 1+deb9u5

export KVERSION_SHORT KVERSION KERNEL_VERSION KERNEL_SUBVERSION


SMPATH=$(SRC_PATH)/sonic-linux-kernel/
SM_DEP_LIST := Makefile
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does this SM stand for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SM stands for Sub Module in the sonic repo
Initially, the DPKG caching framework is supported only for the Linux submodule, later, we have enhanced the DPKG caching framework to cover all the sub modules and Debian packages in the sonic repro.
We will close this one and will raise a new pull request for the enhanced version of the DPKG caching framework.

SM_DEP_LIST += patch/*
SM_DEP_LIST += patch/preconfig/*
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do we have only rule for linux-kernel? We should have rules for other source code?

SMDEP_LIST := $(wildcard $(addprefix $(SMPATH),$(SM_DEP_LIST)))
DEP_LIST := $(SONIC_MAKEFILE_LIST) rules/linux-kernel.mk

LINUX_HEADERS_COMMON = linux-headers-$(KVERSION_SHORT)-common_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_all.deb
$(LINUX_HEADERS_COMMON)_SRC_PATH = $(SRC_PATH)/sonic-linux-kernel
$(LINUX_HEADERS_COMMON)_CACHE_MODE = GIT_COMMIT_SHA
$(LINUX_HEADERS_COMMON)_DEP_SOURCE = $(DEP_LIST)
$(LINUX_HEADERS_COMMON)_SMDEP_SOURCE = $(SMDEP_LIST)
SONIC_MAKE_DEBS += $(LINUX_HEADERS_COMMON)

LINUX_HEADERS = linux-headers-$(KVERSION)_$(KERNEL_VERSION)-$(KERNEL_SUBVERSION)_$(CONFIGURED_ARCH).deb
Expand Down
114 changes: 112 additions & 2 deletions slave.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ export BUILD_NUMBER
export BUILD_TIMESTAMP
export CONFIGURED_PLATFORM
export CONFIGURED_ARCH
SONIC_MAKEFILE_LIST=slave.mk rules/config rules/functions

###############################################################################
## Utility rules
Expand Down Expand Up @@ -193,6 +194,9 @@ $(info "ENABLE_SYSTEM_TELEMETRY" : "$(ENABLE_SYSTEM_TELEMETRY)")
$(info "SONIC_DEBUGGING_ON" : "$(SONIC_DEBUGGING_ON)")
$(info "SONIC_PROFILING_ON" : "$(SONIC_PROFILING_ON)")
$(info "KERNEL_PROCURE_METHOD" : "$(KERNEL_PROCURE_METHOD)")
ifeq ($(SONIC_DPKG_CACHE_METHOD),cache)
$(info "DPKG_CACHE_PATH" : "$(SONIC_DPKG_CACHE_SOURCE)")
endif
$(info "BUILD_TIMESTAMP" : "$(BUILD_TIMESTAMP)")
$(info "BLDENV" : "$(BLDENV)")
$(info "VS_PREPARE_MEM" : "$(VS_PREPARE_MEM)")
Expand All @@ -214,6 +218,85 @@ export kernel_procure_method=$(KERNEL_PROCURE_METHOD)
export vs_build_prepare_mem=$(VS_PREPARE_MEM)

###############################################################################
MOD_CACHE_LOCK_TIMEOUT = 3600
SONIC_DPKG_CACHE_DIR=/dpkg_cache

# Lock macro for debian package level cache
# Lock is implemented through flock command with the timeout value of 1 hour
# Lock file is created in the cache directory and corresponding lock fd is stored as part of DPKG recipe.
define MOD_LOCK
if [[ ! -f $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock ]]; then
touch $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock
chmod 777 $(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock;
fi
$(eval $(1)_lock_fd=$(subst -,_,$(subst +,_,$(subst .,_,$(1)))))
exec {$($(1)_lock_fd)}<"$(SONIC_DPKG_CACHE_DIR)/$(1)_cache_accss.lock";
if ! flock -x -w $(MOD_CACHE_LOCK_TIMEOUT) "$${$($(1)_lock_fd)}" ; then
echo "ERROR: Lock timeout trying to read $(1) from cache.";
exit 1;
fi
endef


# UnLock macro for debian package level cache
define MOD_UNLOCK
eval exec "$${$($(1)_lock_fd)}<&-";
endef


# Loads the deb package from debian cache
# Cache file prefix is formed using SHA value
# The SHA value is derived from one of the keyword type - GIT_COMMIT_SHA or GIT_CONTENT_SHA
# GIT_COMMIT_SHA - SHA value of the last git commit id if it is a submodule
# GIT_CONTENT_SHA - SHA value is calculated from the target dependency files content.
# Cache is loaded only when corresponding cache file is present in the cache direcory and its dependencies are not changed.
define LOAD_CACHE
$(eval MOD_SRC_PATH=$($(1)_SRC_PATH))
$(eval MOD_HASH=$(if $(filter GIT_COMMIT_SHA,$($(1)_CACHE_MODE)),$(shell cd $(MOD_SRC_PATH) && git log -1 --format="%H")
, $(shell git hash-object $($(1)_DEP_SOURCE) $($(1)_SMDEP_SOURCE)|sha1sum|awk '{print $$1}')))
$(eval MOD_CACHE_FILE=$(1)-$(MOD_HASH).tgz)
$(eval $(1)_MOD_CACHE_FILE=$(MOD_CACHE_FILE))
$(eval DRV_DEB=$(foreach pkg,$(addprefix $(DEBS_PATH)/,$(1) $($(1)_DERIVED_DEBS)),$(if $(wildcard $(pkg)),,$(pkg))))
$(eval $(1)_FILES_MODIFIED := $(if $($(1)_DEP_SOURCE),$(shell git status -s $($(1)_DEP_SOURCE))) \
$(if $($(1)_SMDEP_SOURCE),$(shell cd $(MOD_SRC_PATH) && git status -s $(subst $(MOD_SRC_PATH)/,,$($(1)_SMDEP_SOURCE)))) )
#$(filter-out $($(1)_DEP_SOURCE),$($(1)_SMDEP_SOURCE), $?)

$(if $($(1)_FILES_MODIFIED),
echo "Target $(1) dependencies are modifed - load cache skipped";
echo "Modified dependencies are : [$($(1)_FILES_MODIFIED)] ";
,
$(if $(wildcard $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE)),
$(if $(DRV_DEB), tar -xzvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE),echo );
echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is loaded from cache";
$(eval $(1)_CACHE_LOADED := Yes)
,
echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) is not present in the cache !";
)
)
echo ""
endef

# Saves the deb package into debian cache
# A single tared-zip cache is created for .deb and its derived packages in the cache direcory.
# It saves the .deb into cache only when its dependencies are not changed
# The cache save is protected with lock.
# The SAVE_CACHE macro has dependecy with LOAD_CACHE macro
# The target specific variables -_SRC_PATH, _MOD_CACHE_FILE and _FILES_MODIFIED are
# derived from the LOAD_CACHE macro
define SAVE_CACHE
$(eval MOD_SRC_PATH=$($(1)_SRC_PATH))
$(eval MOD_CACHE_FILE=$($(1)_MOD_CACHE_FILE))
$(call MOD_LOCK,$(1))
$(if $($(1)_FILES_MODIFIED),
echo "Target $(1) dependencies are modifed - save cache skipped";
,
tar -czvf $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) $(2) $(addprefix $(DEBS_PATH)/,$($(1)_DERIVED_DEBS));
echo "File $(SONIC_DPKG_CACHE_DIR)/$(MOD_CACHE_FILE) saved in the cache ";
)
$(call MOD_UNLOCK,$(1))
echo ""
endef

## Local targets
###############################################################################

Expand Down Expand Up @@ -311,8 +394,16 @@ SONIC_TARGET_LIST += $(addprefix $(FILES_PATH)/, $(SONIC_MAKE_FILES))
# $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name
# $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ...
# SONIC_MAKE_DEBS += $(SOME_NEW_DEB)
$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)))
$(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
$$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE)
$(HEADER)

# Load the target deb from DPKG cache
$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) )

# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then

# Remove target to force rebuild
rm -f $(addprefix $(DEBS_PATH)/, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS))
# Apply series of patches if exist
Expand All @@ -321,6 +412,12 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS_GENERIC}" make DEST=$(shell pwd)/$(DEBS_PATH) -C $($*_SRC_PATH) $(shell pwd)/$(DEBS_PATH)/$* $(LOG)
# Clean up
if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi

# Save the target deb into DPKG cache
$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check you check the alignment here?


fi

$(FOOTER)

SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS))
Expand All @@ -331,8 +428,16 @@ SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_MAKE_DEBS))
# $(SOME_NEW_DEB)_SRC_PATH = $(SRC_PATH)/project_name
# $(SOME_NEW_DEB)_DEPENDS = $(SOME_OTHER_DEB1) $(SOME_OTHER_DEB2) ...
# SONIC_DPKG_DEBS += $(SOME_NEW_DEB)
$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS)))
$(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(addsuffix -install,$$(addprefix $(DEBS_PATH)/,$$($$*_DEPENDS))) \
$$($$*_DEP_SOURCE) $$($$*_SMDEP_SOURCE)
$(HEADER)

# Load the target deb from DPKG cache
$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call LOAD_CACHE,$*) )

# Skip building the target if it is already loaded from cache
if [ -z '$($*_CACHE_LOADED)' ] ; then

# Remove old build logs if they exist
rm -f $($*_SRC_PATH)/debian/*.debhelper.log
# Apply series of patches if exist
Expand All @@ -349,6 +454,11 @@ $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS)) : $(DEBS_PATH)/% : .platform $$(a
if [ -f $($*_SRC_PATH).patch/series ]; then pushd $($*_SRC_PATH) && quilt pop -a -f; popd; fi
# Take built package(s)
mv $(addprefix $($*_SRC_PATH)/../, $* $($*_DERIVED_DEBS) $($*_EXTRA_DEBS)) $(DEBS_PATH) $(LOG)

# Save the target deb into DPKG cache
$(if $(and $(filter-out none,$(SONIC_DPKG_CACHE_METHOD)),$($*_CACHE_MODE)), $(call SAVE_CACHE,$*,$@))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can check the alignment here?

fi

$(FOOTER)

SONIC_TARGET_LIST += $(addprefix $(DEBS_PATH)/, $(SONIC_DPKG_DEBS))
Expand Down