From c0ec7e40dbfb66126f572781b6ff4b72dd0f1e54 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 19 Jul 2019 02:00:54 -0700 Subject: [PATCH 1/2] [ci] download chromedriver and geckodriver through a caching proxy --- .ci/packer_cache.sh | 14 +------------- src/dev/ci_setup/setup.sh | 26 +++----------------------- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 3dc2b1da658cb..1cf9b406f164e 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -14,17 +14,6 @@ node scripts/es snapshot --license=oss --download-only; # download reporting browsers (cd "x-pack" && yarn gulp prepare); -# cache the chromedriver archive -chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")" -chromedriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")" -if [ -z "$chromedriverDistVersion" ] || [ -z "$chromedriverPkgVersion" ]; then - echo "UNABLE TO DETERMINE CHROMEDRIVER VERSIONS" - exit 1 -fi -mkdir -p .chromedriver -curl "https://chromedriver.storage.googleapis.com/$chromedriverDistVersion/chromedriver_linux64.zip" > .chromedriver/chromedriver.zip -echo "$chromedriverPkgVersion" > .chromedriver/pkgVersion - # archive cacheable directories mkdir -p "$HOME/.kibana/bootstrap_cache" tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ @@ -34,8 +23,7 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ x-pack/legacy/plugins/*/node_modules \ x-pack/legacy/plugins/reporting/.chromium \ test/plugin_functional/plugins/*/node_modules \ - .es \ - .chromedriver; + .es; echo "created $HOME/.kibana/bootstrap_cache/$branch.tar" diff --git a/src/dev/ci_setup/setup.sh b/src/dev/ci_setup/setup.sh index b1b14095a8bc2..db785e2524525 100755 --- a/src/dev/ci_setup/setup.sh +++ b/src/dev/ci_setup/setup.sh @@ -111,29 +111,9 @@ yarn config set yarn-offline-mirror "$cacheDir/yarn-offline-cache" yarnGlobalDir="$(yarn global bin)" export PATH="$PATH:$yarnGlobalDir" -### -### use the chromedriver cache if it exists -### -if [ -d "$dir/.chromedriver" ]; then - branchPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")" - cachedPkgVersion="$(cat "$dir/.chromedriver/pkgVersion")" - if [ "$cachedPkgVersion" == "$branchPkgVersion" ]; then - export CHROMEDRIVER_FILEPATH="$dir/.chromedriver/chromedriver.zip" - export CHROMEDRIVER_SKIP_DOWNLOAD=true - echo " -- Using chromedriver cache at $CHROMEDRIVER_FILEPATH" - else - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo " SKIPPING CHROMEDRIVER CACHE: cached($cachedPkgVersion) branch($branchPkgVersion)" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - fi -else - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo " CHROMEDRIVER CACHE NOT FOUND" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" -fi - -# use a proxy to fetch geckodriver asset -export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/geckodriver_cache" +# use a proxy to fetch chromedriver/geckodriver asset +export GECKODRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" +export CHROMEDRIVER_CDNURL="https://us-central1-elastic-kibana-184716.cloudfunctions.net/kibana-ci-proxy-cache" ### ### install dependencies From 651376be6b27ddbfb92661d8bc99e1339a1f55f0 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 19 Jul 2019 02:02:03 -0700 Subject: [PATCH 2/2] restore chromedriver and geckodriver caching in bootstrap cache --- .ci/packer_cache.sh | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.ci/packer_cache.sh b/.ci/packer_cache.sh index 1cf9b406f164e..b697f22c009d1 100755 --- a/.ci/packer_cache.sh +++ b/.ci/packer_cache.sh @@ -14,6 +14,27 @@ node scripts/es snapshot --license=oss --download-only; # download reporting browsers (cd "x-pack" && yarn gulp prepare); +# cache the chromedriver archive +chromedriverDistVersion="$(node -e "console.log(require('chromedriver').version)")" +chromedriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.chromedriver)")" +if [ -z "$chromedriverDistVersion" ] || [ -z "$chromedriverPkgVersion" ]; then + echo "UNABLE TO DETERMINE CHROMEDRIVER VERSIONS" + exit 1 +fi +mkdir -p .chromedriver +curl "https://chromedriver.storage.googleapis.com/$chromedriverDistVersion/chromedriver_linux64.zip" > .chromedriver/chromedriver.zip +echo "$chromedriverPkgVersion" > .chromedriver/pkgVersion + +# cache the geckodriver archive +geckodriverPkgVersion="$(node -e "console.log(require('./package.json').devDependencies.geckodriver)")" +if [ -z "$geckodriverPkgVersion" ]; then + echo "UNABLE TO DETERMINE geckodriver VERSIONS" + exit 1 +fi +mkdir -p ".geckodriver" +cp "node_modules/geckodriver/geckodriver.tar.gz" .geckodriver/geckodriver.tar.gz +echo "$geckodriverPkgVersion" > .geckodriver/pkgVersion + # archive cacheable directories mkdir -p "$HOME/.kibana/bootstrap_cache" tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ @@ -23,7 +44,9 @@ tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ x-pack/legacy/plugins/*/node_modules \ x-pack/legacy/plugins/reporting/.chromium \ test/plugin_functional/plugins/*/node_modules \ - .es; + .es \ + .chromedriver \ + .geckodriver; echo "created $HOME/.kibana/bootstrap_cache/$branch.tar"