From 8207cd522eb2ae21ee6e23866e0f07df3d7651eb Mon Sep 17 00:00:00 2001 From: Harold Wang Date: Mon, 30 Nov 2020 16:53:46 -0800 Subject: [PATCH 1/5] Headless chrome creation script and readme file --- .../headless-chrome/README_chrome_headless.md | 51 ++++++ .../headless-chrome/build_headless_chrome.sh | 173 ++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md create mode 100644 kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh diff --git a/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md b/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md new file mode 100644 index 00000000..632eba0f --- /dev/null +++ b/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md @@ -0,0 +1,51 @@ +## Chrome Binaries for Kibana Reports used by Puppeteer +Headless Chrome for Linux and Mac are chrome binaries which are significantly smaller than the standard binaries shipped by Google and Puppeteer. +Chrome binary can be built from shell script build_headless_chrome.sh for Mac, Linux x64 and Linux arm64, +output of script is called headless_shell. +​ + +## Puppeteer's Chrome version + +Find the puppeteer version used in Kibana node_modules.json and get the associated chrome SHA to build from crrev.com and puppeteer repositories. Puppeteer 1.9 uses rev 674921 with commit sha as 312d84c8ce62810976feda0d3457108a6dfff9e6) + +## headless Chrome folder structure +-chromium + |-chromium + |-chromium + |-src + |-out + |-headless + |-headless_shell # output of scripts +​ +## How to generate the headless_chrome +This is a shell script to set environment variable, download the source code and build the executable. +​ +## Commands to create headless_chrome +Run below command to create headless_shell for each platform +​ +headless-chrome.sh chrome-version-SHA (arch_name (arm64)) +. Mac x64: ./build_headless_chrome.sh +. Linux x64: ./build_headless_chrome.sh +. Linux arm64: ./build_headless_chrome.sh arm64 +​ +# How to call in Command line: + >> please use what you used for testing this << replace me >> + +## Headless Chromium for MAC +# Files: + headless_shell + libswiftshader_libGLESv2.dylib + +## Headless Chromium for Linux (arm64 and x64) +# Files: + headless_shell + swiftshader + |-libEGL.so + |-libEGL.so.TOC + |-libGLESv2.so + |-libGLESv2.so.TOC +# Additional libaries: + libnss3-dev + fonts-liberation + libfontconfig + (installation command example : sudo apt update && apt install libnss3-dev fonts-liberation libfontconfig) \ No newline at end of file diff --git a/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh b/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh new file mode 100644 index 00000000..4546f9c1 --- /dev/null +++ b/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Initializes a Linux environment. This need only be done once per +# machine. The OS needs to be a flavor that supports apt get, such as Ubuntu. + +function generateArgs { +if [ $1 == 'linux' ]; then + echo 'import("//build/args/headless.gn") +is_component_build = false +remove_webcore_debug_symbols = true +enable_nacl = false +is_debug = false +symbol_level = 0 +use_kerberos = false' > args.gn +elif [ $1 == 'darwin' ]; then + echo '#args configuration + +icu_use_data_file = false +v8_use_external_startup_data = false +remove_webcore_debug_symbols = true +use_kerberos = false +use_libpci = false +use_pulseaudio = false +use_udev = false +is_debug = false +symbol_level = 0 +is_component_build = false +enable_nacl = false +enable_print_preview = false +enable_basic_printing = false +enable_remoting = false +use_alsa = false +use_cups = false +use_dbus = false +use_gio = false +' > args.gn +fi +} + +ARGC=("$#") + +if [ $ARGC -lt 1 ]; +then + echo "format: build_headless_chrome.sh {chrome_source_version} (arch_name)" + echo "Mac x64: ./build_headless_chrome.sh 312d84c8ce62810976feda0d3457108a6dfff9e6" + echo "Linux x64: ./build_headless_chrome.sh 312d84c8ce62810976feda0d3457108a6dfff9e6" + echo "Linux arm64: ./build_headless_chrome.sh 312d84c8ce62810976feda0d3457108a6dfff9e6 arm64" + exit +fi + +source_version=$1 + +if [ $ARGC -lt 2 ]; +then + arch_name="x64" +else + arch_name=$2 +fi + +if ! [ -x "$(command -v python)" ]; then + echo "Python is not found, please install python or setup python environment properly" + exit +fi + +# Launch the cross-platform init script using a relative path +# from this script's location. +mkdir -p ~/chromium + +if [ "$#" -eq 2 ]; then + arch_name=$2 +fi + +current_folder=$(pwd) + +# find the current platform +platform_name='unknown' +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + platform_name='linux' +elif [[ "$OSTYPE" == "darwin"* ]]; then + platform_name='darwin' +elif [[ "$OSTYPE" == "win32" ]]; then + platform_name='windows' +fi + +if [[ "$platform_name" == "unknown" ]]; then + echo "platform is" $platform_name + exit +fi + +echo "source_version = " $source_version +echo "platform_name = " $platform_name +echo "arch_name = " $arch_name +generateArgs $platform_name + +# Configure git +git config --global core.autocrlf false +git config --global core.filemode false +git config --global branch.autosetuprebase always +cd chromium + +# Grab Chromium's custom build tools, if they aren't already installed +# (On Windows, they are installed before this Python script is run) +if ! [ -d "depot_tools" ] +then + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +fi + +# Put depot_tools on the path so we can properly run the fetch command +export PATH="$PATH:${HOME}/chromium/depot_tools" +echo ${HOME}/chromium/depot_tools + +# Fetch the Chromium source code + +if [ -d 'chromium' ]; then + echo "chromium src aready exists, please delete it and retry..." + exit +fi + +mkdir -p chromium +cd chromium +pwd + +# Build Linux deps +echo "fetching chromium..." +fetch chromium + + +# Build Linux deps + +cd src +if [[ arch_name -eq "arm64" ]]; then + ./build/linux/sysroot_scripts/install-sysroot.py --arch=$arch_name + ./build/install-build-deps.sh +fi + + +# Set to "arm" to build for ARM on Linux +echo 'Building Chromium ' $source_version ' for ' $arch_name + +# Sync the codebase to the correct version, syncing master first +# to ensure that we actually have all the versions we may refer to +echo 'Syncing source code' + + +git checkout -f master +git fetch -f origin +gclient sync --with_branch_heads --with_tags --jobs 16 +git checkout $source_version +gclient sync --with_branch_heads --with_tags --jobs 16 +gclient runhooks +echo "current_folder :" $current_folder + +platform_build_args=$current_folder'/args.gn' +#platform_build_args=$current_folder/chromium/build_chromium/$platform_name/args.gn + +outputDir='headless' +mkdir -p 'out/headless' + +echo "platform_build_args :" $platform_build_args + +cp $platform_build_args 'out/headless/args.gn' +echo "platform_build_args :" $platform_build_args +echo 'target_cpu = '\"$arch_name\" >> 'out/headless/args.gn' + +gn gen out/headless + +autoninja -C out/headless headless_shell + +if [[ ($platform_name != "Windows" && $arch_name != 'arm64') ]]; then + echo 'Optimizing headless_shell' + mv out/headless/headless_shell out/headless/headless_shell_raw + strip -o out/headless/headless_shell out/headless/headless_shell_raw +fi From fd2a78d5f989b5cda29639c8c7af9781efd780b5 Mon Sep 17 00:00:00 2001 From: Harold Wang Date: Tue, 1 Dec 2020 10:22:26 -0800 Subject: [PATCH 2/5] Rename README_chrome_headless.md to README.md --- .../headless-chrome/README_chrome_headless.md | 51 ------------------- 1 file changed, 51 deletions(-) delete mode 100644 kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md diff --git a/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md b/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md deleted file mode 100644 index 632eba0f..00000000 --- a/kibana-reports/rendering-engine/headless-chrome/README_chrome_headless.md +++ /dev/null @@ -1,51 +0,0 @@ -## Chrome Binaries for Kibana Reports used by Puppeteer -Headless Chrome for Linux and Mac are chrome binaries which are significantly smaller than the standard binaries shipped by Google and Puppeteer. -Chrome binary can be built from shell script build_headless_chrome.sh for Mac, Linux x64 and Linux arm64, -output of script is called headless_shell. -​ - -## Puppeteer's Chrome version - -Find the puppeteer version used in Kibana node_modules.json and get the associated chrome SHA to build from crrev.com and puppeteer repositories. Puppeteer 1.9 uses rev 674921 with commit sha as 312d84c8ce62810976feda0d3457108a6dfff9e6) - -## headless Chrome folder structure --chromium - |-chromium - |-chromium - |-src - |-out - |-headless - |-headless_shell # output of scripts -​ -## How to generate the headless_chrome -This is a shell script to set environment variable, download the source code and build the executable. -​ -## Commands to create headless_chrome -Run below command to create headless_shell for each platform -​ -headless-chrome.sh chrome-version-SHA (arch_name (arm64)) -. Mac x64: ./build_headless_chrome.sh -. Linux x64: ./build_headless_chrome.sh -. Linux arm64: ./build_headless_chrome.sh arm64 -​ -# How to call in Command line: - >> please use what you used for testing this << replace me >> - -## Headless Chromium for MAC -# Files: - headless_shell - libswiftshader_libGLESv2.dylib - -## Headless Chromium for Linux (arm64 and x64) -# Files: - headless_shell - swiftshader - |-libEGL.so - |-libEGL.so.TOC - |-libGLESv2.so - |-libGLESv2.so.TOC -# Additional libaries: - libnss3-dev - fonts-liberation - libfontconfig - (installation command example : sudo apt update && apt install libnss3-dev fonts-liberation libfontconfig) \ No newline at end of file From 7ebf21ea294ce5ec8a81513d6a33a08ad905e815 Mon Sep 17 00:00:00 2001 From: Harold Wang Date: Tue, 1 Dec 2020 11:47:33 -0800 Subject: [PATCH 3/5] Add README.md --- .../headless-chrome/README.md | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 kibana-reports/rendering-engine/headless-chrome/README.md diff --git a/kibana-reports/rendering-engine/headless-chrome/README.md b/kibana-reports/rendering-engine/headless-chrome/README.md new file mode 100644 index 00000000..632eba0f --- /dev/null +++ b/kibana-reports/rendering-engine/headless-chrome/README.md @@ -0,0 +1,51 @@ +## Chrome Binaries for Kibana Reports used by Puppeteer +Headless Chrome for Linux and Mac are chrome binaries which are significantly smaller than the standard binaries shipped by Google and Puppeteer. +Chrome binary can be built from shell script build_headless_chrome.sh for Mac, Linux x64 and Linux arm64, +output of script is called headless_shell. +​ + +## Puppeteer's Chrome version + +Find the puppeteer version used in Kibana node_modules.json and get the associated chrome SHA to build from crrev.com and puppeteer repositories. Puppeteer 1.9 uses rev 674921 with commit sha as 312d84c8ce62810976feda0d3457108a6dfff9e6) + +## headless Chrome folder structure +-chromium + |-chromium + |-chromium + |-src + |-out + |-headless + |-headless_shell # output of scripts +​ +## How to generate the headless_chrome +This is a shell script to set environment variable, download the source code and build the executable. +​ +## Commands to create headless_chrome +Run below command to create headless_shell for each platform +​ +headless-chrome.sh chrome-version-SHA (arch_name (arm64)) +. Mac x64: ./build_headless_chrome.sh +. Linux x64: ./build_headless_chrome.sh +. Linux arm64: ./build_headless_chrome.sh arm64 +​ +# How to call in Command line: + >> please use what you used for testing this << replace me >> + +## Headless Chromium for MAC +# Files: + headless_shell + libswiftshader_libGLESv2.dylib + +## Headless Chromium for Linux (arm64 and x64) +# Files: + headless_shell + swiftshader + |-libEGL.so + |-libEGL.so.TOC + |-libGLESv2.so + |-libGLESv2.so.TOC +# Additional libaries: + libnss3-dev + fonts-liberation + libfontconfig + (installation command example : sudo apt update && apt install libnss3-dev fonts-liberation libfontconfig) \ No newline at end of file From aaef3fd0ae7ea1128d9f84c8959f88f51a808ed5 Mon Sep 17 00:00:00 2001 From: Harold Wang Date: Mon, 7 Dec 2020 17:00:19 -0800 Subject: [PATCH 4/5] add comments on how to run chrome to get pdf and png report --- .../rendering-engine/headless-chrome/README.md | 16 +++++++++------- .../headless-chrome/build_headless_chrome.sh | 3 +++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/kibana-reports/rendering-engine/headless-chrome/README.md b/kibana-reports/rendering-engine/headless-chrome/README.md index 632eba0f..eaba957d 100644 --- a/kibana-reports/rendering-engine/headless-chrome/README.md +++ b/kibana-reports/rendering-engine/headless-chrome/README.md @@ -16,21 +16,22 @@ Find the puppeteer version used in Kibana node_modules.json and get the associat |-out |-headless |-headless_shell # output of scripts -​ + ## How to generate the headless_chrome This is a shell script to set environment variable, download the source code and build the executable. -​ + ## Commands to create headless_chrome Run below command to create headless_shell for each platform -​ + headless-chrome.sh chrome-version-SHA (arch_name (arm64)) . Mac x64: ./build_headless_chrome.sh . Linux x64: ./build_headless_chrome.sh . Linux arm64: ./build_headless_chrome.sh arm64 -​ + # How to call in Command line: - >> please use what you used for testing this << replace me >> - +. PNG report: ./headless_shell --headless --disable-gpu --screenshot=test.png https://www.google.com +. PDF report: ./headless_shell --headless --disable-gpu --print-to-pdf=test.pdf https://www.google.com + ## Headless Chromium for MAC # Files: headless_shell @@ -48,4 +49,5 @@ headless-chrome.sh chrome-version-SHA (arch_name (arm64)) libnss3-dev fonts-liberation libfontconfig - (installation command example : sudo apt update && apt install libnss3-dev fonts-liberation libfontconfig) \ No newline at end of file + (installation command example : sudo apt update && apt install libnss3-dev fonts-liberation libfontconfig) + diff --git a/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh b/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh index 4546f9c1..219ba8e8 100644 --- a/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh +++ b/kibana-reports/rendering-engine/headless-chrome/build_headless_chrome.sh @@ -130,6 +130,9 @@ fetch chromium cd src if [[ arch_name -eq "arm64" ]]; then ./build/linux/sysroot_scripts/install-sysroot.py --arch=$arch_name +fi + +if [[ platform_name -eq "linux" ]]; then ./build/install-build-deps.sh fi From 3d89dd6fcb4b03e4fcfb384c61bbec7c0e23c07f Mon Sep 17 00:00:00 2001 From: Harold Wang Date: Mon, 7 Dec 2020 17:23:01 -0800 Subject: [PATCH 5/5] Change test web from google.com to https://opendistro.github.io/for-elasticsearch --- kibana-reports/rendering-engine/headless-chrome/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/kibana-reports/rendering-engine/headless-chrome/README.md b/kibana-reports/rendering-engine/headless-chrome/README.md index eaba957d..6cb57034 100644 --- a/kibana-reports/rendering-engine/headless-chrome/README.md +++ b/kibana-reports/rendering-engine/headless-chrome/README.md @@ -2,7 +2,6 @@ Headless Chrome for Linux and Mac are chrome binaries which are significantly smaller than the standard binaries shipped by Google and Puppeteer. Chrome binary can be built from shell script build_headless_chrome.sh for Mac, Linux x64 and Linux arm64, output of script is called headless_shell. -​ ## Puppeteer's Chrome version @@ -29,8 +28,8 @@ headless-chrome.sh chrome-version-SHA (arch_name (arm64)) . Linux arm64: ./build_headless_chrome.sh arm64 # How to call in Command line: -. PNG report: ./headless_shell --headless --disable-gpu --screenshot=test.png https://www.google.com -. PDF report: ./headless_shell --headless --disable-gpu --print-to-pdf=test.pdf https://www.google.com +. PNG report: ./headless_shell --headless --disable-gpu --screenshot=test.png https://opendistro.github.io/for-elasticsearch +. PDF report: ./headless_shell --headless --disable-gpu --print-to-pdf=test.pdf https://opendistro.github.io/for-elasticsearch ## Headless Chromium for MAC # Files: