Skip to content
CI

Hparam: Add selectors to retrieve data written by the new hparams effects #10709

Sign in to view logs

Hparam: Add selectors to retrieve data written by the new hparams effects

Hparam: Add selectors to retrieve data written by the new hparams effects #10709

Workflow file for this run

.github/workflows/ci.yml at 5bf172f
# GitHub Actions CI definition for TensorBoard.
#
# YAML schema for GitHub Actions:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
#
# Helpful YAML parser to clarify YAML syntax:
# https://yaml-online-parser.appspot.com/
name: CI
on:
push:
branches:
- master
- '[0-9]+.*'
- 'ci-*'
tags:
- '*'
pull_request: {}
workflow_call: {}
env:
# Keep this Bazel version in sync with the `versions.check` directive
# in our WORKSPACE file.
BAZEL_VERSION: '4.2.2'
BAZEL_SHA256SUM: '11dea6c7cfd866ed520af19a6bb1d952f3e9f4ee60ffe84e63c0825d95cb5859'
BUILDTOOLS_VERSION: '3.0.0'
BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db'
BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d'
TENSORFLOW_VERSION: 'tf-nightly'
jobs:
build:
runs-on: ubuntu-22.04
needs: lint-python-flake8 # fail fast in case of "undefined variable" errors
strategy:
fail-fast: false
matrix:
tf_version_id: ['tf', 'notf']
python_version: ['3.9']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: ${{ matrix.python_version }}
architecture: 'x64'
- name: 'Set up Bazel'
run: |
ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel
sudo mv ~/bazel /usr/local/bin/bazel
sudo chmod +x /usr/local/bin/bazel
cp ./ci/bazelrc ~/.bazelrc
- name: 'Configure build cache write credentials'
env:
CREDS: ${{ secrets.BAZEL_CACHE_SERVICE_ACCOUNT_CREDS }}
EVENT_TYPE: ${{ github.event_name }}
run: |
if [ -z "${CREDS}" ]; then
printf 'Using read-only cache (no credentials)\n'
exit
fi
if [ "${EVENT_TYPE}" = pull_request ]; then
printf 'Using read-only cache (PR build)\n'
exit
fi
printf 'Using writable cache\n'
creds_file=/tmp/service_account_creds.json
printf '%s\n' "${CREDS}" >"${creds_file}"
printf '%s\n' >>~/.bazelrc \
"common --google_credentials=${creds_file}" \
"common --remote_upload_local_results=true" \
;
- name: 'Install TensorFlow'
run: |
python -m pip install -U pip
pip install "${TENSORFLOW_VERSION}"
if: matrix.tf_version_id != 'notf'
- name: 'Install Python dependencies'
run: |
python -m pip install -U pip
pip install \
-r ./tensorboard/pip_package/requirements.txt \
-r ./tensorboard/pip_package/requirements_dev.txt \
;
- name: 'Check Pip state'
run: pip freeze --all
- name: 'Bazel: fetch'
run: bazel fetch //tensorboard/...
- name: 'Bazel: build'
# Note we suppress a flood of warnings from the proto compiler.
# Googlers see b/222706811 & b/182876485 discussion and preconditions
# for removing this kludge.
run: |
( set -o pipefail;
bazel build //tensorboard/... |&\
grep -v 'external/com_google_protobuf/python: warning: directory' |&\
grep -v 'INFO: From ProtoCompile ' )
- name: 'Bazel: test (with TensorFlow support)'
run: bazel test //tensorboard/...
if: matrix.tf_version_id != 'notf'
- name: 'Bazel: test (non-TensorFlow only)'
run: bazel test //tensorboard/... --test_tag_filters="support_notf"
if: matrix.tf_version_id == 'notf'
- name: 'Bazel: run Pip package test (with TensorFlow support)'
run: |
bazel run //tensorboard/pip_package:test_pip_package -- \
--tf-version "${TENSORFLOW_VERSION}"
if: matrix.tf_version_id != 'notf'
- name: 'Bazel: run Pip package test (non-TensorFlow only)'
run: |
bazel run //tensorboard/pip_package:test_pip_package -- \
--tf-version notf
if: matrix.tf_version_id == 'notf'
- name: 'Bazel: run manual tests'
run: |
bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test &&
bazel test //tensorboard/summary/writer:event_file_writer_s3_test &&
bazel test //tensorboard/compat/tensorflow_stub:gfile_fsspec_test &&
bazel test //tensorboard/summary/writer:event_file_writer_fsspec_test
- name: 'Bazel: build Pip package (master branch TF build only)'
# This pip package is only needed if we will upload it, so run this step
# only under the same conditions as the next step (see comment below).
if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
run: |
./tensorboard/tools/set_nightly_version.sh
rm -rf /tmp/tb_nightly_pip_package && mkdir /tmp/tb_nightly_pip_package
bazel run //tensorboard/pip_package:build_pip_package -- /tmp/tb_nightly_pip_package
- name: 'Upload Pip package as an artifact (master branch TF build only)'
# Prevent uploads when running on forks or non-master branch.
if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: tb-nightly
path: /tmp/tb_nightly_pip_package/*
build-data-server-pip:
runs-on: ${{ matrix.platform }}
strategy:
fail-fast: false
matrix:
mode: ['native']
platform: ['ubuntu-22.04', 'macos-12']
rust_version: ['1.65.0']
include:
- mode: 'universal'
platform: 'ubuntu-22.04'
rust_version: '1.65.0'
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: '3.9'
architecture: 'x64'
- name: 'Cache Cargo artifacts'
if: matrix.mode == 'native'
uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2
with:
path: |
tensorboard/data/server/target/
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
# Needed if we `cargo install` binaries (at time of writing, this
# job doesn't but the files are tiny, and this will mitigate
# confusion if we need to `cargo install` later)
~/.cargo/.crates.toml
~/.cargo/.crates2.json
key: build-data-server-pip-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
- name: 'Install Rust toolchain'
if: matrix.mode == 'native'
uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
with:
toolchain: ${{ matrix.rust_version }}
components: rustfmt
- name: 'Install Python packaging deps'
run: |
python -m pip install -U pip
pip install \
-c ./tensorboard/pip_package/requirements.txt \
-c ./tensorboard/pip_package/requirements_dev.txt \
setuptools wheel
- name: 'Build'
if: matrix.mode == 'native'
run: cd tensorboard/data/server/ && cargo build --release
- name: 'Test'
if: matrix.mode == 'native'
run: cd tensorboard/data/server/ && cargo test --release
- name: 'Package (native)'
if: matrix.mode == 'native'
run: |
rm -rf /tmp/pip_package && mkdir /tmp/pip_package
python tensorboard/data/server/pip_package/build.py \
--server-binary tensorboard/data/server/target/release/rustboard \
--out-dir /tmp/pip_package \
;
- name: 'Package (universal)'
if: matrix.mode == 'universal'
run: |
rm -rf /tmp/pip_package && mkdir /tmp/pip_package
python tensorboard/data/server/pip_package/build.py \
--universal \
--out-dir /tmp/pip_package \
;
- name: 'Upload'
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: tensorboard-data-server
path: /tmp/pip_package/*
lint-python-flake8:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
# flake8 should run on each Python version that we target,
# because the errors and warnings can differ due to language
# changes, and we want to catch them all.
python_version: ['3.9', '3.10', '3.11']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: ${{ matrix.python_version }}
architecture: 'x64'
- name: 'Install flake8'
run: |
python -m pip install -U pip
pip install flake8 -c ./tensorboard/pip_package/requirements_dev.txt
- run: pip freeze --all
- name: 'Lint Python code for errors with flake8'
# See: http://flake8.pycqa.org/en/3.7.8/user/error-codes.html
# Use the comment '# noqa: <error code>' to suppress.
run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics
lint-python-yaml-docs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: '3.10'
architecture: 'x64'
- name: 'Install black, yamllint, and the TensorFlow docs notebook tools'
run: |
python -m pip install -U pip
nbfmt_version="84bd7bea0c468667dd0ec273d426ed50f23e1a15"
pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt
pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version}
# Workaround tensorflow incompatibility with protobuf>4.
# See: https://github.com/tensorflow/tensorboard/issues/5708.
# See: https://github.com/tensorflow/tensorflow/issues/53234.
- name: 'Downgrade to compatible protobuf version'
run: pip install -U "protobuf<3.20"
- run: pip freeze --all
- name: 'Lint Python code for style with Black'
# You can run `black .` to fix all Black complaints.
run: black --check --diff .
- name: 'Lint YAML for gotchas with yamllint'
# Use '# yamllint disable-line rule:foo' to suppress.
run: yamllint -c docs/.yamllint docs docs/.yamllint
- name: 'Lint Colab notebooks for formatting with nbfmt'
run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test
lint-rust:
runs-on: ubuntu-22.04
strategy:
matrix:
rust_version: ['1.65.0']
cargo_raze_version: ['0.16.1']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: 'Cache Cargo artifacts'
uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2
with:
path: |
tensorboard/data/server/target/
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
# Needed for installing binaries (`cargo-raze`) with cache
~/.cargo/.crates.toml
~/.cargo/.crates2.json
key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
- name: 'Install Rust toolchain'
uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
with:
toolchain: ${{ matrix.rust_version }}
components: rustfmt, clippy
- name: 'Install cargo-raze'
run: cargo install cargo-raze --version ${{ matrix.cargo_raze_version }} --locked
- name: 'Run Rustfmt'
run: (cd tensorboard/data/server/ && cargo fmt -- --check)
- name: 'Run Clippy'
# You can run `cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml --fix` to fix all Clippy complaints.
# This will only apply `MachineApplicable` fixes (https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.Applicability.html), so some modifications may need to be done manually.
run: cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml -- -D warnings
- name: 'Check cargo-raze freshness'
run: |
rm -rf third_party/rust/
(cd tensorboard/data/server/ && cargo fetch && cargo raze)
git add .
git diff --staged --exit-code
lint-frontend:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 16
- run: yarn install --ignore-engines
# You can run `yarn fix-lint` to fix all Prettier complaints.
- run: yarn lint
# Make sure no tests are skipped with "focused" tests.
- run: |
! git grep -E 'f(it|describe)\(' 'tensorboard/*_test.ts'
# Make sure no one depends on Angular material and CDK directly. Please
# import the indirection in //tensorboard/webapp/angular.
- run: |
! git grep -E '"@npm//@angular/material"|"@npm//@angular/cdk"' 'tensorboard/*/BUILD' ':!tensorboard/webapp/BUILD' ':!tensorboard/webapp/angular/BUILD'
# Cannot directly depend on d3 in webapp. Must depend on
# `//tensorboard/webapp/third_party:d3` instead.
- run: |
! git grep -E '"@npm//d3"|"@npm//@types/d3"' 'tensorboard/webapp/**/*BUILD' ':!tensorboard/webapp/third_party/**'
- run: |
! git grep -E '"@npm_angular_bazel//:index.bzl"' 'tensorboard/**/BUILD'
- run: |
! git grep -E 'mat-color|$mat-' 'tensorboard/**/*.scss'
# Make sure no one depends on the numeric library directly. Please
# import the indirection in //tensorboard/webapp/third_party.
- run: |
! git grep -E '@npm//numeric' 'tensorboard/*/BUILD' ':!tensorboard/webapp/third_party/**'
lint-misc: # build, protos, etc.
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: 'Set up Buildifier'
run: |
ci/download_buildifier.sh "${BUILDTOOLS_VERSION}" "${BUILDIFIER_SHA256SUM}" ~/buildifier
sudo mv ~/buildifier /usr/local/bin/buildifier
- name: 'Set up Buildozer'
run: |
ci/download_buildozer.sh "${BUILDTOOLS_VERSION}" "${BUILDOZER_SHA256SUM}" ~/buildozer
sudo mv ~/buildozer /usr/local/bin/buildozer
- name: 'Lint BUILD files'
# TODO(tensorboard-team): address all lint warnings and remove the exemption.
run:
git ls-files -z '*BUILD' third_party/js.bzl third_party/workspace.bzl WORKSPACE | xargs -0 buildifier --mode=check --lint=warn
--warnings=-native-py,-native-java
- run: ./tensorboard/tools/mirror_urls_test.sh
- name: 'Lint for no py2 BUILD targets'
# Use | to start a literal so YAML doesn't complain about the '!' character.
run: |
! git grep 'python_version = "PY2"' '*BUILD'
- name: 'No comments on licenses rule'
# Assert buildozer error code for 'success, when no changes were made'.
# https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code
run: |
buildozer '//tensorboard/...:%licenses' remove_comment && false || test $? = 3
- name: clang-format lint
uses: DoozyX/clang-format-lint-action@0138140a2adaafd3032a3ff37f66366fd7dc88e0
with:
source: ./tensorboard
# Exclude tensorboard/compat because the source of truth is TensorFlow.
exclude: ./tensorboard/compat/proto
extensions: 'proto'
clangFormatVersion: 9
- run: ./tensorboard/tools/do_not_submit_test.sh
- run: ./tensorboard/tools/license_test.sh
- run: ./tensorboard/tools/whitespace_hygiene_test.py