From bced07de986b9f7baaa4885d7f03f4e46d1abed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 15 Sep 2022 22:53:10 +0200 Subject: [PATCH] ARROW-17628: [CI][Packaging][Java] Publish latest nightly with SNAPSHOT version (#14135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to help automate nightly distribution upload latest nightly with a fix SNAPSHOT version. The new artifact names will be like: ``` - flight-integration-tests-10.0.0-SNAPSHOT-tests.jar - flight-integration-tests-10.0.0-SNAPSHOT.jar - flight-integration-tests-10.0.0-SNAPSHOT.pom - flight-sql-10.0.0-SNAPSHOT-javadoc.jar - flight-sql-10.0.0-SNAPSHOT-sources.jar - flight-sql-10.0.0-SNAPSHOT-tests.jar - flight-sql-10.0.0-SNAPSHOT.jar - flight-sql-10.0.0-SNAPSHOT.pom ``` Authored-by: Raúl Cumplido Signed-off-by: Sutou Kouhei --- .github/workflows/java_nightly.yml | 8 +- dev/archery/archery/crossbow/core.py | 17 ++- dev/tasks/java-jars/github.yml | 2 +- dev/tasks/tasks.yml | 218 +++++++++++++-------------- 4 files changed, 129 insertions(+), 116 deletions(-) diff --git a/.github/workflows/java_nightly.yml b/.github/workflows/java_nightly.yml index 015e28984fbe0..badb9d94e5247 100644 --- a/.github/workflows/java_nightly.yml +++ b/.github/workflows/java_nightly.yml @@ -97,15 +97,19 @@ jobs: - shell: bash name: Build Repository run: | + DATE=$(date +%Y-%m-%d) if [ -z $PREFIX ]; then - PREFIX=nightly-packaging-$(date +%Y-%m-%d)-0 + PREFIX=nightly-packaging-${DATE}-0 fi - PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+.dev[0-9]+)' + PATTERN_TO_GET_LIB_AND_VERSION='([a-z].+)-([0-9]+.[0-9]+.[0-9]+-SNAPSHOT)' mkdir -p repo/org/apache/arrow/ for LIBRARY in $(ls binaries/$PREFIX/java-jars | grep -E '.jar|.pom' | grep dev); do [[ $LIBRARY =~ $PATTERN_TO_GET_LIB_AND_VERSION ]] mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} + mkdir -p repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} + # Copy twice to maintain a latest snapshot and some earlier versions cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${BASH_REMATCH[2]} + cp binaries/$PREFIX/java-jars/$LIBRARY repo/org/apache/arrow/${BASH_REMATCH[1]}/${DATE} echo "Artifacts $LIBRARY configured" done - name: Prune Repository diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py index 8045254871cec..560a9f1e199f3 100644 --- a/dev/archery/archery/crossbow/core.py +++ b/dev/archery/archery/crossbow/core.py @@ -762,6 +762,14 @@ def __init__(self, head, branch, remote, version, email=None): # '0.16.1-dev10' self.no_rc_semver_version = \ re.sub(r'\.(dev\d+)\Z', r'-\1', self.no_rc_version) + # Substitute dev version for SNAPSHOT + # + # Example: + # + # '10.0.0.dev235' -> + # '10.0.0-SNAPSHOT' + self.no_rc_snapshot_version = re.sub( + r'\.(dev\d+)$', '-SNAPSHOT', self.no_rc_version) @classmethod def from_repo(cls, repo, head=None, branch=None, remote=None, version=None, @@ -1093,15 +1101,16 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None): # instantiate the tasks tasks = {} - versions = {'version': target.version, - 'no_rc_version': target.no_rc_version, - 'no_rc_semver_version': target.no_rc_semver_version} + versions = { + 'version': target.version, + 'no_rc_version': target.no_rc_version, + 'no_rc_semver_version': target.no_rc_semver_version, + 'no_rc_snapshot_version': target.no_rc_snapshot_version} for task_name, task in task_definitions.items(): task = task.copy() artifacts = task.pop('artifacts', None) or [] # because of yaml artifacts = [fn.format(**versions) for fn in artifacts] tasks[task_name] = Task(task_name, artifacts=artifacts, **task) - return cls(target=target, tasks=tasks, params=params, template_searchpath=config.template_searchpath) diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml index abaddc028e900..5375aa5082244 100644 --- a/dev/tasks/java-jars/github.yml +++ b/dev/tasks/java-jars/github.yml @@ -118,7 +118,7 @@ jobs: run: | set -e pushd arrow/java - mvn versions:set -DnewVersion={{ arrow.no_rc_version }} + mvn versions:set -DnewVersion={{ arrow.no_rc_snapshot_version }} popd arrow/ci/scripts/java_full_build.sh \ $GITHUB_WORKSPACE/arrow \ diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 7cb20488009c0..3792dc14905d4 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -818,115 +818,115 @@ tasks: ci: github template: java-jars/github.yml artifacts: - - arrow-algorithm-{no_rc_version}-javadoc.jar - - arrow-algorithm-{no_rc_version}-sources.jar - - arrow-algorithm-{no_rc_version}-tests.jar - - arrow-algorithm-{no_rc_version}.jar - - arrow-algorithm-{no_rc_version}.pom - - arrow-avro-{no_rc_version}-javadoc.jar - - arrow-avro-{no_rc_version}-sources.jar - - arrow-avro-{no_rc_version}-tests.jar - - arrow-avro-{no_rc_version}.jar - - arrow-avro-{no_rc_version}.pom - - arrow-c-data-{no_rc_version}-javadoc.jar - - arrow-c-data-{no_rc_version}-sources.jar - - arrow-c-data-{no_rc_version}-tests.jar - - arrow-c-data-{no_rc_version}.jar - - arrow-c-data-{no_rc_version}.pom - - arrow-compression-{no_rc_version}-javadoc.jar - - arrow-compression-{no_rc_version}-sources.jar - - arrow-compression-{no_rc_version}-tests.jar - - arrow-compression-{no_rc_version}.jar - - arrow-compression-{no_rc_version}.pom - - arrow-dataset-{no_rc_version}-javadoc.jar - - arrow-dataset-{no_rc_version}-sources.jar - - arrow-dataset-{no_rc_version}-tests.jar - - arrow-dataset-{no_rc_version}.jar - - arrow-dataset-{no_rc_version}.pom - - arrow-flight-{no_rc_version}.pom - - arrow-format-{no_rc_version}-javadoc.jar - - arrow-format-{no_rc_version}-sources.jar - - arrow-format-{no_rc_version}-tests.jar - - arrow-format-{no_rc_version}.jar - - arrow-format-{no_rc_version}.pom - - arrow-gandiva-{no_rc_version}-javadoc.jar - - arrow-gandiva-{no_rc_version}-sources.jar - - arrow-gandiva-{no_rc_version}-tests.jar - - arrow-gandiva-{no_rc_version}.jar - - arrow-gandiva-{no_rc_version}.pom - - arrow-java-root-{no_rc_version}-source-release.zip - - arrow-java-root-{no_rc_version}.pom - - arrow-jdbc-{no_rc_version}-javadoc.jar - - arrow-jdbc-{no_rc_version}-sources.jar - - arrow-jdbc-{no_rc_version}-tests.jar - - arrow-jdbc-{no_rc_version}.jar - - arrow-jdbc-{no_rc_version}.pom - - arrow-memory-core-{no_rc_version}-javadoc.jar - - arrow-memory-core-{no_rc_version}-sources.jar - - arrow-memory-core-{no_rc_version}-tests.jar - - arrow-memory-core-{no_rc_version}.jar - - arrow-memory-core-{no_rc_version}.pom - - arrow-memory-netty-{no_rc_version}-javadoc.jar - - arrow-memory-netty-{no_rc_version}-sources.jar - - arrow-memory-netty-{no_rc_version}-tests.jar - - arrow-memory-netty-{no_rc_version}.jar - - arrow-memory-netty-{no_rc_version}.pom - - arrow-memory-unsafe-{no_rc_version}-javadoc.jar - - arrow-memory-unsafe-{no_rc_version}-sources.jar - - arrow-memory-unsafe-{no_rc_version}-tests.jar - - arrow-memory-unsafe-{no_rc_version}.jar - - arrow-memory-unsafe-{no_rc_version}.pom - - arrow-memory-{no_rc_version}.pom - - arrow-orc-{no_rc_version}-javadoc.jar - - arrow-orc-{no_rc_version}-sources.jar - - arrow-orc-{no_rc_version}-tests.jar - - arrow-orc-{no_rc_version}.jar - - arrow-orc-{no_rc_version}.pom - - arrow-performance-{no_rc_version}-sources.jar - - arrow-performance-{no_rc_version}-tests.jar - - arrow-performance-{no_rc_version}.jar - - arrow-performance-{no_rc_version}.pom - - arrow-plasma-{no_rc_version}-javadoc.jar - - arrow-plasma-{no_rc_version}-sources.jar - - arrow-plasma-{no_rc_version}-tests.jar - - arrow-plasma-{no_rc_version}.jar - - arrow-plasma-{no_rc_version}.pom - - arrow-tools-{no_rc_version}-jar-with-dependencies.jar - - arrow-tools-{no_rc_version}-javadoc.jar - - arrow-tools-{no_rc_version}-sources.jar - - arrow-tools-{no_rc_version}-tests.jar - - arrow-tools-{no_rc_version}.jar - - arrow-tools-{no_rc_version}.pom - - arrow-vector-{no_rc_version}-javadoc.jar - - arrow-vector-{no_rc_version}-shade-format-flatbuffers.jar - - arrow-vector-{no_rc_version}-sources.jar - - arrow-vector-{no_rc_version}-tests.jar - - arrow-vector-{no_rc_version}.jar - - arrow-vector-{no_rc_version}.pom - - flight-core-{no_rc_version}-jar-with-dependencies.jar - - flight-core-{no_rc_version}-javadoc.jar - - flight-core-{no_rc_version}-shaded-ext.jar - - flight-core-{no_rc_version}-shaded.jar - - flight-core-{no_rc_version}-sources.jar - - flight-core-{no_rc_version}-tests.jar - - flight-core-{no_rc_version}.jar - - flight-core-{no_rc_version}.pom - - flight-grpc-{no_rc_version}-javadoc.jar - - flight-grpc-{no_rc_version}-sources.jar - - flight-grpc-{no_rc_version}-tests.jar - - flight-grpc-{no_rc_version}.jar - - flight-grpc-{no_rc_version}.pom - - flight-integration-tests-{no_rc_version}-jar-with-dependencies.jar - - flight-integration-tests-{no_rc_version}-javadoc.jar - - flight-integration-tests-{no_rc_version}-sources.jar - - flight-integration-tests-{no_rc_version}-tests.jar - - flight-integration-tests-{no_rc_version}.jar - - flight-integration-tests-{no_rc_version}.pom - - flight-sql-{no_rc_version}-javadoc.jar - - flight-sql-{no_rc_version}-sources.jar - - flight-sql-{no_rc_version}-tests.jar - - flight-sql-{no_rc_version}.jar - - flight-sql-{no_rc_version}.pom + - arrow-algorithm-{no_rc_snapshot_version}-javadoc.jar + - arrow-algorithm-{no_rc_snapshot_version}-sources.jar + - arrow-algorithm-{no_rc_snapshot_version}-tests.jar + - arrow-algorithm-{no_rc_snapshot_version}.jar + - arrow-algorithm-{no_rc_snapshot_version}.pom + - arrow-avro-{no_rc_snapshot_version}-javadoc.jar + - arrow-avro-{no_rc_snapshot_version}-sources.jar + - arrow-avro-{no_rc_snapshot_version}-tests.jar + - arrow-avro-{no_rc_snapshot_version}.jar + - arrow-avro-{no_rc_snapshot_version}.pom + - arrow-c-data-{no_rc_snapshot_version}-javadoc.jar + - arrow-c-data-{no_rc_snapshot_version}-sources.jar + - arrow-c-data-{no_rc_snapshot_version}-tests.jar + - arrow-c-data-{no_rc_snapshot_version}.jar + - arrow-c-data-{no_rc_snapshot_version}.pom + - arrow-compression-{no_rc_snapshot_version}-javadoc.jar + - arrow-compression-{no_rc_snapshot_version}-sources.jar + - arrow-compression-{no_rc_snapshot_version}-tests.jar + - arrow-compression-{no_rc_snapshot_version}.jar + - arrow-compression-{no_rc_snapshot_version}.pom + - arrow-dataset-{no_rc_snapshot_version}-javadoc.jar + - arrow-dataset-{no_rc_snapshot_version}-sources.jar + - arrow-dataset-{no_rc_snapshot_version}-tests.jar + - arrow-dataset-{no_rc_snapshot_version}.jar + - arrow-dataset-{no_rc_snapshot_version}.pom + - arrow-flight-{no_rc_snapshot_version}.pom + - arrow-format-{no_rc_snapshot_version}-javadoc.jar + - arrow-format-{no_rc_snapshot_version}-sources.jar + - arrow-format-{no_rc_snapshot_version}-tests.jar + - arrow-format-{no_rc_snapshot_version}.jar + - arrow-format-{no_rc_snapshot_version}.pom + - arrow-gandiva-{no_rc_snapshot_version}-javadoc.jar + - arrow-gandiva-{no_rc_snapshot_version}-sources.jar + - arrow-gandiva-{no_rc_snapshot_version}-tests.jar + - arrow-gandiva-{no_rc_snapshot_version}.jar + - arrow-gandiva-{no_rc_snapshot_version}.pom + - arrow-java-root-{no_rc_snapshot_version}-source-release.zip + - arrow-java-root-{no_rc_snapshot_version}.pom + - arrow-jdbc-{no_rc_snapshot_version}-javadoc.jar + - arrow-jdbc-{no_rc_snapshot_version}-sources.jar + - arrow-jdbc-{no_rc_snapshot_version}-tests.jar + - arrow-jdbc-{no_rc_snapshot_version}.jar + - arrow-jdbc-{no_rc_snapshot_version}.pom + - arrow-memory-core-{no_rc_snapshot_version}-javadoc.jar + - arrow-memory-core-{no_rc_snapshot_version}-sources.jar + - arrow-memory-core-{no_rc_snapshot_version}-tests.jar + - arrow-memory-core-{no_rc_snapshot_version}.jar + - arrow-memory-core-{no_rc_snapshot_version}.pom + - arrow-memory-netty-{no_rc_snapshot_version}-javadoc.jar + - arrow-memory-netty-{no_rc_snapshot_version}-sources.jar + - arrow-memory-netty-{no_rc_snapshot_version}-tests.jar + - arrow-memory-netty-{no_rc_snapshot_version}.jar + - arrow-memory-netty-{no_rc_snapshot_version}.pom + - arrow-memory-unsafe-{no_rc_snapshot_version}-javadoc.jar + - arrow-memory-unsafe-{no_rc_snapshot_version}-sources.jar + - arrow-memory-unsafe-{no_rc_snapshot_version}-tests.jar + - arrow-memory-unsafe-{no_rc_snapshot_version}.jar + - arrow-memory-unsafe-{no_rc_snapshot_version}.pom + - arrow-memory-{no_rc_snapshot_version}.pom + - arrow-orc-{no_rc_snapshot_version}-javadoc.jar + - arrow-orc-{no_rc_snapshot_version}-sources.jar + - arrow-orc-{no_rc_snapshot_version}-tests.jar + - arrow-orc-{no_rc_snapshot_version}.jar + - arrow-orc-{no_rc_snapshot_version}.pom + - arrow-performance-{no_rc_snapshot_version}-sources.jar + - arrow-performance-{no_rc_snapshot_version}-tests.jar + - arrow-performance-{no_rc_snapshot_version}.jar + - arrow-performance-{no_rc_snapshot_version}.pom + - arrow-plasma-{no_rc_snapshot_version}-javadoc.jar + - arrow-plasma-{no_rc_snapshot_version}-sources.jar + - arrow-plasma-{no_rc_snapshot_version}-tests.jar + - arrow-plasma-{no_rc_snapshot_version}.jar + - arrow-plasma-{no_rc_snapshot_version}.pom + - arrow-tools-{no_rc_snapshot_version}-jar-with-dependencies.jar + - arrow-tools-{no_rc_snapshot_version}-javadoc.jar + - arrow-tools-{no_rc_snapshot_version}-sources.jar + - arrow-tools-{no_rc_snapshot_version}-tests.jar + - arrow-tools-{no_rc_snapshot_version}.jar + - arrow-tools-{no_rc_snapshot_version}.pom + - arrow-vector-{no_rc_snapshot_version}-javadoc.jar + - arrow-vector-{no_rc_snapshot_version}-shade-format-flatbuffers.jar + - arrow-vector-{no_rc_snapshot_version}-sources.jar + - arrow-vector-{no_rc_snapshot_version}-tests.jar + - arrow-vector-{no_rc_snapshot_version}.jar + - arrow-vector-{no_rc_snapshot_version}.pom + - flight-core-{no_rc_snapshot_version}-jar-with-dependencies.jar + - flight-core-{no_rc_snapshot_version}-javadoc.jar + - flight-core-{no_rc_snapshot_version}-shaded-ext.jar + - flight-core-{no_rc_snapshot_version}-shaded.jar + - flight-core-{no_rc_snapshot_version}-sources.jar + - flight-core-{no_rc_snapshot_version}-tests.jar + - flight-core-{no_rc_snapshot_version}.jar + - flight-core-{no_rc_snapshot_version}.pom + - flight-grpc-{no_rc_snapshot_version}-javadoc.jar + - flight-grpc-{no_rc_snapshot_version}-sources.jar + - flight-grpc-{no_rc_snapshot_version}-tests.jar + - flight-grpc-{no_rc_snapshot_version}.jar + - flight-grpc-{no_rc_snapshot_version}.pom + - flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar + - flight-integration-tests-{no_rc_snapshot_version}-javadoc.jar + - flight-integration-tests-{no_rc_snapshot_version}-sources.jar + - flight-integration-tests-{no_rc_snapshot_version}-tests.jar + - flight-integration-tests-{no_rc_snapshot_version}.jar + - flight-integration-tests-{no_rc_snapshot_version}.pom + - flight-sql-{no_rc_snapshot_version}-javadoc.jar + - flight-sql-{no_rc_snapshot_version}-sources.jar + - flight-sql-{no_rc_snapshot_version}-tests.jar + - flight-sql-{no_rc_snapshot_version}.jar + - flight-sql-{no_rc_snapshot_version}.pom ############################## NuGet packages ###############################