ARROW-17233: [Packaging][Linux] Update artifact patterns (apache#13740)

Add `--validate` option to `archery crossbow status`. If `--validate` is specified and there are any missing artifacts, `archery crossbow status --validate` is existed with non-zero exit code. We can use it for CI to detect missing artifacts. We can't use `@github-actions crossbow submit` for this change because this isn't merged into the master branch yet. See https://github.com/ursacomputing/crossbow/branches/all?query=build-674 that is submitted `nightly-packages` manually. Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
zagto · Oct 7, 2022 · 0a9bca9 · 0a9bca9
1 parent 94c8cca
commit 0a9bca9
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 24 deletions.
diff --git a/dev/archery/archery/crossbow/cli.py b/dev/archery/archery/crossbow/cli.py
@@ -17,6 +17,7 @@
 
 from pathlib import Path
 import time
+import sys
 
 import click
 
@@ -273,16 +274,30 @@ def highlight(code):
               help='Fetch references (branches and tags) from the remote')
 @click.option('--task-filter', '-f', 'task_filters', multiple=True,
               help='Glob pattern for filtering relevant tasks')
+@click.option('--validate/--no-validate', default=False,
+              help='Return non-zero exit code '
+                   'if there is any non-success task')
 @click.pass_obj
-def status(obj, job_name, fetch, task_filters):
+def status(obj, job_name, fetch, task_filters, validate):
     output = obj['output']
     queue = obj['queue']
     if fetch:
         queue.fetch()
     job = queue.get(job_name)
 
+    success = True
+
+    def asset_callback(task_name, task, asset):
+        nonlocal success
+        if task.status().combined_state in {'error', 'failure'}:
+            success = False
+        if asset is None:
+            success = False
+
     report = ConsoleReport(job, task_filters=task_filters)
-    report.show(output)
+    report.show(output, asset_callback=asset_callback)
+    if validate and not success:
+        sys.exit(1)
 
 
 @crossbow.command()

diff --git a/dev/archery/archery/crossbow/core.py b/dev/archery/archery/crossbow/core.py
@@ -675,6 +675,7 @@ def put(self, job, prefix='build', increment_job_id=True):
             params = {
                 **job.params,
                 "arrow": job.target,
+                "job": job,
                 "queue_remote_url": self.remote_url
             }
             files = task.render_files(job.template_searchpath, params=params)
@@ -790,7 +791,7 @@ class Task(Serializable):
     submitting the job to a queue.
     """
 
-    def __init__(self, ci, template, artifacts=None, params=None):
+    def __init__(self, name, ci, template, artifacts=None, params=None):
         assert ci in {
             'circle',
             'travis',
@@ -799,6 +800,7 @@ def __init__(self, ci, template, artifacts=None, params=None):
             'github',
             'drone',
         }
+        self.name = name
         self.ci = ci
         self.template = template
         self.artifacts = artifacts or []
@@ -1011,6 +1013,7 @@ def render_tasks(self, params=None):
         params = {
             **self.params,
             "arrow": self.target,
+            "job": self,
             **(params or {})
         }
         for task_name, task in self.tasks.items():
@@ -1081,9 +1084,10 @@ def from_config(cls, config, target, tasks=None, groups=None, params=None):
                     'no_rc_version': target.no_rc_version,
                     'no_rc_semver_version': target.no_rc_semver_version}
         for task_name, task in task_definitions.items():
+            task = task.copy()
             artifacts = task.pop('artifacts', None) or []  # because of yaml
             artifacts = [fn.format(**versions) for fn in artifacts]
-            tasks[task_name] = Task(artifacts=artifacts, **task)
+            tasks[task_name] = Task(task_name, artifacts=artifacts, **task)
 
         return cls(target=target, tasks=tasks, params=params,
                    template_searchpath=config.template_searchpath)
@@ -1219,7 +1223,7 @@ def validate(self):
         # validate that the tasks are constructible
         for task_name, task in self['tasks'].items():
             try:
-                Task(**task)
+                Task(task_name, **task)
             except Exception as e:
                 raise CrossbowError(
                     'Unable to construct a task object from the '
@@ -1236,13 +1240,19 @@ def validate(self):
             version='1.0.0dev123',
             email='[email protected]'
         )
+        job = Job.from_config(config=self,
+                              target=target,
+                              tasks=self['tasks'],
+                              groups=self['groups'],
+                              params={})
 
         for task_name, task in self['tasks'].items():
-            task = Task(**task)
+            task = Task(task_name, **task)
             files = task.render_files(
                 self.template_searchpath,
                 params=dict(
                     arrow=target,
+                    job=job,
                     queue_remote_url='https://github.com/org/crossbow'
                 )
             )

diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
@@ -69,25 +69,43 @@ on:
     uses: actions/setup-python@v4
     with:
       python-version: 3.8
+  - name: Checkout Crossbow
+    uses: actions/checkout@v3
+    with:
+      path: crossbow
+      ref: {{ job.branch }}
   - name: Setup Crossbow
     shell: bash
-    run: pip install -e arrow/dev/archery[crossbow-upload]
+    run: pip install -e arrow/dev/archery[crossbow]
   - name: Upload artifacts
     shell: bash
     run: |
       archery crossbow \
-      --queue-path $(pwd) \
-      --queue-remote {{ queue_remote_url }} \
-      upload-artifacts \
-      --sha {{ task.branch }} \
-      --tag {{ task.tag }} \
-    {% if pattern is string %}
-      "{{ pattern }}"
-    {% elif pattern is iterable %}
-      {% for p in pattern %}
-      "{{ p }}" {{ "\\" if not loop.last else "" }}
-      {% endfor %}
-    {% endif %}
+        --queue-path $(pwd)/crossbow \
+        --queue-remote {{ queue_remote_url }} \
+        upload-artifacts \
+        --sha {{ task.branch }} \
+        --tag {{ task.tag }} \
+      {% if pattern is string %}
+        "{{ pattern }}"
+      {% elif pattern is iterable %}
+        {% for p in pattern %}
+        "{{ p }}" {{ "\\" if not loop.last else "" }}
+        {% endfor %}
+      {% endif %}
+    env:
+      CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }}
+  - name: Verify uploaded artifacts
+    shell: bash
+    run: |
+      archery crossbow \
+        --queue-path $(pwd)/crossbow \
+        --queue-remote {{ queue_remote_url }} \
+        status \
+        --task-filter '{{ task.name }}' \
+        --no-fetch \
+        --validate \
+        {{ job.branch }}
     env:
       CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }}
 {% endmacro %}
@@ -170,7 +188,7 @@ on:
 
 {%- macro travis_upload_releases(pattern) -%}
   - sudo -H pip3 install pygit2==1.0 cryptography==36
-  - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload]
+  - sudo -H pip3 install -e arrow/dev/archery[crossbow]
   - |
     archery crossbow \
       --queue-path $(pwd) \
@@ -185,6 +203,16 @@ on:
       "{{ p }}" {{ "\\" if not loop.last else "" }}
       {% endfor %}
     {% endif %}
+  - git fetch origin {{ job.branch }}:remotes/origin/{{ job.branch }}
+  - |
+    archery crossbow \
+      --queue-path $(pwd) \
+      --queue-remote {{ queue_remote_url }} \
+      status \
+      --task-filter '{{ task.name }}' \
+      --no-fetch \
+      --validate \
+      {{ job.branch }}
 {% endmacro %}
 
 {%- macro travis_upload_gemfury(pattern) -%}

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -627,12 +627,14 @@ tasks:
       - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-glib1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - libarrow-glib1000_{no_rc_version}-1_[a-z0-9]+.deb
+    {% if target != "ubuntu-bionic" %}
       - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python-flight1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - libarrow-python-flight1000_{no_rc_version}-1_[a-z0-9]+.deb
       - libarrow-python1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - libarrow-python1000_{no_rc_version}-1_[a-z0-9]+.deb
+    {% endif %}
       - libarrow1000-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb
       - libarrow1000_{no_rc_version}-1_[a-z0-9]+.deb
       - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb
@@ -720,7 +722,7 @@ tasks:
       - arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
     {% endif %}
       - arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-    {% if not is_rhel7_based %}
+    {% if not is_rhel7_based and architecture == "amd64" %}
       - arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
@@ -746,14 +748,14 @@ tasks:
       - arrow[0-9]+-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
     {% endif %}
       - arrow[0-9]+-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-    {% if target != "amazon-linux-2" %}
+    {% if not is_rhel7_based %}
       - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-        {% if not is_rhel7_based %}
+        {% if architecture == "amd64" %}
       - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow[0-9]+-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow[0-9]+-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
-      - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
         {% endif %}
+      - arrow[0-9]+-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
       - arrow[0-9]+-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm
     {% endif %}
     {% if architecture == "amd64" %}