Merge remote-tracking branch 'upstream/main' into main

alexvorxx · Jul 28, 2024 · bdaed93 · bdaed93
2 parents f815bfd + c3b25dd
commit bdaed93
Show file tree

Hide file tree

Showing 2,979 changed files with 218,756 additions and 109,699 deletions.
diff --git a/.ci-farms-disabled/vmware → .ci-farms/vmware b/.ci-farms-disabled/vmware → .ci-farms/vmware
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
+.cache
 .vscode*
 *.pyc
 *.pyo

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -33,32 +33,32 @@ workflow:
     # merge pipeline
     - if: &is-merge-attempt $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "merge_request_event"
       variables:
-        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         MESA_CI_PERFORMANCE_ENABLED: 1
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: ""  # Empty tags are ignored by gitlab
     # post-merge pipeline
     - if: &is-post-merge $GITLAB_USER_LOGIN == "marge-bot" && $CI_PIPELINE_SOURCE == "push"
     # nightly pipeline
     - if: &is-scheduled-pipeline $CI_PIPELINE_SOURCE == "schedule"
       variables:
-        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         JOB_PRIORITY: 50
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
     # pipeline for direct pushes that bypassed the CI
     - if: &is-direct-push $CI_PROJECT_NAMESPACE == "mesa" && $CI_PIPELINE_SOURCE == "push" && $GITLAB_USER_LOGIN != "marge-bot"
       variables:
-        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         JOB_PRIORITY: 40
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
     # pre-merge or fork pipeline
     - if: $FORCE_KERNEL_TAG != null
       variables:
-        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${FORCE_KERNEL_TAG}
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${FORCE_KERNEL_TAG}
         JOB_PRIORITY: 50
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
     - if: $FORCE_KERNEL_TAG == null
       variables:
-        KERNEL_IMAGE_BASE: https://${S3_HOST}/mesa-lava/${KERNEL_REPO}/${KERNEL_TAG}
+        KERNEL_IMAGE_BASE: https://${S3_HOST}/${S3_KERNEL_BUCKET}/${KERNEL_REPO}/${KERNEL_TAG}
         JOB_PRIORITY: 50
         VALVE_INFRA_VANGOGH_JOB_PRIORITY: priority:low
 
@@ -74,12 +74,22 @@ variables:
           set +o xtrace
   S3_JWT_FILE: /s3_jwt
   S3_HOST: s3.freedesktop.org
+  # This bucket is used to fetch the kernel image
+  S3_KERNEL_BUCKET: mesa-rootfs
+  # Bucket for git cache
+  S3_GITCACHE_BUCKET: git-cache
+  # Bucket for the pipeline artifacts pushed to S3
+  S3_ARTIFACTS_BUCKET: artifacts
+  # Buckets for traces
+  S3_TRACIE_RESULTS_BUCKET: mesa-tracie-results
+  S3_TRACIE_PUBLIC_BUCKET: mesa-tracie-public
+  S3_TRACIE_PRIVATE_BUCKET: mesa-tracie-private
   # per-pipeline artifact storage on MinIO
-  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/artifacts/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
+  PIPELINE_ARTIFACTS_BASE: ${S3_HOST}/${S3_ARTIFACTS_BUCKET}/${CI_PROJECT_PATH}/${CI_PIPELINE_ID}
   # per-job artifact storage on MinIO
   JOB_ARTIFACTS_BASE: ${PIPELINE_ARTIFACTS_BASE}/${CI_JOB_ID}
   # reference images stored for traces
-  PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/mesa-tracie-results/$FDO_UPSTREAM_REPO"
+  PIGLIT_REPLAY_REFERENCE_IMAGES_BASE: "${S3_HOST}/${S3_TRACIE_RESULTS_BUCKET}/$FDO_UPSTREAM_REPO"
   # For individual CI farm status see .ci-farms folder
   # Disable farm with   `git mv .ci-farms{,-disabled}/$farm_name`
   # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
@@ -287,9 +297,9 @@ sanity:
       image_tags=(
         DEBIAN_BASE_TAG
         DEBIAN_BUILD_TAG
-        DEBIAN_X86_64_TEST_ANDROID_TAG
-        DEBIAN_X86_64_TEST_GL_TAG
-        DEBIAN_X86_64_TEST_VK_TAG
+        DEBIAN_TEST_ANDROID_TAG
+        DEBIAN_TEST_GL_TAG
+        DEBIAN_TEST_VK_TAG
         ALPINE_X86_64_BUILD_TAG
         ALPINE_X86_64_LAVA_SSH_TAG
         FEDORA_X86_64_BUILD_TAG

diff --git a/.gitlab-ci/b2c/b2c.yml.jinja2.jinja2 b/.gitlab-ci/b2c/b2c.yml.jinja2.jinja2
@@ -5,17 +5,25 @@ target:
   id: '{{ ci_runner_id }}'
 
 timeouts:
+
   first_console_activity:  # This limits the time it can take to receive the first console log
-    minutes: {{ timeout_first_minutes }}
-    retries: {{ timeout_first_retries }}
+    minutes: {{ timeout_first_console_activity_minutes | default(0, true) }}
+    seconds: {{ timeout_first_console_activity_seconds | default(0, true) }}
+    retries: {{ timeout_first_console_activity_retries }}
+
   console_activity:        # Reset every time we receive a message from the logs
-    minutes: {{ timeout_minutes }}
-    retries: {{ timeout_retries }}
+    minutes: {{ timeout_console_activity_minutes | default(0, true) }}
+    seconds: {{ timeout_console_activity_seconds | default(0, true) }}
+    retries: {{ timeout_console_activity_retries }}
+
   boot_cycle:
-    minutes: {{ timeout_boot_minutes }}
+    minutes: {{ timeout_boot_minutes | default(0, true) }}
+    seconds: {{ timeout_boot_seconds | default(0, true) }}
     retries: {{ timeout_boot_retries }}
+
   overall:                 # Maximum time the job can take, not overrideable by the "continue" deployment
-    minutes: {{ timeout_overall_minutes }}
+    minutes: {{ timeout_overall_minutes | default(0, true) }}
+    seconds: {{ timeout_overall_seconds | default(0, true) }}
     retries: 0
     # no retries possible here
 
@@ -31,29 +39,38 @@ console_patterns:
     job_success:
         regex: >-
           {{ job_success_regex }}
+{% if job_warn_regex %}
     job_warn:
         regex: >-
           {{ job_warn_regex }}
+{% endif %}
 
 # Environment to deploy
 deployment:
   # Initial boot
   start:
+    storage:
+        http:
+          - path: "/b2c-extra-args"
+            data: >
+              b2c.pipefail b2c.poweroff_delay={{ poweroff_delay }}
+              b2c.minio="gateway,{{ '{{' }} minio_url }},{{ '{{' }} job_bucket_access_key }},{{ '{{' }} job_bucket_secret_key }}"
+              b2c.volume="{{ '{{' }} job_bucket }}-results,mirror=gateway/{{ '{{' }} job_bucket }},pull_on=pipeline_start,push_on=changes,overwrite{% for excl in job_volume_exclusions %},exclude={{ excl }}{% endfor %},remove,expiration=pipeline_end,preserve"
+              {% for volume in volumes %}
+              b2c.volume={{ volume }}
+              {% endfor %}
+              b2c.service="--privileged --tls-verify=false --pid=host docker://{{ '{{' }} fdo_proxy_registry }}/gfx-ci/ci-tron/telegraf:latest" b2c.hostname=dut-{{ '{{' }} machine.full_name }}
+              b2c.container="-v {{ '{{' }} job_bucket }}-results:{{ working_dir }} -w {{ working_dir }} {% for mount_volume in mount_volumes %} -v {{ mount_volume }}{% endfor %} --tls-verify=false docker://{{ local_container }} {{ container_cmd }}"
     kernel:
       url: '{{ kernel_url }}'
+
+      # NOTE: b2c.cache_device should not be here, but this works around
+      # a limitation of b2c which will be removed in the next release
       cmdline: >
         SALAD.machine_id={{ '{{' }} machine_id }}
-        console={{ '{{' }} local_tty_device }},115200 earlyprintk=vga,keep
-        loglevel={{ log_level }} no_hash_pointers
-        b2c.service="--privileged --tls-verify=false --pid=host docker://{{ '{{' }} fdo_proxy_registry }}/gfx-ci/ci-tron/telegraf:latest" b2c.hostname=dut-{{ '{{' }} machine.full_name }}
-        b2c.container="-ti --tls-verify=false docker://{{ '{{' }} fdo_proxy_registry }}/gfx-ci/ci-tron/machine-registration:latest check"
-        b2c.ntp_peer=10.42.0.1 b2c.pipefail b2c.cache_device=auto b2c.poweroff_delay={{ poweroff_delay }}
-        b2c.minio="gateway,{{ '{{' }} minio_url }},{{ '{{' }} job_bucket_access_key }},{{ '{{' }} job_bucket_secret_key }}"
-        b2c.volume="{{ '{{' }} job_bucket }}-results,mirror=gateway/{{ '{{' }} job_bucket }},pull_on=pipeline_start,push_on=changes,overwrite{% for excl in job_volume_exclusions %},exclude={{ excl }}{% endfor %},remove,expiration=pipeline_end,preserve"
-{% for volume in volumes %}
-        b2c.volume={{ volume }}
-{% endfor %}
-        b2c.container="-v {{ '{{' }} job_bucket }}-results:{{ working_dir }} -w {{ working_dir }} {% for mount_volume in mount_volumes %} -v {{ mount_volume }}{% endfor %} --tls-verify=false docker://{{ local_container }} {{ container_cmd }}"
+        console={{ '{{' }} local_tty_device }},115200
+        b2c.cache_device=auto b2c.ntp_peer=10.42.0.1
+        b2c.extra_args_url={{ '{{' }} job.http.url }}/b2c-extra-args
         {% if kernel_cmdline_extras is defined %}
         {{ kernel_cmdline_extras }}
         {% endif %}

diff --git a/.gitlab-ci/bare-metal/poe-off b/.gitlab-ci/bare-metal/poe-off
@@ -10,7 +10,7 @@ if [ -z "$BM_POE_ADDRESS" ]; then
     exit 1
 fi
 
-SNMP_KEY="SNMPv2-SMI::mib-2.105.1.1.1.3.1.$((48 + BM_POE_INTERFACE))"
+SNMP_KEY="SNMPv2-SMI::mib-2.105.1.1.1.3.1.$((${BM_POE_BASE:-0} + BM_POE_INTERFACE))"
 SNMP_OFF="i 2"
 
 flock /var/run/poe.lock -c "snmpset -v2c -r 3 -t 30 -cmesaci $BM_POE_ADDRESS $SNMP_KEY $SNMP_OFF"
diff --git a/.gitlab-ci/bare-metal/poe-on b/.gitlab-ci/bare-metal/poe-on
@@ -10,7 +10,7 @@ if [ -z "$BM_POE_ADDRESS" ]; then
     exit 1
 fi
 
-SNMP_KEY="SNMPv2-SMI::mib-2.105.1.1.1.3.1.$((48 + BM_POE_INTERFACE))"
+SNMP_KEY="SNMPv2-SMI::mib-2.105.1.1.1.3.1.$((${BM_POE_BASE:-0} + BM_POE_INTERFACE))"
 SNMP_ON="i 1"
 SNMP_OFF="i 2"
 

diff --git a/.gitlab-ci/bare-metal/poe_run.py b/.gitlab-ci/bare-metal/poe_run.py
@@ -64,7 +64,7 @@ def run(self):
         if not boot_detected:
             self.print_error(
                 "Something wrong; couldn't detect the boot start up sequence")
-            return 1
+            return 2
 
         self.logger.create_job_phase("test")
         for line in self.ser.lines(timeout=self.test_timeout, phase="test"):

diff --git a/.gitlab-ci/build/gitlab-ci.yml b/.gitlab-ci/build/gitlab-ci.yml
@@ -77,7 +77,7 @@ debian-testing:
       -D gallium-nine=true
       -D gallium-va=enabled
       -D gallium-rusticl=true
-    GALLIUM_DRIVERS: "swrast,virgl,radeonsi,zink,crocus,iris,i915,r300,svga"
+    GALLIUM_DRIVERS: "llvmpipe,softpipe,virgl,radeonsi,zink,crocus,iris,i915,r300,svga"
     VULKAN_DRIVERS: "swrast,amd,intel,intel_hasvk,virtio,nouveau"
     BUILDTYPE: "debugoptimized"
     EXTRA_OPTION: >
@@ -138,7 +138,7 @@ debian-testing-msan:
     # GLSL has some issues in sexpression reading.
     # gtest has issues in its test initialization.
     MESON_TEST_ARGS: "--suite glcpp --suite format"
-    GALLIUM_DRIVERS: "freedreno,iris,nouveau,kmsro,r300,r600,swrast,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,radeonsi,tegra,d3d12,crocus"
+    GALLIUM_DRIVERS: "freedreno,iris,nouveau,kmsro,r300,r600,llvmpipe,softpipe,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,radeonsi,tegra,d3d12,crocus"
     VULKAN_DRIVERS: intel,amd,broadcom,virtio
     # Do a host build for intel-clc (msan complains about
     # uninitialized values in the LLVM libs)
@@ -154,7 +154,9 @@ debian-testing-msan:
       -D intel-clc=enabled
       -D install-intel-clc=true
 
-debian-build-testing:
+# Disabled because it is unacceptably slow and blocks too many MRs
+# Should be re-enabled once this problem has been fixed.
+.debian-build-testing:
   extends: .meson-build
   variables:
     BUILDTYPE: debug
@@ -172,9 +174,9 @@ debian-build-testing:
       -D gallium-omx=bellagio
       -D gallium-va=enabled
       -D gallium-xa=enabled
-      -D gallium-nine=true
+      -D gallium-nine=false
       -D gallium-rusticl=false
-    GALLIUM_DRIVERS: "iris,nouveau,kmsro,r300,r600,freedreno,swrast,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,d3d12,asahi,crocus"
+    GALLIUM_DRIVERS: "iris,nouveau,kmsro,r300,r600,freedreno,llvmpipe,softpipe,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,d3d12,asahi,crocus"
     VULKAN_DRIVERS: swrast
     EXTRA_OPTION: >
       -D spirv-to-dxil=true
@@ -190,24 +192,24 @@ debian-build-testing:
     .gitlab-ci/run-shellcheck.sh
     section_switch yamllint "yamllint"
     .gitlab-ci/run-yamllint.sh
-    section_switch meson "meson"
+    section_end yamllint
     .gitlab-ci/meson/build.sh
     .gitlab-ci/prepare-artifacts.sh
   timeout: 15m
 
-shader-db:
+# Disabled because it currently needs debian-build-testing
+.shader-db:
   stage: code-validation
   extends:
     - .use-debian/x86_64_build
     - .container+build-rules
   needs:
-    - debian-build-testing
+    - .debian-build-testing
   variables:
     S3_ARTIFACT_NAME: debian-build-testing
   before_script:
     - !reference [.download_s3, before_script]
   script: |
-    section_switch shader-db "shader-db"
     .gitlab-ci/run-shader-db.sh
   artifacts:
     paths:
@@ -238,7 +240,7 @@ debian-release:
       -D gallium-nine=false
       -D gallium-rusticl=false
       -D llvm=enabled
-    GALLIUM_DRIVERS: "i915,iris,nouveau,kmsro,freedreno,r300,svga,swrast,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,d3d12,crocus"
+    GALLIUM_DRIVERS: "i915,iris,nouveau,kmsro,freedreno,r300,svga,llvmpipe,softpipe,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,d3d12,crocus"
     VULKAN_DRIVERS: "amd,imagination-experimental,microsoft-experimental"
     EXTRA_OPTION: >
       -D spirv-to-dxil=true
@@ -271,7 +273,7 @@ alpine-build-testing:
       -D glvnd=disabled
       -D platforms=wayland
     LLVM_VERSION: "16"
-    GALLIUM_DRIVERS: "crocus,etnaviv,freedreno,iris,kmsro,lima,nouveau,panfrost,r300,r600,radeonsi,svga,swrast,tegra,v3d,vc4,virgl,zink"
+    GALLIUM_DRIVERS: "crocus,etnaviv,freedreno,iris,kmsro,lima,nouveau,panfrost,r300,r600,radeonsi,svga,llvmpipe,softpipe,tegra,v3d,vc4,virgl,zink"
     GALLIUM_ST: >
       -D dri3=enabled
       -D gallium-extra-hud=true
@@ -319,7 +321,7 @@ fedora-release:
       -D intel-rt=enabled
       -D imagination-srv=true
       -D teflon=true
-    GALLIUM_DRIVERS: "crocus,etnaviv,freedreno,i915,iris,kmsro,lima,nouveau,panfrost,r300,r600,radeonsi,svga,swrast,tegra,v3d,vc4,virgl,zink"
+    GALLIUM_DRIVERS: "crocus,etnaviv,freedreno,i915,iris,kmsro,lima,nouveau,panfrost,r300,r600,radeonsi,svga,llvmpipe,softpipe,tegra,v3d,vc4,virgl,zink"
     GALLIUM_ST: >
       -D dri3=enabled
       -D gallium-extra-hud=true
@@ -431,7 +433,7 @@ debian-android:
     - debian/arm64_build
   variables:
     VULKAN_DRIVERS: freedreno,broadcom
-    GALLIUM_DRIVERS: "etnaviv,freedreno,kmsro,lima,nouveau,panfrost,swrast,tegra,v3d,vc4,zink"
+    GALLIUM_DRIVERS: "etnaviv,freedreno,kmsro,lima,nouveau,panfrost,llvmpipe,softpipe,tegra,v3d,vc4,zink"
     BUILDTYPE: "debugoptimized"
   tags:
     - aarch64
@@ -444,6 +446,8 @@ debian-arm32:
     CROSS: armhf
     DRI_LOADERS:
       -D glvnd=disabled
+    # remove llvmpipe from the .meson-arm list because here we have llvm=disabled
+    GALLIUM_DRIVERS: "etnaviv,freedreno,kmsro,lima,nouveau,panfrost,softpipe,tegra,v3d,vc4,zink"
     EXTRA_OPTION: >
       -D llvm=disabled
       -D valgrind=disabled
@@ -482,12 +486,13 @@ debian-arm64:
     DRI_LOADERS:
       -D glvnd=disabled
     EXTRA_OPTION: >
-      -D llvm=disabled
       -D valgrind=disabled
       -D imagination-srv=true
       -D perfetto=true
       -D freedreno-kmds=msm,virtio
       -D teflon=true
+    GALLIUM_ST:
+      -D gallium-rusticl=true
     S3_ARTIFACT_NAME: mesa-arm64-default-${BUILDTYPE}
   script:
     - .gitlab-ci/meson/build.sh
@@ -500,7 +505,6 @@ debian-arm64-asan:
     DRI_LOADERS:
       -D glvnd=disabled
     EXTRA_OPTION: >
-      -D llvm=disabled
       -D b_sanitize=address
       -D valgrind=disabled
       -D tools=dlclose-skip
@@ -571,7 +575,7 @@ debian-clang:
       -D shared-llvm=enabled
       -D opencl-spirv=true
       -D shared-glapi=enabled
-    GALLIUM_DRIVERS: "iris,nouveau,kmsro,r300,r600,freedreno,swrast,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,radeonsi,tegra,d3d12,crocus,i915,asahi"
+    GALLIUM_DRIVERS: "iris,nouveau,kmsro,r300,r600,freedreno,llvmpipe,softpipe,svga,v3d,vc4,virgl,etnaviv,panfrost,lima,zink,radeonsi,tegra,d3d12,crocus,i915,asahi"
     VULKAN_DRIVERS: intel,amd,freedreno,broadcom,virtio,swrast,panfrost,imagination-experimental,microsoft-experimental,nouveau
     EXTRA_OPTION:
       -D spirv-to-dxil=true
@@ -666,7 +670,7 @@ debian-x86_32:
     BUILDTYPE: debug
     CROSS: i386
     VULKAN_DRIVERS: intel,amd,swrast,virtio
-    GALLIUM_DRIVERS: "iris,nouveau,r300,r600,radeonsi,swrast,virgl,zink,crocus,d3d12"
+    GALLIUM_DRIVERS: "iris,nouveau,r300,r600,radeonsi,llvmpipe,softpipe,virgl,zink,crocus,d3d12"
     LLVM_VERSION: 15
     DRI_LOADERS:
       -D glvnd=disabled
@@ -694,7 +698,7 @@ debian-s390x:
     - kvm
   variables:
     CROSS: s390x
-    GALLIUM_DRIVERS: "swrast,zink"
+    GALLIUM_DRIVERS: "llvmpipe,softpipe,zink"
     LLVM_VERSION: 15
     VULKAN_DRIVERS: "swrast"
     DRI_LOADERS:
@@ -708,7 +712,7 @@ debian-ppc64el:
   variables:
     BUILDTYPE: debug
     CROSS: ppc64el
-    GALLIUM_DRIVERS: "nouveau,radeonsi,swrast,virgl,zink"
+    GALLIUM_DRIVERS: "nouveau,radeonsi,llvmpipe,softpipe,virgl,zink"
     VULKAN_DRIVERS: "amd,swrast"
     DRI_LOADERS:
       -D glvnd=disabled
diff --git a/.gitlab-ci/common/generate-env.sh b/.gitlab-ci/common/generate-env.sh
@@ -117,17 +117,19 @@ VARS=(
     TU_DEBUG
     USE_ANGLE
     VIRGL_HOST_API
+    VIRGL_RENDER_SERVER
     WAFFLE_PLATFORM
-    VK_CPU
     VK_DRIVER
-    # required by virglrender CI
-    VK_DRIVER_FILES
     VKD3D_PROTON_RESULTS
     VKD3D_CONFIG
     VKD3D_TEST_EXCLUDE
     ZINK_DESCRIPTORS
     ZINK_DEBUG
     LVP_POISON_MEMORY
+
+    # Dead code within Mesa CI, but required by virglrender CI
+    # (because they include our files in their CI)
+    VK_DRIVER_FILES
 )
 
 for var in "${VARS[@]}"; do
-Original file line number
+Diff line change
@@ -1,3 +1,4 @@
+    .cache
     .vscode*
     *.pyc
     *.pyo
@@ Expand Down @@