jobs: run cosa push-container-manifest privileged

In a recent 4.16 z-stream release, a cri-o backport changed the default seccomp
policy to by default block `clone(CLONE_NEW*)` syscalls:

cri-o/cri-o#8514

This affects us in the FCOS pipeline which runs in a cluster that was
recently updated. The `podman manifest` commands all AIUI also flow
through the default path where it wants to enter a namespace if running
rootless even though we don't strictly need root; we're not running
containers, just creating manifest lists. Ideally podman would be less
eager there.

Anyway, work around this as necessary by running
`cosa push-container-manifest` privileged.

There are two general places where this command is used: in container
image build jobs (e.g. `build-cosa`), and in the release job.

In the former, just use one of the multi-arch builders to do this since
we already have a session there.

In the latter, just run it in supermin. This has some warts: we want to
ideally keep uploading in parallel and while it'd be nice to parallelize
*inside* the supermin VM, the tooling doesn't make that easy. Instead,
we run multiple supermin VMs in parallel which means bumping the
resource request.

jobs/build-cosa.Jenkinsfile

@@ -134,12 +134,16 @@ lock(resource: "build-${containername}") {
                     def arch = architecture
                     images += " --image=docker://${params.CONTAINER_REGISTRY_STAGING_REPO}:${arch}-${shortcommit}"
                 }
-                shwrap("""
-                export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
-                cosa push-container-manifest --v2s2 \
-                    --auth=\$REGISTRY_SECRET --tag ${gitref} \
-                    --repo ${params.CONTAINER_REGISTRY_REPO} ${images}
-                """)
+                // arbitrarily selecting the x86_64 builder; we don't run this
+                // locally because podman wants user namespacing (yes, even just
+                // to push a manifest...)
+                pipeutils.withPodmanRemoteArchBuilder(arch: "x86_64") {
+                    shwrap("""
+                    cosa push-container-manifest --v2s2 \
+                        --auth=\$REGISTRY_SECRET --tag ${gitref} \
+                        --repo ${params.CONTAINER_REGISTRY_REPO} ${images}
+                    """)
+                }
                 // Specifically for the `main` branch let's also update the `latest` tag
                 // If there was a way to alias/tie these two together in the Quay UI
                 // that would be preferable.

jobs/build-fcos-buildroot.Jenkinsfile

@@ -146,12 +146,16 @@ lock(resource: "build-${containername}") {
                     def arch = architecture
                     images += " --image=docker://${params.CONTAINER_REGISTRY_STAGING_REPO}:${arch}-${shortcommit}"
                 }
-                shwrap("""
-                export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
-                cosa push-container-manifest \
-                    --auth=\$REGISTRY_SECRET --tag ${gitref} \
-                    --repo ${params.CONTAINER_REGISTRY_REPO} ${images}
-                """)
+                // arbitrarily selecting the x86_64 builder; we don't run this
+                // locally because podman wants user namespacing (yes, even just
+                // to push a manifest...)
+                pipeutils.withPodmanRemoteArchBuilder(arch: "x86_64") {
+                    shwrap("""
+                        cosa push-container-manifest \
+                            --auth=\$REGISTRY_SECRET --tag ${gitref} \
+                            --repo ${params.CONTAINER_REGISTRY_REPO} ${images}
+                    """)
+                }
             }
 
             stage('Delete Intermediate Tags') {

jobs/build-kola-containers.Jenkinsfile

@@ -186,12 +186,16 @@ lock(resource: "build-kola-containers") {
                         images += " --image=docker://${params.CONTAINER_REGISTRY_STAGING_REPO}:${imageName}-${arch}-${shortcommit}"
                     }
 
-                    shwrap("""
-                        export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
-                        cosa push-container-manifest --v2s2 \
-                            --auth=\$REGISTRY_SECRET --tag latest \
-                            --repo ${params.CONTAINER_REGISTRY_ORG}/${imageName} ${images}
-                    """)
+                    // arbitrarily selecting the x86_64 builder; we don't run this
+                    // locally because podman wants user namespacing (yes, even just
+                    // to push a manifest...)
+                    pipeutils.withPodmanRemoteArchBuilder(arch: "x86_64") {
+                        shwrap("""
+                            cosa push-container-manifest --v2s2 \
+                                --auth=\$REGISTRY_SECRET --tag latest \
+                                --repo ${params.CONTAINER_REGISTRY_ORG}/${imageName} ${images}
+                        """)
+                    }
                 }
             }
 

jobs/release.Jenkinsfile

@@ -88,10 +88,12 @@ def locks = basearches.collect{[resource: "release-${params.VERSION}-${it}"]}
 lock(resource: "release-${params.STREAM}", extra: locks) {
     // We should probably try to change this behavior in the coreos-ci-lib
     // So we won't need to handle the secret case here.
-    def cosaPodDefinition =  [cpu: "1", memory: "1Gi", image: cosa_img,
+    // Request 4.5Gi: in the worst case, we need to upload 4 container images in
+    // parallel via supermin and each VM is 1G.
+    def cosaPodDefinition =  [cpu: "1", memory: "4608Mi", image: cosa_img,
             serviceAccount: "jenkins"]
     if (brew_profile) {
-        cosaPodDefinition = [cpu: "1", memory: "1Gi", image: cosa_img,
+        cosaPodDefinition = [cpu: "1", memory: "4608Mi", image: cosa_img,
             serviceAccount: "jenkins",
             secrets: ["brew-keytab", "brew-ca:ca.crt:/etc/pki/ca.crt",
                       "koji-conf:koji.conf:/etc/koji.conf",
@@ -263,11 +265,14 @@ lock(resource: "release-${params.STREAM}", extra: locks) {
                         def tag_args = registry_repos[configname].tags.collect{"--tag=$it"}
                         def v2s2_arg = registry_repos.v2s2 ? "--v2s2" : ""
                         shwrap("""
-                        export STORAGE_DRIVER=vfs # https://github.com/coreos/fedora-coreos-pipeline/issues/723#issuecomment-1297668507
-                        cosa push-container-manifest --auth=\${REGISTRY_SECRET} \
+                        export COSA_SUPERMIN_MEMORY=1024 # this really shouldn't require much RAM
+                        cp \${REGISTRY_SECRET} tmp/push-secret-${metajsonname}
+                        cosa supermin-run /usr/lib/coreos-assembler/cmd-push-container-manifest \
+                            --auth=tmp/push-secret-${metajsonname} \
                             --repo=${repo} ${tag_args.join(' ')} \
                             --artifact=${artifact} --metajsonname=${metajsonname} \
                             --build=${params.VERSION} ${v2s2_arg}
+                        rm tmp/push-secret-${metajsonname}
                         """)
                     }
                 }]}