Merge pull request #70 from bedroge/remove_config_repo

New client packages (also tar now) that don't use the config repo

.github/workflows/build-test-release-client-packages.yml

@@ -4,13 +4,11 @@ on:
   push:
     branches:
       - main
-      - test_client_packages_ci
     tags:
       - 'v*'
   pull_request:
     branches:
       - main
-      - test_client_packages_ci
 
 jobs:
 
@@ -49,23 +47,32 @@ jobs:
         uses: bpicode/github-action-fpm@master
         with:
           fpm_args: "etc"
-          fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'"
+          fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t rpm -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'"
 
       - name: Build Deb package
         id: build-deb
         uses: bpicode/github-action-fpm@master
         with:
           fpm_args: "etc"
-          fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.'"
+          fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t deb -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'"
+
+      - name: Build tar package
+        id: build-tar
+        uses: bpicode/github-action-fpm@master
+        with:
+          fpm_args: "etc"
+          fpm_opts: "--debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t tar -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.'"
 
       - name: Find filenames of generated packages
         id: find_filenames
         shell: bash
         run: |
           rpmfile="$(ls -1 cvmfs-config-eessi*.rpm)"
           debfile="$(ls -1 cvmfs-config-eessi*.deb)"
+          tarfile="$(ls -1 cvmfs-config-eessi*.tar)"
           echo ::set-output name=rpmfile::${rpmfile}
           echo ::set-output name=debfile::${debfile}
+          echo ::set-output name=tarfile::${tarfile}
 
       - name: Upload Deb package as artifact
         uses: actions/upload-artifact@v2
@@ -79,6 +86,12 @@ jobs:
           name: RPM package
           path: ${{ steps.find_filenames.outputs.rpmfile }}
 
+      - name: Upload tar package as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: Tar package
+          path: ${{ steps.find_filenames.outputs.tarfile }}
+
   build-macos-package:
     runs-on: macos-latest
     steps:
@@ -113,7 +126,7 @@ jobs:
         run: gem install --no-document fpm
 
       - name: Build package
-        run: "fpm --debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t osxpkg -a all -s dir -C ./package --description 'CVMFS config repository package for EESSI.' etc"
+        run: "fpm --debug -n cvmfs-config-eessi -v ${{ steps.get_version.outputs.version }} -t osxpkg -a all -s dir -C ./package --description 'CVMFS configuration package for EESSI.' etc"
 
       - name: Find filename of generated package
         id: find_filename
@@ -157,13 +170,13 @@ jobs:
         run: sudo dpkg -i ./${{ steps.find_filename.outputs.debfile }}
 
       - name: Add local CVMFS configuration (/etc/cvmfs/default.local)
-        run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local
+        run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local
 
       - name: Mount the repositories
         run: sudo cvmfs_config setup
 
       - name: Test repository access
-        run: ls /cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs
+        run: ls /cvmfs/pilot.eessi-hpc.org/
 
   test-rpm-package:
     needs: build-linux-packages
@@ -195,14 +208,51 @@ jobs:
         run: rpm -i ./${{ steps.find_filename.outputs.rpmfile }}
 
       - name: Add local CVMFS configuration (/etc/cvmfs/default.local)
-        run: echo "CVMFS_HTTP_PROXY=DIRECT" | tee /etc/cvmfs/default.local
+        run: echo "CVMFS_CLIENT_PROFILE=single" | tee /etc/cvmfs/default.local
 
       - name: Mount the repositories
         #run: cvmfs_config setup
-        run: mkdir -p /cvmfs/cvmfs-config.eessi-hpc.org && mount -t cvmfs cvmfs-config.eessi-hpc.org /cvmfs/cvmfs-config.eessi-hpc.org
+        run: mkdir -p /cvmfs/pilot.eessi-hpc.org && mount -t cvmfs pilot.eessi-hpc.org /cvmfs/pilot.eessi-hpc.org
+
+      - name: Test repository access
+        run: ls /cvmfs/pilot.eessi-hpc.org/
+
+  test-tar-package:
+    needs: build-linux-packages
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install CVMFS dependencies
+        run: sudo apt-get install lsb-release
+
+      - name: Download and install CVMFS client
+        run: wget https://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-latest_all.deb && sudo dpkg -i cvmfs-release-latest_all.deb
+
+      - name: Install CVMFS client
+        run: sudo apt-get update && sudo apt-get install cvmfs
+
+      - name: Download cvmfs-config-eessi package
+        uses: actions/download-artifact@v2
+        with:
+          name: Tar package
+
+      - name: Find filename of package
+        id: find_filename
+        shell: bash
+        run: |
+          tarfile="$(ls -1 cvmfs-config-eessi*.tar)"
+          echo ::set-output name=tarfile::${tarfile}
+
+      - name: Install package
+        run: sudo tar -C / -xf ./${{ steps.find_filename.outputs.tarfile }}
+
+      - name: Add local CVMFS configuration (/etc/cvmfs/default.local)
+        run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local
+
+      - name: Mount the repositories
+        run: sudo cvmfs_config setup
 
       - name: Test repository access
-        run: ls /cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs
+        run: ls /cvmfs/pilot.eessi-hpc.org/
 
   test-macos-package:
     needs: build-macos-package
@@ -230,16 +280,16 @@ jobs:
         run: sudo installer -target / -pkg ./${{ steps.find_filename.outputs.pkgfile }}
 
       - name: Add local CVMFS configuration (/etc/cvmfs/default.local)
-        run: echo "CVMFS_HTTP_PROXY=DIRECT" | sudo tee /etc/cvmfs/default.local
+        run: echo "CVMFS_CLIENT_PROFILE=single" | sudo tee /etc/cvmfs/default.local
 
       - name: Mount the repositories
-        run: sudo mkdir -p /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org && sudo mount -t cvmfs cvmfs-config.eessi-hpc.org /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org
+        run: sudo mkdir -p /Users/Shared/cvmfs/pilot.eessi-hpc.org && sudo mount -t cvmfs pilot.eessi-hpc.org /Users/Shared/cvmfs/pilot.eessi-hpc.org
 
       - name: Test repository access
-        run: ls /Users/Shared/cvmfs/cvmfs-config.eessi-hpc.org/etc/cvmfs
+        run: ls /Users/Shared/cvmfs/pilot.eessi-hpc.org/
 
   release:
-    needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package]
+    needs: [build-linux-packages, build-macos-package, test-deb-package, test-rpm-package, test-macos-package, test-tar-package]
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-latest
     steps:
@@ -253,6 +303,11 @@ jobs:
         with:
           name: RPM package
 
+      - name: Download tar package
+        uses: actions/download-artifact@v2
+        with:
+          name: Tar package
+
       - name: Download macOS package
         uses: actions/download-artifact@v2
         with:
@@ -265,9 +320,11 @@ jobs:
           rpmfile="$(ls -1 cvmfs-config-eessi*.rpm)"
           debfile="$(ls -1 cvmfs-config-eessi*.deb)"
           pkgfile="$(ls -1 cvmfs-config-eessi*.pkg)"
+          tarfile="$(ls -1 cvmfs-config-eessi*.tar)"
           echo ::set-output name=rpmfile::${rpmfile}
           echo ::set-output name=debfile::${debfile}
           echo ::set-output name=pkgfile::${pkgfile}
+          echo ::set-output name=tarfile::${tarfile}
 
       - name: Create Release
         id: create_release
@@ -309,3 +366,13 @@ jobs:
           asset_path: ${{ steps.find_filenames.outputs.pkgfile }}
           asset_name: ${{ steps.find_filenames.outputs.pkgfile }}
           asset_content_type: application/x-newton-compatible-pkg
+
+      - name: Upload Tar as release asset
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ${{ steps.find_filenames.outputs.tarfile }}
+          asset_name: ${{ steps.find_filenames.outputs.tarfile }}
+          asset_content_type: application/x-tar

.github/workflows/test-playbook.sh

@@ -15,10 +15,12 @@ echo '  - 127.0.0.1' >> inventory/local_site_specific_vars.yml
 # Don't use the GEO API for the Stratum 1, since we do not have a key here.
 export CVMFS_GEO_DB_FILE=NONE
 
-# Only test the cvmfs-config repo on the Stratum 1, as the other ones may be very large.
+# Only test CI suitable repos on the Stratum 1, as the other ones may be very large.
 if [ $playbook == "stratum1.yml" ]
 then
-  echo 'cvmfs_repositories: "[{{ eessi_cvmfs_config_repo.repository }}]"' >> inventory/local_site_specific_vars.yml
+	cat >> inventory/local_site_specific_vars.yml <<-EOF
+		cvmfs_repositories: "{{ eessi_cvmfs_repositories | selectattr('use_for_ci', 'defined') | selectattr('use_for_ci', 'sameas', true) | list }}"
+	EOF
 fi
 
 # Install the Ansible dependencies.

.github/workflows/test-playbooks.yml

@@ -34,8 +34,5 @@ jobs:
         run: docker run -d --workdir /github/workspace --rm -e INPUT_PLAYBOOK -e HOME -e GITHUB_JOB -e GITHUB_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_REPOSITORY_OWNER -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RETENTION_DAYS -e GITHUB_ACTOR -e GITHUB_WORKFLOW -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GITHUB_EVENT_NAME -e GITHUB_SERVER_URL -e GITHUB_API_URL -e GITHUB_GRAPHQL_URL -e GITHUB_WORKSPACE -e GITHUB_ACTION -e GITHUB_EVENT_PATH -e GITHUB_PATH -e GITHUB_ENV -e RUNNER_OS -e RUNNER_TOOL_CACHE -e RUNNER_TEMP -e RUNNER_WORKSPACE -e ACTIONS_RUNTIME_URL -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v $HOME:"/github/home" -v "$HOME/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/_temp/_runner_file_commands":"/github/file_commands" -v "$GITHUB_WORKSPACE":"/github/workspace" -v /sys/fs/cgroup:/sys/fs/cgroup --privileged --device /dev/fuse --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/srv --mount type=bind,source=${GITHUB_WORKSPACE}/srv,target=/var/spool/cvmfs --name ${{ matrix.component }}-${{ matrix.os }} docker.pkg.github.com/$(echo $GITHUB_REPOSITORY | tr '[A-Z]' '[a-z]')/${{ matrix.os }}
       - name: Execute the playbook
         run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}.yml
-      - name: Execute additional playbook for Stratum 0
-        run: docker exec ${{ matrix.component }}-${{ matrix.os }} /test-playbook.sh ${{ matrix.component }}-deploy-cvmfs-config.yml
-        if: ${{ matrix.component == 'stratum0' }}
       - name: Stop the container
         run: docker kill ${{ matrix.component }}-${{ matrix.os }}

README.md

@@ -166,9 +166,13 @@ Next, you need to make a file `/etc/cvmfs/default.local` manually; this file is
 contains, for instance, the URL to your local proxy and the size of the local cache. As an example, you can put
 the following in this file, which corresponds to not using a proxy and setting the local quota limit to 40000MB:
 ```
-CVMFS_HTTP_PROXY=DIRECT
+CVMFS_CLIENT_PROFILE=single
 CVMFS_QUOTA_LIMIT=40000
 ```
+If you do want to use your own proxy, replace the first line by:
+```
+CVMFS_HTTP_PROXY=<hostname of your proxy>:<port>
+```
 For more details about configuring your client, see https://cvmfs.readthedocs.io/en/stable/cpt-configure.html.
 
 Finally, run `cvmfs_config setup` to set up CVMFS.

inventory/group_vars/all.yml

@@ -2,16 +2,23 @@
 # vars file for eessi project
 
 # Automatically configure EESSI CVMFS repos.
-eessi_cvmfs_repos_enabled: config-repo
+eessi_cvmfs_repos_enabled: true
 
-# Email address for the project, which will be put in the contact file on the config repo.
+# Email address for the project.
 eessi_email: [email protected]
 
-# Defaults for eessi-hpc.org config repo, syntax for each key is the same as that of cvmfs_<key | pluralize>
-eessi_cvmfs_config_repo:
-  domain: eessi-hpc.org
-  key:
-    path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub
+#
+# Defaults for eessi-hpc.org repos.
+#
+
+# We do not use a config repo anymore.
+eessi_cvmfs_config_repo: {}
+
+# Public keys for the repositories, which you can find on the Stratum 0 at:
+# /etc/cvmfs/keys/*.pub
+# Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys.
+eessi_cvmfs_keys:
+  - path: /etc/cvmfs/keys/eessi-hpc.org/cvmfs-config.eessi-hpc.org.pub
     key: |
      -----BEGIN PUBLIC KEY-----
      MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqoTBc/Eksr9ECEOoCzHA
@@ -22,25 +29,6 @@ eessi_cvmfs_config_repo:
      +hCt494UsFs3tchmTebpK+x3pVRXybQ66Qm1mQiaz+2VbUJdQEov4RCC5PcOmOOk
      BwIDAQAB
      -----END PUBLIC KEY-----
-  urls:
-    - "http://cvmfs-s1-rug.eessi-hpc.org/cvmfs/@fqrn@"
-    - "http://cvmfs-s1-bgo.eessi-hpc.org/cvmfs/@fqrn@"
-  repository:
-    repository: cvmfs-config.eessi-hpc.org
-    stratum0: cvmfs-s0.eessi-hpc.org
-    owner: "{{ cvmfs_repo_owner | default('root') }}"
-    key_dir: /etc/cvmfs/keys/eessi-hpc.org
-    server_options: []
-    client_options: []
-
-#
-# Defaults for eessi-hpc.org repos.
-#
-
-# Public keys for the repositories, which you can find on the Stratum 0 at:
-# /etc/cvmfs/keys/*.pub
-# Note: you first have to run the stratum0.yml playbook once to generate the repositories and keys.
-eessi_cvmfs_keys:
   - path: /etc/cvmfs/keys/eessi-hpc.org/pilot.eessi-hpc.org.pub
     key: |
      -----BEGIN PUBLIC KEY-----
@@ -52,6 +40,19 @@ eessi_cvmfs_keys:
      HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby
      FQIDAQAB
      -----END PUBLIC KEY-----
+  # The following (duplicate) key can be removed after this PR has been merged:
+  # https://github.com/galaxyproject/ansible-cvmfs/pull/32
+  - path: /etc/cvmfs/keys/eessi-hpc.org/ci.eessi-hpc.org.pub
+    key: |
+     -----BEGIN PUBLIC KEY-----
+     MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAommtaoqQ36Rb/cFnqUWg
+     8FoYCOjJWVBMQ4sVZ87oFRWehOUbcfy/ah85D9055HQws7i/hLNGJJAa7Es3E2o8
+     v8LtSlRdfR1A7b7LA6p3213HHz6dNU6I1riwtE2OP7/w4Q3+Agj1iRtW535GwzGo
+     4ZbewL8T1kbyaEm4HyefoitHXJthoPyTlYAhvAegIil1VJPhJZW+q/NWxnsd9QGJ
+     uAtbSICQg6mBJI50KiCBYmKHOAjH2W8qEnLWc7JoPoNvAHuMx1ya6YerBM5T9Esm
+     HJN0HHOVdRIwNjjPkV2jIqhJoI85no89v8/V45SFPVo+A7N0Cj4QdQrQTezpnlby
+     FQIDAQAB
+     -----END PUBLIC KEY-----
 
 # URLs for all the Stratum 1 servers.
 eessi_cvmfs_server_urls:
@@ -71,8 +72,17 @@ eessi_cvmfs_repositories:
       - CVMFS_AUTO_GC=false
       - CVMFS_AUTO_TAG_TIMESPAN="30 days ago"
       - CVMFS_GARBAGE_COLLECTION=true
-    client_options:
-      - CVMFS_NFILES=4096
+    client_options: []
+    use_for_ci: no
+  - repository: ci.eessi-hpc.org
+    stratum0: cvmfs-s0.eessi-hpc.org
+    owner: "{{ cvmfs_repo_owner | default('root') }}"
+    key_dir: /etc/cvmfs/keys/eessi-hpc.org
+    server_options:
+      - CVMFS_AUTO_GC=false
+      - CVMFS_GARBAGE_COLLECTION=true
+    client_options: []
+    use_for_ci: yes
 
 # Override all the Galaxy defaults by our EESSI defaults.
 # This is required, beucase the galaxy_* variables are used inside the Ansible tasks.

inventory/group_vars/cvmfsclients

@@ -3,6 +3,3 @@
 # List of proxies to be used for the clients.
 # Override this setting in your local_site_specific_vars.yml file.
 cvmfs_http_proxies: "{{ local_cvmfs_http_proxies | default(['DIRECT']) }}"
-
-# Use the CVMFS configuration repository for the clients.
-eessi_cvmfs_repos_enabled: config-repo

inventory/group_vars/cvmfsstratum0servers

@@ -1,3 +1,2 @@
 ---
 
-cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}"

inventory/group_vars/cvmfsstratum1servers

@@ -1,5 +1,3 @@
 ---
 
-cvmfs_repositories: "{{ eessi_cvmfs_repositories + [eessi_cvmfs_config_repo.repository] }}"
-cvmfs_keys: "{{ eessi_cvmfs_keys + [eessi_cvmfs_config_repo.key] }}"
 cvmfs_squid_conf_src: "{{ local_stratum1_cvmfs_squid_conf_src | default('eessi_stratum1_squid.conf.j2') }}"