S3 backend configuration support

README.md

@@ -94,7 +94,8 @@ cvmfs_repositories:
 variable | type | description
 --- | --- | ---
 `cvmfs_private_keys` | list of dicts | Keys to install on Stratum 0 hosts. Separate from `cvmfs_keys` for vaultability and avoiding duplication.
-`cvmfs_config_apache` | boolean | Configure Apache on Stratum 0 and 1 servers. If disabled, you must configure it yourself. Default is `true`.
+`cvmfs_config_apache` | boolean | Configure Apache on Stratum 0 and 1 servers. It can be disabled if using an S3 backend, or you want to configure it yourself. Default is `true`.
+`cvmfs_config_squid` | boolean | Configure Squid on Stratum 0 and 1 servers. It can be disabled if using an S3 backend, or you want to configure it yourself. Default is `true`.
 `cvmfs_manage_firewall` | boolean | Attempt to configure firewalld (EL) or ufw (Debian) to permit traffic to configured ports. Default is `false`.
 `cvmfs_squid_conf_src` | path | Path to template Squid configuration file (for Stratum 1 and local proxy servers). Defaults are in the role `templates/` directory.
 `cvmfs_stratum0_http_ports` | list of integers | Port(s) to configure Apache on Stratum 0 servers to listen on. Default is `80`.
@@ -109,10 +110,35 @@ variable | type | description
 `cvmfs_srv_mount` | path | Path to mount CVMFS data volume on. Default is `/srv` (but is ignored if `cvmfs_srv_device` is unset).
 `cvmfs_union_fs` | string | Union filesystem type (`overlayfs` or `aufs`) for new repositories on Stratum 0 servers.
 `cvmfs_numfiles` | integer | Set the maximum number of open files in `/etc/security/limits.conf`. Useful with the `CVMFS_NFILES` client option on Stratum 0 servers.
+`cvmfs_backend` | string | The storage backend to use for CVMFS (`filesystem` or `s3`). If using `s3`, additional options are available to configure settings.
+
+### S3 backend configuration variables
+
+If using the S3 `cvmfs_backend`, the following additional options can be set. Refer to [cvmfs docs][s3_config] on s3 config for more details.
+
+variable | type | description
+--- | --- | ---
+`cvmfs_s3_access_key` | string | The access key to use when connecting to S3 storage.
+`cvmfs_s3_secret_key` | string | The secret key to use when connecting to S3 storage.
+`cvmfs_s3_host` | string | S3 server hostname, e.g. s3.amazonaws.com. The hostname should NOT be prefixed by “http://”.
+`cvmfs_s3_flavor` | string | Set to “azure” if you store files in Microsoft Azure Blob Storage.
+`cvmfs_s3_region` | string | The S3 region, e.g. eu-central-1. If specified, AWSv4 authorization protocol is used.
+`cvmfs_s3_port` | integer | The port on which the S3 instance is running.
+`cvmfs_s3_bucket` | string | S3 bucket name. The repository name is used as a subdirectory inside the bucket.
+`cvmfs_s3_bucket_url` | string | Public facing URL of the bucket, if using OpenStack or similar. (e.g. https://object-store.rc.nectar.org.au/v1/AUTH_377/cvmfs)
+`cvmfs_s3_use_https` | boolean in MB | Allow to use S3 implementation over HTTPS and not over HTTP.
+`cvmfs_s3_timeout` | integer | Timeout in seconds for the connection to the S3 server.
+`cvmfs_s3_max_retries` | integer | Number of retries for the connection to the S3 server.
+`cvmfs_s3_max_number_of_parallel_connections` | boolean | Number of parallel uploads to the S3 server, e.g. 400.
+`cvmfs_s3_dns_buckets` | boolean | Set to false to disable DNS-style bucket URLs (http://<bucket>.<host>/<object>). Enabled by default.
+`cvmfs_s3_peek_before_put` | boolean | Make PUT requests conditional to a prior HEAD request. Enabled by default.
+`cvmfs_s3_conf_path` | path | Path to store the CVMFS s3 config file. Default is `/etc/cvmfs/s3.conf`.
+
 
 [defaults]: https://github.com/galaxyproject/ansible-cvmfs/blob/master/defaults/main.yml
 [cvmfs-config-repo]: https://cvmfs.readthedocs.io/en/stable/cpt-configure.html#the-config-repository
 [preload]: http://cvmfs.readthedocs.io/en/stable/cpt-hpc.html
+[s3_config]: https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#s3-compatible-storage-systems
 
 Dependencies
 ------------
@@ -203,6 +229,41 @@ defined as:
           MIIBIjAN...
 ```
 
+The following playbook shows an example of how to configure a stratum0/1 server with a custom S3 backend.
+
+```yaml
+- name: CVMFS
+  vars:
+    cvmfs_role: stratum1
+    cvmfs_backend: s3
+    cvmfs_config_apache: false
+    cvmfs_config_squid: false
+
+    cvmfs_s3_access_key: <your_access_key>
+    cvmfs_s3_secret_key: <your_secret_key>
+    cvmfs_s3_host: swift.rc.nectar.org.au
+    cvmfs_s3_port: 8888
+    cvmfs_s3_bucket: cvmfs
+    cvmfs_s3_bucket_url: https://object-store.rc.nectar.org.au/v1/AUTH_377/cvmfs
+    cvmfs_s3_use_https: true
+    cvmfs_s3_dns_buckets: false
+
+    galaxy_cvmfs_repos_enabled: true
+    # override the default
+    galaxy_cvmfs_server_urls:
+      - domain: galaxyproject.org
+        use_ssl_system_ca: true # Set this on both clients and servers to use system CA if accessing S3 byckets over https
+        urls:
+          - "http://cvmfs1-mel0.gvl.org.au/cvmfs/@fqrn@"
+          - "http://cvmfs1-ufr0.galaxyproject.eu/cvmfs/@fqrn@"
+          - "http://cvmfs1-iu0.galaxyproject.org/cvmfs/@fqrn@"
+          - "http://cvmfs1-tacc0.galaxyproject.org/cvmfs/@fqrn@"
+          - "http://cvmfs1-psu0.galaxyproject.org/cvmfs/@fqrn@"
+  roles:
+    - galaxyproject.cvmfs
+  become: yes
+```
+
 License
 -------
 

defaults/main.yml

@@ -35,6 +35,28 @@ cvmfs_srv_mount: /srv
 
 cvmfs_union_fs: overlayfs
 cvmfs_config_apache: true
+cvmfs_config_squid: true
+
+# Backend can be filesystem or s3
+cvmfs_backend: filesystem
+# if s3, the following additional settings can be specified
+# docs: https://cvmfs.readthedocs.io/en/2.9/cpt-repo.html#s3-compatible-storage-systems
+cvmfs_s3_access_key:
+cvmfs_s3_secret_key:
+cvmfs_s3_host:
+cvmfs_s3_flavor: # optional: valid options are azure, awsv2 or awsv4
+cvmfs_s3_region:
+cvmfs_s3_port:
+cvmfs_s3_bucket:
+cvmfs_s3_bucket_url:
+cvmfs_s3_use_https:
+cvmfs_s3_timeout:
+cvmfs_s3_max_retries:
+cvmfs_s3_max_number_of_parallel_connections:
+cvmfs_s3_dns_buckets: # optional: use dns style buckets - true or false
+cvmfs_s3_peek_before_put:
+# the s3 settings above are stored in the following conf file for use by cvmfs
+cvmfs_s3_conf_path: /etc/cvmfs/s3.conf
 
 # Setup an optional cache directory for squid. Otherwise in-memory cache is used.
 # cvmfs_stratum1_cache_dir:
@@ -217,6 +239,7 @@ galaxy_cvmfs_keys:
 galaxy_cvmfs_server_urls:
   - domain: galaxyproject.org
     use_geoapi: false
+    use_ssl_system_ca: true
     urls:
       - "http://cvmfs1-psu0.galaxyproject.org/cvmfs/@fqrn@"
       - "http://cvmfs1-iu0.galaxyproject.org/cvmfs/@fqrn@"

tasks/client.yml

@@ -68,6 +68,7 @@
       CVMFS_SERVER_URL="{{ item.urls | join(';') }}"
       CVMFS_KEYS_DIR=/etc/cvmfs/keys/{{ item.domain }}
       CVMFS_USE_GEOAPI="{{ item.use_geoapi | default(false) | ternary('yes', 'no') }}"
+      CVMFS_USE_SSL_SYSTEM_CA="{{ item.use_ssl_system_ca | default(true) | ternary('yes', 'no') }}"
     dest: /etc/cvmfs/domain.d/{{ item.domain }}.conf
     owner: root
     group: root

tasks/localproxy.yml

@@ -10,6 +10,7 @@
   include_tasks: squid.yml
   vars:
     _cvmfs_squid_conf_src: "{{ cvmfs_squid_conf_src | default('localproxy_squid.conf.j2') }}"
+  when: cvmfs_config_squid
 
 # Need to double check that this actually works (see the hosts_file directive)
 #- name: Create squid hosts file

tasks/s3.yml

@@ -0,0 +1,6 @@
+---
+
+- name: Copy s3 config
+  template:
+    src: "cvmfs_s3.conf.j2"
+    dest: "{{ cvmfs_s3_conf_path }}"

tasks/stratum0.yml

@@ -33,6 +33,11 @@
 
 - name: Include Apache tasks
   include_tasks: apache.yml
+  when: cvmfs_backend == "filesystem" and cvmfs_config_apache
+
+- name: Include S3 tasks
+  include_tasks: s3.yml
+  when: cvmfs_backend in ["s3", "S3"]
 
 - name: Include firewall tasks
   include_tasks: firewall.yml
@@ -41,7 +46,9 @@
   when: cvmfs_manage_firewall
 
 - name: Create repositories
-  command: /usr/bin/cvmfs_server mkfs {{ cvmfs_config_apache_flag }} -o {{ item.owner | default('root') }} -f {{ cvmfs_union_fs }} {{ item.repository }}
+  command: >-
+    /usr/bin/cvmfs_server mkfs {{ cvmfs_config_apache_flag }} -o {{ item.owner | default('root') }}
+      -f {{ cvmfs_union_fs }} {{ item.repository }} -s {{ cvmfs_s3_conf_path }} -w {{ cvmfs_s3_bucket_url }}
   args:
     creates: /srv/cvmfs/{{ item.repository }}
   with_items: "{{ cvmfs_repositories }}"

tasks/stratum1.yml

@@ -24,11 +24,17 @@
 
 - name: Include Apache tasks
   include_tasks: apache.yml
+  when: cvmfs_backend == "filesystem" and cvmfs_config_apache
+
+- name: Include S3 tasks
+  include_tasks: s3.yml
+  when: cvmfs_backend in ["s3", "S3"]
 
 - name: Include squid tasks
   include_tasks: squid.yml
   vars:
     _cvmfs_squid_conf_src: "{{ cvmfs_squid_conf_src | default('stratum1_squid.conf.j2') }}"
+  when: cvmfs_config_squid
 
 - name: Include firewall tasks
   include_tasks: firewall.yml
@@ -47,6 +53,7 @@
 - name: Ensure replicas are configured
   command: >-
     /usr/bin/cvmfs_server add-replica -o {{ item.owner | default('root') }}
+      -s {{ cvmfs_s3_conf_path }} -w {{ cvmfs_s3_bucket_url }}
       http://{{ item.stratum0 }}/cvmfs/{{ item.repository }}
       {{ item.key_dir | default('/etc/cvmfs/keys') }}/{{ item.repository }}.pub
   args:

templates/cvmfs_s3.conf.j2

@@ -0,0 +1,29 @@
+CVMFS_S3_ACCESS_KEY={{ cvmfs_s3_access_key }}
+CVMFS_S3_SECRET_KEY={{ cvmfs_s3_secret_key }}
+CVMFS_S3_HOST={{ cvmfs_s3_host }}
+{% if cvmfs_s3_flavor is defined and cvmfs_s3_flavor %}
+CVMFS_S3_FLAVOR={{ cvmfs_s3_flavor }}
+{% endif %}
+{% if cvmfs_s3_region is defined and cvmfs_s3_region %}
+CVMFS_S3_REGION={{ cvmfs_s3_region }}
+{% endif %}
+CVMFS_S3_PORT={{ cvmfs_s3_port }}
+CVMFS_S3_BUCKET={{ cvmfs_s3_bucket }}
+{% if cvmfs_s3_use_https is defined %}
+CVMFS_S3_USE_HTTPS={{ "true" if (cvmfs_s3_use_https | bool) else "false" }}
+{% endif %}
+{% if cvmfs_s3_timeout is defined and cvmfs_s3_timeout %}
+CVMFS_S3_TIMEOUT={{ cvmfs_s3_timeout }}
+{% endif %}
+{% if cvmfs_s3_max_retries is defined and cvmfs_s3_max_retries %}
+CVMFS_S3_MAX_RETRIES={{ cvmfs_s3_max_retries }}
+{% endif %}
+{% if cvmfs_s3_max_number_of_parallel_connections is defined and cvmfs_s3_max_number_of_parallel_connections %}
+CVMFS_S3_MAX_NUMBER_OF_PARALLEL_CONNECTIONS={{ cvmfs_s3_max_number_of_parallel_connections }}
+{% endif %}
+{% if cvmfs_s3_dns_buckets is defined %}
+CVMFS_S3_DNS_BUCKETS={{ "true" if (cvmfs_s3_dns_buckets | bool) else "false" }}
+{% endif %}
+{% if cvmfs_s3_peek_before_put is defined and cvmfs_s3_peek_before_put %}
+CVMFS_S3_PEEK_BEFORE_PUT={{ cvmfs_s3_peek_before_put }}
+{% endif %}

vars/debian.yml

@@ -9,14 +9,8 @@ cvmfs_squid_user: proxy
 cvmfs_squid_group: proxy
 
 cvmfs_packages:
-  stratum0:
-    - apache2
-    - cvmfs-server
-    - cvmfs-config-default
-  stratum1:
-    - apache2
-    - cvmfs-server
-    - cvmfs-config-default
+  stratum0: "{{ (['cvmfs-server', 'cvmfs-config-default'] + ['apache2'] if cvmfs_config_apache else []) | select() | list }}"
+  stratum1: "{{ (['cvmfs-server', 'cvmfs-config-default'] + ['apache2'] if cvmfs_config_apache else [] + ['squid'] if cvmfs_config_squid else []) | select() | list }}"
   localproxy:
     - squid
   client:

vars/redhat.yml

@@ -9,17 +9,8 @@ cvmfs_squid_user: squid
 cvmfs_squid_group: squid
 
 cvmfs_packages:
-  stratum0:
-    - httpd
-    - cvmfs-server
-    - cvmfs-config-default
-    - cvmfs
-  stratum1:
-    - httpd
-    - mod_wsgi
-    - squid
-    - cvmfs-server
-    - cvmfs-config-default
+  stratum0: "{{ (['cvmfs-server', 'cvmfs-config-default', 'cvmfs'] + ['httpd'] if cvmfs_config_apache else []) | select() | list }}"
+  stratum1: "{{ (['cvmfs-server', 'cvmfs-config-default'] + ['httpd', 'mod_wsgi'] if cvmfs_config_apache else [] + ['squid'] if cvmfs_config_squid else []) | select() | list }}"
   localproxy:
     - squid
   client: