SwissDataScienceCenter · olevski · Oct 25, 2023 · Jul 17, 2023 · Jul 17, 2023 · Aug 7, 2023
@@ -56,6 +56,7 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
@@ -94,6 +95,7 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}"
+          renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
       - name: Check existing renkubot comment
         if: needs.check-deploy.outputs.pr-contains-string == 'true'

@@ -5,21 +5,64 @@
 ------
 
 Renku ``0.41.0`` allows the RenkuLab homepage to be configured to highlight chosen projects.
+In addition we are rolling out a much more comprehensive support for saving and using S3 
+cloud storage.
 
-Internal Changes
+A note to Renku administrators, this release includes breaking changes in our Helm chart values file.
+Refer to the ``Internal Changes`` section below for more details.
+
+User-Facing Changes
 ~~~~~~~~~~~~~~~~~~~
 
 **🌟 New Features**
 
-- 🚀 **UI**: [Showcase] Admins can designate projects to be showcased on the home page, which will show them
-    in the showcase section of the home page. (`#2799 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2799>`_).
+- 🖼 **UI**: Admins can designate projects to be showcased on the home page, which will show them
+  in the showcase section of the home page
+  (`#2799 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2799>`_).
+
+**✨ Improvements**
+
+- 💾 **UI**: Add support for cloud storage configuration per project. There are now more options
+  to customize to support external S3 and S3 compatible storage better
+  (`#2760 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2760>`_).
+- 🌈 **UI**: Improve color contrast and other UX elements
+  (`#2846 <https://github.com/SwissDataScienceCenter/renku-ui/pull/2846>`_).
 
 
-This is a breaking change to the values file and it requires minor edits to the following field:
+Internal Changes
+~~~~~~~~~~~~~~~~~~~
+
+This release is a breaking change to the Helm values file and it requires minor edits to the following field:
 
 - ``ui.homepage`` removed the unused ``projects`` field and added the ``showcase`` field.
+- ``amalthea.scheduler.*`` deprecates all existing child fields and adds new child fields. If you are not defining these fields 
+  in your values file then you are using the default Kubernetes scheduler and this requires no action. But if you are 
+  defining a custom scheduler in your deployment's values file then this requires additional edits to your values file 
+  so that you can retain the same functionality as before.
+- the ``crc`` field in the values file has been renamed to ``dataService``, all child fields remain the same 
+  functionally and by name.
 
-For more details please refer to the explanation in ``helm-chart/values.yaml.changelog.md``.
+For more details on the Helm chart values changes please refer to the explanation in ``helm-chart/values.yaml.changelog.md``.
+
+In addition to this, other notable changes include:
+
+- add node affinities and tolerations for resource classes
+- persist cloud storage configurations at the project level
+- validation of Rclone cloud storage configuration by the backend
+- update the Amalthea scheduler to work with newer versions of Kubernetes
+- ``renku-notebooks`` now get S3 cloud storage configuration from ``renku-data-services``
+- ``renku-gateway`` now provides credentials for the cloud storage potion of ``renku-data-services``
+- UI shows prominent banners during major outages
+- various bug fixes across many components
+
+Individual components
+~~~~~~~~~~~~~~~~~~~~~~
+
+- `renku-data-services 0.1.1 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.1.1>`_
+- `renku-gateway 0.22.0 <https://github.com/SwissDataScienceCenter/renku-gateway/releases/tag/0.22.0>`_
+- `renku-notebooks 1.20.0 <https://github.com/SwissDataScienceCenter/renku-notebooks/releases/tag/1.20.0>`_
+- `renku-ui 3.14.0 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.14.0>`_
+- `amalthea 0.10.0 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.10.0>`_
 
 0.40.1
 ------

@@ -136,9 +136,7 @@ describe("Basic public project functionality", () => {
 
     // Save the changes
     cy.getDataCy("save-session-button").should("be.visible").click();
-    cy.get(".modal")
-      .contains("1 commit will be pushed")
-      .should("be.visible");
+    cy.get(".modal").contains("1 commit will be pushed").should("be.visible");
     cy.getDataCy("save-session-modal-button").should("be.visible").click();
     cy.get(".modal")
       .contains("Saving Session", { timeout: TIMEOUTS.long })
@@ -224,4 +222,110 @@ describe("Basic public project functionality", () => {
     cy.pauseSession();
     cy.deleteSession();
   });
+
+  it("Start a new session with cloud storage attached.", () => {
+    cy.stopAllSessionsForProject(projectIdentifier);
+
+    cy.intercept("/ui-server/api/data/storage*").as("getProjectCloudStorage");
+
+    cy.getProjectSection("Settings").click();
+    cy.getDataCy("settings-navbar")
+      .contains("a.nav-link", "Cloud Storage")
+      .should("be.visible")
+      .click();
+
+    // Add a S3 storage configuration if it doesn't exist
+    cy.wait("@getProjectCloudStorage").then(({ response }) => {
+      const storages = response.body as { storage: { name: string } }[];
+      if (storages.find(({ storage }) => storage.name === "data_s3")) {
+        return;
+      }
+
+      cy.getDataCy("settings-container")
+        .find("button")
+        .contains("Add Cloud Storage")
+        .should("be.visible")
+        .click();
+
+      cy.get(".modal").contains("Add Cloud Storage").should("be.visible");
+
+      cy.get("label").contains("Name").click();
+      cy.get(":focused").type("data_s3");
+
+      cy.get(".modal")
+        .contains("For AWS S3 buckets, supported URLs are of the form")
+        .should("be.visible");
+      cy.get("label").contains("Endpoint URL").click();
+      cy.get(":focused").type("s3://giab");
+
+      cy.get("label")
+        .contains("Requires credentials")
+        .siblings("input")
+        .click()
+        .should("not.be.checked");
+
+      // NOTE: Temporarily removed this from the tests until we enable read-write
+      // cy.get("label")
+      //   .contains("Read-only")
+      //   .siblings("input")
+      //   .should("be.checked");
+
+      cy.get("button[type='submit']")
+        .contains("Add Storage")
+        .should("be.visible")
+        .click();
+    });
+
+    cy.getDataCy("more-menu").should("be.visible").click();
+    cy.getProjectPageLink(projectIdentifier, "sessions/new")
+      .should("be.visible")
+      .first()
+      .click();
+
+    // Wait for the image to be ready and start a session
+    cy.get(".renku-container")
+      .contains("A session gives you an environment")
+      .should("exist");
+    cy.get(".renku-container .badge.bg-success", { timeout: TIMEOUTS.vlong })
+      .contains("available")
+      .should("exist");
+    cy.get(".renku-container button.btn-secondary", { timeout: TIMEOUTS.long })
+      .contains("Start Session")
+      .should("exist")
+      .click();
+    cy.get(".progress-box .progress-title").should("exist"); //.contains("Step 2 of 2");
+    cy.get("button")
+      .contains(projectTestConfig.projectName)
+      .should("be.visible");
+    cy.get(".progress-box .progress-title")
+      .contains("Starting Session")
+      .should("exist");
+    cy.get(".progress-box .progress-title", { timeout: TIMEOUTS.vlong }).should(
+      "not.exist"
+    );
+
+    // Verify that the S3 data is mounted
+    cy.getIframe("iframe#session-iframe").within(() => {
+      cy.get(".jp-DirListing-content", { timeout: TIMEOUTS.long }).should(
+        "be.visible"
+      );
+      cy.get(".jp-DirListing-item")
+        .contains("data_s3")
+        .should("be.visible")
+        .dblclick();
+
+      cy.get(".jp-DirListing-item")
+        .contains("README.s3_structure")
+        .should("be.visible")
+        .dblclick();
+
+      cy.get(".jp-FileEditor", { timeout: TIMEOUTS.long }).should("be.visible");
+      cy.get(".jp-FileEditor")
+        .contains("The GIAB s3 bucket and URLs")
+        .should("be.visible");
+    });
+
+    cy.pauseSession();
+    cy.deleteSession();
+  });
 });
@@ -39,7 +39,7 @@ describe("Verify the infrastructure is ready", () => {
     retryRequest("api/renku/versions", "Core basic");
     retryRequest("api/notebooks/version", "Notebooks");
     retryRequest("api/kg/entities", "Graph");
-    retryRequest("api/data/version", "CRC");
+    retryRequest("api/data/version", "Data Service");
     retryRequest("api/auth/login", "Gateway");
     retryRequest(
       "ui-server/api/allows-iframe/https%3A%2F%2Fgoogle.com",
@@ -80,9 +80,9 @@ describe("Verify the infrastructure is ready", () => {
         throw new Error("Graph backend not working as expected.");
     });
 
-    // CRC should return a list of default resopurce pools
-    const crcUrl = "/ui-server/api/data/resource_pools";
-    cy.request(crcUrl).then((resp) => {
+    // Data service should return a list of default resopurce pools
+    const dataServiceUrl = "/ui-server/api/data/resource_pools";
+    cy.request(dataServiceUrl).then((resp) => {
       if (resp.status >= 400 || !resp.body.length)
         throw new Error("GitLab not working as expected.");
     });

@@ -60,7 +60,7 @@ function stopAllSessionsForProject(
       cy.getDataCy("delete-session-modal-button").should("be.visible").click();
     }
   });
-  cy.contains("No currently running sessions.", { timeout: TIMEOUTS.long });
+  cy.contains("No currently running sessions.", { timeout: TIMEOUTS.vlong });
 }
 
 function deleteSession(fromSessionPage = false) {
@@ -69,7 +69,7 @@ function deleteSession(fromSessionPage = false) {
   cy.getDataCy("delete-session-button").first().should("be.visible").click();
   cy.getDataCy("delete-session-modal-button").should("be.visible").click();
   cy.getDataCy("stopping-btn").should("be.visible");
-  cy.get(".renku-container", { timeout: TIMEOUTS.long })
+  cy.get(".renku-container", { timeout: TIMEOUTS.vlong })
     .contains("No currently running sessions")
     .should("be.visible");
 }

@@ -23,8 +23,8 @@ dependencies:
   alias: jena
 - name: amalthea
   repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-  version: "0.9.1"
+  version: "0.10.0"
 - name: dlf-chart
   repository: "https://swissdatasciencecenter.github.io/datashim/"
-  version: "0.3.0-renku-1"
+  version: "0.3.9-renku-2"
   condition: notebooks.cloudstorage.s3.installDatashim
@@ -1,28 +1,28 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ template "renku.fullname" . }}-crc
+  name: {{ template "renku.fullname" . }}-data-service
   labels:
-    app: renku-crc
+    app: renku-data-service
     chart: {{ template "renku.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  {{- if not .Values.crc.autoscaling.enabled }}
-  replicas: {{ .Values.crc.replicaCount }}
+  {{- if not .Values.dataService.autoscaling.enabled }}
+  replicas: {{ .Values.dataService.replicaCount }}
   {{- end }}
   strategy: 
-    {{- toYaml .Values.crc.updateStrategy | nindent 4 }}
+    {{- toYaml .Values.dataService.updateStrategy | nindent 4 }}
   selector:
     matchLabels:
-      app: renku-crc
+      app: renku-data-service
       release: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app: renku-crc
+        app: renku-data-service
         release: {{ .Release.Name }}
-      {{- with .Values.crc.podAnnotations }}
+      {{- with .Values.dataService.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
@@ -33,9 +33,9 @@ spec:
       initContainers:
         {{- include "certificates.initContainer" . | nindent 8 }}
       containers:
-        - name: crc
-          image: "{{ .Values.crc.image.repository }}:{{ .Values.crc.image.tag }}"
-          imagePullPolicy: {{ .Values.crc.image.pullPolicy }}
+        - name: data-service
+          image: "{{ .Values.dataService.image.repository }}:{{ .Values.dataService.image.tag }}"
+          imagePullPolicy: {{ .Values.dataService.image.pullPolicy }}
           securityContext:
             {{- toYaml .Values.securityContext | nindent 12 }}
           ports:
@@ -57,14 +57,17 @@ spec:
             - name: KEYCLOAK_TOKEN_SIGNATURE_ALGS
               value: "RS256"
             - name: SERVER_DEFAULTS
-              value: /etc/renku-crc/server_options/server_defaults.json
+              value: /etc/renku-data-service/server_options/server_defaults.json
             - name: SERVER_OPTIONS
-              value: /etc/renku-crc/server_options/server_options.json
+              value: /etc/renku-data-service/server_options/server_options.json
             - name: K8S_NAMESPACE
               value: {{ .Release.Namespace | quote }}
+            - name: GITLAB_URL
+              value: {{ .Values.global.gitlab.url | quote}}
+            {{- include "certificates.env.python" $ | nindent 12 }}
           volumeMounts:
             - name: server-options
-              mountPath: /etc/renku-crc/server_options
+              mountPath: /etc/renku-data-service/server_options
             {{- include "certificates.volumeMounts.system" . | nindent 12 }}
           livenessProbe:
             httpGet:
@@ -87,16 +90,16 @@ spec:
             periodSeconds: 5
             failureThreshold: 60
           resources:
-            {{ toYaml .Values.crc.resources | nindent 12 }}
-    {{- with .Values.crc.nodeSelector }}
+            {{ toYaml .Values.dataService.resources | nindent 12 }}
+    {{- with .Values.dataService.nodeSelector }}
       nodeSelector:
         {{ toYaml . | nindent 8 }}
     {{- end }}
-    {{- with .Values.crc.affinity }}
+    {{- with .Values.dataService.affinity }}
       affinity:
         {{ toYaml . | nindent 8 }}
     {{- end }}
-    {{- with .Values.crc.tolerations }}
+    {{- with .Values.dataService.tolerations }}
       tolerations:
         {{ toYaml . | nindent 8 }}
     {{- end }}
@@ -105,4 +108,4 @@ spec:
           configMap:
             name: {{ template "renku.fullname" . }}-server-options
         {{- include "certificates.volumes" . | nindent 8 }}
-      serviceAccountName: {{ template "renku.fullname" . }}-crc
+      serviceAccountName: {{ template "renku.fullname" . }}-data-service