Implementing reading Data Links

.github/CODEOWNERS

@@ -47,3 +47,6 @@ Cargo.toml
 
 # Dashboard, Cloud & Authentication
 /app/ide-desktop/ @PabloBuchu @indiv0 @somebody1234
+# The data-link schema is owned by the libraries team
+/app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json @radeusgd @jdunkerley @GregoryTravis @AdRiley
+/app/ide-desktop/lib/dashboard/src/data/__tests__ @radeusgd @jdunkerley @GregoryTravis @AdRiley @PabloBuchu @indiv0 @somebody1234

CHANGELOG.md

@@ -619,6 +619,7 @@
 - [Separate `Group_By` from `columns` into new argument on `aggregate`.][9027]
 - [Allow `copy_to` and `move_to` to work between local and S3 files.][9054]
 - [Adjusted expression handling and new `Simple_Expression` type.][9128]
+- [Allow reading Data Links configured locally or in the Cloud.][9215]
 
 [debug-shortcuts]:
   https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
@@ -894,6 +895,7 @@
 [9027]: https://github.com/enso-org/enso/pull/9027
 [9054]: https://github.com/enso-org/enso/pull/9054
 [9128]: https://github.com/enso-org/enso/pull/9128
+[9215]: https://github.com/enso-org/enso/pull/9215
 
 #### Enso Compiler
 

app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts

@@ -0,0 +1,66 @@
+/** @file Tests ensuring consistency of example data-link files with the schema. */
+
+import * as fs from 'node:fs'
+import * as path from 'node:path'
+
+import * as v from 'vitest'
+
+import * as validateDataLink from '#/utilities/validateDataLink'
+
+v.test('correctly rejects invalid values as not matching the schema', () => {
+  v.expect(validateDataLink.validateDataLink({})).toBe(false)
+  v.expect(validateDataLink.validateDataLink('foobar')).toBe(false)
+  v.expect(validateDataLink.validateDataLink({ foo: 'BAR' })).toBe(false)
+})
+
+/** Load and parse a data-link description. */
+function loadDataLinkFile(dataLinkPath: string): unknown {
+  const text: string = fs.readFileSync(dataLinkPath, { encoding: 'utf-8' })
+  return JSON.parse(text)
+}
+
+/** Check if the given data-link description matches the schema, reporting any errors. */
+function testSchema(json: unknown, fileName: string): void {
+  const validate = validateDataLink.validateDataLink
+  if (!validate(json)) {
+    v.assert.fail(`Failed to validate ${fileName}:\n${JSON.stringify(validate.errors, null, 2)}`)
+  }
+}
+
+// We need to go up from `app/ide-desktop/lib/dashboard/` to the root of the repo
+const REPO_ROOT = '../../../../'
+const BASE_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/Base_Tests/data/datalinks/')
+const S3_DATA_LINKS_ROOT = path.resolve(REPO_ROOT, 'test/AWS_Tests/data/')
+
+v.test('correctly validates example HTTP .datalink files with the schema', () => {
+  const schemas = [
+    'example-http.datalink',
+    'example-http-format-explicit-default.datalink',
+    'example-http-format-delimited.datalink',
+    'example-http-format-json.datalink',
+  ]
+  for (const schema of schemas) {
+    const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema))
+    testSchema(json, schema)
+  }
+})
+
+v.test('rejects invalid schemas (Base)', () => {
+  const invalidSchemas = ['example-http-format-invalid.datalink']
+  for (const schema of invalidSchemas) {
+    const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema))
+    v.expect(validateDataLink.validateDataLink(json)).toBe(false)
+  }
+})
+
+v.test('correctly validates example S3 .datalink files with the schema', () => {
+  const schemas = [
+    'simple.datalink',
+    'credentials-with-secrets.datalink',
+    'format-delimited.datalink',
+  ]
+  for (const schema of schemas) {
+    const json = loadDataLinkFile(path.resolve(S3_DATA_LINKS_ROOT, schema))
+    testSchema(json, schema)
+  }
+})

app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json

@@ -6,7 +6,8 @@
       "anyOf": [
         { "$ref": "#/$defs/S3DataLink" },
         { "$ref": "#/$defs/HttpFetchDataLink" }
-      ]
+      ],
+      "$comment": "The fields `type` and `libraryName` are required for all data link types, but we currently don't add a top-level `required` setting to the schema, because it was confusing the code that is generating the modal."
     },
     "SecureValue": {
       "title": "Secure Value",
@@ -88,32 +89,35 @@
       "type": "object",
       "properties": {
         "type": { "title": "Type", "const": "S3", "type": "string" },
+        "libraryName": { "const": "Standard.AWS" },
         "uri": {
           "title": "URI",
           "description": "Must start with \"s3://\".",
           "type": "string",
-          "pattern": "^s3://[\\w.~-]+/[/\\w.~-]+$"
+          "pattern": "^s3://[a-z0-9.-]{3,63}/.{1,1024}$"
         },
         "auth": { "title": "Authentication", "$ref": "#/$defs/AwsAuth" },
         "format": { "title": "Format", "$ref": "#/$defs/Format" }
       },
-      "required": ["type", "uri", "auth"]
+      "required": ["type", "libraryName", "uri", "auth"]
     },
     "HttpFetchDataLink": {
-      "$comment": "missing <headers with secrets> OR <query string with secrets>",
+      "$comment": "missing <headers with secrets> and <query string with secrets>",
       "title": "HTTP Fetch",
       "type": "object",
       "properties": {
         "type": { "title": "Type", "const": "HTTP", "type": "string" },
+        "libraryName": { "const": "Standard.Base" },
         "uri": {
           "title": "URI",
           "description": "Must start with \"http://\" or \"https://\".",
           "type": "string",
           "pattern": "^https?://[\\w.~-]+/?.*$"
         },
-        "method": { "title": "Method", "const": "GET", "type": "string" }
+        "method": { "title": "Method", "const": "GET", "type": "string" },
+        "format": { "title": "Format", "$ref": "#/$defs/Format" }
       },
-      "required": ["type", "uri", "method"]
+      "required": ["type", "libraryName", "uri", "method"]
     },
 
     "Format": {
@@ -147,7 +151,8 @@
           "title": "Delimiter",
           "description": "Must not be blank.",
           "type": "string",
-          "minLength": 1
+          "minLength": 1,
+          "maxLength": 1
         },
         "encoding": { "title": "Encoding", "const": "utf8", "type": "string" },
         "headers": {
@@ -162,9 +167,10 @@
       "title": "JSON",
       "type": "object",
       "properties": {
-        "type": { "title": "Type", "const": "json", "type": "string" }
+        "type": { "title": "Type", "const": "format", "type": "string" },
+        "subType": { "title": "Type", "const": "json", "type": "string" }
       },
-      "required": ["type"]
+      "required": ["type", "subType"]
     }
   }
 }

distribution/lib/Standard/AWS/0.0.0-dev/src/Internal/Data_Link_Helpers.enso

@@ -0,0 +1,24 @@
+private
+
+from Standard.Base import all
+import Standard.Base.Errors.Illegal_State.Illegal_State
+from Standard.Base.Enso_Cloud.Public_Utils import get_required_field
+from Standard.Base.Enso_Cloud.Data_Link import parse_secure_value
+
+import project.AWS_Credential.AWS_Credential
+
+## PRIVATE
+   Decodes the JSON representation of `AWS_Credential` as defined in `dataLinkSchema.json#/$defs/AwsAuth`.
+decode_aws_credential json -> AWS_Credential | Nothing =
+    case get_required_field "type" json of
+        "aws_auth" -> case get_required_field "subType" json of
+            "default" -> Nothing
+            "profile" ->
+                profile = get_required_field "profile" json
+                AWS_Credential.Profile profile
+            "access_key" ->
+                access_key_id = get_required_field "accessKeyId" json |> parse_secure_value
+                secret_access_key = get_required_field "secretAccessKey" json |> parse_secure_value
+                AWS_Credential.Key access_key_id secret_access_key
+            unexpected -> Error.throw (Illegal_State.Error "Unexpected subType inside of `auth` field of a datalink: "+unexpected.to_text)
+        unexpected -> Error.throw (Illegal_State.Error "Unexpected type inside of `auth` field of a datalink: "+unexpected.to_text)

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso

@@ -0,0 +1,26 @@
+from Standard.Base import all
+from Standard.Base.Enso_Cloud.Public_Utils import get_required_field
+from Standard.Base.Enso_Cloud.Data_Link import parse_format
+
+import project.AWS_Credential.AWS_Credential
+import project.S3.S3_File.S3_File
+from project.Internal.Data_Link_Helpers import decode_aws_credential
+
+## PRIVATE
+type S3_Data_Link
+    ## PRIVATE
+    Value (uri : Text) format (credentials : AWS_Credential | Nothing)
+
+    ## PRIVATE
+    parse json -> S3_Data_Link =
+        uri = get_required_field "uri" json
+        auth = decode_aws_credential (get_required_field "auth" json)
+        format = parse_format (json.get "format" Nothing)
+        S3_Data_Link.Value uri format auth
+
+    ## PRIVATE
+    as_file self -> S3_File = S3_File.new self.uri self.credentials
+
+    ## PRIVATE
+    read self (on_problems : Problem_Behavior) =
+        self.as_file.read self.format on_problems

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Data_Link.enso

@@ -0,0 +1,93 @@
+import project.Any.Any
+import project.Data.Json.JS_Object
+import project.Data.Text.Encoding.Encoding
+import project.Data.Text.Text
+import project.Enso_Cloud.Enso_Secret.Enso_Secret
+import project.Error.Error
+import project.Errors.Illegal_State.Illegal_State
+import project.Errors.Problem_Behavior.Problem_Behavior
+import project.Errors.Unimplemented.Unimplemented
+import project.Nothing.Nothing
+import project.System.File.File
+import project.System.File.Generic.Writable_File.Writable_File
+import project.System.File_Format.Auto_Detect
+import project.System.File_Format.Infer
+import project.System.File_Format.JSON_Format
+import project.System.File_Format_Metadata.File_Format_Metadata
+import project.System.Input_Stream.Input_Stream
+from project.Enso_Cloud.Public_Utils import get_required_field
+
+polyglot java import org.enso.base.enso_cloud.DataLinkSPI
+polyglot java import org.enso.base.file_format.FileFormatSPI
+
+## PRIVATE
+   A file format for reading data links.
+type Data_Link_Format
+    ## PRIVATE
+       If the File_Format supports reading from the file, return a configured instance.
+    for_read : File_Format_Metadata -> Data_Link_Format | Nothing
+    for_read file:File_Format_Metadata =
+        case file.guess_extension of
+            ".datalink" -> Data_Link_Format
+            _ -> Nothing
+
+    ## PRIVATE
+       Currently writing data links is not supported.
+    for_file_write : Writable_File -> Nothing
+    for_file_write file =
+        _ = file
+        Nothing
+
+    ## PRIVATE
+       Implements the `File.read` for this `File_Format`
+    read : File -> Problem_Behavior -> Any
+    read self file on_problems =
+        json = JSON_Format.read file on_problems
+        read_datalink json on_problems
+
+    ## PRIVATE
+       Implements decoding the format from a stream.
+    read_stream : Input_Stream -> File_Format_Metadata -> Any
+    read_stream self stream:Input_Stream (metadata : File_Format_Metadata) =
+        json = JSON_Format.read_stream stream metadata
+        read_datalink json Problem_Behavior.Report_Error
+
+## PRIVATE
+interpret_json_as_datalink json =
+    typ = get_required_field "type" json
+    case DataLinkSPI.findDataLinkType typ of
+        Nothing ->
+            library_name = get_required_field "libraryName" json
+            Error.throw (Illegal_State.Error "The data link for "+typ+" is provided by the library "+library_name+" which is not loaded. Please import the library, and if necessary, restart the project.")
+        data_link_type ->
+            data_link_type.parse json
+
+## PRIVATE
+read_datalink json on_problems =
+    data_link_instance = interpret_json_as_datalink json
+    data_link_instance.read on_problems
+
+## PRIVATE
+parse_secure_value (json : Text | JS_Object) -> Text | Enso_Secret =
+    case json of
+        raw_text : Text -> raw_text
+        _ : JS_Object ->
+            case get_required_field "type" json of
+                "secret" ->
+                    secret_path = get_required_field "secretPath" json
+                    _ = secret_path
+                    Unimplemented.throw "Reading secrets from a path is not implemented yet, see: https://github.com/enso-org/enso/issues/9048"
+                other -> Error.throw (Illegal_State.Error "Unexpected value inside of a data-link: "+other+".")
+
+## PRIVATE
+parse_format json = case json of
+    Nothing -> Auto_Detect
+    _ : JS_Object -> case get_required_field "subType" json of
+        "default" -> Auto_Detect
+        sub_type : Text ->
+            format_type = FileFormatSPI.findFormatForDataLinkSubType sub_type
+            if format_type.is_nothing then Error.throw (Illegal_State.Error "Unknown format inside of a datalink: "+sub_type+". Perhaps the library providing that format needs to be imported?") else
+                format_type.from json
+        other ->
+            Error.throw (Illegal_State.Error "Expected `subType` to be a string, but got: "+other.to_display_text+".")
+    other -> Error.throw (Illegal_State.Error "Unexpected value inside of a data-link `format` field: "+other.to_display_text+".")

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso

@@ -29,7 +29,8 @@ import project.System.File_Format_Metadata.File_Format_Metadata
 import project.System.Input_Stream.Input_Stream
 import project.System.Output_Stream.Output_Stream
 from project.Data.Boolean import Boolean, False, True
-from project.Enso_Cloud.Utils import get_required_field
+from project.Enso_Cloud.Data_Link import read_datalink
+from project.Enso_Cloud.Public_Utils import get_required_field
 from project.Data.Text.Extensions import all
 from project.System.File.Generic.File_Write_Strategy import generic_copy
 from project.System.File_Format import Auto_Detect, Bytes, File_Format, Plain_Text_Format
@@ -66,7 +67,7 @@ type Enso_File
             Enso_Asset_Type.Directory -> Utils.directory_api + "/" + self.id
             Enso_Asset_Type.File -> Utils.files_api + "/" + self.id
             Enso_Asset_Type.Project -> Utils.projects_api + "/" + self.id
-            Enso_Asset_Type.Data_Link -> Utils.secrets_api + "/" + self.id
+            Enso_Asset_Type.Data_Link -> Utils.datalinks_api + "/" + self.id
             Enso_Asset_Type.Secret -> Error.throw (Illegal_Argument.Error "Secrets cannot be accessed directly.")
 
     ## GROUP Metadata
@@ -183,7 +184,9 @@ type Enso_File
     read self format=Auto_Detect (on_problems=Problem_Behavior.Report_Warning) = case self.asset_type of
         Enso_Asset_Type.Project -> Error.throw (Illegal_Argument.Error "Projects cannot be read within Enso code. Open using the IDE.")
         Enso_Asset_Type.Secret -> Error.throw (Illegal_Argument.Error "Secrets cannot be read directly.")
-        Enso_Asset_Type.Data_Link -> Unimplemented.throw "Reading from a Data Link is not implemented yet."
+        Enso_Asset_Type.Data_Link ->
+            json = Utils.http_request_as_json HTTP_Method.Get self.internal_uri
+            read_datalink json on_problems
         Enso_Asset_Type.Directory -> if format == Auto_Detect then self.list else Error.throw (Illegal_Argument.Error "Directories can only be read using the Auto_Detect format.")
         Enso_Asset_Type.File -> case format of
             Auto_Detect ->
@@ -341,7 +344,7 @@ Enso_Asset_Type.from (that:Text) = case that of
     "directory" -> Enso_Asset_Type.Directory
     "secret" -> Enso_Asset_Type.Secret
     "connector" -> Enso_Asset_Type.Data_Link
-    _ -> Error.throw (Illegal_Argument.Error "Invalid asset type.")
+    _ -> Error.throw (Illegal_Argument.Error "Invalid asset type: "+that.pretty+".")
 
 ## PRIVATE
 File_Format_Metadata.from (that:Enso_File) = File_Format_Metadata.Value Nothing that.name (that.extension.catch _->Nothing)

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_User.enso

@@ -10,7 +10,7 @@ import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
 import project.Nothing.Nothing
 from project.Data.Boolean import Boolean, False, True
-from project.Enso_Cloud.Utils import get_required_field
+from project.Enso_Cloud.Public_Utils import get_required_field
 
 type Enso_User
     ## PRIVATE

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Public_Utils.enso

@@ -0,0 +1,10 @@
+import project.Enso_Cloud.Errors.Enso_Cloud_Error
+import project.Error.Error
+import project.Data.Json.JS_Object
+
+## PRIVATE
+   A helper that extracts a field from a response and handles unexpected
+   response structure.
+get_required_field key js_object = case js_object of
+    _ : JS_Object -> js_object.get key if_missing=(Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Missing required field `"+key+"` in "+js_object.to_display_text+"."))
+    _ -> Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Expected a JSON object, but got "+js_object.to_display_text+".")

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Utils.enso

@@ -6,7 +6,6 @@ import project.Enso_Cloud.Enso_Secret.Enso_Secret
 import project.Enso_Cloud.Errors.Enso_Cloud_Error
 import project.Enso_Cloud.Errors.Not_Logged_In
 import project.Data.Json.Invalid_JSON
-import project.Data.Json.JS_Object
 import project.Data.Map.Map
 import project.Data.Pair.Pair
 import project.Data.Text.Text
@@ -67,6 +66,10 @@ projects_api = cloud_root_uri + "projects"
    Root address for Secrets API
 secrets_api = cloud_root_uri + "secrets"
 
+## PRIVATE
+   Root address for DataLinks API
+datalinks_api = cloud_root_uri + "connectors"
+
 ## PRIVATE
    The current project directory that will be used as the working directory,
    if the user is running in the Cloud.
@@ -109,10 +112,3 @@ http_request (method : HTTP_Method) (url : URI) (body : Request_Body = Request_B
             case handler of
                 Nothing -> Error.throw (Enso_Cloud_Error.Unexpected_Service_Error response.code payload)
                 _ : Function -> handler json_payload
-
-## PRIVATE
-   A helper that extracts a field from a response and handles unexpected
-   response structure.
-get_required_field key js_object = case js_object of
-    _ : JS_Object -> js_object.get key if_missing=(Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Missing required field `"+key+"` in "+js_object.to_display_text+"."))
-    _ -> Error.throw (Enso_Cloud_Error.Invalid_Response_Payload "Expected a JSON object, but got "+js_object.to_display_text+".")