NIFI-12898 Follow on updates from initial implementation

- Remove AssetRequestReplicator and use existing UploadRequestReplicator
- Remove AssetManagerFactoryBean and expose AssetManager from Java based Spring config
- Switch multi-part form upload to direct octect-stream upload
- Adding CLI commands creating and referencing assets
- Add REST end-point for listing assets in a given param context
- Fix param context update merger
- Add REST end-point for retrieving content of an asset
- Add CLI commands for listing assets and getting asset content
- Refactor asset id to a generated uuid from param context id + asset name
- Add AssetSynchronizer and wire into StandardFlowService
- Protect against referencing assets from different context
- Implement syncing assets from the cluster coordinator
- Add system tests for replacing an asset and syncing assets
- Update flow diffing/synchronization to account for changes to referenced assets
- Change ParameterDTO from a list of String assetIds to a list of AssetReferenceDTOs
- Authorize node identities to read parameter contexts
- Improve clean up of unreferenced assets

nifi-api/src/main/java/org/apache/nifi/asset/Asset.java

@@ -18,6 +18,7 @@
 package org.apache.nifi.asset;
 
 import java.io.File;
+import java.util.Optional;
 
 /**
  * An Asset is a representation of some resource that is necessary in order to run a dataflow.
@@ -30,6 +31,11 @@ public interface Asset {
      */
     String getIdentifier();
 
+    /**
+     * Returns the identifier of the parameter context the Asset belongs to
+     */
+    String getParameterContextIdentifier();
+
     /**
      * Returns the name of the Asset
      */
@@ -40,4 +46,9 @@ public interface Asset {
      */
     File getFile();
 
+    /**
+     * Returns the digest of the contents of the local file that the Asset is associated with.
+     * The digest will not be present when the asset is considered missing and the local file does not exist.
+     */
+    Optional<String> getDigest();
 }

nifi-api/src/main/java/org/apache/nifi/flow/VersionedAsset.java

@@ -22,7 +22,6 @@
 public class VersionedAsset {
     private String identifier;
     private String name;
-    private String filename;
 
     @Schema(description = "The identifier of the asset")
     public String getIdentifier() {
@@ -42,12 +41,4 @@ public void setName(final String name) {
         this.name = name;
     }
 
-    @Schema(description = "The filename of the asset")
-    public String getFilename() {
-        return filename;
-    }
-
-    public void setFilename(final String filename) {
-        this.filename = filename;
-    }
 }

nifi-api/src/main/java/org/apache/nifi/flow/VersionedParameter.java

@@ -75,7 +75,7 @@ public void setValue(String value) {
         this.value = value;
     }
 
-    @Schema(description = "The ID's of assets that are referenced by this parameter")
+    @Schema(description = "The assets that are referenced by this parameter")
     public List<VersionedAsset> getReferencedAssets() {
         return referencedAssets;
     }

nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java

@@ -30,7 +30,6 @@ public class Parameter {
     private final boolean provided;
     private final List<Asset> referencedAssets;
 
-
     private Parameter(final Builder builder) {
         this.descriptor = new ParameterDescriptor.Builder()
             .name(builder.name)
@@ -153,7 +152,7 @@ public Builder parameterContextId(final String parameterContextId) {
         }
 
         public Builder provided(final Boolean provided) {
-            this.provided = provided;
+            this.provided = provided != null && provided;
             return this;
         }
 

nifi-api/src/test/java/org/apache/nifi/parameter/TestParameter.java

@@ -22,6 +22,7 @@
 
 import java.io.File;
 import java.util.List;
+import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -45,7 +46,7 @@ public void testCreateParameterWithValue() {
     @Test
     public void testCreateParameterWithSingleReference() {
         final File file = new File("file");
-        final Asset asset = new MockAsset("id", "name", file);
+        final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest");
 
         final Parameter parameter = new Parameter.Builder()
             .name("A")
@@ -64,9 +65,9 @@ public void testCreateParameterWithMultipleReferences() {
         final File file1 = new File("file1");
         final File file2 = new File("file2");
         final File file3 = new File("file3");
-        final Asset asset1 = new MockAsset("id1", "name1", file1);
-        final Asset asset2 = new MockAsset("id2", "name2", file2);
-        final Asset asset3 = new MockAsset("id3", "name3", file3);
+        final Asset asset1 = new MockAsset("id1", "parmContext", "name1", file1, "asset-digest");
+        final Asset asset2 = new MockAsset("id2", "parmContext", "name2", file2, "asset-digest");
+        final Asset asset3 = new MockAsset("id3", "parmContext", "name3", file3, "asset-digest");
 
         final Parameter parameter = new Parameter.Builder()
             .name("A")
@@ -82,7 +83,7 @@ public void testCreateParameterWithMultipleReferences() {
     @Test
     public void testCreateParameterWithValueThenAsset() {
         final File file = new File("file");
-        final Asset asset = new MockAsset("id", "name", file);
+        final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest");
 
         final Parameter parameter = new Parameter.Builder()
             .name("A")
@@ -100,7 +101,7 @@ public void testCreateParameterWithValueThenAsset() {
     @Test
     public void testCreateParameterAssetThenValue() {
         final File file = new File("file");
-        final Asset asset = new MockAsset("id", "name", file);
+        final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest");
 
         final Parameter parameter = new Parameter.Builder()
             .name("A")
@@ -117,7 +118,7 @@ public void testCreateParameterAssetThenValue() {
     @Test
     public void testCreateParameterFromOtherThenOverrideWithAsset() {
         final File file = new File("file");
-        final Asset asset = new MockAsset("id", "name", file);
+        final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest");
 
         final Parameter original = new Parameter.Builder()
             .name("A")
@@ -138,7 +139,7 @@ public void testCreateParameterFromOtherThenOverrideWithAsset() {
     @Test
     public void testCreateParameterFromOtherThenOverrideWithValue() {
         final File file = new File("file");
-        final Asset asset = new MockAsset("id", "name", file);
+        final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest");
 
         final Parameter original = new Parameter.Builder()
             .name("A")
@@ -174,23 +175,31 @@ public void testCreateParameterFromOtherThenOverrideWithDifferentValue() {
         assertEquals(0, parameter.getReferencedAssets().size());
     }
 
-
     private static class MockAsset implements Asset {
         private final String identifier;
+        private final String parameterContextIdentifier;
         private final String name;
         private final File file;
+        private final String digest;
 
-        public MockAsset(final String identifier, final String name, final File file) {
+        public MockAsset(final String identifier, final String parameterContextIdentifier, final String name, final File file, final String digest) {
             this.identifier = identifier;
+            this.parameterContextIdentifier = parameterContextIdentifier;
             this.name = name;
             this.file = file;
+            this.digest = digest;
         }
 
         @Override
         public String getIdentifier() {
             return identifier;
         }
 
+        @Override
+        public String getParameterContextIdentifier() {
+            return parameterContextIdentifier;
+        }
+
         @Override
         public String getName() {
             return name;
@@ -200,5 +209,10 @@ public String getName() {
         public File getFile() {
             return file;
         }
+
+        @Override
+        public Optional<String> getDigest() {
+            return Optional.ofNullable(digest);
+        }
     }
 }

nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java

@@ -582,4 +582,36 @@ public static long getContainerUsableSpace(final Path path) {
         return path.toFile().getUsableSpace();
     }
 
+    // The invalid character list is derived from this Stackoverflow page.
+    // https://stackoverflow.com/questions/1155107/is-there-a-cross-platform-java-method-to-remove-filename-special-chars
+    private final static int[] INVALID_CHARS = {34, 60, 62, 124, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15,
+            16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 58, 42, 63, 92, 47, 32};
+
+    static {
+        Arrays.sort(INVALID_CHARS);
+    }
+
+    /**
+     * Replaces invalid characters for a file system name within a given filename string to underscore '_'.
+     * Be careful not to pass a file path as this method replaces path delimiter characters (i.e forward/back slashes).
+     * @param filename The filename to clean
+     * @return sanitized filename
+     */
+    public static String sanitizeFilename(String filename) {
+        if (filename == null || filename.isEmpty()) {
+            return filename;
+        }
+        int codePointCount = filename.codePointCount(0, filename.length());
+
+        final StringBuilder cleanName = new StringBuilder();
+        for (int i = 0; i < codePointCount; i++) {
+            int c = filename.codePointAt(i);
+            if (Arrays.binarySearch(INVALID_CHARS, c) < 0) {
+                cleanName.appendCodePoint(c);
+            } else {
+                cleanName.append('_');
+            }
+        }
+        return cleanName.toString();
+    }
 }

nifi-docs/src/main/asciidoc/toolkit-guide.adoc

@@ -146,6 +146,11 @@ The following are available commands:
  nifi delete-nar
  nifi list-nars
  nifi list-nar-component-types
+ nifi create-asset
+ nifi list-assets
+ nifi get-asset
+ nifi add-asset-reference
+ nifi remove-asset-reference
  registry current-user
  registry list-buckets
  registry create-bucket

nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManager.java

@@ -19,6 +19,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.List;
 import java.util.Optional;
 
 public interface AssetManager {
@@ -45,6 +46,13 @@ public interface AssetManager {
      */
     Optional<Asset> getAsset(String id);
 
+    /**
+     * Retrieves the Assets that belong to the given parameter context.
+     * @param parameterContextId the id of the parameter context
+     * @return the list of assets for the given context
+     */
+    List<Asset> getAssets(String parameterContextId);
+
     /**
      * Creates an Asset with the given name and associates it with the given parameter context. If the asset already exists, it is returned. Otherwise, an asset is created
      * but the underlying file is not created. This allows the asset to be referenced but any component that attempts to use the asset will still see a File that does not exist, which

nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManagerInitializationContext.java

@@ -21,11 +21,23 @@
 
 import java.util.Map;
 
+/**
+ * Initialization context for AssetManager.
+ */
 public interface AssetManagerInitializationContext {
 
+    /**
+     * @return lookup for obtaining referenced assets
+     */
     AssetReferenceLookup getAssetReferenceLookup();
 
+    /**
+     * @return configuration for the asset manager
+     */
     Map<String, String> getProperties();
 
+    /**
+     * @return the node type provider
+     */
     NodeTypeProvider getNodeTypeProvider();
 }

nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetReferenceLookup.java

@@ -22,7 +22,7 @@
 public interface AssetReferenceLookup {
 
     /**
-     * The Set of identifiers for all Assets that are currently referenced by any parameters.
+     * The Set of all Assets that are currently referenced by any parameters.
      */
     Set<Asset> getReferencedAssets();
 

nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetDTO.java

@@ -17,15 +17,20 @@
 
 package org.apache.nifi.web.api.dto;
 
+import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.xml.bind.annotation.XmlType;
 
+import java.util.Objects;
+
 @XmlType(name = "asset")
 public class AssetDTO {
 
     private String id;
     private String name;
     private String filename;
+    private String digest;
 
+    @Schema(description = "The identifier of the asset.")
     public String getId() {
         return id;
     }
@@ -34,6 +39,7 @@ public void setId(final String id) {
         this.id = id;
     }
 
+    @Schema(description = "The name of the asset.")
     public String getName() {
         return name;
     }
@@ -42,11 +48,38 @@ public void setName(final String name) {
         this.name = name;
     }
 
+    @Schema(description = "The filename of the asset.")
     public String getFilename() {
         return filename;
     }
 
     public void setFilename(final String filename) {
         this.filename = filename;
     }
+
+    @Schema(description = "The digest of the asset.")
+    public String getDigest() {
+        return digest;
+    }
+
+    public void setDigest(final String digest) {
+        this.digest = digest;
+    }
+
+    @Override
+    public boolean equals(final Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        final AssetDTO assetDTO = (AssetDTO) o;
+        return Objects.equals(id, assetDTO.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
 }