From 9437895180cafafcdd5e305876b20d44001170d7 Mon Sep 17 00:00:00 2001 From: Bryan Bende Date: Thu, 6 Jun 2024 14:12:31 -0400 Subject: [PATCH] Refactor standard persistence provider to use hierarchical directories and include properties file --- .../apache/nifi/nar/NarPersistenceInfo.java | 60 +++++ .../nifi/nar/NarPersistenceProvider.java | 21 +- .../org/apache/nifi/nar/NarProperties.java | 214 +++++++++++++++++ .../java/org/apache/nifi/nar/NarProperty.java | 44 ++++ .../apache/nifi/nar/NarPropertiesTest.java | 87 +++++++ .../nifi/controller/FlowController.java | 16 +- .../apache/nifi/nar/StandardNarManager.java | 77 ++++--- .../nar/StandardNarPersistenceProvider.java | 215 ++++++++++++++---- 8 files changed, 640 insertions(+), 94 deletions(-) create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceInfo.java create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperties.java create mode 100644 nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperty.java create mode 100644 nifi-framework-api/src/test/java/org/apache/nifi/nar/NarPropertiesTest.java diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceInfo.java b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceInfo.java new file mode 100644 index 000000000000..5117f80ef375 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceInfo.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.nar; + +import java.io.File; +import java.util.Objects; + +/** + * The information about a NAR persisted by a {@link NarPersistenceProvider}. + */ +public class NarPersistenceInfo { + + private final File narFile; + private final NarProperties narProperties; + + public NarPersistenceInfo(final File narFile, final NarProperties narProperties) { + this.narFile = Objects.requireNonNull(narFile); + this.narProperties = Objects.requireNonNull(narProperties); + } + + public File getNarFile() { + return narFile; + } + + public NarProperties getNarProperties() { + return narProperties; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final NarPersistenceInfo that = (NarPersistenceInfo) o; + return Objects.equals(narProperties.getCoordinate(), that.narProperties.getCoordinate()); + } + + @Override + public int hashCode() { + return Objects.hash(narProperties.getCoordinate()); + } +} diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceProvider.java b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceProvider.java index b380d0bcd937..d5ff5838fc1e 100644 --- a/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceProvider.java +++ b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarPersistenceProvider.java @@ -23,7 +23,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.util.Map; +import java.util.Set; /** * Provides persistence for NAR files dynamically added to NiFi. @@ -35,7 +35,7 @@ public interface NarPersistenceProvider { * * @param initializationContext the context containing and properties and configuration required to initialize the provider */ - void initialize(NarPersistenceProviderInitializationContext initializationContext); + void initialize(NarPersistenceProviderInitializationContext initializationContext) throws IOException; /** * Creates a temporary file containing the contents of the given input stream. @@ -53,11 +53,11 @@ public interface NarPersistenceProvider { * * @param persistenceContext the context information * @param tempNarFile the temporary file created from calling createTempFile - * @return the saved file + * @return the information about the persisted NAR, including the saved file * * @throws IOException if an I/O error occurs saving the temp NAR file */ - File saveNar(NarPersistenceContext persistenceContext, File tempNarFile) throws IOException; + NarPersistenceInfo saveNar(NarPersistenceContext persistenceContext, File tempNarFile) throws IOException; /** * Deletes the contents of the NAR with the given coordinate. @@ -79,6 +79,15 @@ public interface NarPersistenceProvider { */ InputStream readNar(BundleCoordinate narCoordinate) throws FileNotFoundException; + /** + * Retrieves the persistence info for the NAR with the given NAR coordinate. + * + * @param narCoordinate the coordinate of the NAR + * @return the persistence info + * @throws IOException if an I/O error occurs reading the persistence info + */ + NarPersistenceInfo getNarInfo(BundleCoordinate narCoordinate) throws IOException; + /** * Indicates if a NAR with the given coordinate exists in this persistence provider. * @@ -88,9 +97,9 @@ public interface NarPersistenceProvider { boolean exists(BundleCoordinate narCoordinate); /** - * @return a map from NAR coordinate to NAR File for all NARs saved to this persistence provider + * @return the info for all NARs persisted by this persistence provider */ - Map getNarFiles(); + Set getAllNarInfo() throws IOException; /** * Called prior to NiFi shutting down. diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperties.java b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperties.java new file mode 100644 index 000000000000..5a08c02f9085 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperties.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.nar; + +import org.apache.nifi.bundle.BundleCoordinate; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; + +/** + * Properties about a NAR that are persisted by the {@link NarPersistenceProvider}. + */ +public class NarProperties { + + private final String sourceType; + private final String sourceId; + + private final String narGroup; + private final String narId; + private final String narVersion; + + private final String narDependencyGroup; + private final String narDependencyId; + private final String narDependencyVersion; + + private final Instant created; + + private NarProperties(final Builder builder) { + this.sourceType = Objects.requireNonNull(builder.sourceType); + this.sourceId = builder.sourceId; + + this.narGroup = Objects.requireNonNull(builder.narGroup); + this.narId = Objects.requireNonNull(builder.narId); + this.narVersion = Objects.requireNonNull(builder.narVersion); + + this.narDependencyGroup = builder.narDependencyGroup; + this.narDependencyId = builder.narDependencyId; + this.narDependencyVersion = builder.narDependencyVersion; + + this.created = Objects.requireNonNull(builder.created); + } + + public String getSourceType() { + return sourceType; + } + + public String getSourceId() { + return sourceId; + } + + public String getNarGroup() { + return narGroup; + } + + public String getNarId() { + return narId; + } + + public String getNarVersion() { + return narVersion; + } + + public String getNarDependencyGroup() { + return narDependencyGroup; + } + + public String getNarDependencyId() { + return narDependencyId; + } + + public String getNarDependencyVersion() { + return narDependencyVersion; + } + + public Instant getCreated() { + return created; + } + + public BundleCoordinate getCoordinate() { + return new BundleCoordinate(narGroup, narId, narVersion); + } + + public Optional getDependencyCoordinate() { + if (narDependencyGroup == null) { + return Optional.empty(); + } + return Optional.of(new BundleCoordinate(narDependencyGroup, narDependencyId, narDependencyVersion)); + } + + public Properties toProperties() { + final Properties properties = new Properties(); + properties.put(NarProperty.SOURCE_TYPE.getKey(), sourceType); + if (sourceId != null) { + properties.put(NarProperty.SOURCE_ID.getKey(), sourceId); + } + + properties.put(NarProperty.NAR_GROUP.getKey(), narGroup); + properties.put(NarProperty.NAR_ID.getKey(), narId); + properties.put(NarProperty.NAR_VERSION.getKey(), narVersion); + + if (narDependencyGroup != null) { + properties.put(NarProperty.NAR_DEPENDENCY_GROUP.getKey(), narDependencyGroup); + properties.put(NarProperty.NAR_DEPENDENCY_ID.getKey(), narDependencyId); + properties.put(NarProperty.NAR_DEPENDENCY_VERSION.getKey(), narDependencyVersion); + } + + properties.put(NarProperty.CREATED.getKey(), created.toString()); + return properties; + } + + public static NarProperties parse(final InputStream inputStream) throws IOException { + final Properties properties = new Properties(); + properties.load(inputStream); + + final String createdValue = properties.getProperty(NarProperty.CREATED.getKey()); + final Instant created = Instant.parse(createdValue); + + return NarProperties.builder() + .sourceType(properties.getProperty(NarProperty.SOURCE_TYPE.getKey())) + .sourceId(properties.getProperty(NarProperty.SOURCE_ID.getKey())) + .narGroup(properties.getProperty(NarProperty.NAR_GROUP.getKey())) + .narId(properties.getProperty(NarProperty.NAR_ID.getKey())) + .narVersion(properties.getProperty(NarProperty.NAR_VERSION.getKey())) + .narDependencyGroup(properties.getProperty(NarProperty.NAR_DEPENDENCY_GROUP.getKey())) + .narDependencyId(properties.getProperty(NarProperty.NAR_DEPENDENCY_ID.getKey())) + .narDependencyVersion(properties.getProperty(NarProperty.NAR_DEPENDENCY_VERSION.getKey())) + .created(created) + .build(); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + private String sourceType; + private String sourceId; + private String narGroup; + private String narId; + private String narVersion; + private String narDependencyGroup; + private String narDependencyId; + private String narDependencyVersion; + private Instant created; + + public Builder sourceType(final String sourceType) { + this.sourceType = sourceType; + return this; + } + + public Builder sourceId(final String sourceId) { + this.sourceId = sourceId; + return this; + } + + public Builder narGroup(final String narGroup) { + this.narGroup = narGroup; + return this; + } + + public Builder narId(final String narId) { + this.narId = narId; + return this; + } + + public Builder narVersion(final String narVersion) { + this.narVersion = narVersion; + return this; + } + + public Builder narDependencyGroup(final String narDependencyGroup) { + this.narDependencyGroup = narDependencyGroup; + return this; + } + + public Builder narDependencyId(final String narDependencyId) { + this.narDependencyId = narDependencyId; + return this; + } + + public Builder narDependencyVersion(final String narDependencyVersion) { + this.narDependencyVersion = narDependencyVersion; + return this; + } + + public Builder created(final Instant created) { + this.created = created; + return this; + } + + public NarProperties build() { + return new NarProperties(this); + } + } +} diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperty.java b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperty.java new file mode 100644 index 000000000000..436f4cda39a0 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/nar/NarProperty.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.nar; + +/** + * Keys for {@link NarProperties}. + */ +public enum NarProperty { + + SOURCE_TYPE("source.type"), + SOURCE_ID("source.id"), + NAR_GROUP("nar.group"), + NAR_ID("nar.id"), + NAR_VERSION("nar.version"), + NAR_DEPENDENCY_GROUP("nar.dependency.group"), + NAR_DEPENDENCY_ID("nar.dependency.id"), + NAR_DEPENDENCY_VERSION("nar.dependency.version"), + CREATED("created"); + + private final String key; + + NarProperty(final String key) { + this.key = key; + } + + public String getKey() { + return key; + } +} diff --git a/nifi-framework-api/src/test/java/org/apache/nifi/nar/NarPropertiesTest.java b/nifi-framework-api/src/test/java/org/apache/nifi/nar/NarPropertiesTest.java new file mode 100644 index 000000000000..e8d673b417b1 --- /dev/null +++ b/nifi-framework-api/src/test/java/org/apache/nifi/nar/NarPropertiesTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.nar; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Properties; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class NarPropertiesTest { + + private static final String PROPERTIES_FILE_SUFFIX = ".properties"; + + @Test + public void testNarProperties(@TempDir final Path tempDir) throws IOException { + final NarProperties narProperties = NarProperties.builder() + .sourceType(NarSource.UPLOAD.name()) + .narGroup("org.apache.nifi") + .narId("nifi-standard-nar") + .narVersion("2.0.0") + .created(Instant.now()) + .build(); + + final Path tempFile = createTempFile(tempDir); + writeProperties(narProperties, tempFile); + + final NarProperties parsedProperties = readProperties(tempFile); + assertEqual(narProperties, parsedProperties); + } + + private void assertEqual(final NarProperties narProperties1, final NarProperties narProperties2) { + assertEquals(narProperties1.getSourceType(), narProperties2.getSourceType()); + assertEquals(narProperties1.getSourceId(), narProperties2.getSourceId()); + + assertEquals(narProperties1.getNarGroup(), narProperties2.getNarGroup()); + assertEquals(narProperties1.getNarId(), narProperties2.getNarId()); + assertEquals(narProperties1.getNarVersion(), narProperties2.getNarVersion()); + + assertEquals(narProperties1.getNarDependencyGroup(), narProperties2.getNarDependencyGroup()); + assertEquals(narProperties1.getNarDependencyId(), narProperties2.getNarDependencyId()); + assertEquals(narProperties1.getNarDependencyVersion(), narProperties2.getNarDependencyVersion()); + + assertEquals(narProperties1.getCreated(), narProperties2.getCreated()); + } + + private void writeProperties(final NarProperties narProperties, final Path file) throws IOException { + try (final OutputStream outputStream = new FileOutputStream(file.toFile())) { + final Properties properties = narProperties.toProperties(); + properties.store(outputStream, null); + } + } + + private NarProperties readProperties(final Path file) throws IOException { + try (final InputStream inputStream = new FileInputStream(file.toFile())) { + return NarProperties.parse(inputStream); + } + } + + private Path createTempFile(final Path tempDir) throws IOException { + return Files.createTempFile(tempDir, NarProperties.class.getSimpleName(), PROPERTIES_FILE_SUFFIX); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index 8129a2ba0662..a3c9d4bbfe90 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -158,6 +158,7 @@ import org.apache.nifi.nar.ExtensionManager; import org.apache.nifi.nar.NarCloseable; import org.apache.nifi.nar.NarPersistenceContext; +import org.apache.nifi.nar.NarPersistenceInfo; import org.apache.nifi.nar.NarPersistenceProvider; import org.apache.nifi.nar.NarPersistenceProviderInitializationContext; import org.apache.nifi.nar.NarThreadContextClassLoader; @@ -1413,7 +1414,7 @@ private NarPersistenceProvider wrapWithComponentNarLoader(final NarPersistencePr final ClassLoader originalClassLoader = originalInstance.getClass().getClassLoader(); return new NarPersistenceProvider() { @Override - public void initialize(final NarPersistenceProviderInitializationContext initializationContext) { + public void initialize(final NarPersistenceProviderInitializationContext initializationContext) throws IOException { try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(originalClassLoader)) { originalInstance.initialize(initializationContext); } @@ -1434,7 +1435,7 @@ public File createTempFile(final InputStream inputStream) throws IOException { } @Override - public File saveNar(final NarPersistenceContext persistenceContext, final File tempNarFile) throws IOException { + public NarPersistenceInfo saveNar(final NarPersistenceContext persistenceContext, final File tempNarFile) throws IOException { try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(originalClassLoader)) { return originalInstance.saveNar(persistenceContext, tempNarFile); } @@ -1462,9 +1463,16 @@ public boolean exists(final BundleCoordinate narCoordinate) { } @Override - public Map getNarFiles() { + public NarPersistenceInfo getNarInfo(final BundleCoordinate narCoordinate) throws IOException { try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(originalClassLoader)) { - return originalInstance.getNarFiles(); + return originalInstance.getNarInfo(narCoordinate); + } + } + + @Override + public Set getAllNarInfo() throws IOException { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(originalClassLoader)) { + return originalInstance.getAllNarInfo(); } } }; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java index eaf8993a3f8d..46dd88b4e39f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java @@ -94,54 +94,63 @@ public StandardNarManager(final FlowController flowController, final NarLoader n // 1. Any previously stored NARs need to have their extensions loaded and made available for use during start up since they won't be in any of the standard NAR directories // 2. NarLoader keeps track of NARs that were missing dependencies to consider them on future loads, so this restores state that may have been lost on a restart @Override - public void afterPropertiesSet() { - final Map narFiles = persistenceProvider.getNarFiles(); + public void afterPropertiesSet() throws IOException { + final Collection narFiles = persistenceProvider.getAllNarInfo().stream() + .map(NarPersistenceInfo::getNarFile) + .collect(Collectors.toList()); LOGGER.info("Initializing NAR Manager, loading {} previously stored NARs", narFiles.size()); - narLoader.load(narFiles.values()); + narLoader.load(narFiles); } @Override public synchronized BundleCoordinate addNar(final String filename, final InputStream inputStream) throws IOException { final File tempNarFile = persistenceProvider.createTempFile(inputStream); - final NarManifest manifest = getNarManifest(tempNarFile); - final BundleCoordinate coordinate = manifest.getCoordinate(); + try { + final NarManifest manifest = getNarManifest(tempNarFile); + final BundleCoordinate coordinate = manifest.getCoordinate(); - final Bundle existingBundle = extensionManager.getBundle(coordinate); - final boolean previouslyExistedInExtensionManager = existingBundle != null; - final boolean previouslyExistedInPersistenceProvider = persistenceProvider.exists(coordinate); + final Bundle existingBundle = extensionManager.getBundle(coordinate); + final boolean previouslyExistedInExtensionManager = existingBundle != null; + final boolean previouslyExistedInPersistenceProvider = persistenceProvider.exists(coordinate); - if (previouslyExistedInExtensionManager && !previouslyExistedInPersistenceProvider) { - deleteFileQuietly(tempNarFile); - throw new IllegalStateException("Another NAR is registered with the same coordinate and can not be replaced because it is not part of the NAR Manager"); - } + if (previouslyExistedInExtensionManager && !previouslyExistedInPersistenceProvider) { + deleteFileQuietly(tempNarFile); + throw new IllegalStateException("Another NAR is registered with the same coordinate and can not be replaced because it is not part of the NAR Manager"); + } - final NarPersistenceContext persistenceContext = NarPersistenceContext.builder() - .manifest(manifest) - .source(NarSource.UPLOAD) - .sourceIdentifier(NarSource.UPLOAD.name().toLowerCase()) - .build(); + final NarPersistenceContext persistenceContext = NarPersistenceContext.builder() + .manifest(manifest) + .source(NarSource.UPLOAD) + .sourceIdentifier(NarSource.UPLOAD.name().toLowerCase()) + .build(); - final File narFile = persistenceProvider.saveNar(persistenceContext, tempNarFile); + final NarPersistenceInfo narPersistenceInfo = persistenceProvider.saveNar(persistenceContext, tempNarFile); + final File narFile = narPersistenceInfo.getNarFile(); - final StoppedComponents stoppedComponents = new StoppedComponents(controllerServiceProvider); - if (previouslyExistedInExtensionManager) { - LOGGER.info("Unloading NAR and components for coordinate [{}] in order to replace NAR", coordinate); - narLoader.unload(existingBundle); - unloadComponents(coordinate, stoppedComponents); - } + final StoppedComponents stoppedComponents = new StoppedComponents(controllerServiceProvider); + if (previouslyExistedInExtensionManager) { + LOGGER.info("Unloading NAR and components for coordinate [{}] in order to replace NAR", coordinate); + narLoader.unload(existingBundle); + unloadComponents(coordinate, stoppedComponents); + } - // Load the NAR and attempt to un-ghost any components that can be provided by one of the loaded NARs, this handles a general ghosting case where - // the NAR now becomes available, as well as restoring any component that may have been purposely unloaded above for replacing an existing NAR - final NarLoadResult narLoadResult = narLoader.load(Collections.singleton(narFile), ALLOWED_EXTENSION_TYPES); - for (final Bundle loadedBundle : narLoadResult.getLoadedBundles()) { - final BundleCoordinate loadedCoordinate = loadedBundle.getBundleDetails().getCoordinate(); - loadMissingComponents(loadedCoordinate, stoppedComponents); - } + // Load the NAR and attempt to un-ghost any components that can be provided by one of the loaded NARs, this handles a general ghosting case where + // the NAR now becomes available, as well as restoring any component that may have been purposely unloaded above for replacing an existing NAR + final NarLoadResult narLoadResult = narLoader.load(Collections.singleton(narFile), ALLOWED_EXTENSION_TYPES); + for (final Bundle loadedBundle : narLoadResult.getLoadedBundles()) { + final BundleCoordinate loadedCoordinate = loadedBundle.getBundleDetails().getCoordinate(); + loadMissingComponents(loadedCoordinate, stoppedComponents); + } - // Restore previously running/enabled components to their original state - stoppedComponents.startAll(); + // Restore previously running/enabled components to their original state + stoppedComponents.startAll(); - return coordinate; + return coordinate; + } finally { + if (tempNarFile.exists() && !tempNarFile.delete()) { + LOGGER.warn("Failed to delete temp NAR file at [{}], file must be cleaned up manually", tempNarFile.getAbsolutePath()); + } + } } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java index a82d7b995d0c..710d28cf820d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java @@ -26,14 +26,23 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; -import java.util.Collections; -import java.util.HashMap; +import java.time.Instant; +import java.util.Date; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Properties; +import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; /** * Standard implementation of {@link NarPersistenceProvider} that stores NARs in a directory on the local filesystem. @@ -46,16 +55,19 @@ public class StandardNarPersistenceProvider implements NarPersistenceProvider { private static final String TEMP_STORAGE_DIR = "temp"; private static final String INSTALLED_STORAGE_DIR = "installed"; - private static final String NAR_FILENAME_SEPARATOR = "::"; + private static final String NAR_PATH_FORMAT = "%s/%s/%s"; private static final String NAR_FILENAME_EXTENSION = ".nar"; - private static final String NAR_FILENAME_FORMAT = "%s" + NAR_FILENAME_SEPARATOR + "%s" + NAR_FILENAME_SEPARATOR + "%s" + NAR_FILENAME_EXTENSION; + private static final String NAR_FILENAME_FORMAT = "%s-%s" + NAR_FILENAME_EXTENSION; + private static final String NAR_PROPERTIES_FILE_EXTENSION = ".properties"; + private static final String NAR_PROPERTIES_FILE_FORMAT = "%s-%s" + NAR_PROPERTIES_FILE_EXTENSION; private volatile File storageLocation; private volatile File tempStorageLocation; private volatile File installedStorageLocation; + private final Map narPersistenceInfoMap = new ConcurrentHashMap<>(); @Override - public void initialize(final NarPersistenceProviderInitializationContext initializationContext) { + public void initialize(final NarPersistenceProviderInitializationContext initializationContext) throws IOException { final String storageLocationPropertyValue = getRequiredValue(initializationContext, STORAGE_LOCATION_PROPERTY); storageLocation = new File(storageLocationPropertyValue); try { @@ -75,6 +87,15 @@ public void initialize(final NarPersistenceProviderInitializationContext initial throw new RuntimeException("Unable to create installed storage location at [" + installedStorageLocation.getAbsolutePath() + "]"); } + final Set narPersistenceInfos = loadNarPersistenceInfo(); + LOGGER.info("Loaded persistence info for {} NARs", narPersistenceInfos.size()); + + narPersistenceInfoMap.clear(); + narPersistenceInfos.forEach(narPersistenceInfo -> { + final BundleCoordinate narCoordinate = narPersistenceInfo.getNarProperties().getCoordinate(); + narPersistenceInfoMap.put(narCoordinate, narPersistenceInfo); + }); + LOGGER.info("NarManager initialization completed - NARs will be stored at [{}]", storageLocation.getAbsolutePath()); } @@ -93,77 +114,106 @@ public File createTempFile(final InputStream inputStream) throws IOException { } @Override - public File saveNar(final NarPersistenceContext persistenceContext, final File tempNarFile) throws IOException { + public NarPersistenceInfo saveNar(final NarPersistenceContext persistenceContext, final File tempNarFile) throws IOException { final NarManifest manifest = persistenceContext.getManifest(); final BundleCoordinate coordinate = manifest.getCoordinate(); - final File narFile = getFile(coordinate); + final File narVersionDirectory = getNarVersionDirectory(coordinate); + if (!narVersionDirectory.exists() && !narVersionDirectory.mkdirs()) { + throw new IOException("Unable to create NAR directories: " + narVersionDirectory.getAbsolutePath()); + } + + final File narFile = getNarFile(narVersionDirectory, coordinate); try { Files.move(tempNarFile.toPath(), narFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + LOGGER.info("Saved NAR to [{}]", narFile.getAbsolutePath()); } catch (final IOException e) { throw new IOException("Failed to move NAR from [" + tempNarFile.getAbsolutePath() + "] to [" + narFile.getAbsolutePath() + "]", e); } - LOGGER.info("Saved NAR to [{}]", narFile.getAbsolutePath()); - return narFile; + final NarProperties narProperties = createNarProperties(persistenceContext); + + final File narPropertiesFile = getNarProperties(narVersionDirectory, coordinate); + try (final OutputStream outputStream = new FileOutputStream(narPropertiesFile)) { + final Properties properties = narProperties.toProperties(); + properties.store(outputStream, null); + LOGGER.info("Saved NAR Properties to [{}]", narPropertiesFile.getAbsolutePath()); + } catch (final IOException e) { + throw new IOException("Failed to create NAR properties file for [" + coordinate + "]", e); + } + + final NarPersistenceInfo narPersistenceInfo = new NarPersistenceInfo(narFile, narProperties); + narPersistenceInfoMap.put(narPersistenceInfo.getNarProperties().getCoordinate(), narPersistenceInfo); + return narPersistenceInfo; } @Override public void deleteNar(final BundleCoordinate narCoordinate) throws IOException { - final File file = getFile(narCoordinate); - if (!file.exists()) { - throw new FileNotFoundException("NAR file [" + file.getAbsolutePath() + "] does not exist"); + narPersistenceInfoMap.remove(narCoordinate); + + final File narVersionDirectory = getNarVersionDirectory(narCoordinate); + final File narPropertiesFile = getNarProperties(narVersionDirectory, narCoordinate); + final File narFile = getNarFile(narVersionDirectory, narCoordinate); + + if (narPropertiesFile.exists()) { + try { + Files.delete(narPropertiesFile.toPath()); + LOGGER.info("Deleted NAR Properties [{}]", narPropertiesFile.getAbsolutePath()); + } catch (final IOException e) { + throw new IOException("Failed to delete NAR Properties [" + narPropertiesFile.getAbsolutePath() + "]", e); + } } - try { - Files.delete(file.toPath()); - LOGGER.info("Deleted NAR [{}]", file.getAbsolutePath()); - } catch (final IOException e) { - throw new IOException("Failed to delete NAR [" + file.getAbsolutePath() + "]", e); + if (narFile.exists()) { + try { + Files.delete(narFile.toPath()); + LOGGER.info("Deleted NAR [{}]", narFile.getAbsolutePath()); + } catch (final IOException e) { + throw new IOException("Failed to delete NAR [" + narFile.getAbsolutePath() + "]", e); + } + } + + if (narVersionDirectory.exists()) { + try { + Files.delete(narVersionDirectory.toPath()); + LOGGER.info("Deleted NAR directory [{}]", narVersionDirectory.getAbsolutePath()); + } catch (final IOException e) { + throw new IOException("Failed to delete NAR directory [" + narVersionDirectory.getAbsolutePath() + "]", e); + } } } @Override public InputStream readNar(final BundleCoordinate narCoordinate) throws FileNotFoundException { - final File file = getFile(narCoordinate); - if (!file.exists()) { - throw new FileNotFoundException("NAR file [" + file.getAbsolutePath() + "] does not exist"); + final File narVersionDirectory = getNarVersionDirectory(narCoordinate); + final File narFile = getNarFile(narVersionDirectory, narCoordinate); + if (!narFile.exists()) { + throw new FileNotFoundException("NAR file [" + narFile.getAbsolutePath() + "] does not exist"); } - return new FileInputStream(file); + return new FileInputStream(narFile); } @Override - public boolean exists(final BundleCoordinate narCoordinate) { - final File file = getFile(narCoordinate); - return file.exists(); + public NarPersistenceInfo getNarInfo(final BundleCoordinate narCoordinate) throws IOException { + final File narVersionDirectory = getNarVersionDirectory(narCoordinate); + final File narPropertiesFile = getNarProperties(narVersionDirectory, narCoordinate); + if (!narPropertiesFile.exists()) { + throw new FileNotFoundException("NAR Properties file [" + narPropertiesFile.getAbsolutePath() + "] does not exist"); + } + + final NarProperties narProperties = readNarProperties(narPropertiesFile); + final File narFile = getNarFile(narVersionDirectory, narCoordinate); + return new NarPersistenceInfo(narFile, narProperties); } @Override - public Map getNarFiles() { - final File[] files = installedStorageLocation.listFiles(f -> f.isFile() && f.getName().endsWith(NAR_FILENAME_EXTENSION)); - if (files == null || files.length == 0) { - return Collections.emptyMap(); - } - - final Map narCoordinateMap = new HashMap<>(); - for (final File narFile : files) { - final BundleCoordinate coordinate = getCoordinate(narFile); - if (coordinate != null) { - narCoordinateMap.put(coordinate, narFile); - } - } - return narCoordinateMap; + public boolean exists(final BundleCoordinate narCoordinate) { + return narPersistenceInfoMap.containsKey(narCoordinate); } - private BundleCoordinate getCoordinate(final File narFile) { - final String filenameWithoutExtension = narFile.getName().replace(NAR_FILENAME_EXTENSION, ""); - final String[] filenameParts = filenameWithoutExtension.split(NAR_FILENAME_SEPARATOR); - if (filenameParts.length == 3) { - return new BundleCoordinate(filenameParts[0], filenameParts[1], filenameParts[2]); - } else { - LOGGER.warn("Unable to determine coordinate from unexpected NAR filename: {}", narFile.getName()); - return null; - } + @Override + public Set getAllNarInfo() { + return new HashSet<>(narPersistenceInfoMap.values()); } @Override @@ -171,9 +221,19 @@ public void shutdown() { } - private File getFile(final BundleCoordinate coordinate) { - final String filename = NAR_FILENAME_FORMAT.formatted(coordinate.getGroup(), coordinate.getId(), coordinate.getVersion()); - return new File(installedStorageLocation, filename); + private File getNarVersionDirectory(final BundleCoordinate coordinate) { + final String path = NAR_PATH_FORMAT.formatted(coordinate.getGroup(), coordinate.getId(), coordinate.getVersion()); + return new File(installedStorageLocation, path); + } + + private File getNarFile(final File directory, final BundleCoordinate coordinate) { + final String filename = NAR_FILENAME_FORMAT.formatted(coordinate.getId(), coordinate.getVersion()); + return new File(directory, filename); + } + + private File getNarProperties(final File directory, final BundleCoordinate coordinate) { + final String filename = NAR_PROPERTIES_FILE_FORMAT.formatted(coordinate.getId(), coordinate.getVersion()); + return new File(directory, filename); } private File getTempFile() { @@ -188,4 +248,59 @@ private String getRequiredValue(final NarPersistenceProviderInitializationContex } return value; } + + private NarProperties createNarProperties(final NarPersistenceContext persistenceContext) { + final NarManifest narManifest = persistenceContext.getManifest(); + + return NarProperties.builder() + .sourceType(persistenceContext.getSource().name()) + .sourceId(persistenceContext.getSourceIdentifier()) + .narGroup(narManifest.getGroup()) + .narId(narManifest.getId()) + .narVersion(narManifest.getVersion()) + .narDependencyGroup(narManifest.getDependencyGroup()) + .narDependencyId(narManifest.getDependencyId()) + .narDependencyVersion(narManifest.getDependencyVersion()) + .created(Instant.now()) + .build(); + } + + public Set loadNarPersistenceInfo() throws IOException { + try (final Stream pathStream = Files.walk(installedStorageLocation.toPath())) { + final List narPropertyFiles = pathStream.filter(Files::isRegularFile) + .map(Path::toFile) + .filter(file -> file.getName().endsWith(NAR_PROPERTIES_FILE_EXTENSION)) + .toList(); + + final Set narPersistenceInfos = new HashSet<>(); + for (final File narPropertiesFile : narPropertyFiles) { + try { + final NarPersistenceInfo narPersistenceInfo = loadNarPersistenceInfo(narPropertiesFile); + narPersistenceInfos.add(narPersistenceInfo); + } catch (final Exception e) { + LOGGER.warn("Failed to load persistence info for NAR Properties [{}]", narPropertiesFile.getAbsolutePath(), e); + } + } + return narPersistenceInfos; + } + } + + private NarPersistenceInfo loadNarPersistenceInfo(final File narPropertiesFile) throws IOException { + LOGGER.debug("Loading persistence info from NAR Properties [{}]", narPropertiesFile.getAbsolutePath()); + + final NarProperties narProperties = readNarProperties(narPropertiesFile); + final BundleCoordinate narCoordinate = narProperties.getCoordinate(); + LOGGER.debug("Loaded NAR Properties for [{}]", narCoordinate); + + final File narVersionDirectory = getNarVersionDirectory(narCoordinate); + final File narFile = getNarFile(narVersionDirectory, narCoordinate); + return new NarPersistenceInfo(narFile, narProperties); + } + + private NarProperties readNarProperties(final File narPropertiesFile) throws IOException { + try (final InputStream inputStream = new FileInputStream(narPropertiesFile)) { + return NarProperties.parse(inputStream); + } + } + }