diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/nar/NarManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/nar/NarManager.java index 5bc17a1e3cf00..21a405c5bdd5e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/nar/NarManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/nar/NarManager.java @@ -24,7 +24,7 @@ public interface NarManager { - BundleCoordinate saveNar(String filename, InputStream inputStream) throws IOException; + BundleCoordinate addNar(String filename, InputStream inputStream) throws IOException; boolean deleteNar(BundleCoordinate narCoordinate) throws IOException; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index e56ef95d67348..077b390022fb5 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -96,6 +96,11 @@ nifi-framework-nar-utils 2.0.0-SNAPSHOT + + org.apache.nifi + nifi-framework-nar-loading-utils + 2.0.0-SNAPSHOT + org.apache.nifi.registry diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java index 9f2fa1f216057..7ea4023bf5cb3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java @@ -37,15 +37,17 @@ public class StandardNarManager implements NarManager { private final FlowManager flowManager; private final ExtensionManager extensionManager; private final NarPersistenceProvider persistenceProvider; + private final NarLoader narLoader; - public StandardNarManager(final FlowController flowController) { + public StandardNarManager(final FlowController flowController, final NarLoader narLoader) { this.flowManager = flowController.getFlowManager(); this.extensionManager = flowController.getExtensionManager(); this.persistenceProvider = flowController.getNarPersistenceProvider(); + this.narLoader = narLoader; } @Override - public BundleCoordinate saveNar(final String filename, final InputStream inputStream) throws IOException { + public BundleCoordinate addNar(final String filename, final InputStream inputStream) throws IOException { final File tempFile = Files.createTempFile("nifi", "nar").toFile(); LOGGER.debug("Created temporary file {} to hold contents of NAR for {}", tempFile.getAbsolutePath(), filename); try { @@ -67,6 +69,7 @@ public BundleCoordinate saveNar(final String filename, final InputStream inputSt @Override public boolean deleteNar(final BundleCoordinate narCoordinate) throws IOException { + // TODO verify the bundle exists, no components are instantiate from it, then remove from extension manager return persistenceProvider.deleteNar(narCoordinate); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java index 98dc79c9cac4b..e07dfcd0c1115 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarPersistenceProvider.java @@ -74,8 +74,6 @@ public void initialize(final NarPersistenceProviderInitializationContext initial } private void loadNarInformation() { - // TODO what if a NAR is copied into this location manually and doesn't match the filename format being used here? - // we'll end up loading the NAR info, but then a call to delete or stream would fail final File[] files = storageLocation.listFiles(f -> f.isFile() && f.getName().endsWith(NAR_FILENAME_EXTENSION)); if (files == null) { LOGGER.info("No existing NARs found at [{}]", storageLocation.getAbsolutePath()); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/NarLoaderFactoryBean.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/NarLoaderFactoryBean.java new file mode 100644 index 0000000000000..0f122f8261be1 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/spring/NarLoaderFactoryBean.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.spring; + +import org.apache.nifi.nar.NarLoader; +import org.apache.nifi.nar.NarLoaderHolder; +import org.springframework.beans.factory.FactoryBean; + +/** + * Factory bean that provides the NarLoader instance from the holder. + */ +public class NarLoaderFactoryBean implements FactoryBean { + + @Override + public NarLoader getObject() throws Exception { + return NarLoaderHolder.getNarLoader(); + } + + @Override + public Class getObjectType() { + return NarLoader.class; + } + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml index 59097e4b17030..031b8dd66b1f6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/resources/nifi-context.xml @@ -31,11 +31,9 @@ - - + - - + @@ -89,8 +87,11 @@ + + - + + diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties index 74f2d13cebd20..6571285728e35 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties @@ -63,7 +63,7 @@ nifi.provenance.repository.rollover.time=30 secs nifi.provenance.repository.rollover.size=100 MB # NAR Persistence Provider Properties -nifi.nar.persistence.provider.properties.directory=./extensions +nifi.nar.persistence.provider.properties.directory=./nar_storage # Site to Site properties nifi.remote.input.socket.port=9990 diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarLoaderHolder.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarLoaderHolder.java new file mode 100644 index 0000000000000..b1543169ac9e4 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-loading-utils/src/main/java/org/apache/nifi/nar/NarLoaderHolder.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.nar; + +public class NarLoaderHolder { + + private static volatile NarLoader INSTANCE; + + public static void init(final NarLoader narLoader) { + if (INSTANCE == null) { + synchronized (NarLoader.class) { + if (INSTANCE == null) { + INSTANCE = narLoader; + } else { + throw new IllegalStateException("Cannot reinitialize NarLoaderHolder"); + } + } + } else { + throw new IllegalStateException("Cannot reinitialize NarLoaderHolder"); + } + } + + public static NarLoader getNarLoader() { + if (INSTANCE == null) { + synchronized (NarLoader.class) { + if (INSTANCE == null) { + throw new IllegalStateException("NarLoaderHolder was never initialized"); + } + } + } + + return INSTANCE; + } + + // Private access + private NarLoaderHolder() { + + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index 13438e8678645..b3a8ee179a058 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -119,7 +119,7 @@ org.apache.nifi.nar.StandardNarPersistenceProvider - ./extensions + ./nar_storage diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/pom.xml index 0923f7a8a45d9..79ae89a85d831 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/pom.xml @@ -60,11 +60,6 @@ nifi-framework-core-api 2.0.0-SNAPSHOT - - org.apache.nifi - nifi-framework-nar-loading-utils - 2.0.0-SNAPSHOT - org.apache.nifi nifi-framework-external-resource-utils diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java index b0a1ea77a55bd..adb60862b3109 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-jetty/src/main/java/org/apache/nifi/web/server/JettyServer.java @@ -83,6 +83,7 @@ import org.apache.nifi.nar.NarAutoLoader; import org.apache.nifi.nar.NarClassLoadersHolder; import org.apache.nifi.nar.NarLoader; +import org.apache.nifi.nar.NarLoaderHolder; import org.apache.nifi.nar.NarThreadContextClassLoader; import org.apache.nifi.nar.NarUnpackMode; import org.apache.nifi.nar.StandardExtensionDiscoveringManager; @@ -758,7 +759,7 @@ public void start() { .targetDirectory(new File(props.getProperty(NiFiProperties.NAR_LIBRARY_AUTOLOAD_DIRECTORY, NiFiProperties.DEFAULT_NAR_LIBRARY_AUTOLOAD_DIR))) .conflictResolutionStrategy(props.getProperty(NAR_PROVIDER_CONFLICT_RESOLUTION, DEFAULT_NAR_PROVIDER_CONFLICT_RESOLUTION)) .pollInterval(props.getProperty(NAR_PROVIDER_POLL_INTERVAL_PROPERTY, DEFAULT_NAR_PROVIDER_POLL_INTERVAL)) - .restrainingStartup(Boolean.parseBoolean(props.getProperty(NAR_PROVIDER_RESTRAIN_PROPERTY, "true"))) + .restrainingStartup(Boolean.parseBoolean(props.getProperty(NAR_PROVIDER_RESTRAIN_PROPERTY, "true"))) .build(); narProviderService.start(); @@ -774,6 +775,9 @@ public void start() { this, unpackMode); + // Set the NarLoader into the holder which makes it available to the Spring context via a factory bean + NarLoaderHolder.init(narLoader); + narAutoLoader = new NarAutoLoader(props, narLoader); narAutoLoader.start(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index b372d5cc83cd8..b701d5304db9e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -92,6 +92,7 @@ import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.BulletinEntity; +import org.apache.nifi.web.api.entity.BundleEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; import org.apache.nifi.web.api.entity.ConfigurationAnalysisEntity; import org.apache.nifi.web.api.entity.ConnectionEntity; @@ -144,6 +145,8 @@ import org.apache.nifi.web.api.entity.VersionedReportingTaskImportResponseEntity; import org.apache.nifi.web.api.request.FlowMetricsRegistry; +import java.io.IOException; +import java.io.InputStream; import java.util.Collection; import java.util.Date; import java.util.List; @@ -2821,4 +2824,10 @@ ControllerServiceReferencingComponentsEntity updateControllerServiceReferencingC * @return rule violations produced by the analysis of the process group */ FlowAnalysisResultEntity getFlowAnalysisResult(String processGroupId); + + // ---------------------------------------- + // NAR Management methods + // ---------------------------------------- + + BundleEntity addNar(String filename, InputStream inputStream) throws IOException; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacadeLock.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacadeLock.java index 67c977f456b80..87d31907eab2b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacadeLock.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacadeLock.java @@ -114,6 +114,12 @@ public Object populateLock(ProceedingJoinPoint proceedingJoinPoint) throws Throw return proceedWithWriteLock(proceedingJoinPoint); } + @Around("within(org.apache.nifi.web.NiFiServiceFacade+) && " + + "execution(* add*(..))") + public Object addLock(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { + return proceedWithWriteLock(proceedingJoinPoint); + } + @Around("within(org.apache.nifi.web.NiFiServiceFacade+) && " + "execution(* get*(..))") public Object getLock(ProceedingJoinPoint proceedingJoinPoint) throws Throwable { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index dfa6f0df90a92..3cafdfd9d3bc3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -127,6 +127,7 @@ import org.apache.nifi.history.PreviousValue; import org.apache.nifi.metrics.jvm.JmxJvmMetrics; import org.apache.nifi.nar.ExtensionManager; +import org.apache.nifi.nar.NarManager; import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterContextLookup; @@ -286,6 +287,7 @@ import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.BulletinEntity; +import org.apache.nifi.web.api.entity.BundleEntity; import org.apache.nifi.web.api.entity.ComponentReferenceEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; import org.apache.nifi.web.api.entity.ConfigurationAnalysisEntity; @@ -380,6 +382,7 @@ import org.springframework.security.oauth2.core.OAuth2Token; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -462,8 +465,8 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private final ClusterMetricsRegistry clusterMetricsRegistry = new ClusterMetricsRegistry(); private RuleViolationsManager ruleViolationsManager; - private PredictionBasedParallelProcessingService parallelProcessingService; + private NarManager narManager; // ----------------------------------------- // Synchronization methods @@ -6562,6 +6565,13 @@ public FlowAnalysisResultEntity createFlowAnalysisResultEntity(Collection + @@ -421,10 +422,7 @@ - - - diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties index 7a071789b3411..c8a06649b53b5 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties @@ -128,7 +128,7 @@ nifi.components.status.repository.buffer.size=1440 nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties -nifi.nar.persistence.provider.properties.directory=./extensions +nifi.nar.persistence.provider.properties.directory=./nar_storage # Site to Site properties nifi.remote.input.host= diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties index c2139237fb5a8..f272cd5519e71 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties @@ -128,7 +128,7 @@ nifi.components.status.repository.buffer.size=1440 nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties -nifi.nar.persistence.provider.properties.directory=./extensions +nifi.nar.persistence.provider.properties.directory=./nar_storage # Site to Site properties nifi.remote.input.host= diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties index 3d9fef8b14a52..117ff2b453be6 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties @@ -129,7 +129,7 @@ nifi.components.status.repository.buffer.size=1440 nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties -nifi.nar.persistence.provider.properties.directory=./extensions +nifi.nar.persistence.provider.properties.directory=./nar_storage # Site to Site properties nifi.remote.input.host= diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties index 4eddee10ad1f2..91b132a24b6a9 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties @@ -133,7 +133,7 @@ nifi.components.status.repository.buffer.size=1440 nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties -nifi.nar.persistence.provider.properties.directory=./extensions +nifi.nar.persistence.provider.properties.directory=./nar_storage # Site to Site properties nifi.remote.input.host=