diff --git a/core/deployment/src/main/java/io/quarkus/deployment/recording/BytecodeRecorderImpl.java b/core/deployment/src/main/java/io/quarkus/deployment/recording/BytecodeRecorderImpl.java
index cb7c478af64c0..5704944d44a29 100644
--- a/core/deployment/src/main/java/io/quarkus/deployment/recording/BytecodeRecorderImpl.java
+++ b/core/deployment/src/main/java/io/quarkus/deployment/recording/BytecodeRecorderImpl.java
@@ -45,6 +45,7 @@
 import org.jboss.jandex.Type;
 import org.wildfly.common.Assert;
 
+import io.quarkus.bootstrap.classloading.QuarkusClassLoader;
 import io.quarkus.deployment.proxy.ProxyConfiguration;
 import io.quarkus.deployment.proxy.ProxyFactory;
 import io.quarkus.deployment.recording.AnnotationProxyProvider.AnnotationProxy;
@@ -339,6 +340,15 @@ public Object invoke(Object proxy, Method method, Object[] args) throws Throwabl
             T recordingProxy = factory.newInstance(invocationHandler);
             existingProxyClasses.put(theClass, recordingProxy);
             recordingProxyFactories.put(theClass, factory);
+
+            if (theClass.getClassLoader() instanceof QuarkusClassLoader) {
+                ((QuarkusClassLoader) theClass.getClassLoader()).addCloseTask(new Runnable() {
+                    @Override
+                    public void run() {
+                        recordingProxyFactories.remove(theClass);
+                    }
+                });
+            }
             return recordingProxy;
         } catch (IllegalAccessException | InstantiationException e) {
             throw new RuntimeException(e);