[JENKINS-62757] Introduce fingerprint migration in External Storage API

core/src/main/java/hudson/model/Fingerprint.java

@@ -46,6 +46,7 @@
 
 import jenkins.fingerprints.FileFingerprintStorage;
 import jenkins.fingerprints.FingerprintStorage;
+import jenkins.fingerprints.GlobalFingerprintConfiguration;
 import jenkins.model.FingerprintFacet;
 import jenkins.model.Jenkins;
 import jenkins.model.TransientFingerprintFacetFactory;
@@ -1251,9 +1252,20 @@ public synchronized void save() throws IOException {
         if(logger.isLoggable(Level.FINE))
             start = System.currentTimeMillis();
 
+        FingerprintStorage configuredFingerprintStorage = FingerprintStorage.get();
+        FingerprintStorage fileFingerprintStorage = FingerprintStorage.getFileFingerprintStorage();
+
         // Implementations are expected to invoke SaveableListener on their own if relevant
         // TODO: Consider improving Saveable Listener API: https://issues.jenkins-ci.org/browse/JENKINS-62543
-        FingerprintStorage.get().save(this);
+        configuredFingerprintStorage.save(this);
+
+        // In the case that external fingerprint storage is configured, there may be some fingerprints in memory that
+        // get saved before a load call (because they are already in memory). This ensures that they get deleted from
+        // the file fingerprint storage.
+        // TODO: Consider improving KeyedDataStorage so it provides an API for clearing the fingerprints in memory.
+        if (!(configuredFingerprintStorage instanceof FileFingerprintStorage) && fileFingerprintStorage.isReady()) {
+            fileFingerprintStorage.delete(this.getHashString());
+        }
 
         if(logger.isLoggable(Level.FINE))
             logger.fine("Saving fingerprint " + getHashString() + " took " + (System.currentTimeMillis() - start) + "ms");
@@ -1316,15 +1328,33 @@ public Api getApi() {
      * Loads a {@link Fingerprint} from the Storage with the given unique id.
      * @return Loaded {@link Fingerprint}. {@code null} if the config file does not exist or
      * malformed.
+     *
+     * In case an external storage is configured on top of a file system based storage:
+     * 1. External storage is polled to retrieve the fingerprint
+     * 2. If not found, then the local storage is polled to retrieve the fingerprint
      */
     public static @CheckForNull Fingerprint load(@NonNull String id) throws IOException {
         long start=0;
         if(logger.isLoggable(Level.FINE)) {
             start = System.currentTimeMillis();
         }
 
-        Fingerprint loaded = FingerprintStorage.get().load(id);
-        initFacets(loaded);
+        FingerprintStorage configuredFingerprintStorage = FingerprintStorage.get();
+        FingerprintStorage fileFingerprintStorage = FileFingerprintStorage.getFileFingerprintStorage();
+
+        Fingerprint loaded = configuredFingerprintStorage.load(id);
+
+        if (loaded == null && !(configuredFingerprintStorage instanceof FileFingerprintStorage) &&
+                fileFingerprintStorage.isReady()) {
+            loaded = fileFingerprintStorage.load(id);
+            if (loaded != null) {
+                initFacets(loaded);
+                configuredFingerprintStorage.save(loaded);
+                fileFingerprintStorage.delete(id);
+            }
+        } else {
+            initFacets(loaded);
+        }
 
         if(logger.isLoggable(Level.FINE)) {
             logger.fine("Loading fingerprint took " + (System.currentTimeMillis() - start) + "ms");
@@ -1357,12 +1387,19 @@ public Api getApi() {
     }
 
     /**
-     * Deletes the {@link Fingerprint} in the Storage with the given unique id.
+     * Deletes the {@link Fingerprint} in the configured storage with the given unique id.
      *
      * @since TODO
      */
     public static void delete(@NonNull String id) throws IOException {
-        FingerprintStorage.get().delete(id);
+        FingerprintStorage configuredFingerprintStorage = FingerprintStorage.get();
+        FingerprintStorage fileFingerprintStorage = FingerprintStorage.getFileFingerprintStorage();
+
+        configuredFingerprintStorage.delete(id);
+
+        if (!(configuredFingerprintStorage instanceof FileFingerprintStorage) && fileFingerprintStorage.isReady()) {
+            fileFingerprintStorage.delete(id);
+        }
     }
 
     /**

test/src/test/java/jenkins/fingerprints/FingerprintStorageTest.java

@@ -0,0 +1,130 @@
+/*
+ * The MIT License
+ *
+ * Copyright (c) 2020, Jenkins project contributors
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package jenkins.fingerprints;
+
+import hudson.Util;
+import hudson.model.Fingerprint;
+import hudson.model.FingerprintCleanupThreadTest;
+import org.junit.Rule;
+import org.junit.Test;
+import org.jvnet.hudson.test.JenkinsRule;
+
+import java.io.IOException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.nullValue;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.IsNot.not;
+
+public class FingerprintStorageTest {
+
+    @Rule
+    public JenkinsRule jenkinsRule = new JenkinsRule();
+
+    @Test
+    public void testLoadingAndSavingLocalStorageFingerprintWithExternalStorage() throws IOException {
+        String id = Util.getDigestOf("testLoadingAndSavingLocalStorageFingerprintWithExternalStorage");
+        Fingerprint fingerprintSaved = new Fingerprint(null, "foo.jar", Util.fromHexString(id));
+        Fingerprint fingerprintLoaded = Fingerprint.load(id);
+        assertThat(fingerprintLoaded, is(not(nullValue())));
+        assertThat(fingerprintSaved.toString(), is(equalTo(fingerprintLoaded.toString())));
+
+        // After external storage is configured, check if local storage fingerprint is still accessible.
+        FingerprintStorage externalFingerprintStorage = configureExternalStorage();
+        fingerprintLoaded = Fingerprint.load(id);
+        assertThat(fingerprintLoaded, is(not(nullValue())));
+        assertThat(fingerprintSaved.toString(), is(equalTo(fingerprintLoaded.toString())));
+
+        // After loading the fingerprint, ensure it was moved to external storage.
+        fingerprintLoaded = externalFingerprintStorage.load(id);
+        assertThat(fingerprintLoaded, is(not(nullValue())));
+        assertThat(fingerprintSaved.toString(), is(equalTo(fingerprintLoaded.toString())));
+
+        // Ensure that the loaded fingerprint was deleted from local storage after being loaded.
+        fingerprintLoaded = FingerprintStorage.getFileFingerprintStorage().load(id);
+        assertThat(fingerprintLoaded, is(nullValue()));
+    }
+
+    @Test
+    public void testLoadingAndSavingFingerprintWithExternalStorage() throws IOException {
+        FingerprintStorage externalFingerprintStorage = configureExternalStorage();
+        String id = Util.getDigestOf("testLoadingAndSavingFingerprintWithExternalStorage");
+        Fingerprint fingerprintSaved = new Fingerprint(null, "bar.jar", Util.fromHexString(id));
+        Fingerprint fingerprintLoaded = externalFingerprintStorage.load(id);
+        assertThat(fingerprintLoaded, is(not(nullValue())));
+        assertThat(fingerprintSaved.toString(), is(equalTo(fingerprintLoaded.toString())));
+    }
+
+    @Test
+    public void testDeletingLocalStorageFingerprintWithExternalStorageBeforeMigration() throws IOException {
+        String id = Util.getDigestOf("testLoadingAndSavingLocalStorageFingerprintWithExternalStorage");
+        new Fingerprint(null, "foo.jar", Util.fromHexString(id));
+        configureExternalStorage();
+        Fingerprint.delete(id);
+        Fingerprint fingerprintLoaded = FingerprintStorage.getFileFingerprintStorage().load(id);
+        assertThat(fingerprintLoaded, is(nullValue()));
+    }
+
+    @Test
+    public void testDeletingLocalStorageFingerprintWithExternalStorageAfterMigration() throws IOException {
+        String id = Util.getDigestOf("testLoadingAndSavingLocalStorageFingerprintWithExternalStorage");
+        new Fingerprint(null, "foo.jar", Util.fromHexString(id));
+        FingerprintStorage externalFingerprintStorage = configureExternalStorage();
+        Fingerprint.load(id);
+        Fingerprint.delete(id);
+        Fingerprint fingerprintLoaded = externalFingerprintStorage.load(id);
+        assertThat(fingerprintLoaded, is(nullValue()));
+    }
+
+    @Test
+    public void testDeletingFingerprintWithExternalStorage() throws IOException {
+        configureExternalStorage();
+        String id = Util.getDigestOf("testLoadingAndSavingLocalStorageFingerprintWithExternalStorage");
+        new Fingerprint(null, "foo.jar", Util.fromHexString(id));
+        Fingerprint.delete(id);
+        Fingerprint fingerprintLoaded = Fingerprint.load(id);
+        assertThat(fingerprintLoaded, is(nullValue()));
+    }
+
+    @Test
+    public void testMigrationDeletesFingerprintsInMemoryFromFileStorage() throws IOException {
+        String id = Util.getDigestOf("testMigrationDeletesFingerprintsInMemoryFromFileStorage");
+        Fingerprint fingerprintSaved = new Fingerprint(null, "foo.jar", Util.fromHexString(id));
+        configureExternalStorage();
+        fingerprintSaved.add("test", 3);
+        // This fingerprint is now implicitly saved without making a load call.
+        // We want the file storage to not have this fingerprint now.
+        Fingerprint fingerprintLoaded = FingerprintStorage.getFileFingerprintStorage().load(id);
+        assertThat(fingerprintLoaded, is(nullValue()));
+
+    }
+
+    private FingerprintStorage configureExternalStorage() {
+        FingerprintStorage fingerprintStorage = new FingerprintCleanupThreadTest.TestExternalFingerprintStorage();
+        GlobalFingerprintConfiguration.get().setStorage(fingerprintStorage);
+        return fingerprintStorage;
+    }
+
+}