Skip to content

Commit

Permalink
Integrated TopToc into repository
Browse files Browse the repository at this point in the history
  • Loading branch information
@phax
phax committed Dec 18, 2023
1 parent f7f4eea commit 6d4754d
Show file tree
Hide file tree
Showing 22 changed files with 523 additions and 160 deletions.
11 changes: 6 additions & 5 deletions ph-diver-repo-http/src/main/java/com/helger/diver/repo/http/RepoStorageHttp.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,9 @@
import com.helger.diver.repo.ERepoWritable;
import com.helger.diver.repo.IRepoStorage;
import com.helger.diver.repo.RepoStorageKey;
import com.helger.diver.repo.RepoStorageKeyOfArtefact;
import com.helger.diver.repo.RepoStorageType;
import com.helger.diver.repo.impl.AbstractRepoStorageWithToc;
import com.helger.diver.repo.toc.IRepoTopTocService;
import com.helger.httpclient.HttpClientManager;
import com.helger.httpclient.response.ResponseHandlerByteArray;

Expand All @@ -64,9 +64,10 @@ public RepoStorageHttp (@Nonnull @WillNotClose final HttpClientManager aHttpClie
@Nonnull @Nonempty final String sURLPrefix,
@Nonnull @Nonempty final String sID,
@Nonnull final ERepoWritable eWriteEnabled,
@Nonnull final ERepoDeletable eDeleteEnabled)
@Nonnull final ERepoDeletable eDeleteEnabled,
@Nonnull final IRepoTopTocService aTopTocService)
{
super (RepoStorageType.HTTP, sID, eWriteEnabled, eDeleteEnabled);
super (RepoStorageType.HTTP, sID, eWriteEnabled, eDeleteEnabled, aTopTocService);
ValueEnforcer.notNull (aHttpClient, "HttpClient");
ValueEnforcer.notEmpty (sURLPrefix, "URLPrefix");
m_aHttpClient = aHttpClient;
Expand Down Expand Up @@ -108,7 +109,7 @@ protected InputStream getInputStream (@Nonnull final RepoStorageKey aKey)

@Override
@Nonnull
protected ESuccess writeObject (@Nonnull final RepoStorageKeyOfArtefact aKey, @Nonnull final byte [] aPayload)
protected ESuccess writeObject (@Nonnull final RepoStorageKey aKey, @Nonnull final byte [] aPayload)
{
final String sURL = FilenameHelper.getCleanConcatenatedUrlPath (m_sURLPrefix, aKey.getPath ());
if (LOGGER.isInfoEnabled ())
Expand Down Expand Up @@ -139,7 +140,7 @@ protected ESuccess writeObject (@Nonnull final RepoStorageKeyOfArtefact aKey, @N

@Override
@Nonnull
protected ESuccess deleteObject (@Nonnull final RepoStorageKeyOfArtefact aKey)
protected ESuccess deleteObject (@Nonnull final RepoStorageKey aKey)
{
final String sURL = FilenameHelper.getCleanConcatenatedUrlPath (m_sURLPrefix, aKey.getPath ());
if (LOGGER.isInfoEnabled ())
Expand Down
30 changes: 20 additions & 10 deletions ph-diver-repo-http/src/test/java/com/helger/diver/repo/http/RepoStorageChainFuncTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import com.helger.diver.repo.http.mock.MockRepoStorageLocalFileSystem;
import com.helger.diver.repo.impl.RepoStorageInMemory;
import com.helger.diver.repo.impl.RepoStorageLocalFileSystem;
import com.helger.diver.repo.toc.RepoTopTocServiceRepoBasedXML;
import com.helger.httpclient.HttpClientManager;

/**
Expand Down Expand Up @@ -82,7 +83,8 @@ public void testReadAndCacheAndRead ()
LocalJettyRunner.DEFAULT_ACCESS_URL,
"unittest-http",
ERepoWritable.WITHOUT_WRITE,
ERepoDeletable.WITHOUT_DELETE);
ERepoDeletable.WITHOUT_DELETE,
new RepoTopTocServiceRepoBasedXML ());

final RepoStorageChain aRepoChain = RepoStorageChain.of (new CommonsArrayList <> (aRepoInMemory,
aRepoLocalFS,
Expand Down Expand Up @@ -119,23 +121,30 @@ public void testReadAndCacheAndRead ()
}
finally
{
final File fBase = MockRepoStorageLocalFileSystem.TEST_REPO_DIR;

// Cleanup from local FS
File f = new File (MockRepoStorageLocalFileSystem.TEST_REPO_DIR, "com/ecosio/http-only/1/http-only-1.txt");
File f = new File (fBase, "com/ecosio/http-only/1/http-only-1.txt");
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (MockRepoStorageLocalFileSystem.TEST_REPO_DIR,
"com/ecosio/http-only/1/http-only-1.txt" + RepoStorageKey.SUFFIX_SHA256);
f = new File (fBase, "com/ecosio/http-only/1/http-only-1.txt" + RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);

// Delete ToC as well
f = new File (MockRepoStorageLocalFileSystem.TEST_REPO_DIR,
"com/ecosio/http-only/" + RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML);
f = new File (fBase, "com/ecosio/http-only/" + RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML);
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (MockRepoStorageLocalFileSystem.TEST_REPO_DIR,
f = new File (fBase,
"com/ecosio/http-only/" +
RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML +
RepoStorageKey.SUFFIX_SHA256);
RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML +
RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);

// Delete Top-ToC as well
f = new File (fBase, RepoTopTocServiceRepoBasedXML.FILENAME_TOP_TOC_DIVER_XML);
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (fBase, RepoTopTocServiceRepoBasedXML.FILENAME_TOP_TOC_DIVER_XML + RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);
}
}
Expand All @@ -155,7 +164,8 @@ public void testReadNoCacheAndRead ()
LocalJettyRunner.DEFAULT_ACCESS_URL,
"unittest-http",
ERepoWritable.WITHOUT_WRITE,
ERepoDeletable.WITHOUT_DELETE);
ERepoDeletable.WITHOUT_DELETE,
new RepoTopTocServiceRepoBasedXML ());
final RepoStorageChain aChain = RepoStorageChain.of (new CommonsArrayList <> (aInMemory, aLocalFS, aHttp),
new CommonsArrayList <> (aInMemory, aLocalFS))
.setCacheRemoteContent (false);
Expand Down
30 changes: 20 additions & 10 deletions ph-diver-repo-http/src/test/java/com/helger/diver/repo/http/RepoStorageHttpTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import com.helger.diver.repo.RepoStorageKey;
import com.helger.diver.repo.RepoStorageKeyOfArtefact;
import com.helger.diver.repo.http.mock.LocalJettyRunner;
import com.helger.diver.repo.toc.RepoTopTocServiceRepoBasedXML;
import com.helger.httpclient.HttpClientManager;

/**
Expand Down Expand Up @@ -73,7 +74,8 @@ private static RepoStorageHttp _createRepoReadOnly ()
LocalJettyRunner.DEFAULT_ACCESS_URL,
"unittest",
ERepoWritable.WITHOUT_WRITE,
ERepoDeletable.WITHOUT_DELETE);
ERepoDeletable.WITHOUT_DELETE,
new RepoTopTocServiceRepoBasedXML ());
}

@Test
Expand All @@ -100,7 +102,8 @@ private static RepoStorageHttp _createRepoWritable ()
LocalJettyRunner.DEFAULT_ACCESS_URL,
"unittest",
ERepoWritable.WITH_WRITE,
ERepoDeletable.WITH_DELETE);
ERepoDeletable.WITH_DELETE,
new RepoTopTocServiceRepoBasedXML ());
}

@Test
Expand Down Expand Up @@ -148,23 +151,30 @@ public void testWritableWriteAndRead ()
}
finally
{
final File fBase = LocalJettyRunner.DEFAULT_TEST_RESOURCE_BASE;

// Cleanup
File f = new File (LocalJettyRunner.DEFAULT_TEST_RESOURCE_BASE, "com/ecosio/http-written/1/http-written-1.txt");
File f = new File (fBase, "com/ecosio/http-written/1/http-written-1.txt");
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (LocalJettyRunner.DEFAULT_TEST_RESOURCE_BASE,
"com/ecosio/http-written/1/http-written-1.txt" + RepoStorageKey.SUFFIX_SHA256);
f = new File (fBase, "com/ecosio/http-written/1/http-written-1.txt" + RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);

// Delete ToC as well
f = new File (LocalJettyRunner.DEFAULT_TEST_RESOURCE_BASE,
"com/ecosio/http-written/" + RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML);
f = new File (fBase, "com/ecosio/http-written/" + RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML);
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (LocalJettyRunner.DEFAULT_TEST_RESOURCE_BASE,
f = new File (fBase,
"com/ecosio/http-written/" +
RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML +
RepoStorageKey.SUFFIX_SHA256);
RepoStorageKeyOfArtefact.FILENAME_TOC_DIVER_XML +
RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);

// Delete Top-ToC as well
f = new File (fBase, RepoTopTocServiceRepoBasedXML.FILENAME_TOP_TOC_DIVER_XML);
FileOperationManager.INSTANCE.deleteFile (f);

f = new File (fBase, RepoTopTocServiceRepoBasedXML.FILENAME_TOP_TOC_DIVER_XML + RepoStorageKey.SUFFIX_SHA256);
FileOperationManager.INSTANCE.deleteFile (f);
}
}
Expand Down
3 changes: 2 additions & 1 deletion ...po-http/src/test/java/com/helger/diver/repo/http/mock/MockRepoStorageLocalFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.helger.diver.repo.ERepoDeletable;
import com.helger.diver.repo.ERepoWritable;
import com.helger.diver.repo.impl.RepoStorageLocalFileSystem;
import com.helger.diver.repo.toc.RepoTopTocServiceRepoBasedXML;

public final class MockRepoStorageLocalFileSystem extends RepoStorageLocalFileSystem
{
Expand All @@ -31,6 +32,6 @@ public final class MockRepoStorageLocalFileSystem extends RepoStorageLocalFileSy
public MockRepoStorageLocalFileSystem (@Nonnull final ERepoWritable eWriteEnabled,
@Nonnull final ERepoDeletable eDeleteEnabled)
{
super (TEST_REPO_DIR, "unittest-fs", eWriteEnabled, eDeleteEnabled);
super (TEST_REPO_DIR, "unittest-fs", eWriteEnabled, eDeleteEnabled, new RepoTopTocServiceRepoBasedXML ());
}
}
11 changes: 6 additions & 5 deletions ph-diver-repo-s3/src/main/java/com/helger/diver/repo/s3/RepoStorageS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import com.helger.diver.repo.ERepoWritable;
import com.helger.diver.repo.IRepoStorage;
import com.helger.diver.repo.RepoStorageKey;
import com.helger.diver.repo.RepoStorageKeyOfArtefact;
import com.helger.diver.repo.RepoStorageType;
import com.helger.diver.repo.impl.AbstractRepoStorageWithToc;
import com.helger.diver.repo.toc.IRepoTopTocService;

import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
Expand Down Expand Up @@ -64,9 +64,10 @@ public RepoStorageS3 (@Nonnull final S3Client aS3Client,
@Nonnull @Nonempty final String sBucketName,
@Nonnull @Nonempty final String sID,
@Nonnull final ERepoWritable eWriteEnabled,
@Nonnull final ERepoDeletable eDeleteEnabled)
@Nonnull final ERepoDeletable eDeleteEnabled,
@Nonnull final IRepoTopTocService aTopTocService)
{
super (AWS_S3, sID, eWriteEnabled, eDeleteEnabled);
super (AWS_S3, sID, eWriteEnabled, eDeleteEnabled, aTopTocService);
ValueEnforcer.notNull (aS3Client, "S3Client");
ValueEnforcer.notEmpty (sBucketName, "BucketName");
ValueEnforcer.isEqual (sBucketName, sBucketName.trim (), "BucketName must be trimmed");
Expand Down Expand Up @@ -132,7 +133,7 @@ protected ResponseInputStream <GetObjectResponse> getInputStream (@Nonnull final

@Override
@Nonnull
protected ESuccess writeObject (@Nonnull final RepoStorageKeyOfArtefact aKey, @Nonnull final byte [] aPayload)
protected ESuccess writeObject (@Nonnull final RepoStorageKey aKey, @Nonnull final byte [] aPayload)
{
final String sRealKey = aKey.getPath ();

Expand Down Expand Up @@ -160,7 +161,7 @@ protected ESuccess writeObject (@Nonnull final RepoStorageKeyOfArtefact aKey, @N

@Override
@Nonnull
protected ESuccess deleteObject (@Nonnull final RepoStorageKeyOfArtefact aKey)
protected ESuccess deleteObject (@Nonnull final RepoStorageKey aKey)
{
ValueEnforcer.notNull (aKey, "Key");

Expand Down
30 changes: 29 additions & 1 deletion ph-diver-repo-s3/src/test/java/com/helger/diver/repo/s3/RepoStorageS3Test.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static org.junit.Assert.assertTrue;

import java.util.concurrent.ThreadLocalRandom;
import java.util.function.Consumer;

import javax.annotation.Nonnull;

Expand All @@ -39,6 +40,9 @@
import com.helger.diver.repo.RepoStorageItem;
import com.helger.diver.repo.RepoStorageKey;
import com.helger.diver.repo.RepoStorageKeyOfArtefact;
import com.helger.diver.repo.toc.IRepoStorageWithToc;
import com.helger.diver.repo.toc.IRepoTopTocGroupNameConsumer;
import com.helger.diver.repo.toc.IRepoTopTocService;

import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.core.exception.SdkClientException;
Expand Down Expand Up @@ -79,7 +83,31 @@ private static RepoStorageS3 _createRepoWritable ()
"bucket",
"unittest.s3",
ERepoWritable.WITH_WRITE,
ERepoDeletable.WITH_DELETE);
ERepoDeletable.WITH_DELETE,
new IRepoTopTocService ()
{
@Nonnull
public ESuccess registerGroupAndArtifact (final String sGroupID,
final String sArtifactID)
{
return ESuccess.SUCCESS;
}

public void iterateAllTopLevelGroupNames (final Consumer <String> aGroupNameConsumer)
{}

public void iterateAllSubGroups (final String sGroupID,
final IRepoTopTocGroupNameConsumer aGroupNameConsumer,
final boolean bRecursive)
{}

public void iterateAllArtifacts (final String sGroupID,
final Consumer <String> aArtifactNameConsumer)
{}

public void initForRepo (final IRepoStorageWithToc aRepo)
{}
});
}

@Test
Expand Down
6 changes: 3 additions & 3 deletions ph-diver-repo/src/main/java/com/helger/diver/repo/IRepoStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ default String getRepoTypeID ()
* @return {@link ESuccess}
*/
@Nonnull
default ESuccess write (@Nonnull final RepoStorageKeyOfArtefact aKey, @Nonnull final RepoStorageItem aItem)
default ESuccess write (@Nonnull final RepoStorageKey aKey, @Nonnull final RepoStorageItem aItem)
{
return write (aKey, aItem, (OffsetDateTime) null);
}
Expand All @@ -95,7 +95,7 @@ default ESuccess write (@Nonnull final RepoStorageKeyOfArtefact aKey, @Nonnull f
* @return {@link ESuccess}
*/
@Nonnull
ESuccess write (@Nonnull RepoStorageKeyOfArtefact aKey,
ESuccess write (@Nonnull RepoStorageKey aKey,
@Nonnull RepoStorageItem aItem,
@Nullable OffsetDateTime aPublicationDT);

Expand All @@ -114,5 +114,5 @@ ESuccess write (@Nonnull RepoStorageKeyOfArtefact aKey,
* {@link ESuccess#FAILURE} if not.
*/
@Nonnull
ESuccess delete (@Nonnull RepoStorageKeyOfArtefact aKey);
ESuccess delete (@Nonnull RepoStorageKey aKey);
}
6 changes: 5 additions & 1 deletion ph-diver-repo/src/main/java/com/helger/diver/repo/RepoStorageChain.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,10 @@ public RepoStorageItem read (@Nonnull final RepoStorageKey aKey)
LOGGER.debug (sMsg);
}

// Only copy the artefact if:
// 1. The source repository is remote
// 2. Caching of content is enabled
// 3. It is a real artifact and not just "any" data (like TopToC)
if (aStorage.getRepoType ().isRemote () && m_bCacheRemoteContent && aKey instanceof RepoStorageKeyOfArtefact)
{
// Item was read from remote
Expand All @@ -178,7 +182,7 @@ public RepoStorageItem read (@Nonnull final RepoStorageKey aKey)
"' to " +
m_aWritableStorages.getAllMapped (IRepoStorage::getRepoTypeID));
for (final IRepoStorage aWritableStorage : m_aWritableStorages)
aWritableStorage.write ((RepoStorageKeyOfArtefact) aKey, aItem);
aWritableStorage.write (aKey, aItem);
}
}
return aItem;
Expand Down
15 changes: 1 addition & 14 deletions ph-diver-repo/src/main/java/com/helger/diver/repo/RepoStorageKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,11 @@ public class RepoStorageKey
*/
public static final String SUFFIX_SHA256 = ".sha256";

/** The default filename for the top-level table of contents. */
public static final String FILENAME_TOP_TOC_DIVER_XML = "toptoc-diver.xml";

private static final Logger LOGGER = LoggerFactory.getLogger (RepoStorageKey.class);

private final String m_sPath;

protected RepoStorageKey (@Nonnull @Nonempty final String sPath)
public RepoStorageKey (@Nonnull @Nonempty final String sPath)
{
ValueEnforcer.notEmpty (sPath, "Path");
ValueEnforcer.isFalse (sPath.startsWith ("/"), "Path should not start with a Slash");
Expand Down Expand Up @@ -102,14 +99,4 @@ public String toString ()
{
return new ToStringGenerator (null).append ("Path", m_sPath).getToString ();
}

/**
* @return The {@link RepoStorageKey} associated with the top-level table of
* contents. Never <code>null</code>.
*/
@Nonnull
public static RepoStorageKey ofTopToc ()
{
return new RepoStorageKey (FILENAME_TOP_TOC_DIVER_XML);
}
}
16 changes: 16 additions & 0 deletions ph-diver-repo/src/main/java/com/helger/diver/repo/RepoStorageKeyOfArtefact.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,19 @@
/*
* Copyright (C) 2023 Philip Helger & ecosio
* philip[at]helger[dot]com
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.helger.diver.repo;

import javax.annotation.Nonnull;
Expand Down
Loading

0 comments on commit 6d4754d

Please sign in to comment.