Delete sequence files from file storage via sequence run page

CHANGELOG.md

@@ -19,6 +19,7 @@
 * [Developer]: Deprecated "/api/projects/{projectId}/samples/bySequencerId/{seqeuncerId}" in favour of "/api/projects/{projectId}/samples/bySampleName", which accepts a json property "sampleName"
 * [Developer]: Fixed bug in setting a `default_sequencing_object and default_genome_assembly to `NULL` for a sample when the default sequencing object or genome assembly were removed. [See PR 1466](https://github.com/phac-nml/irida/pull/1466)
 * [Developer]: Fixed bug preventing a `sample` with an analysis submission from being deleted. [See PR 1467](https://github.com/phac-nml/irida/pull/1467)
+* [Developer]: Added functionality to delete sequence files from file system when a sequence run is removed. [See PR 1468](https://github.com/phac-nml/irida/pull/1468)
 
 ## [22.09.7] - 2023/01/24
 * [UI]: Fixed bugs on NCBI Export page preventing the NCBI `submission.xml` file from being properly written. See [PR 1451](https://github.com/phac-nml/irida/pull/1451)

src/main/java/ca/corefacility/bioinformatics/irida/repositories/analysis/AnalysisOutputFileRepositoryImpl.java

@@ -14,10 +14,7 @@
 import ca.corefacility.bioinformatics.irida.repositories.filesystem.IridaFileStorageUtility;
 
 /**
- * Custom implementation of {@link FilesystemSupplementedRepositoryImpl} for
- * {@link AnalysisOutputFile}.
- * 
- *
+ * Custom implementation of {@link FilesystemSupplementedRepositoryImpl} for {@link AnalysisOutputFile}.
  */
 @Repository
 public class AnalysisOutputFileRepositoryImpl extends FilesystemSupplementedRepositoryImpl<AnalysisOutputFile> {
@@ -37,4 +34,13 @@ public AnalysisOutputFile save(AnalysisOutputFile entity) {
 		return super.saveInternal(entity);
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	@Transactional
+	public void delete(AnalysisOutputFile entity) {
+		super.deleteInternal(entity);
+	}
+
 }

src/main/java/ca/corefacility/bioinformatics/irida/repositories/assembly/GenomeAssemblyRepositoryImpl.java

@@ -3,6 +3,7 @@
 import java.nio.file.Path;
 
 import javax.persistence.EntityManager;
+import javax.transaction.Transactional;
 
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -21,7 +22,8 @@ public class GenomeAssemblyRepositoryImpl extends FilesystemSupplementedReposito
 
 	@Autowired
 	public GenomeAssemblyRepositoryImpl(EntityManager entityManager,
-			@Qualifier("assemblyFileBaseDirectory") Path baseDirectory, IridaFileStorageUtility iridaFileStorageUtility) {
+			@Qualifier("assemblyFileBaseDirectory") Path baseDirectory,
+			IridaFileStorageUtility iridaFileStorageUtility) {
 		super(entityManager, baseDirectory, iridaFileStorageUtility);
 	}
 
@@ -34,4 +36,13 @@ public GenomeAssembly save(GenomeAssembly entity) {
 		}
 
 	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	@Transactional
+	public void delete(GenomeAssembly entity) {
+		this.deleteInternal(entity);
+	}
 }

src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/FilesystemSupplementedRepositoryImpl.java

@@ -19,6 +19,7 @@
 
 import ca.corefacility.bioinformatics.irida.model.IridaThing;
 import ca.corefacility.bioinformatics.irida.model.VersionedFileFields;
+import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequenceFile;
 
 /**
  * Custom implementation of a repository that writes the {@link Path} part of an entity to disk.
@@ -166,6 +167,24 @@ protected Type saveInternal(final Type entity) {
 		return entityManager.merge(entity);
 	}
 
+	/**
+	 * Actually delete the entity from disk and the database.
+	 *
+	 * @param entity the entity to delete.
+	 */
+	protected void deleteInternal(final Type entity) {
+		Path deletePath = null;
+
+		if (entity instanceof SequenceFile) {
+			//remove all sequence file copies from revision folders
+			deletePath = ((SequenceFile) entity).getFile().getParent().getParent();
+
+			if (deletePath != null) {
+				iridaFileStorageUtility.deleteFolder(baseDirectory.resolve(deletePath));
+			}
+		}
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */
@@ -186,6 +205,14 @@ public Type saveMetadata(final Type entity) {
 	 */
 	public abstract Type save(final Type entity);
 
+	/**
+	 * Delete an entity from disk and database. Implementors of this method are recommended to call
+	 * {@link FilesystemSupplementedRepositoryImpl#deleteInternal} to avoid repeated boilerplate code.
+	 *
+	 * @param entity the entity to delete.
+	 */
+	public abstract void delete(final Type entity);
+
 	/**
 	 * Write any files to disk and update the {@link Path} location. This method works using reflection to automagically
 	 * find and update any internal {@link Path} members on the {@link VersionedFileFields}. This class **does not**

src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAwsUtilityImpl.java

@@ -1,6 +1,9 @@
 package ca.corefacility.bioinformatics.irida.repositories.filesystem;
 
-import java.io.*;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
 import java.nio.channels.FileChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -25,6 +28,7 @@
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
+import com.amazonaws.SdkClientException;
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.client.builder.AwsClientBuilder;
@@ -33,6 +37,7 @@
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
 
 /**
  * Implementation of file utilities for aws storage
@@ -177,6 +182,37 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque
 		}
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFile(Path file) {
+		try {
+			logger.trace("Deleting file: [" + file.toString() + "]");
+			s3.deleteObject(bucketName, getAwsFileAbsolutePath(file));
+		} catch (SdkClientException e) {
+			logger.error("Unable to delete file", e);
+			throw new StorageException("Unable to delete file", e);
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFolder(Path folder) {
+		try {
+			logger.trace("Deleting folder: [" + folder.toString() + "]");
+			for (S3ObjectSummary file : s3.listObjects(bucketName, getAwsFileAbsolutePath(folder))
+					.getObjectSummaries()) {
+				s3.deleteObject(bucketName, file.getKey());
+			}
+		} catch (SdkClientException e) {
+			logger.error("Unable to delete folder", e);
+			throw new StorageException("Unable to delete folder", e);
+		}
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */
@@ -354,7 +390,7 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) {
 			 below uses getBucketAcl. So if bucket permissions aren't set then the else code is used.
 			 */
 			GetObjectRequest rangeObjectRequest = new GetObjectRequest(bucketName,
-					getAwsFileAbsolutePath(file)).withRange(seek, (chunk+seek) - 1);
+					getAwsFileAbsolutePath(file)).withRange(seek, (chunk + seek) - 1);
 			try (S3Object s3Object = s3.getObject(rangeObjectRequest);
 					S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent()) {
 				byte[] bytes = s3ObjectInputStream.readAllBytes();

src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageAzureUtilityImpl.java

@@ -4,12 +4,10 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.channels.FileChannel;
-
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
-
 import java.time.Duration;
 import java.util.List;
 import java.util.Optional;
@@ -25,7 +23,10 @@
 import ca.corefacility.bioinformatics.irida.model.sequenceFile.SequencingObject;
 import ca.corefacility.bioinformatics.irida.ria.web.ajax.dto.analysis.FileChunkResponse;
 
-import com.azure.storage.blob.*;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
 import com.azure.storage.blob.models.*;
 import com.azure.storage.blob.specialized.BlobInputStream;
 import com.azure.storage.common.StorageSharedKeyCredential;
@@ -168,6 +169,39 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque
 		}
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFile(Path file) {
+		try {
+			logger.trace("Deleting file: [" + file.toString() + "]");
+			BlobClient blobClient = containerClient.getBlobClient(getAzureFileAbsolutePath(file));
+			blobClient.deleteIfExists();
+		} catch (BlobStorageException e) {
+			logger.error("Unable to delete file", e);
+			throw new StorageException("Unable to delete file", e);
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFolder(Path folder) {
+		try {
+			logger.trace("Deleting folder: [" + folder.toString() + "]");
+			ListBlobsOptions options = new ListBlobsOptions().setPrefix(getAzureFileAbsolutePath(folder));
+			containerClient.listBlobs(options, null).forEach(blob -> {
+				BlobClient blobClient = containerClient.getBlobClient(blob.getName());
+				blobClient.deleteIfExists();
+			});
+		} catch (BlobStorageException e) {
+			logger.error("Unable to delete folder", e);
+			throw new StorageException("Unable to delete folder", e);
+		}
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */

src/main/java/ca/corefacility/bioinformatics/irida/repositories/filesystem/IridaFileStorageLocalUtilityImpl.java

@@ -16,6 +16,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.util.FileSystemUtils;
 
 import ca.corefacility.bioinformatics.irida.exceptions.StorageException;
 import ca.corefacility.bioinformatics.irida.model.enums.StorageType;
@@ -61,13 +62,13 @@ public IridaTemporaryFile getTemporaryFile(Path file, String prefix) {
 	@Override
 	public void cleanupDownloadedLocalTemporaryFiles(IridaTemporaryFile iridaTemporaryFile) {
 		if (iridaTemporaryFile.getFile() != null) {
-			logger.trace("File resides on local filesystem. Not cleaning up file [" + iridaTemporaryFile.getFile()
-					.toString() + "]");
+			logger.trace(
+					"File resides on local filesystem. Not cleaning up file [" + iridaTemporaryFile.getFile().toString()
+							+ "]");
 		}
 		if (iridaTemporaryFile.getDirectoryPath() != null) {
 			logger.trace("Directory resides on local filesystem. Not cleaning up directory ["
-					+ iridaTemporaryFile.getDirectoryPath()
-					.toString() + "]");
+					+ iridaTemporaryFile.getDirectoryPath().toString() + "]");
 		}
 	}
 
@@ -100,13 +101,40 @@ public void writeFile(Path source, Path target, Path sequenceFileDir, Path seque
 		}
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFile(Path file) {
+		try {
+			logger.trace("Deleting file: [" + file.toString() + "]");
+			Files.deleteIfExists(file);
+		} catch (IOException e) {
+			logger.error("Unable to delete file", e);
+			throw new StorageException("Unable to delete file", e);
+		}
+	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Override
+	public void deleteFolder(Path folder) {
+		try {
+			logger.trace("Deleting folder: [" + folder.toString() + "]");
+			FileSystemUtils.deleteRecursively(folder);
+		} catch (IOException e) {
+			logger.error("Unable to delete folder", e);
+			throw new StorageException("Unable to delete folder", e);
+		}
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */
 	public String getFileName(Path file) {
 		String fileName = "";
-		fileName = file.getFileName()
-				.toString();
+		fileName = file.getFileName().toString();
 		return fileName;
 	}
 
@@ -171,9 +199,7 @@ public String getFileExtension(List<? extends SequencingObject> sequencingObject
 		for (SequencingObject object : sequencingObjects) {
 
 			for (SequenceFile file : object.getFiles()) {
-				String fileName = file.getFile()
-						.toFile()
-						.getName();
+				String fileName = file.getFile().toFile().getName();
 
 				Optional<String> currentExtensionOpt = VALID_CONCATENATION_EXTENSIONS.stream()
 						.filter(e -> fileName.endsWith(e))
@@ -245,7 +271,7 @@ public FileChunkResponse readChunk(Path file, Long seek, Long chunk) {
 			}
 
 			return new FileChunkResponse(chunkResponse, randomAccessFile.getFilePointer());
-		} catch (IOException e ) {
+		} catch (IOException e) {
 			logger.error("Could not read output file ", e);
 		}
 		return null;