Added in coverage to Project Samples table and export

CHANGELOG.md

@@ -20,6 +20,7 @@
 * [Developer]: Add identifier to project drop-downs. See [PR 1352](https://github.com/phac-nml/irida/pull/1352)
 * [UI]: Fixed issue with Biohansel pipeline being launched without selecting an option for a required parameter. See [PR 1356](https://github.com/phac-nml/irida/pull/1356)
 * [Developer]: Unified validate sample names into one endpoint. See [PR 1353](https://github.com/phac-nml/irida/pull/1353)
+* [Developer/UI]: Increased speed of Project Samples table export and added estimated coverage to Project Samples table and exports. See [PR 1360](https://github.com/phac-nml/irida/pull/1360)
 
 ## [22.05.5] - 2022/06/28
 * [UI]: Fixed bug preventing export of project samples table due to invalid url. [PR 1331](https://github.com/phac-nml/irida/pull/1331)

doc/user/user/samples/index.md

@@ -21,11 +21,12 @@ Viewing samples in a project
 The samples listing shows high-level sample details, such as:
 
 * The name of the sample,
+* The estimated coverage of the Sample (if the project has set the genome size),
 * The user-defined organism of the sample (if provided),
 * The name of the person who collected the sample (if provided),
 * The project that the sample belongs to (if from a related project),
 * The date that the sample was created in IRIDA.
-* Whether the sample failed IRIDA's quality control checks.  If a sample has failed QC, the cell will have a red background and warning icon <i class="fa fa-exclamation-triangle" aria-hidden="true"></i>.  Note if you are displaying [associated project samples](#viewing-associated-samples) QC information will come from the sample's parent project.
+* Whether the sample failed IRIDA's quality control checks.  If a sample has failed QC, the cell will have a red background and warning icon <i class="fa fa-exclamation-triangle" aria-hidden="true"></i>.  If a sample does not have any Sequencing Objects associated with it, the cell will have a grey background with the text N.A.  Note if you are displaying [associated project samples](#viewing-associated-samples) QC information will come from the sample's parent project.
 * Whether the sample can be modified in this project.  If the project does not have modification access, a lock icon will be displayed <i class="fa fa-lock" aria-hidden="true"></i>.
 
 ### Viewing individual sample details
@@ -129,7 +130,7 @@ Adding a new sample
 Filtering samples
 ------------------------
 
-Each column in the projects' samples table, (except the quality control column) can be filtered by typing in the filter
+Each column in the projects' samples table, (except the quality control column and coverage column) can be filtered by typing in the filter
 box. The filter box is located in the right corner of the column header.
 
 ![Location of the column filters.](images/sample-table-header-filters.png)

src/main/java/ca/corefacility/bioinformatics/irida/repositories/joins/project/ProjectSampleJoinRepository.java

@@ -2,6 +2,8 @@
 
 import java.util.List;
 
+import javax.persistence.Tuple;
+
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.domain.Specification;
@@ -95,4 +97,13 @@ public interface ProjectSampleJoinRepository
 	@Query("SELECT j.sample.id FROM ProjectSampleJoin j where j.owner=false AND j.project=?1")
 	public List<Long> getLockedSamplesForProject(Project project);
 
+	/**
+	 * Calculate the coverage for a list of Samples within a Project.
+	 * 
+	 * @param project   {@link Project} to get sample coverage for.
+	 * @param sampleIds {@link Sample} ids
+	 * @return A List of {@link Tuple}s with first entry as the sample id and second entry as the coverage
+	 */
+	@Query("SELECT ps.sample.id, ROUND(SUM(qc.totalBases)/ps.project.genomeSize) FROM ProjectSampleJoin ps join SampleSequencingObjectJoin sso on sso.sample = ps.sample join CoverageQCEntry qc on qc.sequencingObject = sso.sequencingObject where ps.project=?1 and ps.sample.id in ?2 group by ps.sample.id")
+	public List<Tuple> calculateCoverageForSamplesInProject(Project project, List<Long> sampleIds);
 }

src/main/java/ca/corefacility/bioinformatics/irida/repositories/sample/QCEntryRepository.java

@@ -2,6 +2,8 @@
 
 import java.util.List;
 
+import javax.persistence.Tuple;
+
 import org.springframework.data.jpa.repository.Query;
 
 import ca.corefacility.bioinformatics.irida.model.sample.QCEntry;
@@ -14,13 +16,21 @@
  */
 public interface QCEntryRepository extends IridaJpaRepository<QCEntry, Long> {
 	/**
-	 * find all the {@link QCEntry} associated with {@link SequencingObject}s in
-	 * a given {@link Sample}
+	 * find all the {@link QCEntry} associated with {@link SequencingObject}s in a given {@link Sample}
 	 * 
-	 * @param sample
-	 *            the {@link Sample} to get {@link QCEntry} for
+	 * @param sample the {@link Sample} to get {@link QCEntry} for
 	 * @return a list of {@link QCEntry}
 	 */
 	@Query("FROM QCEntry e WHERE e.sequencingObject.sample.sample = ?1")
 	public List<QCEntry> getQCEntriesForSample(Sample sample);
+
+	/**
+	 * find all the {@link QCEntry} associated with {@link SequencingObject}s in a given {@link Sample} for a list of
+	 * {@link Sample}s.
+	 * 
+	 * @param samples the List of {@link Sample}s to get {@link QCEntry}s for
+	 * @return a List of {@link Tuple}s containing the {@link Sample} id and {@link QCEntry}
+	 */
+	@Query("SELECT e.sequencingObject.sample.sample.id, e FROM QCEntry e WHERE e.sequencingObject.sample.sample in ?1")
+	public List<Tuple> getQCEntriesForSamples(List<Sample> samples);
 }

src/main/java/ca/corefacility/bioinformatics/irida/ria/web/projects/dto/ProjectSampleTableItem.java

@@ -15,13 +15,17 @@ public class ProjectSampleTableItem extends AntTableItem {
 	private final SampleObject sample;
 	private final ProjectObject project;
 	private final List<String> quality;
+	private final String qcStatus;
+	private final Long coverage;
 
-	public ProjectSampleTableItem(ProjectSampleJoin join, List<String> quality) {
+	public ProjectSampleTableItem(ProjectSampleJoin join, List<String> quality, String qcStatus, Long coverage) {
 		super(join.getId());
 		this.owner = join.isOwner();
 		this.sample = new SampleObject(join.getObject());
 		this.project = new ProjectObject(join.getSubject());
 		this.quality = quality;
+		this.qcStatus = qcStatus;
+		this.coverage = coverage;
 	}
 
 	public SampleObject getSample() {
@@ -39,4 +43,12 @@ public Boolean getOwner() {
 	public List<String> getQuality() {
 		return quality;
 	}
+
+	public String getQcStatus() {
+		return qcStatus;
+	}
+
+	public Long getCoverage() {
+		return coverage;
+	}
 }

src/main/java/ca/corefacility/bioinformatics/irida/ria/web/services/UISampleService.java

@@ -94,9 +94,9 @@ public class UISampleService {
 	 These correspond to their internationalized strings in the messages file
 	 */
 	private final List<String> TABLE_HEADERS = ImmutableList.of("server.SamplesTable.sampleName",
-			"server.SamplesTable.sampleId", "server.SamplesTable.quality", "server.SamplesTable.organism",
-			"server.SamplesTable.project", "server.SamplesTable.projectId", "server.SamplesTable.collectedBy",
-			"server.SamplesTable.created", "server.SamplesTable.modified");
+			"server.SamplesTable.sampleId", "server.SamplesTable.quality", "server.SamplesTable.coverage",
+			"server.SamplesTable.organism", "server.SamplesTable.project", "server.SamplesTable.projectId",
+			"server.SamplesTable.collectedBy", "server.SamplesTable.created", "server.SamplesTable.modified");
 
 	@Autowired
 	public UISampleService(SampleService sampleService, ProjectService projectService,
@@ -175,8 +175,8 @@ public SampleFilesResponse getFilesForSamples(List<Long> sampleIds, Long project
 	 * @return list of paired end sequence files
 	 */
 	public List<SequencingObject> getPairedSequenceFilesForSample(Sample sample, Project project) {
-		Collection<SampleSequencingObjectJoin> filePairJoins = sequencingObjectService.getSequencesForSampleOfType(
-				sample, SequenceFilePair.class);
+		Collection<SampleSequencingObjectJoin> filePairJoins = sequencingObjectService
+				.getSequencesForSampleOfType(sample, SequenceFilePair.class);
 		// add project to qc entries and filter any unavailable entries
 		List<SequencingObject> filePairs = new ArrayList<>();
 		for (SampleSequencingObjectJoin join : filePairJoins) {
@@ -196,8 +196,8 @@ public List<SequencingObject> getPairedSequenceFilesForSample(Sample sample, Pro
 	 * @return list of single end sequence files
 	 */
 	public List<SequencingObject> getSingleEndSequenceFilesForSample(Sample sample, Project project) {
-		Collection<SampleSequencingObjectJoin> singleFileJoins = sequencingObjectService.getSequencesForSampleOfType(
-				sample, SingleEndSequenceFile.class);
+		Collection<SampleSequencingObjectJoin> singleFileJoins = sequencingObjectService
+				.getSequencesForSampleOfType(sample, SingleEndSequenceFile.class);
 
 		List<SequencingObject> singles = new ArrayList<>();
 		for (SampleSequencingObjectJoin join : singleFileJoins) {
@@ -216,8 +216,8 @@ public List<SequencingObject> getSingleEndSequenceFilesForSample(Sample sample,
 	 * @return list of fast5 sequence files
 	 */
 	public List<SequencingObject> getFast5FilesForSample(Sample sample) {
-		Collection<SampleSequencingObjectJoin> fast5FileJoins = sequencingObjectService.getSequencesForSampleOfType(
-				sample, Fast5Object.class);
+		Collection<SampleSequencingObjectJoin> fast5FileJoins = sequencingObjectService
+				.getSequencesForSampleOfType(sample, Fast5Object.class);
 		return fast5FileJoins.stream().map(SampleSequencingObjectJoin::getObject).collect(Collectors.toList());
 	}
 
@@ -431,16 +431,15 @@ public StreamingResponseBody downloadSamples(Long projectId, List<Long> sampleId
 
 			try {
 				for (Sample sample : samples) {
-					Collection<SampleSequencingObjectJoin> sequencingObjectsForSample = sequencingObjectService.getSequencingObjectsForSample(
-							sample);
+					Collection<SampleSequencingObjectJoin> sequencingObjectsForSample = sequencingObjectService
+							.getSequencingObjectsForSample(sample);
 
 					for (SampleSequencingObjectJoin join : sequencingObjectsForSample) {
 						for (SequenceFile file : join.getObject().getFiles()) {
 							Path path = file.getFile();
 
-							String fileName =
-									project.getName() + "/" + sample.getSampleName() + "/" + path.getFileName()
-											.toString();
+							String fileName = project.getName() + "/" + sample.getSampleName() + "/"
+									+ path.getFileName().toString();
 							if (usedFileNames.contains(fileName)) {
 								fileName = handleDuplicate(fileName, usedFileNames);
 							}
@@ -555,21 +554,50 @@ public void downloadSamplesSpreadsheet(Long projectId, String type, ProjectSampl
 	 * @return List of {@link ProjectSampleTableItem}
 	 */
 	private List<ProjectSampleTableItem> formatSamplesForTable(Page<ProjectSampleJoin> page, Locale locale) {
+		Map<Project, List<Long>> projectSampleIdsMap = page.getContent()
+				.stream()
+				.collect(Collectors.groupingBy(ProjectSampleJoin::getSubject,
+						Collectors.mapping(join -> join.getObject().getId(), Collectors.toList())));
+
+		Map<Project, Map<Long, Long>> projectSamplesCoverageMap = new HashMap<Project, Map<Long, Long>>();
+		projectSampleIdsMap.forEach((project, sampleIds) -> projectSamplesCoverageMap.put(project,
+				sampleService.getCoverageForSamplesInProject(project, sampleIds)));
+
+		List<Sample> samples = page.getContent().stream().map(join -> join.getObject()).collect(Collectors.toList());
+		Map<Long, List<QCEntry>> sampleQCEntries = sampleService.getQCEntriesForSamples(samples);
+
 		return page.getContent().stream().map(join -> {
 			Sample sample = join.getObject();
 			Project project = join.getSubject();
 
-			List<QCEntry> qcEntriesForSample = sampleService.getQCEntriesForSample(sample);
-			List<String> quality = new ArrayList<>();
+			Long coverage = null;
+			if (projectSamplesCoverageMap.containsKey(project)
+					&& projectSamplesCoverageMap.get(project).containsKey(sample.getId())) {
+				coverage = projectSamplesCoverageMap.get(project).get(sample.getId());
+			}
 
-			qcEntriesForSample.forEach(entry -> {
-				entry.addProjectSettings(project);
-				if (entry.getStatus() == QCEntry.QCEntryStatus.NEGATIVE) {
-					quality.add(messageSource.getMessage("sample.files.qc." + entry.getType(),
-							new Object[] { entry.getMessage() }, locale));
+			List<QCEntry> qcEntriesForSample = sampleQCEntries.get(sample.getId());
+			List<String> quality = new ArrayList<>();
+			String qcStatus = null;
+
+			// If the sample has any SequencingObjects we will have at minimum CoverageQCEntry's
+			// which can be checked to set QCStatus.
+			if (qcEntriesForSample != null) {
+				qcEntriesForSample.forEach(entry -> {
+					entry.addProjectSettings(project);
+					if (entry.getStatus() == QCEntry.QCEntryStatus.NEGATIVE) {
+						quality.add(messageSource.getMessage("sample.files.qc." + entry.getType(),
+								new Object[] { entry.getMessage() }, locale));
+					}
+				});
+				// set qcStatus based on filtered qcEntries
+				if (quality.size() == 0) {
+					qcStatus = "pass";
+				} else {
+					qcStatus = "fail";
 				}
-			});
-			return new ProjectSampleTableItem(join, quality);
+			}
+			return new ProjectSampleTableItem(join, quality, qcStatus, coverage);
 		}).collect(Collectors.toList());
 	}
 
@@ -618,6 +646,11 @@ private void writeToExcel(HttpServletResponse response, String filename, List<Pr
 			Cell sampleQualityCell = row.createCell(cellNum++);
 			sampleQualityCell.setCellValue(StringUtils.join(item.getQuality(), "; "));
 
+			Cell sampleCoverageCell = row.createCell(cellNum++);
+			if (item.getCoverage() != null) {
+				sampleCoverageCell.setCellValue(item.getCoverage());
+			}
+
 			Cell sampleOrganismCell = row.createCell(cellNum++);
 			sampleOrganismCell.setCellValue(sample.getOrganism());
 
@@ -651,8 +684,7 @@ private void writeToExcel(HttpServletResponse response, String filename, List<Pr
 	 *
 	 * @param response {@link HttpServletResponse}
 	 * @param filename {@link String} name of the file to download.
-	 * @param items    {@link ProjectSampleTableItem} details about each row of the table   Data to download in the
-	 *                 table
+	 * @param items    {@link ProjectSampleTableItem} details about each row of the table Data to download in the table
 	 * @param headers  for the table
 	 * @throws IOException thrown if file cannot be written
 	 */
@@ -666,9 +698,16 @@ private void writeToCSV(HttpServletResponse response, String filename, List<Proj
 			SampleObject sample = item.getSample();
 			ProjectObject project = item.getProject();
 			String[] row = {
-					sample.getSampleName(), sample.getId().toString(), StringUtils.join(item.getQuality(), "; "),
-					sample.getOrganism(), project.getName(), project.getId().toString(), sample.getCollectedBy(),
-					sample.getCreatedDate().toString(), sample.getModifiedDate().toString() };
+					sample.getSampleName(),
+					sample.getId().toString(),
+					StringUtils.join(item.getQuality(), "; "),
+					item.getCoverage() != null ? item.getCoverage().toString() : "",
+					sample.getOrganism(),
+					project.getName(),
+					project.getId().toString(),
+					sample.getCollectedBy(),
+					sample.getCreatedDate().toString(),
+					sample.getModifiedDate().toString() };
 			results.add(row);
 		}
 

src/main/java/ca/corefacility/bioinformatics/irida/service/impl/sample/SampleServiceImpl.java

@@ -664,6 +664,19 @@ public List<QCEntry> getQCEntriesForSample(Sample sample) {
 		return qcEntryRepository.getQCEntriesForSample(sample);
 	}
 
+	/**
+	 * {@inheritDoc}
+	 */
+	@Transactional(readOnly = true)
+	@Override
+	@PreAuthorize("hasRole('ROLE_ADMIN') or hasPermission(#samples, 'canReadSample')")
+	public Map<Long, List<QCEntry>> getQCEntriesForSamples(List<Sample> samples) {
+		return qcEntryRepository.getQCEntriesForSamples(samples)
+				.stream()
+				.collect(Collectors.groupingBy(sampleQCEntryTuple -> (Long) sampleQCEntryTuple.get(0), Collectors
+						.mapping(sampleQCEntryTuple -> (QCEntry) sampleQCEntryTuple.get(1), Collectors.toList())));
+	}
+
 	/**
 	 * {@inheritDoc}
 	 */
@@ -813,4 +826,18 @@ public Long getSamplesCreated(Date createdDate) {
 	public List<GenericStatModel> getSamplesCreatedGrouped(Date createdDate, StatisticTimePeriod statisticTimePeriod) {
 		return sampleRepository.countSamplesCreatedGrouped(createdDate, statisticTimePeriod.getGroupByFormat());
 	}
+
+	/**
+	 * {@inheritDoc}
+	 */
+	@Transactional(readOnly = true)
+	@PreAuthorize("hasRole('ROLE_ADMIN') or hasPermission(#project, 'canReadProject')")
+	public Map<Long, Long> getCoverageForSamplesInProject(Project project, List<Long> sampleIds) {
+		return psjRepository.calculateCoverageForSamplesInProject(project, sampleIds)
+				.stream()
+				.collect(HashMap::new,
+						(sampleCoverageMap, sampleCoverageTuple) -> sampleCoverageMap
+								.put((Long) sampleCoverageTuple.get(0), (Long) sampleCoverageTuple.get(1)),
+						Map::putAll);
+	}
 }

src/main/java/ca/corefacility/bioinformatics/irida/service/sample/SampleService.java

@@ -275,6 +275,15 @@ public Page<ProjectSampleJoin> getFilteredSamplesForProjects(List<Project> proje
 	 */
 	public List<QCEntry> getQCEntriesForSample(Sample sample);
 
+	/**
+	 * Find all the {@link QCEntry} associated with {@link SequencingObject}s in a given {@link Sample} for a list of
+	 * {@link Sample}s
+	 *
+	 * @param samples the {@link Sample}s to get {@link QCEntry}s for
+	 * @return a map of sample id to {@link QCEntry}s
+	 */
+	public Map<Long, List<QCEntry>> getQCEntriesForSamples(List<Sample> samples);
+
 	/**
 	 * Search all {@link Sample}s in projects the current logged in user has access to
 	 *
@@ -316,4 +325,13 @@ public Page<ProjectSampleJoin> searchAllSamples(String query, final Integer page
 	 * @return An {@link GenericStatModel} list
 	 */
 	public List<GenericStatModel> getSamplesCreatedGrouped(Date createdDate, StatisticTimePeriod statisticTimePeriod);
+
+	/**
+	 * Get Coverage for samples within a project.
+	 * 
+	 * @param project   the {@link Project} to use to calculate the coverage
+	 * @param sampleIds the {@link Sample} ids to get coverage for
+	 * @return a map of sample id to coverage value
+	 */
+	public Map<Long, Long> getCoverageForSamplesInProject(Project project, List<Long> sampleIds);
 }

src/main/resources/i18n/messages.properties

@@ -2803,6 +2803,7 @@ SampleMenu.fileFilter.clear=Clear File Filter: {0}
 SampleIcon.locked=You do not have ownership of this sample and cannot modify it.
 SamplesTable.Column.sampleName=Name
 SamplesTable.Column.quality=QC
+SamplesTable.Column.coverage=Coverage
 SamplesTable.Column.organism=Organism
 SamplesTable.Column.project=Project
 SamplesTable.Column.collectedBy=Collected By
@@ -2813,6 +2814,7 @@ SamplesTable.Summary=Selected: {0} of {1}
 server.SamplesTable.sampleName=Sample Name
 server.SamplesTable.sampleId=Sample Identifier
 server.SamplesTable.quality=QC
+server.SamplesTable.coverage=Coverage
 server.SamplesTable.organism=Organism
 server.SamplesTable.project=Project
 server.SamplesTable.projectId=Project Identifier
@@ -2834,6 +2836,7 @@ RemoveModal.error=There was an error trying to remove these samples from this pr
 AssociatedSamplesList.header=These samples are from an associated project and cannot be removed from there
 LockedSamplesList.header=You do not have permission to modify these samples
 
+SampleQuality.na=N/A
 SampleQuality.pass=Pass
 SampleQuality.fail=Fail