From c4167e2375d0541809021f515824fa25627950bb Mon Sep 17 00:00:00 2001 From: karimGl Date: Fri, 25 Oct 2024 19:45:17 +0200 Subject: [PATCH] chore(): one shot batch --- .../infra/migration/ZipReportMigration.java | 83 +++++++++++++++++++ .../DatabaseExecutionHistoryRepository.java | 2 + .../infra/storage/jpa/ReportConverter.java | 44 ++++++---- .../index/infra/IndexRepository.java | 11 +++ ...cenarioExecutionReportIndexRepository.java | 15 +++- .../infra/raw/DatabaseTestCaseRepository.java | 2 +- .../changelog/db.changelog-master.xml | 2 +- 7 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 chutney/server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java diff --git a/chutney/server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java b/chutney/server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java new file mode 100644 index 000000000..143d9270b --- /dev/null +++ b/chutney/server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: 2017-2024 Enedis + * + * SPDX-License-Identifier: Apache-2.0 + * + */ + +package com.chutneytesting.execution.infra.migration; + +import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.SCENARIO_EXECUTION_REPORT; +import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.WHAT; + +import com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository; +import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity; +import com.chutneytesting.index.infra.IndexRepository; +import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository; +import jakarta.persistence.EntityManager; +import java.util.List; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +@Component +public class ZipReportMigration implements CommandLineRunner { + + + private final ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository; + private final ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository; + private final IndexRepository indexRepository; + private final EntityManager entityManager; + private static final Logger LOGGER = LoggerFactory.getLogger(ZipReportMigration.class); + + + public ZipReportMigration(ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository, ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository, IndexRepository indexRepository, EntityManager entityManager) { + this.scenarioExecutionReportIndexRepository = scenarioExecutionReportIndexRepository; + this.scenarioExecutionReportJpaRepository = scenarioExecutionReportJpaRepository; + this.indexRepository = indexRepository; + this.entityManager = entityManager; + } + + @Override + @Transactional + public void run(String... args) { + if (isMigrationDone()) { + LOGGER.info("Report compression already done, skipping..."); + return; + } + List reportsInDb = scenarioExecutionReportJpaRepository.findAll(); + compressAndSaveInDb(reportsInDb); + index(reportsInDb); + LOGGER.info("{} report(s) successfully compressed and indexed", reportsInDb.size()); + } + + private void index(List reportsInDb) { + LOGGER.info("{} report(s) will be indexed", reportsInDb.size()); + scenarioExecutionReportIndexRepository.saveAll(reportsInDb); + } + + private void compressAndSaveInDb(List reportsInDb) { + LOGGER.info("{} report(s) will be compressed", reportsInDb.size()); + + // calling scenarioExecutionReportJpaRepository save/saveAll doesn't call ReportConverter + // ReportConverter will be called by entityManager update. So compression will be done + reportsInDb.forEach(report -> { + entityManager.createQuery( + "UPDATE SCENARIO_EXECUTIONS_REPORTS SET report = :report WHERE id = :id") + .setParameter("report", report.getReport()) + .setParameter("id", report.scenarioExecutionId()) + .executeUpdate(); + }); + } + + private boolean isMigrationDone() { + Query whatQuery = new TermQuery(new Term(WHAT, SCENARIO_EXECUTION_REPORT)); + int indexedReports = indexRepository.count(whatQuery); + return indexedReports > 0; + } +} diff --git a/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionHistoryRepository.java b/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionHistoryRepository.java index 6fc7713d1..d172923e3 100644 --- a/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionHistoryRepository.java +++ b/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/DatabaseExecutionHistoryRepository.java @@ -186,6 +186,7 @@ private void updateReport(Execution execution) throws ReportNotFoundException { ); scenarioExecutionReport.updateReport(execution); scenarioExecutionReportJpaRepository.save(scenarioExecutionReport); + scenarioExecutionReportIndexRepository.save(scenarioExecutionReport); } @Override @@ -212,6 +213,7 @@ public void deleteExecutions(Set executionsIds) { campaignExecutionJpaRepository.deleteAllByIdInBatch(campaignExecutionsIds); scenarioExecutionReportJpaRepository.deleteAllById(executionsIds); + scenarioExecutionReportIndexRepository.deleteAllById(executionsIds); scenarioExecutionsJpaRepository.deleteAllByIdInBatch(executionsIds); } diff --git a/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/jpa/ReportConverter.java b/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/jpa/ReportConverter.java index f0c4ea8dc..a05fd19d8 100644 --- a/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/jpa/ReportConverter.java +++ b/chutney/server/src/main/java/com/chutneytesting/execution/infra/storage/jpa/ReportConverter.java @@ -8,39 +8,49 @@ package com.chutneytesting.execution.infra.storage.jpa; import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; -import org.apache.commons.lang3.StringUtils; +@Converter public class ReportConverter implements AttributeConverter { @Override public byte[] convertToDatabaseColumn(String report) { - if (StringUtils.isNoneEmpty()) { - try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { - - gzipOutputStream.write(report.getBytes(StandardCharsets.UTF_8)); - gzipOutputStream.finish(); - return byteArrayOutputStream.toByteArray(); - - } catch (IOException e) { - throw new RuntimeException("Failed to compress report content", e); - } - } - return null; + return compress(report); } @Override public String convertToEntityAttribute(byte[] zippedReport) { - if (zippedReport == null || zippedReport.length == 0) { - return null; + if (!isCompressed(zippedReport)) { + return new String(zippedReport, StandardCharsets.UTF_8); } + return decompress(zippedReport); + } + + private boolean isCompressed(byte[] data) { + return (data != null && data.length >= 2 && + (data[0] == (byte) 0x1f && data[1] == (byte) 0x8b)); + } + + private byte[] compress(String report) { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { + + gzipOutputStream.write(report.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.finish(); + return byteArrayOutputStream.toByteArray(); + + } catch (IOException e) { + throw new RuntimeException("Failed to compress report content", e); + } + } - try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(zippedReport); + private String decompress(byte[] compressedData) { + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData); GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { diff --git a/chutney/server/src/main/java/com/chutneytesting/index/infra/IndexRepository.java b/chutney/server/src/main/java/com/chutneytesting/index/infra/IndexRepository.java index cbc197e8e..68b8a0aad 100644 --- a/chutney/server/src/main/java/com/chutneytesting/index/infra/IndexRepository.java +++ b/chutney/server/src/main/java/com/chutneytesting/index/infra/IndexRepository.java @@ -55,6 +55,17 @@ public List search(Query query, int limit, Sort sort) { return result; } + public int count(Query query) { + int count = 0; + try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + IndexSearcher searcher = new IndexSearcher(reader); + count = searcher.count(query); + + } catch (IOException ignored) { + } + return count; + } + public void delete(Query query) { try { indexWriter.deleteDocuments(query); diff --git a/chutney/server/src/main/java/com/chutneytesting/index/infra/ScenarioExecutionReportIndexRepository.java b/chutney/server/src/main/java/com/chutneytesting/index/infra/ScenarioExecutionReportIndexRepository.java index f689540ea..ec60d8924 100644 --- a/chutney/server/src/main/java/com/chutneytesting/index/infra/ScenarioExecutionReportIndexRepository.java +++ b/chutney/server/src/main/java/com/chutneytesting/index/infra/ScenarioExecutionReportIndexRepository.java @@ -11,6 +11,7 @@ import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity; import java.util.List; +import java.util.Set; import org.apache.lucene.document.Document; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; @@ -31,8 +32,8 @@ public class ScenarioExecutionReportIndexRepository { public static final String SCENARIO_EXECUTION_REPORT = "scenario_execution_report"; public static final String WHAT = "what"; - public static final String SCENARIO_EXECUTION_ID = "scenarioExecutionId"; - public static final String REPORT = "report"; + private static final String SCENARIO_EXECUTION_ID = "scenarioExecutionId"; + private static final String REPORT = "report"; private final IndexRepository indexRepository; public ScenarioExecutionReportIndexRepository(IndexRepository indexRepository) { @@ -51,7 +52,11 @@ public void save(ScenarioExecutionReportEntity report) { indexRepository.index(document); } - public void remove(Long scenarioExecutionId) { + public void saveAll(List reports) { + reports.forEach(this::save); + } + + public void delete(Long scenarioExecutionId) { Query whatQuery = new TermQuery(new Term(WHAT, SCENARIO_EXECUTION_REPORT)); Query idQuery = new TermQuery(new Term(SCENARIO_EXECUTION_ID, scenarioExecutionId.toString())); BooleanQuery query = new BooleanQuery.Builder() @@ -61,6 +66,10 @@ public void remove(Long scenarioExecutionId) { indexRepository.delete(query); } + public void deleteAllById(Set scenarioExecutionIds) { + scenarioExecutionIds.forEach(this::delete); + } + public List idsByKeywordInReport(String keyword) { Query whatQuery = new TermQuery(new Term(WHAT, SCENARIO_EXECUTION_REPORT)); diff --git a/chutney/server/src/main/java/com/chutneytesting/scenario/infra/raw/DatabaseTestCaseRepository.java b/chutney/server/src/main/java/com/chutneytesting/scenario/infra/raw/DatabaseTestCaseRepository.java index 4e7622920..e61550f0b 100644 --- a/chutney/server/src/main/java/com/chutneytesting/scenario/infra/raw/DatabaseTestCaseRepository.java +++ b/chutney/server/src/main/java/com/chutneytesting/scenario/infra/raw/DatabaseTestCaseRepository.java @@ -121,7 +121,7 @@ public void removeById(String scenarioId) { allExecutions.forEach(e -> { e.forCampaignExecution(null); scenarioExecutionsJpaRepository.save(e); - scenarioExecutionReportIndexRepository.remove(e.id()); + scenarioExecutionReportIndexRepository.delete(e.id()); }); List allCampaignScenarioEntities = campaignScenarioJpaRepository.findAllByScenarioId(scenarioId); diff --git a/chutney/server/src/main/resources/changelog/db.changelog-master.xml b/chutney/server/src/main/resources/changelog/db.changelog-master.xml index cefe86bc5..293e1fc0d 100644 --- a/chutney/server/src/main/resources/changelog/db.changelog-master.xml +++ b/chutney/server/src/main/resources/changelog/db.changelog-master.xml @@ -1072,7 +1072,7 @@ - +