-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(server): zip and index scenario execution report (#208)
- Loading branch information
Showing
22 changed files
with
715 additions
and
15 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
53 changes: 53 additions & 0 deletions
53
...in/java/com/chutneytesting/execution/infra/aop/ScenarioExecutionReportIndexingAspect.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,53 @@ | ||
| /* | ||
| * SPDX-FileCopyrightText: 2017-2024 Enedis | ||
| * | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package com.chutneytesting.execution.infra.aop; | ||
|
|
||
| import com.chutneytesting.execution.infra.storage.DatabaseExecutionJpaRepository; | ||
| import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionEntity; | ||
| import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity; | ||
| import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository; | ||
| import com.chutneytesting.scenario.infra.jpa.ScenarioEntity; | ||
| import java.util.List; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
| import org.aspectj.lang.annotation.After; | ||
| import org.aspectj.lang.annotation.Aspect; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| @Aspect | ||
| @Component | ||
| public class ScenarioExecutionReportIndexingAspect { | ||
| private final ScenarioExecutionReportIndexRepository reportIndexRepository; | ||
| private final DatabaseExecutionJpaRepository scenarioExecutionRepository; | ||
|
|
||
| public ScenarioExecutionReportIndexingAspect(ScenarioExecutionReportIndexRepository reportIndexRepository, DatabaseExecutionJpaRepository scenarioExecutionRepository) { | ||
| this.reportIndexRepository = reportIndexRepository; | ||
| this.scenarioExecutionRepository = scenarioExecutionRepository; | ||
| } | ||
|
|
||
| @After("execution(* com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository.save(..)) && args(reportEntity)") | ||
| public void index(ScenarioExecutionReportEntity reportEntity) { | ||
| if (reportEntity.status().isFinal()){ | ||
| reportIndexRepository.save(reportEntity); | ||
| } | ||
| } | ||
|
|
||
| @After("execution(* com.chutneytesting.scenario.infra.raw.ScenarioJpaRepository.save(..)) && args(scenario)") | ||
| public void deleteDeactivatedScenarioExecutions(ScenarioEntity scenario) { | ||
| if (!scenario.isActivated()){ | ||
| List<ScenarioExecutionEntity> executions = scenarioExecutionRepository.findAllByScenarioId(String.valueOf(scenario.getId())); | ||
| reportIndexRepository.deleteAllById(executions.stream().map(ScenarioExecutionEntity::getId).collect(Collectors.toSet())); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| @After("execution(* com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository.deleteAllById(..)) && args(scenarioExecutionIds)") | ||
| public void deleteById(Set<Long> scenarioExecutionIds) { | ||
| reportIndexRepository.deleteAllById(scenarioExecutionIds); | ||
| } | ||
| } |
97 changes: 97 additions & 0 deletions
97
...server/src/main/java/com/chutneytesting/execution/infra/migration/ZipReportMigration.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| /* | ||
| * SPDX-FileCopyrightText: 2017-2024 Enedis | ||
| * | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package com.chutneytesting.execution.infra.migration; | ||
|
|
||
| import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.SCENARIO_EXECUTION_REPORT; | ||
| import static com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository.WHAT; | ||
|
|
||
| import com.chutneytesting.execution.infra.storage.ScenarioExecutionReportJpaRepository; | ||
| import com.chutneytesting.execution.infra.storage.jpa.ScenarioExecutionReportEntity; | ||
| import com.chutneytesting.index.infra.IndexRepository; | ||
| import com.chutneytesting.index.infra.ScenarioExecutionReportIndexRepository; | ||
| import jakarta.persistence.EntityManager; | ||
| import java.util.List; | ||
| import org.apache.lucene.index.Term; | ||
| import org.apache.lucene.search.Query; | ||
| import org.apache.lucene.search.TermQuery; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
| import org.springframework.boot.CommandLineRunner; | ||
| import org.springframework.data.domain.PageRequest; | ||
| import org.springframework.data.domain.Pageable; | ||
| import org.springframework.data.domain.Slice; | ||
| import org.springframework.stereotype.Component; | ||
| import org.springframework.transaction.annotation.Transactional; | ||
|
|
||
| @Component | ||
| public class ZipReportMigration implements CommandLineRunner { | ||
|
|
||
|
|
||
| private final ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository; | ||
| private final ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository; | ||
| private final IndexRepository indexRepository; | ||
| private final EntityManager entityManager; | ||
| private static final Logger LOGGER = LoggerFactory.getLogger(ZipReportMigration.class); | ||
|
|
||
|
|
||
| public ZipReportMigration(ScenarioExecutionReportIndexRepository scenarioExecutionReportIndexRepository, ScenarioExecutionReportJpaRepository scenarioExecutionReportJpaRepository, IndexRepository indexRepository, EntityManager entityManager) { | ||
| this.scenarioExecutionReportIndexRepository = scenarioExecutionReportIndexRepository; | ||
| this.scenarioExecutionReportJpaRepository = scenarioExecutionReportJpaRepository; | ||
| this.indexRepository = indexRepository; | ||
| this.entityManager = entityManager; | ||
| } | ||
|
|
||
| @Override | ||
| @Transactional | ||
| public void run(String... args) { | ||
| if (isMigrationDone()) { | ||
| LOGGER.info("Report compression & indexing already done, skipping..."); | ||
| return; | ||
| } | ||
| PageRequest firstPage = PageRequest.of(0, 10); | ||
| int count = 0; | ||
| compressAndIndex(firstPage, count); | ||
| } | ||
|
|
||
| private void compressAndIndex(Pageable pageable, int previousCount) { | ||
| Slice<ScenarioExecutionReportEntity> slice = scenarioExecutionReportJpaRepository.findAll(pageable); | ||
| List<ScenarioExecutionReportEntity> reports = slice.getContent(); | ||
|
|
||
| compressAndSaveInDb(reports); | ||
| index(reports); | ||
|
|
||
| int count = previousCount + slice.getNumberOfElements(); | ||
| if (slice.hasNext()) { | ||
| compressAndIndex(slice.nextPageable(), count); | ||
| } else { | ||
| LOGGER.info("{} report(s) successfully compressed and indexed", count); | ||
| } | ||
| } | ||
|
|
||
| private void compressAndSaveInDb(List<ScenarioExecutionReportEntity> reportsInDb) { | ||
| // calling scenarioExecutionReportJpaRepository find() and then save() doesn't call ReportConverter | ||
| // ReportConverter will be called by entityManager update. So compression will be done | ||
| reportsInDb.forEach(report -> { | ||
| entityManager.createQuery( | ||
| "UPDATE SCENARIO_EXECUTIONS_REPORTS SET report = :report WHERE id = :id") | ||
| .setParameter("report", report.getReport()) | ||
| .setParameter("id", report.scenarioExecutionId()) | ||
| .executeUpdate(); | ||
| }); | ||
| } | ||
|
|
||
| private void index(List<ScenarioExecutionReportEntity> reportsInDb) { | ||
| scenarioExecutionReportIndexRepository.saveAll(reportsInDb); | ||
| } | ||
|
|
||
| private boolean isMigrationDone() { | ||
| Query whatQuery = new TermQuery(new Term(WHAT, SCENARIO_EXECUTION_REPORT)); | ||
| int indexedReports = indexRepository.count(whatQuery); | ||
| return indexedReports > 0; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
63 changes: 63 additions & 0 deletions
63
.../server/src/main/java/com/chutneytesting/execution/infra/storage/jpa/ReportConverter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| /* | ||
| * SPDX-FileCopyrightText: 2017-2024 Enedis | ||
| * | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| */ | ||
|
|
||
| package com.chutneytesting.execution.infra.storage.jpa; | ||
|
|
||
| import jakarta.persistence.AttributeConverter; | ||
| import jakarta.persistence.Converter; | ||
| import java.io.ByteArrayInputStream; | ||
| import java.io.ByteArrayOutputStream; | ||
| import java.io.IOException; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.util.zip.GZIPInputStream; | ||
| import java.util.zip.GZIPOutputStream; | ||
|
|
||
| @Converter | ||
| public class ReportConverter implements AttributeConverter<String, byte[]> { | ||
| @Override | ||
| public byte[] convertToDatabaseColumn(String report) { | ||
| return compress(report); | ||
| } | ||
|
|
||
| @Override | ||
| public String convertToEntityAttribute(byte[] zippedReport) { | ||
| if (!isCompressed(zippedReport)) { | ||
| return new String(zippedReport, StandardCharsets.UTF_8); | ||
| } | ||
| return decompress(zippedReport); | ||
| } | ||
|
|
||
| private boolean isCompressed(byte[] data) { | ||
| return (data != null && data.length >= 2 && | ||
| (data[0] == (byte) 0x1f && data[1] == (byte) 0x8b)); | ||
| } | ||
|
|
||
| private byte[] compress(String report) { | ||
| try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); | ||
| GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) { | ||
|
|
||
| gzipOutputStream.write(report.getBytes(StandardCharsets.UTF_8)); | ||
| gzipOutputStream.finish(); | ||
| return byteArrayOutputStream.toByteArray(); | ||
|
|
||
| } catch (IOException e) { | ||
| throw new RuntimeException("Failed to compress report content", e); | ||
| } | ||
| } | ||
|
|
||
| private String decompress(byte[] compressedData) { | ||
| try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(compressedData); | ||
| GZIPInputStream gzipInputStream = new GZIPInputStream(byteArrayInputStream); | ||
| ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { | ||
| byteArrayOutputStream.write(gzipInputStream.readAllBytes()); | ||
| return byteArrayOutputStream.toString(StandardCharsets.UTF_8); | ||
|
|
||
| } catch (IOException e) { | ||
| throw new RuntimeException("Failed to decompress report content", e); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.