Skip to content

Commit

Permalink
ADM-914:[backend][frontend] feat: set file name with time range when …
Browse files Browse the repository at this point in the history 
…generate file (#1386)

* ADM-914:[backend][frontend] feat: set file name with time range when generate file

* ADM-914:[backend][frontend]fix: fix code issue and update export url

* ADM-914:[backend]fix: update export path

* ADM-914:[frontend]fix: fix download url

* ADM-914:[backend]fix: check filename

* ADM-914:[backend]fix: check invalid time range time stamp

* ADM-914:[backend]fix: fix sonar issue

* ADM-914:[backend]feat: add start time and end time param in api

* ADM-914:[backend]fix: fix sonar issue

* ADM-914:[backend]fix: format code

* ADM-914:[backend]fix: fix sonar issue

* ADM-914:[backend]fix: fix un Security code

* ADM-914:[frontend]fix: fix mock url

* ADM-914:[backend]fix: fix sonar and call back url

* ADM-914:[backend]fix: fix sonar issue

* ADM-914:[backend]fix: add pattern for time

* ADM-914:[backend]fix: fix sonar issue

* ADM-914:[backend]fix: format issue code

* ADM-914:[backend]fix: fix sonar error

---------

Co-authored-by: Tingyu Dong <[email protected]>
  • Loading branch information
@yulongcai @TingyuDong
yulongcai and TingyuDong authored Apr 19, 2024
1 parent 6984484 commit bb3e0b9
Show file tree
Hide file tree
Showing 19 changed files with 421 additions and 234 deletions.
37 changes: 23 additions & 14 deletions backend/src/main/java/heartbeat/controller/report/ReportController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import heartbeat.controller.report.dto.response.ReportResponse;
import heartbeat.service.report.GenerateReporterService;
import heartbeat.service.report.ReportService;
import heartbeat.util.TimeUtil;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.tags.Tag;
import lombok.RequiredArgsConstructor;
Expand All @@ -15,12 +17,12 @@
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.PostMapping;

@RestController
@RequiredArgsConstructor
Expand All @@ -37,30 +39,37 @@ public class ReportController {
@Value("${callback.interval}")
private Integer interval;

@GetMapping("/{reportType}/{filename}")
@GetMapping("/{reportType}/{timeStamp}")
public InputStreamResource exportCSV(
@Schema(type = "string", allowableValues = { "metric", "pipeline", "board" },
accessMode = Schema.AccessMode.READ_ONLY) @PathVariable() ReportType reportType,
@PathVariable String filename) {
log.info("Start to export CSV file_reportType: {}, filename: {}", reportType.getValue(), filename);
InputStreamResource result = reportService.exportCsv(reportType, Long.parseLong(filename));
log.info("Successfully get CSV file_reportType: {}, filename: {}, _result: {}", reportType.getValue(), filename,
result);
accessMode = Schema.AccessMode.READ_ONLY) @PathVariable ReportType reportType,
@PathVariable String timeStamp,
@Schema(type = "string", example = "20240310", pattern = "^[0-9]{8}$") @Parameter String startTime,
@Schema(type = "string", example = "20240409", pattern = "^[0-9]{8}$") @Parameter String endTime) {
log.info("Start to export CSV file_reportType: {}, filename: {}", reportType.getValue(), timeStamp);
InputStreamResource result = reportService.exportCsv(reportType, timeStamp, startTime, endTime);
log.info("Successfully get CSV file_reportType: {}, filename: {}, _result: {}", reportType.getValue(),
timeStamp, result);
return result;
}

@GetMapping("/{reportId}")
public ResponseEntity<ReportResponse> generateReport(@PathVariable String reportId) {
log.info("Start to generate report_reportId: {}", reportId);
ReportResponse reportResponse = generateReporterService.getComposedReportResponse(reportId);
@GetMapping("/{timeStamp}")
public ResponseEntity<ReportResponse> generateReport(@PathVariable String timeStamp,
@Schema(type = "string", example = "20240310", pattern = "^[0-9]{8}$") @Parameter String startTime,
@Schema(type = "string", example = "20240409", pattern = "^[0-9]{8}$") @Parameter String endTime) {
log.info("Start to generate report_reportId: {}", timeStamp);
ReportResponse reportResponse = generateReporterService.getComposedReportResponse(timeStamp, startTime,
endTime);
return ResponseEntity.status(HttpStatus.OK).body(reportResponse);
}

@PostMapping
public ResponseEntity<CallbackResponse> generateReport(@RequestBody GenerateReportRequest request) {
log.info("Start to generate report");
reportService.generateReport(request);
String callbackUrl = "/reports/" + request.getCsvTimeStamp();
String callbackUrl = "/reports/" + request.getCsvTimeStamp() + "?startTime="
+ TimeUtil.convertToChinaSimpleISOFormat(Long.parseLong(request.getStartTime())) + "&endTime="
+ TimeUtil.convertToChinaSimpleISOFormat(Long.parseLong(request.getEndTime()));
log.info("Successfully generate report");
return ResponseEntity.status(HttpStatus.ACCEPTED)
.body(CallbackResponse.builder().callbackUrl(callbackUrl).interval(interval).build());
Expand Down
20 changes: 14 additions & 6 deletions backend/src/main/java/heartbeat/controller/report/dto/request/GenerateReportRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import heartbeat.util.IdUtil;
import heartbeat.util.MetricsUtil;
import heartbeat.util.TimeUtil;
import jakarta.validation.constraints.NotBlank;
import lombok.AllArgsConstructor;
import lombok.Builder;
Expand Down Expand Up @@ -69,18 +70,25 @@ public List<String> getBoardMetrics() {
}

@JsonIgnore
public String getPipelineReportId() {
return IdUtil.getPipelineReportId(this.csvTimeStamp);
public String getTimeRangeAndTimeStamp() {
return TimeUtil.convertToChinaSimpleISOFormat(Long.parseLong(this.startTime)) + "-"
+ TimeUtil.convertToChinaSimpleISOFormat(Long.parseLong(this.endTime)) + "-" + this.csvTimeStamp;

}

@JsonIgnore
public String getPipelineReportFileId() {
return IdUtil.getPipelineReportFileId(this.getTimeRangeAndTimeStamp());
}

@JsonIgnore
public String getSourceControlReportId() {
return IdUtil.getSourceControlReportId(this.csvTimeStamp);
public String getSourceControlReportFileId() {
return IdUtil.getSourceControlReportFileId(this.getTimeRangeAndTimeStamp());
}

@JsonIgnore
public String getBoardReportId() {
return IdUtil.getBoardReportId(this.csvTimeStamp);
public String getBoardReportFileId() {
return IdUtil.getBoardReportFileId(this.getTimeRangeAndTimeStamp());
}

@JsonIgnore
Expand Down
33 changes: 21 additions & 12 deletions backend/src/main/java/heartbeat/service/report/CSVFileGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -65,14 +66,18 @@ public class CSVFileGenerator {

public static final String FILE_LOCAL_PATH = "./app/output/csv";

private static final Path FILE_PATH = new File(FILE_LOCAL_PATH).toPath().normalize();

private static final String CANCELED_STATUS = "canceled";

private static final String REWORK_FIELD = "Rework";

private static InputStreamResource readStringFromCsvFile(String fileName) {
public static InputStreamResource readStringFromCsvFile(File file) {
if (!file.toPath().normalize().startsWith(FILE_PATH)) {
throw new IllegalArgumentException("Invalid file path");
}
try {
InputStream inputStream = new FileInputStream(fileName);

InputStream inputStream = new FileInputStream(file);
return new InputStreamResource(inputStream);
}
catch (IOException e) {
Expand Down Expand Up @@ -157,14 +162,18 @@ private String[] getRowData(PipelineCSVInfo csvInfo) {
pipelineFinishTime, totalTime, prLeadTime, pipelineLeadTime, state, branch, isRevert };
}

public InputStreamResource getDataFromCSV(ReportType reportDataType, long csvTimeStamp) {
public InputStreamResource getDataFromCSV(ReportType reportDataType, String timeRangeAndTimeStamp) {
if (timeRangeAndTimeStamp.contains("..") || timeRangeAndTimeStamp.contains("/")
|| timeRangeAndTimeStamp.contains("\\")) {
throw new IllegalArgumentException("Invalid time range time stamp");
}
return switch (reportDataType) {
case METRIC -> readStringFromCsvFile(
CSVFileNameEnum.METRIC.getValue() + FILENAME_SEPARATOR + csvTimeStamp + CSV_EXTENSION);
case PIPELINE -> readStringFromCsvFile(
CSVFileNameEnum.PIPELINE.getValue() + FILENAME_SEPARATOR + csvTimeStamp + CSV_EXTENSION);
default -> readStringFromCsvFile(
CSVFileNameEnum.BOARD.getValue() + FILENAME_SEPARATOR + csvTimeStamp + CSV_EXTENSION);
case METRIC -> readStringFromCsvFile(new File(FILE_LOCAL_PATH,
ReportType.METRIC.getValue() + FILENAME_SEPARATOR + timeRangeAndTimeStamp + CSV_EXTENSION));
case PIPELINE -> readStringFromCsvFile(new File(FILE_LOCAL_PATH,
ReportType.PIPELINE.getValue() + FILENAME_SEPARATOR + timeRangeAndTimeStamp + CSV_EXTENSION));
default -> readStringFromCsvFile(new File(FILE_LOCAL_PATH,
ReportType.BOARD.getValue() + FILENAME_SEPARATOR + timeRangeAndTimeStamp + CSV_EXTENSION));
};
}

Expand All @@ -182,10 +191,10 @@ public void convertBoardDataToCSV(List<JiraCardDTO> cardDTOList, List<BoardCSVCo
writeDataToCSV(csvTimeStamp, mergedArrays);
}

public void writeDataToCSV(String csvTimeStamp, String[][] mergedArrays) {
public void writeDataToCSV(String csvTimeRangeTimeStamp, String[][] mergedArrays) {
createCsvDirToConvertData();

String fileName = CSVFileNameEnum.BOARD.getValue() + FILENAME_SEPARATOR + csvTimeStamp + CSV_EXTENSION;
String fileName = CSVFileNameEnum.BOARD.getValue() + FILENAME_SEPARATOR + csvTimeRangeTimeStamp + CSV_EXTENSION;
if (!fileName.contains("..") && fileName.startsWith(FILE_LOCAL_PATH)) {
try (CSVWriter writer = new CSVWriter(new FileWriter(fileName))) {
writer.writeAll(Arrays.asList(mergedArrays));
Expand Down
71 changes: 39 additions & 32 deletions backend/src/main/java/heartbeat/service/report/GenerateReporterService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
import heartbeat.handler.AsyncMetricsDataHandler;
import heartbeat.handler.AsyncReportRequestHandler;
import heartbeat.handler.base.AsyncExceptionDTO;
import heartbeat.service.report.calculator.DevChangeFailureRateCalculator;
import heartbeat.service.report.calculator.ClassificationCalculator;
import heartbeat.service.report.calculator.CycleTimeCalculator;
import heartbeat.service.report.calculator.DeploymentFrequencyCalculator;
import heartbeat.service.report.calculator.DevChangeFailureRateCalculator;
import heartbeat.service.report.calculator.LeadTimeForChangesCalculator;
import heartbeat.service.report.calculator.MeanToRecoveryCalculator;
import heartbeat.service.report.calculator.ReworkCalculator;
Expand Down Expand Up @@ -84,8 +84,10 @@ public class GenerateReporterService {

private final AsyncExceptionHandler asyncExceptionHandler;

private static final char FILENAME_SEPARATOR = '-';

public void generateBoardReport(GenerateReportRequest request) {
String boardReportId = request.getBoardReportId();
String boardReportId = request.getBoardReportFileId();
removePreviousAsyncException(boardReportId);
log.info(
"Start to generate board report, _metrics: {}, _considerHoliday: {}, _startTime: {}, _endTime: {}, _boardReportId: {}",
Expand All @@ -102,14 +104,14 @@ public void generateBoardReport(GenerateReportRequest request) {
asyncExceptionHandler.put(boardReportId, e);
if (List.of(401, 403, 404).contains(e.getStatus()))
asyncMetricsDataHandler.updateMetricsDataCompletedInHandler(
IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()), BOARD, false);
IdUtil.getDataCompletedPrefix(request.getBoardReportFileId()), BOARD, false);

}
}

public void generateDoraReport(GenerateReportRequest request) {
removePreviousAsyncException(request.getPipelineReportId());
removePreviousAsyncException(request.getSourceControlReportId());
removePreviousAsyncException(request.getPipelineReportFileId());
removePreviousAsyncException(request.getSourceControlReportFileId());
FetchedData fetchedData = new FetchedData();
if (CollectionUtils.isNotEmpty(request.getPipelineMetrics())) {
GenerateReportRequest pipelineRequest = request.toPipelineRequest();
Expand All @@ -121,14 +123,14 @@ public void generateDoraReport(GenerateReportRequest request) {
}

MetricsDataCompleted previousMetricsCompleted = asyncMetricsDataHandler
.getMetricsDataCompleted(IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()));
.getMetricsDataCompleted(IdUtil.getDataCompletedPrefix(request.getTimeRangeAndTimeStamp()));
if (Boolean.FALSE.equals(previousMetricsCompleted.doraMetricsCompleted())) {
CompletableFuture.runAsync(() -> generateCSVForPipeline(request, fetchedData.getBuildKiteData()));
}
}

private void generatePipelineReport(GenerateReportRequest request, FetchedData fetchedData) {
String pipelineReportId = request.getPipelineReportId();
String pipelineReportId = request.getPipelineReportFileId();
log.info(
"Start to generate pipeline report, _metrics: {}, _considerHoliday: {}, _startTime: {}, _endTime: {}, _pipelineReportId: {}",
request.getPipelineMetrics(), request.getConsiderHoliday(), request.getStartTime(),
Expand All @@ -145,12 +147,12 @@ private void generatePipelineReport(GenerateReportRequest request, FetchedData f
asyncExceptionHandler.put(pipelineReportId, e);
if (List.of(401, 403, 404).contains(e.getStatus()))
asyncMetricsDataHandler.updateMetricsDataCompletedInHandler(
IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()), DORA, false);
IdUtil.getDataCompletedPrefix(request.getTimeRangeAndTimeStamp()), DORA, false);
}
}

private void generateSourceControlReport(GenerateReportRequest request, FetchedData fetchedData) {
String sourceControlReportId = request.getSourceControlReportId();
String sourceControlReportId = request.getSourceControlReportFileId();
log.info(
"Start to generate source control report, _metrics: {}, _considerHoliday: {}, _startTime: {}, _endTime: {}, _sourceControlReportId: {}",
request.getSourceControlMetrics(), request.getConsiderHoliday(), request.getStartTime(),
Expand All @@ -167,7 +169,7 @@ private void generateSourceControlReport(GenerateReportRequest request, FetchedD
asyncExceptionHandler.put(sourceControlReportId, e);
if (List.of(401, 403, 404).contains(e.getStatus()))
asyncMetricsDataHandler.updateMetricsDataCompletedInHandler(
IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()), DORA, false);
IdUtil.getDataCompletedPrefix(request.getTimeRangeAndTimeStamp()), DORA, false);
}
}

Expand Down Expand Up @@ -225,8 +227,8 @@ private synchronized ReportResponse generateBoardReporter(GenerateReportRequest
private void generateCsvForBoard(GenerateReportRequest request, FetchedData fetchedData) {
kanbanCsvService.generateCsvInfo(request, fetchedData.getCardCollectionInfo().getRealDoneCardCollection(),
fetchedData.getCardCollectionInfo().getNonDoneCardCollection());
asyncMetricsDataHandler
.updateMetricsDataCompletedInHandler(IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()), BOARD, true);
asyncMetricsDataHandler.updateMetricsDataCompletedInHandler(
IdUtil.getDataCompletedPrefix(request.getTimeRangeAndTimeStamp()), BOARD, true);
}

private void assembleVelocity(FetchedData fetchedData, ReportResponse reportResponse) {
Expand Down Expand Up @@ -300,13 +302,13 @@ private void generateCSVForPipeline(GenerateReportRequest request, BuildKiteData
List<PipelineCSVInfo> pipelineData = pipelineService.generateCSVForPipeline(request.getStartTime(),
request.getEndTime(), buildKiteData, request.getBuildKiteSetting().getDeploymentEnvList());

csvFileGenerator.convertPipelineDataToCSV(pipelineData, request.getCsvTimeStamp());
asyncMetricsDataHandler
.updateMetricsDataCompletedInHandler(IdUtil.getDataCompletedPrefix(request.getCsvTimeStamp()), DORA, true);
csvFileGenerator.convertPipelineDataToCSV(pipelineData, request.getTimeRangeAndTimeStamp());
asyncMetricsDataHandler.updateMetricsDataCompletedInHandler(
IdUtil.getDataCompletedPrefix(request.getTimeRangeAndTimeStamp()), DORA, true);
}

public void generateCSVForMetric(ReportResponse reportContent, String csvTimeStamp) {
csvFileGenerator.convertMetricDataToCSV(reportContent, csvTimeStamp);
public void generateCSVForMetric(ReportResponse reportContent, String csvTimeRangeTimeStamp) {
csvFileGenerator.convertMetricDataToCSV(reportContent, csvTimeRangeTimeStamp);
}

private void saveReporterInHandler(ReportResponse reportContent, String reportId) {
Expand Down Expand Up @@ -365,30 +367,34 @@ private ReportResponse getReportFromHandler(String reportId) {
return asyncReportRequestHandler.getReport(reportId);
}

public MetricsDataCompleted checkReportReadyStatus(String reportTimeStamp) {
if (validateExpire(System.currentTimeMillis(), Long.parseLong(reportTimeStamp))) {
public MetricsDataCompleted checkReportReadyStatus(String timeRangeAndTimeStamp) {
String timeStamp = timeRangeAndTimeStamp.substring(timeRangeAndTimeStamp.lastIndexOf(FILENAME_SEPARATOR) + 1);
if (validateExpire(System.currentTimeMillis(), Long.parseLong(timeStamp))) {
throw new GenerateReportException("Failed to get report due to report time expires");
}
return asyncMetricsDataHandler.getMetricsDataCompleted(IdUtil.getDataCompletedPrefix(reportTimeStamp));
return asyncMetricsDataHandler.getMetricsDataCompleted(IdUtil.getDataCompletedPrefix(timeRangeAndTimeStamp));
}

public ReportResponse getComposedReportResponse(String reportId) {
MetricsDataCompleted reportReadyStatus = checkReportReadyStatus(reportId);
public ReportResponse getComposedReportResponse(String timeStamp, String startTime, String endTime) {
String timeRangeAndTimeStamp = startTime + FILENAME_SEPARATOR + endTime + FILENAME_SEPARATOR + timeStamp;
MetricsDataCompleted reportReadyStatus = checkReportReadyStatus(timeRangeAndTimeStamp);

ReportResponse boardReportResponse = getReportFromHandler(IdUtil.getBoardReportId(reportId));
ReportResponse pipleineReportResponse = getReportFromHandler(IdUtil.getPipelineReportId(reportId));
ReportResponse sourceControlReportResponse = getReportFromHandler(IdUtil.getSourceControlReportId(reportId));
ReportResponse boardReportResponse = getReportFromHandler(IdUtil.getBoardReportFileId(timeRangeAndTimeStamp));
ReportResponse pipelineReportResponse = getReportFromHandler(
IdUtil.getPipelineReportFileId(timeRangeAndTimeStamp));
ReportResponse sourceControlReportResponse = getReportFromHandler(
IdUtil.getSourceControlReportFileId(timeRangeAndTimeStamp));

ReportMetricsError reportMetricsError = getReportErrorAndHandleAsyncException(reportId);
ReportMetricsError reportMetricsError = getReportErrorAndHandleAsyncException(timeRangeAndTimeStamp);
return ReportResponse.builder()
.velocity(getValueOrNull(boardReportResponse, ReportResponse::getVelocity))
.classificationList(getValueOrNull(boardReportResponse, ReportResponse::getClassificationList))
.cycleTime(getValueOrNull(boardReportResponse, ReportResponse::getCycleTime))
.rework(getValueOrNull(boardReportResponse, ReportResponse::getRework))
.exportValidityTime(EXPORT_CSV_VALIDITY_TIME)
.deploymentFrequency(getValueOrNull(pipleineReportResponse, ReportResponse::getDeploymentFrequency))
.devChangeFailureRate(getValueOrNull(pipleineReportResponse, ReportResponse::getDevChangeFailureRate))
.devMeanTimeToRecovery(getValueOrNull(pipleineReportResponse, ReportResponse::getDevMeanTimeToRecovery))
.deploymentFrequency(getValueOrNull(pipelineReportResponse, ReportResponse::getDeploymentFrequency))
.devChangeFailureRate(getValueOrNull(pipelineReportResponse, ReportResponse::getDevChangeFailureRate))
.devMeanTimeToRecovery(getValueOrNull(pipelineReportResponse, ReportResponse::getDevMeanTimeToRecovery))
.leadTimeForChanges(getValueOrNull(sourceControlReportResponse, ReportResponse::getLeadTimeForChanges))
.boardMetricsCompleted(reportReadyStatus.boardMetricsCompleted())
.doraMetricsCompleted(reportReadyStatus.doraMetricsCompleted())
Expand All @@ -400,9 +406,10 @@ public ReportResponse getComposedReportResponse(String reportId) {
}

private ReportMetricsError getReportErrorAndHandleAsyncException(String reportId) {
AsyncExceptionDTO boardException = asyncExceptionHandler.get(IdUtil.getBoardReportId(reportId));
AsyncExceptionDTO pipelineException = asyncExceptionHandler.get(IdUtil.getPipelineReportId(reportId));
AsyncExceptionDTO sourceControlException = asyncExceptionHandler.get(IdUtil.getSourceControlReportId(reportId));
AsyncExceptionDTO boardException = asyncExceptionHandler.get(IdUtil.getBoardReportFileId(reportId));
AsyncExceptionDTO pipelineException = asyncExceptionHandler.get(IdUtil.getPipelineReportFileId(reportId));
AsyncExceptionDTO sourceControlException = asyncExceptionHandler
.get(IdUtil.getSourceControlReportFileId(reportId));
return ReportMetricsError.builder()
.boardMetricsError(handleAsyncExceptionAndGetErrorInfo(boardException))
.pipelineMetricsError(handleAsyncExceptionAndGetErrorInfo(pipelineException))
Expand Down
Loading

0 comments on commit bb3e0b9

Please sign in to comment.