Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#311 keep archive on download #520

Merged
merged 4 commits into from
Jan 11, 2023
Merged

#311 keep archive on download #520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
808f7fb
#311 keep archive on download
yzerk Dec 28, 2022
5e0fe7e
#311 - fix review notes
yzerk Jan 3, 2023
19c7c23
#311 - fix review notes
yzerk Jan 4, 2023
e5fe369
#311 - fix review notes
yzerk Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/crowdin/cli/commands/Actions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public interface Actions {

NewAction<PropertiesWithFiles, ProjectClient> download(
FilesInterface files, boolean noProgress, List<String> languageIds, boolean pseudo, String branchName,
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean userServerSources
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean userServerSources, boolean keepArchive
);

NewAction<NoProperties, NoClient> generate(FilesInterface files, Path destinationPath, boolean skipGenerateDescription);
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/crowdin/cli/commands/actions/CliActions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public class CliActions implements Actions {
@Override
public NewAction<PropertiesWithFiles, ProjectClient> download(
FilesInterface files, boolean noProgress, List<String> languageIds, boolean pseudo, String branchName,
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources, boolean keepArchive
) {
return new DownloadAction(files, noProgress, languageIds, pseudo, branchName, ignoreMatch, isVerbose, plainView, useServerSources);
return new DownloadAction(files, noProgress, languageIds, pseudo, branchName, ignoreMatch, isVerbose, plainView, useServerSources, keepArchive);
}

@Override
Expand Down
22 changes: 14 additions & 8 deletions src/main/java/com/crowdin/cli/commands/actions/DownloadAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,13 @@ class DownloadAction implements NewAction<PropertiesWithFiles, ProjectClient> {
private boolean isVerbose;
private boolean plainView;
private boolean useServerSources;
private boolean keepArchive;

private Outputter out;

public DownloadAction(
FilesInterface files, boolean noProgress, List<String> languageIds, boolean pseudo, String branchName,
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources
boolean ignoreMatch, boolean isVerbose, boolean plainView, boolean useServerSources, boolean keepArchive
) {
this.files = files;
this.noProgress = noProgress || plainView;
Expand All @@ -80,6 +81,7 @@ public DownloadAction(
this.isVerbose = isVerbose;
this.plainView = plainView;
this.useServerSources = useServerSources;
this.keepArchive = keepArchive;
}

@Override
Expand Down Expand Up @@ -146,7 +148,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
: RequestBuilder.crowdinTranslationCreateProjectPseudoBuildForm(true, null, null, null, null);
}

Pair<File, List<String>> downloadedFiles = this.download(request, client, pb.getBasePath());
Pair<File, List<String>> downloadedFiles = this.download(request, client, pb.getBasePath(), keepArchive);
for (FileBean fb : pb.getFiles()) {
Map<String, String> filesWithMapping = this.getFiles(fb, pb.getBasePath(), serverLanguageMapping, forLanguages, placeholderUtil, new ArrayList<>(serverSources.keySet()), pb.getPreserveHierarchy());
fileBeansWithDownloadedFiles.putIfAbsent(downloadedFiles.getLeft(), new ArrayList<>());
Expand Down Expand Up @@ -192,7 +194,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
out.println(WARNING.withIcon(RESOURCE_BUNDLE.getString("error.export_strings_that_passed_workflow_not_supported")));
}
}
Pair<File, List<String>> downloadedFiles = this.download(buildRequest, client, pb.getBasePath());
Pair<File, List<String>> downloadedFiles = this.download(buildRequest, client, pb.getBasePath(), keepArchive);
for (FileBean fb : pb.getFiles()) {
if (fb.getSkipTranslatedOnly() == downloadConfiguration.getLeft().getLeft()
&& fb.getSkipUntranslatedFiles() == downloadConfiguration.getLeft().getRight()
Expand Down Expand Up @@ -325,7 +327,7 @@ public void act(Outputter out, PropertiesWithFiles pb, ProjectClient client) {
* @param basePath base path
* @return pair of temporary directory and list of files in it(relative paths to that directory)
*/
private Pair<File, List<String>> download(BuildProjectTranslationRequest request, ProjectClient client, String basePath) {
private Pair<File, List<String>> download(BuildProjectTranslationRequest request, ProjectClient client, String basePath, Boolean keepArchive) {
ProjectBuild projectBuild = buildTranslation(client, request);
String randomHash = RandomStringUtils.random(11, false, true);
File baseTempDir =
Expand All @@ -344,10 +346,14 @@ private Pair<File, List<String>> download(BuildProjectTranslationRequest request
.removeStart(f.getAbsolutePath(), baseTempDir.getAbsolutePath() + Utils.PATH_SEPARATOR))
.collect(Collectors.toList());

try {
files.deleteFile(downloadedZipArchive);
} catch (IOException e) {
out.println(ERROR.withIcon(String.format(RESOURCE_BUNDLE.getString("error.deleting_archive"), downloadedZipArchive)));
if (!keepArchive) {
andrii-bodnar marked this conversation as resolved.
Show resolved Hide resolved
try {
files.deleteFile(downloadedZipArchive);
} catch (IOException e) {
out.println(ERROR.withIcon(String.format(RESOURCE_BUNDLE.getString("error.deleting_archive"), downloadedZipArchive)));
}
} else {
out.println(OK.withIcon(String.format(RESOURCE_BUNDLE.getString("message.archive"), downloadedZipArchivePath)));
}
return Pair.of(baseTempDir, downloadedFilesProc);
}
Expand Down
5 changes: 4 additions & 1 deletion src/main/java/com/crowdin/cli/commands/picocli/DownloadSubcommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,17 @@ class DownloadSubcommand extends ActCommandWithFiles {
@CommandLine.Option(names = {"--export-only-approved"}, descriptionKey = "params.exportOnlyApproved")
protected Boolean exportApprovedOnly;

@CommandLine.Option(names = {"--keep-archive"}, descriptionKey = "params.keepArchive")
protected boolean keepArchive;

@CommandLine.Option(names = {"--all"})
protected boolean all;

@Override
protected NewAction<PropertiesWithFiles, ProjectClient> getAction(Actions actions) {
return (dryrun)
? actions.listTranslations(noProgress, treeView, false, plainView, all, true)
: actions.download(new FsFiles(), noProgress, languageIds, pseudo, branchName, ignoreMatch, isVerbose, plainView, all);
: actions.download(new FsFiles(), noProgress, languageIds, pseudo, branchName, ignoreMatch, isVerbose, plainView, all, keepArchive);
}

@CommandLine.Option(names = {"--plain"}, descriptionKey = "crowdin.list.usage.plain")
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/messages/messages.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ params.translation=Path to the translation files
params.dest=Specify file name in Crowdin
params.skipUntranslatedStrings=Skip untranslated strings in exported files (does not work with .docx, .html, .md and other document files)
params.skipUntranslatedFiles=Omit downloading not fully translated files
params.keepArchive=Do not remove the downloaded archive with translations after it's extracting
params.exportOnlyApproved=Include approved translations only in exported files. If not combined with --skip-untranslated-strings option, strings without approval are fulfilled with the source language
params.label=Attach labels to strings (multiple labels could be specified)
params.excluded-languages=Specify excluded languages (multiple languages could be specified)
Expand Down Expand Up @@ -405,6 +406,7 @@ message.build_archive_pseudo=Building ZIP archive with the pseudo-localized tran
message.building_translation=Building translation @|bold (%d%%)|@
message.building_reviewed_sources=Building reviewed sources @|bold (%d%%)|@
message.extracted_file=Extracted: @|bold '%s'|@
message.archive=Archive: @|bold '%s'|@
message.downloaded_files_omitted=Downloaded translations don't match the current project configuration. The translations for the following sources will be omitted (use --verbose to get the list of the omitted translations)\:
message.downloaded_files_omitted_without_sources=Due to missing respective sources, the following translations will be omitted\:
message.item_list=\t- %s
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/crowdin/cli/commands/actions/CliActionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ public class CliActionsTest {

@Test
public void testDownload() {
assertNotNull(actions.download(new FsFiles(), false, null, false, null, false, false, false, false));
assertNotNull(actions.download(new FsFiles(), false, null, false, null, false, false, false, false, false));
}

@Test
Expand Down
79 changes: 67 additions & 12 deletions src/test/java/com/crowdin/cli/commands/actions/DownloadActionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public void testEmptyProject() throws ResponseException, IOException {
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -137,7 +137,7 @@ public void testProjectOneFittingFile() throws ResponseException, IOException {
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -198,7 +198,7 @@ public void testProjectOneFittingFile_WithExportApprovedOnly_WithSkipUntranslate
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand All @@ -219,6 +219,61 @@ public void testProjectOneFittingFile_WithExportApprovedOnly_WithSkipUntranslate
verifyNoMoreInteractions(files);
}

@Test
public void testProjectDownloadWithKeepArchive() throws IOException {
NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
.minimalBuiltPropertiesBean("*", Utils.PATH_SEPARATOR + "%original_file_name%-CR-%locale%")
.setBasePath(project.getBasePath());
PropertiesWithFiles pb = pbBuilder.build();

project.addFile("first.po");

ProjectClient client = mock(ProjectClient.class);
when(client.downloadFullProject())
.thenReturn(ProjectBuilder.emptyProject(Long.parseLong(pb.getProjectId()))
.addFile("first.po", "gettext", 101L, null, null, "/%original_file_name%-CR-%locale%").build());
CrowdinTranslationCreateProjectBuildForm buildProjectTranslationRequest = new CrowdinTranslationCreateProjectBuildForm();
long buildId = 42L;
when(client.startBuildingTranslation(eq(buildProjectTranslationRequest)))
.thenReturn(buildProjectBuild(buildId, Long.parseLong(pb.getProjectId()), "finished", 100));
URL urlMock = MockitoUtils.getMockUrl(getClass());
when(client.downloadBuild(eq(buildId)))
.thenReturn(urlMock);

FilesInterface files = mock(FilesInterface.class);
AtomicReference<File> zipArchive = new AtomicReference<>();
AtomicReference<File> tempDir = new AtomicReference<>();
when(files.extractZipArchive(any(), any()))
.thenAnswer((invocation -> {
zipArchive.set(invocation.getArgument(0));
tempDir.set(invocation.getArgument(1));
return new ArrayList<File>() {{
add(new File(tempDir.get().getAbsolutePath() + Utils.PATH_SEPARATOR + "first.po-CR-uk-UA"));
add(new File(tempDir.get().getAbsolutePath() + Utils.PATH_SEPARATOR + "first.po-CR-ru-RU"));
}};
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false, true);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
verify(client).startBuildingTranslation(eq(buildProjectTranslationRequest));
verify(client).downloadBuild(eq(buildId));
verifyNoMoreInteractions(client);

verify(files).writeToFile(any(), any());
verify(files).extractZipArchive(any(), any());
verify(files).copyFile(
new File(tempDir.get().getAbsolutePath() + Utils.PATH_SEPARATOR + "first.po-CR-ru-RU"),
new File(pb.getBasePath() + "first.po-CR-ru-RU"));
verify(files).copyFile(
new File(tempDir.get().getAbsolutePath() + Utils.PATH_SEPARATOR + "first.po-CR-uk-UA"),
new File(pb.getBasePath() + "first.po-CR-uk-UA"));
verify(files).deleteDirectory(tempDir.get());
verifyNoMoreInteractions(files);
}

@Test
public void testProjectOneFittingFile_LongBuild() throws ResponseException, IOException {
NewPropertiesWithFilesUtilBuilder pbBuilder = NewPropertiesWithFilesUtilBuilder
Expand Down Expand Up @@ -258,7 +313,7 @@ public void testProjectOneFittingFile_LongBuild() throws ResponseException, IOEx
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -319,7 +374,7 @@ public void testProjectOneFittingOneUnfittingFile_LongBuild() throws ResponseExc
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -380,7 +435,7 @@ public void testProjectOneFittingOneUnfittingOneWithUnfoundSourceFile_LongBuild(
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, true, false, false);
new DownloadAction(files, false, null, false, null, false, true, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -442,7 +497,7 @@ public void testProjectOneFittingFile_WithLanguageMapping() throws ResponseExcep
}));

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, true, false, false);
new DownloadAction(files, false, null, false, null, false, true, false, false, false);
action.act(Outputter.getDefault(), pb, client);

verify(client).downloadFullProject();
Expand Down Expand Up @@ -487,7 +542,7 @@ public void testProjectOneFittingFile_FailBuild() throws ResponseException, IOEx
FilesInterface files = mock(FilesInterface.class);

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, client));

verify(client).downloadFullProject();
Expand All @@ -513,7 +568,7 @@ public void testProjectOneFittingFile_failDownloadProject() throws ResponseExcep
FilesInterface files = mock(FilesInterface.class);

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, client));

verify(client).downloadFullProject();
Expand Down Expand Up @@ -561,7 +616,7 @@ public void testProjectOneFittingFile_failDeleteFile() throws ResponseException,
.when(files).deleteFile(any());

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
// assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, client));
action.act(Outputter.getDefault(), pb, client);

Expand Down Expand Up @@ -606,7 +661,7 @@ public void testProjectOneFittingFile_failDownloadingException() throws Response
FilesInterface files = mock(FilesInterface.class);

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, client));

verify(client).downloadFullProject();
Expand Down Expand Up @@ -644,7 +699,7 @@ public void testProjectOneFittingFile_failWritingFile() throws ResponseException
.writeToFile(any(), any());

NewAction<PropertiesWithFiles, ProjectClient> action =
new DownloadAction(files, false, null, false, null, false, false, false, false);
new DownloadAction(files, false, null, false, null, false, false, false, false, false);
assertThrows(RuntimeException.class, () -> action.act(Outputter.getDefault(), pb, client));

verify(client).downloadFullProject();
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/crowdin/cli/commands/picocli/DownloadSubcommandTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class DownloadSubcommandTest extends PicocliTestUtils {
public void testDownload() {
this.execute(CommandNames.DOWNLOAD, "--debug");
verify(actionsMock)
.download(any(), anyBoolean(), any(), anyBoolean(), any(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean());
.download(any(), anyBoolean(), any(), anyBoolean(), any(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean());
this.check(true);
}

Expand Down
2 changes: 1 addition & 1 deletion src/test/java/com/crowdin/cli/commands/picocli/PicocliTestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ void mockActions() {
actionsMock = mock(Actions.class);
actionMock = mock(NewAction.class);

when(actionsMock.download(any(), anyBoolean(), any(), anyBoolean(), any(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean()))
when(actionsMock.download(any(), anyBoolean(), any(), anyBoolean(), any(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean(), anyBoolean()))
.thenReturn(actionMock);
when(actionsMock.generate(any(), any(), anyBoolean()))
.thenReturn(actionMock);
Expand Down