Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[JENKINS-67963] Add option to save bandwidth and resources, which are wasted unnecessarily #245

Merged
merged 7 commits into from
Jun 2, 2022
Merged

[JENKINS-67963] Add option to save bandwidth and resources, which are wasted unnecessarily #245

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
46c4a7a
initial implementation, no configuration
parvit May 22, 2022
fce0581
integrated control parameter in the server host configuration
parvit May 23, 2022
7ec1fe8
documentation
parvit May 23, 2022
55e7e8d
fix java target level
parvit May 23, 2022
76857d5
Revert space removal
parvit May 23, 2022
dbb244b
better skip message
parvit May 29, 2022
1df29c0
fixed again the space removed
parvit May 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,28 +25,27 @@

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>plugin</artifactId>
<version>4.19</version>
</parent>

<artifactId>publish-over-ssh</artifactId>
<properties>
<revision>1.25</revision>
<changelist>-SNAPSHOT</changelist>
<gitHubRepo>jenkinsci/publish-over-ssh-plugin</gitHubRepo>
<jenkins.version>2.289.1</jenkins.version>
<java.level>8</java.level>
</properties>

<artifactId>publish-over-ssh</artifactId>
<packaging>hpi</packaging>
<name>Publish Over SSH</name>
<version>${revision}${changelist}</version>
<description>Send build artifacts over SSH</description>
<url>https://github.com/jenkinsci/publish-over-ssh-plugin</url>

<properties>
<revision>1.25</revision>
<changelist>-SNAPSHOT</changelist>
<gitHubRepo>jenkinsci/publish-over-ssh-plugin</gitHubRepo>
<jenkins.version>2.289.1</jenkins.version>
<java.level>8</java.level>
</properties>

<licenses>
<license>
<name>The MIT license</name>
Expand Down Expand Up @@ -89,6 +88,11 @@
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>structs</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.13.0</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
Expand Down
45 changes: 39 additions & 6 deletions src/main/java/jenkins/plugins/publish_over_ssh/BapSshClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import hudson.FilePath;
import hudson.Util;
import jenkins.plugins.publish_over.BPBuildEnv;
import jenkins.plugins.publish_over.BPBuildInfo;
import jenkins.plugins.publish_over.BPDefaultClient;
import jenkins.plugins.publish_over.BapPublisherException;
Expand Down Expand Up @@ -60,14 +61,18 @@ public class BapSshClient extends BPDefaultClient<BapSshTransfer> {
private final Stack<Session> sessions = new Stack<>();
private final boolean disableExec;
private ChannelSftp sftp;
private final boolean avoidSameFileUpload;

private BapSshTransferCache remoteResourceCache;

public BapSshClient(final BPBuildInfo buildInfo, final Session session) {
this(buildInfo, session, false);
this(buildInfo, session, false, false);
}

public BapSshClient(final BPBuildInfo buildInfo, final Session session, final boolean disableExec) {
this.buildInfo = buildInfo;
public BapSshClient(final BPBuildInfo buildInfo, final Session session, final boolean disableExec, final boolean avoidSameFileUpload) {
this.buildInfo = buildInfo;
this.disableExec = disableExec;
this.avoidSameFileUpload = avoidSameFileUpload;
addSession(session);
}

Expand All @@ -83,6 +88,10 @@ public boolean isDisableExec() {
return disableExec;
}

public boolean isAvoidSameFileUpload() {
return avoidSameFileUpload;
}

public BPBuildInfo getBuildInfo() {
return buildInfo;
}
Expand All @@ -103,6 +112,16 @@ public void beginTransfers(final BapSshTransfer transfer) {
if (!transfer.hasConfiguredSourceFiles() && !transfer.hasExecCommand())
throw new BapPublisherException(Messages.exception_badTransferConfig());
}

BPBuildEnv buildEnv = getBuildInfo().getCurrentBuildEnv();
String jobName = buildEnv.getEnvVars().get(BPBuildEnv.ENV_JOB_NAME);

FilePath jobConfigPath = getBuildInfo().getConfigDir()
.child("jobs")
.child(jobName);

if( isAvoidSameFileUpload() )
this.remoteResourceCache = new BapSshTransferCache(jobConfigPath);
}

public boolean changeDirectory(final String directory) {
Expand Down Expand Up @@ -180,9 +199,18 @@ public boolean makeDirectory(final String directory) {

public void transferFile(final BapSshTransfer bapSshTransfer, final FilePath filePath,
final InputStream inputStream) throws SftpException {
buildInfo.printIfVerbose(Messages.console_put(filePath.getName()));
sftp.put(inputStream, filePath.getName());
success();

if( !isAvoidSameFileUpload() || remoteResourceCache.checkCachedResource(filePath) ) {
buildInfo.printIfVerbose(Messages.console_put(filePath.getName()));
sftp.put(inputStream, filePath.getName());
success();
}
else
{
buildInfo.println(Messages._console_warning(
Messages.console_message_transferskip(filePath.getName()) )
.toString());
}
}

private void success() {
Expand All @@ -194,6 +222,11 @@ private boolean hasSubDirs(final String directory) {
}

public void endTransfers(final BapSshTransfer transfer) {
if( isAvoidSameFileUpload() ) {
remoteResourceCache.save();
remoteResourceCache = null;
}

if (!disableExec && transfer.hasExecCommand()) {
if (transfer.isUseSftpForExec())
sftpExec(transfer);
Expand Down
32 changes: 22 additions & 10 deletions src/main/java/jenkins/plugins/publish_over_ssh/BapSshHostConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@

@SuppressWarnings("PMD.TooManyMethods")
public class BapSshHostConfiguration extends BPHostConfiguration<BapSshClient, BapSshCommonConfiguration> implements Describable<BapSshHostConfiguration> {

static final String LOCALHOST = "127.0.0.1";
private static final long serialVersionUID = 1L;
public static final int DEFAULT_PORT = 22;
Expand All @@ -62,6 +62,8 @@ public class BapSshHostConfiguration extends BPHostConfiguration<BapSshClient, B
public static final String SOCKS_4_PROXY_TYPE = "socks4";
public static final String SOCKS_5_PROXY_TYPE = "socks5";

public static final boolean DEFAULT_AVOID_SAME_FILES_UPLOAD = false;

private int timeout;
private boolean overrideKey;
private boolean disableExec;
Expand All @@ -78,6 +80,8 @@ public class BapSshHostConfiguration extends BPHostConfiguration<BapSshClient, B
private String proxyPassword;
private Secret secretProxyPassword;

private boolean avoidSameFileUploads;

public BapSshHostConfiguration() {
// use this constructor instead of the default w/o parameters because there is some
// business logic in there...
Expand All @@ -90,20 +94,21 @@ public BapSshHostConfiguration() {
@DataBoundConstructor
public BapSshHostConfiguration(final String name, final String hostname, final String username, final String encryptedPassword,
final String remoteRootDir, final int port, final int timeout, final boolean overrideKey, final String keyPath,
final String key, final boolean disableExec) {
final String key, final boolean disableExec, final boolean avoidSameFileUploads) {
// CSON: ParameterNumberCheck
super(name, hostname, username, null, remoteRootDir, port);
this.timeout = timeout;
this.overrideKey = overrideKey;
this.keyInfo = new BapSshKeyInfo(encryptedPassword, key, keyPath);
this.disableExec = disableExec;
this.avoidSameFileUploads = avoidSameFileUploads;
}

@DataBoundSetter
public void setJumpHost(final String jumpHost) {
this.jumpHost = jumpHost;
}

public String getJumpHost() {
return jumpHost;
}
Expand Down Expand Up @@ -225,6 +230,10 @@ public void setSecretProxyPassword(Secret secretProxyPassword) {
this.secretProxyPassword = secretProxyPassword;
}

public boolean isAvoidSameFileUploads() {
return avoidSameFileUploads;
}

@Override
public Object readResolve() {
if(StringUtils.isNotEmpty(proxyPassword)) {
Expand Down Expand Up @@ -263,7 +272,7 @@ public BapSshClient createClient(final BPBuildInfo buildInfo, final boolean conn
String[] hosts = getHosts();
Session session = createSession(buildInfo, ssh, hosts[0], getPort());
configureAuthentication(buildInfo, ssh, session);
final BapSshClient bapClient = new BapSshClient(buildInfo, session, isEffectiveDisableExec());
final BapSshClient bapClient = new BapSshClient(buildInfo, session, isEffectiveDisableExec(), isAvoidSameFileUploads());
try {
connect(buildInfo, session);
for (int i = 1; i < hosts.length; i++) {
Expand All @@ -277,14 +286,14 @@ public BapSshClient createClient(final BPBuildInfo buildInfo, final boolean conn
setupSftp(bapClient);
} catch (IOException | JSchException | BapPublisherException e) {
bapClient.disconnectQuietly();
throw new BapPublisherException(Messages.exception_failedToCreateClient(e.getLocalizedMessage()), e);
throw new BapPublisherException(Messages.exception_failedToCreateClient(e.getLocalizedMessage()), e);
}
return bapClient;
}

/**
* create a list of hosts from the explicit stated target host and an optional list of jumphosts
*
*
* @return list of hosts
*/
String[] getHosts() {
Expand Down Expand Up @@ -467,7 +476,8 @@ protected EqualsBuilder addToEquals(final EqualsBuilder builder, final BapSshHos
.append(proxyHost, that.proxyHost)
.append(proxyPort, that.proxyPort)
.append(proxyUser, that.proxyUser)
.append(secretProxyPassword, that.secretProxyPassword);
.append(secretProxyPassword, that.secretProxyPassword)
.append(avoidSameFileUploads, that.avoidSameFileUploads);
}

@Override
Expand All @@ -482,7 +492,8 @@ protected HashCodeBuilder addToHashCode(final HashCodeBuilder builder) {
.append(proxyHost)
.append(proxyPort)
.append(proxyUser)
.append(secretProxyPassword.getPlainText());
.append(secretProxyPassword.getPlainText())
.append(avoidSameFileUploads);
}

@Override
Expand All @@ -497,7 +508,8 @@ protected ToStringBuilder addToToString(final ToStringBuilder builder) {
.append("proxyHost", proxyHost)
.append("proxyPort", proxyPort)
.append("proxyUser", proxyUser)
.append("proxyPassword", "xxxxxxx");
.append("proxyPassword", "xxxxxxx")
.append("avoidSameFileUploads", avoidSameFileUploads);
}

@Override
Expand Down
Loading