diff --git a/lang-java/pom.xml b/lang-java/pom.xml index db24bc8fc..e929c399c 100644 --- a/lang-java/pom.xml +++ b/lang-java/pom.xml @@ -94,7 +94,7 @@ commons-compress test - + com.xebialabs.restito @@ -110,18 +110,48 @@ - org.apache.maven.plugins maven-shade-plugin + + + + 1 package shade + + + ch.uzh.ifi.seal:changedistiller + + + + + ch.uzh.ifi.seal:changedistiller + + META-INF/** + + + + + + + + + + 2 + package + + shade + + + jar-with-dependencies + true @@ -132,18 +162,29 @@ - com.sap.research.security.vulas:vulas-core - ch.uzh.ifi.seal:changedistiller + *:* + + ch.uzh.ifi.seal:changedistiller + - ch.uzh.ifi.seal:changedistiller + *:* - META-INF/** + META-INF/MANIFEST.MF + + + org.apache.http + org.eclipse.steady.repackaged.org.apache.http + + org.apache.http.* + + + @@ -164,38 +205,6 @@ - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - true - - true - true - - - - com.sap.psr.vulas.monitor.DynamicTransformer - - false - - - - - make-assembly - package - - single - - - - diff --git a/lang/pom.xml b/lang/pom.xml index 5a4438d72..d9ff1b28b 100644 --- a/lang/pom.xml +++ b/lang/pom.xml @@ -190,45 +190,6 @@ - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - - - - com.sap.psr.vulas.monitor.DynamicTransformer - - - - - - com.sap.research.security.vulas:vulas-core - ch.uzh.ifi.seal:changedistiller - - - - - ch.uzh.ifi.seal:changedistiller - - META-INF/** - - - - - - - - org.apache.maven.plugins diff --git a/lang/src/main/java/com/sap/psr/vulas/backend/requests/BasicHttpRequest.java b/lang/src/main/java/com/sap/psr/vulas/backend/requests/BasicHttpRequest.java old mode 100644 new mode 100755 index 672a9d6ab..e72df7307 --- a/lang/src/main/java/com/sap/psr/vulas/backend/requests/BasicHttpRequest.java +++ b/lang/src/main/java/com/sap/psr/vulas/backend/requests/BasicHttpRequest.java @@ -19,28 +19,32 @@ */ package com.sap.psr.vulas.backend.requests; -import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.ObjectInputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import java.util.Map; - +import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - +import org.apache.http.Header; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.HttpClients; import com.sap.psr.vulas.backend.BackendConnectionException; import com.sap.psr.vulas.backend.HttpMethod; import com.sap.psr.vulas.backend.HttpResponse; @@ -297,216 +301,221 @@ public void deletePayloadFromDisk() throws IOException { } private final HttpResponse sendRequest() throws BackendConnectionException { - HttpResponse response = null; - HttpURLConnection connection = null; - int response_code = -1; - final URI uri = this.getUri(); - Map> request_fields = null; - final RequestRepeater repeater = new RequestRepeater(this.getVulasConfiguration().getConfiguration().getLong(CoreConfiguration.REPEAT_MAX, 50), this.getVulasConfiguration().getConfiguration().getLong(CoreConfiguration.REPEAT_WAIT, 60000)); - - boolean is_503; - try { - do { - is_503 = false; - - final long start_nano = System.nanoTime(); - - connection = (HttpURLConnection)uri.toURL().openConnection(); - connection.setRequestMethod(this.method.toString().toUpperCase()); - - // Include tenant and space Http headers - String tenant_token = null, space_token = null; - if(this.context!=null && this.context.hasTenant()) { - tenant_token = this.context.getTenant().getTenantToken(); - connection.setRequestProperty(Constants.HTTP_TENANT_HEADER, tenant_token); - } - if(this.context!=null && this.context.hasSpace()) { - space_token = this.context.getSpace().getSpaceToken(); - connection.setRequestProperty(Constants.HTTP_SPACE_HEADER, space_token); - } - - // Include version and component as request header - connection.setRequestProperty(Constants.HTTP_VERSION_HEADER, CoreConfiguration.getVulasRelease()); - connection.setRequestProperty(Constants.HTTP_COMPONENT_HEADER, Constants.VulasComponent.client.toString()); - - // Include additional headers from configuration (if any) - final Map add_headers = this.getVulasConfiguration().getServiceHeaders(this.service); - if(add_headers!=null && add_headers.size()>0) { - for(String key: add_headers.keySet()) { - connection.setRequestProperty(key, add_headers.get(key)); - } - } - - // Only if put something in the body - if(this.hasPayload()) { - connection.setRequestProperty("Content-Type", "application/json; charset=utf-8"); - connection.setRequestProperty("Content-Length", Integer.toString(this.payload.getBytes().length)); - connection.setRequestProperty("Content-Language", "en-US"); - } - else if(this.binPayload!=null){ - connection.setRequestProperty("Content-Type", this.contentType); - } - - if(!this.hasPayload()) - BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); - else if(this.binPayload==null) - BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + ", size=" + StringUtil.byteToKBString(this.payload.getBytes().length) + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); - else - BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + ", size=" + this.binPayload.available() + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); - - connection.setUseCaches(false); - connection.setDoInput(true); - - if(this.hasPayload()) { - connection.setDoOutput(true); - request_fields = connection.getRequestProperties(); - final DataOutputStream wr = new DataOutputStream (connection.getOutputStream ()); - wr.write(this.payload.getBytes("UTF-8")); - wr.flush(); - wr.close(); - } - else if(this.binPayload!=null){ - connection.setDoOutput(true); - //required only for multipart/fileupload - // String boundary = Long.toHexString(System.currentTimeMillis()); // Just generate some unique random value. - // String CRLF = "\r\n"; // Line separator required by multipart/form-data. - - request_fields = connection.getRequestProperties(); - final OutputStream output = connection.getOutputStream(); - // PrintWriter writer = new PrintWriter(new OutputStreamWriter(output), true); - - byte[] buffer = new byte[4096]; - int length; - while ((length = this.binPayload.read(buffer)) > 0) { - output.write(buffer, 0, length); - } - output.flush(); - output.close(); - // writer.append(CRLF).flush(); - // writer.append("--" + boundary + "--").append(CRLF).flush(); - } - else { - connection.setDoOutput(false); - request_fields = connection.getRequestProperties(); - connection.connect(); - } - - // Read response - response_code = connection.getResponseCode(); - response = new HttpResponse(response_code); - - // If the response body contains a JAR file, save it - if(response.isOk() && connection.getContentType()!=null && connection.getContentType().contains("application/java-archive")){ - String fileName = ""; - String disposition = connection.getHeaderField("Content-Disposition"); - if (disposition != null) { - // Extracts file name from header field - int index = disposition.indexOf("filename="); - if (index > 0) { - fileName = disposition.substring(index + 9, disposition.length() ); - } - } else { - // Extracts file name from URL - fileName = this.path.substring(this.path.lastIndexOf("/") + 1, this.path.length()); - } - - // Opens input stream from the HTTP connection - InputStream inputStream = connection.getInputStream(); - String saveFilePath = null; - if (this.dir!=null){ - //create directories if not existing - if(!Files.exists(Paths.get(dir))){ - Files.createDirectories(Paths.get(dir)); - } - saveFilePath= dir + File.separator + fileName; - } - else - saveFilePath= Paths.get(this.getVulasConfiguration().getTmpDir().toString()).toString()+ File.separator + fileName; - - // Opens an output stream to save into file - FileOutputStream outputStream = new FileOutputStream(saveFilePath); + int response_code = -1; + org.apache.http.HttpResponse httpResponse = null; + final URI uri = this.getUri(); + HttpUriRequest httpUriRequest = null; + final RequestRepeater repeater = new RequestRepeater(this.getVulasConfiguration().getConfiguration().getLong(CoreConfiguration.REPEAT_MAX, 50), this.getVulasConfiguration().getConfiguration().getLong(CoreConfiguration.REPEAT_WAIT, 60000)); + + boolean is_503; + RequestBuilder requestBuilder = null; + switch (this.method) { + case GET: + requestBuilder = RequestBuilder.get(); + break; + case PUT: + requestBuilder = RequestBuilder.put(); + break; + case POST: + requestBuilder = RequestBuilder.post(); + break; + case OPTIONS: + requestBuilder = RequestBuilder.options(); + break; + case DELETE: + requestBuilder = RequestBuilder.delete(); + break; + } + + requestBuilder = requestBuilder.setUri(uri); + + // Include tenant and space Http headers + String tenant_token = null, space_token = null; + if(this.context!=null && this.context.hasTenant()) { + tenant_token = this.context.getTenant().getTenantToken(); + requestBuilder.addHeader(Constants.HTTP_TENANT_HEADER, tenant_token); + } + if(this.context!=null && this.context.hasSpace()) { + space_token = this.context.getSpace().getSpaceToken(); + requestBuilder.addHeader(Constants.HTTP_SPACE_HEADER, space_token); + } + + // Include version and component as request header + requestBuilder.addHeader(Constants.HTTP_VERSION_HEADER, CoreConfiguration.getVulasRelease()); + requestBuilder.addHeader(Constants.HTTP_COMPONENT_HEADER, Constants.VulasComponent.client.toString()); + + // Include additional headers from configuration (if any) + final Map add_headers = this.getVulasConfiguration().getServiceHeaders(this.service); + if(add_headers!=null && !add_headers.isEmpty()) { + for(String key: add_headers.keySet()) { + requestBuilder.addHeader(key, add_headers.get(key)); + } + } + + // Only if put something in the body + if(this.hasPayload()) { + requestBuilder.addHeader("Content-Type", "application/json; charset=utf-8"); + requestBuilder.addHeader("Content-Language", "en-US"); + }else if(this.binPayload!=null){ + requestBuilder.addHeader("Content-Type", this.contentType); + } + + if(this.hasPayload()) { + requestBuilder.setEntity(new StringEntity(this.payload, StandardCharsets.UTF_8)); + }else if(this.binPayload!=null){ + requestBuilder.setEntity(new InputStreamEntity(this.binPayload)); + } + + httpUriRequest = requestBuilder.build(); + + try { + do { + is_503 = false; + + final long start_nano = System.nanoTime(); + + if(!this.hasPayload()) { + BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); + }else if(this.binPayload==null) { + BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + ", size=" + StringUtil.byteToKBString(this.payload.getBytes().length) + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); + }else { + BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " [uri=" + uri + ", size=" + this.binPayload.available() + (tenant_token==null?"":", tenant=" + tenant_token) + (space_token==null?"":", space=" + space_token) + "]"); + } + + SocketConfig socketConfig = SocketConfig.custom().setSoKeepAlive(true).build(); + HttpClient client = HttpClients.custom().setDefaultSocketConfig(socketConfig).useSystemProperties().build(); + + // Read response + httpResponse = client.execute(httpUriRequest); + response_code = httpResponse.getStatusLine().getStatusCode(); + response = new HttpResponse(response_code); + + // If the response body contains a JAR file, save it + if(response.isOk() && httpResponse.getFirstHeader("Content-Type")!=null && httpResponse.getFirstHeader("Content-Type").getValue().contains("application/java-archive")){ + String fileName = ""; + Header disposition = httpResponse.getFirstHeader("Content-Disposition"); + if (disposition != null) { + String dispositionValue = disposition.getValue(); + // Extracts file name from header field + int index = dispositionValue.indexOf("filename="); + if (index > 0) { + fileName = dispositionValue.substring(index + 9, dispositionValue.length() ); + } + } else { + // Extracts file name from URL + fileName = this.path.substring(this.path.lastIndexOf("/") + 1, this.path.length()); + } + + String saveFilePath = null; + if (this.dir!=null){ + //create directories if not existing + if(!Files.exists(Paths.get(dir))){ + Files.createDirectories(Paths.get(dir)); + } + saveFilePath= dir + File.separator + fileName; + }else { + saveFilePath= Paths.get(this.getVulasConfiguration().getTmpDir().toString()).toString()+ File.separator + fileName; + } + + try( + // Opens input stream from the HTTP connection + InputStream inputStream = httpResponse.getEntity().getContent(); + // Opens an output stream to save into file + FileOutputStream outputStream = new FileOutputStream(saveFilePath); + ){ + int bytesRead = -1; + byte[] buffer = new byte[inputStream.available()]; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + response.setBody(saveFilePath); + } + }else if(response.isOk() || response.isCreated()) { + final String body = this.readResponse(httpResponse); + if(StringUtils.isNotBlank(body)) response.setBody(body); + } + + // Stats + final long end_nano = System.nanoTime(); + BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " completed with response code [" + response_code + "] in " + StringUtil.nanoToFlexDurationString(end_nano-start_nano) + " (proxy=" + isProxySet() + ")") ; + + // 503: Retry + if(response.isServiceUnavailable()) { + is_503 = true; + } + // 5xx: Throw exception + else if(response.isServerError() || response.getStatus()==400) { + final BackendConnectionException bce = new BackendConnectionException(this.method, uri, response_code, null); + throwBceException(httpResponse, bce); + } + } + while(repeater.repeat(is_503)); + if(is_503) + throw new BackendConnectionException(this.method, uri, 503, null); + } catch(BackendConnectionException bce) { + this.logHeaderFields(" Request-header", httpUriRequest.getAllHeaders()); + this.logHeaderFields(" Response-header", httpResponse.getAllHeaders()); + if(bce.getHttpResponseBody()!=null) + BasicHttpRequest.log.error(" Response-body: [" + bce.getHttpResponseBody().replaceAll("[\\t\\n\\x0B\\f\\r]*", "") + "]"); + BasicHttpRequest.log.error(" Exception message: [" + bce.getMessage() + "]"); + if(this.hasPayload()) + BasicHttpRequest.log.error(" HTTP Request body: [" + this.payload.toString() + "]"); + } catch(Exception e) { + final BackendConnectionException bce = new BackendConnectionException(this.method, uri, response_code, e); + throwBceException(httpResponse, bce); + } + return response; + } - int bytesRead = -1; - byte[] buffer = new byte[inputStream.available()]; - while ((bytesRead = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } + private boolean isProxySet() { + return StringUtils.isNotBlank(System.getProperty("http.proxyHost")) ? true + : false || StringUtils.isNotBlank(System.getProperty("https.proxyHost")) ? true : false; + } - response.setBody(saveFilePath); - outputStream.close(); - inputStream.close(); - } - else if(response.isOk() || response.isCreated()) - response.setBody(FileUtil.readInputStream(connection.getInputStream(), FileUtil.getCharset())); + /** + * @param _httpResponse + * @param _bce + * @throws BackendConnectionException + */ + private void throwBceException(org.apache.http.HttpResponse _httpResponse, + final BackendConnectionException _bce) throws BackendConnectionException { + try { + final String body = this.readResponse(_httpResponse); + if(StringUtils.isNotBlank(body)) _bce.setHttpResponseBody(body); + } catch (IOException e) { + //BasicHttpRequest.log.error("Cannot read input stream: " + e1.getMessage()); + } + throw _bce; + } - // Stats - final long end_nano = System.nanoTime(); - BasicHttpRequest.log.info("HTTP " + this.method.toString().toUpperCase() + " completed with response code [" + response_code + "] in " + StringUtil.nanoToFlexDurationString(end_nano-start_nano) + " (proxy=" + connection.usingProxy() + ")") ; + /** + * @param _prefix + * @param _fields + */ + private void logHeaderFields(String _prefix, Header[] _fields) { + if(_fields==null) { + return; + } - // 503: Retry - if(response.isServiceUnavailable()) { - is_503 = true; - } - // 5xx: Throw exception - else if(response.isServerError() || response.getStatus()==400) { - final BackendConnectionException bce = new BackendConnectionException(this.method, uri, response_code, null); - try { - final String body = this.readErrorStream(connection); - if(body!=null && !body.trim().equals("")) - bce.setHttpResponseBody(body); - } catch (IOException e1) { - //BasicHttpRequest.log.error("Cannot read input stream: " + e1.getMessage()); - } - throw bce; - } - } - while(repeater.repeat(is_503)); - if(is_503) - throw new BackendConnectionException(this.method, uri, 503, null); - } catch(BackendConnectionException bce) { - this.logHeaderFields(" Request-header", request_fields); - this.logHeaderFields(" Response-header", connection.getHeaderFields()); - if(bce.getHttpResponseBody()!=null) - BasicHttpRequest.log.error(" Response-body: [" + bce.getHttpResponseBody().replaceAll("[\\t\\n\\x0B\\f\\r]*", "") + "]"); - BasicHttpRequest.log.error(" Exception message: [" + bce.getMessage() + "]"); - if(this.hasPayload()) - BasicHttpRequest.log.error(" HTTP Request body: [" + this.payload.toString() + "]"); - //throw bce; - } catch(Exception e) { - final BackendConnectionException bce = new BackendConnectionException(this.method, uri, response_code, e); - try { - bce.setHttpResponseBody(this.readErrorStream(connection)); - } catch (IOException e1) { - //BasicHttpRequest.log.error("Cannot read error stream: " + e1.getMessage()); - } - throw bce; - } - finally { - if(connection != null) connection.disconnect(); - } - return response; - } - - private void logHeaderFields(String _prefix, Map> _fields) { - for(Map.Entry> entry: _fields.entrySet()) - BasicHttpRequest.log.error(_prefix + " " + (entry.getKey()==null?"":"["+entry.getKey()+"]" + " = ") + entry.getValue()); + for(Header header: _fields) + BasicHttpRequest.log.error(_prefix + " " + "["+header.getName()+"]" + " = " + header.getValue()); } /** * @param _c + * @throws IOException */ - private String readErrorStream(HttpURLConnection _c) throws IOException { - String error = null; - if(_c!=null) { - InputStream is = _c.getErrorStream(); - if(is==null) - is = _c.getInputStream(); - if(is!=null) { - error = FileUtil.readInputStream(is, FileUtil.getCharset()); - } - } - return error; + private String readResponse(org.apache.http.HttpResponse _c) throws IOException { + String response = null; + if(_c!=null) { + InputStream is = _c.getEntity().getContent(); + if(is!=null) { + response = FileUtil.readInputStream(is, FileUtil.getCharset()); + } + } + return response; } private URI getUri() {