Fix windows encoding issues (opensearch-project#2206)

Adds spotbugs [1] to detect internalization before they are added to the codebase, also fixed several encoding bugs that impact windows users. [1] https://spotbugs.readthedocs.io/en/stable/index.html Closes opensearch-project#2194 Signed-off-by: Peter Nied <[email protected]> Signed-off-by: Stephen Crawford <[email protected]>
stephen-crawford · Nov 10, 2022 · a3133fa · a3133fa
1 parent f283ad8
commit a3133fa
Show file tree

Hide file tree

Showing 11 changed files with 44 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -34,6 +34,7 @@ jobs:
           -x spotlessCheck
           -x checkstyleMain
           -x checkstyleTest
+          -x spotbugsMain
 
     - name: Coverage
       uses: codecov/codecov-action@v1
@@ -80,6 +81,7 @@ jobs:
           -x spotlessCheck
           -x checkstyleMain
           -x checkstyleTest
+          -x spotbugsMain
 
   backward-compatibility:
     runs-on: ubuntu-latest

diff --git a/.github/workflows/code-hygiene.yml b/.github/workflows/code-hygiene.yml
@@ -42,3 +42,18 @@ jobs:
       - uses: gradle/gradle-build-action@v2
         with:
           arguments: checkstyleMain checkstyleTest
+
+  spotbugs:
+    runs-on: ubuntu-latest
+    name: Spotbugs scan
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v2
+        with:
+          distribution: temurin # Temurin is a distribution of adoptium
+          java-version: 11
+
+      - uses: gradle/gradle-build-action@v2
+        with:
+          arguments: spotbugsMain
diff --git a/build.gradle b/build.gradle
@@ -57,6 +57,7 @@ plugins {
     id 'nebula.ospackage'  version "8.3.0"
     id "org.gradle.test-retry" version "1.3.1"
     id 'eclipse'
+    id "com.github.spotbugs" version "5.0.13"
 }
 
 allprojects {
@@ -85,6 +86,14 @@ spotless {
   }
 }
 
+spotbugs {
+    includeFilter = file('spotbugs-include.xml')
+}
+
+spotbugsTest {
+    enabled = false
+}
+
 java.sourceCompatibility = JavaVersion.VERSION_11
 java.targetCompatibility = JavaVersion.VERSION_11
 

diff --git a/spotbugs-include.xml b/spotbugs-include.xml
@@ -0,0 +1,5 @@
+<FindBugsFilter>
+    <Match>
+        <Bug category="I18N" />
+    </Match>
+</FindBugsFilter>
diff --git a/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java b/src/main/java/com/amazon/dlic/auth/http/saml/AuthTokenProcessorHandler.java
@@ -14,6 +14,7 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
 import java.security.PrivilegedActionException;
 import java.security.PrivilegedExceptionAction;
@@ -152,7 +153,7 @@ private AuthTokenProcessorAction.Response handleImpl(RestRequest restRequest, Re
             SettingsException {
         if (token_log.isDebugEnabled()) {
             try {
-                token_log.debug("SAMLResponse for {}\n{}", samlRequestId, new String(Util.base64decoder(samlResponseBase64), "UTF-8"));
+                token_log.debug("SAMLResponse for {}\n{}", samlRequestId, new String(Util.base64decoder(samlResponseBase64), StandardCharsets.UTF_8));
             } catch (Exception e) {
                 token_log.warn(
                         "SAMLResponse for {} cannot be decoded from base64\n{}",

diff --git a/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java b/src/main/java/org/opensearch/security/auditlog/impl/AbstractAuditLog.java
@@ -419,7 +419,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index
                     try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, originalResult.internalSourceRef(), XContentType.JSON)) {
                         Object base64 = parser.map().values().iterator().next();
                         if(base64 instanceof String) {
-                            originalSource = (new String(BaseEncoding.base64().decode((String) base64)));
+                            originalSource = (new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8));
                         } else {
                             originalSource = XContentHelper.convertToJson(originalResult.internalSourceRef(), false, XContentType.JSON);
                         }
@@ -430,7 +430,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index
                     try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, currentIndex.source(), XContentType.JSON)) {
                         Object base64 = parser.map().values().iterator().next();
                         if(base64 instanceof String) {
-                            currentSource = (new String(BaseEncoding.base64().decode((String) base64)));
+                            currentSource = new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8);
                         } else {
                             currentSource = XContentHelper.convertToJson(currentIndex.source(), false, XContentType.JSON);
                         }
@@ -457,7 +457,7 @@ public void logDocumentWritten(ShardId shardId, GetResult originalResult, Index
                 try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, THROW_UNSUPPORTED_OPERATION, currentIndex.source(), XContentType.JSON)) {
                    Object base64 = parser.map().values().iterator().next();
                    if(base64 instanceof String) {
-                       msg.addSecurityConfigContentToRequestBody(new String(BaseEncoding.base64().decode((String) base64)), id);
+                       msg.addSecurityConfigContentToRequestBody(new String(BaseEncoding.base64().decode((String) base64), StandardCharsets.UTF_8), id);
                    } else {
                        msg.addSecurityConfigTupleToRequestBody(new Tuple<XContentType, BytesReference>(XContentType.JSON, currentIndex.source()), id);
                    }

diff --git a/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java b/src/main/java/org/opensearch/security/configuration/ConfigurationLoaderSecurity7.java
@@ -27,6 +27,7 @@
 package org.opensearch.security.configuration;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
@@ -253,7 +254,7 @@ private SecurityDynamicConfiguration<?> toConfig(GetResponse singleGetResponse,
 
             parser.nextToken();
 
-            final String jsonAsString = SecurityUtils.replaceEnvVars(new String(parser.binaryValue()), settings);
+            final String jsonAsString = SecurityUtils.replaceEnvVars(new String(parser.binaryValue(), StandardCharsets.UTF_8), settings);
             final JsonNode jsonNode = DefaultObjectMapper.readTree(jsonAsString);
             int configVersion = 1;
 

diff --git a/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java b/src/main/java/org/opensearch/security/ssl/DefaultSecurityKeyStore.java
@@ -20,6 +20,7 @@
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.LinkOption;
 import java.nio.file.Path;
@@ -646,7 +647,7 @@ private boolean areSameCerts(final X509Certificate[] currentX509Certs, final X50
 
         final Function<? super X509Certificate, String> certificateSignature = cert -> {
             final byte[] signature = cert !=null && cert.getSignature() != null ? cert.getSignature() : null;
-            return new String(signature);
+            return new String(signature, StandardCharsets.UTF_8);
         };
 
         final Set<String> currentCertSignatureSet = Arrays.stream(currentX509Certs)

diff --git a/src/main/java/org/opensearch/security/support/ConfigHelper.java b/src/main/java/org/opensearch/security/support/ConfigHelper.java
@@ -31,6 +31,7 @@
 import java.io.IOException;
 import java.io.Reader;
 import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -91,7 +92,7 @@ public static void uploadFile(Client tc, String filepath, String index, CType cT
     public static Reader createFileOrStringReader(CType cType, int configVersion, String filepath, boolean populateEmptyIfFileMissing) throws Exception {
         Reader reader;
         if (!populateEmptyIfFileMissing || new File(filepath).exists()) {
-            reader = new FileReader(filepath);
+            reader = new FileReader(filepath, StandardCharsets.UTF_8);
         } else {
             reader = new StringReader(createEmptySdcYaml(cType, configVersion));
         }
@@ -143,7 +144,7 @@ public static <T> SecurityDynamicConfiguration<T> fromYamlReader(Reader yamlRead
     }
 
     public static <T> SecurityDynamicConfiguration<T> fromYamlFile(String filepath, CType ctype, int version, long seqNo, long primaryTerm) throws IOException {
-        return fromYamlReader(new FileReader(filepath), ctype, version, seqNo, primaryTerm);
+        return fromYamlReader(new FileReader(filepath, StandardCharsets.UTF_8), ctype, version, seqNo, primaryTerm);
     }
 
     public static <T> SecurityDynamicConfiguration<T> fromYamlString(String yamlString, CType ctype, int version, long seqNo, long primaryTerm) throws IOException {

diff --git a/src/main/java/org/opensearch/security/tools/SecurityAdmin.java b/src/main/java/org/opensearch/security/tools/SecurityAdmin.java
@@ -896,7 +896,7 @@ private static boolean retrieveFile(final RestHighLevelClient restHighLevelClien
         }
 
         System.out.println("Will retrieve '"+"/" +id+"' into "+filepath+" "+(legacy?"(legacy mode)":""));
-        try (Writer writer = new FileWriter(filepath)) {
+        try (Writer writer = new FileWriter(filepath, StandardCharsets.UTF_8)) {
 
 			final GetResponse response = restHighLevelClient.get(new GetRequest(index).id(id).refresh(true).realtime(false), RequestOptions.DEFAULT);
 

diff --git a/src/test/java/org/opensearch/security/IntegrationTests.java b/src/test/java/org/opensearch/security/IntegrationTests.java
@@ -34,7 +34,6 @@
 import org.apache.hc.core5.http.message.BasicHeader;
 import org.junit.Assert;
 import org.junit.Assume;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -270,7 +269,6 @@ public void testSingle() throws Exception {
     }
 
     @Test
-    @Ignore // https://github.com/opensearch-project/security/issues/2194
     public void testSpecialUsernames() throws Exception {
 
         setup();