Explicit Text Support in Serialization (Azure#32277)

Explicit Text Support in Serialization

eng/versioning/version_client.txt

@@ -400,6 +400,7 @@ com.azure.tools:azure-sdk-build-tool;1.0.0-beta.1;1.0.0-beta.2
 # note: The unreleased dependencies will not be manipulated with the automatic PR creation code.
 # In the pom, the version update tag after the version should name the unreleased package and the dependency version:
 # <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
+unreleased_com.azure:azure-core;1.36.0-beta.1
 
 
 # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current

sdk/containerregistry/azure-containers-containerregistry/pom.xml

@@ -48,7 +48,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core</artifactId>
-      <version>1.35.0</version> <!-- {x-version-update;com.azure:azure-core;dependency} -->
+      <version>1.36.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
     </dependency>
      <dependency>
       <groupId>com.azure</groupId>

sdk/containerregistry/azure-containers-containerregistry/src/main/java/com/azure/containers/containerregistry/RegistryArtifact.java

@@ -221,7 +221,7 @@ public Response<ArtifactManifestProperties> getManifestPropertiesWithResponse(Co
         String res = this.getDigest();
         try {
             return this.serviceClient.getManifestPropertiesWithResponse(getRepositoryName(), res, enableSync(getTracingContext(context)));
-        } catch (AcrErrorsException exception) {
+        } catch (HttpResponseException exception) {
             throw LOGGER.logExceptionAsError(mapAcrErrorsException(exception));
         }
     }

sdk/containerregistry/azure-containers-containerregistry/src/main/java/com/azure/containers/containerregistry/implementation/UtilsImpl.java

@@ -6,7 +6,6 @@
 import com.azure.containers.containerregistry.ContainerRegistryServiceVersion;
 import com.azure.containers.containerregistry.implementation.authentication.ContainerRegistryCredentialsPolicy;
 import com.azure.containers.containerregistry.implementation.authentication.ContainerRegistryTokenService;
-import com.azure.containers.containerregistry.implementation.models.AcrErrorsException;
 import com.azure.containers.containerregistry.implementation.models.ManifestAttributesBase;
 import com.azure.containers.containerregistry.implementation.models.TagAttributesBase;
 import com.azure.containers.containerregistry.models.ArtifactManifestProperties;
@@ -178,13 +177,9 @@ public static HttpPipeline buildHttpPipeline(
         return httpPipeline;
     }
 
+    @SuppressWarnings("unchecked")
     private static ArrayList<HttpPipelinePolicy> clone(ArrayList<HttpPipelinePolicy> policies) {
-        ArrayList<HttpPipelinePolicy> clonedPolicy = new ArrayList<>();
-        for (HttpPipelinePolicy policy:policies) {
-            clonedPolicy.add(policy);
-        }
-
-        return clonedPolicy;
+        return (ArrayList<HttpPipelinePolicy>) policies.clone();
     }
 
     /**
@@ -249,41 +244,26 @@ private static <T> Response<Void> getAcceptedDeleteResponse(Response<T> response
      * @return The exception returned by the public methods.
      */
     public static Throwable mapException(Throwable exception) {
-        AcrErrorsException acrException = null;
+        HttpResponseException acrException = null;
 
-        if (exception instanceof AcrErrorsException) {
-            acrException = ((AcrErrorsException) exception);
+        if (exception instanceof HttpResponseException) {
+            acrException = ((HttpResponseException) exception);
         } else if (exception instanceof RuntimeException) {
             RuntimeException runtimeException = (RuntimeException) exception;
             Throwable throwable = runtimeException.getCause();
-            if (throwable instanceof AcrErrorsException) {
-                acrException = (AcrErrorsException) throwable;
+            if (throwable instanceof HttpResponseException) {
+                acrException = (HttpResponseException) throwable;
             }
         }
 
         if (acrException == null) {
             return exception;
         }
 
-        final HttpResponse errorHttpResponse = acrException.getResponse();
-        final int statusCode = errorHttpResponse.getStatusCode();
-        final String errorDetail = acrException.getMessage();
-
-        switch (statusCode) {
-            case 401:
-                return new ClientAuthenticationException(errorDetail, acrException.getResponse(), exception);
-            case 404:
-                return new ResourceNotFoundException(errorDetail, acrException.getResponse(), exception);
-            case 409:
-                return new ResourceExistsException(errorDetail, acrException.getResponse(), exception);
-            case 412:
-                return new ResourceModifiedException(errorDetail, acrException.getResponse(), exception);
-            default:
-                return new HttpResponseException(errorDetail, acrException.getResponse(), exception);
-        }
+        return mapAcrErrorsException(acrException);
     }
 
-    public static HttpResponseException mapAcrErrorsException(AcrErrorsException acrException) {
+    public static HttpResponseException mapAcrErrorsException(HttpResponseException acrException) {
         final HttpResponse errorHttpResponse = acrException.getResponse();
         final int statusCode = errorHttpResponse.getStatusCode();
         final String errorDetail = acrException.getMessage();

sdk/core/azure-core/src/main/java/com/azure/core/implementation/AccessibleByteArrayOutputStream.java

@@ -4,12 +4,17 @@
 package com.azure.core.implementation;
 
 import java.io.ByteArrayOutputStream;
+import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.util.Arrays;
 
 /**
  * This class is an extension of {@link ByteArrayOutputStream} which allows access to the backing {@code byte[]} without
  * requiring a copying of the data. The only use of this class is for internal purposes where we know it is safe to
  * directly access the {@code byte[]} without copying.
+ * <p>
+ * This class isn't meant to be thread-safe as usage should be internal to azure-core and should be guarded
+ * appropriately when used.
  */
 public class AccessibleByteArrayOutputStream extends ByteArrayOutputStream {
     /**
@@ -30,8 +35,38 @@ public AccessibleByteArrayOutputStream(int initialCapacity) {
     }
 
     @Override
-    public synchronized byte[] toByteArray() {
-        return buf;
+    public byte[] toByteArray() {
+        return Arrays.copyOf(buf, count);
+    }
+
+    // Commenting out as this isn't used but may be added back in the future.
+//    /**
+//     * Returns the internal {@code byte[]} without copying.
+//     * <p>
+//     * This will be the full {@code byte[]}, so if writing required it to be resized to 8192 bytes but only 6000 bytes
+//     * were written the final 2192 bytes will be undefined data. If this is used in an API where a {@code byte[]} is
+//     * accepted you must use the range based overload with {@link #count()}, if a range based overload isn't available
+//     * use {@link #toByteArray()} which will copy the range of bytes written.
+//     *
+//     * @return A direct reference to the internal {@code byte[]} where data is being written.
+//     */
+//    public byte[] toByteArrayUnsafe() {
+//        return buf;
+//    }
+
+    /**
+     * Returns a {@link ByteBuffer} representation of the content written to this stream.
+     * <p>
+     * The {@link ByteBuffer} will use a direct reference to the internal {@code byte[]} being written, so any
+     * modifications to the content already written will be reflected in the {@link ByteBuffer}. Given the direct
+     * reference to the internal {@code byte[]} the {@link ByteBuffer} returned by the API will be read-only. Further
+     * writing to this stream won't be reflected in the {@link ByteBuffer} as the ByteBuffer will be created using
+     * {@code ByteBuffer.wrap(bytes, 0, count())}.
+     *
+     * @return A read-only {@link ByteBuffer} represented by the internal buffer being written to.
+     */
+    public ByteBuffer toByteBuffer() {
+        return ByteBuffer.wrap(buf, 0, count).asReadOnlyBuffer();
     }
 
     /**
@@ -52,4 +87,17 @@ public int count() {
     public String toString(Charset charset) {
         return new String(buf, 0, count, charset);
     }
+
+    /**
+     * Gets a BOM aware string representation of the stream.
+     * <p>
+     * This method is the equivalent of calling
+     * {@code ImplUtils.bomAwareToString(toByteBufferUnsafe(), 0, count(), contentType)}.
+     *
+     * @param contentType The {@code Content-Type} header value.
+     * @return A string representation of the stream encoded to the found encoding.
+     */
+    public String bomAwareToString(String contentType) {
+        return ImplUtils.bomAwareToString(buf, 0, count, contentType);
+    }
 }

sdk/core/azure-core/src/main/java/com/azure/core/implementation/ImplUtils.java

@@ -15,6 +15,10 @@
 import java.io.OutputStream;
 import java.net.URL;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.IllegalCharsetNameException;
+import java.nio.charset.StandardCharsets;
+import java.nio.charset.UnsupportedCharsetException;
 import java.time.DateTimeException;
 import java.time.Duration;
 import java.time.OffsetDateTime;
@@ -26,6 +30,8 @@
 import java.util.NoSuchElementException;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Utility class containing implementation specific methods.
@@ -37,6 +43,16 @@ public final class ImplUtils {
     // future improvement - make this configurable
     public static final int MAX_CACHE_SIZE = 10000;
 
+    private static final Charset UTF_32BE = Charset.forName("UTF-32BE");
+    private static final Charset UTF_32LE = Charset.forName("UTF-32LE");
+    private static final byte ZERO = (byte) 0x00;
+    private static final byte BB = (byte) 0xBB;
+    private static final byte BF = (byte) 0xBF;
+    private static final byte EF = (byte) 0xEF;
+    private static final byte FE = (byte) 0xFE;
+    private static final byte FF = (byte) 0xFF;
+    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=(\\S+)\\b", Pattern.CASE_INSENSITIVE);
+
     /**
      * Attempts to extract a retry after duration from a given set of {@link HttpHeaders}.
      * <p>
@@ -247,6 +263,61 @@ public Map.Entry<String, String> next() {
         }
     }
 
+    /**
+     * Attempts to convert a byte stream into the properly encoded String.
+     * <p>
+     * This utility method will attempt to find the encoding for the String in this order.
+     * <ol>
+     *     <li>Find the byte order mark in the byte array.</li>
+     *     <li>Find the charset in the {@code contentType} header.</li>
+     *     <li>Default to {@code UTF-8}.</li>
+     * </ol>
+     *
+     * @param bytes The byte array.
+     * @param offset The starting offset in the byte array.
+     * @param count The number of bytes to process in the byte array.
+     * @param contentType The {@code Content-Type} header value.
+     * @return A string representation of the byte encoded to the found encoding, or null if {@code bytes} is null.
+     */
+    public static String bomAwareToString(byte[] bytes, int offset, int count, String contentType) {
+        if (bytes == null) {
+            return null;
+        }
+
+        if (count >= 3 && bytes[offset] == EF && bytes[offset + 1] == BB && bytes[offset + 2] == BF) {
+            return new String(bytes, 3, bytes.length - 3, StandardCharsets.UTF_8);
+        } else if (count >= 4 && bytes[offset] == ZERO && bytes[offset + 1] == ZERO
+            && bytes[offset + 2] == FE && bytes[offset + 3] == FF) {
+            return new String(bytes, 4, bytes.length - 4, UTF_32BE);
+        } else if (count >= 4 && bytes[offset] == FF && bytes[offset + 1] == FE
+            && bytes[offset + 2] == ZERO && bytes[offset + 3] == ZERO) {
+            return new String(bytes, 4, bytes.length - 4, UTF_32LE);
+        } else if (count >= 2 && bytes[offset] == FE && bytes[offset + 1] == FF) {
+            return new String(bytes, 2, bytes.length - 2, StandardCharsets.UTF_16BE);
+        } else if (count >= 2 && bytes[offset] == FF && bytes[offset + 1] == FE) {
+            return new String(bytes, 2, bytes.length - 2, StandardCharsets.UTF_16LE);
+        } else {
+            /*
+             * Attempt to retrieve the default charset from the 'Content-Encoding' header, if the value isn't
+             * present or invalid fallback to 'UTF-8' for the default charset.
+             */
+            if (!CoreUtils.isNullOrEmpty(contentType)) {
+                try {
+                    Matcher charsetMatcher = CHARSET_PATTERN.matcher(contentType);
+                    if (charsetMatcher.find()) {
+                        return new String(bytes, offset, count, Charset.forName(charsetMatcher.group(1)));
+                    } else {
+                        return new String(bytes, offset, count, StandardCharsets.UTF_8);
+                    }
+                } catch (IllegalCharsetNameException | UnsupportedCharsetException ex) {
+                    return new String(bytes, offset, count, StandardCharsets.UTF_8);
+                }
+            } else {
+                return new String(bytes, offset, count, StandardCharsets.UTF_8);
+            }
+        }
+    }
+
     private ImplUtils() {
     }
 }

sdk/core/azure-core/src/main/java/com/azure/core/implementation/http/rest/ReflectionSerializable.java

@@ -202,7 +202,7 @@ static ByteBuffer serializeAsJsonSerializable(Object jsonSerializable) throws IO
             JSON_WRITER_WRITE_JSON_SERIALIZABLE.writeJson(jsonWriter, jsonSerializable);
             JSON_WRITER_FLUSH.flush(jsonWriter);
 
-            return ByteBuffer.wrap(outputStream.toByteArray(), 0, outputStream.count());
+            return outputStream.toByteBuffer();
         }
     }
 
@@ -289,7 +289,7 @@ static ByteBuffer serializeAsXmlSerializable(Object bodyContent) throws IOExcept
             XML_WRITER_WRITE_XML_SERIALIZABLE.writeXml(xmlWriter, bodyContent);
             XML_WRITER_FLUSH.flush(xmlWriter);
 
-            return ByteBuffer.wrap(outputStream.toByteArray(), 0, outputStream.count());
+            return outputStream.toByteBuffer();
         } catch (IOException ex) {
             throw ex;
         } catch (Exception ex) {

sdk/core/azure-core/src/main/java/com/azure/core/implementation/http/rest/RestProxyBase.java

@@ -25,6 +25,7 @@
 import com.azure.core.implementation.TypeUtil;
 import com.azure.core.implementation.http.UnexpectedExceptionInformation;
 import com.azure.core.implementation.serializer.HttpResponseDecoder;
+import com.azure.core.implementation.serializer.MalformedValueException;
 import com.azure.core.util.BinaryData;
 import com.azure.core.util.Context;
 import com.azure.core.util.UrlBuilder;
@@ -352,6 +353,16 @@ public Exception instantiateUnexpectedException(final UnexpectedExceptionInforma
             exceptionMessage.append("\"").append(new String(responseContent, StandardCharsets.UTF_8)).append("\"");
         }
 
+        // If the decoded response content is on of these exception types there was a failure in creating the actual
+        // exception body type. In this case return an HttpResponseException to maintain the exception having a
+        // reference to the HttpResponse and information about what caused the deserialization failure.
+        if (responseDecodedContent instanceof IOException
+            || responseDecodedContent instanceof MalformedValueException
+            || responseDecodedContent instanceof IllegalStateException) {
+            return new HttpResponseException(exceptionMessage.toString(), httpResponse,
+                (Throwable) responseDecodedContent);
+        }
+
         // For HttpResponseException types that exist in azure-core, call the constructor directly.
         Class<? extends HttpResponseException> exceptionType = exception.getExceptionType();
         if (exceptionType == HttpResponseException.class) {

sdk/core/azure-core/src/main/java/com/azure/core/implementation/serializer/HttpResponseBodyDecoder.java

@@ -61,11 +61,16 @@ static Object decodeByteArray(byte[] body, HttpResponse httpResponse, Serializer
                 return deserializeBody(body,
                     decodeData.getUnexpectedException(httpResponse.getStatusCode()).getExceptionBodyType(),
                     null, serializer, SerializerEncoding.fromHeaders(httpResponse.getHeaders()));
-            } catch (IOException | MalformedValueException ex) {
-                // This translates in RestProxy as a RestException with no deserialized body.
-                // The response content will still be accessible via the .response() member.
+            } catch (IOException | MalformedValueException | IllegalStateException ex) {
+                // MalformedValueException is thrown by Jackson, IllegalStateException is thrown by the TEXT
+                // serialization encoding handler, and IOException can be thrown by both Jackson and TEXT.
+                //
+                // There has been an issue deserializing the error response body. This may be an error in the service
+                // return.
+                //
+                // Return the exception as the body type, RestProxyBase will handle this later.
                 LOGGER.warning("Failed to deserialize the error entity.", ex);
-                return null;
+                return ex;
             }
         } else {
             if (!decodeData.isReturnTypeDecodeable()) {