Expose a Utf8String type.

src/classlibnative/bcltype/CMakeLists.txt

@@ -18,6 +18,7 @@ set(BCLTYPE_SOURCES
     objectnative.cpp
     stringnative.cpp
     system.cpp
+    utf8stringnative.cpp
     varargsnative.cpp
     variant.cpp
 )

src/classlibnative/bcltype/utf8stringnative.cpp

@@ -0,0 +1,51 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+//
+// File: StringNative.cpp
+//
+
+//
+// Purpose: The implementation of the Utf8String class.
+//
+
+//
+
+#include "common.h"
+
+#include "object.h"
+#include "utilcode.h"
+#include "excep.h"
+#include "frames.h"
+#include "field.h"
+#include "vars.hpp"
+#include "utf8stringnative.h"
+#include "comutilnative.h"
+#include "metasig.h"
+#include "excep.h"
+
+// Compile the string functionality with these pragma flags (equivalent of the command line /Ox flag)
+// Compiling this functionality differently gives us significant throughout gain in some cases.
+#if defined(_MSC_VER) && defined(_TARGET_X86_)
+#pragma optimize("tgy", on)
+#endif
+
+FCIMPL1(Utf8StringObject *, Utf8StringFastAllocate, DWORD length)
+{
+    FCALL_CONTRACT;
+
+    UTF8STRINGREF rv = NULL; // not protected
+
+    HELPER_METHOD_FRAME_BEGIN_RET_1(rv);
+    rv = SlowAllocateUtf8String(length);
+    HELPER_METHOD_FRAME_END();
+
+    return UTF8STRINGREFToObject(rv);
+}
+FCIMPLEND
+
+
+// Revert to command line compilation flags
+#if defined(_MSC_VER) && defined(_TARGET_X86_)
+#pragma optimize ("", on)
+#endif

src/classlibnative/bcltype/utf8stringnative.h

@@ -0,0 +1,24 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+//
+// File: StringNative.h
+//
+
+//
+// Purpose: Contains types and method signatures for the Utf8String class
+//
+
+//
+
+#include "fcall.h"
+#include "qcall.h"
+#include "excep.h"
+
+#ifndef _UTF8STRINGNATIVE_H_
+#define _UTF8STRINGNATIVE_H_
+
+FCDECL1(Utf8StringObject *, Utf8StringFastAllocate, DWORD length);
+
+#endif // _UTF8STRINGNATIVE_H_
+

src/inc/dacvars.h

@@ -172,6 +172,7 @@ DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pObjectClass, ::g_pObjectClass
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pRuntimeTypeClass, ::g_pRuntimeTypeClass)
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pCanonMethodTableClass, ::g_pCanonMethodTableClass)
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pStringClass, ::g_pStringClass)
+DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pUtf8StringClass, ::g_pUtf8StringClass)
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pArrayClass, ::g_pArrayClass)
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pSZArrayHelperClass, ::g_pSZArrayHelperClass)
 DEFINE_DACVAR(ULONG, UNKNOWN_POINTER_TYPE, dac__g_pNullableClass, ::g_pNullableClass)

src/mscorlib/System.Private.CoreLib.csproj

@@ -357,6 +357,7 @@
     <Compile Include="$(BclSourcesRoot)\System\TypeNameParser.cs" />
     <Compile Include="$(BclSourcesRoot)\System\TypedReference.cs" />
     <Compile Include="$(BclSourcesRoot)\System\TypeLoadException.cs" />
+    <Compile Include="$(BclSourcesRoot)\System\Utf8String.cs" />
     <Compile Include="$(BclSourcesRoot)\System\ValueType.cs" />
     <Compile Include="$(BclSourcesRoot)\System\WeakReference.cs" />
     <Compile Include="$(BclSourcesRoot)\System\WeakReferenceOfT.cs" />
@@ -667,4 +668,4 @@
   </ItemGroup>
   <Import Project="ILLink.targets" />
   <Import Project="GenerateCompilerResponseFile.targets" />
-</Project>
+</Project>

src/mscorlib/src/System/Utf8String.cs

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Runtime.CompilerServices;
+
+namespace System
+{
+    // This is an experimental type and not referenced from CoreFx but needs to exists and be public so we can prototype in CoreFxLab.
+    public sealed class Utf8String
+    {
+        // Do not reorder these fields. Must match layout of Utf8StringObject in object.h.
+        private int _length;
+        [CLSCompliant(false)]
+        public byte _firstByte; // TODO: Is public for experimentation in CoreFxLab. Will be private in its ultimate form.
+
+        private Utf8String() { } // Suppress creation of the public constructor. No one actually calls this.
+
+        public int Length => _length;
+
+        // Creates a new zero-initialized instance of the specified length. Actual storage allocated is "length + 1" bytes (the extra
+        // +1 is for the NUL terminator.)
+        [MethodImpl(MethodImplOptions.InternalCall)]
+        public static extern Utf8String FastAllocate(int length);  //TODO: Is public for experimentation in CoreFxLab. Will be private in its ultimate form.
+    }
+}

src/strongname/api/common.h

@@ -146,6 +146,7 @@ typedef DPTR(class ReJitManager)        PTR_ReJitManager;
 typedef DPTR(struct ReJitInfo)          PTR_ReJitInfo;
 typedef DPTR(struct SharedReJitInfo)    PTR_SharedReJitInfo;
 typedef DPTR(class StringObject)        PTR_StringObject;
+typedef DPTR(class Utf8StringObject)    PTR_Utf8StringObject;
 typedef DPTR(class TypeHandle)          PTR_TypeHandle;
 #ifdef STUB_DISPATCH
 typedef VPTR(class VirtualCallStubManager) PTR_VirtualCallStubManager;

src/vm/appdomain.cpp

@@ -2762,6 +2762,11 @@ void SystemDomain::LoadBaseSystemClasses()
     _ASSERTE(g_pStringClass->GetBaseSize() == ObjSizeOf(StringObject)+sizeof(WCHAR));
     _ASSERTE(g_pStringClass->GetComponentSize() == 2);
 
+    // Load Utf8String
+    g_pUtf8StringClass = MscorlibBinder::GetClass(CLASS__UTF8_STRING);
+    _ASSERTE(g_pUtf8StringClass->GetBaseSize() == ObjSizeOf(Utf8StringObject)+sizeof(BYTE));
+    _ASSERTE(g_pUtf8StringClass->GetComponentSize() == 1);
+
     // Used by Buffer::BlockCopy
     g_pByteArrayMT = ClassLoader::LoadArrayTypeThrowing(
         TypeHandle(MscorlibBinder::GetElementType(ELEMENT_TYPE_U1))).AsArray()->GetMethodTable();

src/vm/classnames.h

@@ -149,6 +149,8 @@
 #define g_TransparentProxyName "__TransparentProxy"
 #define g_TypeClassName   "System.Type"
 
+#define g_Utf8StringName "Utf8String"
+
 #define g_VariantClassName "System.Variant"
 #define g_GuidClassName "System.Guid"
 

src/vm/common.h

@@ -168,6 +168,7 @@ typedef DPTR(struct ReJitInfo)          PTR_ReJitInfo;
 typedef DPTR(struct SharedReJitInfo)    PTR_SharedReJitInfo;
 typedef DPTR(class StringObject)        PTR_StringObject;
 typedef DPTR(class TypeHandle)          PTR_TypeHandle;
+typedef DPTR(class Utf8StringObject)    PTR_Utf8StringObject;
 typedef VPTR(class VirtualCallStubManager) PTR_VirtualCallStubManager;
 typedef VPTR(class VirtualCallStubManagerManager) PTR_VirtualCallStubManagerManager;
 typedef VPTR(class IGCHeap)             PTR_IGCHeap;

src/vm/ecalllist.h

@@ -116,6 +116,10 @@ FCFuncStart(gStringFuncs)
 #endif // FEATURE_COMINTEROP
 FCFuncEnd()
 
+FCFuncStart(gUtf8StringFuncs)
+    FCFuncElement("FastAllocate", Utf8StringFastAllocate)
+FCFuncEnd()
+
 FCFuncStart(gValueTypeFuncs)
     FCFuncElement("CanCompareBits", ValueTypeHelper::CanCompareBits)
     FCFuncElement("FastEqualsCheck", ValueTypeHelper::FastEqualsCheck)
@@ -1384,6 +1388,7 @@ FCClassElement("TypedReference", "System", gTypedReferenceFuncs)
 #ifdef FEATURE_COMINTEROP
 FCClassElement("UriMarshaler", "System.StubHelpers", gUriMarshalerFuncs)
 #endif
+FCClassElement("Utf8String", "System", gUtf8StringFuncs)
 FCClassElement("ValueClassMarshaler", "System.StubHelpers", gValueClassMarshalerFuncs)
 FCClassElement("ValueType", "System", gValueTypeFuncs)
 #ifdef FEATURE_COMINTEROP

src/vm/gchelpers.cpp

@@ -1059,6 +1059,73 @@ STRINGREF SlowAllocateString( DWORD cchStringLength )
     return( ObjectToSTRINGREF(orObject) );
 }
 
+UTF8STRINGREF SlowAllocateUtf8String(DWORD cchStringLength)
+{
+    CONTRACTL{
+        THROWS;
+        GC_TRIGGERS;
+        MODE_COOPERATIVE; // returns an objref without pinning it => cooperative
+    } CONTRACTL_END;
+
+    Utf8StringObject    *orObject = NULL;
+
+#ifdef _DEBUG
+    if (g_pConfig->ShouldInjectFault(INJECTFAULT_GCHEAP))
+    {
+        char *a = new char;
+        delete a;
+    }
+#endif
+
+    // Limit the maximum string size to <2GB to mitigate risk of security issues caused by 32-bit integer
+    // overflows in buffer size calculations.
+    if (cchStringLength > 0x7FFFFFDF)
+        ThrowOutOfMemory();
+
+    SIZE_T ObjectSize = PtrAlign(Utf8StringObject::GetSize(cchStringLength));
+    _ASSERTE(ObjectSize > cchStringLength);
+
+    SetTypeHandleOnThreadForAlloc(TypeHandle(g_pUtf8StringClass));
+
+    orObject = (Utf8StringObject *)Alloc(ObjectSize, FALSE, FALSE);
+
+    // Object is zero-init already
+    _ASSERTE(orObject->HasEmptySyncBlockInfo());
+
+    // Initialize Object
+    orObject->SetMethodTable(g_pUtf8StringClass);
+    orObject->SetUtf8StringLength(cchStringLength);
+
+    if (ObjectSize >= LARGE_OBJECT_SIZE)
+    {
+        GCHeapUtilities::GetGCHeap()->PublishObject((BYTE*)orObject);
+    }
+
+    // Notify the profiler of the allocation
+    if (TrackAllocations())
+    {
+        OBJECTREF objref = ObjectToOBJECTREF((Object*)orObject);
+        GCPROTECT_BEGIN(objref);
+        ProfilerObjectAllocatedCallback(objref, (ClassID)orObject->GetTypeHandle().AsPtr());
+        GCPROTECT_END();
+
+        orObject = (Utf8StringObject *)OBJECTREFToObject(objref);
+    }
+
+#ifdef FEATURE_EVENT_TRACE
+    // Send ETW event for allocation
+    if (ETW::TypeSystemLog::IsHeapAllocEventEnabled())
+    {
+        ETW::TypeSystemLog::SendObjectAllocatedEvent(orObject);
+    }
+#endif // FEATURE_EVENT_TRACE
+
+    LogAlloc(ObjectSize, g_pUtf8StringClass, orObject);
+
+    return(ObjectToUTF8STRINGREF(orObject));
+}
+
+
 #ifdef FEATURE_COMINTEROP_UNMANAGED_ACTIVATION
 // OBJECTREF AllocateComClassObject(ComClassFactory* pComClsFac)
 void AllocateComClassObject(ComClassFactory* pComClsFac, OBJECTREF* ppRefClass)

src/vm/gchelpers.h

@@ -71,6 +71,8 @@ STRINGREF AllocateString( DWORD cchStringLength );
     // The slow version, implemented in gcscan.cpp
 STRINGREF SlowAllocateString( DWORD cchStringLength );
 
+UTF8STRINGREF SlowAllocateUtf8String( DWORD cchStringLength );
+
 #else
 
 // On other platforms, go to the (somewhat less efficient) implementations in gcscan.cpp
@@ -83,6 +85,8 @@ OBJECTREF AllocateObjectArray(DWORD cElements, TypeHandle ElementType, BOOL bAll
 
 STRINGREF SlowAllocateString( DWORD cchStringLength );
 
+UTF8STRINGREF SlowAllocateUtf8String( DWORD cchStringLength );
+
 inline STRINGREF AllocateString( DWORD cchStringLength )
 {
     WRAPPER_NO_CONTRACT;

src/vm/methodtable.h

@@ -1882,7 +1882,7 @@ class MethodTable
     BOOL IsString()
     {
         LIMITED_METHOD_DAC_CONTRACT;
-        return HasComponentSize() && !IsArray();
+        return HasComponentSize() && !IsArray() && RawGetComponentSize() == 2;
     }
 
     BOOL            HasComponentSize() const

src/vm/methodtablebuilder.cpp

@@ -9744,6 +9744,17 @@ void MethodTableBuilder::CheckForSystemTypes()
 
             pMT->SetComponentSize(2);
         }
+        else if (strcmp(name, g_Utf8StringName) == 0 && strcmp(nameSpace, g_SystemNS) == 0)
+        {
+            // Utf8Strings are not "normal" objects, so we need to mess with their method table a bit
+            // so that the GC can figure out how big each string is...
+            DWORD baseSize = ObjSizeOf(Utf8StringObject) + sizeof(BYTE);
+            pMT->SetBaseSize(baseSize); // NULL character included
+
+            GetHalfBakedClass()->SetBaseSizePadding(baseSize - bmtFP->NumInstanceFieldBytes);
+
+            pMT->SetComponentSize(1);
+        }
         else if (strcmp(name, g_CriticalFinalizerObjectName) == 0 && strcmp(nameSpace, g_ConstrainedExecutionNS) == 0)
         {
             // To introduce a class with a critical finalizer,

src/vm/mscorlib.cpp

@@ -26,6 +26,7 @@
 //
 #include "arraynative.h"
 #include "stringnative.h"
+#include "utf8stringnative.h"
 #include "objectnative.h"
 #include "comdelegate.h"
 #include "customattribute.h"

src/vm/mscorlib.h

@@ -884,6 +884,8 @@ DEFINE_METHOD(STRING,               INTERNAL_COPY,          InternalCopy,
 DEFINE_METHOD(STRING,               WCSLEN,                 wcslen,                     SM_PtrChar_RetInt)
 DEFINE_PROPERTY(STRING,             LENGTH,                 Length,                     Int)
 
+DEFINE_CLASS(UTF8_STRING,           System,                 Utf8String)
+
 DEFINE_CLASS(STRING_BUILDER,        Text,                   StringBuilder)
 DEFINE_PROPERTY(STRING_BUILDER,     LENGTH,                 Length,                     Int)
 DEFINE_PROPERTY(STRING_BUILDER,     CAPACITY,               Capacity,                   Int)

src/vm/object.h

@@ -866,6 +866,7 @@ typedef DPTR(U8Array)   PTR_U8Array;
 typedef DPTR(PTRArray)  PTR_PTRArray;
 
 class StringObject;
+class Utf8StringObject;
 
 #ifdef USE_CHECKED_OBJECTREFS
 typedef REF<ArrayBase>  BASEARRAYREF;
@@ -883,6 +884,7 @@ typedef REF<U8Array>    U8ARRAYREF;
 typedef REF<CHARArray>  CHARARRAYREF;
 typedef REF<PTRArray>   PTRARRAYREF;  // Warning: Use PtrArray only for single dimensional arrays, not multidim arrays.
 typedef REF<StringObject> STRINGREF;
+typedef REF<Utf8StringObject> UTF8STRINGREF;
 
 #else   // USE_CHECKED_OBJECTREFS
 
@@ -901,6 +903,7 @@ typedef PTR_U8Array     U8ARRAYREF;
 typedef PTR_CHARArray   CHARARRAYREF;
 typedef PTR_PTRArray    PTRARRAYREF;  // Warning: Use PtrArray only for single dimensional arrays, not multidim arrays.
 typedef PTR_StringObject STRINGREF;
+typedef PTR_Utf8StringObject UTF8STRINGREF;
 
 #endif // USE_CHECKED_OBJECTREFS
 
@@ -935,6 +938,7 @@ typedef PTR_StringObject STRINGREF;
 #ifdef _MSC_VER
 #pragma warning(disable : 4200)     // disable zero-sized array warning
 #endif
+
 class StringObject : public Object
 {
 #ifdef DACCESS_COMPILE
@@ -1205,6 +1209,32 @@ class ReflectClassBaseObject : public BaseObjectWithCachedData
 
 };
 
+class Utf8StringObject : public Object
+{
+#ifdef DACCESS_COMPILE
+    friend class ClrDataAccess;
+#endif
+
+private:
+    DWORD   m_StringLength;
+    BYTE    m_Characters[0];
+    // GC will see a Utf8StringObject like this:
+    //   DWORD m_StringLength
+    //   BYTE  m_Characters[0]
+    //   DWORD m_OptionalPadding (this is an optional field and will appear based on need)
+
+public:
+    VOID    SetUtf8StringLength(DWORD len) { LIMITED_METHOD_CONTRACT; _ASSERTE(len >= 0); m_StringLength = len; }
+
+protected:
+    Utf8StringObject() { LIMITED_METHOD_CONTRACT; }
+    ~Utf8StringObject() { LIMITED_METHOD_CONTRACT; }
+
+public:
+    static SIZE_T GetSize(DWORD stringLength);
+};
+
+
 // This is the Method version of the Reflection object.
 //  A Method has adddition information.
 //   m_pMD - A pointer to the actual MethodDesc of the method.