Single-File: Run from Bundle (#36052)

* Single-File: Run from Bundle This change implements: * Runtime changes necessary to load assemblies directly from the bundle: * Design notes about [Load from Bundle](https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#peimage-loader) * Most of these changes are directly from dotnet/coreclr#26504 and dotnet/coreclr#26904 * Hostpolicy change to not add bundled assemblies to TPA list: * Design notes about [Dependency Resolution](https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#dependency-resolution) * TBD (separately) items: Fix for hammer servicing #36031 Fixes #32822 * Address some code review feedback * Address futher review feedback * Fix Assembly.ni.dll and Assembly.dll path usage. * Remove unused full-path computation for corelib when loading from bundle. * Add tracing when loading from bundle. * Add bundle-probing for satellite assemblies. * Adapt the BundleRename test to the .net5 scenario where assemblies are loaded from bundle.
dotnet · May 26, 2020 · f9d3e6f · f9d3e6f
2 parents df3f5d1 + c08d909
commit f9d3e6f
Show file tree

Hide file tree

Showing 40 changed files with 713 additions and 257 deletions.
diff --git a/src/coreclr/src/binder/assemblybinder.cpp b/src/coreclr/src/binder/assemblybinder.cpp
diff --git a/src/coreclr/src/binder/coreclrbindercommon.cpp b/src/coreclr/src/binder/coreclrbindercommon.cpp
@@ -7,6 +7,7 @@
 #include "assemblybinder.hpp"
 #include "coreclrbindercommon.h"
 #include "clrprivbindercoreclr.h"
+#include "bundle.h"
 
 using namespace BINDER_SPACE;
 

diff --git a/src/coreclr/src/binder/inc/assembly.hpp b/src/coreclr/src/binder/inc/assembly.hpp
@@ -27,15 +27,18 @@
 #include "clrprivbinderassemblyloadcontext.h"
 #endif // !defined(DACCESS_COMPILE) && !defined(CROSSGEN_COMPILE)
 
-STDAPI BinderAcquirePEImage(LPCTSTR   szAssemblyPath,
-                            PEImage **ppPEImage,
-                            PEImage **ppNativeImage,
-                            BOOL      fExplicitBindToNativeImage);
-
-STDAPI BinderAcquireImport(PEImage                  *pPEImage,
-                           IMDInternalImport       **pIMetaDataAssemblyImport,
-                           DWORD                    *pdwPAFlags,
-                           BOOL                     bNativeImage);
+#include "bundle.h"
+
+STDAPI BinderAcquirePEImage(LPCTSTR            szAssemblyPath,
+                            PEImage          **ppPEImage,
+                            PEImage          **ppNativeImage,
+                            BOOL               fExplicitBindToNativeImage,
+                            BundleFileLocation bundleFileLocation);
+
+STDAPI BinderAcquireImport(PEImage            *pPEImage,
+                           IMDInternalImport **pIMetaDataAssemblyImport,
+                           DWORD              *pdwPAFlags,
+                           BOOL                bNativeImage);
 
 STDAPI BinderHasNativeHeader(PEImage *pPEImage,
                              BOOL    *result);

diff --git a/src/coreclr/src/binder/inc/assemblybinder.hpp b/src/coreclr/src/binder/inc/assemblybinder.hpp
@@ -18,6 +18,7 @@
 #include "bindertypes.hpp"
 #include "bindresult.hpp"
 #include "coreclrbindercommon.h"
+#include "bundle.h"
 
 class CLRPrivBinderAssemblyLoadContext;
 class CLRPrivBinderCoreCLR;
@@ -54,7 +55,8 @@ namespace BINDER_SPACE
                                    /* in */  BOOL         fIsInGAC,
                                    /* in */  BOOL         fExplicitBindToNativeImage,
                                    /* out */ Assembly   **ppAssembly,
-                                   /* in */  LPCTSTR      szMDAssemblyPath = NULL);
+                                   /* in */  LPCTSTR      szMDAssemblyPath = NULL,
+                                   /* in */  BundleFileLocation bundleFileLocation = BundleFileLocation::Invalid());
 
 #if !defined(DACCESS_COMPILE) && !defined(CROSSGEN_COMPILE)
         static HRESULT BindUsingHostAssemblyResolver (/* in */ INT_PTR pManagedAssemblyLoadContextToBindWithin,

diff --git a/src/coreclr/src/binder/inc/bindertracing.h b/src/coreclr/src/binder/inc/bindertracing.h
@@ -180,7 +180,8 @@ namespace BinderTracing
         AppNativeImagePaths,
         AppPaths,
         PlatformResourceRoots,
-        SatelliteSubdirectory
+        SatelliteSubdirectory,
+        Bundle
     };
 
     void PathProbed(const WCHAR *path, PathSource source, HRESULT hr);

diff --git a/src/coreclr/src/binder/utils.cpp b/src/coreclr/src/binder/utils.cpp
@@ -80,7 +80,7 @@ namespace BINDER_SPACE
         SString platformPathSeparator(SString::Literal, GetPlatformPathSeparator());
         combinedPath.Set(pathA);
 
-        if (!combinedPath.EndsWith(platformPathSeparator))
+        if (!combinedPath.IsEmpty() && !combinedPath.EndsWith(platformPathSeparator))
         {
             combinedPath.Append(platformPathSeparator);
         }

diff --git a/src/coreclr/src/dlls/mscoree/unixinterface.cpp b/src/coreclr/src/dlls/mscoree/unixinterface.cpp
@@ -18,6 +18,7 @@
 #ifdef FEATURE_GDBJIT
 #include "../../vm/gdbjithelpers.h"
 #endif // FEATURE_GDBJIT
+#include "bundle.h"
 
 #define ASSERTE_ALL_BUILDS(expr) _ASSERTE_ALL_BUILDS(__FILE__, (expr))
 
@@ -112,7 +113,8 @@ static void ConvertConfigPropertiesToUnicode(
     const char** propertyValues,
     int propertyCount,
     LPCWSTR** propertyKeysWRef,
-    LPCWSTR** propertyValuesWRef)
+    LPCWSTR** propertyValuesWRef,
+    BundleProbe** bundleProbe)
 {
     LPCWSTR* propertyKeysW = new (nothrow) LPCWSTR[propertyCount];
     ASSERTE_ALL_BUILDS(propertyKeysW != nullptr);
@@ -124,6 +126,13 @@ static void ConvertConfigPropertiesToUnicode(
     {
         propertyKeysW[propertyIndex] = StringToUnicode(propertyKeys[propertyIndex]);
         propertyValuesW[propertyIndex] = StringToUnicode(propertyValues[propertyIndex]);
+
+        if (strcmp(propertyKeys[propertyIndex], "BUNDLE_PROBE") == 0)
+        {
+            // If this application is a single-file bundle, the bundle-probe callback 
+            // is passed in as the value of "BUNDLE_PROBE" property (encoded as a string).
+            *bundleProbe = (BundleProbe*)_wcstoui64(propertyValuesW[propertyIndex], nullptr, 0);
+        }
     }
 
     *propertyKeysWRef = propertyKeysW;
@@ -183,12 +192,21 @@ int coreclr_initialize(
 
     LPCWSTR* propertyKeysW;
     LPCWSTR* propertyValuesW;
+    BundleProbe* bundleProbe = nullptr;
+
     ConvertConfigPropertiesToUnicode(
         propertyKeys,
         propertyValues,
         propertyCount,
         &propertyKeysW,
-        &propertyValuesW);
+        &propertyValuesW,
+        &bundleProbe);
+
+    if (bundleProbe != nullptr)
+    {
+        static Bundle bundle(StringToUnicode(exePath), bundleProbe);
+        Bundle::AppBundle = &bundle;
+    }
 
     // This will take ownership of propertyKeysWTemp and propertyValuesWTemp
     Configuration::InitializeConfigurationKnobs(propertyCount, propertyKeysW, propertyValuesW);

diff --git a/src/coreclr/src/inc/bundle.h b/src/coreclr/src/inc/bundle.h
@@ -0,0 +1,62 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+/*****************************************************************************
+ **                                                                         **
+ ** bundle.h - Information about applications bundled as a single-file  **
+ **                                                                         **
+ *****************************************************************************/
+
+#ifndef _BUNDLE_H_
+#define _BUNDLE_H_
+
+#include <sstring.h>
+
+class Bundle;
+
+struct BundleFileLocation
+{
+    INT64 Size;
+    INT64 Offset;
+
+    BundleFileLocation()
+    { 
+        LIMITED_METHOD_CONTRACT;
+
+        Size = 0; 
+        Offset = 0; 
+    }
+
+    static BundleFileLocation Invalid() { LIMITED_METHOD_CONTRACT; return BundleFileLocation(); }
+
+    const SString &Path() const;
+
+    bool IsValid() const { LIMITED_METHOD_CONTRACT; return Offset != 0; }
+};
+
+typedef bool(__stdcall BundleProbe)(LPCWSTR, INT64*, INT64*);
+
+class Bundle
+{
+public:
+    Bundle(LPCWSTR bundlePath, BundleProbe *probe);
+    BundleFileLocation Probe(LPCWSTR path, bool pathIsBundleRelative = false) const;
+
+    const SString &Path() const { LIMITED_METHOD_CONTRACT; return m_path; }
+    const SString &BasePath() const { LIMITED_METHOD_CONTRACT; return m_basePath; }
+
+    static Bundle* AppBundle; // The BundleInfo for the current app, initialized by coreclr_initialize.
+    static bool AppIsBundle() { LIMITED_METHOD_CONTRACT; return AppBundle != nullptr; }
+    static BundleFileLocation ProbeAppBundle(LPCWSTR path, bool pathIsBundleRelative = false);
+
+private:
+
+    SString m_path; // The path to single-file executable
+    BundleProbe *m_probe;
+
+    SString m_basePath; // The prefix to denote a path within the bundle
+};
+
+#endif // _BUNDLE_H_
+// EOF =======================================================================
diff --git a/src/coreclr/src/pal/inc/pal.h b/src/coreclr/src/pal/inc/pal.h
@@ -47,6 +47,9 @@ Module Name:
 #include <string.h>
 #include <errno.h>
 #include <ctype.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
 #endif
 
 #ifdef  __cplusplus
@@ -2543,6 +2546,7 @@ Abstract
 
 Parameters:
     IN hFile    - The file to load
+    IN offset - offset within hFile where the PE "file" is located
 
 Return value:
     A valid base address if successful.
@@ -2551,7 +2555,7 @@ Return value:
 PALIMPORT
 PVOID
 PALAPI
-PAL_LOADLoadPEFile(HANDLE hFile);
+PAL_LOADLoadPEFile(HANDLE hFile, size_t offset);
 
 /*++
     PAL_LOADUnloadPEFile

diff --git a/src/coreclr/src/pal/src/include/pal/map.hpp b/src/coreclr/src/pal/src/include/pal/map.hpp
@@ -79,13 +79,14 @@ extern "C"
 
     Parameters:
         IN hFile - file to map
+        IN offset - offset within hFile where the PE "file" is located
 
     Return value:
         non-NULL - the base address of the mapped image
         NULL - error, with last error set.
     --*/
 
-    void * MAPMapPEFile(HANDLE hFile);
+    void* MAPMapPEFile(HANDLE hFile, off_t offset);
 
     /*++
     Function :

diff --git a/src/coreclr/src/pal/src/include/pal/module.h b/src/coreclr/src/pal/src/include/pal/module.h
@@ -145,12 +145,13 @@ Abstract
 
 Parameters:
     IN hFile    - The file to load
+    IN offset - offset within hFile where the PE "file" is located
 
 Return value:
     A valid base address if successful.
     0 if failure
 --*/
-void * PAL_LOADLoadPEFile(HANDLE hFile);
+void* PAL_LOADLoadPEFile(HANDLE hFile, size_t offset);
 
 /*++
     PAL_LOADUnloadPEFile

diff --git a/src/coreclr/src/pal/src/loader/module.cpp b/src/coreclr/src/pal/src/loader/module.cpp
@@ -752,18 +752,19 @@ PAL_UnregisterModule(
 
 Parameters:
     IN hFile - file to map
+    IN offset - offset within hFile where the PE "file" is located
 
 Return value:
     non-NULL - the base address of the mapped image
     NULL - error, with last error set.
 --*/
 PVOID
 PALAPI
-PAL_LOADLoadPEFile(HANDLE hFile)
+PAL_LOADLoadPEFile(HANDLE hFile, size_t offset)
 {
-    ENTRY("PAL_LOADLoadPEFile (hFile=%p)\n", hFile);
+    ENTRY("PAL_LOADLoadPEFile (hFile=%p, offset=%zx)\n", hFile, offset);
 
-    void * loadedBase = MAPMapPEFile(hFile);
+    void* loadedBase = MAPMapPEFile(hFile, offset);
 
 #ifdef _DEBUG
     if (loadedBase != nullptr)
@@ -775,7 +776,7 @@ PAL_LOADLoadPEFile(HANDLE hFile)
             {
                 TRACE("Forcing failure of PE file map, and retry\n");
                 PAL_LOADUnloadPEFile(loadedBase); // unload it
-                loadedBase = MAPMapPEFile(hFile); // load it again
+                loadedBase = MAPMapPEFile(hFile, offset); // load it again
             }
 
             free(envVar);
@@ -1547,7 +1548,8 @@ static MODSTRUCT *LOADAddModule(NATIVE_LIBRARY_HANDLE dl_handle, LPCSTR libraryN
         {
             /* found the handle. increment the refcount and return the
                existing module structure */
-            TRACE("Found matching module %p for module name %s\n", module, libraryNameOrPath);
+            TRACE("Found matching module %p for module name %s\n", module,
+                (libraryNameOrPath != nullptr) ? libraryNameOrPath : "nullptr");
 
             if (module->refcount != -1)
             {

diff --git a/src/coreclr/src/pal/src/map/map.cpp b/src/coreclr/src/pal/src/map/map.cpp
@@ -1088,6 +1088,8 @@ CorUnix::InternalMapViewOfFile(
     CFileMappingImmutableData *pImmutableData = NULL;
     CFileMappingProcessLocalData *pProcessLocalData = NULL;
     IDataLock *pProcessLocalDataLock = NULL;
+    INT64 offset = ((INT64)dwFileOffsetHigh << 32) | (INT64)dwFileOffsetLow;
+
 #if ONE_SHARED_MAPPING_PER_FILEREGION_PER_PROCESS
     PMAPPED_VIEW_LIST pReusedMapping = NULL;
 #endif
@@ -1102,9 +1104,9 @@ CorUnix::InternalMapViewOfFile(
         goto InternalMapViewOfFileExit;
     }
 
-    if ( 0 != dwFileOffsetHigh || 0 != dwFileOffsetLow )
+    if (offset < 0)
     {
-        ASSERT( "dwFileOffsetHigh and dwFileOffsetLow are always 0.\n" );
+        ASSERT("dwFileOffsetHigh | dwFileOffsetLow should be non-negative.\n");
         palError = ERROR_INVALID_PARAMETER;
         goto InternalMapViewOfFileExit;
     }
@@ -1182,7 +1184,7 @@ CorUnix::InternalMapViewOfFile(
             PROT_READ|PROT_WRITE,
             flags,
             pProcessLocalData->UnixFd,
-            0
+            offset
             );
     }
     else
@@ -1205,7 +1207,7 @@ CorUnix::InternalMapViewOfFile(
                 prot,
                 flags,
                 pProcessLocalData->UnixFd,
-                0
+                offset
                 );
 
 #if ONE_SHARED_MAPPING_PER_FILEREGION_PER_PROCESS
@@ -2210,13 +2212,14 @@ MAPmmapAndRecord(
 
 Parameters:
     IN hFile - file to map
+    IN offset - offset within hFile where the PE "file" is located
 
 Return value:
     non-NULL - the base address of the mapped image
     NULL - error, with last error set.
 --*/
 
-void * MAPMapPEFile(HANDLE hFile)
+void * MAPMapPEFile(HANDLE hFile, off_t offset)
 {
     PAL_ERROR palError = 0;
     IPalObject *pFileObject = NULL;
@@ -2231,7 +2234,7 @@ void * MAPMapPEFile(HANDLE hFile)
     char* envVar;
 #endif
 
-    ENTRY("MAPMapPEFile (hFile=%p)\n", hFile);
+    ENTRY("MAPMapPEFile (hFile=%p offset=%zx)\n", hFile, offset);
 
     //Step 0: Verify values, find internal pal data structures.
     if (INVALID_HANDLE_VALUE == hFile)
@@ -2270,13 +2273,13 @@ void * MAPMapPEFile(HANDLE hFile)
     //Step 1: Read the PE headers and reserve enough space for the whole image somewhere.
     IMAGE_DOS_HEADER dosHeader;
     IMAGE_NT_HEADERS ntHeader;
-    if (sizeof(dosHeader) != pread(fd, &dosHeader, sizeof(dosHeader), 0))
+    if (sizeof(dosHeader) != pread(fd, &dosHeader, sizeof(dosHeader), offset))
     {
         palError = FILEGetLastErrorFromErrno();
         ERROR_(LOADER)( "reading dos header failed\n" );
         goto done;
     }
-    if (sizeof(ntHeader) != pread(fd, &ntHeader, sizeof(ntHeader), dosHeader.e_lfanew))
+    if (sizeof(ntHeader) != pread(fd, &ntHeader, sizeof(ntHeader), offset + dosHeader.e_lfanew))
     {
         palError = FILEGetLastErrorFromErrno();
         goto done;
@@ -2418,7 +2421,7 @@ void * MAPMapPEFile(HANDLE hFile)
 
     //first, map the PE header to the first page in the image.  Get pointers to the section headers
     palError = MAPmmapAndRecord(pFileObject, loadedBase,
-                    loadedBase, headerSize, PROT_READ, MAP_FILE|MAP_PRIVATE|MAP_FIXED, fd, 0,
+                    loadedBase, headerSize, PROT_READ, MAP_FILE|MAP_PRIVATE|MAP_FIXED, fd, offset,
                     (void**)&loadedHeader);
     if (NO_ERROR != palError)
     {
@@ -2511,7 +2514,7 @@ void * MAPMapPEFile(HANDLE hFile)
                         prot,
                         MAP_FILE|MAP_PRIVATE|MAP_FIXED,
                         fd,
-                        currentHeader.PointerToRawData,
+                        offset + currentHeader.PointerToRawData,
                         &sectionData);
         if (NO_ERROR != palError)
         {
@@ -2541,7 +2544,7 @@ void * MAPMapPEFile(HANDLE hFile)
         palError = MAPRecordMapping(pFileObject,
                         loadedBase,
                         prevSectionEnd,
-                        (char*)imageEnd - (char*)prevSectionEnd,
+                        offset + (char*)imageEnd - (char*)prevSectionEnd,
                         PROT_NONE);
         if (NO_ERROR != palError)
         {