Ref assembly output part1

azure-pipelines.yml

@@ -353,21 +353,18 @@ stages:
           - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release
             displayName: End to end build tests
 
-        # Determinism
+        # Determinism, we want to run it only in PR builds
         - job: Determinism_Debug
+          condition: eq(variables['Build.Reason'], 'PullRequest')
+          variables:
+          - name: _SignType
+            value: Test
           pool:
-            name: NetCore1ESPool-Public
-            demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
+            vmImage: windows-latest
           timeoutInMinutes: 90
           steps:
-          - checkout: none
-          - script: |
-              @echo on
-              git init
-              git remote add origin "$(Build.Repository.Uri)"
-              git fetch --progress --no-tags --depth=1 origin "$(Build.SourceVersion)"
-              git checkout "$(Build.SourceVersion)"
-            displayName: Shallow checkout
+          - checkout: self
+            clean: true
           - script: .\eng\test-determinism.cmd -configuration Debug
             displayName: Determinism tests with Debug configuration
           - task: PublishPipelineArtifact@1

eng/Version.Details.xml

@@ -8,9 +8,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21569.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21606.6">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>97463777ee9a8445d4a4c5911ede0f0cd71fa8aa</Sha>
+      <Sha>5d969787afb2fd87f642458687e3ad41094ac3ab</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>

eng/build-utils.ps1

@@ -247,7 +247,7 @@ function Make-BootstrapBuild() {
     $args = "build $buildToolsProject -c $bootstrapConfiguration -v $verbosity" + $argNoRestore + $argNoIncremental
     if ($binaryLog) {
         $logFilePath = Join-Path $LogDir "toolsBootstrapLog.binlog"
-        $args += " /bl:$logFilePath"
+        $args += " /bl:`"$logFilePath`""
     }
     Exec-Console $dotnetExe $args
 
@@ -260,7 +260,7 @@ function Make-BootstrapBuild() {
     $args = "build $protoProject -c $bootstrapConfiguration -v $verbosity -f $bootstrapTfm" + $argNoRestore + $argNoIncremental
     if ($binaryLog) {
         $logFilePath = Join-Path $LogDir "protoBootstrapLog.binlog"
-        $args += " /bl:$logFilePath"
+        $args += " /bl:`"$logFilePath`""
     }
     Exec-Console $dotnetExe $args
 

eng/common/build.sh

@@ -187,10 +187,6 @@ function InitializeCustomToolset {
 }
 
 function Build {
-
-  if [[ "$ci" == true ]]; then
-    TryLogClientIpAddress
-  fi
   InitializeToolset
   InitializeCustomToolset
 

eng/common/native/init-compiler.sh

@@ -2,6 +2,7 @@
 #
 # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
 #
+# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! 
 
 if [[ "$#" -lt 3 ]]; then
   echo "Usage..."
@@ -111,12 +112,10 @@ if [[ -z "$CC" ]]; then
     exit 1
 fi
 
-if [[ "$compiler" == "clang" ]]; then
-    if command -v "lld$desired_version" > /dev/null; then
-        # Only lld version >= 9 can be considered stable
-        if [[ "$majorVersion" -ge 9 ]]; then
-            LDFLAGS="-fuse-ld=lld"
-        fi
+# Only lld version >= 9 can be considered stable
+if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 ]]; then
+    if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
+        LDFLAGS="-fuse-ld=lld"
     fi
 fi
 

eng/common/sdk-task.ps1

@@ -83,9 +83,6 @@ try {
   }
 
   if ($restore) {
-    if ($ci) {
-      Try-LogClientIpAddress
-    }
     Build 'Restore'
   }
 

eng/common/sdl/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Microsoft.Guardian.Cli" version="0.53.3"/>
+  <package id="Microsoft.Guardian.Cli" version="0.109.0"/>
 </packages>

eng/common/templates/job/execute-sdl.yml

@@ -54,7 +54,7 @@ jobs:
     # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in
     # sync with the packages.config file.
     - name: DefaultGuardianVersion
-      value: 0.53.3
+      value: 0.109.0
     - name: GuardianVersion
       value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }}
     - name: GuardianPackagesConfigFile

eng/common/templates/job/job.yml

@@ -114,6 +114,7 @@ jobs:
         continueOnError: ${{ parameters.continueOnError }}
         condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
 
+  - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}:
     - task: NuGetAuthenticate@0
 
   - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:

eng/common/tools.ps1

@@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
   # Disable telemetry on CI.
   if ($ci) {
     $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
-
-    # In case of network error, try to log the current IP for reference
-    Try-LogClientIpAddress
   }
 
   # Source Build uses DotNetCoreSdkDir variable
@@ -895,24 +892,6 @@ if (!$disableConfigureToolsetImport) {
   }
 }
 
-function Try-LogClientIpAddress()
-{
-    Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
-    try
-    {
-        $result = Invoke-WebRequest -Uri "http://co1r5a.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
-        $lines = $result.Content.Split([Environment]::NewLine) 
-        $socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
-        Write-Host $socketIp
-        $clientIp = $lines | Select-String -Pattern "^Client IP:.*"
-        Write-Host $clientIp
-    }
-    catch
-    {
-        Write-Host "Unable to get this machine's effective IP address for logging: $_"
-    }
-}
-
 #
 # If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
 #

eng/common/tools.sh

@@ -178,7 +178,7 @@ function InstallDotNetSdk {
   if [[ $# -ge 3 ]]; then
     architecture=$3
   fi
-  InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key
+  InstallDotNet "$root" "$version" $architecture 'sdk' 'true' $runtime_source_feed $runtime_source_feed_key
 }
 
 function InstallDotNet {
@@ -405,13 +405,6 @@ function StopProcesses {
   return 0
 }
 
-function TryLogClientIpAddress () {
-  echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes'
-  if command -v curl > /dev/null; then
-    curl -s 'http://co1r5a.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true
-  fi
-}
-
 function MSBuild {
   local args=$@
   if [[ "$pipelines_log" == true ]]; then

eng/pipelines/checkout-windows-task.yml

@@ -0,0 +1,11 @@
+# Shallow checkout sources on Windows
+steps:
+  - checkout: none
+
+  - script: |
+      @echo on
+      git init
+      git remote add origin "$(Build.Repository.Uri)"
+      git fetch --progress --no-tags --depth=1 origin "$(Build.SourceVersion)"
+      git checkout "$(Build.SourceVersion)"
+    displayName: Shallow Checkout

eng/pipelines/publish-logs.yml

@@ -0,0 +1,17 @@
+# Build on windows desktop
+parameters:
+- name: jobName
+  type: string
+  default: ''
+- name: configuration
+  type: string
+  default: 'Debug'
+
+steps:
+  - task: PublishPipelineArtifact@1
+    displayName: Publish Logs
+    inputs:
+      targetPath: '$(Build.SourcesDirectory)/artifacts/log/${{ parameters.configuration }}'
+      artifactName: '${{ parameters.jobName }} Attempt $(System.JobAttempt) Logs'
+    continueOnError: true
+    condition: not(succeeded())

eng/test-determinism.cmd

@@ -1,2 +1,2 @@
 @echo off
-powershell -noprofile -executionPolicy RemoteSigned -file "%~dp0\test-determinism.ps1" %*
+powershell -noprofile -executionPolicy RemoteSigned -file "%~dp0\test-determinism.ps1" %*

eng/test-determinism.ps1

@@ -39,7 +39,7 @@ function Run-Build([string]$rootDir, [string]$logFileName) {
   $stopWatch.Stop()
   Write-Host "Cleaning took $($stopWatch.Elapsed)"
 
-  $solution = Join-Path $rootDir "FSharp.sln"
+  $solution = Join-Path $rootDir (Join-Path "service" "FSharp.Compiler.Service.sln")
 
   if ($logFileName -eq "") {
     $logFileName = [IO.Path]::GetFileNameWithoutExtension($projectFilePath)
@@ -59,6 +59,9 @@ function Run-Build([string]$rootDir, [string]$logFileName) {
     /p:Rebuild=false `
     /p:Pack=false `
     /p:Sign=false `
+    /p:SignType=Test `
+    /p:DotNetSignType=Test `
+    /p:MicroBuild_SigningEnabled=false `
     /p:Publish=false `
     /p:ContinuousIntegrationBuild=false `
     /p:OfficialBuildId="" `
@@ -194,10 +197,12 @@ function Test-Build([string]$rootDir, $dataMap, [string]$logFileName) {
     }
 
     $oldfileData = $datamap[$fileId]
-    if ($fileData.Hash -ne $oldFileData.Hash) {
-      Write-Host "`tERROR! $relativeDir\$fileName contents don't match"
+    $oldHash = $oldfileData.Hash
+    $newHash = $fileData.Hash
+    if ($newHash -ne $oldHash) {
+      Write-Host "`tERROR! $relativeDir\$fileName hashes don't match"
       $allGood = $false
-      $errorList += $fileName
+      $errorList += "$fileName (old hash: $oldHash; new hash: $newHash)"
 
       $errorCurrentDirLeft = Join-Path $errorDirLeft $relativeDir
       Create-Directory $errorCurrentDirLeft

global.json

@@ -14,7 +14,7 @@
     }
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21569.2",
+    "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21606.6",
     "Microsoft.DotNet.Helix.Sdk": "2.0.0-beta.19069.2"
   }
 }

src/fsharp/AugmentWithHashCompare.fs

@@ -823,7 +823,7 @@ let TyconIsCandidateForAugmentationWithCompare (g: TcGlobals) (tycon: Tycon) =
     // This type gets defined in prim-types, before we can add attributes to F# type definitions
     let isUnit = g.compilingFslib && tycon.DisplayName = "Unit"
     not isUnit && 
-    not (TyconRefHasAttribute g tycon.Range g.attrib_IsByRefLikeAttribute (mkLocalTyconRef tycon)) &&
+    not (isByrefLikeTyconRef g tycon.Range (mkLocalTyconRef tycon)) &&
     match getAugmentationAttribs g tycon with 
     // [< >] 
     | true, true, None, None, None, None, None, None, None
@@ -838,7 +838,7 @@ let TyconIsCandidateForAugmentationWithEquals (g: TcGlobals) (tycon: Tycon) =
     // This type gets defined in prim-types, before we can add attributes to F# type definitions
     let isUnit = g.compilingFslib && tycon.DisplayName = "Unit"
     not isUnit && 
-    not (TyconRefHasAttribute g tycon.Range g.attrib_IsByRefLikeAttribute (mkLocalTyconRef tycon)) &&
+    not (isByrefLikeTyconRef g tycon.Range (mkLocalTyconRef tycon)) &&
 
     match getAugmentationAttribs g tycon with 
     // [< >] 

src/fsharp/CompilerConfig.fs

@@ -325,6 +325,12 @@ type PackageManagerLine =
     static member StripDependencyManagerKey (packageKey: string) (line: string): string =
         line.Substring(packageKey.Length + 1).Trim()
 
+[<RequireQualifiedAccess>]
+type MetadataAssemblyGeneration =
+    | None
+    | ReferenceOut of outputPath: string
+    | ReferenceOnly
+
 [<NoEquality; NoComparison>]
 type TcConfigBuilder =
     {
@@ -437,6 +443,7 @@ type TcConfigBuilder =
       mutable emitTailcalls: bool
       mutable deterministic: bool
       mutable concurrentBuild: bool
+      mutable emitMetadataAssembly: MetadataAssemblyGeneration
       mutable preferredUiLang: string option
       mutable lcid: int option
       mutable productNameForBannerText: string
@@ -642,6 +649,7 @@ type TcConfigBuilder =
           emitTailcalls = true
           deterministic = false
           concurrentBuild = true
+          emitMetadataAssembly = MetadataAssemblyGeneration.None
           preferredUiLang = None
           lcid = None
           productNameForBannerText = FSharpProductName
@@ -1022,6 +1030,7 @@ type TcConfig private (data: TcConfigBuilder, validate: bool) =
     member x.emitTailcalls = data.emitTailcalls
     member x.deterministic = data.deterministic
     member x.concurrentBuild = data.concurrentBuild
+    member x.emitMetadataAssembly = data.emitMetadataAssembly
     member x.pathMap = data.pathMap
     member x.langVersion = data.langVersion
     member x.preferredUiLang = data.preferredUiLang

src/fsharp/CompilerConfig.fsi

@@ -143,6 +143,16 @@ type PackageManagerLine =
     static member SetLinesAsProcessed: string -> Map<string, PackageManagerLine list> -> Map<string, PackageManagerLine list>
     static member StripDependencyManagerKey: string -> string -> string
 
+[<RequireQualifiedAccess>]
+type MetadataAssemblyGeneration =
+    | None
+    /// Includes F# signature and optimization metadata as resources in the emitting assembly.
+    /// Implementation assembly will still be emitted normally, but will emit the reference assembly with the specified output path. 
+    | ReferenceOut of outputPath: string
+    /// Includes F# signature and optimization metadata as resources in the emitting assembly.
+    /// Only emits the assembly as a reference assembly.
+    | ReferenceOnly
+
 [<NoEquality; NoComparison>]
 type TcConfigBuilder =
     { mutable primaryAssembly: PrimaryAssembly
@@ -249,6 +259,7 @@ type TcConfigBuilder =
       mutable emitTailcalls: bool
       mutable deterministic: bool
       mutable concurrentBuild: bool
+      mutable emitMetadataAssembly: MetadataAssemblyGeneration
       mutable preferredUiLang: string option
       mutable lcid        : int option
       mutable productNameForBannerText: string
@@ -439,6 +450,7 @@ type TcConfig =
     member emitTailcalls: bool
     member deterministic: bool
     member concurrentBuild: bool
+    member emitMetadataAssembly: MetadataAssemblyGeneration
     member pathMap: PathMap
     member preferredUiLang: string option
     member optsOn       : bool

src/fsharp/CompilerOptions.fs

@@ -406,6 +406,23 @@ let SetTailcallSwitch (tcConfigB: TcConfigBuilder) switch =
 let SetDeterministicSwitch (tcConfigB: TcConfigBuilder) switch =
     tcConfigB.deterministic <- (switch = OptionSwitch.On)
 
+let SetReferenceAssemblyOnlySwitch (tcConfigB: TcConfigBuilder) switch =
+    match tcConfigB.emitMetadataAssembly with
+    | MetadataAssemblyGeneration.None ->
+        tcConfigB.emitMetadataAssembly <- if (switch = OptionSwitch.On) then MetadataAssemblyGeneration.ReferenceOnly else MetadataAssemblyGeneration.None
+    | _ ->
+        error(Error(FSComp.SR.optsInvalidRefAssembly(), rangeCmdArgs))
+
+let SetReferenceAssemblyOutSwitch (tcConfigB: TcConfigBuilder) outputPath =
+    match tcConfigB.emitMetadataAssembly with
+    | MetadataAssemblyGeneration.None ->      
+        if FileSystem.IsInvalidPathShim outputPath then
+            error(Error(FSComp.SR.optsInvalidRefOut(), rangeCmdArgs))
+        else
+            tcConfigB.emitMetadataAssembly <- MetadataAssemblyGeneration.ReferenceOut outputPath
+    | _ ->
+        error(Error(FSComp.SR.optsInvalidRefAssembly(), rangeCmdArgs))
+
 let AddPathMapping (tcConfigB: TcConfigBuilder) (pathPair: string) =
     match pathPair.Split([|'='|], 2) with
     | [| oldPrefix; newPrefix |] ->
@@ -723,6 +740,16 @@ let outputFileFlagsFsc (tcConfigB: TcConfigBuilder) =
            ("nocopyfsharpcore", tagNone,
             OptionUnit (fun () -> tcConfigB.copyFSharpCore <- CopyFSharpCoreFlag.No), None,
             Some (FSComp.SR.optsNoCopyFsharpCore()))
+
+        CompilerOption
+           ("refonly", tagNone,
+            OptionSwitch (SetReferenceAssemblyOnlySwitch tcConfigB), None,
+            Some (FSComp.SR.optsRefOnly()))
+
+        CompilerOption
+           ("refout", tagFile,
+            OptionString (SetReferenceAssemblyOutSwitch tcConfigB), None,
+            Some (FSComp.SR.optsRefOut()))
     ]