merge main to feature/datalakestg82

.github/workflows/event.yml

@@ -1,13 +1,22 @@
-name: GitHub Event Handler
+# NOTE: currently azure-sdk-actions only hosts check enforcer code.
+# If further functionality is added, this name should be updated to reflect
+# the more generic behavior
+name: Check Enforcer
 
 on:
   check_suite:
     types: [completed]
   issue_comment:
     types: [created]
 
+permissions: {}
+
 jobs:
   event-handler:
+    permissions:
+      statuses: write # to set status (azure/azure-sdk-actions)
+      pull-requests: read # to read pull requests (azure/azure-sdk-actions)
+      checks: read # to read check status (azure/azure-sdk-actions)
     name: Handle ${{ github.event_name }} ${{ github.event.action }} event
     runs-on: ubuntu-latest
     steps:

CONTRIBUTING.md

@@ -15,7 +15,7 @@ Thank you for your interest in contributing to Azure SDK for C++.
 - **DO** submit all code changes via pull requests (PRs) rather than through a direct commit. PRs will be reviewed and potentially merged by the repo maintainers after a peer review that includes at least one maintainer.
 - **DO** review your own PR to make sure there aren't any unintended changes or commits before submitting it.
 - **DO NOT** submit "work in progress" PRs. A PR should only be submitted when it is considered ready for review and subsequent merging by the contributor.
-  - If the change is work-in-progress or an experiment, **DO** start if off as a temporary draft PR.
+  - If the change is work-in-progress or an experiment, **DO** start it off as a temporary draft PR.
 - **DO** give PRs short-but-descriptive names (e.g. "Improve code coverage for Azure.Core by 10%", not "Fix #1234") and add a description which explains why the change is being made.
 - **DO** refer to any relevant issues, and include [keywords](https://docs.github.com/articles/closing-issues-via-commit-messages/) that automatically close issues when the PR is merged.
 - **DO** tag any users that should know about and/or review the change.
@@ -48,13 +48,13 @@ Codespaces is new technology that allows you to use a container as your developm
 ### GitHub Codespaces
 
 1. From the Azure SDK GitHub repo, click on the "Code -> Open with Codespaces" button.
-1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing).
+1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project).
 
 ### VS Code Codespaces
 
 1. Install the [VS Code Remote Extension Pack](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack)
 1. When you open the Azure SDK for C++ repo in VS Code, it will prompt you to open the project in the Dev Container. If it does not prompt you, then hit CTRL+P, and select "Remote-Containers: Open Folder in Container..."
-1. Open a Terminal. The development environment will be ready for you. Continue to [Building and Testing](https://github.com/Azure/azure-sdk-for-cpp/blob/main/CONTRIBUTING.md#building-and-testing).
+1. Open a Terminal. The development environment will be ready for you. Continue to [Building the project](#building-the-project).
 
 ## Full Local Setup
 

eng/common/pipelines/templates/steps/sparse-checkout.yml

@@ -8,16 +8,12 @@ parameters:
       - Name: $(Build.Repository.Name)
         Commitish: $(Build.SourceVersion)
         WorkingDirectory: $(System.DefaultWorkingDirectory)
-  # NOTE: SkipDefaultCheckout is being deprecated in favor of SkipCheckoutNone
-  - name: SkipDefaultCheckout
-    type: boolean
-    default: false
   - name: SkipCheckoutNone
     type: boolean
     default: false
 
 steps:
-  - ${{ if and(not(parameters.SkipDefaultCheckout), not(parameters.SkipCheckoutNone)) }}:
+  - ${{ if not(parameters.SkipCheckoutNone) }}:
       - checkout: none
 
   - task: PowerShell@2

eng/common/scripts/Helpers/DevOps-WorkItem-Helpers.ps1

@@ -108,8 +108,8 @@ function LoginToAzureDevops([string]$devops_pat)
   if (!$devops_pat) {
     return
   }
-  $azCmdStr = "'$devops_pat' | az devops login $($ReleaseDevOpsOrgParameters -join ' ')"
-  Invoke-Expression $azCmdStr
+  # based on the docs at https://aka.ms/azure-devops-cli-auth the recommendation is to set this env variable to login
+  $env:AZURE_DEVOPS_EXT_PAT = $devops_pat
 }
 
 function BuildHashKeyNoNull()
@@ -234,6 +234,7 @@ function FindPackageWorkItem($lang, $packageName, $version, $outputCommand = $tr
   $fields += "Custom.PackagePatchVersions"
   $fields += "Custom.Generated"
   $fields += "Custom.RoadmapState"
+  $fields += "Microsoft.VSTS.Common.StateChangeDate"
 
   $fieldList = ($fields | ForEach-Object { "[$_]"}) -join ", "
   $query = "SELECT ${fieldList} FROM WorkItems WHERE [Work Item Type] = 'Package'"
@@ -466,7 +467,7 @@ function CreateOrUpdatePackageWorkItem($lang, $pkg, $verMajorMinor, $existingIte
     if ($pkgName -ne $existingItem.fields["Custom.Package"]) { $changedField = "Custom.Package" }
     if ($verMajorMinor -ne $existingItem.fields["Custom.PackageVersionMajorMinor"]) { $changedField = "Custom.PackageVersionMajorMinor" }
     if ($pkgDisplayName -ne $existingItem.fields["Custom.PackageDisplayName"]) { $changedField = "Custom.PackageDisplayName" }
-    if ($pkgType -ne $existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" }
+    if ($pkgType -ne [string]$existingItem.fields["Custom.PackageType"]) { $changedField = "Custom.PackageType" }
     if ($pkgNewLibrary -ne $existingItem.fields["Custom.PackageTypeNewLibrary"]) { $changedField = "Custom.PackageTypeNewLibrary" }
     if ($pkgRepoPath -ne $existingItem.fields["Custom.PackageRepoPath"]) { $changedField = "Custom.PackageRepoPath" }
     if ($serviceName -ne $existingItem.fields["Custom.ServiceName"]) { $changedField = "Custom.ServiceName" }
@@ -884,6 +885,25 @@ function UpdatePackageVersions($pkgWorkItem, $plannedVersions, $shippedVersions)
   "value": "$shippedPackages"
 }
 "@
+
+    # If we shipped a version after we set "In Release" state then reset the state to "Next Release Unknown"
+    if ($pkgWorkItem.fields["System.State"] -eq "In Release")
+    {
+      $lastShippedDate = [DateTime]$newShippedVersions[0].Date
+      $markedInReleaseDate = ([DateTime]$pkgWorkItem.fields["Microsoft.VSTS.Common.StateChangeDate"])
+
+      # We just shipped so lets set the state to "Next Release Unknown"
+      if ($markedInReleaseDate -le $lastShippedDate)
+      {
+        $fieldUpdates += @'
+{
+  "op": "replace",
+  "path": "/fields/State",
+  "value": "Next Release Unknown"
+}
+'@
+      }
+    }
   }
 
   # Full merged version set

eng/common/scripts/Update-DevOps-Release-WorkItem.ps1

@@ -33,6 +33,10 @@ az extension show -n azure-devops *> $null
 if (!$?){
   Write-Host 'Installing azure-devops extension'
   az extension add --name azure-devops
+} else {
+  # Force update the extension to the latest version if it was already installed
+  # this is needed to ensure we have the authentication issue fixed from earlier versions
+  az extension update -n azure-devops *> $null
 }
 
 . (Join-Path $PSScriptRoot SemVer.ps1)

eng/common/testproxy/target_version.txt

@@ -1 +1 @@
-1.0.0-dev.20220810.2
+1.0.0-dev.20220921.2

eng/common/testproxy/test-proxy-docker.yml

@@ -2,24 +2,30 @@ parameters:
   rootFolder: '$(Build.SourcesDirectory)'
   targetVersion: ''
   templateRoot: '$(Build.SourcesDirectory)'
+  condition: true
 
 steps:
   - pwsh: |
         ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1
     displayName: 'Language Specific Certificate Trust'
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       docker info
     displayName: 'Dump active docker information'
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       ${{ parameters.templateRoot }}/eng/common/testproxy/docker-start-proxy.ps1 -Mode start -TargetFolder "${{ parameters.rootFolder }}" -VersionOverride="${{ parameters.targetVersion }}"
     displayName: 'Run the docker container'
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       docker container ls -a
     displayName: Check running container
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true"
     displayName: 'Set PROXY_MANUAL_START'
+    condition: and(succeeded(), ${{ parameters.condition }})

eng/common/testproxy/test-proxy-tool.yml

@@ -3,11 +3,13 @@ parameters:
   runProxy: true
   targetVersion: ''
   templateRoot: '$(Build.SourcesDirectory)'
+  condition: true
 
 steps:
   - pwsh: |
         ${{ parameters.templateRoot }}/eng/common/scripts/trust-proxy-certificate.ps1
     displayName: 'Language Specific Certificate Trust'
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       $version = $(Get-Content "${{ parameters.templateRoot }}/eng/common/testproxy/target_version.txt" -Raw).Trim()
@@ -23,6 +25,7 @@ steps:
         --add-source https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-net/nuget/v3/index.json `
         --version $version
     displayName: "Install test-proxy"
+    condition: and(succeeded(), ${{ parameters.condition }})
 
   - pwsh: |
       Write-Host "##vso[task.prependpath]$(Build.BinariesDirectory)/test-proxy"
@@ -34,19 +37,20 @@ steps:
         Write-Host "##vso[task.setvariable variable=ASPNETCORE_Kestrel__Certificates__Default__Password]password"
         Write-Host "##vso[task.setvariable variable=PROXY_MANUAL_START]true"
       displayName: 'Configure Kestrel and PROXY_MANUAL_START Variables'
+      condition: and(succeeded(), ${{ parameters.condition }})
 
     - pwsh: |
         Start-Process $(Build.BinariesDirectory)/test-proxy/test-proxy.exe `
           -ArgumentList "--storage-location ${{ parameters.rootFolder }}" `
           -NoNewWindow -PassThru -RedirectStandardOutput ${{ parameters.templateRoot }}/test-proxy.log
       displayName: 'Run the testproxy - windows'
-      condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'))
+      condition: and(succeeded(), eq(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }})
 
     # nohup does NOT continue beyond the current session if you use it within powershell
     - bash: |
         nohup $(Build.BinariesDirectory)/test-proxy/test-proxy > ${{ parameters.templateRoot }}/test-proxy.log &
       displayName: "Run the testproxy - linux/mac"
-      condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'))
+      condition: and(succeeded(), ne(variables['Agent.OS'],'Windows_NT'), ${{ parameters.condition }})
       workingDirectory: "${{ parameters.rootFolder }}"
 
     - pwsh: |
@@ -62,4 +66,4 @@ steps:
         Write-Error "Could not connect to test proxy."
         exit 1
       displayName: Test Proxy IsAlive
-
+      condition: and(succeeded(), ${{ parameters.condition }})

eng/pipelines/templates/stages/archetype-cpp-release.yml

@@ -144,6 +144,7 @@ stages:
                         parameters:
                           RepoOwner: Azure
                           RepoName: azure-sdk-vcpkg-betas
+                          SetVcpkgRoot: false
 
                       - template: /eng/pipelines/templates/steps/vcpkg-publish.yml
                         parameters:

eng/pipelines/templates/steps/vcpkg-clone.yml

@@ -3,6 +3,7 @@ parameters:
   RepoOwner: not-set
   RepoName: vcpkg
   PRBranchName: not-set
+  SetVcpkgRoot: true
 
 steps:
   - pwsh: |
@@ -14,6 +15,13 @@ steps:
       }
     displayName: Clone vcpkg (${{ parameters.RepoOwner }}/${{ parameters.RepoName }})
 
+  - ${{ if eq(parameters.SetVcpkgRoot, true) }}:
+    - pwsh: |
+        $vcpkgRoot = Resolve-Path "${{ parameters.Workspace }}/${{ parameters.RepoName }}"
+        Write-Host "Set VCPKG_ROOT: $vcpkgRoot"
+        Write-Host "##vso[task.setvariable variable=VCPKG_ROOT]$vcpkgRoot"
+      displayName: Set VCPKG_ROOT
+
   # Check out the PR branch if it's already in remote. Ignore failures.
   - pwsh: |
       $ErrorActionPreference = "Continue"

sdk/attestation/azure-security-attestation/CHANGELOG.md

@@ -6,6 +6,11 @@
 
 ### Breaking Changes
 
+- Changed `AttestationClient::AttestTpm` to match `AttestOpenEnclave` and `AttestSgxEnclave` 
+ - Added `std::vector<uint8_t>` dataToAttest parameter.
+ - Removed `PayLoad` in `TpmAttestationOptions`
+ - Changed `TpmResult` in `TpmAttestationResult` to type `std::vector<uint8_t>`
+
 ### Bugs Fixed
 
 ### Other Changes

sdk/attestation/azure-security-attestation/cgmanifest.json

@@ -1,4 +1,5 @@
 {
+  "$schema": "https://json.schemastore.org/component-detection-manifest.json",
   "Registrations": [
     {
       "Component": {

sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client.hpp

@@ -223,26 +223,20 @@ namespace Azure { namespace Security { namespace Attestation {
         Azure::Core::Context const& context = Azure::Core::Context{}) const;
 
     /**
-     * @brief Perform a single leg
-     *
-     * Processes attestation evidence from a VBS enclave, producing an attestation result.
-     *
+     * @brief Sends TPM-based attestation data to the service.
      * The TPM attestation protocol is defined
      * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol')
      *
-     * Unlike OpenEnclave reports and SGX enclave quotes, TPM attestation is implemented using
-     * JSON encoded strings.
      *
-     * The client formats a string serialized JSON request to the
-     * service, which responds with a JSON response. The serialized JSON object exchange continues
-     * until the service responds with a JSON string with a property named {@code "report"}, whose
-     * value will be an attestation result token.
+     * @param dataToAttest - Attestation request data.
+     * @param options - Options to the attestation request.
+     * @param context - Context for the operation.
      *
-     * @param options sent to the service for Trusted Platform Module (TPM) attestation.
-     * @return attestation response for Trusted Platform Module (TPM) attestation.
+     * @return Response<TpmAttestationResult> - The result of the attestation operation
      */
     Response<Models::TpmAttestationResult> AttestTpm(
-        AttestTpmOptions const& options,
+        std::vector<uint8_t> const& dataToAttest,
+        AttestTpmOptions const& options = AttestTpmOptions{},
         Azure::Core::Context const& context = Azure::Core::Context{}) const;
 
   private:

sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_models.hpp

@@ -448,12 +448,13 @@ namespace Azure { namespace Security { namespace Attestation { namespace Models
    */
   struct TpmAttestationResult final
   {
-    /** @brief The JSON encoded value returned from TPM attestation.
+    /** @brief Attestation response data.
+     *
      * The TPM attestation protocol is defined
      * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol')
      *
      */
-    std::string TpmResult;
+    std::vector<uint8_t> TpmResult;
   };
 
   /**

sdk/attestation/azure-security-attestation/inc/azure/attestation/attestation_client_options.hpp

@@ -261,12 +261,6 @@ namespace Azure { namespace Security { namespace Attestation {
    */
   struct AttestTpmOptions final
   {
-    /**
-     * @brief JSON Data to send to the attestation service for TPM attestation.
-     * @details The TPM attestation protocol is defined
-     * [here](https://docs.microsoft.com/azure/attestation/virtualization-based-security-protocol')
-     */
-    std::string Payload;
   };
 
   /** @brief The AttestationSigningKey represents a tuple of asymmetric private cryptographic key

sdk/attestation/azure-security-attestation/src/attestation_client.cpp

@@ -196,13 +196,14 @@ Azure::Response<AttestationToken<AttestationResult>> AttestationClient::AttestOp
 }
 
 Azure::Response<TpmAttestationResult> AttestationClient::AttestTpm(
-    AttestTpmOptions const& attestTpmOptions,
+    std::vector<uint8_t> const& dataToAttest,
+    AttestTpmOptions const&,
     Azure::Core::Context const& context) const
 {
   auto tracingContext(m_tracingFactory.CreateTracingContext("AttestTpm", context));
   try
   {
-    std::string jsonToSend = TpmDataSerializer::Serialize(attestTpmOptions.Payload);
+    std::string jsonToSend = TpmDataSerializer::Serialize(dataToAttest);
     auto encodedVector = std::vector<uint8_t>(jsonToSend.begin(), jsonToSend.end());
     Azure::Core::IO::MemoryBodyStream stream(encodedVector);
 
@@ -212,7 +213,7 @@ Azure::Response<TpmAttestationResult> AttestationClient::AttestTpm(
     // Send the request to the service.
     auto response
         = AttestationCommonRequest::SendRequest(*m_pipeline, request, tracingContext.Context);
-    std::string returnedBody(TpmDataSerializer::Deserialize(response));
+    std::vector<uint8_t> returnedBody{TpmDataSerializer::Deserialize(response)};
     return Response<TpmAttestationResult>(TpmAttestationResult{returnedBody}, std::move(response));
   }
   catch (std::runtime_error const& ex)

sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.cpp

@@ -397,21 +397,21 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail
         returnValue.CertificateThumbprint, jsonResult, "x-ms-certificate-thumbprint");
     return returnValue;
   }
-  std::string TpmDataSerializer::Serialize(std::string const& tpmData)
+  std::string TpmDataSerializer::Serialize(std::vector<uint8_t> const& tpmData)
   {
     Azure::Core::Json::_internal::json jsonData;
-    jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode(
-        std::vector<uint8_t>(tpmData.begin(), tpmData.end()));
+    jsonData["data"] = Azure::Core::_internal::Base64Url::Base64UrlEncode(tpmData);
     return jsonData.dump();
   }
-  std::string TpmDataSerializer::Deserialize(Azure::Core::Json::_internal::json const& jsonData)
+  std::vector<uint8_t> TpmDataSerializer::Deserialize(
+      Azure::Core::Json::_internal::json const& jsonData)
   {
     std::vector<uint8_t> returnValue;
     JsonOptional::SetIfExists<std::string, std::vector<uint8_t>>(
         returnValue, jsonData, "data", Azure::Core::_internal::Base64Url::Base64UrlDecode);
-    return std::string(returnValue.begin(), returnValue.end());
+    return returnValue;
   }
-  std::string TpmDataSerializer::Deserialize(
+  std::vector<uint8_t> TpmDataSerializer::Deserialize(
       std::unique_ptr<Azure::Core::Http::RawResponse> const& response)
   {
     return TpmDataSerializer::Deserialize(

sdk/attestation/azure-security-attestation/src/private/attestation_deserializers_private.hpp

@@ -136,9 +136,10 @@ namespace Azure { namespace Security { namespace Attestation { namespace _detail
 
   struct TpmDataSerializer
   {
-    static std::string Serialize(std::string const& tpmData);
-    static std::string Deserialize(Azure::Core::Json::_internal::json const& jsonData);
-    static std::string Deserialize(std::unique_ptr<Azure::Core::Http::RawResponse> const& response);
+    static std::string Serialize(std::vector<uint8_t> const& tpmData);
+    static std::vector<uint8_t> Deserialize(Azure::Core::Json::_internal::json const& jsonData);
+    static std::vector<uint8_t> Deserialize(
+        std::unique_ptr<Azure::Core::Http::RawResponse> const& response);
   };
 
 }}}} // namespace Azure::Security::Attestation::_detail