Run service samples on CI gate with ubuntu (#3105)

* samples on ubuntu-default

eng/pipelines/templates/jobs/archetype-sdk-tests.yml

@@ -62,6 +62,15 @@ jobs:
         CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON -DCMAKE_BUILD_TYPE=Release'
         AZURE_CORE_ENABLE_JSON_TESTS: 1
         BuildArgs: '-j 4'
+      Ubuntu20_samples:
+        Pool: azsdk-pool-mms-ubuntu-1804-general
+        OSVmImage: MMSUbuntu18.04
+        VcpkgInstall: 'curl[ssl] libxml2 openssl'
+        VCPKG_DEFAULT_TRIPLET: 'x64-linux'
+        # Builds samples and run them against the deployed resources. Samples requires Azure Account, so only works on live tests.
+        CmakeArgs: ' -DBUILD_TESTING=ON -DBUILD_SAMPLES=ON -DBUILD_STORAGE_SAMPLES=ON'
+        BuildArgs: '-j 4'
+        RunSamples: 1
       Ubuntu20_x64_no_rtti:
         Pool: azsdk-pool-mms-ubuntu-1804-general
         OSVmImage: MMSUbuntu18.04
@@ -201,6 +210,11 @@ jobs:
   - script: ctest -C Debug --tests-regex ${{ parameters.CtestRegex }} --no-compress-output -T Test
     workingDirectory: build
     displayName: ctest
+    # Runs only if test-resources are happly deployed.
+    # unit-tests runs for those configs where samples are not ran.
+    # This enables to run tests and samples at the same time as different matrix configuration.
+    # Then unit-tests runs, samples should not run.
+    condition: and(succeeded(), ne(variables['RunSamples'], '1'))
 
   - task: PublishTestResults@2
     inputs:
@@ -211,7 +225,32 @@ jobs:
       mergeTestResults: true
       publishRunAttachments: true
     displayName: Publish test results
-    condition: succeededOrFailed()
+    # this step only makes sense when ctest has run
+    condition: and(succeededOrFailed(), ne(variables['RunSamples'], '1'))
+
+    # Running Samples step.
+    # Will run samples described on a file name [service]-samples.txt within the build directory.
+    # For example keyvault-samples.txt.
+    # The file is written by CMake during configuration when building samples.
+  - bash: |
+      IFS=$'\n'
+      if [[ -f "./${{ parameters.ServiceDirectory }}-samples.txt" ]]; then
+        for sample in `cat ./${{ parameters.ServiceDirectory }}-samples.txt`
+        do
+          echo "**********Running sample: ${sample}"
+          bash -c "$sample"
+          status=$?
+          if [[ $status -eq 0 ]]; then
+            echo "*********Sample completed*********"
+          else
+            echo "*Sample returned a failed code: $status"
+            exit 1
+          fi
+        done
+      fi
+    workingDirectory: build
+    displayName: "Run Samples for : ${{ parameters.ServiceDirectory }}"
+    condition: and(succeeded(), eq(variables['RunSamples'], '1'))
 
   # Make coverage targets (specified in coverage_targets.txt) and assemble
   # coverage report
@@ -220,7 +259,7 @@ jobs:
       ../tools/reportgenerator "-reports:${{ parameters.CoverageReportPath }}" "-targetdir:." "-reporttypes:Cobertura"
     workingDirectory: build
     displayName: Generate Code Coverage Data
-    condition: and(succeededOrFailed(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], ''))
+    condition: and(succeeded(), ne(variables['CODE_COVERAGE'], 'disabled'), ne(variables['CODE_COVERAGE'], ''))
   
   - task: PublishCodeCoverageResults@1
     inputs:

eng/pipelines/templates/stages/platform-matrix.json

@@ -8,7 +8,7 @@
                 "macOS-10.14": {
                     "OSVmImage": "macOS-10.14"
                 },
-                "macOS-10.15": { 
+                "macOS-10.15": {
                     "OSVmImage": "macOS-10.15"
                 },
                 "macOS-11": {
@@ -131,10 +131,13 @@
                     "CODE_COVERAGE_COLLECT_ONLY": "1",
                     "CODE_COVERAGE": "enabled"
                 },
-                "included": { },
+                "included": {},
                 "included_release": {
                     "CMAKE_BUILD_TYPE": "Release",
                     "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_PERFORMANCE_TESTS=ON -DRUN_LONG_UNIT_TESTS=ON"
+                },
+                "included_samples": {
+                    "CmakeArgs": " -DBUILD_TESTING=ON -DBUILD_SAMPLES=ON -DBUILD_STORAGE_SAMPLES=ON"
                 }
             }
         },
@@ -157,7 +160,6 @@
                 },
                 "included_release": {
                     "CMAKE_BUILD_TYPE": "Release"
-
                 }
             }
         }

sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-basic-operations/CMakeLists.txt

@@ -13,3 +13,7 @@ add_executable (
 )
 
 target_link_libraries(certificate-basic-operations PRIVATE azure-security-keyvault-certificates azure-identity )
+
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/certificate-basic-operations\n")

sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/CMakeLists.txt

@@ -13,3 +13,7 @@ add_executable (
 )
 
 target_link_libraries(certificate-get-certificates PRIVATE azure-security-keyvault-certificates azure-identity )
+
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/certificate-get-certificates\n")

sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-get-certificates/certificate_get_certificates.cpp

@@ -46,8 +46,8 @@ int main()
 
   try
   {
-    std::string certificateName1 = "Sample1";
-    std::string certificateName2 = "Sample2";
+    std::string certificateName1 = "SampleCertificate1";
+    std::string certificateName2 = "SampleCertificate2";
     KeyVaultCertificateWithPolicy certificate1;
     KeyVaultCertificateWithPolicy certificate2;
     // create and get two certificates

sdk/keyvault/azure-security-keyvault-certificates/test/samples/certificate-import-certificate/CMakeLists.txt

@@ -13,3 +13,7 @@ add_executable (
 )
 
 target_link_libraries(certificate-import-certificate PRIVATE azure-security-keyvault-certificates azure-identity )
+
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/certificate-import-certificate\n")

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/CMakeLists.txt

@@ -12,4 +12,8 @@ add_executable (
   sample1_hello_world.cpp
 )
 
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/sample1-hello-world\n")
+
 target_link_libraries(sample1-hello-world PRIVATE azure-security-keyvault-keys azure-identity)

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample1-hello-world/sample1_hello_world.cpp

@@ -30,13 +30,14 @@ using namespace Azure::Security::KeyVault::Keys;
 
 int main()
 {
-  auto tenantId = std::getenv("AZURE_TENANT_ID");
-  auto clientId = std::getenv("AZURE_CLIENT_ID");
-  auto clientSecret = std::getenv("AZURE_CLIENT_SECRET");
+  auto const tenantId = std::string(std::getenv("AZURE_TENANT_ID"));
+  auto const clientId = std::string(std::getenv("AZURE_CLIENT_ID"));
+  auto const clientSecret = std::string(std::getenv("AZURE_CLIENT_SECRET"));
+  auto const keyVaultUrl = std::string(std::getenv("AZURE_KEYVAULT_URL"));
   auto credential
       = std::make_shared<Azure::Identity::ClientSecretCredential>(tenantId, clientId, clientSecret);
 
-  KeyClient keyClient(std::getenv("AZURE_KEYVAULT_URL"), credential);
+  KeyClient keyClient(keyVaultUrl, credential);
 
   std::string rsaKeyName("CloudRsaKey" + Azure::Core::Uuid::CreateUuid().ToString());
   try

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample2-backup-and-restore/CMakeLists.txt

@@ -12,4 +12,8 @@ add_executable (
   sample2_backup_and_restore.cpp
 )
 
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/sample2-backup-and-restore\n")
+
 target_link_libraries(sample2-backup-and-restore PRIVATE azure-security-keyvault-keys azure-identity)

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/CMakeLists.txt

@@ -12,4 +12,8 @@ add_executable (
   sample3_get_keys.cpp
 )
 
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/sample3-get-keys\n")
+
 target_link_libraries(sample3-get-keys PRIVATE azure-security-keyvault-keys azure-identity)

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample3-get-keys/sample3_get_keys.cpp

@@ -39,14 +39,7 @@ int main()
       = std::make_shared<Azure::Identity::ClientSecretCredential>(tenantId, clientId, clientSecret);
 
   KeyClient keyClient(std::getenv("AZURE_KEYVAULT_URL"), credential);
-  try
-  {
-    KeyVaultKey key = keyClient.GetKey("some_key").Value;
-  }
-  catch (const Azure::Core::RequestFailedException& ex)
-  {
-    std::cout << std::underlying_type<Azure::Core::Http::HttpStatusCode>::type(ex.StatusCode);
-  }
+
   try
   {
     std::string rsaKeyName("CloudRsaKey-" + Azure::Core::Uuid::CreateUuid().ToString());

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample4-encrypt-decrypt/CMakeLists.txt

@@ -12,4 +12,8 @@ add_executable (
   sample4_encrypt_decrypt.cpp
 )
 
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/sample4-encrypt-decrypt\n")
+
 target_link_libraries(sample4-encrypt-decrypt PRIVATE azure-security-keyvault-keys azure-identity)

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/CMakeLists.txt

@@ -12,4 +12,8 @@ add_executable (
   sample5_sign_verify.cpp
 )
 
+# Add the sample to be run on CI. 
+# CI pipeline reads the {service}-samples.txt and runs the binaries listed there.
+file(APPEND ${CMAKE_BINARY_DIR}/keyvault-samples.txt "${CMAKE_CURRENT_BINARY_DIR}/sample5-sign-verify\n")
+
 target_link_libraries(sample5-sign-verify PRIVATE azure-security-keyvault-keys azure-identity)

sdk/keyvault/azure-security-keyvault-keys/test/samples/sample5-sign-verify/sample5_sign_verify.cpp

@@ -31,6 +31,7 @@ using namespace std::chrono_literals;
 
 int main()
 {
+
   auto tenantId = std::getenv("AZURE_TENANT_ID");
   auto clientId = std::getenv("AZURE_CLIENT_ID");
   auto clientSecret = std::getenv("AZURE_CLIENT_SECRET");
@@ -47,80 +48,92 @@ int main()
   auto ecKeyOptions = CreateEcKeyOptions(ecKeyName, false);
   ecKeyOptions.CurveName = KeyCurveName::P256K;
 
-  KeyVaultKey cloudRsaKey = keyClient.CreateRsaKey(keyOptions).Value;
-  std::cout << " - Key is returned with name " << cloudRsaKey.Name() << " and type "
-            << cloudRsaKey.GetKeyType().ToString() << std::endl;
-
-  KeyVaultKey cloudEcKey = keyClient.CreateEcKey(ecKeyOptions).Value;
-  std::cout << " - Key is returned with name " << cloudEcKey.Name() << " and type "
-            << cloudEcKey.GetKeyType().ToString() << std::endl;
-
-  CryptographyClient rsaCryptoClient(cloudRsaKey.Id(), credential);
-
-  CryptographyClient ecCryptoClient(cloudEcKey.Id(), credential);
-
-  uint8_t const dataSource[]
-      = "This is some sample data which we will use to demonstrate sign and verify";
-  std::vector<uint8_t> data(std::begin(dataSource), std::end(dataSource));
-
-  // digestRaw simulates some text data that has been hashed using the SHA256 algorithm
-  // and then base 64 encoded. It is not relevant for the sample how to create the SHA256
-  // hashed digest.
-  std::vector<uint8_t> digest
-      // cspell: disable-next-line
-      = Azure::Core::Convert::Base64Decode("DU9EdhpwhJqnGnieD0qKYEz6e8QPKlOVpYZZro");
-
-  // Sign and Verify from digest
-  SignResult rsaSignResult = rsaCryptoClient.Sign(SignatureAlgorithm::RS256, digest).Value;
-  std::cout << " - Signed digest using the algorithm " << rsaSignResult.Algorithm.ToString()
-            << ", with key " << rsaSignResult.KeyId << ". The resulting signature is: "
-            << Azure::Core::Convert::Base64Encode(rsaSignResult.Signature) << std::endl;
-
-  SignResult ecSignResult = ecCryptoClient.Sign(SignatureAlgorithm::ES256K, digest).Value;
-  std::cout << " - Signed digest using the algorithm " << ecSignResult.Algorithm.ToString()
-            << ", with key " << ecSignResult.KeyId << ". The resulting signature is: "
-            << Azure::Core::Convert::Base64Encode(ecSignResult.Signature) << std::endl;
-
-  VerifyResult rsaVerifyResult
-      = rsaCryptoClient.Verify(SignatureAlgorithm::RS256, digest, rsaSignResult.Signature).Value;
-  std::cout << " - Verified the signature using the algorithm "
-            << rsaVerifyResult.Algorithm.ToString() << ", with key " << rsaVerifyResult.KeyId
-            << ". Signature is valid: " << (rsaVerifyResult.IsValid ? "True" : "False")
-            << std::endl;
-
-  VerifyResult ecVerifyResult
-      = ecCryptoClient.Verify(SignatureAlgorithm::ES256K, digest, ecSignResult.Signature).Value;
-  std::cout << " - Verified the signature using the algorithm "
-            << ecVerifyResult.Algorithm.ToString() << ", with key " << ecVerifyResult.KeyId
-            << ". Signature is valid: " << (ecVerifyResult.IsValid ? "True" : "False") << std::endl;
-
-  // Sign and Verify from data
-  SignResult rsaSignDataResult = rsaCryptoClient.SignData(SignatureAlgorithm::RS256, data).Value;
-  std::cout << " - Signed data using the algorithm " << rsaSignDataResult.Algorithm.ToString()
-            << ", with key " << rsaSignDataResult.KeyId << ". The resulting signature is: "
-            << Azure::Core::Convert::Base64Encode(rsaSignDataResult.Signature) << std::endl;
-
-  SignResult ecSignDataResult = ecCryptoClient.SignData(SignatureAlgorithm::ES256K, data).Value;
-  std::cout << " - Signed data using the algorithm " << ecSignDataResult.Algorithm.ToString()
-            << ", with key " << ecSignDataResult.KeyId << ". The resulting signature is: "
-            << Azure::Core::Convert::Base64Encode(ecSignDataResult.Signature) << std::endl;
-
-  VerifyResult rsaVerifyDataResult
-      = rsaCryptoClient.VerifyData(SignatureAlgorithm::RS256, data, rsaSignDataResult.Signature)
-            .Value;
-  std::cout << " - Verified the signature using the algorithm "
-            << rsaVerifyDataResult.Algorithm.ToString() << ", with key "
-            << rsaVerifyDataResult.KeyId
-            << ". Signature is valid: " << (rsaVerifyDataResult.IsValid ? "True" : "False")
-            << std::endl;
-
-  VerifyResult ecVerifyDataResult
-      = ecCryptoClient.VerifyData(SignatureAlgorithm::ES256K, data, ecSignDataResult.Signature)
-            .Value;
-  std::cout << " - Verified the signature using the algorithm "
-            << ecVerifyDataResult.Algorithm.ToString() << ", with key " << ecVerifyDataResult.KeyId
-            << ". Signature is valid: " << (ecVerifyDataResult.IsValid ? "True" : "False")
-            << std::endl;
+  auto returnValue = 0;
+  try
+  {
+    KeyVaultKey cloudRsaKey = keyClient.CreateRsaKey(keyOptions).Value;
+    std::cout << " - Key is returned with name " << cloudRsaKey.Name() << " and type "
+              << cloudRsaKey.GetKeyType().ToString() << std::endl;
+
+    KeyVaultKey cloudEcKey = keyClient.CreateEcKey(ecKeyOptions).Value;
+    std::cout << " - Key is returned with name " << cloudEcKey.Name() << " and type "
+              << cloudEcKey.GetKeyType().ToString() << std::endl;
+
+    CryptographyClient rsaCryptoClient(cloudRsaKey.Id(), credential);
+
+    CryptographyClient ecCryptoClient(cloudEcKey.Id(), credential);
+
+    uint8_t const dataSource[]
+        = "This is some sample data which we will use to demonstrate sign and verify";
+    std::vector<uint8_t> data(std::begin(dataSource), std::end(dataSource));
+
+    // digestRaw simulates some text data that has been hashed using the SHA256 algorithm
+    // and then base 64 encoded. It is not relevant for the sample how to create the SHA256
+    // hashed digest.
+    std::vector<uint8_t> digest
+        // cspell: disable-next-line
+        = Azure::Core::Convert::Base64Decode("RUE3Nzg4NTQ4QjQ5RjFFN0U2NzAyQzhDNEMwMkJDOTA=");
+
+    // Sign and Verify from digest
+    SignResult rsaSignResult = rsaCryptoClient.Sign(SignatureAlgorithm::RS256, digest).Value;
+    std::cout << " - Signed digest using the algorithm " << rsaSignResult.Algorithm.ToString()
+              << ", with key " << rsaSignResult.KeyId << ". The resulting signature is: "
+              << Azure::Core::Convert::Base64Encode(rsaSignResult.Signature) << std::endl;
+
+    SignResult ecSignResult = ecCryptoClient.Sign(SignatureAlgorithm::ES256K, digest).Value;
+    std::cout << " - Signed digest using the algorithm " << ecSignResult.Algorithm.ToString()
+              << ", with key " << ecSignResult.KeyId << ". The resulting signature is: "
+              << Azure::Core::Convert::Base64Encode(ecSignResult.Signature) << std::endl;
+
+    VerifyResult rsaVerifyResult
+        = rsaCryptoClient.Verify(SignatureAlgorithm::RS256, digest, rsaSignResult.Signature).Value;
+    std::cout << " - Verified the signature using the algorithm "
+              << rsaVerifyResult.Algorithm.ToString() << ", with key " << rsaVerifyResult.KeyId
+              << ". Signature is valid: " << (rsaVerifyResult.IsValid ? "True" : "False")
+              << std::endl;
+
+    VerifyResult ecVerifyResult
+        = ecCryptoClient.Verify(SignatureAlgorithm::ES256K, digest, ecSignResult.Signature).Value;
+    std::cout << " - Verified the signature using the algorithm "
+              << ecVerifyResult.Algorithm.ToString() << ", with key " << ecVerifyResult.KeyId
+              << ". Signature is valid: " << (ecVerifyResult.IsValid ? "True" : "False")
+              << std::endl;
+
+    // Sign and Verify from data
+    SignResult rsaSignDataResult = rsaCryptoClient.SignData(SignatureAlgorithm::RS256, data).Value;
+    std::cout << " - Signed data using the algorithm " << rsaSignDataResult.Algorithm.ToString()
+              << ", with key " << rsaSignDataResult.KeyId << ". The resulting signature is: "
+              << Azure::Core::Convert::Base64Encode(rsaSignDataResult.Signature) << std::endl;
+
+    SignResult ecSignDataResult = ecCryptoClient.SignData(SignatureAlgorithm::ES256K, data).Value;
+    std::cout << " - Signed data using the algorithm " << ecSignDataResult.Algorithm.ToString()
+              << ", with key " << ecSignDataResult.KeyId << ". The resulting signature is: "
+              << Azure::Core::Convert::Base64Encode(ecSignDataResult.Signature) << std::endl;
+
+    VerifyResult rsaVerifyDataResult
+        = rsaCryptoClient.VerifyData(SignatureAlgorithm::RS256, data, rsaSignDataResult.Signature)
+              .Value;
+    std::cout << " - Verified the signature using the algorithm "
+              << rsaVerifyDataResult.Algorithm.ToString() << ", with key "
+              << rsaVerifyDataResult.KeyId
+              << ". Signature is valid: " << (rsaVerifyDataResult.IsValid ? "True" : "False")
+              << std::endl;
+
+    VerifyResult ecVerifyDataResult
+        = ecCryptoClient.VerifyData(SignatureAlgorithm::ES256K, data, ecSignDataResult.Signature)
+              .Value;
+    std::cout << " - Verified the signature using the algorithm "
+              << ecVerifyDataResult.Algorithm.ToString() << ", with key "
+              << ecVerifyDataResult.KeyId
+              << ". Signature is valid: " << (ecVerifyDataResult.IsValid ? "True" : "False")
+              << std::endl;
+  }
+  catch (Azure::Core::RequestFailedException const& e)
+  {
+    auto const b = e.RawResponse->GetBody();
+    std::cout << "Error: " + std::string(b.begin(), b.end());
+    returnValue = 1;
+  }
 
   // Delete the key
   auto deleteOperation = keyClient.StartDeleteKey(rsaKeyName);
@@ -129,4 +142,6 @@ int main()
   ecDeleteOperation.PollUntilDone(2min);
   keyClient.PurgeDeletedKey(rsaKeyName);
   keyClient.PurgeDeletedKey(ecKeyName);
+
+  return returnValue;
 }