Merge branch 'main' into users/prebansa/servicenowv2exp

.github/PULL_REQUEST_TEMPLATE/control_plane_template.md

@@ -10,7 +10,7 @@
 
 Please understand this diagram before proceeding. It explains how to get your PR approved & merged.
 
-![diagram](https://github.com/Azure/azure-rest-api-specs/assets/4429827/f5cdc159-40bd-42a5-8a1c-1e398a1d80fd)
+![diagram](https://github.com/Azure/azure-rest-api-specs/assets/20296335/9e51cff7-28f7-4ae7-99a3-0f4a116c759f)
 
 <details>
 <summary>Click here to see the details of Step 1, Breaking Changes review</summary>

.github/comment.yml

@@ -9,14 +9,10 @@
     label: Approved-OkToMerge
     onLabeledComments: >-
       Hi @${PRAuthor}! Your PR is approved. Congratulations. :partying_face::rocket: </br>
-      <li>
       If this PR is targeting `main` branch, then it cannot be merged, as `azure-rest-api-specs-pr` repo `main` branch 
-      is mirrored from `azure-rest-api-specs` `main`` branch. <br/>
+      is mirrored from `azure-rest-api-specs` `main` branch.<br/>
       If you want to publish the PR to the public repo (`Azure/azure-rest-api-specs`) `main` branch,
-      please use [OpenAPIHub Publish PR](${openapiHub}/tools/publishpullrequest?pr=${owner}/${repo}/${PRNumber}&to=${to}).
-      </li><li>
-      For further guidance see [Spec Repos](https://eng.ms/docs/products/azure-developer-experience/design/api-specs-pr/api-repos).
-      </li>
+      see [aka.ms/azsdk/move-pr](https://aka.ms/azsdk/move-pr).
 
 - rule:
     type: label

cSpell.json

@@ -841,6 +841,13 @@
             ]
         },
 
+        {
+            "filename": "**/specification/maps/data-plane/Weather/stable/1.1/weather.json",
+            "words": [
+                "locationally"
+            ]
+        },
+
         {
             "filename": "**/specification/maps/data-plane/Microsoft.Maps/Render/preview/1.0/render.json",
             "words": [
@@ -880,9 +887,11 @@
             ]
         },
         {
-            "filename": "**/specification/communication/data-plane/Chat/preview/2020-09-21-preview2/communicationserviceschat.json",
+            "filename": "**/specification/communication/data-plane/Chat/**/communicationserviceschat.json",
             "words": [
-                "readreceipts"
+                "readreceipts",
+                "heic",
+                "webp"
             ]
         },
         {
@@ -1167,6 +1176,15 @@
                 "wihtin"
             ]
         },
+        {
+           "filename": "**/specification/confidentialledger/data-plane/Microsoft.CodeTransparency/preview/**/*.json",
+           "words": [
+                "cbor",
+                "txids",
+                "operationid",
+                "scitt"
+            ]
+        },
         {
             "filename": "**/specification/applicationinsights/data-plane/LiveMetrics/preview/2024-04-01-preview/livemetrics.json",
             "words": [
@@ -1177,6 +1195,13 @@
                 "Comparand",
                 "SDK"
             ]
+        },
+        {
+            "filename": "**/specification/cognitiveservices/data-plane/ContentSafety/**/*.json",
+            "words": [
+                "Groundedness",
+                "ungroundedness"
+            ]
         }
     ],
     "enableFiletypes": [

custom-words.txt

@@ -3964,6 +3964,11 @@ personalvoice
 personalvoices
 basemodels
 Multistyle
+batchsynthesis
+batchsyntheses
+fadein
+aoai
+AOAI
 vectorizers
 ONTAP
 SVM

documentation/ci-fix.md

@@ -26,10 +26,12 @@
   - [`Swagger ApiDocPreview`](#swagger-apidocpreview)
   - [`TypeSpec Validation`](#typespec-validation)
     - [Run `tsv` locally](#run-tsv-locally)
+  - [APIView Failures: troubleshooting guides](#apiview-failures-troubleshooting-guides)
   - [Suppression Process](#suppression-process)
   - [Checks not covered by this guide](#checks-not-covered-by-this-guide)
   - [Obsolete checks](#obsolete-checks)
 
+
 # CI Fix Guide
 
 Short link: https://aka.ms/azsdk/ci-fix
@@ -124,20 +126,11 @@ Refer to [Semantic and Model Violations Reference](https://github.com/Azure/azur
 
 ## `Swagger BreakingChange` and `BreakingChange(Cross-Version)`
 
-- An API contract is identified by its api-version value. Once published, no changes to this API contract are allowed. This applies regardless of whether the API contract is for private preview, public preview, or GA (stable).
-    - The same-version breaking change linter rules check for changes to an existing api-version OpenAPI spec.
-        - When introducing a new API contract (preview or not), the new API contract must be backwards compatible with the previous GA’s API contract.
-            - However, during a (private or public) preview cycle, a new preview API contract does not have to be backwards compatible with the previous preview API contract although it must still be backwards compatible with the latest GA API contract.
-            - The cross version breaking change linter rules checks for this by comparing the new OpenAPI spec with the latest GA OpenAPI spec. If there is no latest GA OpenAPI spec, then the latest preview if it > 1 year old. If nether a GA or preview > 1 year old exists, then the OpenAPI spec is considered good.
-
-### Adding label on PR automatically
-
-The breaking change check has two types of violations: one is breaking change in the same version but not breaking change in a new version, the other is breaking change even in a new version.
-For the former, a label 'NewApiVersionRequired' will be added automatically; For the latter, a label 'BreakingChangeReviewRequired' will be added automatically. Adding each label will trigger a github comment with guldance on how to fix.
+See [aka.ms/azsdk/pr-brch-deep](https://aka.ms/azsdk/pr-brch-deep). If you want a quick read, see only [the `summary` section](https://aka.ms/azsdk/pr-brch-deep#summary).
 
 ### Run `oad` locally
 
-To repro issues with "breaking changes" checks, you can locally run the tool that powers them [Azure/openapi-diff](https://github.com/Azure/openapi-diff) aka `oad`:
+To repro issues with "breaking changes" checks, you can locally run the tool that powers them: [Azure/openapi-diff](https://github.com/Azure/openapi-diff), aka `oad`:
 ```
 npm install -g @azure/oad
 oad compare <old-spec-path> <new-spec-path>
@@ -262,6 +255,25 @@ that the generated OpenAPI spec files were not in-sync with the TypeSpec project
 
 If none of the above helped, please reach out on [TypeSpec Discussions Teams channel].
 
+## APIView Failures: troubleshooting guides
+Various APIViews are generated as part of the Azure REST API specs PR build. Among these are TypeSpec and Swagger as well as any other language that is being generated in the run. When everything is successful you should see a comment box similar to the picture below showing the APIViews generated for TypeSpec or Swagger, plus all other languages being generated.
+
+![alt text](image-3.png)
+
+#### If an expected APIView was not generated, follow the step below to troubleshoot.
+
+- On the CI check click on `details` > `View Azure DevOps build log for more details` to view the devOps logs.
+- Investigate the CI job for the languge with error. TypeSpec and Swagger APIViews are generated as part of the `AzureRestApiSpecsPipeline` stage in the `TypeSpecAPIView` and `SwaggerAPIView` jobs respectively, while APIViews for other SDK languges are generated in their respective language jobs in the `SDK Automation` stage.
+- Ensure that all previous checks in the job are green before proceeding. 
+
+#### Diagnosing APIView failure for SDK Language (not Swagger or TypeSpec)
+1. Check for an unexpected skip of the `Publish SDK APIView Artifact to Pipeline Artifacts` and `Generate SDK APIView` step.
+2. Look in `SDK Automation` step to verify that the API token generation completed successfully.
+3. Search logs for `Read Temp File`
+4. Below `Read Temp File` find the .json object and search within to locate the `apiViewArtifact` property.
+5. If not present, the APIView parser for the language failed to generate the `APIView Token Artifacts`.
+6. Please contact [APIView Support Teams Channel] for assistance.
+
 ## Suppression Process
 
 In case there are validation errors reported against your service that you believe do not apply, we have a suppression process you can follow to permanently remove these reported errors for your specs. Refer to the [suppression guide](https://aka.ms/pr-suppressions) for detailed guidance.
@@ -285,3 +297,4 @@ Following checks have been removed from the validation toolchain as of August 20
 [aka.ms/azsdk/support/specreview-channel]: https://aka.ms/azsdk/support/specreview-channel
 [aka.ms/azsdk/support]: https://aka.ms/azsdk/support
 [TypeSpec Discussions Teams channel]: https://teams.microsoft.com/l/channel/19%3A906c1efbbec54dc8949ac736633e6bdf%40thread.skype/TypeSpec%20Discussion%20%F0%9F%90%AE?groupId=3e17dcb0-4257-4a30-b843-77f47f1d4121&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47
+[APIView Support Teams Channel]: https://teams.microsoft.com/l/channel/19%3A3adeba4aa1164f1c889e148b1b3e3ddd%40thread.skype/APIView?groupId=3e17dcb0-4257-4a30-b843-77f47f1d4121&tenantId=72f988bf-86f1-41af-91ab-2d7cd011db47

documentation/code-gen/configure-python-sdk.md

@@ -86,7 +86,7 @@ This is not used by Autorest itself.
 
 ``` yaml $(swagger-to-sdk)
 swagger-to-sdk:
-  - repo: azure-sdk-for-python-track2
+  - repo: azure-sdk-for-python
   - ...
 
 

documentation/samplefiles/readme.java.md

@@ -0,0 +1,7 @@
+## Java
+
+These settings apply only when `--java` is specified on the command line.
+
+``` yaml $(java)
+client-flattened-annotation-target: disabled
+```

documentation/samplefiles/samplereadme.md

@@ -49,7 +49,7 @@ This is not used by Autorest itself.
 
 ```yaml $(swagger-to-sdk)
 swagger-to-sdk:
-  - repo: azure-sdk-for-python-track2
+  - repo: azure-sdk-for-python
   - repo: azure-sdk-for-java
   - repo: azure-sdk-for-go
   - repo: azure-sdk-for-js
@@ -76,3 +76,7 @@ See configuration in [readme.typescript.md](./readme.typescript.md)
 ## CSharp
 
 See configuration in [readme.csharp.md](./readme.csharp.md)
+
+## Java
+
+See configuration in [readme.java.md](./readme.java.md)

documentation/sdkautomation/sdk-suppressions.md

@@ -0,0 +1,39 @@
+# SDK Breaking Change Review Workflow
+
+When a specification pull request has breaking change for a specific SDK language, such as `Go`, the pull request will be labelled with "BreakingChange-Go-Sdk". According to [the design principles of Azure SDK](https://azure.github.io/azure-sdk/general_introduction.html#dependable), breaking changes are more harmful to a user’s experience than most new features and improvements are beneficial. Therefore, the pull request author will be strongly encouraged to update the API specification to remove the breaking changes for the SDK. If the breaking change cannot be mitigated by a specification change, the author will need to suppress the breaking changes and have the suppression reviewed. See the following section for details on how to suppress breaking changes.
+
+Once the suppression is added to the pull request, SDK reviewers will evaluate the suppressions and either provide feedback or approve the suppressions. When the suppressions have been approved, the SDK breaking change review is complete and the pull request can proceed to the next stage. 
+
+> Note: both the suppressions review and the SDK breaking change review will be conducted proactively by SDK reviewers, and the expected completion time is around 48 business hours.
+
+## How to Suppress the SDK Breaking Changes
+
+To suppress SDK breaking changes, go to the SDK generation CI check result for a specific SDK, such as "azure-sdk-for-go", and look for the `Absent SDK breaking changes suppressions` section, as shown in the following code block:
+```
+Absent SDK breaking changes suppressions
+- Function `*LinkerClient.BeginCreateOrUpdate` has been removed
+- Function `*LinkerClient.NewListPager` has been removed
+```
+### Next Steps
+
+1. The pull request authors create a file called `sdk-suppressions.yaml` if it doesn't already exist in your service folder in the `azure-rest-api-specs` repository. 
+2. The pull request authors copy every line under the `Absent SDK breaking changes suppressions` section to the suppression file in the following format:
+```
+suppressions:
+  azure-sdk-for-go:
+    - package: 'sdk/resourcemanager/servicelinker/armservicelinker'
+      breaking-changes:
+        - Function `*LinkerClient.BeginCreateOrUpdate` has been removed
+        - Function `*LinkerClient.NewListPager` has been removed
+
+```
+3. The pull request authors add this suppression file change to your spec pull request. Then the suppressions will be recognized by the bot, and the corresponding suppression label will be added to the spec pull request. For example, for the  `azure-sdk-for-go`, the label `BreakingChange-Go-Sdk-Suppression` will be added to the pull request.
+4. The SDK reviewers will evaluate the suppressions and either approve them or provide feedback. If the suppressions are approved, the corresponding approval label will be added to the pull request. For example, for the  `azure-sdk-for-go`, the label `BreakingChange-Go-Sdk-Suppression-Approved` will be added to the pull request.
+
+## How Does the Suppression Work
+
+The suppression file will remain with the specification move with it to other branches or repositories(such as from the private specification repository to the public specification repository). The suppressions can be deleted after the SDK has been released to the public.
+
+## How to Get the Suppression Reviewed
+
+SDK reviewers will monitor the specification pull requests with breaking change suppression lables. For example, for the  `azure-sdk-for-go`, the label is `BreakingChange-Go-Sdk-Suppression`. They will evaluate the suppressions and either approve them or provide feedback. If the suppressions are approved, the corresponding approval label will be added to the pull request. For example, for the  `azure-sdk-for-go`, the label `BreakingChange-Go-Sdk-Suppression-Approved` will be added to the pull request. Once the suppression review is complete, the pull request can proceed to the next stage.

eng/scripts/TypeSpec-Requirement.ps1

@@ -82,6 +82,7 @@ else {
   #   - specification/foo/data-plane/Foo/stable/2023-01-01/Foo.json
   #   - specification/foo/data-plane/Foo/bar/stable/2023-01-01/Foo.json
   #   - specification/foo/resource-manager/Microsoft.Foo/stable/2023-01-01/Foo.json
+  # - Doc: https://github.com/Azure/azure-rest-api-specs/blob/main/README.md#directory-structure
   foreach ($file in $filesToCheck) {
     LogInfo "Checking $file"
 
@@ -98,27 +99,51 @@ else {
 
     try {
       $jsonContent = Get-Content $fullPath | ConvertFrom-Json -AsHashtable
+    }
+    catch {
+      LogWarning "  OpenAPI cannot be parsed as JSON, so assuming not generated from TypeSpec"
+      LogWarning "    $_"
+    }
 
+    if ($jsonContent) {
       if ($null -ne ${jsonContent}?["info"]?["x-typespec-generated"]) {
         LogInfo "  OpenAPI was generated from TypeSpec (contains '/info/x-typespec-generated')"
+
+        if ($file -match "specification/(?<rpFolder>[^/]+)/") {
+          $rpFolder = $Matches["rpFolder"];
+          $tspConfigs = @(Get-ChildItem -Path (Join-Path $repoPath "specification" $rpFolder) -Recurse -File
+            | Where-Object { $_.Name -eq "tspconfig.yaml" })
+
+          if ($tspConfigs) {
+            LogInfo "  Folder 'specification/$rpFolder' contains $($tspConfigs.Count) file(s) named 'tspconfig.yaml'"
+          }
+          else {
+            LogError ("OpenAPI was generated from TypeSpec, but folder 'specification/$rpFolder' contains no files named 'tspconfig.yaml'." `
+              + "  The TypeSpec used to generate OpenAPI must be added to this folder.")
+            LogJobFailure
+            exit 1
+          }
+        }
+        else {
+          LogError "Path to OpenAPI did not match expected regex.  Unable to extract RP folder."
+          LogJobFailure
+          exit 1
+        }
+
         # Skip further checks, since spec is already using TypeSpec
         continue
       }
       else {
         LogInfo "  OpenAPI was not generated from TypeSpec (missing '/info/x-typespec-generated')"
       }
     }
-    catch {
-      LogWarning "  OpenAPI cannot be parsed as JSON, so assuming not generated from TypeSpec"
-      LogWarning "    $_"
-    }
 
     # Extract path between "specification/" and "/(preview|stable)"
     if ($file -match "specification/(?<servicePath>[^/]+/($SpecType).*?)/(preview|stable)/[^/]+/[^/]+\.json$") {
       $servicePath = $Matches["servicePath"]
     }
     else {
-      LogError "  Path to OpenAPI did not match expected regex.  Unable to extract service path."
+      LogError "Path to OpenAPI did not match expected regex.  Unable to extract service path."
       LogJobFailure
       exit 1
     }
@@ -142,7 +167,7 @@ else {
         $responseCache[$urlToStableFolder] = $responseStatus
       }
       catch {
-        LogError "  Exception making web request to ${logUrlToStableFolder}: $_"
+        LogError "Exception making web request to ${logUrlToStableFolder}: $_"
         LogJobFailure
         exit 1
       }

eng/tools/typespec-validation/src/index.ts

@@ -4,7 +4,6 @@ import { EmitAutorestRule } from "./rules/emit-autorest.js";
 import { FlavorAzureRule } from "./rules/flavor-azure.js";
 import { FolderStructureRule } from "./rules/folder-structure.js";
 import { FormatRule } from "./rules/format.js";
-import { GitDiffRule } from "./rules/git-diff.js";
 import { LinterRulesetRule } from "./rules/linter-ruleset.js";
 import { NpmPrefixRule } from "./rules/npm-prefix.js";
 import { TsvRunnerHost } from "./tsv-runner-host.js";
@@ -40,7 +39,6 @@ export async function main() {
     new LinterRulesetRule(),
     new CompileRule(),
     new FormatRule(),
-    new GitDiffRule(),
   ];
   let success = true;
   for (let i = 0; i < rules.length; i++) {

eng/tools/typespec-validation/src/rules/compile.ts

@@ -44,6 +44,15 @@ export class CompileRule implements Rule {
       errorOutput += stderr;
     }
 
+    if (success) {
+      const gitDiffResult = await host.gitDiffTopSpecFolder(host, folder);
+      stdOutput += gitDiffResult.stdOutput;
+      if (!gitDiffResult.success) {
+        success = false;
+        errorOutput += gitDiffResult.errorOutput;
+        errorOutput += `\nFiles have been changed after \`tsp compile\`. Run \`tsp compile\` and ensure all files are included in your change.`;
+      }
+    }
     return {
       success: success,
       stdOutput: stdOutput,

eng/tools/typespec-validation/src/rules/format.ts

@@ -33,6 +33,16 @@ export class FormatRule implements Rule {
     stdOutput += stdout;
     errorOutput += stderr;
 
+    if (success) {
+      const gitDiffResult = await host.gitDiffTopSpecFolder(host, folder);
+      stdOutput += gitDiffResult.stdOutput;
+      if (!gitDiffResult.success) {
+        success = false;
+        errorOutput += gitDiffResult.errorOutput;
+        errorOutput += `\nFiles have been changed after \`tsp format\`. Run \`tsp format\` and ensure all files are included in your change.`;
+      }
+    }
+
     return {
       success: success,
       stdOutput: stdOutput,