diff --git a/.github/workflows/tryit-comment.yml b/.github/workflows/tryit-comment.yml
deleted file mode 100644
index e75184be02..0000000000
--- a/.github/workflows/tryit-comment.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-# IMPORTANT: DO NOT CHECKOUT
-name: Make try it comment
-
-on:
-  pull_request_target:
-    types: [opened, reopened]
-
-concurrency: ${{ github.workflow }}-${{ github.ref }}
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  pull-requests: write
-
-jobs:
-  # Create PR
-  pr_created:
-    name: Provide try it comments
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/github-script@v5
-        with:
-          github-token: ${{secrets.GITHUB_TOKEN}}
-          script: |
-            const prNumber = ${{ github.event.pull_request.number }};
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: [
-                `Changes in this PR will be published to the following url to try(check status of TypeSpec Pull Request Try It pipeline for publish status):`,
-                `Playground: https://cadlplayground.z22.web.core.windows.net/prs/${prNumber}/`,
-                "",
-                `Website: https://tspwebsitepr.z22.web.core.windows.net/prs/${prNumber}/`,
-
-              ].join("\n")
-            })
diff --git a/eng/pipelines/pr-tryit.yml b/eng/pipelines/pr-tryit.yml
index fcb2be7690..4ac394283c 100644
--- a/eng/pipelines/pr-tryit.yml
+++ b/eng/pipelines/pr-tryit.yml
@@ -47,3 +47,8 @@ jobs:
               --destination-path $(TYPESPEC_WEBSITE_BASE_PATH) \
               --source "./packages/website/build/" \
               --overwrite
+
+      - script: node eng/scripts/create-tryit-comment.js "cadl-azure"
+        displayName: Check already commented
+        env:
+          GH_TOKEN: $(azuresdk-github-pat)
diff --git a/eng/scripts/create-tryit-comment.js b/eng/scripts/create-tryit-comment.js
index 641bc94802..d9be7631c4 100644
--- a/eng/scripts/create-tryit-comment.js
+++ b/eng/scripts/create-tryit-comment.js
@@ -26,7 +26,11 @@ async function main() {
   const folderName = process.argv.length > 2 ? `/${process.argv[2]}` : "";
   const repo = process.env["BUILD_REPOSITORY_ID"];
   const prNumber = process.env["SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"];
-  const ghAuth = getGithubAuthHeader(repo);
+  const ghToken = process.env.GH_TOKEN;
+  if (ghToken === undefined) {
+    throw new Error("GH_TOKEN environment variable is not set");
+  }
+  const ghAuth = `Bearer ${ghToken}`;
 
   console.log("Looking for comments in", { repo, prNumber });
   const data = await listComments(repo, prNumber, ghAuth);