Merge branch 'main' into fix-flaky-aws-restoration

.github/pull_request_template.md

@@ -27,9 +27,21 @@ Fixes: #issue_number
 ## Contributor checklist
 Please verify that the PR meets the following requirements, where applicable:
 
-- [ ] An overview of proposed schema changes is included in a linked GitHub issue.
-- [ ] A design document PR is created in the [design-notes repository](https://github.com/radius-project/design-notes/), if new APIs are being introduced.
-- [ ] If applicable, design document has been reviewed and approved by Radius maintainers/approvers.
-- [ ] A PR for the [samples repository](https://github.com/radius-project/samples) is created, if existing samples are affected by the changes in this PR.
-- [ ] A PR for the [documentation repository](https://github.com/radius-project/docs) is created, if the changes in this PR affect the documentation or any user facing updates are made.
-- [ ] A PR for the [recipes repository](https://github.com/radius-project/recipes) is created, if existing recipes are affected by the changes in this PR.
+- An overview of proposed schema changes is included in a linked GitHub issue.
+    - [ ] Yes
+    - [ ] Not applicable
+- A design document PR is created in the [design-notes repository](https://github.com/radius-project/design-notes/), if new APIs are being introduced.
+    - [ ] Yes
+    - [ ] Not applicable
+- The design document has been reviewed and approved by Radius maintainers/approvers.
+    - [ ] Yes
+    - [ ] Not applicable
+- A PR for the [samples repository](https://github.com/radius-project/samples) is created, if existing samples are affected by the changes in this PR.
+    - [ ] Yes
+    - [ ] Not applicable
+- A PR for the [documentation repository](https://github.com/radius-project/docs) is created, if the changes in this PR affect the documentation or any user facing updates are made.
+    - [ ] Yes
+    - [ ] Not applicable
+- A PR for the [recipes repository](https://github.com/radius-project/recipes) is created, if existing recipes are affected by the changes in this PR.
+    - [ ] Yes
+    - [ ] Not applicable

.github/workflows/functional-test-cloud.yaml

@@ -657,6 +657,50 @@ jobs:
             --set global.azureWorkloadIdentity.enabled=true \
             --set global.aws.irsa.enabled=true 
           
+          echo "*** Verify manifests are registered ***"
+          rm -f registermanifest_logs.txt
+          # Find the pod with container "ucp"
+          POD_NAME=$(
+            kubectl get pods -n radius-system \
+              -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
+            | grep "ucp" \
+            | head -n1 \
+            | cut -d" " -f1
+          )
+          echo "Found ucp pod: $POD_NAME"
+
+          if [ -z "$POD_NAME" ]; then
+            echo "No pod with container 'ucp' found in namespace radius-system."
+            exit 1
+          fi
+
+          # Poll logs for up to  iterations, 30 seconds each (upto 3 minutes total)
+          for i in {1..6}; do
+            kubectl logs "$POD_NAME" -n radius-system | tee registermanifest_logs.txt > /dev/null
+
+            # Exit on error
+            if grep -qi "Service initializer terminated with error" registermanifest_logs.txt; then
+              echo "Error found in ucp logs:"
+              grep -i "Service initializer terminated with error" registermanifest_logs.txt
+              exit 1
+            fi
+
+            # Check for success
+            if grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+              echo "Successfully registered manifests - message found."
+              break
+            fi
+
+            echo "Logs not ready, waiting 30 seconds..."
+            sleep 30
+          done
+
+          # Final check to ensure success message was found
+          if ! grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+            echo "Manifests not registered after 3 minutes."
+            exit 1
+          fi
+
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
           rad group create kind-radius
@@ -678,7 +722,7 @@ jobs:
           
           rad credential register aws irsa \
             --iam-role ${{ secrets.FUNC_TEST_RAD_IRSA_ROLE }}
-          
+
       - uses: marocchino/sticky-pull-request-comment@v2
         if: failure() && env.PR_NUMBER != ''
         continue-on-error: true
@@ -909,4 +953,4 @@ jobs:
               title: `Scheduled functional test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
               body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
-            })
+            })

.github/workflows/functional-test-noncloud.yaml

@@ -272,6 +272,50 @@ jobs:
           echo "*** Installing Radius to Kubernetes ***"
           eval $RAD_COMMAND
 
+          echo "*** Verify manifests are registered ***"
+          rm -f registermanifest_logs.txt
+          # Find the pod with container "ucp"
+          POD_NAME=$(
+            kubectl get pods -n radius-system \
+              -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
+            | grep "ucp" \
+            | head -n1 \
+            | cut -d" " -f1
+          )
+          echo "Found ucp pod: $POD_NAME"
+
+          if [ -z "$POD_NAME" ]; then
+            echo "No pod with container 'ucp' found in namespace radius-system."
+            exit 1
+          fi
+
+          # Poll logs for up to  iterations, 30 seconds each (upto 3 minutes total)
+          for i in {1..6}; do
+            kubectl logs "$POD_NAME" -n radius-system | tee registermanifest_logs.txt > /dev/null
+            
+            # Exit on error
+            if grep -qi "Service initializer terminated with error" registermanifest_logs.txt; then
+              echo "Error found in ucp logs."
+              grep -i "Service initializer terminated with error" registermanifest_logs.txt
+              exit 1
+            fi
+
+            # Check for success
+            if grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+              echo "Successfully registered manifests - message found."
+              break
+            fi
+
+            echo "Logs not ready, waiting 30 seconds..."
+            sleep 30
+          done
+
+          # Final check to ensure success message was found
+          if ! grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+            echo "Manifests not registered after 3 minutes."
+            exit 1
+          fi
+
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
           rad group create kind-radius
@@ -280,6 +324,8 @@ jobs:
           # The functional test is designed to use default namespace. So you must create the environment for default namespace.
           rad env create kind-radius --namespace default
           rad env switch kind-radius
+        continue-on-error: true  
+
         env:
           USE_CERT_FILE: "true"
           TEMP_CERT_DIR: ${{ steps.create-local-registry.outputs.temp-cert-dir }}

build/docker.mk

@@ -17,6 +17,7 @@
 DOCKER_REGISTRY?=$(shell whoami)
 DOCKER_TAG_VERSION?=latest
 IMAGE_SRC?=https://github.com/radius-project/radius
+MANIFEST_DIR?=deploy/manifest/built-in-providers/self-hosted
 
 ##@ Docker Images
 
@@ -106,6 +107,17 @@ APPS_MAP := ucpd:./deploy/images/ucpd \
 	testrp:./test/testrp \
 	magpiego:./test/magpiego
 
+# copy_manifests copies the manifests to the output directory
+.PHONY: copy-manifests
+copy-manifests:
+	@if [ ! -d "$(MANIFEST_DIR)" ] || [ -z "$$(ls -A $(MANIFEST_DIR))" ]; then \
+		echo "MANIFEST_DIR '$(MANIFEST_DIR)' does not exist or is empty"; \
+		exit 1; \
+	fi
+	@mkdir -p $(OUT_DIR)/manifest/built-in-providers/
+	@echo "Copying manifests from $(MANIFEST_DIR) to $(OUT_DIR)/manifest/built-in-providers/"
+	@cp -v $(MANIFEST_DIR)/* $(OUT_DIR)/manifest/built-in-providers/
+
 # Function to extract the name and the directory of the Dockerfile from the app string
 define parseApp
 $(eval NAME := $(shell echo $(1) | cut -d: -f1))
@@ -132,15 +144,15 @@ DOCKER_PUSH_MULTI_TARGETS := $(foreach APP,$(APPS_MAP),$(eval $(call parseApp,$(
 
 # targets to build development images
 .PHONY: docker-build
-docker-build: $(DOCKER_BUILD_TARGETS) ## Builds all Docker images.
+docker-build: copy-manifests $(DOCKER_BUILD_TARGETS) ## Builds all Docker images.
 
 .PHONY: docker-push
 docker-push: $(DOCKER_PUSH_TARGETS) ## Pushes all Docker images (without building).
 
 # targets to build and push multi arch images. If you run this target in your machine,
 # ensure you have qemu and buildx installed by running make configure-buildx.
 .PHONY: docker-multi-arch-build
-docker-multi-arch-build: $(DOCKER_BUILD_MULTI_TARGETS) ## Builds all docker images for multiple architectures.
+docker-multi-arch-build: copy-manifests $(DOCKER_BUILD_MULTI_TARGETS) ## Builds all docker images for multiple architectures.
 
 .PHONY: docker-multi-arch-push
-docker-multi-arch-push: $(DOCKER_PUSH_MULTI_TARGETS) ## Pushes all docker images for multiple architectures after building.
+docker-multi-arch-push: copy-manifests $(DOCKER_PUSH_MULTI_TARGETS) ## Pushes all docker images for multiple architectures after building.

cmd/ucpd/ucp-dev.yaml

@@ -53,7 +53,7 @@ initialization:
           Microsoft.Resources: "http://localhost:5017"
         kind: "UCPNative"
   # This is the directory location which contains manifests to be registered.
-  manifestDirectory: ""      
+  manifestDirectory: "manifest/built-in-providers/"      
 
 identity:
   authMethod: default

deploy/Chart/templates/ucp/configmaps.yaml

@@ -50,7 +50,7 @@ data:
         - id: "/planes/aws/aws"
           properties:
             kind: "AWS"
-      manifestDirectory: ""      
+      manifestDirectory: "/manifest/built-in-providers"
 
     identity:
       authMethod: UCPCredential

deploy/images/ucpd/Dockerfile

@@ -10,6 +10,9 @@ WORKDIR /
 # Copy the application binary for the specified architecture
 COPY ./linux_${TARGETARCH:-amd64}/release/ucpd /
 
+# Copy the manifest files for the built-in providers
+COPY ./manifest/built-in-providers/ /manifest/built-in-providers/
+
 # Set the user to non-root (65532:65532 is the default non-root user in distroless)
 USER 65532:65532
 

deploy/manifest/built-in-providers/applications_core.yaml → deploy/manifest/built-in-providers/dev/applications_core.yaml

@@ -1,37 +1,40 @@
 name: Applications.Core
+location:
+  global:
+    "http://localhost:8080"
 types:
   containers:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   applications:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   environments:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   gateways:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   secretStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   extenders:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   volumes:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []

deploy/manifest/built-in-providers/applications_dapr.yaml → deploy/manifest/built-in-providers/dev/applications_dapr.yaml

@@ -1,22 +1,25 @@
 name: Applications.Dapr
+location:
+  global:
+    "http://localhost:8080"
 types:
   configurationStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   pubSubBrokers:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   secretStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   stateStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/applications_datastores.yaml → deploy/manifest/built-in-providers/dev/applications_datastores.yaml

@@ -1,17 +1,20 @@
 name: Applications.Datastores
+location:
+  global:
+    "http://localhost:8080"
 types:
   mongoDatabases:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   sqlDatabases:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   redisCaches:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/applications_messaging.yaml → deploy/manifest/built-in-providers/dev/applications_messaging.yaml

@@ -1,7 +1,10 @@
 name: Applications.Messaging
+location:
+  global:
+    "http://localhost:8080"
 types:
   rabbitMQQueues:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/dev/microsoft_resources.yaml

@@ -0,0 +1,12 @@
+name: Microsoft.Resources
+location:
+  global:
+    "http://localhost:5017"
+types:
+  deployments:
+    apiVersions:
+      "2020-10-01":
+        schema: {}
+      "2022-09-01":
+        schema: {}  
+    capabilities: []