Merge from ci_dev into ci_prod

.pipelines/push-helm-chart-as-oci-artifact.sh → .pipelines/push-helm-chart-to-canary-repos.sh

@@ -1,8 +1,9 @@
 #!/bin/bash
-# push the helm chart as an OCI artifact to specified ACR
 # working directory of this script should be charts/azuremonitor-containers
 
-export REPO_PATH="batch1/test/azure-monitor-containers"
+# note: this repo registered in arc k8s extension for canary region
+export REPO_PATH="public/azuremonitor/containerinsights/canary/preview/azuremonitor-containers"
+
 export  HELM_EXPERIMENTAL_OCI=1
 
 for ARGUMENT in "$@"
@@ -11,32 +12,33 @@ do
    VALUE=$(echo $ARGUMENT | cut -f2 -d=)
 
    case "$KEY" in
-           CIARCACR) CIARCACR=$VALUE ;;
+           CIACR) CIACR=$VALUE ;;
            CICHARTVERSION) CHARTVERSION=$VALUE ;;
            *)
     esac
 done
 
-echo "CI ARC K8S ACR: ${CIARCACR}"
+echo "CI ARC K8S ACR: ${CIACR}"
 echo "CI HELM CHART VERSION: ${CHARTVERSION}"
 
 echo "start: read appid and appsecret"
 ACR_APP_ID=$(cat ~/acrappid)
 ACR_APP_SECRET=$(cat ~/acrappsecret)
 echo "end: read appid and appsecret"
 
-ACR=${CIARCACR}
+ACR=${CIACR}
+
+echo "login to acr:${ACR} using helm"
+helm registry login $ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
 
-echo "login to acr:${ACR} using oras"
-oras login $ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
 echo "login to acr:${ACR} completed: ${ACR}"
 
 echo "start: push the chart version: ${CHARTVERSION} to acr repo: ${ACR}"
 
-echo "generate helm package"
-helm package .
+echo "save the chart locally with acr full path"
+helm chart save . ${ACR}/${REPO_PATH}:${CHARTVERSION}
 
-echo "pushing the helm chart as an OCI artifact"
-oras push ${ACR}/${REPO_PATH}:${CHARTVERSION} --manifest-config /dev/null:application/vnd.unknown.config.v1+json  ./azuremonitor-containers-${CHARTVERSION}.tgz:application/tar+gzip
+echo "pushing the helm chart to ACR: ${ACR}"
+helm chart push ${ACR}/${REPO_PATH}:${CHARTVERSION}
 
 echo "end: push the chart version: ${CHARTVERSION} to acr repo: ${ACR}"

.pipelines/push-helm-chart-to-prod-repos.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# working directory of this script should be charts/azuremonitor-containers
+
+# this repo used without extension public preview release
+export PROD_REPO_PATH="public/azuremonitor/containerinsights/preview/azuremonitor-containers"
+
+# note: this repo registered in arc k8s extension for prod group1 regions.
+export EXTENSION_PROD_REPO_PATH="public/azuremonitor/containerinsights/prod1/preview/azuremonitor-containers"
+
+export  HELM_EXPERIMENTAL_OCI=1
+
+for ARGUMENT in "$@"
+do
+   KEY=$(echo $ARGUMENT | cut -f1 -d=)
+   VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+
+   case "$KEY" in
+           CIACR) CIACR=$VALUE ;;
+           CICHARTVERSION) CHARTVERSION=$VALUE ;;
+           *)
+    esac
+done
+
+echo "CI ARC K8S ACR: ${CIACR}"
+echo "CI HELM CHART VERSION: ${CHARTVERSION}"
+
+echo "start: read appid and appsecret"
+ACR_APP_ID=$(cat ~/acrappid)
+ACR_APP_SECRET=$(cat ~/acrappsecret)
+echo "end: read appid and appsecret"
+
+ACR=${CIACR}
+
+echo "login to acr:${ACR} using helm"
+helm registry login $ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
+
+echo "login to acr:${ACR} completed: ${ACR}"
+
+echo "start: push the chart version: ${CHARTVERSION} to acr repo: ${ACR}"
+
+echo "save the chart locally with acr full path: ${ACR}/${EXTENSION_PROD_REPO_PATH}:${CHARTVERSION}"
+helm chart save . ${ACR}/${EXTENSION_PROD_REPO_PATH}:${CHARTVERSION}
+
+echo "save the chart locally with acr full path: ${ACR}/${PROD_REPO_PATH}:${CHARTVERSION}"
+helm chart save . ${ACR}/${PROD_REPO_PATH}:${CHARTVERSION}
+
+echo "pushing the helm chart to ACR: ${ACR}/${EXTENSION_PROD_REPO_PATH}:${CHARTVERSION}"
+helm chart push ${ACR}/${EXTENSION_PROD_REPO_PATH}:${CHARTVERSION}
+
+echo "pushing the helm chart to ACR: ${ACR}/${PROD_REPO_PATH}:${CHARTVERSION}"
+helm chart push ${ACR}/${PROD_REPO_PATH}:${CHARTVERSION}
+
+echo "end: push the chart version: ${CHARTVERSION} to acr repo: ${ACR}"

Kubecon/README.MD

@@ -0,0 +1,36 @@
+# Kubecon Azure Monitor for containers lab
+
+## Overview
+
+### This Azure Monitor for containers lab will give you hands on experience to monitor AKS workloads. In this lab you will be working Azure Monitor, Log Analytics and Azure Monitor for Container Insights.
+
+## Instructions for lab
+
+1. Set-up environment [Setup Guide](https://github.com/rkuehfus/pre-ready-2019-H1/blob/master/Student/Guides/Deployment%20Setup%20Guide.docx?raw=true)
+
+2. Tasks for the lab
+  *  From your Visual Studio Server, deploy the eShoponWeb application to AKS using Dev Spaces
+  *  From Azure Monitor, locate the container running the eShoponWeb application
+  *  Generate an exception in the eShoponWeb application(Hint: Try to change your password)
+  *  Optimize the Azure Monitor for contains ingestion cost by fine tuning log-collection parameters like std-out/std-error, namespace.
+
+## Outcome
+
+### Understand Azure Monitor capabilities, facilitate an Azure Monitor customer conversation, and demo key features of Azure Monitor.
+
+## Target Audience
+
+This content has been targeted to devops/SRE intended to build their knowledge on Azure Monitor also for people that have a passion around Monitoring are more than welcome to attend.
+
+## Prerequisites
+ 1.	Please review the following content before the event
+  a.  [Azure Monitor for containers Overview](https://docs.microsoft.com/azure/azure-monitor/insights/container-insights-overview)
+  b.  [Optimize Azure Monitor for containers cost ](https://medium.com/microsoftazure/azure-monitor-for-containers-optimizing-data-collection-settings-for-cost-ce6f848aca32)
+
+2.	Attendees have access to an Azure Subscription where they can each deploy the provided ARM template that will build a very detailed infrastructure to monitor.  This includes the Vnet, subnets, NSG(s), LB(s), NAT rules, scales set and a fully functional .NET Core Application (eShopOnWeb) to monitor.
+3.	Attendees should have a level 200-300 understanding of the Azure platform.  Understand concepts like PowerShell, Azure Cli, ARM, resource groups, RBAC, network, storage, compute, scale sets, virtual machines and security.  Previous experience working with ARM templates is recommended.
+4.	Access to a machine with Visual Studio Code and the Azure PowerShell Modules loaded or Azure CLI. VS Code ARM and PowerShell extensions should be configured.
+
+![alt text](https://raw.githubusercontent.com/rkuehfus/pre-ready-2019-H1/master/monitoringhackdiagram.png)
+
+

README.md

@@ -70,7 +70,6 @@ The general directory structure is:
 │   ├── windows/                              - scripts to build the Docker image for Windows Agent
 │   │   ├── dockerbuild                       - script to build the code and docker imag, and publish docker image
 │   │   ├── acrworkflows/                     - acr work flows for the Windows Agent container image
-│   │   ├── baseimage/                        - windowsservercore base image for the windows agent container
 │   │   ├── DockerFile                        - DockerFile for Windows Agent Container Image
 │   │   ├── main.ps1                          - Windows Agent container entry point
 │   │   ├── setup.ps1                         - setup file for Windows Agent Container Image
@@ -140,7 +139,7 @@ bash ~/Docker-Provider/scripts/build/linux/install-build-pre-requisites.sh
 
 ### Build Docker Provider Shell Bundle and Docker Image and Publish Docker Image
 
-> Note: If you are using WSL2, ensure Docker for windows running Linux containers mode to build Linux agent image successfully
+> Note: If you are using WSL2, ensure `Docker for windows` running with Linux containers mode on your windows machine to build Linux agent image successfully
 
 ```
 cd ~/Docker-Provider/kubernetes/linux/dockerbuild
@@ -167,9 +166,23 @@ docker push <repo>/<imagename>:<imagetag>
 ```
 ## Windows Agent
 
+To build the windows agent, you will have to build .NET and Go code, and docker image for windows agent.
+Docker image for windows agent can only build on Windows machine with `Docker for windows` with Windows containers mode but the .NET code and Go code can be built either on Windows or Linux or WSL2.
+
 ### Install Pre-requisites
 
-If you are planning to build the .net and go code for windows agent on Linux machine and you have already have Docker for Windows on Windows machine, then you may skip this.
+Install pre-requisites based on OS platform you will be using to build the windows agent code
+
+#### Option 1 - Using Windows Machine to Build the Windows agent
+
+```
+powershell # launch powershell with elevated admin on your windows machine
+Set-ExecutionPolicy -ExecutionPolicy bypass # set the execution policy
+cd %userprofile%\Docker-Provider\scripts\build\windows # based on your repo path
+.\install-build-pre-requisites.ps1 #
+```
+
+#### Option 2 - Using WSL2 to Build the Windows agent
 
 ```
 powershell # launch powershell with elevated admin on your windows machine
@@ -178,20 +191,36 @@ net use z: \\wsl$\Ubuntu-16.04 # map the network drive of the ubuntu app to wind
 cd z:\home\sshadmin\Docker-Provider\scripts\build\windows # based on your repo path
 .\install-build-pre-requisites.ps1 #
 ```
-#### Build Certificate Generator Source code and Out OMS Go plugin code
 
-> Note: .net and go code for windows agent can built on Ubuntu
+
+### Build Windows Agent code and Docker Image
+
+> Note: format of the windows agent imagetag will be `win-ci<release><MMDDYYYY>`. possible values for release are test, dev, preview, dogfood, prod etc.
+
+#### Option 1 - Using Windows Machine to Build the Windows agent
+
+Execute below instructions on elevated command prompt to build windows agent code and docker image, publishing the image to acr or docker hub
 
 ```
-cd ~/Docker-Provider/build/windows # based on your repo path on ubuntu or WSL2
+cd %userprofile%\Docker-Provider\kubernetes\windows\dockerbuild # based on your repo path
+docker login # if you want to publish the image to acr then login to acr via `docker login <acr-name>`
+powershell -ExecutionPolicy bypass  # switch to powershell if you are not on powershell already
+.\build-and-publish-docker-image.ps1 -image <repo>/<imagename>:<imagetag> # trigger build code and image and publish docker hub or acr
+```
+
+#### Option 2 - Using WSL2 to Build the Windows agent
+
+##### On WSL2, Build Certificate Generator Source code and Out OMS Go plugin code
+
+```
+cd ~/Docker-Provider/build/windows # based on your repo path on WSL2 Ubuntu app
 pwsh #switch to powershell
 .\Makefile.ps1 # trigger build and publish of .net and go code
 ```
-> Note: format of the imagetag will be `win-ci<release><MMDDYYYY>`. possible values for release are test, dev, preview, dogfood, prod etc.
 
-####  Build and Push Docker Image
+####  On Windows machine, build and Push Docker Image
 
-> Note: windows container can only built on windows hence you will have to execute below commands on windows via accessing network share or copying published bits omsagentwindows under kubernetes directory on to windows machine
+> Note: Docker image for windows container can only built on windows hence you will have to execute below commands on windows via accessing network share or copying published bits omsagentwindows under kubernetes directory on to windows machine
 
 ```
 net use z: \\wsl$\Ubuntu-16.04 # map the network drive of the ubuntu app to windows
@@ -200,17 +229,6 @@ docker build -t <repo>/<imagename>:<imagetag> --build-arg IMAGE_TAG=<imagetag> .
 docker push <repo>/<imagename>:<imagetag>
 ```
 
-### Build Cert generator, Out OMS Plugun and Docker Image and Publish Docker Image
-
-If you have code cloned on to windows, you can built everything for windows agent on windows machine via below instructions
-
-```
-cd %userprofile%\Docker-Provider\kubernetes\windows\dockerbuild # based on your repo path
-docker login # if you want to publish the image to acr then login to acr via `docker login <acr-name>`
-powershell -ExecutionPolicy bypass  # switch to powershell if you are not on powershell already
-.\build-and-publish-docker-image.ps1 -image <repo>/<imagename>:<imagetag> # trigger build code and image and publish docker hub or acr
-```
-
 # Azure DevOps Build Pipeline
 
 Navigate to https://github-private.visualstudio.com/microsoft/_build?view=pipelines to see Linux and Windows Agent build pipelines. These pipelines are configured with CI triggers for ci_dev and ci_prod.

ReleaseNotes.md

@@ -11,6 +11,35 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 10/05/2020 -
+##### Version microsoft/oms:ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10052020 (linux)
+##### Version microsoft/oms:win-ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
+##### Code change log
+- Health CRD to version v1 (from v1beta1) for k8s versions >= 1.19.0
+- Collection of PV usage metrics for PVs mounted by pods (kube-system pods excluded by default)(doc-link-needed)
+- Zero fill few custom metrics under a timer, also add zero filling for new PV usage metrics
+- Collection of additional Kubelet metrics ('kubelet_running_pod_count','volume_manager_total_volumes','kubelet_node_config_error','process_resident_memory_bytes','process_cpu_seconds_total','kubelet_runtime_operations_total','kubelet_runtime_operations_errors_total'). This also includes updates to 'kubelet' workbook to include these new metrics
+- Collection of Azure NPM (Network Policy Manager) metrics (basic & advanced. By default, NPM metrics collection is turned OFF)(doc-link-needed)
+- Support log collection when docker root is changed with knode. Tracked by [this](https://github.com/Azure/AKS/issues/1373) issue
+- Support for Pods in 'Terminating' state for nodelost scenarios
+- Fix for reduction in telemetry for custom metrics ingestion failures
+- Fix CPU capacity/limits metrics being 0 for Virtual nodes (VK)
+- Add new custom metric regions (eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth)
+- Enable strict SSL validation for AppInsights Ruby SDK
+- Turn off custom metrics upload for unsupported cluster types
+- Install CA certs from wire server for windows (in certain clouds)
+
+### 09/16/2020 -
+> Note: This agent release targetted ONLY for non-AKS clusters via Azure Monitor for containers HELM chart update
+##### Version microsoft/oms:ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod09162020 (linux)
+##### Version microsoft/oms:win-ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod09162020 (windows)
+##### Code change log
+- Collection of Azure Network Policy Manager Basic and Advanced metrics
+- Add support in Windows Agent for Container log collection of CRI runtimes such as ContainerD
+- Alertable metrics support Arc K8s cluster to parity with AKS
+- Support for multiple container log mount paths when docker is updated through knode
+- Bug fix related to MDM telemetry
+
 ### 08/07/2020 -
 ##### Version microsoft/oms:ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020 (linux)
 ##### Version microsoft/oms:win-ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08072020 (windows)

ReleaseProcess.md

@@ -45,7 +45,10 @@ Make PR against [AKS-Engine](https://github.com/Azure/aks-engine). Refer PR http
 
 ## ARO v4, On-prem K8s, Azure Arc K8s and OpenShift v4 clusters
 
-Make PR against [HELM-charts](https://github.com/helm/charts) with Azure Monitor for containers chart update.
+Make sure azuremonitor-containers chart yamls updates with all changes going with the release and also make sure to bump the chart version, imagetag and docker provider version etc. Similar to agent container image, build pipeline automatically push the chart to container insights prod acr for canary and prod repos accordingly.
+Both the agent and helm chart will be replicated to `mcr.microsoft.com`.
+
+The way, customers will be onboard the monitoring to these clusters using onboarding scripts under `onboarding\managed` directory so please bump chart version for prod release. Once we move to Arc K8s Monitoring extension Public preview, these will be taken care so at that point of time no manual changes like this required.
 
 # 4. Monitor agent roll-out status