git commit -m "merge master"

Signed-off-by: Chris Martin <[email protected]>

.github/workflows/build.yml

@@ -32,16 +32,16 @@ jobs:
           DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
 
       - name: Output full commit sha
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/patch/v')) }}
         run: echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_ENV
 
       - name: Save Docker image tarballs
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/patch/v')) }}
         run: |
           scripts/docker-save.sh -t ${{ env.sha_full }} -o /tmp/imgs
 
       - name: Save Docker image tarballs as artifacts
-        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        if: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/patch/v')) }}
         uses: actions/upload-artifact@v4
         with:
           name: armada-image-tarballs

.github/workflows/release-rc.yml

@@ -6,6 +6,7 @@ on:
     workflows: [CI]
     branches:
       - master
+      - patch/v*
 
 permissions:
   contents: write
@@ -30,11 +31,20 @@ jobs:
           ref='${{ github.event.workflow_run.head_branch }}'
           sha='${{ github.event.workflow_run.head_sha }}'
 
-          [ "$ref" == "master" ] &&
-          [ $(git branch --contains=$sha master | wc -l) -eq 1 ] &&
-          [ $(git rev-list --count $sha..master) -le 2 ]
+          if [[ "$ref" == "master" ]]; then
+            [ $(git branch --contains=$sha master | wc -l) -eq 1 ] &&
+            [ $(git rev-list --count $sha..master) -le 2 ]
+          elif [[ "$ref" =~ ^patch/v[0-9]+\.[0-9]+\.[0-9]+-patch[0-9]+$ ]]; then
+            [ $(git branch --contains=$sha "$ref" | wc -l) -eq 1 ] &&
+            base_tag=${ref#patch/}
+            base_tag=${base_tag%-*}
+            [ $(git rev-list --count $sha..$base_tag) -le 5 ]
+          else
+            false
+          fi
+
           if [ $? -ne 0 ]; then
-            echo "::error ::Invalid ref $ref $sha: must be a merge to master branch and not more than 2 commits away"
+            echo "::error ::Invalid ref $ref $sha: must be a merge to master branch (not more than 2 commits away) or a patch branch (not more than 5 commits away from base tag)"
             exit 1
           fi
   release:

.github/workflows/release.yml

@@ -30,13 +30,32 @@ jobs:
           ref='${{ github.event.workflow_run.head_branch }}'
           sha='${{ github.event.workflow_run.head_sha }}'
 
-          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
-          [ $(git tag --points-at $sha | grep -E "^$ref\$" | wc -l) -eq 1 ] &&
-          [ $(git branch --contains=$sha master | wc -l) -eq 1 ]
-          if [ $? -ne 0 ]; then
-            echo "::error ::Invalid ref $ref $sha: must be a tag, belong to the master branch and match the semver regex"
+          echo "Validating ref: $ref, sha: $sha"
+
+          # Check if it's a valid tag format
+          if [[ ! $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-patch[0-9]+)?$ ]]; then
+            echo "::error::Invalid tag format: $ref"
+            exit 1
+          fi
+
+          # Check if the tag points to the current SHA
+          if [ $(git tag --points-at $sha | grep -E "^$ref\$" | wc -l) -ne 1 ]; then
+            echo "::error::Tag $ref does not point to the current SHA $sha"
+            exit 1
+          fi
+
+          # Fetch all branches to ensure we have the necessary information
+          git fetch --all
+
+          # Check if it's from master branch or a patch branch
+          if [ $(git branch -r --contains=$sha | grep -E "origin/(master|patch/v[0-9]+\.[0-9]+\.[0-9]+-patch[0-9]+)$" | wc -l) -eq 0 ]; then
+            echo "::error::$sha is not in master or any patch branch"
+            echo "Branches containing this SHA:"
+            git branch -r --contains=$sha
             exit 1
           fi
+
+          echo "Validation successful"
   release:
     name: "Release"
     needs: validate
@@ -76,7 +95,22 @@ jobs:
         run: |
           current_tag='${{ github.event.workflow_run.head_branch }}'
           echo "GORELEASER_CURRENT_TAG=$current_tag" >> $GITHUB_ENV
-          previous_tag=$(git -c 'versionsort.suffix=-rc' tag --list --sort=version:refname | grep -Eo '^v[0-9]{1,}.[0-9]{1,}.[0-9]{1,}$' | tail -n 2 | head -n 1)
+
+          # Function to extract base version
+          get_base_version() {
+            echo "$1" | sed -E 's/^v?([0-9]+\.[0-9]+\.[0-9]+).*$/\1/'
+          }
+
+          # Check if current tag is a patch release
+          if [[ $current_tag =~ ^v?[0-9]+\.[0-9]+\.[0-9]+-patch[0-9]+$ ]]; then
+            # For patch releases, find the previous patch or the base version
+            base_version=$(get_base_version "$current_tag")
+            previous_tag=$(git tag --list "v${base_version}*" --sort=-v:refname | grep -v "$current_tag" | head -n1)
+          else
+            # For master releases, find the previous master release
+            previous_tag=$(git tag --list 'v*' --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | grep -v "$current_tag" | head -n1)
+          fi
+
           echo "GORELEASER_PREVIOUS_TAG=$previous_tag" >> $GITHUB_ENV
 
       - name: Run GoReleaser

internal/scheduler/metrics/metrics.go

@@ -393,7 +393,6 @@ func appendLabelsFromJob(labels []string, job *jobdb.Job) []string {
 	executor := executorNameFromRun(job.LatestRun())
 	labels = append(labels, job.Queue())
 	labels = append(labels, executor)
-	labels = append(labels, "") // No nodeType.
 	return labels
 }
 
@@ -498,7 +497,7 @@ func (m *Metrics) counterVectorsFromResource(resource v1.ResourceName) (*prometh
 				Name:      name,
 				Help:      resource.String() + "resource counter.",
 			},
-			[]string{"state", "category", "subCategory", "queue", "cluster", "nodeType", "node"},
+			[]string{"state", "category", "subCategory", "queue", "cluster"},
 		)
 		m.resourceCounters[resource] = c
 	}
@@ -514,7 +513,7 @@ func (m *Metrics) counterVectorsFromResource(resource v1.ResourceName) (*prometh
 				Name:      name,
 				Help:      resource.String() + "-second resource counter.",
 			},
-			[]string{"priorState", "state", "category", "subCategory", "queue", "cluster", "nodeType", "node"},
+			[]string{"priorState", "state", "category", "subCategory", "queue", "cluster"},
 		)
 		m.resourceCounters[resourceSeconds] = cSeconds
 	}

internal/scheduler/metrics/metrics_test.go

@@ -1,21 +1,74 @@
 package metrics
 
 import (
-	"regexp"
 	"testing"
+	"time"
 
-	"github.com/stretchr/testify/assert"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+
+	"github.com/armadaproject/armada/internal/common/armadacontext"
+	"github.com/armadaproject/armada/internal/scheduler/configuration"
+	"github.com/armadaproject/armada/internal/scheduler/context"
+	"github.com/armadaproject/armada/internal/scheduler/schedulerobjects"
+	"github.com/armadaproject/armada/internal/scheduler/testfixtures"
+	"github.com/armadaproject/armada/pkg/armadaevents"
 )
 
-func TestFoo(t *testing.T) {
-	r, err := regexp.Compile("foo.*bar")
+func TestUpdate(t *testing.T) {
+	ctx := armadacontext.Background()
+
+	metrics, err := New(configuration.MetricsConfig{
+		TrackedErrorRegexes:  nil,
+		TrackedResourceNames: []v1.ResourceName{"cpu"},
+		ResetInterval:        24 * time.Hour,
+	})
 	require.NoError(t, err)
-	assert.True(t, r.MatchString("foobar"))
-	assert.True(t, r.MatchString("foo bar"))
-	assert.True(t, r.MatchString("foo and bar"))
-	assert.True(t, r.MatchString("this is foo and bar so"))
-	assert.False(t, r.MatchString("barfoo"))
-	assert.False(t, r.MatchString("foo"))
-	assert.False(t, r.MatchString("bar"))
+
+	now := time.Now()
+
+	queuedJob := testfixtures.NewJob(uuid.NewString(),
+		"test-jobset",
+		"test-queue",
+		1,
+		&schedulerobjects.JobSchedulingInfo{},
+		true,
+		0,
+		false,
+		false,
+		false,
+		time.Now().UnixNano(),
+		true)
+
+	jobRunErrorsByRunId := map[uuid.UUID]*armadaevents.Error{
+		uuid.MustParse(queuedJob.Id()): {
+			Terminal: true,
+			Reason: &armadaevents.Error_PodError{
+				PodError: &armadaevents.PodError{
+					Message: "my error",
+				},
+			},
+		},
+	}
+
+	leasedJob := queuedJob.WithNewRun("test-executor", "node1", "test-node", "test-pool", 1)
+	pendingJob := leasedJob.WithUpdatedRun(leasedJob.LatestRun().WithPendingTime(addSeconds(now, 1)))
+	runningJob := pendingJob.WithUpdatedRun(pendingJob.LatestRun().WithRunningTime(addSeconds(now, 2)))
+	finishedJob := runningJob.WithUpdatedRun(runningJob.LatestRun().WithTerminatedTime(addSeconds(now, 3)))
+	preemptedJob := finishedJob.WithUpdatedRun(runningJob.LatestRun().WithPreemptedTime(addSeconds(now, 4)))
+
+	require.NoError(t, metrics.UpdateQueued(queuedJob))
+	require.NoError(t, metrics.UpdateLeased(context.JobSchedulingContextFromJob(leasedJob)))
+	require.NoError(t, metrics.UpdatePending(pendingJob))
+	require.NoError(t, metrics.UpdateRunning(runningJob))
+	require.NoError(t, metrics.UpdateSucceeded(finishedJob))
+	require.NoError(t, metrics.UpdateCancelled(finishedJob))
+	require.NoError(t, metrics.UpdateFailed(ctx, finishedJob, jobRunErrorsByRunId))
+	require.NoError(t, metrics.UpdatePreempted(preemptedJob))
+}
+
+func addSeconds(t time.Time, seconds int) *time.Time {
+	t = t.Add(time.Duration(seconds) * time.Second)
+	return &t
 }

internal/scheduler/nodedb/nodedb.go

@@ -259,13 +259,6 @@ func NewNodeDb(
 	for _, key := range indexedNodeLabels {
 		indexedNodeLabelValues[key] = make(map[string]struct{})
 	}
-	mapFromSlice := func(vs []string) map[string]interface{} {
-		rv := make(map[string]interface{})
-		for _, v := range vs {
-			rv[v] = true
-		}
-		return rv
-	}
 
 	indexedResourceResolution, err := makeIndexedResourceResolution(indexedResources, resourceListFactory)
 	if err != nil {
@@ -304,6 +297,14 @@ func NewNodeDb(
 	return &nodeDb, nil
 }
 
+func mapFromSlice(vs []string) map[string]interface{} {
+	rv := make(map[string]interface{})
+	for _, v := range vs {
+		rv[v] = true
+	}
+	return rv
+}
+
 func makeIndexedResourceResolution(indexedResourceTypes []configuration.ResourceType, resourceListFactory *internaltypes.ResourceListFactory) ([]int64, error) {
 	if len(indexedResourceTypes) < 1 {
 		return nil, errors.New("must specify at least one entry in indexedResources in config")