diff --git a/.changelog/20344.txt b/.changelog/20344.txt new file mode 100644 index 00000000000..de6252d5b2a --- /dev/null +++ b/.changelog/20344.txt @@ -0,0 +1,3 @@ +```release-note:bug +consul: Fixed a bug where services with interpolation would not get correctly signed Workload Identities +``` diff --git a/client/allocrunner/alloc_runner.go b/client/allocrunner/alloc_runner.go index db0e5dee481..9a781b62b8b 100644 --- a/client/allocrunner/alloc_runner.go +++ b/client/allocrunner/alloc_runner.go @@ -31,6 +31,7 @@ import ( "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/vaultclient" "github.com/hashicorp/nomad/client/widmgr" "github.com/hashicorp/nomad/helper/pointer" @@ -272,8 +273,17 @@ func NewAllocRunner(config *config.AllocRunnerConfig) (interfaces.AllocRunner, e ar.shutdownDelayCtx = shutdownDelayCtx ar.shutdownDelayCancelFn = shutdownDelayCancel + // Create a *taskenv.Builder for the allocation so the WID manager can + // interpolate services with the allocation and tasks as needed + envBuilder := taskenv.NewBuilder( + config.ClientConfig.Node, + ar.Alloc(), + nil, + config.ClientConfig.Region, + ).SetAllocDir(ar.allocDir.AllocDir) + // initialize the workload identity manager - widmgr := widmgr.NewWIDMgr(ar.widsigner, alloc, ar.stateDB, ar.logger) + widmgr := widmgr.NewWIDMgr(ar.widsigner, alloc, ar.stateDB, ar.logger, envBuilder) ar.widmgr = widmgr // Initialize the runners hooks. diff --git a/client/allocrunner/alloc_runner_hooks.go b/client/allocrunner/alloc_runner_hooks.go index f36001cb264..8a299763d78 100644 --- a/client/allocrunner/alloc_runner_hooks.go +++ b/client/allocrunner/alloc_runner_hooks.go @@ -110,7 +110,7 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { SetAllocDir(ar.allocDir.AllocDir) } - // Create a taskenv.TaskEnv which is used for read only purposes by the + // Create a *taskenv.TaskEnv which is used for read only purposes by the // newNetworkHook and newChecksHook. builtTaskEnv := newEnvBuilder().Build() @@ -128,6 +128,7 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error { consulConfigs: ar.clientConfig.GetConsulConfigs(hookLogger), consulClientConstructor: consul.NewConsulClient, hookResources: ar.hookResources, + envBuilder: newEnvBuilder, logger: hookLogger, }), newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher), diff --git a/client/allocrunner/consul_hook.go b/client/allocrunner/consul_hook.go index 22b04086463..6b642cfbce1 100644 --- a/client/allocrunner/consul_hook.go +++ b/client/allocrunner/consul_hook.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/consul" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/widmgr" "github.com/hashicorp/nomad/nomad/structs" structsc "github.com/hashicorp/nomad/nomad/structs/config" @@ -24,6 +25,7 @@ type consulHook struct { consulConfigs map[string]*structsc.ConsulConfig consulClientConstructor func(*structsc.ConsulConfig, log.Logger) (consul.Client, error) hookResources *cstructs.AllocHookResources + envBuilder *taskenv.Builder logger log.Logger } @@ -42,6 +44,9 @@ type consulHookConfig struct { // hookResources is used for storing and retrieving Consul tokens hookResources *cstructs.AllocHookResources + // envBuilder is used to interpolate services + envBuilder func() *taskenv.Builder + logger log.Logger } @@ -53,6 +58,7 @@ func newConsulHook(cfg consulHookConfig) *consulHook { consulConfigs: cfg.consulConfigs, consulClientConstructor: cfg.consulClientConstructor, hookResources: cfg.hookResources, + envBuilder: cfg.envBuilder(), } h.logger = cfg.logger.Named(h.Name()) return h @@ -81,11 +87,12 @@ func (h *consulHook) Prerun() error { } var mErr *multierror.Error - if err := h.prepareConsulTokensForServices(tg.Services, tg, tokens); err != nil { + if err := h.prepareConsulTokensForServices(tg.Services, tg, tokens, h.envBuilder.Build()); err != nil { mErr = multierror.Append(mErr, err) } for _, task := range tg.Tasks { - if err := h.prepareConsulTokensForServices(task.Services, tg, tokens); err != nil { + h.envBuilder.UpdateTask(h.alloc, task) + if err := h.prepareConsulTokensForServices(task.Services, tg, tokens, h.envBuilder.Build()); err != nil { mErr = multierror.Append(mErr, err) } if err := h.prepareConsulTokensForTask(task, tg, tokens); err != nil { @@ -155,7 +162,7 @@ func (h *consulHook) prepareConsulTokensForTask(task *structs.Task, tg *structs. return nil } -func (h *consulHook) prepareConsulTokensForServices(services []*structs.Service, tg *structs.TaskGroup, tokens map[string]map[string]*consulapi.ACLToken) error { +func (h *consulHook) prepareConsulTokensForServices(services []*structs.Service, tg *structs.TaskGroup, tokens map[string]map[string]*consulapi.ACLToken, env *taskenv.TaskEnv) error { if len(services) == 0 { return nil } @@ -174,8 +181,8 @@ func (h *consulHook) prepareConsulTokensForServices(services []*structs.Service, } // Find signed identity workload. - identity := *service.IdentityHandle() - jwt, err := h.widmgr.Get(identity) + handle := *service.IdentityHandle(env.ReplaceEnv) + jwt, err := h.widmgr.Get(handle) if err != nil { mErr = multierror.Append(mErr, fmt.Errorf( "error getting signed identity for service %s: %v", @@ -189,7 +196,7 @@ func (h *consulHook) prepareConsulTokensForServices(services []*structs.Service, JWT: jwt.JWT, AuthMethodName: consulConfig.ServiceIdentityAuthMethod, Meta: map[string]string{ - "requested_by": fmt.Sprintf("nomad_service_%s", identity.WorkloadIdentifier), + "requested_by": fmt.Sprintf("nomad_service_%s", handle.InterpolatedWorkloadIdentifier), }, } token, err := h.getConsulToken(clusterName, req) diff --git a/client/allocrunner/consul_hook_test.go b/client/allocrunner/consul_hook_test.go index 2234c3b107e..8176041652b 100644 --- a/client/allocrunner/consul_hook_test.go +++ b/client/allocrunner/consul_hook_test.go @@ -13,7 +13,9 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/client/allocrunner/interfaces" "github.com/hashicorp/nomad/client/consul" + cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/widmgr" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" @@ -41,45 +43,30 @@ func consulHookTestHarness(t *testing.T) *consulHook { Provider: structs.ServiceProviderConsul, Identity: &structs.WorkloadIdentity{Name: "consul-service_webservice", Audience: []string{"consul.io"}}, Cluster: "default", - Name: "webservice", - TaskName: "web", + Name: "${NOMAD_TASK_NAME}service", + TaskName: "web", // note: this doesn't interpolate }, } - identitiesToSign := []*structs.WorkloadIdentity{} - identitiesToSign = append(identitiesToSign, task.Identities...) - for _, service := range task.Services { - identitiesToSign = append(identitiesToSign, service.Identity) - } + // setup mock signer but don't sign identities, as we're going to want them + // interpolated by the WIDMgr + mockSigner := widmgr.NewMockWIDSigner(nil) + db := cstate.NewMemDB(logger) - // setup mock signer and sign the identities - mockSigner := widmgr.NewMockWIDSigner(identitiesToSign) - signedIDs, err := mockSigner.SignIdentities(1, []*structs.WorkloadIdentityRequest{ - { - AllocID: alloc.ID, - WIHandle: structs.WIHandle{ - WorkloadIdentifier: task.Name, - IdentityName: task.Identities[0].Name, - }, - }, - { - AllocID: alloc.ID, - WIHandle: structs.WIHandle{ - WorkloadIdentifier: task.Services[0].Name, - IdentityName: task.Services[0].Identity.Name, - WorkloadType: structs.WorkloadTypeService, - }, - }, - }) - must.NoError(t, err) + // the WIDMgr env builder never has the task available + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") - mockWIDMgr := widmgr.NewMockWIDMgr(signedIDs) + mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger, envBuilder) + mockWIDMgr.SignForTesting() consulConfigs := map[string]*structsc.ConsulConfig{ "default": structsc.DefaultConsulConfig(), } hookResources := cstructs.NewAllocHookResources() + envBuilderFn := func() *taskenv.Builder { + return taskenv.NewBuilder(mock.Node(), alloc, task, "global") + } consulHookCfg := consulHookConfig{ alloc: alloc, @@ -88,6 +75,7 @@ func consulHookTestHarness(t *testing.T) *consulHook { consulConfigs: consulConfigs, consulClientConstructor: consul.NewMockConsulClient, hookResources: hookResources, + envBuilder: envBuilderFn, logger: logger, } return newConsulHook(consulHookCfg) @@ -103,13 +91,11 @@ func Test_consulHook_prepareConsulTokensForTask(t *testing.T) { ti := *task.IdentityHandle(wid) jwt, err := hook.widmgr.Get(ti) must.NoError(t, err) - hashJWT := md5.Sum([]byte(jwt.JWT)) tests := []struct { name string task *structs.Task - tokens map[string]map[string]*consulapi.ACLToken wantErr bool errMsg string expectedTokens map[string]map[string]*consulapi.ACLToken @@ -117,7 +103,6 @@ func Test_consulHook_prepareConsulTokensForTask(t *testing.T) { { name: "empty task", task: nil, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: true, errMsg: "no task specified", expectedTokens: map[string]map[string]*consulapi.ACLToken{}, @@ -125,7 +110,6 @@ func Test_consulHook_prepareConsulTokensForTask(t *testing.T) { { name: "task with signed identity", task: task, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: false, errMsg: "", expectedTokens: map[string]map[string]*consulapi.ACLToken{ @@ -144,7 +128,6 @@ func Test_consulHook_prepareConsulTokensForTask(t *testing.T) { {Name: structs.ConsulTaskIdentityNamePrefix + "_default"}}, Name: "foo", }, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: true, errMsg: "unable to find token for workload", expectedTokens: map[string]map[string]*consulapi.ACLToken{}, @@ -152,13 +135,14 @@ func Test_consulHook_prepareConsulTokensForTask(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := hook.prepareConsulTokensForTask(tt.task, nil, tt.tokens) + tokens := map[string]map[string]*consulapi.ACLToken{} + err := hook.prepareConsulTokensForTask(tt.task, nil, tokens) if tt.wantErr { must.Error(t, err) must.ErrorContains(t, err, tt.errMsg) } else { must.NoError(t, err) - must.Eq(t, tt.tokens, tt.expectedTokens) + must.Eq(t, tt.expectedTokens, tokens) } }) } @@ -170,21 +154,21 @@ func Test_consulHook_prepareConsulTokensForServices(t *testing.T) { hook := consulHookTestHarness(t) task := hook.alloc.LookupTask("web") services := task.Services - + env := hook.envBuilder.Build() hashedJWT := make(map[string]string) + for _, s := range services { - widHandle := *s.IdentityHandle() + widHandle := *s.IdentityHandle(env.ReplaceEnv) jwt, err := hook.widmgr.Get(widHandle) must.NoError(t, err) hash := md5.Sum([]byte(jwt.JWT)) - hashedJWT[s.Name] = hex.EncodeToString(hash[:]) + hashedJWT[widHandle.InterpolatedWorkloadIdentifier] = hex.EncodeToString(hash[:]) } tests := []struct { name string services []*structs.Service - tokens map[string]map[string]*consulapi.ACLToken wantErr bool errMsg string expectedTokens map[string]map[string]*consulapi.ACLToken @@ -192,7 +176,6 @@ func Test_consulHook_prepareConsulTokensForServices(t *testing.T) { { name: "empty services", services: nil, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: false, errMsg: "", expectedTokens: map[string]map[string]*consulapi.ACLToken{}, @@ -200,7 +183,6 @@ func Test_consulHook_prepareConsulTokensForServices(t *testing.T) { { name: "services with signed identity", services: services, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: false, errMsg: "", expectedTokens: map[string]map[string]*consulapi.ACLToken{ @@ -224,7 +206,6 @@ func Test_consulHook_prepareConsulTokensForServices(t *testing.T) { TaskName: "web", }, }, - tokens: map[string]map[string]*consulapi.ACLToken{}, wantErr: true, errMsg: "unable to find token for workload", expectedTokens: map[string]map[string]*consulapi.ACLToken{}, @@ -232,13 +213,14 @@ func Test_consulHook_prepareConsulTokensForServices(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := hook.prepareConsulTokensForServices(tt.services, nil, tt.tokens) + tokens := map[string]map[string]*consulapi.ACLToken{} + err := hook.prepareConsulTokensForServices(tt.services, nil, tokens, env) if tt.wantErr { must.Error(t, err) must.ErrorContains(t, err, tt.errMsg) } else { must.NoError(t, err) - must.Eq(t, tt.tokens, tt.expectedTokens) + must.Eq(t, tt.expectedTokens, tokens) } }) } diff --git a/client/allocrunner/identity_hook_test.go b/client/allocrunner/identity_hook_test.go index 5571079b4ab..bdf0ca27826 100644 --- a/client/allocrunner/identity_hook_test.go +++ b/client/allocrunner/identity_hook_test.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/nomad/ci" "github.com/hashicorp/nomad/client/allocrunner/interfaces" cstate "github.com/hashicorp/nomad/client/state" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/client/widmgr" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" @@ -47,9 +48,12 @@ func TestIdentityHook_Prerun(t *testing.T) { logger := testlog.HCLogger(t) db := cstate.NewMemDB(logger) + // the WIDMgr env builder never has the task available + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") + // setup mock signer and WIDMgr mockSigner := widmgr.NewMockWIDSigner(task.Identities) - mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger) + mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger, envBuilder) allocrunner.widmgr = mockWIDMgr allocrunner.widsigner = mockSigner diff --git a/client/allocrunner/taskrunner/identity_hook_test.go b/client/allocrunner/taskrunner/identity_hook_test.go index 7993ea3ff8a..b3b7fd21dc7 100644 --- a/client/allocrunner/taskrunner/identity_hook_test.go +++ b/client/allocrunner/taskrunner/identity_hook_test.go @@ -81,7 +81,9 @@ func TestIdentityHook_RenewAll(t *testing.T) { logger := testlog.HCLogger(t) db := cstate.NewMemDB(logger) mockSigner := widmgr.NewMockWIDSigner(task.Identities) - mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger) + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") + + mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger, envBuilder) mockWIDMgr.SetMinWait(time.Second) // fast renewals, because the default is 10s mockLifecycle := trtesting.NewMockTaskHooks() @@ -188,7 +190,8 @@ func TestIdentityHook_RenewOne(t *testing.T) { logger := testlog.HCLogger(t) db := cstate.NewMemDB(logger) mockSigner := widmgr.NewMockWIDSigner(task.Identities) - mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger) + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") + mockWIDMgr := widmgr.NewWIDMgr(mockSigner, alloc, db, logger, envBuilder) mockWIDMgr.SetMinWait(time.Second) // fast renewals, because the default is 10s h := &identityHook{ diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index 876061f6bcb..d1030782bbd 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -33,6 +33,7 @@ import ( "github.com/hashicorp/nomad/client/serviceregistration/wrapper" cstate "github.com/hashicorp/nomad/client/state" cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/helper" structsc "github.com/hashicorp/nomad/nomad/structs/config" @@ -136,6 +137,9 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri if vault != nil { vaultFunc = func(_ string) (vaultclient.VaultClient, error) { return vault, nil } } + // the envBuilder for the WIDMgr never has access to the task, so don't + // include it here + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") conf := &Config{ Alloc: alloc, @@ -157,7 +161,7 @@ func testTaskRunnerConfig(t *testing.T, alloc *structs.Allocation, taskName stri ServiceRegWrapper: wrapperMock, Getter: getter.TestSandbox(t), Wranglers: proclib.MockWranglers(t), - WIDMgr: widmgr.NewWIDMgr(widsigner, alloc, db, logger), + WIDMgr: widmgr.NewWIDMgr(widsigner, alloc, db, logger, envBuilder), AllocHookResources: cstructs.NewAllocHookResources(), } diff --git a/client/allocrunner/taskrunner/vault_hook_test.go b/client/allocrunner/taskrunner/vault_hook_test.go index fb086592f78..36e62e44233 100644 --- a/client/allocrunner/taskrunner/vault_hook_test.go +++ b/client/allocrunner/taskrunner/vault_hook_test.go @@ -99,8 +99,8 @@ func setupTestVaultHook(t *testing.T, config *vaultHookConfig) *vaultHook { if config.widmgr == nil { db := cstate.NewMemDB(config.logger) signer := widmgr.NewMockWIDSigner(config.task.Identities) - - config.widmgr = widmgr.NewWIDMgr(signer, config.alloc, db, config.logger) + envBuilder := taskenv.NewBuilder(mock.Node(), config.alloc, nil, "global") + config.widmgr = widmgr.NewWIDMgr(signer, config.alloc, db, config.logger, envBuilder) err := config.widmgr.Run() must.NoError(t, err) } diff --git a/client/taskenv/services.go b/client/taskenv/services.go index 0b2c57c844e..ddbc6137697 100644 --- a/client/taskenv/services.go +++ b/client/taskenv/services.go @@ -221,3 +221,16 @@ func interpolateTaskResources(taskEnv *TaskEnv, resources *structs.Resources) { } } } + +// InterpolateWIHandle returns a copy of the WIHandle with only the +// InterpolatedWorkloadIdentifier field interpolated. The original +// WorkloadIdentifier should never be altered so the server can find +// uninterpolated services associated with the handle. +func InterpolateWIHandle(taskEnv *TaskEnv, orig structs.WIHandle) structs.WIHandle { + return structs.WIHandle{ + IdentityName: orig.IdentityName, + WorkloadIdentifier: orig.WorkloadIdentifier, + WorkloadType: orig.WorkloadType, + InterpolatedWorkloadIdentifier: taskEnv.ReplaceEnv(orig.WorkloadIdentifier), + } +} diff --git a/client/widmgr/mock.go b/client/widmgr/mock.go index 90469bb1792..0202c92910d 100644 --- a/client/widmgr/mock.go +++ b/client/widmgr/mock.go @@ -113,35 +113,3 @@ func (m *MockWIDSigner) SignIdentities(minIndex uint64, req []*structs.WorkloadI } return swids, nil } - -// MockWIDMgr mocks IdentityManager interface allowing to only get identities -// signed by the mock signer. -type MockWIDMgr struct { - swids map[structs.WIHandle]*structs.SignedWorkloadIdentity -} - -func NewMockWIDMgr(swids []*structs.SignedWorkloadIdentity) *MockWIDMgr { - swidmap := map[structs.WIHandle]*structs.SignedWorkloadIdentity{} - for _, id := range swids { - swidmap[id.WIHandle] = id - } - return &MockWIDMgr{swids: swidmap} -} - -// Run does not run a renewal loop in this mock -func (m MockWIDMgr) Run() error { return nil } - -func (m MockWIDMgr) Get(id structs.WIHandle) (*structs.SignedWorkloadIdentity, error) { - sid, ok := m.swids[id] - if !ok { - return nil, fmt.Errorf("unable to find token for workload %q and identity %q", id.WorkloadIdentifier, id.IdentityName) - } - return sid, nil -} - -// Watch does not do anything, this mock doesn't support watching. -func (m MockWIDMgr) Watch(identity structs.WIHandle) (<-chan *structs.SignedWorkloadIdentity, func()) { - return nil, nil -} - -func (m MockWIDMgr) Shutdown() {} diff --git a/client/widmgr/widmgr.go b/client/widmgr/widmgr.go index 0e7420d6323..c0a043d6f9a 100644 --- a/client/widmgr/widmgr.go +++ b/client/widmgr/widmgr.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/go-hclog" cstate "github.com/hashicorp/nomad/client/state" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" ) @@ -53,13 +54,16 @@ type WIDMgr struct { logger hclog.Logger } -func NewWIDMgr(signer IdentitySigner, a *structs.Allocation, db cstate.StateDB, logger hclog.Logger) *WIDMgr { +func NewWIDMgr(signer IdentitySigner, a *structs.Allocation, db cstate.StateDB, logger hclog.Logger, envBuilder *taskenv.Builder) *WIDMgr { widspecs := map[structs.WIHandle]*structs.WorkloadIdentity{} tg := a.Job.LookupTaskGroup(a.TaskGroup) + allocEnv := envBuilder.Build() + for _, service := range tg.Services { if service.Identity != nil { - widspecs[*service.IdentityHandle()] = service.Identity + handle := *service.IdentityHandle(allocEnv.ReplaceEnv) + widspecs[handle] = service.Identity } } @@ -69,9 +73,12 @@ func NewWIDMgr(signer IdentitySigner, a *structs.Allocation, db cstate.StateDB, widspecs[*task.IdentityHandle(id)] = id } + // update the builder for this task + taskEnv := envBuilder.UpdateTask(a, task).Build() for _, service := range task.Services { if service.Identity != nil { - widspecs[*service.IdentityHandle()] = service.Identity + handle := *service.IdentityHandle(taskEnv.ReplaceEnv) + widspecs[handle] = service.Identity } } } @@ -239,6 +246,12 @@ func (m *WIDMgr) restoreStoredIdentities() (bool, error) { return hasExpired, nil } +// SignForTesting signs all the identities in m.widspec, typically with the mock +// signer. This should only be used for testing downstream hooks. +func (m *WIDMgr) SignForTesting() { + m.getInitialIdentities() +} + // getInitialIdentities fetches all signed identities or returns an error. It // should be run once when the WIDMgr first runs. func (m *WIDMgr) getInitialIdentities() error { diff --git a/client/widmgr/widmgr_test.go b/client/widmgr/widmgr_test.go index 2617747f274..8133840a660 100644 --- a/client/widmgr/widmgr_test.go +++ b/client/widmgr/widmgr_test.go @@ -8,6 +8,7 @@ import ( "time" cstate "github.com/hashicorp/nomad/client/state" + "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -29,9 +30,10 @@ func TestWIDMgr_Restore(t *testing.T) { } alloc.Job.TaskGroups[0].Tasks[0].Services[0].Identity = widSpecs[0] alloc.Job.TaskGroups[0].Tasks[0].Identities = widSpecs[1:] + envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, "global") signer := NewMockWIDSigner(widSpecs) - mgr := NewWIDMgr(signer, alloc, db, logger) + mgr := NewWIDMgr(signer, alloc, db, logger, envBuilder) // restore, but we haven't previously saved to the db hasExpired, err := mgr.restoreStoredIdentities() @@ -51,7 +53,8 @@ func TestWIDMgr_Restore(t *testing.T) { signer.mockNow = time.Now().Add(-1 * time.Minute) widSpecs[2].TTL = time.Second signer.setWIDs(widSpecs) - wiHandle := service.IdentityHandle() + + wiHandle := service.IdentityHandle(envBuilder.Build().ReplaceEnv) mgr.widSpecs[*wiHandle].TTL = time.Second // force a re-sign to re-populate the lastToken and save to the db diff --git a/nomad/alloc_endpoint.go b/nomad/alloc_endpoint.go index 4dd75662545..83aee52191c 100644 --- a/nomad/alloc_endpoint.go +++ b/nomad/alloc_endpoint.go @@ -637,13 +637,13 @@ func (a *Alloc) signServices( // services can be on the level of task groups or tasks for _, tg := range job.TaskGroups { for _, service := range tg.Services { - if service.IdentityHandle().Equal(wid) { + if service.IdentityHandle(nil).Equal(wid) { return true, a.signIdentities(alloc, service.Identity, idReq, reply, now) } } for _, task := range tg.Tasks { for _, service := range task.Services { - if service.IdentityHandle().Equal(wid) { + if service.IdentityHandle(nil).Equal(wid) { return true, a.signIdentities(alloc, service.Identity, idReq, reply, now) } } diff --git a/nomad/structs/services.go b/nomad/structs/services.go index 6fc1a3d9e03..e18b28eef9f 100644 --- a/nomad/structs/services.go +++ b/nomad/structs/services.go @@ -802,15 +802,21 @@ func (s *Service) MakeUniqueIdentityName() string { return fmt.Sprintf("%s_%v-%v", prefix, s.Name, s.PortLabel) } +type envReplacer func(string) string + // IdentityHandle returns a WorkloadIdentityHandle which is a pair of service // identity name and service name. -func (s *Service) IdentityHandle() *WIHandle { +func (s *Service) IdentityHandle(replace envReplacer) *WIHandle { if s.Identity != nil { - return &WIHandle{ + wi := &WIHandle{ IdentityName: s.Identity.Name, WorkloadIdentifier: s.Name, WorkloadType: WorkloadTypeService, } + if replace != nil { + wi.InterpolatedWorkloadIdentifier = replace(s.Name) + } + return wi } return nil } diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index e527015b778..daeae2687b0 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -11378,6 +11378,9 @@ func NewIdentityClaims(job *Job, alloc *Allocation, wihandle *WIHandle, wid *Wor switch wihandle.WorkloadType { case WorkloadTypeService: serviceName := wihandle.WorkloadIdentifier + if wihandle.InterpolatedWorkloadIdentifier != "" { + serviceName = wihandle.InterpolatedWorkloadIdentifier + } claims.ServiceName = serviceName // Find task name if this is a task service. diff --git a/nomad/structs/structs_test.go b/nomad/structs/structs_test.go index 7060d95eba0..5ef71d3c1d7 100644 --- a/nomad/structs/structs_test.go +++ b/nomad/structs/structs_test.go @@ -8418,7 +8418,7 @@ func TestNewIdentityClaims(t *testing.T) { name: path, group: tg.Name, wid: s.Identity, - wiHandle: s.IdentityHandle(), + wiHandle: s.IdentityHandle(nil), expectedClaims: expectedClaims[path], }) } @@ -8447,7 +8447,7 @@ func TestNewIdentityClaims(t *testing.T) { name: path, group: tg.Name, wid: s.Identity, - wiHandle: s.IdentityHandle(), + wiHandle: s.IdentityHandle(nil), expectedClaims: expectedClaims[path], }) } diff --git a/nomad/structs/workload_id.go b/nomad/structs/workload_id.go index 7aa4ff80109..165b8701135 100644 --- a/nomad/structs/workload_id.go +++ b/nomad/structs/workload_id.go @@ -310,12 +310,18 @@ type WIHandle struct { // WorkloadIdentifier is either a ServiceName or a TaskName WorkloadIdentifier string WorkloadType WorkloadType + + // InterpolatedWorkloadIdentifier is the WorkloadIdentifier, interpolated by + // the client. It is used only to provide an override for the identity + // claims + InterpolatedWorkloadIdentifier string } func (w *WIHandle) Equal(o WIHandle) bool { if w == nil { return false } + // note: we're intentionally ignoring InterpolatedWorkloadIdentifier here return w.IdentityName == o.IdentityName && w.WorkloadIdentifier == o.WorkloadIdentifier && w.WorkloadType == o.WorkloadType