Skip to content

Commit

Permalink
incusd/instance: Keep track of API operations
Browse files Browse the repository at this point in the history 
Signed-off-by: Stéphane Graber <[email protected]>
  • Loading branch information
@stgraber
stgraber committed Aug 5, 2024
1 parent 2964b8c commit 392077c
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 26 deletions.
1 change: 1 addition & 0 deletions cmd/incusd/instance_put.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,7 @@ func instancePut(d *Daemon, r *http.Request) response.Response {

// Update container configuration
do = func(op *operations.Operation) error {
inst.SetOperation(op)
defer unlock()

args := db.InstanceArgs{
Expand Down
8 changes: 5 additions & 3 deletions internal/server/instance/drivers/driver_common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,8 @@ func (d *common) Snapshots() ([]instance.Instance, error) {
return nil, err
}

snapInst.SetOperation(d.op)

snapshots = append(snapshots, instance.Instance(snapInst))
}

Expand Down Expand Up @@ -533,7 +535,7 @@ func (d *common) expandConfig() error {
// restartCommon handles the common part of instance restarts.
func (d *common) restartCommon(inst instance.Instance, timeout time.Duration) error {
// Setup a new operation for the stop/shutdown phase.
op, err := operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestart, true, true)
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, true, true)
if err != nil {
return fmt.Errorf("Create restart operation: %w", err)
}
Expand Down Expand Up @@ -597,7 +599,7 @@ func (d *common) restartCommon(inst instance.Instance, timeout time.Duration) er
}

// Setup a new operation for the start phase.
op, err = operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestart, true, true)
op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, true, true)
if err != nil {
return fmt.Errorf("Create restart (for start) operation: %w", err)
}
Expand Down Expand Up @@ -902,7 +904,7 @@ func (d *common) onStopOperationSetup(target string) (*operationlock.InstanceOpe
action = operationlock.ActionRestart
}

op, err = operationlock.Create(d.Project().Name, d.Name(), action, false, false)
op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, action, false, false)
if err != nil {
return nil, fmt.Errorf("Failed creating %q operation: %w", action, err)
}
Expand Down
23 changes: 14 additions & 9 deletions internal/server/instance/drivers/driver_lxc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2628,7 +2628,7 @@ func (d *lxc) Start(stateful bool) error {
}

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -2882,7 +2882,7 @@ func (d *lxc) Stop(stateful bool) error {
}

// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -3058,7 +3058,7 @@ func (d *lxc) Shutdown(timeout time.Duration) error {
}

// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -3220,6 +3220,11 @@ func (d *lxc) onStop(args map[string]string) error {
d.fromHook = false
err = nil

// Set operation if missing.
if d.op == nil {
d.op = op.GetOperation()
}

// Unlock on return
defer op.Done(nil)

Expand Down Expand Up @@ -3771,7 +3776,7 @@ func (d *lxc) Snapshot(name string, expiry time.Time, stateful bool) error {
func (d *lxc) Restore(sourceContainer instance.Instance, stateful bool) error {
var ctxMap logger.Ctx

op, err := operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestore, false, false)
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
Expand Down Expand Up @@ -3817,7 +3822,7 @@ func (d *lxc) Restore(sourceContainer instance.Instance, stateful bool) error {
}

// Refresh the operation as that one is now complete.
op, err = operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestore, false, false)
op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
Expand Down Expand Up @@ -3992,7 +3997,7 @@ func (d *lxc) Delete(force bool) error {
defer unlock()

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionDelete, nil, false, false)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionDelete, nil, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance delete operation: %w", err)
}
Expand Down Expand Up @@ -4365,7 +4370,7 @@ func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error {
defer unlock()

// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionCreate, operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionCreate, operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance update operation: %w", err)
}
Expand Down Expand Up @@ -5505,7 +5510,7 @@ func (d *lxc) MigrateSend(args instance.MigrateSendArgs) error {
defer d.logger.Debug("Migration send stopped")

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionMigrate, nil, false, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionMigrate, nil, false, true)
if err != nil {
return err
}
Expand Down Expand Up @@ -8408,7 +8413,7 @@ func (d *lxc) LockExclusive() (*operationlock.InstanceOperation, error) {
}

// Prevent concurrent operations the instance.
op, err := operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionCreate, false, false)
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionCreate, false, false)
if err != nil {
return nil, err
}
Expand Down
23 changes: 14 additions & 9 deletions internal/server/instance/drivers/driver_qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,11 @@ func (d *qemu) onStop(target string) error {
// Unlock on return
defer op.Done(nil)

// Set operation if missing.
if d.op == nil {
d.op = op.GetOperation()
}

// Wait for QEMU process to end (to avoiding racing start when restarting).
// Wait up to 5 minutes to allow for flushing any pending data to disk.
d.logger.Debug("Waiting for VM process to finish")
Expand Down Expand Up @@ -767,7 +772,7 @@ func (d *qemu) Shutdown(timeout time.Duration) error {
// Allow reuse when creating a new stop operation. This allows the Stop() function to inherit operation.
// Allow reuse of a reusable ongoing stop operation as Shutdown() may be called earlier, which allows reuse
// of its operations. This allow for multiple Shutdown() attempts.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -1211,7 +1216,7 @@ func (d *qemu) start(stateful bool, op *operationlock.InstanceOperation) error {

// Setup a new operation if needed.
if op == nil {
op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -4750,7 +4755,7 @@ func (d *qemu) Stop(stateful bool) error {
// Don't allow reuse when creating a new stop operation. This prevents other operations from intefering.
// Allow reuse of a reusable ongoing stop operation as Shutdown() may be called first, which allows reuse
// of its operations. This allow for Stop() to inherit from Shutdown() where instance is stuck.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
Expand Down Expand Up @@ -4962,7 +4967,7 @@ func (d *qemu) Snapshot(name string, expiry time.Time, stateful bool) error {

// Restore restores an instance snapshot.
func (d *qemu) Restore(source instance.Instance, stateful bool) error {
op, err := operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestore, false, false)
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
Expand Down Expand Up @@ -5012,7 +5017,7 @@ func (d *qemu) Restore(source instance.Instance, stateful bool) error {
}

// Refresh the operation as that one is now complete.
op, err = operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionRestore, false, false)
op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
Expand Down Expand Up @@ -5279,7 +5284,7 @@ func (d *qemu) Update(args db.InstanceArgs, userRequested bool) error {
defer unlock()

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance update operation: %w", err)
}
Expand Down Expand Up @@ -6007,7 +6012,7 @@ func (d *qemu) Delete(force bool) error {
defer unlock()

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionDelete, nil, false, false)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionDelete, nil, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance delete operation: %w", err)
}
Expand Down Expand Up @@ -6431,7 +6436,7 @@ func (d *qemu) MigrateSend(args instance.MigrateSendArgs) error {
}

// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), operationlock.ActionMigrate, nil, false, true)
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionMigrate, nil, false, true)
if err != nil {
return err
}
Expand Down Expand Up @@ -7955,7 +7960,7 @@ func (d *qemu) LockExclusive() (*operationlock.InstanceOperation, error) {
}

// Prevent concurrent operations the instance.
op, err := operationlock.Create(d.Project().Name, d.Name(), operationlock.ActionCreate, false, false)
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionCreate, false, false)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion internal/server/instance/instance_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -852,7 +852,7 @@ func CreateInternal(s *state.State, args db.InstanceArgs, op *operations.Operati
}

// Prevent concurrent create requests for same instance.
opl, err := operationlock.Create(args.Project, args.Name, operationlock.ActionCreate, false, false)
opl, err := operationlock.Create(args.Project, args.Name, op, operationlock.ActionCreate, false, false)
if err != nil {
return nil, nil, nil, err
}
Expand Down
21 changes: 17 additions & 4 deletions internal/server/instance/operationlock/operationlock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"sync"

"github.com/lxc/incus/v6/internal/server/operations"
"github.com/lxc/incus/v6/internal/server/project"
"github.com/lxc/incus/v6/shared/logger"
)
Expand Down Expand Up @@ -52,13 +53,14 @@ type InstanceOperation struct {
instanceName string
reusable bool
instanceInitiated bool
op *operations.Operation
}

// Create creates a new operation lock for an Instance if one does not already exist and returns it.
// The lock will be released after TimeoutDefault or when Done() is called, which ever occurs first.
// If createReusuable is set as true then future lock attempts can specify the reuseExisting argument as true
// which will then trigger a reset of the timeout to TimeoutDefault on the existing lock and return it.
func Create(projectName string, instanceName string, action Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) {
func Create(projectName string, instanceName string, apiOp *operations.Operation, action Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) {
if projectName == "" || instanceName == "" {
return nil, fmt.Errorf("Invalid project or instance name")
}
Expand All @@ -85,6 +87,7 @@ func Create(projectName string, instanceName string, action Action, createReusua
op.action = action
op.reusable = createReusuable
op.chanDone = make(chan error)
op.op = apiOp

instanceOperations[opKey] = op
logger.Debug("Instance operation lock created", logger.Ctx{"project": op.projectName, "instance": op.instanceName, "action": op.action, "reusable": op.reusable})
Expand All @@ -106,12 +109,12 @@ func Create(projectName string, instanceName string, action Action, createReusua
//
// Returns ErrWaitedForMatching if it waited for a matching operation to finish and it's finished successfully and
// so didn't return create a new operation.
func CreateWaitGet(projectName string, instanceName string, action Action, inheritableActions []Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) {
func CreateWaitGet(projectName string, instanceName string, apiOp *operations.Operation, action Action, inheritableActions []Action, createReusuable bool, reuseExisting bool) (*InstanceOperation, error) {
op := Get(projectName, instanceName)

// No existing operation, call create.
if op == nil {
op, err := Create(projectName, instanceName, action, createReusuable, reuseExisting)
op, err := Create(projectName, instanceName, apiOp, action, createReusuable, reuseExisting)
return op, err
}

Expand All @@ -136,7 +139,7 @@ func CreateWaitGet(projectName string, instanceName string, action Action, inher
}

// Send the rest to Create to try and create a new operation.
op, err := Create(projectName, instanceName, action, createReusuable, reuseExisting)
op, err := Create(projectName, instanceName, apiOp, action, createReusuable, reuseExisting)

return op, err
}
Expand Down Expand Up @@ -230,3 +233,13 @@ func (op *InstanceOperation) GetInstanceInitiated() bool {

return op.instanceInitiated
}

// GetOperation gets the API background operation.
func (op *InstanceOperation) GetOperation() *operations.Operation {
// This function can be called on a nil struct.
if op == nil {
return nil
}

return op.op
}

0 comments on commit 392077c

Please sign in to comment.