incusd: Add support for local live-migration between storage pools

Signed-off-by: Piotr Resztak <[email protected]>

Author: presztak

cmd/incusd/instance_post.go

@@ -19,6 +19,7 @@ import (
 	dbCluster "github.com/lxc/incus/v6/internal/server/db/cluster"
 	"github.com/lxc/incus/v6/internal/server/db/operationtype"
 	"github.com/lxc/incus/v6/internal/server/instance"
+	"github.com/lxc/incus/v6/internal/server/instance/instancetype"
 	"github.com/lxc/incus/v6/internal/server/operations"
 	"github.com/lxc/incus/v6/internal/server/project"
 	"github.com/lxc/incus/v6/internal/server/request"
@@ -234,9 +235,15 @@ func instancePost(d *Daemon, r *http.Request) response.Response {
 				return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved statelessly"))
 			}
 
-			// Storage pool changes require a stopped instance.
+			// Storage pool changes require a target flag.
 			if req.Pool != "" {
-				return response.BadRequest(fmt.Errorf("Instance must be stopped to be moved across storage pools"))
+				if inst.Type() != instancetype.VM {
+					return response.BadRequest(fmt.Errorf("Storage pool change supported only by virtual-machines"))
+				}
+
+				if target == "" {
+					return response.BadRequest(fmt.Errorf("Storage pool can be specified only together with target flag"))
+				}
 			}
 
 			// Project changes require a stopped instance.
@@ -433,7 +440,7 @@ func instancePost(d *Daemon, r *http.Request) response.Response {
 	}
 
 	// Cross-server instance migration.
-	ws, err := newMigrationSource(inst, req.Live, req.InstanceOnly, req.AllowInconsistent, "", req.Target)
+	ws, err := newMigrationSource(inst, req.Live, req.InstanceOnly, req.AllowInconsistent, "", "", req.Target)
 	if err != nil {
 		return response.InternalError(err)
 	}
@@ -476,6 +483,11 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
 		return fmt.Errorf("Failed loading instance storage pool: %w", err)
 	}
 
+	// Check that we're not requested to move to the same storage pool we're currently use.
+	if req.Pool != "" && req.Pool == sourcePool.Name() {
+		return fmt.Errorf("Requested storage pool is the same as current pool")
+	}
+
 	// Get the DB volume type for the instance.
 	volType, err := storagePools.InstanceTypeToVolumeType(inst.Type())
 	if err != nil {
@@ -593,8 +605,8 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
 		req.Name = ""
 	}
 
-	// Handle pool and project moves.
-	if req.Project != "" || req.Pool != "" {
+	// Handle pool and project moves for stopped instances.
+	if (req.Project != "" || req.Pool != "") && !req.Live {
 		// Get a local client.
 		args := &incus.ConnectionArgs{
 			SkipGetServer: true,
@@ -756,7 +768,7 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
 		req.Project = ""
 	}
 
-	// Handle remote migrations (location changes).
+	// Handle remote migrations (location and storage pool changes).
 	if targetMemberInfo != nil && inst.Location() != targetMemberInfo.Name {
 		// Get the client.
 		networkCert := s.Endpoints.NetworkCert()
@@ -794,7 +806,7 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
 		}
 
 		// Setup a new migration source.
-		sourceMigration, err := newMigrationSource(inst, req.Live, false, req.AllowInconsistent, inst.Name(), nil)
+		sourceMigration, err := newMigrationSource(inst, req.Live, false, req.AllowInconsistent, inst.Name(), req.Pool, nil)
 		if err != nil {
 			return fmt.Errorf("Failed setting up instance migration on source: %w", err)
 		}
@@ -918,8 +930,9 @@ func migrateInstance(ctx context.Context, s *state.State, inst instance.Instance
 			return err
 		}
 
-		// Cleanup instance paths on source member if using remote shared storage.
-		if sourcePool.Driver().Info().Remote {
+		// Cleanup instance paths on source member if using remote shared storage
+		// and there was no storage pool change.
+		if sourcePool.Driver().Info().Remote && req.Pool == "" {
 			err = sourcePool.CleanupInstancePaths(inst, nil)
 			if err != nil {
 				return fmt.Errorf("Failed cleaning up instance paths on source member: %w", err)

cmd/incusd/instance_snapshot.go

@@ -676,7 +676,7 @@ func snapshotPost(s *state.State, r *http.Request, snapInst instance.Instance) r
 			}
 		}
 
-		ws, err := newMigrationSource(snapInst, reqNew.Live, true, false, "", req.Target)
+		ws, err := newMigrationSource(snapInst, reqNew.Live, true, false, "", "", req.Target)
 		if err != nil {
 			return response.SmartError(err)
 		}

cmd/incusd/instances_post.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -363,6 +364,7 @@ func createFromMigration(ctx context.Context, s *state.State, r *http.Request, p
 		ClusterMoveSourceName: clusterMoveSourceName,
 		Refresh:               req.Source.Refresh,
 		RefreshExcludeOlder:   req.Source.RefreshExcludeOlder,
+		StoragePool:           storagePool,
 	}
 
 	sink, err := newMigrationSink(&migrationArgs)
@@ -388,6 +390,48 @@ func createFromMigration(ctx context.Context, s *state.State, r *http.Request, p
 		}
 
 		instOp.Done(nil) // Complete operation that was created earlier, to release lock.
+
+		// Update root device for instance.
+		err = s.DB.Cluster.Transaction(context.Background(), func(ctx context.Context, tx *db.ClusterTx) error {
+			devs := inst.LocalDevices().CloneNative()
+			rootDevKey, _, err := internalInstance.GetRootDiskDevice(devs)
+			if err != nil {
+				if !errors.Is(err, internalInstance.ErrNoRootDisk) {
+					return err
+				}
+
+				rootDev := map[string]string{}
+				rootDev["type"] = "disk"
+				rootDev["path"] = "/"
+				rootDev["pool"] = storagePool
+
+				devs["root"] = rootDev
+			} else {
+				// Apply the override.
+				devs[rootDevKey]["pool"] = storagePool
+			}
+
+			devices, err := dbCluster.APIToDevices(devs)
+			if err != nil {
+				return err
+			}
+
+			id, err := dbCluster.GetInstanceID(ctx, tx.Tx(), inst.Project().Name, inst.Name())
+			if err != nil {
+				return fmt.Errorf("Failed to get ID of moved instance: %w", err)
+			}
+
+			err = dbCluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(id), devices)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
 		runRevert.Success()
 
 		return instanceCreateFinish(s, req, args, op)

cmd/incusd/migrate.go

@@ -37,6 +37,7 @@ type migrationFields struct {
 	// storage specific fields
 	volumeOnly        bool
 	allowInconsistent bool
+	storagePool       string
 }
 
 func (c *migrationFields) send(m proto.Message) error {
@@ -207,8 +208,9 @@ type migrationSinkArgs struct {
 	Snapshots             []*migration.Snapshot
 
 	// Storage specific fields
-	VolumeOnly bool
-	VolumeSize int64
+	StoragePool string
+	VolumeOnly  bool
+	VolumeSize  int64
 
 	// Transport specific fields
 	RsyncFeatures []string

cmd/incusd/migrate_instance.go

@@ -20,11 +20,12 @@ import (
 	"github.com/lxc/incus/v6/shared/logger"
 )
 
-func newMigrationSource(inst instance.Instance, stateful bool, instanceOnly bool, allowInconsistent bool, clusterMoveSourceName string, pushTarget *api.InstancePostTarget) (*migrationSourceWs, error) {
+func newMigrationSource(inst instance.Instance, stateful bool, instanceOnly bool, allowInconsistent bool, clusterMoveSourceName string, storagePool string, pushTarget *api.InstancePostTarget) (*migrationSourceWs, error) {
 	ret := migrationSourceWs{
 		migrationFields: migrationFields{
 			instance:          inst,
 			allowInconsistent: allowInconsistent,
+			storagePool:       storagePool,
 		},
 		clusterMoveSourceName: clusterMoveSourceName,
 	}
@@ -144,6 +145,7 @@ func (s *migrationSourceWs) Do(state *state.State, migrateOp *operations.Operati
 				}
 			},
 			ClusterMoveSourceName: s.clusterMoveSourceName,
+			StoragePool:           s.storagePool,
 		},
 		AllowInconsistent: s.allowInconsistent,
 	})
@@ -164,6 +166,7 @@ func newMigrationSink(args *migrationSinkArgs) (*migrationSink, error) {
 			instance:     args.Instance,
 			instanceOnly: args.InstanceOnly,
 			live:         args.Live,
+			storagePool:  args.StoragePool,
 		},
 		url:                   args.URL,
 		clusterMoveSourceName: args.ClusterMoveSourceName,
@@ -275,6 +278,7 @@ func (c *migrationSink) Do(state *state.State, instOp *operationlock.InstanceOpe
 				}
 			},
 			ClusterMoveSourceName: c.clusterMoveSourceName,
+			StoragePool:           c.storagePool,
 		},
 		InstanceOperation:   instOp,
 		Refresh:             c.refresh,