implement soft-delete functionality for deployments

pkg/debugcmd/deps.go

@@ -11,12 +11,13 @@ import (
 	"github.com/threefoldtech/zosbase/pkg/gridtypes/zos"
 )
 
-// Provision is the subset of the provision zbus interface used by debug commands.
-type Provision interface {
-	ListTwins(ctx context.Context) ([]uint32, error)
-	List(ctx context.Context, twin uint32) ([]gridtypes.Deployment, error)
-	Get(ctx context.Context, twin uint32, contract uint64) (gridtypes.Deployment, error)
-	Changes(ctx context.Context, twin uint32, contract uint64) ([]gridtypes.Workload, error)
+// Storage is the subset of the provision interface used by debug commands.
+type Storage interface {
+	GetDeployment(ctx context.Context, twin uint32, contractID uint64) (gridtypes.Deployment, error)
+	GetDeployments(ctx context.Context, twin uint32) ([]gridtypes.Deployment, error)
+	GetTwins(ctx context.Context) ([]uint32, error)
+	Changes(ctx context.Context, twin uint32, contractID uint64) ([]gridtypes.Workload, error)
+	GetWorkload(ctx context.Context, twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error)
 }
 
 // VM is the subset of the vmd zbus interface used by debug commands.
@@ -33,9 +34,9 @@ type Network interface {
 }
 
 type Deps struct {
-	Provision Provision
-	VM        VM
-	Network   Network
+	VM      VM
+	Network Network
+	Storage Storage
 }
 
 // ParseDeploymentID parses a deployment identifier in the format "twin-id:contract-id"

pkg/debugcmd/get.go

@@ -29,8 +29,7 @@ func Get(ctx context.Context, deps Deps, req GetRequest) (GetResponse, error) {
 		return GetResponse{}, err
 	}
 
-	// TODO: only return active deployment. should return all
-	deployment, err := deps.Provision.Get(ctx, twinID, contractID)
+	deployment, err := deps.Storage.GetDeployment(ctx, twinID, contractID)
 	if err != nil {
 		return GetResponse{}, err
 	}

pkg/debugcmd/health.go

@@ -69,7 +69,7 @@ func Health(ctx context.Context, deps Deps, req HealthRequest) (HealthResponse,
 	}
 
 	if req.Deployment != "" {
-		deployment, err := deps.Provision.Get(ctx, twinID, contractID)
+		deployment, err := deps.Storage.GetDeployment(ctx, twinID, contractID)
 		if err != nil {
 			return HealthResponse{}, fmt.Errorf("failed to get deployment: %w", err)
 		}

pkg/debugcmd/history.go

@@ -39,8 +39,7 @@ func History(ctx context.Context, deps Deps, req HistoryRequest) (HistoryRespons
 		return HistoryResponse{}, err
 	}
 
-	// TODO: only return history for active deployment.
-	history, err := deps.Provision.Changes(ctx, twinID, contractID)
+	history, err := deps.Storage.Changes(ctx, twinID, contractID)
 	if err != nil {
 		return HistoryResponse{}, err
 	}

pkg/debugcmd/info.go

@@ -41,20 +41,11 @@ func Info(ctx context.Context, deps Deps, req InfoRequest) (InfoResponse, error)
 		return InfoResponse{}, err
 	}
 
-	deployment, err := deps.Provision.Get(ctx, twinID, contractID)
+	workload, found, err := deps.Storage.GetWorkload(ctx, twinID, contractID, gridtypes.Name(req.Workload))
 	if err != nil {
-		return InfoResponse{}, fmt.Errorf("failed to get deployment: %w", err)
+		return InfoResponse{}, fmt.Errorf("failed to get workload: %w", err)
 	}
-
-	var workload *gridtypes.Workload
-	for i := range deployment.Workloads {
-		if string(deployment.Workloads[i].Name) == req.Workload {
-			workload = &deployment.Workloads[i]
-			break
-		}
-	}
-
-	if workload == nil {
+	if !found {
 		return InfoResponse{}, fmt.Errorf("workload '%s' not found in deployment", req.Workload)
 	}
 
@@ -70,7 +61,7 @@ func Info(ctx context.Context, deps Deps, req InfoRequest) (InfoResponse, error)
 	case zos.ZMachineType, zos.ZMachineLightType:
 		return handleZMachineInfo(ctx, deps, workloadID.String(), req.Verbose, resp)
 	case zos.NetworkType, zos.NetworkLightType:
-		return handleNetworkInfo(ctx, deps, twinID, workload, resp)
+		return handleNetworkInfo(ctx, deps, twinID, &workload, resp)
 	default:
 		return InfoResponse{}, fmt.Errorf("workload type '%s' not supported for info command", workload.Type)
 	}

pkg/debugcmd/list.go

@@ -40,19 +40,16 @@ func ParseListRequest(payload []byte) (ListRequest, error) {
 func List(ctx context.Context, deps Deps, req ListRequest) (ListResponse, error) {
 	twins := []uint32{req.TwinID}
 	if req.TwinID == 0 {
-		allTwins, err := deps.Provision.ListTwins(ctx)
+		var err error
+		twins, err = deps.Storage.GetTwins(ctx)
 		if err != nil {
 			return ListResponse{}, err
 		}
-
-		twins = allTwins
 	}
 
 	deployments := make([]ListDeployment, 0)
 	for _, twin := range twins {
-		// TODO: this is only returning active deployments,
-		// cause when deprovision the workload is removed from the key list.
-		deploymentList, err := deps.Provision.List(ctx, twin)
+		deploymentList, err := deps.Storage.GetDeployments(ctx, twin)
 		if err != nil {
 			return ListResponse{}, err
 		}

pkg/network/nr/net_resource.go

@@ -606,7 +606,7 @@ func (nr *NetResource) ConfigureWG(privateKey string) error {
 // Delete removes all the interfaces and namespaces created by the Create method
 func (nr *NetResource) Delete() error {
 	log.Info().Str("network-id", nr.ID()).Str("subnet", nr.resource.Subnet.String()).Msg("deleting network resource")
-	
+
 	netnsName, err := nr.Namespace()
 	if err != nil {
 		return err

pkg/provision.go

@@ -21,6 +21,15 @@ type Provision interface {
 	ListTwins() ([]uint32, error)
 	ListPublicIPs() ([]string, error)
 	ListPrivateIPs(twin uint32, network gridtypes.Name) ([]string, error)
+	// GetDeployment returns a deployment including soft-deleted ones.
+	GetDeployment(twin uint32, contractID uint64) (gridtypes.Deployment, error)
+	// GetDeployments returns all deployments for a twin including soft-deleted ones.
+	GetDeployments(twin uint32) ([]gridtypes.Deployment, error)
+	// GetTwins returns all twins including soft-deleted ones.
+	GetTwins() ([]uint32, error)
+	// GetWorkload returns the latest workload state by name including soft-deleted ones.
+	// Returns (workload, true, nil) if found, (zero, false, nil) if not found.
+	GetWorkload(twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error)
 }
 
 type Statistics interface {

pkg/provision/debug.go

@@ -0,0 +1,44 @@
+package provision
+
+import "github.com/threefoldtech/zosbase/pkg/gridtypes"
+
+func (e *NativeEngine) GetDeployment(twin uint32, contractID uint64) (gridtypes.Deployment, error) {
+	return e.storage.Get(twin, contractID, WithDeleted())
+}
+
+func (e *NativeEngine) GetDeployments(twin uint32) ([]gridtypes.Deployment, error) {
+	ids, err := e.storage.ByTwin(twin, WithDeleted())
+	if err != nil {
+		return nil, err
+	}
+	deployments := make([]gridtypes.Deployment, 0, len(ids))
+	for _, id := range ids {
+		dep, err := e.storage.Get(twin, id, WithDeleted())
+		if err != nil {
+			return nil, err
+		}
+		deployments = append(deployments, dep)
+	}
+	return deployments, nil
+}
+
+func (e *NativeEngine) GetTwins() ([]uint32, error) {
+	return e.storage.Twins(WithDeleted())
+}
+
+func (e *NativeEngine) GetWorkload(twin uint32, contractID uint64, name gridtypes.Name) (gridtypes.Workload, bool, error) {
+	dep, err := e.storage.Get(twin, contractID, WithDeleted())
+	if err != nil {
+		return gridtypes.Workload{}, false, err
+	}
+	for i := range dep.Workloads {
+		if dep.Workloads[i].Name == name {
+			return dep.Workloads[i], true, nil
+		}
+	}
+	wl, err := e.storage.Current(twin, contractID, name, WithDeleted())
+	if err != nil {
+		return gridtypes.Workload{}, false, nil // not found, no error
+	}
+	return wl, true, nil
+}

pkg/provision/interface.go

@@ -110,6 +110,19 @@ type StorageCapacity struct {
 // and or workload that returns true from the capacity calculation.
 type Exclude = func(dl *gridtypes.Deployment, wl *gridtypes.Workload) bool
 
+// QueryOpt is a functional option for Storage query methods
+type QueryOpt func(*QueryOpts)
+
+// QueryOpts holds query options for Storage methods
+type QueryOpts struct {
+	Deleted bool
+}
+
+// WithDeleted returns a QueryOpt that includes soft-deleted items in query results
+func WithDeleted() QueryOpt {
+	return func(o *QueryOpts) { o.Deleted = true }
+}
+
 // Storage interface
 type Storage interface {
 	// Create a new deployment in storage, it sets the initial transactions
@@ -120,7 +133,7 @@ type Storage interface {
 	// Delete deletes a deployment from storage.
 	Delete(twin uint32, deployment uint64) error
 	// Get gets the current state of a deployment from storage
-	Get(twin uint32, deployment uint64) (gridtypes.Deployment, error)
+	Get(twin uint32, deployment uint64, opts ...QueryOpt) (gridtypes.Deployment, error)
 	// Error sets global deployment error
 	Error(twin uint32, deployment uint64, err error) error
 	// Add workload to deployment, if no active deployment exists with same name
@@ -132,11 +145,11 @@ type Storage interface {
 	// Changes return all the historic transactions of a deployment
 	Changes(twin uint32, deployment uint64) (changes []gridtypes.Workload, err error)
 	// Current gets last state of a workload by name
-	Current(twin uint32, deployment uint64, name gridtypes.Name) (gridtypes.Workload, error)
+	Current(twin uint32, deployment uint64, name gridtypes.Name, opts ...QueryOpt) (gridtypes.Workload, error)
 	// Twins list twins in storage
-	Twins() ([]uint32, error)
+	Twins(opts ...QueryOpt) ([]uint32, error)
 	// ByTwin return list of deployments for a twin
-	ByTwin(twin uint32) ([]uint64, error)
+	ByTwin(twin uint32, opts ...QueryOpt) ([]uint64, error)
 	// return total capacity and active deployments
 	Capacity(exclude ...Exclude) (StorageCapacity, error)
 }

pkg/provision/provisiner.go

@@ -122,7 +122,7 @@ func (p *mapProvisioner) Initialize(ctx context.Context) error {
 // Provision implements provision.Provisioner
 func (p *mapProvisioner) Provision(ctx context.Context, wl *gridtypes.WorkloadWithID) (result gridtypes.Result, err error) {
 	log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("provisioning workload")
-	
+
 	manager, ok := p.managers[wl.Type]
 	if !ok {
 		return result, fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID)
@@ -139,7 +139,7 @@ func (p *mapProvisioner) Provision(ctx context.Context, wl *gridtypes.WorkloadWi
 // Decommission implementation for provision.Provisioner
 func (p *mapProvisioner) Deprovision(ctx context.Context, wl *gridtypes.WorkloadWithID) error {
 	log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("deprovisioning workload")
-	
+
 	manager, ok := p.managers[wl.Type]
 	if !ok {
 		return fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID)
@@ -151,7 +151,7 @@ func (p *mapProvisioner) Deprovision(ctx context.Context, wl *gridtypes.Workload
 // Pause a workload
 func (p *mapProvisioner) Pause(ctx context.Context, wl *gridtypes.WorkloadWithID) (gridtypes.Result, error) {
 	log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("pausing workload")
-	
+
 	if wl.Result.State != gridtypes.StateOk {
 		return wl.Result, fmt.Errorf("can only pause workloads in ok state")
 	}
@@ -180,7 +180,7 @@ func (p *mapProvisioner) Pause(ctx context.Context, wl *gridtypes.WorkloadWithID
 // Resume a workload
 func (p *mapProvisioner) Resume(ctx context.Context, wl *gridtypes.WorkloadWithID) (gridtypes.Result, error) {
 	log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("resuming workload")
-	
+
 	if wl.Result.State != gridtypes.StatePaused {
 		return wl.Result, fmt.Errorf("can only resume workloads in paused state")
 	}
@@ -208,7 +208,7 @@ func (p *mapProvisioner) Resume(ctx context.Context, wl *gridtypes.WorkloadWithI
 // Provision implements provision.Provisioner
 func (p *mapProvisioner) Update(ctx context.Context, wl *gridtypes.WorkloadWithID) (result gridtypes.Result, err error) {
 	log.Info().Str("workload-id", string(wl.ID)).Str("workload-type", string(wl.Type)).Msg("updating workload")
-	
+
 	manager, ok := p.managers[wl.Type]
 	if !ok {
 		return result, fmt.Errorf("unknown workload type '%s' for reservation id '%s'", wl.Type, wl.ID)