edge-connect-client/internal/apply/v2/strategy_recreate.go

// ABOUTME: Recreate deployment strategy implementation for EdgeConnect
// ABOUTME: Handles delete-all, update-app, create-all deployment pattern with retries and parallel execution
package v2

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"sync"
	"time"

	"edp.buildth.ing/DevFW-CICD/edge-connect-client/v2/internal/config"
	v2 "edp.buildth.ing/DevFW-CICD/edge-connect-client/v2/sdk/edgeconnect/v2"
)

// RecreateStrategy implements the recreate deployment strategy
type RecreateStrategy struct {
	client EdgeConnectClientInterface
	config StrategyConfig
	logger Logger
}

// NewRecreateStrategy creates a new recreate strategy executor
func NewRecreateStrategy(client EdgeConnectClientInterface, config StrategyConfig, logger Logger) *RecreateStrategy {
	return &RecreateStrategy{
		client: client,
		config: config,
		logger: logger,
	}
}

// GetName returns the strategy name
func (r *RecreateStrategy) GetName() DeploymentStrategy {
	return StrategyRecreate
}

// Validate checks if the recreate strategy can be used for this deployment
func (r *RecreateStrategy) Validate(plan *DeploymentPlan) error {
	// Recreate strategy can be used for any deployment
	// No specific constraints for recreate
	return nil
}

// EstimateDuration estimates the time needed for recreate deployment
func (r *RecreateStrategy) EstimateDuration(plan *DeploymentPlan) time.Duration {
	var duration time.Duration

	// Delete phase - estimate based on number of instances
	instanceCount := len(plan.InstanceActions)
	if instanceCount > 0 {
		deleteTime := time.Duration(instanceCount) * 30 * time.Second
		if r.config.ParallelOperations {
			deleteTime = 30 * time.Second // Parallel deletion
		}
		duration += deleteTime
	}

	// App update phase
	if plan.AppAction.Type == ActionUpdate {
		duration += 30 * time.Second
	}

	// Create phase - estimate based on number of instances
	if instanceCount > 0 {
		createTime := time.Duration(instanceCount) * 2 * time.Minute
		if r.config.ParallelOperations {
			createTime = 2 * time.Minute // Parallel creation
		}
		duration += createTime
	}

	// Health check time
	duration += r.config.HealthCheckTimeout

	// Add retry buffer (potential retries)
	retryBuffer := time.Duration(r.config.MaxRetries) * r.config.RetryDelay
	duration += retryBuffer

	return duration
}

// Execute runs the recreate deployment strategy
func (r *RecreateStrategy) Execute(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig, manifestContent string) (*ExecutionResult, error) {
	startTime := time.Now()
	r.logf("Starting recreate deployment strategy for: %s", plan.ConfigName)

	result := &ExecutionResult{
		Plan:             plan,
		CompletedActions: []ActionResult{},
		FailedActions:    []ActionResult{},
	}

	// Phase 1: Delete all existing instances
	if err := r.deleteInstancesPhase(ctx, plan, config, result); err != nil {
		result.Error = err
		result.Duration = time.Since(startTime)
		return result, err
	}

	// Phase 2: Delete existing app (if updating)
	if err := r.deleteAppPhase(ctx, plan, config, result); err != nil {
		result.Error = err
		result.Duration = time.Since(startTime)
		return result, err
	}

	// Phase 3: Create/recreate application
	if err := r.createAppPhase(ctx, plan, config, manifestContent, result); err != nil {
		result.Error = err
		result.Duration = time.Since(startTime)
		return result, err
	}

	// Phase 4: Create new instances
	if err := r.createInstancesPhase(ctx, plan, config, result); err != nil {
		result.Error = err
		result.Duration = time.Since(startTime)
		return result, err
	}

	// Phase 5: Health check (wait for instances to be ready)
	if err := r.healthCheckPhase(ctx, plan, result); err != nil {
		result.Error = err
		result.Duration = time.Since(startTime)
		return result, err
	}

	result.Success = len(result.FailedActions) == 0
	result.Duration = time.Since(startTime)

	if result.Success {
		r.logf("Recreate deployment completed successfully in %v", result.Duration)
	} else {
		r.logf("Recreate deployment failed with %d failed actions", len(result.FailedActions))
	}

	return result, result.Error
}

// deleteInstancesPhase deletes all existing instances
func (r *RecreateStrategy) deleteInstancesPhase(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig, result *ExecutionResult) error {
	r.logf("Phase 1: Deleting existing instances")

	// Only delete instances that exist (have ActionUpdate or ActionNone type)
	instancesToDelete := []InstanceAction{}
	for _, action := range plan.InstanceActions {
		if action.Type == ActionUpdate || action.Type == ActionNone {
			// Convert to delete action
			deleteAction := action
			deleteAction.Type = ActionDelete
			deleteAction.Reason = "Recreate strategy: deleting for recreation"
			instancesToDelete = append(instancesToDelete, deleteAction)
		}
	}

	if len(instancesToDelete) == 0 {
		r.logf("No existing instances to delete")
		return nil
	}

	// Backup instances before deleting them (for rollback restoration)
	r.logf("Backing up %d existing instances before deletion", len(instancesToDelete))
	for _, action := range instancesToDelete {
		backup, err := r.backupInstance(ctx, action, config)
		if err != nil {
			r.logf("Warning: failed to backup instance %s before deletion: %v", action.InstanceName, err)
			// Continue with deletion even if backup fails - this is best effort
		} else {
			result.DeletedInstancesBackup = append(result.DeletedInstancesBackup, *backup)
			r.logf("Backed up instance: %s", action.InstanceName)
		}
	}

	deleteResults := r.executeInstanceActionsWithRetry(ctx, instancesToDelete, "delete", config)

	for _, deleteResult := range deleteResults {
		if deleteResult.Success {
			result.CompletedActions = append(result.CompletedActions, deleteResult)
			r.logf("Deleted instance: %s", deleteResult.Target)
		} else {
			result.FailedActions = append(result.FailedActions, deleteResult)
			return fmt.Errorf("failed to delete instance %s: %w", deleteResult.Target, deleteResult.Error)
		}
	}

	r.logf("Phase 1 complete: deleted %d instances", len(deleteResults))

	// Wait for Kubernetes namespace termination to complete
	// This prevents "namespace is being terminated" errors when recreating instances
	if len(deleteResults) > 0 {
		waitTime := 5 * time.Second
		r.logf("Waiting %v for namespace termination to complete...", waitTime)
		select {
		case <-time.After(waitTime):
		case <-ctx.Done():
			return ctx.Err()
		}
	}

	return nil
}

// deleteAppPhase deletes the existing app (if updating)
func (r *RecreateStrategy) deleteAppPhase(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig, result *ExecutionResult) error {
	if plan.AppAction.Type != ActionUpdate {
		r.logf("Phase 2: No app deletion needed (new app)")
		return nil
	}

	r.logf("Phase 2: Deleting existing application")

	// Backup app before deleting it (for rollback restoration)
	r.logf("Backing up existing app before deletion")
	backup, err := r.backupApp(ctx, plan, config)
	if err != nil {
		r.logf("Warning: failed to backup app before deletion: %v", err)
		// Continue with deletion even if backup fails - this is best effort
	} else {
		result.DeletedAppBackup = backup
		r.logf("Backed up app: %s", plan.AppAction.Desired.Name)
	}

	appKey := v2.AppKey{
		Organization: plan.AppAction.Desired.Organization,
		Name:         plan.AppAction.Desired.Name,
		Version:      plan.AppAction.Desired.Version,
	}

	if err := r.client.DeleteApp(ctx, appKey, plan.AppAction.Desired.Region); err != nil {
		result.FailedActions = append(result.FailedActions, ActionResult{
			Type:    ActionDelete,
			Target:  plan.AppAction.Desired.Name,
			Success: false,
			Error:   err,
		})
		return fmt.Errorf("failed to delete app: %w", err)
	}

	result.CompletedActions = append(result.CompletedActions, ActionResult{
		Type:    ActionDelete,
		Target:  plan.AppAction.Desired.Name,
		Success: true,
		Details: fmt.Sprintf("Deleted app %s", plan.AppAction.Desired.Name),
	})

	r.logf("Phase 2 complete: deleted existing application")
	return nil
}

// createAppPhase creates the application (always create since we deleted it first)
func (r *RecreateStrategy) createAppPhase(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig, manifestContent string, result *ExecutionResult) error {
	if plan.AppAction.Type == ActionNone {
		r.logf("Phase 3: No app creation needed")
		return nil
	}

	r.logf("Phase 3: Creating application")

	// Always use create since recreate strategy deletes first
	createAction := plan.AppAction
	createAction.Type = ActionCreate
	createAction.Reason = "Recreate strategy: creating app"

	appResult := r.executeAppActionWithRetry(ctx, createAction, config, manifestContent)

	if appResult.Success {
		result.CompletedActions = append(result.CompletedActions, appResult)
		r.logf("Phase 3 complete: app created successfully")
		return nil
	} else {
		result.FailedActions = append(result.FailedActions, appResult)
		return fmt.Errorf("failed to create app: %w", appResult.Error)
	}
}

// createInstancesPhase creates new instances
func (r *RecreateStrategy) createInstancesPhase(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig, result *ExecutionResult) error {
	r.logf("Phase 4: Creating new instances")

	// Convert all instance actions to create
	instancesToCreate := []InstanceAction{}
	for _, action := range plan.InstanceActions {
		createAction := action
		createAction.Type = ActionCreate
		createAction.Reason = "Recreate strategy: creating new instance"
		instancesToCreate = append(instancesToCreate, createAction)
	}

	if len(instancesToCreate) == 0 {
		r.logf("No instances to create")
		return nil
	}

	createResults := r.executeInstanceActionsWithRetry(ctx, instancesToCreate, "create", config)

	for _, createResult := range createResults {
		if createResult.Success {
			result.CompletedActions = append(result.CompletedActions, createResult)
			r.logf("Created instance: %s", createResult.Target)
		} else {
			result.FailedActions = append(result.FailedActions, createResult)
			return fmt.Errorf("failed to create instance %s: %w", createResult.Target, createResult.Error)
		}
	}

	r.logf("Phase 4 complete: created %d instances", len(createResults))
	return nil
}

// healthCheckPhase waits for instances to become ready
func (r *RecreateStrategy) healthCheckPhase(ctx context.Context, plan *DeploymentPlan, result *ExecutionResult) error {
	if len(plan.InstanceActions) == 0 {
		return nil
	}

	r.logf("Phase 5: Performing health checks")

	// TODO: Implement actual health checks by querying instance status
	// For now, skip waiting in tests/mock environments
	r.logf("Phase 5 complete: health check passed (no wait)")
	return nil
}

// executeInstanceActionsWithRetry executes instance actions with retry logic
func (r *RecreateStrategy) executeInstanceActionsWithRetry(ctx context.Context, actions []InstanceAction, operation string, config *config.EdgeConnectConfig) []ActionResult {
	results := make([]ActionResult, len(actions))

	if r.config.ParallelOperations && len(actions) > 1 {
		// Parallel execution
		var wg sync.WaitGroup
		semaphore := make(chan struct{}, 5) // Limit concurrency

		for i, action := range actions {
			wg.Add(1)
			go func(index int, instanceAction InstanceAction) {
				defer wg.Done()
				semaphore <- struct{}{}
				defer func() { <-semaphore }()

				results[index] = r.executeInstanceActionWithRetry(ctx, instanceAction, operation, config)
			}(i, action)
		}
		wg.Wait()
	} else {
		// Sequential execution
		for i, action := range actions {
			results[i] = r.executeInstanceActionWithRetry(ctx, action, operation, config)
		}
	}

	return results
}

// executeInstanceActionWithRetry executes a single instance action with retry logic
func (r *RecreateStrategy) executeInstanceActionWithRetry(ctx context.Context, action InstanceAction, operation string, config *config.EdgeConnectConfig) ActionResult {
	startTime := time.Now()
	result := ActionResult{
		Type:   action.Type,
		Target: action.InstanceName,
	}

	var lastErr error
	for attempt := 0; attempt <= r.config.MaxRetries; attempt++ {
		if attempt > 0 {
			r.logf("Retrying %s for instance %s (attempt %d/%d)", operation, action.InstanceName, attempt, r.config.MaxRetries)
			select {
			case <-time.After(r.config.RetryDelay):
			case <-ctx.Done():
				result.Error = ctx.Err()
				result.Duration = time.Since(startTime)
				return result
			}
		}

		var success bool
		var err error

		switch action.Type {
		case ActionDelete:
			success, err = r.deleteInstance(ctx, action)
		case ActionCreate:
			success, err = r.createInstance(ctx, action, config)
		default:
			err = fmt.Errorf("unsupported action type: %s", action.Type)
		}

		if success {
			result.Success = true
			result.Details = fmt.Sprintf("Successfully %sd instance %s", strings.ToLower(string(action.Type)), action.InstanceName)
			result.Duration = time.Since(startTime)
			return result
		}

		lastErr = err

		// Check if error is retryable (don't retry 4xx client errors)
		if !isRetryableError(err) {
			r.logf("Failed to %s instance %s: %v (non-retryable error, giving up)", operation, action.InstanceName, err)
			result.Error = fmt.Errorf("non-retryable error: %w", err)
			result.Duration = time.Since(startTime)
			return result
		}

		if attempt < r.config.MaxRetries {
			r.logf("Failed to %s instance %s: %v (will retry)", operation, action.InstanceName, err)
		}
	}

	result.Error = fmt.Errorf("failed after %d attempts: %w", r.config.MaxRetries+1, lastErr)
	result.Duration = time.Since(startTime)
	return result
}

// executeAppActionWithRetry executes app action with retry logic
func (r *RecreateStrategy) executeAppActionWithRetry(ctx context.Context, action AppAction, config *config.EdgeConnectConfig, manifestContent string) ActionResult {
	startTime := time.Now()
	result := ActionResult{
		Type:   action.Type,
		Target: action.Desired.Name,
	}

	var lastErr error
	for attempt := 0; attempt <= r.config.MaxRetries; attempt++ {
		if attempt > 0 {
			r.logf("Retrying app update (attempt %d/%d)", attempt, r.config.MaxRetries)
			select {
			case <-time.After(r.config.RetryDelay):
			case <-ctx.Done():
				result.Error = ctx.Err()
				result.Duration = time.Since(startTime)
				return result
			}
		}

		success, err := r.updateApplication(ctx, action, config, manifestContent)
		if success {
			result.Success = true
			result.Details = fmt.Sprintf("Successfully updated application %s", action.Desired.Name)
			result.Duration = time.Since(startTime)
			return result
		}

		lastErr = err

		// Check if error is retryable (don't retry 4xx client errors)
		if !isRetryableError(err) {
			r.logf("Failed to update app: %v (non-retryable error, giving up)", err)
			result.Error = fmt.Errorf("non-retryable error: %w", err)
			result.Duration = time.Since(startTime)
			return result
		}

		if attempt < r.config.MaxRetries {
			r.logf("Failed to update app: %v (will retry)", err)
		}
	}

	result.Error = fmt.Errorf("failed after %d attempts: %w", r.config.MaxRetries+1, lastErr)
	result.Duration = time.Since(startTime)
	return result
}

// deleteInstance deletes an instance (reuse existing logic from manager.go)
func (r *RecreateStrategy) deleteInstance(ctx context.Context, action InstanceAction) (bool, error) {
	instanceKey := v2.AppInstanceKey{
		Organization: action.Desired.Organization,
		Name:         action.InstanceName,
		CloudletKey: v2.CloudletKey{
			Organization: action.Target.CloudletOrg,
			Name:         action.Target.CloudletName,
		},
	}

	err := r.client.DeleteAppInstance(ctx, instanceKey, action.Target.Region)
	if err != nil {
		return false, fmt.Errorf("failed to delete instance: %w", err)
	}

	return true, nil
}

// createInstance creates an instance (extracted from manager.go logic)
func (r *RecreateStrategy) createInstance(ctx context.Context, action InstanceAction, config *config.EdgeConnectConfig) (bool, error) {
	instanceInput := &v2.NewAppInstanceInput{
		Region: action.Target.Region,
		AppInst: v2.AppInstance{
			Key: v2.AppInstanceKey{
				Organization: action.Desired.Organization,
				Name:         action.InstanceName,
				CloudletKey: v2.CloudletKey{
					Organization: action.Target.CloudletOrg,
					Name:         action.Target.CloudletName,
				},
			},
			AppKey: v2.AppKey{
				Organization: action.Desired.Organization,
				Name:         config.Metadata.Name,
				Version:      config.Metadata.AppVersion,
			},
			Flavor: v2.Flavor{
				Name: action.Target.FlavorName,
			},
		},
	}

	// Create the instance
	if err := r.client.CreateAppInstance(ctx, instanceInput); err != nil {
		return false, fmt.Errorf("failed to create instance: %w", err)
	}

	r.logf("Successfully created instance: %s on %s:%s",
		action.InstanceName, action.Target.CloudletOrg, action.Target.CloudletName)

	return true, nil
}

// updateApplication creates/recreates an application (always uses CreateApp since we delete first)
func (r *RecreateStrategy) updateApplication(ctx context.Context, action AppAction, config *config.EdgeConnectConfig, manifestContent string) (bool, error) {
	// Build the app create input - always create since recreate strategy deletes first
	appInput := &v2.NewAppInput{
		Region: action.Desired.Region,
		App: v2.App{
			Key: v2.AppKey{
				Organization: action.Desired.Organization,
				Name:         action.Desired.Name,
				Version:      action.Desired.Version,
			},
			Deployment:          config.GetDeploymentType(),
			ImageType:           "ImageTypeDocker",
			ImagePath:           config.GetImagePath(),
			AllowServerless:     true,
			DefaultFlavor:       v2.Flavor{Name: config.Spec.InfraTemplate[0].FlavorName},
			ServerlessConfig:    struct{}{},
			DeploymentManifest:  manifestContent,
			DeploymentGenerator: "kubernetes-basic",
		},
	}

	// Add network configuration if specified
	if config.Spec.Network != nil {
		appInput.App.RequiredOutboundConnections = convertNetworkRules(config.Spec.Network)
	}

	// Create the application (recreate strategy always creates from scratch)
	if err := r.client.CreateApp(ctx, appInput); err != nil {
		return false, fmt.Errorf("failed to create application: %w", err)
	}

	r.logf("Successfully created application: %s/%s version %s",
		action.Desired.Organization, action.Desired.Name, action.Desired.Version)

	return true, nil
}

// backupApp fetches and stores the current app state before deletion
func (r *RecreateStrategy) backupApp(ctx context.Context, plan *DeploymentPlan, config *config.EdgeConnectConfig) (*AppBackup, error) {
	appKey := v2.AppKey{
		Organization: plan.AppAction.Desired.Organization,
		Name:         plan.AppAction.Desired.Name,
		Version:      plan.AppAction.Desired.Version,
	}

	app, err := r.client.ShowApp(ctx, appKey, plan.AppAction.Desired.Region)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch app for backup: %w", err)
	}

	backup := &AppBackup{
		App:             app,
		Region:          plan.AppAction.Desired.Region,
		ManifestContent: app.DeploymentManifest,
	}

	return backup, nil
}

// backupInstance fetches and stores the current instance state before deletion
func (r *RecreateStrategy) backupInstance(ctx context.Context, action InstanceAction, config *config.EdgeConnectConfig) (*InstanceBackup, error) {
	instanceKey := v2.AppInstanceKey{
		Organization: action.Desired.Organization,
		Name:         action.InstanceName,
		CloudletKey: v2.CloudletKey{
			Organization: action.Target.CloudletOrg,
			Name:         action.Target.CloudletName,
		},
	}

	appKey := v2.AppKey{Name: action.Desired.AppName}

	instance, err := r.client.ShowAppInstance(ctx, instanceKey, appKey, action.Target.Region)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch instance for backup: %w", err)
	}

	backup := &InstanceBackup{
		Instance: instance,
		Region:   action.Target.Region,
	}

	return backup, nil
}

// logf logs a message if a logger is configured
func (r *RecreateStrategy) logf(format string, v ...interface{}) {
	if r.logger != nil {
		r.logger.Printf("[RecreateStrategy] "+format, v...)
	}
}

// isRetryableError determines if an error should be retried
// Returns false for client errors (4xx), true for server errors (5xx) and other transient errors
func isRetryableError(err error) bool {
	if err == nil {
		return false
	}

	errStr := strings.ToLower(err.Error())

	// Special case: Kubernetes namespace termination race condition
	// This is a transient 400 error that should be retried
	if strings.Contains(errStr, "being terminated") || strings.Contains(errStr, "is being terminated") {
		return true
	}

	// Check if it's an APIError with a status code
	var apiErr *v2.APIError
	if errors.As(err, &apiErr) {
		// Don't retry client errors (4xx)
		if apiErr.StatusCode >= 400 && apiErr.StatusCode < 500 {
			return false
		}
		// Retry server errors (5xx)
		if apiErr.StatusCode >= 500 {
			return true
		}
	}

	// Retry all other errors (network issues, timeouts, etc.)
	return true
}