Release (2026-06-01)

.github/workflows/dependency-review.yaml

@@ -14,6 +14,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: actions/dependency-review-action@2031cfc080254a8a887f58cffee85186f0e49e48 # v4.9.0
+    - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
       with:
         config-file: .github/dependency-review-config.yaml

app/Console/Commands/DispatchProjectPurgeJobsCommand.php

@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Models\PolydockAppInstance;
+use FreedomtechHosting\PolydockApp\Enums\PolydockAppInstanceStatus;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+/**
+ * Picks REMOVED app instances whose grace period has elapsed (or which have
+ * been flagged for force purge) and transitions them to PENDING_PURGE so the
+ * ProcessProjectPurgeJob can attempt full Lagoon project deletion.
+ *
+ * Designed to run on a short cron (every 10 minutes by default). Backoff
+ * between attempts is enforced via purge_last_attempted_at.
+ */
+class DispatchProjectPurgeJobsCommand extends Command
+{
+    protected $signature = 'polydock:dispatch-project-purge
+                          {--dry-run : List eligible instances without dispatching}
+                          {--limit= : Override the per-run limit}';
+
+    protected $description = 'Dispatch full Lagoon project purge jobs for REMOVED instances past their grace period';
+
+    public function handle(): int
+    {
+        $isDryRun = (bool) $this->option('dry-run');
+        $limit = (int) ($this->option('limit') ?? config('polydock.cleanup.purge_max_per_run', 25));
+        $pollInterval = (int) config('polydock.cleanup.purge_poll_interval_minutes', 10);
+        $maxAttempts = (int) config('polydock.cleanup.purge_max_poll_attempts', 144);
+
+        $now = now();
+        $backoffCutoff = $now->copy()->subMinutes($pollInterval);
+
+        // First, push instances that have exceeded the polling cap to PURGE_FAILED
+        // so they stop being re-dispatched (admin must explicitly retry).
+        $stuck = PolydockAppInstance::query()
+            ->where('status', PolydockAppInstanceStatus::REMOVED)
+            ->where('purge_attempts', '>=', $maxAttempts)
+            ->get();
+
+        foreach ($stuck as $instance) {
+            $message = sprintf(
+                'Purge polling cap reached (%d attempts) without success',
+                $instance->purge_attempts,
+            );
+
+            $this->warn(sprintf('[stuck] %s (id=%d): %s', $instance->name, $instance->id, $message));
+            Log::warning('Marking instance PURGE_FAILED after polling cap', [
+                'app_instance_id' => $instance->id,
+                'attempts' => $instance->purge_attempts,
+            ]);
+
+            if (! $isDryRun) {
+                $instance->purge_failure_reason = $message;
+                $instance->setStatus(PolydockAppInstanceStatus::PURGE_FAILED, $message);
+                $instance->save();
+            }
+        }
+
+        // Now find eligible candidates.
+        $candidates = PolydockAppInstance::query()
+            ->where('status', PolydockAppInstanceStatus::REMOVED)
+            ->where('purge_attempts', '<', $maxAttempts)
+            ->where(function ($query) use ($now) {
+                // Either grace period elapsed naturally...
+                $query->where(function ($q) use ($now) {
+                    $q->whereNotNull('purge_eligible_at')
+                        ->where('purge_eligible_at', '<=', $now);
+                })
+                    // ...or admin-forced.
+                    ->orWhereNotNull('force_purge_requested_at');
+            })
+            ->where(function ($query) use ($backoffCutoff) {
+                $query->whereNull('purge_last_attempted_at')
+                    ->orWhere('purge_last_attempted_at', '<=', $backoffCutoff);
+            })
+            ->orderBy('purge_eligible_at')
+            ->limit($limit)
+            ->get();
+
+        if ($candidates->isEmpty()) {
+            $this->info('No instances eligible for project purge.');
+
+            return self::SUCCESS;
+        }
+
+        $this->info(sprintf('Found %d instance(s) eligible for project purge.', $candidates->count()));
+
+        foreach ($candidates as $instance) {
+            $reason = $instance->force_purge_requested_at !== null ? 'forced' : 'grace-period elapsed';
+            $line = sprintf(
+                ' - %s (id=%d, attempts=%d, %s)',
+                $instance->name,
+                $instance->id,
+                $instance->purge_attempts ?? 0,
+                $reason,
+            );
+
+            if ($isDryRun) {
+                $this->line('[dry-run]'.$line);
+
+                continue;
+            }
+
+            $this->line($line);
+
+            Log::info('Dispatching ProcessProjectPurgeJob', [
+                'app_instance_id' => $instance->id,
+                'reason' => $reason,
+                'attempts' => $instance->purge_attempts ?? 0,
+            ]);
+
+            // Setting status triggers the listener which dispatches the job.
+            $instance->setStatus(PolydockAppInstanceStatus::PENDING_PURGE, 'Project purge dispatched ('.$reason.')');
+            $instance->save();
+        }
+
+        return self::SUCCESS;
+    }
+}

app/Console/Commands/MarkStuckInstancesFailedCommand.php

@@ -0,0 +1,133 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Models\PolydockAppInstance;
+use FreedomtechHosting\PolydockApp\Enums\PolydockAppInstanceStatus;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Log;
+
+class MarkStuckInstancesFailedCommand extends Command
+{
+    protected $signature = 'polydock:mark-stuck-instances-failed
+                            {--threshold=30 : Minutes an instance can remain in a running/pending state before being considered stuck}
+                            {--dry-run : Show what would be marked failed without making changes}
+                            {--chunk=200 : Number of instances to process per batch}';
+
+    protected $description = 'Detect instances stuck in intermediate statuses and mark them as failed';
+
+    /**
+     * Statuses that indicate an instance is actively progressing through the pipeline.
+     * If an instance has been in one of these statuses longer than the threshold, it is stuck.
+     *
+     * @return array<int, PolydockAppInstanceStatus>
+     */
+    private static function intermediateStatuses(): array
+    {
+        return PolydockAppInstance::unallocatedInProgressStatuses();
+    }
+
+    /**
+     * Resolve the corresponding failed status for a given intermediate status.
+     */
+    private static function resolveFailedStatus(PolydockAppInstanceStatus $status): PolydockAppInstanceStatus
+    {
+        return match ($status) {
+            PolydockAppInstanceStatus::NEW,
+            PolydockAppInstanceStatus::PENDING_PRE_CREATE,
+            PolydockAppInstanceStatus::PRE_CREATE_RUNNING,
+            PolydockAppInstanceStatus::PRE_CREATE_COMPLETED => PolydockAppInstanceStatus::PRE_CREATE_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_CREATE,
+            PolydockAppInstanceStatus::CREATE_RUNNING,
+            PolydockAppInstanceStatus::CREATE_COMPLETED => PolydockAppInstanceStatus::CREATE_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_POST_CREATE,
+            PolydockAppInstanceStatus::POST_CREATE_RUNNING,
+            PolydockAppInstanceStatus::POST_CREATE_COMPLETED => PolydockAppInstanceStatus::POST_CREATE_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_PRE_DEPLOY,
+            PolydockAppInstanceStatus::PRE_DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::PRE_DEPLOY_COMPLETED => PolydockAppInstanceStatus::PRE_DEPLOY_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_DEPLOY,
+            PolydockAppInstanceStatus::DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::DEPLOY_COMPLETED => PolydockAppInstanceStatus::DEPLOY_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_POST_DEPLOY,
+            PolydockAppInstanceStatus::POST_DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::POST_DEPLOY_COMPLETED => PolydockAppInstanceStatus::POST_DEPLOY_FAILED,
+
+            PolydockAppInstanceStatus::PENDING_POLYDOCK_CLAIM,
+            PolydockAppInstanceStatus::POLYDOCK_CLAIM_RUNNING,
+            PolydockAppInstanceStatus::POLYDOCK_CLAIM_COMPLETED => PolydockAppInstanceStatus::POLYDOCK_CLAIM_FAILED,
+
+            default => throw new \LogicException("No failed status mapping for: {$status->value}"),
+        };
+    }
+
+    public function handle(): int
+    {
+        $threshold = (int) $this->option('threshold');
+        $dryRun = (bool) $this->option('dry-run');
+        $chunkSize = (int) $this->option('chunk');
+        $cutoff = now()->subMinutes($threshold);
+
+        $totalMarked = 0;
+        $rows = [];
+
+        PolydockAppInstance::query()
+            ->whereIn('status', self::intermediateStatuses())
+            ->where('updated_at', '<=', $cutoff)
+            ->chunkById($chunkSize, function ($instances) use ($dryRun, $threshold, &$totalMarked, &$rows) {
+                foreach ($instances as $instance) {
+                    $failedStatus = self::resolveFailedStatus($instance->status);
+
+                    $rows[] = [
+                        $instance->id,
+                        $instance->uuid ?? $instance->id,
+                        $instance->status->value,
+                        $failedStatus->value,
+                        $instance->updated_at->diffForHumans(),
+                    ];
+
+                    if (! $dryRun) {
+                        $previousStatus = $instance->status->value;
+
+                        $instance->update([
+                            'status' => $failedStatus,
+                            'status_message' => "Automatically marked failed: stuck at {$previousStatus} for >{$threshold} minutes",
+                        ]);
+                    }
+
+                    $totalMarked++;
+                }
+            });
+
+        if ($totalMarked === 0) {
+            $this->info('No stuck instances found.');
+            Log::info('polydock:mark-stuck-instances-failed: no stuck instances found', [
+                'threshold_minutes' => $threshold,
+            ]);
+
+            return Command::SUCCESS;
+        }
+
+        $this->info("Found {$totalMarked} stuck instance(s) (threshold: {$threshold} minutes):");
+        $this->table(['ID', 'UUID', 'Was', 'Now', 'Last Updated'], $rows);
+
+        if ($dryRun) {
+            $this->warn('Dry run — no changes made.');
+        } else {
+            $this->info("Marked {$totalMarked} instance(s) as failed.");
+            Log::warning('polydock:mark-stuck-instances-failed: marked instances as failed', [
+                'count' => $totalMarked,
+                'threshold_minutes' => $threshold,
+            ]);
+        }
+
+        return Command::SUCCESS;
+    }
+}