Release (2026-07-21)

.lagoon.yml

@@ -25,7 +25,16 @@ tasks:
         shell: bash
     - run:
         name: Run migrations
-        command: php artisan -n migrate --force
+        # Pause horizon workers (broadcast via redis to all pods) so long
+        # queue jobs can't hold metadata locks that stall migration DDL.
+        # The EXIT/TERM/INT trap resumes horizon on every exit path —
+        # migrate failure, task cancellation, pod termination — so a dead
+        # deploy can't leave the queue paused. Migrate's exit status still
+        # propagates as the task result.
+        command: |
+          php artisan horizon:pause || true
+          trap 'php artisan horizon:continue || true' EXIT TERM INT
+          php artisan -n migrate --force
         service: cli
         shell: bash
     - run:

app/Console/Commands/RemoveStaleFailedInstancesCommand.php

@@ -0,0 +1,162 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Console\Commands;
+
+use App\Models\PolydockAppInstance;
+use App\Polydock\Core\Enums\PolydockAppInstanceStatus;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+
+/**
+ * Sweeps instances that died mid-lifecycle and pushes them into the removal
+ * pipeline so their Lagoon projects don't accumulate forever.
+ *
+ * Two groups, both older than the retention window:
+ * - create/deploy/claim failures -> PENDING_PRE_REMOVE (normal remove flow,
+ *   app-specific pre/post-remove hooks still run)
+ * - remove-stage failures -> REMOVED (the remove flow already failed once;
+ *   project purge deletes the whole Lagoon project, and treats an
+ *   already-gone project as success)
+ *
+ * Both get force_purge_requested_at stamped so the purge skips the 14-day
+ * grace period — failed instances hold no user data worth a grace window.
+ * PURGE_FAILED is deliberately excluded: it means the purge polling cap was
+ * reached and an admin must explicitly retry.
+ */
+class RemoveStaleFailedInstancesCommand extends BaseCommand
+{
+    protected $signature = 'polydock:remove-stale-failed-instances
+                          {--dry-run : List eligible instances without making changes}
+                          {--days= : Override the retention window in days}
+                          {--limit= : Override the per-run limit}';
+
+    protected $description = 'Push failed app instances older than the retention window into the remove/purge pipeline';
+
+    /**
+     * Failures before or during the remove stage get the full remove flow.
+     *
+     * @return array<int, PolydockAppInstanceStatus>
+     */
+    private static function preRemovalFailedStatuses(): array
+    {
+        return [
+            PolydockAppInstanceStatus::PRE_CREATE_FAILED,
+            PolydockAppInstanceStatus::CREATE_FAILED,
+            PolydockAppInstanceStatus::POST_CREATE_FAILED,
+            PolydockAppInstanceStatus::PRE_DEPLOY_FAILED,
+            PolydockAppInstanceStatus::DEPLOY_FAILED,
+            PolydockAppInstanceStatus::POST_DEPLOY_FAILED,
+            PolydockAppInstanceStatus::POLYDOCK_CLAIM_FAILED,
+        ];
+    }
+
+    /**
+     * Failures inside the remove stage skip straight to REMOVED so the
+     * project purge can delete the whole Lagoon project.
+     *
+     * @return array<int, PolydockAppInstanceStatus>
+     */
+    private static function removeStageFailedStatuses(): array
+    {
+        return [
+            PolydockAppInstanceStatus::PRE_REMOVE_FAILED,
+            PolydockAppInstanceStatus::REMOVE_FAILED,
+            PolydockAppInstanceStatus::POST_REMOVE_FAILED,
+        ];
+    }
+
+    public function handle(): int
+    {
+        $isDryRun = (bool) $this->option('dry-run');
+        $days = (int) ($this->option('days') ?? config('polydock.cleanup.failed_instance_retention_days', 7));
+        $limit = (int) ($this->option('limit') ?? config('polydock.cleanup.failed_sweep_max_per_run', 25));
+        $cutoff = now()->subDays($days);
+
+        $eligible = PolydockAppInstance::query()
+            ->whereIn('status', array_merge(self::preRemovalFailedStatuses(), self::removeStageFailedStatuses()))
+            ->where('updated_at', '<=', $cutoff)
+            ->orderBy('updated_at')
+            ->limit($limit)
+            ->get();
+
+        if ($eligible->isEmpty()) {
+            $this->info("No failed instances older than {$days} days found.");
+
+            return self::SUCCESS;
+        }
+
+        $this->info(sprintf('Found %d stale failed instance(s) (older than %d days).', $eligible->count(), $days));
+
+        $swept = 0;
+
+        foreach ($eligible as $instance) {
+            $isRemoveStageFailure = in_array($instance->status, self::removeStageFailedStatuses(), true);
+            $target = $isRemoveStageFailure
+                ? PolydockAppInstanceStatus::REMOVED
+                : PolydockAppInstanceStatus::PENDING_PRE_REMOVE;
+
+            $line = sprintf(
+                ' - %s (id=%d, %s -> %s)',
+                $instance->name,
+                $instance->id,
+                $instance->status->value,
+                $target->value,
+            );
+
+            if ($isDryRun) {
+                $this->line('[dry-run]'.$line);
+
+                continue;
+            }
+
+            // Re-check under a row lock before writing: a retry or operator
+            // may have recovered the instance between the eligibility query
+            // and this write, and saving the stale snapshot would shove a
+            // live instance into removal/purge.
+            $sweptThisOne = DB::transaction(function () use ($instance, $cutoff, $target, $days) {
+                $fresh = PolydockAppInstance::query()
+                    ->whereKey($instance->id)
+                    ->lockForUpdate()
+                    ->first();
+
+                if ($fresh === null
+                    || $fresh->status !== $instance->status
+                    || $fresh->updated_at > $cutoff) {
+                    return false;
+                }
+
+                $fresh->force_purge_requested_at = $fresh->force_purge_requested_at ?? now();
+                // Status change fires PolydockAppInstanceStatusChanged, whose
+                // listener dispatches the stage job / purge transition.
+                $fresh->setStatus($target, "Stale failed instance swept after {$days} days");
+                $fresh->save();
+
+                return true;
+            });
+
+            if (! $sweptThisOne) {
+                $this->line(sprintf(' - %s (id=%d) skipped: state changed since selection', $instance->name, $instance->id));
+
+                continue;
+            }
+
+            $this->line($line);
+
+            Log::info('Sweeping stale failed instance into removal pipeline', [
+                'app_instance_id' => $instance->id,
+                'previous_status' => $instance->status->value,
+                'target_status' => $target->value,
+            ]);
+
+            $swept++;
+        }
+
+        if (! $isDryRun) {
+            Log::info('polydock:remove-stale-failed-instances swept instances', ['count' => $swept]);
+        }
+
+        return self::SUCCESS;
+    }
+}

app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php

@@ -40,7 +40,7 @@ public function handle(): int
 
             // Build the query for unclaimed instances older than specified days
             $query = PolydockAppInstance::where('status', PolydockAppInstanceStatus::RUNNING_HEALTHY_UNCLAIMED)
-                ->whereDate('created_at', '<=', now()->subDay());
+                ->whereDate('created_at', '<=', now()->subDays($days));
 
             // Add app filter if specified
             if ($appId) {

app/Models/PolydockAppInstanceStatusTransition.php

@@ -7,11 +7,18 @@
 use App\Polydock\Core\Enums\PolydockAppInstanceStatus;
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Database\Eloquent\Relations\BelongsTo;
+use Illuminate\Support\Carbon;
 
 /**
  * One row per status change of an app instance — the timing source of truth
  * for per-stage durations (the activity log deliberately does not record
  * status). Rows are immutable and cascade-delete with the instance.
+ *
+ * @property int $id
+ * @property int $polydock_app_instance_id
+ * @property PolydockAppInstanceStatus|null $from_status
+ * @property PolydockAppInstanceStatus $to_status
+ * @property Carbon $created_at
  */
 class PolydockAppInstanceStatusTransition extends Model
 {

config/polydock.php

@@ -55,6 +55,10 @@
         'purge_poll_interval_minutes' => (int) env('POLYDOCK_PURGE_POLL_INTERVAL', 10),
         'purge_max_poll_attempts' => (int) env('POLYDOCK_PURGE_MAX_POLLS', 144),
         'purge_max_per_run' => (int) env('POLYDOCK_PURGE_MAX_PER_RUN', 25),
+        // Failed instances (create/deploy/claim/remove failures) are swept
+        // into the remove/purge pipeline after this many days.
+        'failed_instance_retention_days' => (int) env('POLYDOCK_FAILED_RETENTION_DAYS', 7),
+        'failed_sweep_max_per_run' => (int) env('POLYDOCK_FAILED_SWEEP_MAX_PER_RUN', 25),
     ],
     'deploy' => [
         // Max instances a single scheduled-redeploy tick will trigger. Combined
@@ -77,6 +81,7 @@
     'register_simulate_round_robin' => env('POLYDOCK_REGISTER_SIMULATE_ROUND_ROBIN', false),
     'register_simulate_error' => env('POLYDOCK_REGISTER_SIMULATE_ERROR', false),
     'lagoon_deploy_private_key_file' => env('POLYDOCK_LAGOON_DEPLOY_PRIVATE_KEY_FILE', 'tests/fixtures/lagoon-deploy-private-key'),
+    'ftlagoon_private_key_content' => env('FTLAGOON_PRIVATE_KEY_CONTENT'),
     'service_providers_singletons' => $serviceProviderSingletons,
     'lagoon_cores' => [
         'http://lagoon-api.172.22.0.240.nip.io/graphql' => [

database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php

@@ -2,28 +2,63 @@
 
 use Illuminate\Database\Migrations\Migration;
 use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Schema;
 
 return new class extends Migration
 {
+    private const TABLE = 'polydock_app_instance_status_transitions';
+
     public function up(): void
     {
-        Schema::create('polydock_app_instance_status_transitions', function (Blueprint $table) {
-            $table->id();
-            $table->foreignId('polydock_app_instance_id')
-                ->constrained('polydock_app_instances')
-                ->cascadeOnDelete();
-            // Null for the row recorded when an instance is created as NEW.
-            $table->string('from_status')->nullable();
-            $table->string('to_status');
-            $table->timestamp('created_at');
-
-            $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx');
-        });
+        if (! Schema::hasTable(self::TABLE)) {
+            Schema::create(self::TABLE, function (Blueprint $table) {
+                $table->id();
+                $table->foreignId('polydock_app_instance_id')
+                    ->constrained('polydock_app_instances')
+                    ->cascadeOnDelete();
+                // Null for the row recorded when an instance is created as NEW.
+                $table->string('from_status')->nullable();
+                $table->string('to_status');
+                $table->timestamp('created_at');
+
+                $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx');
+            });
+
+            return;
+        }
+
+        // Self-heal: MySQL DDL is not transactional, so a migrate run killed
+        // between `create table` and the trailing FK alter leaves the table
+        // behind with the migration unrecorded — every later deploy then dies
+        // with "table already exists". Finish the missing pieces instead.
+        if (! in_array('papist_instance_created_idx', Schema::getIndexListing(self::TABLE))) {
+            Schema::table(self::TABLE, function (Blueprint $table) {
+                $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx');
+            });
+        }
+
+        $hasForeignKey = collect(Schema::getForeignKeys(self::TABLE))
+            ->contains(fn (array $fk) => $fk['columns'] === ['polydock_app_instance_id']);
+
+        if (! $hasForeignKey) {
+            // Rows orphaned while the cascade FK was missing would block the
+            // constraint from applying.
+            DB::table(self::TABLE)
+                ->whereNotIn('polydock_app_instance_id', DB::table('polydock_app_instances')->select('id'))
+                ->delete();
+
+            Schema::table(self::TABLE, function (Blueprint $table) {
+                $table->foreign('polydock_app_instance_id')
+                    ->references('id')
+                    ->on('polydock_app_instances')
+                    ->cascadeOnDelete();
+            });
+        }
     }
 
     public function down(): void
     {
-        Schema::dropIfExists('polydock_app_instance_status_transitions');
+        Schema::dropIfExists(self::TABLE);
     }
 };

database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php

@@ -0,0 +1,46 @@
+<?php
+
+declare(strict_types=1);
+
+use App\Polydock\Core\Enums\PolydockAppInstanceStatus;
+use Carbon\Carbon;
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Support\Facades\DB;
+
+/**
+ * Instances that reached REMOVED before the project-purge feature shipped
+ * have purge_eligible_at = NULL, so DispatchProjectPurgeJobsCommand never
+ * selects them and their Lagoon projects linger forever. Backfill the grace
+ * clock from removed_at (or updated_at for rows predating that column too).
+ */
+return new class extends Migration
+{
+    public function up(): void
+    {
+        $graceDays = (int) config('polydock.cleanup.project_grace_period_days', 14);
+
+        DB::table('polydock_app_instances')
+            ->where('status', PolydockAppInstanceStatus::REMOVED->value)
+            ->whereNull('removed_at')
+            ->update(['removed_at' => DB::raw('updated_at')]);
+
+        DB::table('polydock_app_instances')
+            ->where('status', PolydockAppInstanceStatus::REMOVED->value)
+            ->whereNull('purge_eligible_at')
+            ->orderBy('id')
+            ->chunkById(500, function ($rows) use ($graceDays) {
+                foreach ($rows as $row) {
+                    DB::table('polydock_app_instances')
+                        ->where('id', $row->id)
+                        ->update([
+                            'purge_eligible_at' => Carbon::parse($row->removed_at)->addDays($graceDays),
+                        ]);
+                }
+            });
+    }
+
+    public function down(): void
+    {
+        // Backfill only; nothing sensible to restore.
+    }
+};

routes/console.php

@@ -62,6 +62,12 @@
     ->everyFifteenMinutes()
     ->withoutOverlapping();
 
+// ///// Stale Failed Instance Sweep ///////
+Schedule::command('polydock:remove-stale-failed-instances')
+    ->hourlyAt(50)
+    ->withoutOverlapping()
+    ->onOneServer();
+
 // ///// Project Purge (full Lagoon project deletion after grace period) ///////
 Schedule::command('polydock:dispatch-project-purge')
     ->everyTenMinutes()