From 2d1f97430508f198c553e8726f534f9d0201066b Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 21 Jul 2026 23:20:18 +0200 Subject: [PATCH 1/7] chore: auto-cleanup for stale failed instances and pre-purge-era removed rows --- .../RemoveStaleFailedInstancesCommand.php | 136 +++++++++++++++ .../RemoveUnclaimedAppInstancesCommand.php | 2 +- config/polydock.php | 4 + ...purge_eligible_at_on_removed_instances.php | 46 +++++ routes/console.php | 6 + .../RemoveStaleFailedInstancesCommandTest.php | 165 ++++++++++++++++++ 6 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 app/Console/Commands/RemoveStaleFailedInstancesCommand.php create mode 100644 database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php create mode 100644 tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php diff --git a/app/Console/Commands/RemoveStaleFailedInstancesCommand.php b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php new file mode 100644 index 00000000..d21e57d4 --- /dev/null +++ b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php @@ -0,0 +1,136 @@ + PENDING_PRE_REMOVE (normal remove flow, + * app-specific pre/post-remove hooks still run) + * - remove-stage failures -> REMOVED (the remove flow already failed once; + * project purge deletes the whole Lagoon project, and treats an + * already-gone project as success) + * + * Both get force_purge_requested_at stamped so the purge skips the 14-day + * grace period — failed instances hold no user data worth a grace window. + * PURGE_FAILED is deliberately excluded: it means the purge polling cap was + * reached and an admin must explicitly retry. + */ +class RemoveStaleFailedInstancesCommand extends BaseCommand +{ + protected $signature = 'polydock:remove-stale-failed-instances + {--dry-run : List eligible instances without making changes} + {--days= : Override the retention window in days} + {--limit= : Override the per-run limit}'; + + protected $description = 'Push failed app instances older than the retention window into the remove/purge pipeline'; + + /** + * Failures before or during the remove stage get the full remove flow. + * + * @return array + */ + private static function preRemovalFailedStatuses(): array + { + return [ + PolydockAppInstanceStatus::PRE_CREATE_FAILED, + PolydockAppInstanceStatus::CREATE_FAILED, + PolydockAppInstanceStatus::POST_CREATE_FAILED, + PolydockAppInstanceStatus::PRE_DEPLOY_FAILED, + PolydockAppInstanceStatus::DEPLOY_FAILED, + PolydockAppInstanceStatus::POST_DEPLOY_FAILED, + PolydockAppInstanceStatus::POLYDOCK_CLAIM_FAILED, + ]; + } + + /** + * Failures inside the remove stage skip straight to REMOVED so the + * project purge can delete the whole Lagoon project. + * + * @return array + */ + private static function removeStageFailedStatuses(): array + { + return [ + PolydockAppInstanceStatus::PRE_REMOVE_FAILED, + PolydockAppInstanceStatus::REMOVE_FAILED, + PolydockAppInstanceStatus::POST_REMOVE_FAILED, + ]; + } + + public function handle(): int + { + $isDryRun = (bool) $this->option('dry-run'); + $days = (int) ($this->option('days') ?? config('polydock.cleanup.failed_instance_retention_days', 7)); + $limit = (int) ($this->option('limit') ?? config('polydock.cleanup.failed_sweep_max_per_run', 25)); + $cutoff = now()->subDays($days); + + $eligible = PolydockAppInstance::query() + ->whereIn('status', array_merge(self::preRemovalFailedStatuses(), self::removeStageFailedStatuses())) + ->where('updated_at', '<=', $cutoff) + ->orderBy('updated_at') + ->limit($limit) + ->get(); + + if ($eligible->isEmpty()) { + $this->info("No failed instances older than {$days} days found."); + + return self::SUCCESS; + } + + $this->info(sprintf('Found %d stale failed instance(s) (older than %d days).', $eligible->count(), $days)); + + $swept = 0; + + foreach ($eligible as $instance) { + $isRemoveStageFailure = in_array($instance->status, self::removeStageFailedStatuses(), true); + $target = $isRemoveStageFailure + ? PolydockAppInstanceStatus::REMOVED + : PolydockAppInstanceStatus::PENDING_PRE_REMOVE; + + $line = sprintf( + ' - %s (id=%d, %s -> %s)', + $instance->name, + $instance->id, + $instance->status->value, + $target->value, + ); + + if ($isDryRun) { + $this->line('[dry-run]'.$line); + + continue; + } + + $this->line($line); + + Log::info('Sweeping stale failed instance into removal pipeline', [ + 'app_instance_id' => $instance->id, + 'previous_status' => $instance->status->value, + 'target_status' => $target->value, + ]); + + $instance->force_purge_requested_at = $instance->force_purge_requested_at ?? now(); + // Status change fires PolydockAppInstanceStatusChanged, whose + // listener dispatches the stage job / purge transition. + $instance->setStatus($target, "Stale failed instance swept after {$days} days"); + $instance->save(); + + $swept++; + } + + if (! $isDryRun) { + Log::info('polydock:remove-stale-failed-instances swept instances', ['count' => $swept]); + } + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php b/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php index 74c21db5..14edda1f 100644 --- a/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php +++ b/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php @@ -40,7 +40,7 @@ public function handle(): int // Build the query for unclaimed instances older than specified days $query = PolydockAppInstance::where('status', PolydockAppInstanceStatus::RUNNING_HEALTHY_UNCLAIMED) - ->whereDate('created_at', '<=', now()->subDay()); + ->whereDate('created_at', '<=', now()->subDays($days)); // Add app filter if specified if ($appId) { diff --git a/config/polydock.php b/config/polydock.php index 408ed7a4..ba23c1a5 100644 --- a/config/polydock.php +++ b/config/polydock.php @@ -55,6 +55,10 @@ 'purge_poll_interval_minutes' => (int) env('POLYDOCK_PURGE_POLL_INTERVAL', 10), 'purge_max_poll_attempts' => (int) env('POLYDOCK_PURGE_MAX_POLLS', 144), 'purge_max_per_run' => (int) env('POLYDOCK_PURGE_MAX_PER_RUN', 25), + // Failed instances (create/deploy/claim/remove failures) are swept + // into the remove/purge pipeline after this many days. + 'failed_instance_retention_days' => (int) env('POLYDOCK_FAILED_RETENTION_DAYS', 7), + 'failed_sweep_max_per_run' => (int) env('POLYDOCK_FAILED_SWEEP_MAX_PER_RUN', 25), ], 'deploy' => [ // Max instances a single scheduled-redeploy tick will trigger. Combined diff --git a/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php b/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php new file mode 100644 index 00000000..703712f8 --- /dev/null +++ b/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php @@ -0,0 +1,46 @@ +where('status', PolydockAppInstanceStatus::REMOVED->value) + ->whereNull('removed_at') + ->update(['removed_at' => DB::raw('updated_at')]); + + DB::table('polydock_app_instances') + ->where('status', PolydockAppInstanceStatus::REMOVED->value) + ->whereNull('purge_eligible_at') + ->orderBy('id') + ->chunkById(500, function ($rows) use ($graceDays) { + foreach ($rows as $row) { + DB::table('polydock_app_instances') + ->where('id', $row->id) + ->update([ + 'purge_eligible_at' => Carbon::parse($row->removed_at)->addDays($graceDays), + ]); + } + }); + } + + public function down(): void + { + // Backfill only; nothing sensible to restore. + } +}; diff --git a/routes/console.php b/routes/console.php index c763f131..5d1afddf 100644 --- a/routes/console.php +++ b/routes/console.php @@ -62,6 +62,12 @@ ->everyFifteenMinutes() ->withoutOverlapping(); +// ///// Stale Failed Instance Sweep /////// +Schedule::command('polydock:remove-stale-failed-instances') + ->hourlyAt(50) + ->withoutOverlapping() + ->onOneServer(); + // ///// Project Purge (full Lagoon project deletion after grace period) /////// Schedule::command('polydock:dispatch-project-purge') ->everyTenMinutes() diff --git a/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php b/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php new file mode 100644 index 00000000..f516a6c8 --- /dev/null +++ b/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php @@ -0,0 +1,165 @@ +uuid = 'test-'.uniqid(); + $instance->polydock_store_app_id = $storeApp->id; + $instance->name = 'test-instance-'.uniqid(); + $instance->status = $status; + $instance->app_type = 'test_app_type'; + $instance->data = []; + $instance->saveQuietly(); + + if ($updatedAt !== null) { + PolydockAppInstance::where('id', $instance->id) + ->update(['updated_at' => $updatedAt]); + $instance->refresh(); + } + + return $instance; + } + + private function createStoreApp(): PolydockStoreApp + { + $store = PolydockStore::factory()->create(); + + return PolydockStoreApp::factory()->create([ + 'polydock_store_id' => $store->id, + ]); + } + + public function test_sweeps_old_failed_instance_into_remove_flow(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::PENDING_PRE_REMOVE, $instance->status); + $this->assertNotNull($instance->force_purge_requested_at); + } + + public function test_remove_stage_failures_go_straight_to_removed(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::REMOVE_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertNotNull($instance->force_purge_requested_at); + $this->assertNotNull($instance->removed_at); + // The force-purge listener path may immediately advance REMOVED to + // PENDING_PURGE; both mean the purge pipeline now owns it. + $this->assertContains($instance->status, [ + PolydockAppInstanceStatus::REMOVED, + PolydockAppInstanceStatus::PENDING_PURGE, + ]); + } + + public function test_recent_failures_are_left_alone(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(2)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::DEPLOY_FAILED, $instance->status); + $this->assertNull($instance->force_purge_requested_at); + } + + public function test_purge_failed_is_excluded(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::PURGE_FAILED, + now()->subDays(30)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::PURGE_FAILED, $instance->status); + } + + public function test_dry_run_does_not_mutate(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', [ + '--days' => 7, + '--dry-run' => true, + ])->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::DEPLOY_FAILED, $instance->status); + $this->assertNull($instance->force_purge_requested_at); + } + + public function test_respects_limit(): void + { + $storeApp = $this->createStoreApp(); + $old = now()->subDays(10)->toDateTimeString(); + $a = $this->createInstance($storeApp, PolydockAppInstanceStatus::DEPLOY_FAILED, $old); + $b = $this->createInstance($storeApp, PolydockAppInstanceStatus::DEPLOY_FAILED, $old); + + $this->artisan('polydock:remove-stale-failed-instances', [ + '--days' => 7, + '--limit' => 1, + ])->assertSuccessful(); + + $statuses = collect([$a->fresh()->status, $b->fresh()->status]); + $this->assertCount(1, $statuses->filter( + fn ($s) => $s === PolydockAppInstanceStatus::PENDING_PRE_REMOVE, + )); + } +} From 3aa3f7f7a2b537967102035016f2ee461a636397 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 21 Jul 2026 23:42:15 +0200 Subject: [PATCH 2/7] chore: wire ftlagoon_private_key_content to FTLAGOON_PRIVATE_KEY_CONTENT env --- config/polydock.php | 1 + 1 file changed, 1 insertion(+) diff --git a/config/polydock.php b/config/polydock.php index 408ed7a4..b2f2ca36 100644 --- a/config/polydock.php +++ b/config/polydock.php @@ -77,6 +77,7 @@ 'register_simulate_round_robin' => env('POLYDOCK_REGISTER_SIMULATE_ROUND_ROBIN', false), 'register_simulate_error' => env('POLYDOCK_REGISTER_SIMULATE_ERROR', false), 'lagoon_deploy_private_key_file' => env('POLYDOCK_LAGOON_DEPLOY_PRIVATE_KEY_FILE', 'tests/fixtures/lagoon-deploy-private-key'), + 'ftlagoon_private_key_content' => env('FTLAGOON_PRIVATE_KEY_CONTENT'), 'service_providers_singletons' => $serviceProviderSingletons, 'lagoon_cores' => [ 'http://lagoon-api.172.22.0.240.nip.io/graphql' => [ From bf8b027fdbce6feaab48deb5265e49492a8746fd Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 21 Jul 2026 23:51:29 +0200 Subject: [PATCH 3/7] fix: make status transitions migration self-heal after partial DDL failure --- ..._app_instance_status_transitions_table.php | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php b/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php index bf05df2a..90b88d8d 100644 --- a/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php +++ b/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php @@ -2,28 +2,63 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; return new class extends Migration { + private const TABLE = 'polydock_app_instance_status_transitions'; + public function up(): void { - Schema::create('polydock_app_instance_status_transitions', function (Blueprint $table) { - $table->id(); - $table->foreignId('polydock_app_instance_id') - ->constrained('polydock_app_instances') - ->cascadeOnDelete(); - // Null for the row recorded when an instance is created as NEW. - $table->string('from_status')->nullable(); - $table->string('to_status'); - $table->timestamp('created_at'); - - $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); - }); + if (! Schema::hasTable(self::TABLE)) { + Schema::create(self::TABLE, function (Blueprint $table) { + $table->id(); + $table->foreignId('polydock_app_instance_id') + ->constrained('polydock_app_instances') + ->cascadeOnDelete(); + // Null for the row recorded when an instance is created as NEW. + $table->string('from_status')->nullable(); + $table->string('to_status'); + $table->timestamp('created_at'); + + $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); + }); + + return; + } + + // Self-heal: MySQL DDL is not transactional, so a migrate run killed + // between `create table` and the trailing FK alter leaves the table + // behind with the migration unrecorded — every later deploy then dies + // with "table already exists". Finish the missing pieces instead. + if (! in_array('papist_instance_created_idx', Schema::getIndexListing(self::TABLE))) { + Schema::table(self::TABLE, function (Blueprint $table) { + $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); + }); + } + + $hasForeignKey = collect(Schema::getForeignKeys(self::TABLE)) + ->contains(fn (array $fk) => $fk['columns'] === ['polydock_app_instance_id']); + + if (! $hasForeignKey) { + // Rows orphaned while the cascade FK was missing would block the + // constraint from applying. + DB::table(self::TABLE) + ->whereNotIn('polydock_app_instance_id', DB::table('polydock_app_instances')->select('id')) + ->delete(); + + Schema::table(self::TABLE, function (Blueprint $table) { + $table->foreign('polydock_app_instance_id') + ->references('id') + ->on('polydock_app_instances') + ->cascadeOnDelete(); + }); + } } public function down(): void { - Schema::dropIfExists('polydock_app_instance_status_transitions'); + Schema::dropIfExists(self::TABLE); } }; From 858605752e733b2ad861ea9f5953f7b84d681a87 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 21 Jul 2026 23:54:20 +0200 Subject: [PATCH 4/7] chore: pause horizon around deploy migrations --- .lagoon.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.lagoon.yml b/.lagoon.yml index 06c66938..edf1d835 100644 --- a/.lagoon.yml +++ b/.lagoon.yml @@ -25,7 +25,16 @@ tasks: shell: bash - run: name: Run migrations - command: php artisan -n migrate --force + # Pause horizon workers (broadcast via redis to all pods) so long + # queue jobs can't hold metadata locks that stall migration DDL. + # Resume even when migrate fails — a failed deploy must not leave + # the queue paused. + command: | + php artisan horizon:pause || true + php artisan -n migrate --force + status=$? + php artisan horizon:continue || true + exit $status service: cli shell: bash - run: From 7197688f2227a4af1db76ba7f3646b31c2180272 Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Tue, 21 Jul 2026 23:58:49 +0200 Subject: [PATCH 5/7] fix: re-check instance state under row lock before stale-failure sweep --- .../RemoveStaleFailedInstancesCommand.php | 38 ++++++++++++++++--- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/app/Console/Commands/RemoveStaleFailedInstancesCommand.php b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php index d21e57d4..76b2242d 100644 --- a/app/Console/Commands/RemoveStaleFailedInstancesCommand.php +++ b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php @@ -6,6 +6,7 @@ use App\Models\PolydockAppInstance; use App\Polydock\Core\Enums\PolydockAppInstanceStatus; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Log; /** @@ -110,6 +111,37 @@ public function handle(): int continue; } + // Re-check under a row lock before writing: a retry or operator + // may have recovered the instance between the eligibility query + // and this write, and saving the stale snapshot would shove a + // live instance into removal/purge. + $sweptThisOne = DB::transaction(function () use ($instance, $cutoff, $target, $days) { + $fresh = PolydockAppInstance::query() + ->whereKey($instance->id) + ->lockForUpdate() + ->first(); + + if ($fresh === null + || $fresh->status !== $instance->status + || $fresh->updated_at > $cutoff) { + return false; + } + + $fresh->force_purge_requested_at = $fresh->force_purge_requested_at ?? now(); + // Status change fires PolydockAppInstanceStatusChanged, whose + // listener dispatches the stage job / purge transition. + $fresh->setStatus($target, "Stale failed instance swept after {$days} days"); + $fresh->save(); + + return true; + }); + + if (! $sweptThisOne) { + $this->line(sprintf(' - %s (id=%d) skipped: state changed since selection', $instance->name, $instance->id)); + + continue; + } + $this->line($line); Log::info('Sweeping stale failed instance into removal pipeline', [ @@ -118,12 +150,6 @@ public function handle(): int 'target_status' => $target->value, ]); - $instance->force_purge_requested_at = $instance->force_purge_requested_at ?? now(); - // Status change fires PolydockAppInstanceStatusChanged, whose - // listener dispatches the stage job / purge transition. - $instance->setStatus($target, "Stale failed instance swept after {$days} days"); - $instance->save(); - $swept++; } From 494a92b763d0630214199ef10fd32d7a7921f79c Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Wed, 22 Jul 2026 00:01:00 +0200 Subject: [PATCH 6/7] fix: add property docblock to status transition model for phpstan --- app/Models/PolydockAppInstanceStatusTransition.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/Models/PolydockAppInstanceStatusTransition.php b/app/Models/PolydockAppInstanceStatusTransition.php index b235fa4c..b4628c3e 100644 --- a/app/Models/PolydockAppInstanceStatusTransition.php +++ b/app/Models/PolydockAppInstanceStatusTransition.php @@ -7,11 +7,18 @@ use App\Polydock\Core\Enums\PolydockAppInstanceStatus; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Support\Carbon; /** * One row per status change of an app instance — the timing source of truth * for per-stage durations (the activity log deliberately does not record * status). Rows are immutable and cascade-delete with the instance. + * + * @property int $id + * @property int $polydock_app_instance_id + * @property PolydockAppInstanceStatus|null $from_status + * @property PolydockAppInstanceStatus $to_status + * @property Carbon $created_at */ class PolydockAppInstanceStatusTransition extends Model { From d798bf5ce2956ea7b06b6abcca9cad66fac5412c Mon Sep 17 00:00:00 2001 From: Dan Lemon Date: Wed, 22 Jul 2026 00:04:26 +0200 Subject: [PATCH 7/7] fix: resume horizon via shell trap so a killed deploy task cannot leave queues paused --- .lagoon.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.lagoon.yml b/.lagoon.yml index edf1d835..39d95d60 100644 --- a/.lagoon.yml +++ b/.lagoon.yml @@ -27,14 +27,14 @@ tasks: name: Run migrations # Pause horizon workers (broadcast via redis to all pods) so long # queue jobs can't hold metadata locks that stall migration DDL. - # Resume even when migrate fails — a failed deploy must not leave - # the queue paused. + # The EXIT/TERM/INT trap resumes horizon on every exit path — + # migrate failure, task cancellation, pod termination — so a dead + # deploy can't leave the queue paused. Migrate's exit status still + # propagates as the task result. command: | php artisan horizon:pause || true + trap 'php artisan horizon:continue || true' EXIT TERM INT php artisan -n migrate --force - status=$? - php artisan horizon:continue || true - exit $status service: cli shell: bash - run: