diff --git a/.lagoon.yml b/.lagoon.yml index 06c66938..39d95d60 100644 --- a/.lagoon.yml +++ b/.lagoon.yml @@ -25,7 +25,16 @@ tasks: shell: bash - run: name: Run migrations - command: php artisan -n migrate --force + # Pause horizon workers (broadcast via redis to all pods) so long + # queue jobs can't hold metadata locks that stall migration DDL. + # The EXIT/TERM/INT trap resumes horizon on every exit path — + # migrate failure, task cancellation, pod termination — so a dead + # deploy can't leave the queue paused. Migrate's exit status still + # propagates as the task result. + command: | + php artisan horizon:pause || true + trap 'php artisan horizon:continue || true' EXIT TERM INT + php artisan -n migrate --force service: cli shell: bash - run: diff --git a/app/Console/Commands/RemoveStaleFailedInstancesCommand.php b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php new file mode 100644 index 00000000..76b2242d --- /dev/null +++ b/app/Console/Commands/RemoveStaleFailedInstancesCommand.php @@ -0,0 +1,162 @@ + PENDING_PRE_REMOVE (normal remove flow, + * app-specific pre/post-remove hooks still run) + * - remove-stage failures -> REMOVED (the remove flow already failed once; + * project purge deletes the whole Lagoon project, and treats an + * already-gone project as success) + * + * Both get force_purge_requested_at stamped so the purge skips the 14-day + * grace period — failed instances hold no user data worth a grace window. + * PURGE_FAILED is deliberately excluded: it means the purge polling cap was + * reached and an admin must explicitly retry. + */ +class RemoveStaleFailedInstancesCommand extends BaseCommand +{ + protected $signature = 'polydock:remove-stale-failed-instances + {--dry-run : List eligible instances without making changes} + {--days= : Override the retention window in days} + {--limit= : Override the per-run limit}'; + + protected $description = 'Push failed app instances older than the retention window into the remove/purge pipeline'; + + /** + * Failures before or during the remove stage get the full remove flow. + * + * @return array + */ + private static function preRemovalFailedStatuses(): array + { + return [ + PolydockAppInstanceStatus::PRE_CREATE_FAILED, + PolydockAppInstanceStatus::CREATE_FAILED, + PolydockAppInstanceStatus::POST_CREATE_FAILED, + PolydockAppInstanceStatus::PRE_DEPLOY_FAILED, + PolydockAppInstanceStatus::DEPLOY_FAILED, + PolydockAppInstanceStatus::POST_DEPLOY_FAILED, + PolydockAppInstanceStatus::POLYDOCK_CLAIM_FAILED, + ]; + } + + /** + * Failures inside the remove stage skip straight to REMOVED so the + * project purge can delete the whole Lagoon project. + * + * @return array + */ + private static function removeStageFailedStatuses(): array + { + return [ + PolydockAppInstanceStatus::PRE_REMOVE_FAILED, + PolydockAppInstanceStatus::REMOVE_FAILED, + PolydockAppInstanceStatus::POST_REMOVE_FAILED, + ]; + } + + public function handle(): int + { + $isDryRun = (bool) $this->option('dry-run'); + $days = (int) ($this->option('days') ?? config('polydock.cleanup.failed_instance_retention_days', 7)); + $limit = (int) ($this->option('limit') ?? config('polydock.cleanup.failed_sweep_max_per_run', 25)); + $cutoff = now()->subDays($days); + + $eligible = PolydockAppInstance::query() + ->whereIn('status', array_merge(self::preRemovalFailedStatuses(), self::removeStageFailedStatuses())) + ->where('updated_at', '<=', $cutoff) + ->orderBy('updated_at') + ->limit($limit) + ->get(); + + if ($eligible->isEmpty()) { + $this->info("No failed instances older than {$days} days found."); + + return self::SUCCESS; + } + + $this->info(sprintf('Found %d stale failed instance(s) (older than %d days).', $eligible->count(), $days)); + + $swept = 0; + + foreach ($eligible as $instance) { + $isRemoveStageFailure = in_array($instance->status, self::removeStageFailedStatuses(), true); + $target = $isRemoveStageFailure + ? PolydockAppInstanceStatus::REMOVED + : PolydockAppInstanceStatus::PENDING_PRE_REMOVE; + + $line = sprintf( + ' - %s (id=%d, %s -> %s)', + $instance->name, + $instance->id, + $instance->status->value, + $target->value, + ); + + if ($isDryRun) { + $this->line('[dry-run]'.$line); + + continue; + } + + // Re-check under a row lock before writing: a retry or operator + // may have recovered the instance between the eligibility query + // and this write, and saving the stale snapshot would shove a + // live instance into removal/purge. + $sweptThisOne = DB::transaction(function () use ($instance, $cutoff, $target, $days) { + $fresh = PolydockAppInstance::query() + ->whereKey($instance->id) + ->lockForUpdate() + ->first(); + + if ($fresh === null + || $fresh->status !== $instance->status + || $fresh->updated_at > $cutoff) { + return false; + } + + $fresh->force_purge_requested_at = $fresh->force_purge_requested_at ?? now(); + // Status change fires PolydockAppInstanceStatusChanged, whose + // listener dispatches the stage job / purge transition. + $fresh->setStatus($target, "Stale failed instance swept after {$days} days"); + $fresh->save(); + + return true; + }); + + if (! $sweptThisOne) { + $this->line(sprintf(' - %s (id=%d) skipped: state changed since selection', $instance->name, $instance->id)); + + continue; + } + + $this->line($line); + + Log::info('Sweeping stale failed instance into removal pipeline', [ + 'app_instance_id' => $instance->id, + 'previous_status' => $instance->status->value, + 'target_status' => $target->value, + ]); + + $swept++; + } + + if (! $isDryRun) { + Log::info('polydock:remove-stale-failed-instances swept instances', ['count' => $swept]); + } + + return self::SUCCESS; + } +} diff --git a/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php b/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php index 74c21db5..14edda1f 100644 --- a/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php +++ b/app/Console/Commands/RemoveUnclaimedAppInstancesCommand.php @@ -40,7 +40,7 @@ public function handle(): int // Build the query for unclaimed instances older than specified days $query = PolydockAppInstance::where('status', PolydockAppInstanceStatus::RUNNING_HEALTHY_UNCLAIMED) - ->whereDate('created_at', '<=', now()->subDay()); + ->whereDate('created_at', '<=', now()->subDays($days)); // Add app filter if specified if ($appId) { diff --git a/app/Models/PolydockAppInstanceStatusTransition.php b/app/Models/PolydockAppInstanceStatusTransition.php index b235fa4c..b4628c3e 100644 --- a/app/Models/PolydockAppInstanceStatusTransition.php +++ b/app/Models/PolydockAppInstanceStatusTransition.php @@ -7,11 +7,18 @@ use App\Polydock\Core\Enums\PolydockAppInstanceStatus; use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Support\Carbon; /** * One row per status change of an app instance — the timing source of truth * for per-stage durations (the activity log deliberately does not record * status). Rows are immutable and cascade-delete with the instance. + * + * @property int $id + * @property int $polydock_app_instance_id + * @property PolydockAppInstanceStatus|null $from_status + * @property PolydockAppInstanceStatus $to_status + * @property Carbon $created_at */ class PolydockAppInstanceStatusTransition extends Model { diff --git a/config/polydock.php b/config/polydock.php index 408ed7a4..2c885fe4 100644 --- a/config/polydock.php +++ b/config/polydock.php @@ -55,6 +55,10 @@ 'purge_poll_interval_minutes' => (int) env('POLYDOCK_PURGE_POLL_INTERVAL', 10), 'purge_max_poll_attempts' => (int) env('POLYDOCK_PURGE_MAX_POLLS', 144), 'purge_max_per_run' => (int) env('POLYDOCK_PURGE_MAX_PER_RUN', 25), + // Failed instances (create/deploy/claim/remove failures) are swept + // into the remove/purge pipeline after this many days. + 'failed_instance_retention_days' => (int) env('POLYDOCK_FAILED_RETENTION_DAYS', 7), + 'failed_sweep_max_per_run' => (int) env('POLYDOCK_FAILED_SWEEP_MAX_PER_RUN', 25), ], 'deploy' => [ // Max instances a single scheduled-redeploy tick will trigger. Combined @@ -77,6 +81,7 @@ 'register_simulate_round_robin' => env('POLYDOCK_REGISTER_SIMULATE_ROUND_ROBIN', false), 'register_simulate_error' => env('POLYDOCK_REGISTER_SIMULATE_ERROR', false), 'lagoon_deploy_private_key_file' => env('POLYDOCK_LAGOON_DEPLOY_PRIVATE_KEY_FILE', 'tests/fixtures/lagoon-deploy-private-key'), + 'ftlagoon_private_key_content' => env('FTLAGOON_PRIVATE_KEY_CONTENT'), 'service_providers_singletons' => $serviceProviderSingletons, 'lagoon_cores' => [ 'http://lagoon-api.172.22.0.240.nip.io/graphql' => [ diff --git a/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php b/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php index bf05df2a..90b88d8d 100644 --- a/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php +++ b/database/migrations/2026_07_17_140000_create_polydock_app_instance_status_transitions_table.php @@ -2,28 +2,63 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; return new class extends Migration { + private const TABLE = 'polydock_app_instance_status_transitions'; + public function up(): void { - Schema::create('polydock_app_instance_status_transitions', function (Blueprint $table) { - $table->id(); - $table->foreignId('polydock_app_instance_id') - ->constrained('polydock_app_instances') - ->cascadeOnDelete(); - // Null for the row recorded when an instance is created as NEW. - $table->string('from_status')->nullable(); - $table->string('to_status'); - $table->timestamp('created_at'); - - $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); - }); + if (! Schema::hasTable(self::TABLE)) { + Schema::create(self::TABLE, function (Blueprint $table) { + $table->id(); + $table->foreignId('polydock_app_instance_id') + ->constrained('polydock_app_instances') + ->cascadeOnDelete(); + // Null for the row recorded when an instance is created as NEW. + $table->string('from_status')->nullable(); + $table->string('to_status'); + $table->timestamp('created_at'); + + $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); + }); + + return; + } + + // Self-heal: MySQL DDL is not transactional, so a migrate run killed + // between `create table` and the trailing FK alter leaves the table + // behind with the migration unrecorded — every later deploy then dies + // with "table already exists". Finish the missing pieces instead. + if (! in_array('papist_instance_created_idx', Schema::getIndexListing(self::TABLE))) { + Schema::table(self::TABLE, function (Blueprint $table) { + $table->index(['polydock_app_instance_id', 'created_at'], 'papist_instance_created_idx'); + }); + } + + $hasForeignKey = collect(Schema::getForeignKeys(self::TABLE)) + ->contains(fn (array $fk) => $fk['columns'] === ['polydock_app_instance_id']); + + if (! $hasForeignKey) { + // Rows orphaned while the cascade FK was missing would block the + // constraint from applying. + DB::table(self::TABLE) + ->whereNotIn('polydock_app_instance_id', DB::table('polydock_app_instances')->select('id')) + ->delete(); + + Schema::table(self::TABLE, function (Blueprint $table) { + $table->foreign('polydock_app_instance_id') + ->references('id') + ->on('polydock_app_instances') + ->cascadeOnDelete(); + }); + } } public function down(): void { - Schema::dropIfExists('polydock_app_instance_status_transitions'); + Schema::dropIfExists(self::TABLE); } }; diff --git a/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php b/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php new file mode 100644 index 00000000..703712f8 --- /dev/null +++ b/database/migrations/2026_07_21_000001_backfill_purge_eligible_at_on_removed_instances.php @@ -0,0 +1,46 @@ +where('status', PolydockAppInstanceStatus::REMOVED->value) + ->whereNull('removed_at') + ->update(['removed_at' => DB::raw('updated_at')]); + + DB::table('polydock_app_instances') + ->where('status', PolydockAppInstanceStatus::REMOVED->value) + ->whereNull('purge_eligible_at') + ->orderBy('id') + ->chunkById(500, function ($rows) use ($graceDays) { + foreach ($rows as $row) { + DB::table('polydock_app_instances') + ->where('id', $row->id) + ->update([ + 'purge_eligible_at' => Carbon::parse($row->removed_at)->addDays($graceDays), + ]); + } + }); + } + + public function down(): void + { + // Backfill only; nothing sensible to restore. + } +}; diff --git a/routes/console.php b/routes/console.php index c763f131..5d1afddf 100644 --- a/routes/console.php +++ b/routes/console.php @@ -62,6 +62,12 @@ ->everyFifteenMinutes() ->withoutOverlapping(); +// ///// Stale Failed Instance Sweep /////// +Schedule::command('polydock:remove-stale-failed-instances') + ->hourlyAt(50) + ->withoutOverlapping() + ->onOneServer(); + // ///// Project Purge (full Lagoon project deletion after grace period) /////// Schedule::command('polydock:dispatch-project-purge') ->everyTenMinutes() diff --git a/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php b/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php new file mode 100644 index 00000000..f516a6c8 --- /dev/null +++ b/tests/Feature/Console/Commands/RemoveStaleFailedInstancesCommandTest.php @@ -0,0 +1,165 @@ +uuid = 'test-'.uniqid(); + $instance->polydock_store_app_id = $storeApp->id; + $instance->name = 'test-instance-'.uniqid(); + $instance->status = $status; + $instance->app_type = 'test_app_type'; + $instance->data = []; + $instance->saveQuietly(); + + if ($updatedAt !== null) { + PolydockAppInstance::where('id', $instance->id) + ->update(['updated_at' => $updatedAt]); + $instance->refresh(); + } + + return $instance; + } + + private function createStoreApp(): PolydockStoreApp + { + $store = PolydockStore::factory()->create(); + + return PolydockStoreApp::factory()->create([ + 'polydock_store_id' => $store->id, + ]); + } + + public function test_sweeps_old_failed_instance_into_remove_flow(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::PENDING_PRE_REMOVE, $instance->status); + $this->assertNotNull($instance->force_purge_requested_at); + } + + public function test_remove_stage_failures_go_straight_to_removed(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::REMOVE_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertNotNull($instance->force_purge_requested_at); + $this->assertNotNull($instance->removed_at); + // The force-purge listener path may immediately advance REMOVED to + // PENDING_PURGE; both mean the purge pipeline now owns it. + $this->assertContains($instance->status, [ + PolydockAppInstanceStatus::REMOVED, + PolydockAppInstanceStatus::PENDING_PURGE, + ]); + } + + public function test_recent_failures_are_left_alone(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(2)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::DEPLOY_FAILED, $instance->status); + $this->assertNull($instance->force_purge_requested_at); + } + + public function test_purge_failed_is_excluded(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::PURGE_FAILED, + now()->subDays(30)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', ['--days' => 7]) + ->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::PURGE_FAILED, $instance->status); + } + + public function test_dry_run_does_not_mutate(): void + { + $storeApp = $this->createStoreApp(); + $instance = $this->createInstance( + $storeApp, + PolydockAppInstanceStatus::DEPLOY_FAILED, + now()->subDays(10)->toDateTimeString(), + ); + + $this->artisan('polydock:remove-stale-failed-instances', [ + '--days' => 7, + '--dry-run' => true, + ])->assertSuccessful(); + + $instance->refresh(); + $this->assertEquals(PolydockAppInstanceStatus::DEPLOY_FAILED, $instance->status); + $this->assertNull($instance->force_purge_requested_at); + } + + public function test_respects_limit(): void + { + $storeApp = $this->createStoreApp(); + $old = now()->subDays(10)->toDateTimeString(); + $a = $this->createInstance($storeApp, PolydockAppInstanceStatus::DEPLOY_FAILED, $old); + $b = $this->createInstance($storeApp, PolydockAppInstanceStatus::DEPLOY_FAILED, $old); + + $this->artisan('polydock:remove-stale-failed-instances', [ + '--days' => 7, + '--limit' => 1, + ])->assertSuccessful(); + + $statuses = collect([$a->fresh()->status, $b->fresh()->status]); + $this->assertCount(1, $statuses->filter( + fn ($s) => $s === PolydockAppInstanceStatus::PENDING_PRE_REMOVE, + )); + } +}