GH#1839: fix(ci): retry killed PHPUnit suites

.github/workflows/tests.yml

@@ -109,11 +109,9 @@ jobs:
 
       - name: Run PHPUnit Tests
         if: matrix.php-version != '8.3'
-        run: |
-          # The full suite exceeds the container memory limit when run in one PHP process.
-          # Restart PHPUnit between small batches so every test file still runs.
-          find tests -name '*_Test.php' -type f -print0 \
-            | xargs -0 -r -n 50 vendor/bin/phpunit
+        # Run bounded XML suites rather than passing many positional paths to PHPUnit.
+        # Ten files keep the largest suites below the runner memory limit.
+        run: PHPUNIT_BATCH_SIZE=10 php scripts/run-phpunit-batches.php
 
       - name: Run PHPUnit Tests with Coverage
         if: matrix.php-version == '8.3'

scripts/run-phpunit-batches.php

@@ -0,0 +1,313 @@
+<?php
+/**
+ * Run PHPUnit test files in deterministic, memory-bounded suites.
+ *
+ * PHPUnit accepts one file or directory as its positional test target. Passing
+ * many files through xargs silently runs only the first file in each batch.
+ * This runner creates a temporary configuration for every batch and removes
+ * plugin-owned test tables between processes so reset WordPress IDs cannot
+ * collide with stale Ultimate Multisite records. A killed multi-file suite is
+ * retried one file at a time, preserving failures while lowering peak memory.
+ *
+ * @package WP_Ultimo
+ */
+
+// WordPress is not loaded in this CLI runner, so native file, database, and process APIs are required.
+// phpcs:disable WordPress.WP.AlternativeFunctions, WordPress.PHP.DiscouragedPHPFunctions.system_calls_proc_open, WordPress.DB.RestrictedFunctions
+
+$project_root = dirname(__DIR__);
+$phpunit      = $project_root . '/vendor/phpunit/phpunit/phpunit';
+$bootstrap    = $project_root . '/tests/bootstrap.php';
+$batch_size   = (int) (getenv('PHPUNIT_BATCH_SIZE') ?: 1);
+$batch_number = (int) (getenv('PHPUNIT_BATCH_NUMBER') ?: 0);
+$input_paths  = array_slice($argv, 1) ?: [$project_root . '/tests'];
+
+if ($batch_size < 1) {
+	fwrite(STDERR, "PHPUNIT_BATCH_SIZE must be at least 1.\n");
+	exit(2);
+}
+
+if ($batch_number < 0) {
+	fwrite(STDERR, "PHPUNIT_BATCH_NUMBER cannot be negative.\n");
+	exit(2);
+}
+
+if ( ! is_file($phpunit) || ! is_file($bootstrap)) {
+	fwrite(STDERR, "Install Composer dependencies before running PHPUnit batches.\n");
+	exit(2);
+}
+
+$tests_directory = getenv('WP_TESTS_DIR') ?: rtrim(sys_get_temp_dir(), '/\\') . '/wordpress-tests-lib';
+$config_path     = getenv('WP_TESTS_CONFIG_FILE_PATH') ?: $tests_directory . '/wp-tests-config.php';
+
+if (is_dir($config_path)) {
+	$config_path = rtrim($config_path, '/\\') . '/wp-tests-config.php';
+}
+
+if ( ! is_file($config_path)) {
+	fwrite(STDERR, "The WordPress test configuration could not be found.\n");
+	exit(2);
+}
+
+require $config_path;
+
+if ( ! class_exists('mysqli') || ! defined('DB_NAME') || ! defined('DB_USER') || ! defined('DB_PASSWORD') || ! defined('DB_HOST') || ! isset($table_prefix)) {
+	fwrite(STDERR, "The WordPress test database configuration is incomplete.\n");
+	exit(2);
+}
+
+$test_files = [];
+
+foreach ($input_paths as $input_path) {
+	$path = realpath($input_path);
+
+	if (false === $path) {
+		fwrite(STDERR, sprintf("Test path does not exist: %s\n", $input_path));
+		exit(2);
+	}
+
+	if (is_file($path)) {
+		if (str_ends_with($path, 'Test.php')) {
+			$test_files[] = $path;
+		}
+
+		continue;
+	}
+
+	$iterator = new RecursiveIteratorIterator(
+		new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS)
+	);
+
+	foreach ($iterator as $file) {
+		if ($file->isFile() && str_ends_with($file->getPathname(), 'Test.php')) {
+			$test_files[] = $file->getPathname();
+		}
+	}
+}
+
+$test_files = array_values(array_unique($test_files));
+sort($test_files, SORT_STRING);
+
+if (empty($test_files)) {
+	fwrite(STDERR, "No PHPUnit test files were found.\n");
+	exit(2);
+}
+
+$clean_plugin_tables = static function () use ($table_prefix) {
+	$db_host = DB_HOST;
+	$db_port = 0;
+
+	if (preg_match('/^([^:]+):(\d+)$/', DB_HOST, $host_parts)) {
+		$db_host = $host_parts[1];
+		$db_port = (int) $host_parts[2];
+	}
+
+	mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT);
+	$database = mysqli_init();
+	$database->real_connect($db_host, DB_USER, DB_PASSWORD, DB_NAME, $db_port);
+	$tables   = $database->query('SHOW TABLES');
+	$prefixes = [
+		$table_prefix . 'wu_',
+		$table_prefix . 'actionscheduler_',
+	];
+
+	$database->query('SET FOREIGN_KEY_CHECKS = 0');
+
+	while (true) {
+		$table = $tables->fetch_row();
+
+		if (null === $table) {
+			break;
+		}
+
+		foreach ($prefixes as $prefix) {
+			if (str_starts_with($table[0], $prefix)) {
+				$table_name = str_replace('`', '``', $table[0]);
+				$database->query("DROP TABLE IF EXISTS `{$table_name}`");
+				break;
+			}
+		}
+	}
+
+	$database->query('SET FOREIGN_KEY_CHECKS = 1');
+	$database->close();
+};
+
+$count_executed_files = static function ($junit_path, $batch) {
+	if ( ! is_file($junit_path)) {
+		throw new RuntimeException('PHPUnit did not write its JUnit report.');
+	}
+
+	$report = simplexml_load_file($junit_path);
+
+	if (false === $report) {
+		throw new RuntimeException('PHPUnit wrote an unreadable JUnit report.');
+	}
+
+	$test_cases = $report->xpath('//testcase[@file]');
+
+	if (false === $test_cases) {
+		throw new RuntimeException('PHPUnit JUnit report test cases could not be read.');
+	}
+
+	$expected_files = array_fill_keys($batch, true);
+	$executed_files = [];
+
+	foreach ($test_cases as $test_case) {
+		$test_file = realpath((string) $test_case['file']);
+
+		if (false !== $test_file && isset($expected_files[ $test_file ])) {
+			$executed_files[ $test_file ] = true;
+		}
+	}
+
+	return count($executed_files);
+};
+
+$batches        = array_chunk($test_files, $batch_size);
+$total_batches  = count($batches);
+$expected_files = count($test_files);
+$executed_files = 0;
+$batch_offset   = 0;
+$exit_status    = 0;
+
+if ($batch_number) {
+	if ( ! isset($batches[ $batch_number - 1 ])) {
+		fwrite(STDERR, sprintf("PHPUnit batch %d does not exist; expected 1-%d.\n", $batch_number, $total_batches));
+		exit(2);
+	}
+
+	$batch_offset   = $batch_number - 1;
+	$batches        = [$batches[ $batch_offset ]];
+	$expected_files = count($batches[0]);
+}
+
+foreach ($batches as $index => $batch) {
+	$current_batch = $batch_offset + $index + 1;
+
+	try {
+		$clean_plugin_tables();
+	} catch (Throwable $exception) {
+		fwrite(STDERR, sprintf("Unable to reset plugin test tables: %s\n", $exception->getMessage()));
+		exit(2);
+	}
+
+	$config_path = tempnam(sys_get_temp_dir(), 'wu-phpunit-batch-');
+	$junit_path  = tempnam(sys_get_temp_dir(), 'wu-phpunit-junit-');
+
+	if (false === $config_path || false === $junit_path) {
+		if (false !== $config_path) {
+			unlink($config_path);
+		}
+
+		if (false !== $junit_path) {
+			unlink($junit_path);
+		}
+
+		fwrite(STDERR, "Unable to create a temporary PHPUnit configuration.\n");
+		exit(2);
+	}
+
+	$file_nodes = array_map(
+		static fn($file) => '      <file>' . htmlspecialchars($file, ENT_XML1 | ENT_QUOTES, 'UTF-8') . '</file>',
+		$batch
+	);
+	$config     = sprintf(
+		"<?xml version=\"1.0\"?>\n<phpunit bootstrap=\"%s\" backupGlobals=\"false\" colors=\"true\">\n  <php>\n    <const name=\"WP_TESTS_MULTISITE\" value=\"1\"/>\n  </php>\n  <testsuites>\n    <testsuite name=\"batch-%d\">\n%s\n    </testsuite>\n  </testsuites>\n</phpunit>\n",
+		htmlspecialchars($bootstrap, ENT_XML1 | ENT_QUOTES, 'UTF-8'),
+		$current_batch,
+		implode("\n", $file_nodes)
+	);
+
+	if (strlen($config) !== file_put_contents($config_path, $config)) {
+		unlink($config_path);
+		unlink($junit_path);
+		fwrite(STDERR, "Unable to write a temporary PHPUnit configuration.\n");
+		exit(2);
+	}
+
+	fwrite(STDOUT, sprintf("Running PHPUnit batch %d/%d (%d files)\n", $current_batch, $total_batches, count($batch)));
+
+	$process = proc_open(
+		[PHP_BINARY, $phpunit, '--configuration', $config_path, '--no-coverage', '--log-junit', $junit_path],
+		[STDIN, STDOUT, STDERR],
+		$pipes,
+		$project_root
+	);
+
+	if ( ! is_resource($process)) {
+		unlink($config_path);
+		unlink($junit_path);
+		fwrite(STDERR, "Unable to start PHPUnit.\n");
+		exit(2);
+	}
+
+	$batch_status = proc_close($process);
+
+	if (0 !== $batch_status) {
+		unlink($config_path);
+		unlink($junit_path);
+
+		if (9 === $batch_status && 1 < count($batch)) {
+			fwrite(STDERR, sprintf("PHPUnit batch %d/%d was killed; retrying each file separately.\n", $current_batch, $total_batches));
+
+			foreach ($batch as $test_file) {
+				$retry_environment                         = getenv();
+				$retry_environment['PHPUNIT_BATCH_NUMBER'] = '0';
+				$retry_process                             = proc_open(
+					[PHP_BINARY, __FILE__, $test_file],
+					[STDIN, STDOUT, STDERR],
+					$retry_pipes,
+					$project_root,
+					$retry_environment
+				);
+
+				if ( ! is_resource($retry_process) || 0 !== proc_close($retry_process)) {
+					fwrite(STDERR, sprintf("PHPUnit retry failed for %s.\n", $test_file));
+					$exit_status = 1;
+					continue;
+				}
+
+				++$executed_files;
+			}
+
+			continue;
+		}
+
+		fwrite(STDERR, sprintf("PHPUnit batch %d/%d failed with exit code %d.\n", $current_batch, $total_batches, $batch_status));
+		$exit_status = 1;
+		continue;
+	}
+
+	try {
+		$executed_files += $count_executed_files($junit_path, $batch);
+	} catch (RuntimeException $exception) {
+		unlink($config_path);
+		unlink($junit_path);
+		fwrite(STDERR, sprintf("Unable to verify PHPUnit batch %d: %s\n", $current_batch, $exception->getMessage()));
+		exit(2);
+	}
+
+	unlink($config_path);
+	unlink($junit_path);
+}
+
+try {
+	$clean_plugin_tables();
+} catch (Throwable $exception) {
+	fwrite(STDERR, sprintf("Unable to reset plugin test tables: %s\n", $exception->getMessage()));
+	exit(2);
+}
+
+if (0 !== $exit_status) {
+	exit(1);
+}
+
+fwrite(STDOUT, sprintf("PHPUnit test-file accounting: %d/%d executed.\n", $executed_files, $expected_files));
+
+if ($executed_files !== $expected_files) {
+	fwrite(STDERR, "PHPUnit did not execute every discovered test file.\n");
+	exit(2);
+}
+
+exit(0);