diff --git a/bin/pest b/bin/pest index 9e6e703c7..c1fc6cceb 100755 --- a/bin/pest +++ b/bin/pest @@ -112,6 +112,17 @@ use Symfony\Component\Console\Output\ConsoleOutput; } } + if (str_contains($value, '--shard-timing=')) { + $_SERVER['PEST_SHARD_TIMING'] = substr($value, strlen('--shard-timing=')); + unset($arguments[$key]); + } elseif ($value === '--shard-timing') { + if (isset($arguments[$key + 1])) { + $_SERVER['PEST_SHARD_TIMING'] = $arguments[$key + 1]; + unset($arguments[$key + 1]); + } + unset($arguments[$key]); + } + if (str_contains($value, '--teamcity')) { unset($arguments[$key]); $arguments[] = '--no-output'; diff --git a/src/Plugins/ShardByTime.php b/src/Plugins/ShardByTime.php new file mode 100644 index 000000000..8ea633585 --- /dev/null +++ b/src/Plugins/ShardByTime.php @@ -0,0 +1,351 @@ +hasArgument('--shard', $arguments)) { + return $arguments; + } + + $timingFile = $_SERVER['PEST_SHARD_TIMING'] ?? null; + + if ($timingFile === null) { + return $arguments; + } + + // @phpstan-ignore-next-line + $input = new ArgvInput($arguments); + + ['index' => $index, 'total' => $total] = Shard::getShard($input); + + $arguments = $this->popArgument("--shard=$index/$total", $this->popArgument('--shard', $this->popArgument( + "$index/$total", + $arguments, + ))); + + /** @phpstan-ignore-next-line */ + $tests = $this->allTests($arguments); + + $classTimes = $this->loadClassTimes($timingFile, array_values($arguments)); + + $testsToRun = $classTimes !== null + ? $this->shardByTime($tests, $total, $index, $classTimes) + : (array_chunk($tests, max(1, (int) ceil(count($tests) / $total))))[$index - 1] ?? []; + + self::$shard = [ + 'index' => $index, + 'total' => $total, + 'testsRan' => count($testsToRun), + 'testsCount' => count($tests), + ]; + + return [...$arguments, '--filter', $this->buildFilterArgument($testsToRun)]; + } + + /** + * Shards tests by execution time using the LPT bin-packing algorithm. + * + * @param list $tests + * @param array $classTimes + * @return list + */ + private function shardByTime(array $tests, int $total, int $index, array $classTimes): array + { + usort($tests, fn (string $a, string $b): int => ($classTimes[$b] ?? self::DEFAULT_TIME) <=> ($classTimes[$a] ?? self::DEFAULT_TIME)); + + /** @var list> $shards */ + $shards = array_fill(0, $total, []); + /** @var list $shardTimes */ + $shardTimes = array_fill(0, $total, 0.0); + + foreach ($tests as $test) { + $minIndex = 0; + $minTime = $shardTimes[0]; + + for ($i = 1; $i < $total; $i++) { + if ($shardTimes[$i] < $minTime) { + $minTime = $shardTimes[$i]; + $minIndex = $i; + } + } + + $shards[$minIndex][] = $test; + $shardTimes[$minIndex] += $classTimes[$test] ?? self::DEFAULT_TIME; + } + + return $shards[$index - 1] ?? []; + } + + /** + * Loads per-class execution times, preferring JUnit XML, falling back to result cache. + * + * @param list $arguments + * @return array|null + */ + private function loadClassTimes(string $timingFile, array $arguments): ?array + { + return $this->loadClassTimesFromJunitXml($timingFile) + ?? $this->loadClassTimesFromResultCache($arguments); + } + + /** + * Loads per-class execution times from a JUnit XML report. + * + * @return array|null + */ + private function loadClassTimesFromJunitXml(string $file): ?array + { + if (! file_exists($file)) { + return null; + } + + $xml = simplexml_load_file($file); + + if ($xml === false) { + return null; + } + + /** @var array $classTimes */ + $classTimes = []; + + foreach ($xml->xpath('//testsuite[@class]') as $suite) { + $class = (string) $suite['class']; + $time = (float) $suite['time']; + + $classTimes[$class] = ($classTimes[$class] ?? 0.0) + $time; + } + + if ($classTimes === []) { + foreach ($xml->xpath('//testcase[@class]') as $testcase) { + $class = (string) $testcase['class']; + $time = (float) $testcase['time']; + + $classTimes[$class] = ($classTimes[$class] ?? 0.0) + $time; + } + } + + return $classTimes !== [] ? $classTimes : null; + } + + /** + * Loads per-class execution times from the PHPUnit result cache. + * + * @param list $arguments + * @return array|null + */ + private function loadClassTimesFromResultCache(array $arguments): ?array + { + $cacheFile = $this->findCacheFile($arguments); + + if ($cacheFile === null || ! file_exists($cacheFile)) { + return null; + } + + $contents = file_get_contents($cacheFile); + + if ($contents === false) { + return null; + } + + $data = json_decode($contents, true); + + if (! is_array($data) || ! isset($data['times']) || ! is_array($data['times'])) { // @phpstan-ignore booleanNot.alwaysFalse + return null; + } + + /** @var array $classTimes */ + $classTimes = []; + + /** @var array $times */ + $times = $data['times']; + + foreach ($times as $method => $time) { + $parts = explode('::', $method, 2); + $class = preg_replace('/^P\\\\/', '', $parts[0]); + + $classTimes[$class] = ($classTimes[$class] ?? 0.0) + $time; + } + + return $classTimes; + } + + /** + * Finds the result cache file path. + * + * @param list $arguments + */ + private function findCacheFile(array $arguments): ?string + { + $cacheDirectory = $this->getCacheDirectoryFromArguments($arguments); + + if ($cacheDirectory === null) { + $cacheDirectory = $this->getCacheDirectoryFromXml(); + } + + if ($cacheDirectory === null) { + $tempFolder = realpath(self::TEMPORARY_FOLDER); + $cacheDirectory = $tempFolder !== false ? $tempFolder : null; + } + + if ($cacheDirectory === null) { + return null; + } + + $testResults = $cacheDirectory.DIRECTORY_SEPARATOR.'test-results'; + + if (file_exists($testResults)) { + return $testResults; + } + + return $cacheDirectory.DIRECTORY_SEPARATOR.'.phpunit.result.cache'; + } + + /** + * @param list $arguments + */ + private function getCacheDirectoryFromArguments(array $arguments): ?string + { + foreach ($arguments as $i => $argument) { + if ($argument === '--cache-directory' && isset($arguments[$i + 1])) { + return $arguments[$i + 1]; + } + + if (str_starts_with($argument, '--cache-directory=')) { + return substr($argument, strlen('--cache-directory=')); + } + } + + return null; + } + + private function getCacheDirectoryFromXml(): ?string + { + $cliConfiguration = (new CliConfigurationBuilder)->fromParameters([]); + $configurationFile = (new XmlConfigurationFileFinder)->find($cliConfiguration); + $xmlConfiguration = DefaultConfiguration::create(); + + if (is_string($configurationFile)) { + $xmlConfiguration = (new Loader)->load($configurationFile); + } + + if ($xmlConfiguration->phpunit()->hasCacheDirectory()) { + return $xmlConfiguration->phpunit()->cacheDirectory(); + } + + return null; + } + + /** + * @param list $arguments + * @return list + */ + private function allTests(array $arguments): array + { + $output = (new Process([ + 'php', + ...$this->removeParallelArguments($arguments), + '--list-tests', + ]))->mustRun()->getOutput(); + + preg_match_all('/ - (?:P\\\\)?(Tests\\\\[^:]+)::/', $output, $matches); + + return array_values(array_unique($matches[1])); + } + + /** + * @param array $arguments + * @return array + */ + private function removeParallelArguments(array $arguments): array + { + return array_filter($arguments, fn (string $argument): bool => ! in_array($argument, ['--parallel', '-p'], strict: true)); + } + + /** + * @param list $testsToRun + */ + private function buildFilterArgument(array $testsToRun): string + { + return addslashes(implode('|', $testsToRun)); + } + + /** + * Adds output after the Test Suite execution. + */ + public function addOutput(int $exitCode): int + { + if (self::$shard === null) { + return $exitCode; + } + + [ + 'index' => $index, + 'total' => $total, + 'testsRan' => $testsRan, + 'testsCount' => $testsCount, + ] = self::$shard; + + $this->output->writeln(sprintf( + ' Shard: %d of %d — %d file%s ran, out of %d (time-balanced).', + $index, + $total, + $testsRan, + $testsRan === 1 ? '' : 's', + $testsCount, + )); + + return $exitCode; + } +} diff --git a/tests/Unit/Plugins/ShardByTime.php b/tests/Unit/Plugins/ShardByTime.php new file mode 100644 index 000000000..48fdbf920 --- /dev/null +++ b/tests/Unit/Plugins/ShardByTime.php @@ -0,0 +1,227 @@ +invoke($plugin, ...$args); +} + +it('skips when --shard is not present', function () { + $plugin = new ShardByTime(new NullOutput); + + $arguments = $plugin->handleArguments(['./vendor/bin/pest', '--filter', 'SomeTest']); + + expect($arguments)->toBe(['./vendor/bin/pest', '--filter', 'SomeTest']); +}); + +it('skips when --shard-timing is not set', function () { + unset($_SERVER['PEST_SHARD_TIMING']); + + $plugin = new ShardByTime(new NullOutput); + + $arguments = $plugin->handleArguments(['./vendor/bin/pest', '--shard=1/2']); + + expect($arguments)->toBe(['./vendor/bin/pest', '--shard=1/2']); +}); + +it('parses JUnit XML testsuite class times', function () { + $junitXml = tempnam(sys_get_temp_dir(), 'pest-junit-'); + file_put_contents($junitXml, <<<'XML' + + + + + + + + + + +XML); + + $result = callShardByTimeMethod('loadClassTimesFromJunitXml', [$junitXml]); + + expect($result)->toBe([ + 'Tests\Unit\FooTest' => 1.5, + 'Tests\Unit\BarTest' => 3.0, + ]); + + unlink($junitXml); +}); + +it('falls back to testcase elements when no testsuite has class attribute', function () { + $junitXml = tempnam(sys_get_temp_dir(), 'pest-junit-'); + file_put_contents($junitXml, <<<'XML' + + + + + + + + +XML); + + $result = callShardByTimeMethod('loadClassTimesFromJunitXml', [$junitXml]); + + expect($result)->toBe([ + 'Tests\Unit\FooTest' => 1.5, + 'Tests\Unit\BarTest' => 2.0, + ]); + + unlink($junitXml); +}); + +it('returns null for non-existent JUnit XML file', function () { + $result = callShardByTimeMethod('loadClassTimesFromJunitXml', ['/tmp/non-existent-junit.xml']); + + expect($result)->toBeNull(); +}); + +it('returns null for empty JUnit XML', function () { + $junitXml = tempnam(sys_get_temp_dir(), 'pest-junit-'); + file_put_contents($junitXml, <<<'XML' + + + +XML); + + $result = callShardByTimeMethod('loadClassTimesFromJunitXml', [$junitXml]); + + expect($result)->toBeNull(); + + unlink($junitXml); +}); + +it('distributes tests using LPT bin-packing', function () { + $tests = [ + 'Tests\Unit\FastTest', + 'Tests\Unit\MediumTest', + 'Tests\Unit\SlowTest', + ]; + $classTimes = [ + 'Tests\Unit\SlowTest' => 10.0, + 'Tests\Unit\MediumTest' => 5.0, + 'Tests\Unit\FastTest' => 1.0, + ]; + + $shard1 = callShardByTimeMethod('shardByTime', [$tests, 2, 1, $classTimes]); + $shard2 = callShardByTimeMethod('shardByTime', [$tests, 2, 2, $classTimes]); + + // LPT: SlowTest(10) -> shard1, MediumTest(5) -> shard2, FastTest(1) -> shard2 + expect($shard1)->toBe(['Tests\Unit\SlowTest']) + ->and($shard2)->toBe(['Tests\Unit\MediumTest', 'Tests\Unit\FastTest']); +}); + +it('uses default time of 1s for tests without timing data', function () { + $tests = [ + 'Tests\Unit\KnownTest', + 'Tests\Unit\UnknownTest', + ]; + $classTimes = [ + 'Tests\Unit\KnownTest' => 5.0, + ]; + + $shard1 = callShardByTimeMethod('shardByTime', [$tests, 2, 1, $classTimes]); + $shard2 = callShardByTimeMethod('shardByTime', [$tests, 2, 2, $classTimes]); + + // KnownTest(5.0) -> shard1, UnknownTest(1.0 default) -> shard2 + expect($shard1)->toBe(['Tests\Unit\KnownTest']) + ->and($shard2)->toBe(['Tests\Unit\UnknownTest']); +}); + +it('balances shards evenly across multiple bins', function () { + $tests = [ + 'Tests\Unit\A', + 'Tests\Unit\B', + 'Tests\Unit\C', + 'Tests\Unit\D', + ]; + $classTimes = [ + 'Tests\Unit\A' => 4.0, + 'Tests\Unit\B' => 3.0, + 'Tests\Unit\C' => 2.0, + 'Tests\Unit\D' => 1.0, + ]; + + $shard1 = callShardByTimeMethod('shardByTime', [$tests, 3, 1, $classTimes]); + $shard2 = callShardByTimeMethod('shardByTime', [$tests, 3, 2, $classTimes]); + $shard3 = callShardByTimeMethod('shardByTime', [$tests, 3, 3, $classTimes]); + + // LPT: A(4)->s1, B(3)->s2, C(2)->s3, D(1)->s3 + expect($shard1)->toBe(['Tests\Unit\A']) + ->and($shard2)->toBe(['Tests\Unit\B']) + ->and($shard3)->toBe(['Tests\Unit\C', 'Tests\Unit\D']); +}); + +it('returns empty array for out-of-range shard index', function () { + $tests = ['Tests\Unit\A']; + $classTimes = ['Tests\Unit\A' => 1.0]; + + $result = callShardByTimeMethod('shardByTime', [$tests, 2, 3, $classTimes]); + + expect($result)->toBe([]); +}); + +it('parses result cache with P prefix', function () { + $cacheFile = tempnam(sys_get_temp_dir(), 'pest-cache-'); + file_put_contents($cacheFile, json_encode([ + 'version' => 1, + 'defects' => [], + 'times' => [ + 'P\Tests\Unit\FooTest::test_one' => 0.5, + 'P\Tests\Unit\FooTest::test_two' => 0.3, + 'P\Tests\Unit\BarTest::test_bar' => 1.2, + ], + ])); + + $result = callShardByTimeMethod('loadClassTimesFromResultCache', [['--cache-directory', dirname($cacheFile)]]); + + // The method looks for 'test-results' or '.phpunit.result.cache' in the cache directory, + // not an arbitrary file. We need to place the file correctly. + unlink($cacheFile); + + // Test with properly named file + $cacheDir = sys_get_temp_dir().'/pest-cache-test-'.uniqid(); + mkdir($cacheDir); + file_put_contents($cacheDir.'/test-results', json_encode([ + 'version' => 1, + 'defects' => [], + 'times' => [ + 'P\Tests\Unit\FooTest::test_one' => 0.5, + 'P\Tests\Unit\FooTest::test_two' => 0.3, + 'P\Tests\Unit\BarTest::test_bar' => 1.2, + ], + ])); + + $result = callShardByTimeMethod('loadClassTimesFromResultCache', [['--cache-directory', $cacheDir]]); + + expect($result)->toBe([ + 'Tests\Unit\FooTest' => 0.8, + 'Tests\Unit\BarTest' => 1.2, + ]); + + unlink($cacheDir.'/test-results'); + rmdir($cacheDir); +}); + +it('returns null for missing result cache', function () { + $result = callShardByTimeMethod('loadClassTimesFromResultCache', [['--cache-directory', '/tmp/non-existent-cache-dir']]); + + expect($result)->toBeNull(); +}); + +it('does not produce output when shard was not used', function () { + $output = new \Symfony\Component\Console\Output\BufferedOutput; + $plugin = new ShardByTime($output); + + $exitCode = $plugin->addOutput(0); + + expect($exitCode)->toBe(0) + ->and($output->fetch())->toBe(''); +});