Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion src/Analyser/Parallel/ParallelAnalysisNodeExtractor.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
use function array_search;
use function array_sum;
use function arsort;
use function ceil;
use function count;
use function dirname;
use function explode;
Expand All @@ -36,9 +37,11 @@
use function is_array;
use function is_dir;
use function is_string;
use function max;
use function min;
use function mkdir;
use function proc_close;
use function reset;
use function rmdir;
use function serialize;
use function sprintf;
Expand Down Expand Up @@ -363,12 +366,20 @@ private function buildWorkerBuckets(array $files, int $workerCount): array

arsort($fileSizes);

// The largest file cannot be split, so a worker whose byte share is
// below it would only add a process boot without finishing sooner.
$totalSize = array_sum($fileSizes);
$largestSize = reset($fileSizes);
if ($largestSize > 0) {
$workerCount = max(1, min($workerCount, (int) ceil($totalSize / $largestSize)));
}

$buckets = array_fill(0, $workerCount, []);
$bucketSizes = array_fill(0, $workerCount, 0);

// The largest files fill one worker up to an even byte share first, so
// only that worker pays their AST peaks; the rest are balanced by LPT.
$share = array_sum($fileSizes) / $workerCount;
$share = $totalSize / $workerCount;

foreach ($fileSizes as $file => $fileSize) {
$minIdx = $bucketSizes[0] < $share
Expand Down
48 changes: 48 additions & 0 deletions tests/Analyser/Parallel/ParallelAnalysisNodeExtractorTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -314,6 +314,54 @@ public function finish(): void
$this->assertSame($files, $progress->files);
}

public function testExtractSpawnsNoMoreWorkersThanTheLargestFileShareAllows(): void
{
$dir = $this->makeTemporaryDirectory('structarmed-parallel-test');
$files = [$dir . '/Large.php'];

// larger than the three small files together: two byte shares fit the
// files, so a third and fourth worker would only add a process boot
file_put_contents($files[0], <<<'PHP'
<?php

namespace App;

final class Large
{
public function first(): string
{
return 'first';
}

public function second(): string
{
return 'second';
}
}
PHP);

for ($index = 0; $index < 3; ++$index) {
$file = sprintf('%s/Small%d.php', $dir, $index);
file_put_contents($file, sprintf('<?php namespace App; final class Small%d {}', $index));

$files[] = $file;
}

$GLOBALS['mock_tracked_tempnam_files'] = [];

try {
$extractionResult = (new ParallelAnalysisNodeExtractor($dir, [], [], 4))->extract($files);
$tempnamFiles = $GLOBALS['mock_tracked_tempnam_files'];
} finally {
$GLOBALS['mock_tracked_tempnam_files'] = [];
}

$this->assertCount(4, $extractionResult->classNodes);
// input, output and stderr file per worker
$this->assertIsArray($tempnamFiles);
$this->assertCount(6, $tempnamFiles);
}

public function testExtractBuffersProgressLineSplitAcrossReads(): void
{
// Simulates a worker whose progress lines reach the coordinator in pieces: index 10 arrives as "1" then
Expand Down
Loading