Added a source filter to pre-process files to eliminate unnecessary scanning of exact replica of files.

A SourceFilter class was added which goal is to compare two source trees and filter out any duplicate source code files. It does so by hashing each file content and comparing the before and after hash sets. The intersection set is removed from the before and after list of files since there is no point to scan files with the exact same source code.

The benefits of this a important both in term of cpu usage and memory usage since we do not parse code that will in the end reveal itself not to have changed.

For example, here are the before and after of this change, comparing Symfony 2.5.9 with 2.6.3.

Before
[Symfony 2.5.9] 2936 php files in src
[Symfony 2.6.3] 3078 php files in src

2564 files are exactly the same

After
[Symfony 2.5.9] 367 scanned and parsed (12.5% of the original amount)
[Symfony 2.6.3] 509 scanned and parsed (16.5% of the original amount)

But most important of all, it can compare the whole source trees in a reasonable amount of time, something it couldn't before.

Author: tomzx

src/PHPSemVerChecker/Console/Command/CompareCommand.php

@@ -4,6 +4,7 @@
 
 use File_Iterator_Facade;
 use PHPSemVerChecker\Analyzer\Analyzer;
+use PHPSemVerChecker\Filter\SourceFilter;
 use PHPSemVerChecker\Reporter\Reporter;
 use PHPSemVerChecker\Scanner\Scanner;
 use Symfony\Component\Console\Command\Command;
@@ -43,6 +44,14 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
 		$progress = new ProgressBar($output, count($sourceBefore) + count($sourceAfter));
 		$progress->setFormat("%message%\n%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% %memory:6s%");
+		$progress->setMessage('Pre-processing before/after files');
+		$progress->start();
+
+		$sourceFilter = new SourceFilter();
+		$identicalCount = $sourceFilter->filter($sourceBefore, $sourceAfter);
+
+		$progress->start(count($sourceBefore) + count($sourceAfter));
+
 		$progress->setMessage('Scanning before files');
 		foreach ($sourceBefore as $file) {
 			$scannerBefore->scan($file);
@@ -68,6 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
 		$duration = microtime(true) - $startTime;
 		$output->writeln('');
+		$output->writeln('[Scanned files] Before: ' . count($sourceBefore) . ', After: ' . count($sourceAfter) . ', Identical: ' . $identicalCount);
 		$output->writeln('Time: ' . round($duration, 3) . ' seconds, Memory: ' . round(memory_get_peak_usage() / 1024 / 1024, 3) . ' MB');
 	}
 }

src/PHPSemVerChecker/Filter/SourceFilter.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace PHPSemVerChecker\Filter;
+
+class SourceFilter
+{
+	/**
+	 * @param array $filesBefore
+	 * @param array $filesAfter
+	 */
+	public function filter(array &$filesBefore, array &$filesAfter)
+	{
+		$hashedBefore = [];
+		foreach ($filesBefore as $fileBefore) {
+			$hashedBefore[sha1(file_get_contents($fileBefore))] = $fileBefore;
+		}
+
+		$hashedAfter = [];
+		foreach ($filesAfter as $fileAfter) {
+			$hashedAfter[sha1(file_get_contents($fileAfter))] = $fileAfter;
+		}
+
+		$intersection = array_intersect_key($hashedBefore, $hashedAfter);
+		$filesBefore = array_values(array_diff_key($hashedBefore, $intersection));
+		$filesAfter = array_values(array_diff_key($hashedAfter, $intersection));
+		return count($intersection);
+	}
+}