Faster search by file contents using file memory mapping

Author: VioletGiraffe

cpputils

@@ -159,7 +159,7 @@ FileOperationResultCode CFileManipulator::copyChunk(const uint64_t chunkSize, co
 
 		_destinationFilePath = destFolder + (newName.isEmpty() ? _srcObject.fullName() : newName);
 
-		if (!_destFile.open(_destinationFilePath.toUtf8().constData(), thin_io::file_definitions::Write)) [[unlikely]]
+		if (!_destFile.open(_destinationFilePath.toUtf8().constData(), thin_io::file::open_mode::Write)) [[unlikely]]
 		{
 			_lastErrorMessage = getLastFileError();
 

file-commander-core/src/cfilemanipulator.cpp

@@ -3,17 +3,19 @@
 #include "directoryscanner.h"
 
 #include "qtcore_helpers/qstring_helpers.hpp"
+#include "file.hpp"
 
 #include "hash/jenkins_hash.hpp"
 #include "threading/thread_helpers.h"
+#include "utility_functions/memory_functions.h"
 
 DISABLE_COMPILER_WARNINGS
 #include <QDebug>
 #include <QRegularExpression>
 #include <QTextStream>
 RESTORE_COMPILER_WARNINGS
 
-static QString queryToRegex(const QString& query)
+[[nodiscard]] static QString queryToRegex(const QString& query, bool startToEnd)
 {
 	// Escape the dots
 	QString regExString = QString{ query }.replace('.', QLatin1StringView{ "\\." });
@@ -22,18 +24,102 @@ static QString queryToRegex(const QString& query)
 	if (nameQueryHasWildcards)
 	{
 		regExString.replace('?', '.').replace('*', ".*");
-		regExString.prepend("\\A").append("\\z");
+		//regExString.prepend("\\A").append("\\z");
 	}
 
-	if (!regExString.startsWith('^'))
-		regExString.prepend('^');
+	if (startToEnd)
+	{
+		if (!regExString.startsWith('^'))
+			regExString.prepend('^');
 
-	if (!regExString.endsWith('$'))
-		regExString.append('$');
+		if (!regExString.endsWith('$'))
+			regExString.append('$');
+	}
 
 	return regExString;
 }
 
+#ifndef __ARM_ARCH_ISA_A64
+
+#include <smmintrin.h>  // SSE4.1
+
+inline void replace_byte(uint8_t* array, size_t size) noexcept
+{
+	const __m128i old_sse = _mm_set1_epi8(0);
+	const __m128i new_sse = _mm_set1_epi8(' ');
+
+	for (size_t i = 0; i < size; i += 16)
+	{
+		__m128i data = _mm_loadu_si128(reinterpret_cast<__m128i*>(array + i));  // Load 16 bytes
+		__m128i mask = _mm_cmpeq_epi8(data, old_sse);           // Compare with old_value
+		__m128i result = _mm_blendv_epi8(data, new_sse, mask);  // Blend new_value where mask is true
+		_mm_storeu_si128(reinterpret_cast<__m128i*>(array + i), result); // Store the result back
+	}
+}
+
+#else // ARM64
+
+#include <arm_neon.h>
+
+inline void replace_byte(uint8_t* array, size_t size)
+{
+	uint8x16_t old_neon = vdupq_n_u8(0);  // Duplicate old_value across all 16 bytes in the vector
+	uint8x16_t new_neon = vdupq_n_u8(' ');  // Duplicate new_value across all 16 bytes in the vector
+
+	for (size_t i = 0; i < size; i += 16)
+	{
+		uint8x16_t data = vld1q_u8(&array[i]);           // Load 16 bytes
+		uint8x16_t mask = vceqq_u8(data, old_neon);      // Compare with old_value
+		uint8x16_t result = vbslq_u8(mask, new_neon, data);  // Select new_value where mask is true, else original value
+		vst1q_u8(&array[i], result);                     // Store the result back
+	}
+}
+
+#endif
+
+[[nodiscard]] static bool fileContentsMatches(const QString& path, const QRegularExpression& regex)
+{
+	thin_io::file file;
+	if (!file.open(path.toUtf8().constData(), thin_io::file::open_mode::Read)) [[unlikely]]
+		return false;
+
+	const auto fileSize = file.size().value_or(0);
+	if (fileSize == 0) [[unlikely]]
+		return false;
+
+	static constexpr auto toBytePtr = [](const void* ptr) -> const std::byte* {
+		return reinterpret_cast<const std::byte*>(ptr);
+	};
+
+	auto* mappedFile = toBytePtr(file.mmap(thin_io::file::mmap_access_mode::ReadOnly, 0, fileSize));
+	if (!mappedFile) [[unlikely]]
+	{
+		assert_debug_only(mappedFile);
+		return false;
+	}
+
+
+	static constexpr size_t maxLineLength = 8 * 1024;
+	char buffer[maxLineLength];
+
+	for (size_t offset = 0; offset < fileSize; )
+	{
+		const auto maxSearchLength = std::min(fileSize - offset, maxLineLength);
+		const auto lineStart = mappedFile + offset;
+		offset += maxSearchLength;
+
+		::memcpy(buffer, lineStart, maxSearchLength);
+		replace_byte((uint8_t*)buffer, (maxSearchLength + 15) / 16);
+
+		QString line = QString::fromUtf8(buffer, maxSearchLength);
+		assert(!line.isEmpty());
+		if (regex.match(line).hasMatch())
+			return true;
+	}
+
+	return false;
+}
+
 CFileSearchEngine::CFileSearchEngine(CController& controller) :
 	_controller(controller),
 	_workerThread("File search thread")
@@ -92,7 +178,7 @@ void CFileSearchEngine::searchThread(const QString& what, bool subjectCaseSensit
 	QRegularExpression queryRegExp;
 	if (!noFileNameFilter)
 	{
-		queryRegExp.setPattern(queryToRegex(what));
+		queryRegExp.setPattern(queryToRegex(what, true));
 		assert_r(queryRegExp.isValid());
 
 		if (!subjectCaseSensitive)
@@ -104,34 +190,29 @@ void CFileSearchEngine::searchThread(const QString& what, bool subjectCaseSensit
 	QRegularExpression fileContentsRegExp;
 	if (searchByContents)
 	{
-		if (contentsToFind.contains(QRegularExpression(QSL("[*?]"))))
-		{
-			fileContentsRegExp.setPattern(QRegularExpression::wildcardToRegularExpression(contentsToFind));
-			assert_r(fileContentsRegExp.isValid());
-		}
-		else if (contentsWholeWords)
-		{
-			fileContentsRegExp.setPattern("\\b" + contentsToFind + "\\b");
-			assert_r(fileContentsRegExp.isValid());
-		}
+		QString pattern = queryToRegex(contentsToFind, false);
+		if (contentsWholeWords)
+			pattern.prepend("\\b").append("\\b");
+
+		fileContentsRegExp.setPattern(pattern);
 
 		if (!contentsCaseSensitive)
 			fileContentsRegExp.setPatternOptions(QRegularExpression::CaseInsensitiveOption);
-	}
 
-	const bool useFileContentsRegExp = !fileContentsRegExp.pattern().isEmpty();
+		assert_r(fileContentsRegExp.isValid());
+	}
 
 	const int uniqueJobTag = static_cast<int>(jenkins_hash("CFileSearchEngine")) + rand();
 
 	QString line;
 
+	const QByteArray contentsUtf8 = contentsToFind.toUtf8();
+
 	for (const QString& pathToLookIn : where)
 	{
 		scanDirectory(CFileSystemObject(pathToLookIn),
 			[&](const CFileSystemObject& item) {
 
-				++itemCounter;
-
 				if (itemCounter % 8192 == 0)
 				{
 					// No need to report every single item and waste CPU cycles
@@ -141,6 +222,8 @@ void CFileSearchEngine::searchThread(const QString& what, bool subjectCaseSensit
 						}, uniqueJobTag);
 				}
 
+				++itemCounter;
+
 				if (searchByContents && !item.isFile())
 					return;
 
@@ -152,21 +235,7 @@ void CFileSearchEngine::searchThread(const QString& what, bool subjectCaseSensit
 				bool matchFound = false;
 
 				if (searchByContents)
-				{
-					QFile file{ item.fullAbsolutePath() };
-					if (!file.open(QFile::ReadOnly))
-						return;
-
-					const auto contentsCaseSensitivity = contentsCaseSensitive ? Qt::CaseSensitive : Qt::CaseInsensitive;
-
-					QTextStream stream{ &file };
-
-					while (!matchFound && !_workerThread.terminationFlag() && stream.readLineInto(&line))
-					{
-						// contains() is faster than RegEx match (as of Qt 5.4.2, but this was for QRegExp, not tested with QRegularExpression)
-						matchFound = useFileContentsRegExp ? fileContentsRegExp.match(line).hasMatch() : line.contains(contentsToFind, contentsCaseSensitivity);
-					}
-				}
+					matchFound = fileContentsMatches(item.fullAbsolutePath(), fileContentsRegExp);
 				else
 					matchFound = nameMatches;