Skip to content

Commit 45a8342

Browse files
makutomakuto
committed
WIP Removing dependence on file modification times
1 parent 74a80e8 commit 45a8342

8 files changed

+218
-19
lines changed

‎src/Build.cpp

+61-3
Original file line numberDiff line numberDiff line change
@@ -407,8 +407,9 @@ void makeTargetPlatformVersionArgument(char* resolvedArgumentOut, int resolvedAr
407407
#endif
408408
}
409409

410-
void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
411-
ArtifactCrcTable& newCommandCrcs)
410+
static void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
411+
ArtifactCrcTable& newCommandCrcs,
412+
HashedSourceArtifactCrcTable& sourceArtifactFileCrcs)
412413
{
413414
char outputFilename[MAX_PATH_LENGTH] = {0};
414415
if (!outputFilenameFromSourceFilename(buildOutputDir, "Cache", "cake", outputFilename,
@@ -430,6 +431,7 @@ void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCom
430431
const Token openParen = {TokenType_OpenParen, EmptyString, "Build.cpp", 1, 0, 0};
431432
const Token closeParen = {TokenType_CloseParen, EmptyString, "Build.cpp", 1, 0, 0};
432433
const Token crcInvoke = {TokenType_Symbol, "command-crc", "Build.cpp", 1, 0, 0};
434+
const Token sourceArtifactInvoke = {TokenType_Symbol, "source-artifact-crc", "Build.cpp", 1, 0, 0};
433435

434436
for (ArtifactCrcTablePair& crcPair : outputCrcs)
435437
{
@@ -445,6 +447,21 @@ void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCom
445447
outputTokens.push_back(closeParen);
446448
}
447449

450+
for (const HashedSourceArtifactCrcTablePair& crcPair : sourceArtifactFileCrcs)
451+
{
452+
outputTokens.push_back(openParen);
453+
outputTokens.push_back(sourceArtifactInvoke);
454+
455+
Token sourceArtifactName = {
456+
TokenType_String, std::to_string(crcPair.first), "Build.cpp", 1, 0, 0};
457+
outputTokens.push_back(sourceArtifactName);
458+
459+
Token crcToken = {TokenType_Symbol, std::to_string(crcPair.second), "Build.cpp", 1, 0, 0};
460+
outputTokens.push_back(crcToken);
461+
462+
outputTokens.push_back(closeParen);
463+
}
464+
448465
if (outputTokens.empty())
449466
{
450467
Log("no tokens to write to cache file");
@@ -464,7 +481,8 @@ void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCom
464481
}
465482

466483
// Returns false if there were errors; the file not existing is not an error
467-
bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs)
484+
bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
485+
HashedSourceArtifactCrcTable& sourceArtifactFileCrcs)
468486
{
469487
char inputFilename[MAX_PATH_LENGTH] = {0};
470488
if (!outputFilenameFromSourceFilename(buildOutputDir, "Cache", "cake", inputFilename,
@@ -512,6 +530,27 @@ bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedComm
512530
cachedCommandCrcs[(*tokens)[artifactIndex].contents] =
513531
static_cast<uint32_t>(std::stoul((*tokens)[crcIndex].contents));
514532
}
533+
else if (invocationToken.contents.compare("source-artifact-crc") == 0)
534+
{
535+
int artifactSourceIndex = getExpectedArgument("expected artifact source CRC",
536+
(*tokens), i, 1, endInvocationIndex);
537+
if (artifactSourceIndex == -1)
538+
{
539+
delete tokens;
540+
return false;
541+
}
542+
int crcIndex =
543+
getExpectedArgument("expected crc", (*tokens), i, 2, endInvocationIndex);
544+
if (crcIndex == -1)
545+
{
546+
delete tokens;
547+
return false;
548+
}
549+
550+
sourceArtifactFileCrcs[static_cast<uint32_t>(
551+
std::stoul((*tokens)[artifactSourceIndex].contents))] =
552+
static_cast<uint32_t>(std::stoul((*tokens)[crcIndex].contents));
553+
}
515554
else
516555
{
517556
Logf("error: unrecognized invocation in %s: %s\n", inputFilename,
@@ -528,6 +567,25 @@ bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedComm
528567
return true;
529568
}
530569

570+
void buildReadMergeWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
571+
ArtifactCrcTable& newCommandCrcs,
572+
HashedSourceArtifactCrcTable& sourceArtifactFileCrcs)
573+
{
574+
ArtifactCrcTable mergedCachedCommandCrcs;
575+
HashedSourceArtifactCrcTable mergedSourceArtifactFileCrcs;
576+
577+
buildReadCacheFile(buildOutputDir, mergedCachedCommandCrcs, mergedSourceArtifactFileCrcs);
578+
579+
// Merge, using our version as latest
580+
for (ArtifactCrcTablePair& crcPair : newCommandCrcs)
581+
mergedCachedCommandCrcs[crcPair.first] = crcPair.second;
582+
for (const HashedSourceArtifactCrcTablePair& crcPair : sourceArtifactFileCrcs)
583+
mergedSourceArtifactFileCrcs[crcPair.first] = crcPair.second;
584+
585+
buildWriteCacheFile(buildOutputDir, mergedCachedCommandCrcs, newCommandCrcs,
586+
mergedSourceArtifactFileCrcs);
587+
}
588+
531589
// It is essential to scan the #include files to determine if any of the headers have been modified,
532590
// because changing them could require a rebuild (for e.g., you change the size or order of a struct
533591
// declared in a header; all source files now need updated sizeof calls). This is annoyingly

‎src/Build.hpp

+12-3
Original file line numberDiff line numberDiff line change
@@ -71,11 +71,20 @@ typedef std::unordered_map<std::string, FileModifyTime> HeaderModificationTimeTa
7171
typedef std::unordered_map<std::string, uint32_t> ArtifactCrcTable;
7272
typedef std::pair<const std::string, uint32_t> ArtifactCrcTablePair;
7373

74-
void buildWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
75-
ArtifactCrcTable& newCommandCrcs);
74+
// Uses a hash of the artifact name, then the source name
75+
typedef std::unordered_map<uint32_t, uint32_t> HashedSourceArtifactCrcTable;
76+
typedef std::pair<uint32_t, uint32_t> HashedSourceArtifactCrcTablePair;
77+
78+
// Why read, merge, write? Because it's possible we ran another instance of cakelisp in the same
79+
// directory during our build phase. The caches are shared state, so we don't want to blow away
80+
// their data.
81+
void buildReadMergeWriteCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
82+
ArtifactCrcTable& newCommandCrcs,
83+
HashedSourceArtifactCrcTable& sourceArtifactFileCrcs);
7684

7785
// Returns false if there were errors; the file not existing is not an error
78-
bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs);
86+
bool buildReadCacheFile(const char* buildOutputDir, ArtifactCrcTable& cachedCommandCrcs,
87+
HashedSourceArtifactCrcTable& sourceArtifactFileCrcs);
7988

8089
// commandArguments should have terminating null sentinel
8190
bool commandEqualsCachedCommand(ArtifactCrcTable& cachedCommandCrcs, const char* artifactKey,

‎src/Evaluator.cpp

+94-5
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,7 @@ struct ComptimeBuildObject
770770
bool hasAnyRefs = false;
771771
std::string artifactsName;
772772
std::string dynamicLibraryPath;
773+
std::string sourceOutputName;
773774
std::string buildObjectName;
774775
std::vector<std::string> importLibraries;
775776
ObjectDefinition* definition = nullptr;
@@ -997,6 +998,7 @@ bool ComptimePrepareHeaders(EvaluatorEnvironment& environment)
997998
{
998999
environment.comptimeHeadersPrepared = true;
9991000
environment.comptimeCombinedHeaderFilename = combinedHeaderName;
1001+
setSourceArtifactCrc(environment, combinedHeaderRelativePath, precompiledHeaderFilename);
10001002
return true;
10011003
}
10021004

@@ -1203,6 +1205,7 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
12031205
char sourceOutputName[MAX_PATH_LENGTH] = {0};
12041206
PrintfBuffer(sourceOutputName, "%s/%s.cpp", cakelispWorkingDir,
12051207
buildObject.artifactsName.c_str());
1208+
buildObject.sourceOutputName = sourceOutputName;
12061209

12071210
char buildObjectName[MAX_PATH_LENGTH] = {0};
12081211
PrintfBuffer(buildObjectName, "%s/%s.%s", cakelispWorkingDir,
@@ -1436,6 +1439,9 @@ int BuildExecuteCompileTimeFunctions(EvaluatorEnvironment& environment,
14361439
if (logging.buildProcess)
14371440
Logf("Linked %s successfully\n", buildObject.definition->name.c_str());
14381441

1442+
setSourceArtifactCrc(environment, buildObject.sourceOutputName.c_str(),
1443+
buildObject.dynamicLibraryPath.c_str());
1444+
14391445
DynamicLibHandle builtLib = loadDynamicLibrary(buildObject.dynamicLibraryPath.c_str());
14401446
if (!builtLib)
14411447
{
@@ -1728,7 +1734,8 @@ bool EvaluateResolveReferences(EvaluatorEnvironment& environment)
17281734
{
17291735
// We're about to start compiling comptime code; read the cache
17301736
// TODO: Multiple comptime configurations require different working dir
1731-
if (!buildReadCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs))
1737+
if (!buildReadCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs,
1738+
environment.sourceArtifactFileCrcs))
17321739
return false;
17331740

17341741
// Print state
@@ -1881,9 +1888,10 @@ bool EvaluateResolveReferences(EvaluatorEnvironment& environment)
18811888

18821889
// Only write CRCs if we did some comptime compilation. Otherwise, the tokenizer will complain
18831890
// about loading a completely empty file
1884-
if (!environment.comptimeNewCommandCrcs.empty())
1885-
buildWriteCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs,
1886-
environment.comptimeNewCommandCrcs);
1891+
if (!environment.comptimeNewCommandCrcs.empty() || !environment.sourceArtifactFileCrcs.empty())
1892+
buildReadMergeWriteCacheFile(cakelispWorkingDir, environment.comptimeCachedCommandCrcs,
1893+
environment.comptimeNewCommandCrcs,
1894+
environment.sourceArtifactFileCrcs);
18871895

18881896
return errors == 0 && numBuildResolveErrors == 0;
18891897
}
@@ -2038,11 +2046,92 @@ void resetGeneratorOutput(GeneratorOutput& output)
20382046
output.imports.clear();
20392047
}
20402048

2049+
uint32_t cacheUpdateFileCrc(EvaluatorEnvironment& environment, const char* filename)
2050+
{
2051+
uint32_t sourceCrc = getFileCrc32(filename);
2052+
environment.cachedIntraBuildFileCrcs[filename] = sourceCrc;
2053+
return sourceCrc;
2054+
}
2055+
2056+
uint32_t getSourceArtifactKey(const char* source,
2057+
const char* artifact)
2058+
{
2059+
uint32_t artifactSourceNameCrc = 0;
2060+
crc32(artifact, strlen(artifact), &artifactSourceNameCrc);
2061+
crc32(source, strlen(source), &artifactSourceNameCrc);
2062+
return artifactSourceNameCrc;
2063+
}
2064+
2065+
// Call after an artifact has been built such to have source's latest changes
2066+
void setSourceArtifactCrc(EvaluatorEnvironment& environment, const char* source,
2067+
const char* artifact)
2068+
{
2069+
uint32_t sourceCrc = 0;
2070+
{
2071+
ArtifactCrcTable::iterator findIt = environment.cachedIntraBuildFileCrcs.find(source);
2072+
if (findIt != environment.cachedIntraBuildFileCrcs.end())
2073+
sourceCrc = findIt->second;
2074+
else
2075+
sourceCrc = cacheUpdateFileCrc(environment, source);
2076+
}
2077+
uint32_t artifactSourceNameCrc = getSourceArtifactKey(source, artifact);
2078+
2079+
// We haven't recorded this association yet, so we cannot confidently say artifact is up to
2080+
// date with source's changes. We now record the association with the assumption that by
2081+
// virtue of this function returning false, artifact will be updated with source's changes.
2082+
// if (logging.buildReasons)
2083+
// Logf("Artifact %s now has source %s changes (%u)\n", artifact, source, sourceCrc);
2084+
environment.sourceArtifactFileCrcs[artifactSourceNameCrc] = sourceCrc;
2085+
}
2086+
2087+
// If either of the files have a CRC that doesn't match what we had last build, we need to re-build
2088+
bool crcsMatchExpectedUpdateCrcPairing(EvaluatorEnvironment& environment, const char* source,
2089+
const char* artifact)
2090+
{
2091+
uint32_t artifactSourceNameCrc = getSourceArtifactKey(source, artifact);
2092+
HashedSourceArtifactCrcTable::iterator findIt =
2093+
environment.sourceArtifactFileCrcs.find(artifactSourceNameCrc);
2094+
if (findIt == environment.sourceArtifactFileCrcs.end())
2095+
{
2096+
if (logging.buildReasons)
2097+
Logf(
2098+
"Artifact %s was not associated with source %s before. There are %d associations\n",
2099+
artifact, source, (int)environment.sourceArtifactFileCrcs.size());
2100+
// We didn't know previously that artifact depends on source
2101+
return false;
2102+
}
2103+
else
2104+
{
2105+
uint32_t sourceCrc = 0;
2106+
{
2107+
ArtifactCrcTable::iterator findIt = environment.cachedIntraBuildFileCrcs.find(source);
2108+
if (findIt != environment.cachedIntraBuildFileCrcs.end())
2109+
sourceCrc = findIt->second;
2110+
else
2111+
sourceCrc = cacheUpdateFileCrc(environment, source);
2112+
}
2113+
bool sourceCrcMatchesLastUse = sourceCrc == findIt->second;
2114+
if (logging.buildReasons)
2115+
{
2116+
if (sourceCrcMatchesLastUse)
2117+
{
2118+
// Logf("Artifact %s not rebuilding because source %s CRC is same as last time
2119+
// (%u)\n", artifact, source, sourceCrc);
2120+
}
2121+
else
2122+
Logf("Artifact %s needs to build because source %s CRC is now %u\n", artifact,
2123+
source, sourceCrc);
2124+
}
2125+
return sourceCrcMatchesLastUse;
2126+
}
2127+
}
2128+
20412129
bool canUseCachedFile(EvaluatorEnvironment& environment, const char* filename,
20422130
const char* reference)
20432131
{
20442132
if (environment.useCachedFiles)
2045-
return !fileIsMoreRecentlyModified(filename, reference);
2133+
return !fileIsMoreRecentlyModified(filename, reference) &&
2134+
crcsMatchExpectedUpdateCrcPairing(environment, filename, reference);
20462135
else
20472136
return false;
20482137
}

‎src/Evaluator.hpp

+7
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,10 @@ struct EvaluatorEnvironment
336336
// If any artifact no longer matches its crc in cachedCommandCrcs, the change will appear here
337337
ArtifactCrcTable comptimeNewCommandCrcs;
338338

339+
// We cannot fully trust file modification times. Track the file contents hash to be sure
340+
ArtifactCrcTable cachedIntraBuildFileCrcs;
341+
HashedSourceArtifactCrcTable sourceArtifactFileCrcs;
342+
339343
// When a definition is replaced (e.g. by ReplaceAndEvaluateDefinition()), the original
340344
// definition's output is still used, but no longer has a definition to keep track of it. This
341345
// is also used for splices that don't have an owning object. We'll make sure the orphans get
@@ -500,6 +504,9 @@ CAKELISP_API bool GetCompileTimeVariable(EvaluatorEnvironment& environment, cons
500504
bool canUseCachedFile(EvaluatorEnvironment& environment, const char* filename,
501505
const char* reference);
502506

507+
void setSourceArtifactCrc(EvaluatorEnvironment& environment, const char* source,
508+
const char* artifact);
509+
503510
const char* objectTypeToString(ObjectType type);
504511

505512
// shortPath can be "Example.cake" or e.g. "../tests/Example.cake"

‎src/FileUtilities.cpp

+20
Original file line numberDiff line numberDiff line change
@@ -538,3 +538,23 @@ bool changeExtension(char* buffer, const char* newExtension)
538538
*extensionWrite = '\0';
539539
return true;
540540
}
541+
542+
uint32_t getFileCrc32(const char* filename)
543+
{
544+
FILE* file = fileOpen(filename, "rb");
545+
if (!file)
546+
return 0;
547+
548+
uint32_t crc = 0;
549+
550+
char buffer[4096] = {0};
551+
size_t numCharsRead = fread(buffer, 1, sizeof(buffer), file);
552+
while (numCharsRead)
553+
{
554+
crc32(buffer, numCharsRead, &crc);
555+
numCharsRead = fread(buffer, 1, sizeof(buffer), file);
556+
}
557+
558+
fclose(file);
559+
return crc;
560+
}

‎src/FileUtilities.hpp

+2
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,5 @@ CAKELISP_API void makeBackslashFilename(char* buffer, int bufferSize, const char
4646
// Does NOT validate whether your buffer can fit the new extension + null terminator
4747
// TODO: Safer version
4848
CAKELISP_API bool changeExtension(char* buffer, const char* newExtension);
49+
50+
CAKELISP_API uint32_t getFileCrc32(const char* filename);

‎src/ModuleManager.cpp

+20-8
Original file line numberDiff line numberDiff line change
@@ -1315,7 +1315,11 @@ bool moduleManagerBuild(ModuleManager& manager, std::vector<BuildObject*>& build
13151315
// Forget that the command was changed because the artifact wasn't successfully built
13161316
manager.newCommandCrcs.erase(object->filename.c_str());
13171317
succeededBuild = false;
1318-
continue;
1318+
}
1319+
else
1320+
{
1321+
setSourceArtifactCrc(manager.environment, object->sourceFilename.c_str(),
1322+
object->filename.c_str());
13191323
}
13201324
}
13211325

@@ -1548,6 +1552,10 @@ bool moduleManagerLink(ModuleManager& manager, std::vector<BuildObject*>& buildO
15481552
return false;
15491553
}
15501554

1555+
for (BuildObject* object : buildObjects)
1556+
setSourceArtifactCrc(manager.environment, object->filename.c_str(),
1557+
outputExecutableName.c_str());
1558+
15511559
Logf("Successfully built and linked %s\n", finalOutputName.c_str());
15521560
builtOutputs.push_back(finalOutputName);
15531561
}
@@ -1558,7 +1566,8 @@ bool moduleManagerLink(ModuleManager& manager, std::vector<BuildObject*>& buildO
15581566

15591567
bool moduleManagerBuildAndLink(ModuleManager& manager, std::vector<std::string>& builtOutputs)
15601568
{
1561-
if (!buildReadCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs))
1569+
if (!buildReadCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1570+
manager.environment.sourceArtifactFileCrcs))
15621571
return false;
15631572

15641573
// Pointer because the objects can't move, status codes are pointed to
@@ -1571,22 +1580,25 @@ bool moduleManagerBuildAndLink(ModuleManager& manager, std::vector<std::string>&
15711580
{
15721581
// Remember any succeeded artifact command CRCs so they don't get forgotten just because
15731582
// some others failed
1574-
buildWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1575-
manager.newCommandCrcs);
1583+
buildReadMergeWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1584+
manager.newCommandCrcs,
1585+
manager.environment.sourceArtifactFileCrcs);
15761586
return false;
15771587
}
15781588

15791589
if (!moduleManagerLink(manager, buildObjects, buildOptions, builtOutputs))
15801590
{
15811591
// Remember any succeeded artifact command CRCs so they don't get forgotten just because
15821592
// some others failed
1583-
buildWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1584-
manager.newCommandCrcs);
1593+
buildReadMergeWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1594+
manager.newCommandCrcs,
1595+
manager.environment.sourceArtifactFileCrcs);
15851596
return false;
15861597
}
15871598

1588-
buildWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1589-
manager.newCommandCrcs);
1599+
buildReadMergeWriteCacheFile(manager.buildOutputDir.c_str(), manager.cachedCommandCrcs,
1600+
manager.newCommandCrcs,
1601+
manager.environment.sourceArtifactFileCrcs);
15901602

15911603
return true;
15921604
}

0 commit comments

Comments
 (0)