Added ability to build arbitrary C++ files

This is necessary to interface with existing C++ code easily.

Author: makuto

BuildAndRunTests.sh

@@ -8,4 +8,5 @@
 # jam -j4 && ./bin/cakelisp --enable-hot-reloading runtime/TestMain.cake runtime/TextAdventure.cake \
 	# && cd runtime && jam -j4
 
-jam -j4 && ./bin/cakelisp --verbose-build-process runtime/TestMain.cake runtime/TextAdventure.cake
+jam -j4 && ./bin/cakelisp --verbose-build-process --verbose-processes \
+						  runtime/TestMain.cake runtime/TextAdventure.cake

runtime/HotReloading.cake

@@ -4,18 +4,21 @@
 ;; Include cakelisp source for DynamicLoader.hpp
 (set-module-option build-time-compile-arguments
                    "-std=c++11" "-Wall" "-Wextra" "-Wno-unused-parameter" "-O0"
+                   ;; TODO: Multiplatform support
+                   "-DUNIX"
                    "-g" "-c" 'source-input "-o" 'object-output "-fPIC" "-Isrc/")
 ;; TODO: This only makes sense on a per-target basis. Instead, modules should be able to append
 ;; arguments to the link command only
 (set-module-option build-time-linker "/usr/bin/clang++")
 ;; This needs to link -ldl and such (depending on platform...)
 (set-cakelisp-option build-time-link-arguments
                      "-shared" "-o" 'executable-output 'object-input
-                     ;; TODO: Make this get built by cakelisp too?
-                     "src/DynamicLoader.o"
                      ;; TODO: OS dependent
                      "-ldl" "-lpthread")
 
+;; TODO: Relative vs. absolute paths
+(add-cpp-build-dependency "../src/DynamicLoader.cpp")
+
 (import &comptime-only "Macros.cake")
 (c-import "<unordered_map>" "<vector>")
 (c-import "DynamicLoader.hpp")

src/Building.cpp

@@ -0,0 +1,17 @@
+#include "Building.hpp"
+
+#include "Utilities.hpp"
+#include "FileUtilities.hpp"
+
+bool objectFilenameFromSourceFilename(const char* cakelispWorkingDir, const char* sourceFilename,
+                                      char* bufferOut, int bufferSize)
+{
+	char buildFilename[MAX_NAME_LENGTH] = {0};
+	getFilenameFromPath(sourceFilename, buildFilename, sizeof(buildFilename));
+	if (!buildFilename[0])
+		return false;
+
+	// TODO: Trim .cake.cpp (etc.)
+	SafeSnprinf(bufferOut, bufferSize, "%s/%s.o", cakelispWorkingDir, buildFilename);
+	return true;
+}

src/Building.hpp

@@ -0,0 +1,4 @@
+#pragma once
+
+bool objectFilenameFromSourceFilename(const char* cakelispWorkingDir, const char* sourceFilename,
+                                      char* bufferOut, int bufferSize);

src/FileUtilities.cpp

@@ -69,3 +69,12 @@ void getFilenameFromPath(const char* path, char* bufferOut, int bufferSize)
 #error Need to be able to strip path to get filename on this platform
 #endif
 }
+
+void makePathRelativeToFile(const char* filePath, const char* referencedFilePath, char* bufferOut,
+                            int bufferSize)
+{
+	getDirectoryFromPath(filePath, bufferOut, bufferSize);
+	// TODO: Need to make this safe!
+	strcat(bufferOut, "/");
+	strcat(bufferOut, referencedFilePath);
+}

src/FileUtilities.hpp

@@ -9,3 +9,5 @@ void makeDirectory(const char* path);
 
 void getDirectoryFromPath(const char* path, char* bufferOut, int bufferSize);
 void getFilenameFromPath(const char* path, char* bufferOut, int bufferSize);
+void makePathRelativeToFile(const char* filePath, const char* referencedFilePath, char* bufferOut,
+                            int bufferSize);

src/Generators.cpp

@@ -395,6 +395,39 @@ bool ImportGenerator(EvaluatorEnvironment& environment, const EvaluatorContext&
 	return true;
 }
 
+bool AddDependencyGenerator(EvaluatorEnvironment& environment, const EvaluatorContext& context,
+                            const std::vector<Token>& tokens, int startTokenIndex,
+                            GeneratorOutput& output)
+{
+	if (!context.module)
+	{
+		ErrorAtToken(tokens[startTokenIndex],
+		             "module cannot track dependency (potential internal error)");
+		return false;
+	}
+
+	int endInvocationIndex = FindCloseParenTokenIndex(tokens, startTokenIndex);
+
+	int invocationIndex = startTokenIndex + 1;
+
+	int nameIndex = getExpectedArgument("expected dependency name", tokens, startTokenIndex, 1,
+	                                    endInvocationIndex);
+	if (nameIndex == -1)
+		return false;
+	if (!ExpectTokenType("add dependency", tokens[nameIndex], TokenType_String))
+		return false;
+
+	if (tokens[invocationIndex].contents.compare("add-cpp-build-dependency") == 0)
+	{
+		ModuleDependency newDependency = {};
+		newDependency.type = ModuleDependency_CFile;
+		newDependency.name = tokens[nameIndex].contents;
+		context.module->dependencies.push_back(newDependency);
+	}
+
+	return true;
+}
+
 bool DefunGenerator(EvaluatorEnvironment& environment, const EvaluatorContext& context,
                     const std::vector<Token>& tokens, int startTokenIndex, GeneratorOutput& output)
 {
@@ -1970,6 +2003,7 @@ void importFundamentalGenerators(EvaluatorEnvironment& environment)
 	environment.generators["set-module-option"] = SetModuleOption;
 
 	environment.generators["skip-build"] = SkipBuildGenerator;
+	environment.generators["add-cpp-build-dependency"] = AddDependencyGenerator;
 
 	// Dispatches based on invocation name
 	const char* cStatementKeywords[] = {

src/Jamfile

@@ -16,7 +16,9 @@ RunProcess.cpp
 OutputPreambles.cpp
 DynamicLoader.cpp
 ModuleManager.cpp
-Logging.cpp ;
+Logging.cpp
+Building.cpp
+;
 
 MakeLocate cakelisp$(SUFEXE) : bin ;
 MakeLocate libCakelisp.a : lib ;

src/ModuleManager.cpp

@@ -1,5 +1,6 @@
 #include "ModuleManager.hpp"
 
+#include "Building.hpp"
 #include "Converters.hpp"
 #include "DynamicLoader.hpp"
 #include "Evaluator.hpp"
@@ -292,7 +293,10 @@ static void OnCompileProcessOutput(const char* output)
 struct BuiltObject
 {
 	int buildStatus;
+	std::string sourceFilename;
 	std::string filename;
+
+	ProcessCommand* buildCommandOverride;
 };
 
 void builtObjectsFree(std::vector<BuiltObject*>& objects)
@@ -315,6 +319,12 @@ bool moduleManagerBuild(ModuleManager& manager)
 	{
 		Module* module = manager.modules[moduleIndex];
 
+		ProcessCommand* buildCommandOverride =
+		    (!module->buildTimeBuildCommand.fileToExecute.empty() &&
+		     !module->buildTimeBuildCommand.arguments.empty()) ?
+		        &module->buildTimeBuildCommand :
+		        nullptr;
+
 		if (log.buildProcess)
 			printf("Build module %s\n", module->sourceOutputName.c_str());
 		for (ModuleDependency& dependency : module->dependencies)
@@ -326,48 +336,70 @@ bool moduleManagerBuild(ModuleManager& manager)
 			// ourselves with them
 			if (dependency.type == ModuleDependency_Cakelisp)
 				continue;
+
+			if (dependency.type == ModuleDependency_CFile)
+			{
+				BuiltObject* newBuiltObject = new BuiltObject;
+				newBuiltObject->buildStatus = 0;
+				// TODO: Better to use search directories
+				char sourceName[MAX_PATH_LENGTH] = {0};
+				makePathRelativeToFile(module->sourceOutputName.c_str(), dependency.name.c_str(),
+				                       sourceName, sizeof(sourceName));
+				newBuiltObject->sourceFilename = sourceName;
+
+				char buildObjectName[MAX_PATH_LENGTH] = {0};
+				if (!objectFilenameFromSourceFilename(cakelispWorkingDir,
+				                                      newBuiltObject->sourceFilename.c_str(),
+				                                      buildObjectName, sizeof(buildObjectName)))
+				{
+					builtObjectsFree(builtObjects);
+					return false;
+				}
+
+				newBuiltObject->filename = buildObjectName;
+				newBuiltObject->buildCommandOverride = buildCommandOverride;
+				builtObjects.push_back(newBuiltObject);
+			}
 		}
 
 		// TODO: Importing needs to set this on the module, not the dependency...
 		if (module->skipBuild)
 			continue;
 
-		// TODO: Lots of overlap between this and compile-time building
-		char buildFilename[MAX_NAME_LENGTH] = {0};
-		getFilenameFromPath(module->sourceOutputName.c_str(), buildFilename, sizeof(buildFilename));
-		if (!buildFilename[0])
+		char buildObjectName[MAX_PATH_LENGTH] = {0};
+		if (!objectFilenameFromSourceFilename(cakelispWorkingDir, module->sourceOutputName.c_str(),
+		                                      buildObjectName, sizeof(buildObjectName)))
 		{
 			builtObjectsFree(builtObjects);
 			return false;
 		}
 
-		// TODO: Trim .cake.cpp
-		char buildObjectName[MAX_PATH_LENGTH] = {0};
-		PrintfBuffer(buildObjectName, "%s/%s.o", cakelispWorkingDir, buildFilename);
-
 		// At this point, we do want to build the object. We might skip building it if it is cached.
 		// In that case, the status code should still be 0, as if we built and succeeded building it
 		BuiltObject* newBuiltObject = new BuiltObject;
 		newBuiltObject->buildStatus = 0;
+		newBuiltObject->sourceFilename = module->sourceOutputName.c_str();
 		newBuiltObject->filename = buildObjectName;
+		newBuiltObject->buildCommandOverride = buildCommandOverride;
 		builtObjects.push_back(newBuiltObject);
+	}
 
-		if (!fileIsMoreRecentlyModified(module->sourceOutputName.c_str(), buildObjectName))
+	for (BuiltObject* object : builtObjects)
+	{
+		if (!fileIsMoreRecentlyModified(object->sourceFilename.c_str(), object->filename.c_str()))
 		{
 			if (log.buildProcess)
-				printf("Skipping compiling %s (using cached object)\n",
-				       module->sourceOutputName.c_str());
+				printf("Skipping compiling %s (using cached object)\n", object->sourceFilename.c_str());
 		}
 		else
 		{
-			ProcessCommand& buildCommand = (!module->buildTimeBuildCommand.fileToExecute.empty() &&
-			                                !module->buildTimeBuildCommand.arguments.empty()) ?
-			                                   module->buildTimeBuildCommand :
+			ProcessCommand& buildCommand = object->buildCommandOverride ?
+			                                   *object->buildCommandOverride :
 			                                   manager.environment.buildTimeBuildCommand;
 
 			ProcessCommandInput buildTimeInputs[] = {
-			    {ProcessCommandArgumentType_SourceInput, {module->sourceOutputName.c_str()}},
-			    {ProcessCommandArgumentType_ObjectOutput, {buildObjectName}}};
+			    {ProcessCommandArgumentType_SourceInput, {object->sourceFilename.c_str()}},
+			    {ProcessCommandArgumentType_ObjectOutput, {object->filename.c_str()}}};
 			const char** buildArguments = MakeProcessArgumentsFromCommand(
 			    buildCommand, buildTimeInputs, ArraySize(buildTimeInputs));
 			if (!buildArguments)
@@ -381,7 +413,7 @@ bool moduleManagerBuild(ModuleManager& manager)
 			compileArguments.arguments = buildArguments;
 			// PrintProcessArguments(buildArguments);
 
-			if (runProcess(compileArguments, &newBuiltObject->buildStatus) != 0)
+			if (runProcess(compileArguments, &object->buildStatus) != 0)
 			{
 				printf("error: failed to invoke compiler\n");
 				free(buildArguments);
@@ -408,7 +440,6 @@ bool moduleManagerBuild(ModuleManager& manager)
 	// TODO: Make configurable
 	const char* outputExecutableName = "a.out";
 
-	int objectNameBufferSize = 0;
 	int numObjectsToLink = 0;
 	bool succeededBuild = true;
 	bool needsLink = false;
@@ -424,7 +455,7 @@ bool moduleManagerBuild(ModuleManager& manager)
 		}
 
 		if (log.buildProcess)
-			printf("Linking %s\n", object->filename.c_str());
+			printf("Need to link %s\n", object->filename.c_str());
 
 		++numObjectsToLink;