Merge pull request #8302 from roc-lang/import-builtins

Import builtins from .roc files

Author: rtfeldman

.github/workflows/ci_zig.yml

@@ -64,6 +64,10 @@ jobs:
         run: |
           zig build test-playground -Doptimize=ReleaseSmall -- --verbose
 
+      - name: Verify Serialized Type Sizes
+        run: |
+          zig build test-serialization-sizes
+
   zig-tests:
     needs: check-once
     runs-on: ${{ matrix.os }}

build.zig

@@ -237,6 +237,12 @@ pub fn build(b: *std.Build) void {
         }
     }
 
+    // Also copy builtin_indices.bin
+    _ = write_compiled_builtins.addCopyFile(
+        .{ .cwd_relative = "zig-out/builtins/builtin_indices.bin" },
+        "builtin_indices.bin",
+    );
+
     // Generate compiled_builtins.zig dynamically based on discovered .roc files
     const builtins_source_str = generateCompiledBuiltinsSource(b, roc_files) catch |err| {
         std.debug.print("Failed to generate compiled_builtins.zig: {}\n", .{err});
@@ -252,6 +258,9 @@ pub fn build(b: *std.Build) void {
         .root_source_file = compiled_builtins_source,
     });
 
+    roc_modules.repl.addImport("compiled_builtins", compiled_builtins_module);
+    roc_modules.compile.addImport("compiled_builtins", compiled_builtins_module);
+
     // Manual rebuild command: zig build rebuild-builtins
     // Use this after making compiler changes to ensure those changes are reflected in builtins
     const rebuild_builtins_step = b.step(
@@ -334,7 +343,11 @@ pub fn build(b: *std.Build) void {
     }));
     playground_exe.entry = .disabled;
     playground_exe.rdynamic = true;
+    playground_exe.link_function_sections = true;
+    playground_exe.import_memory = false;
     roc_modules.addAll(playground_exe);
+    playground_exe.root_module.addImport("compiled_builtins", compiled_builtins_module);
+    playground_exe.step.dependOn(&write_compiled_builtins.step);
 
     add_tracy(b, roc_modules.build_options, playground_exe, b.resolveTargetQuery(.{
         .cpu_arch = .wasm32,
@@ -429,8 +442,8 @@ pub fn build(b: *std.Build) void {
     const tests_summary = TestsSummaryStep.create(b);
     const module_tests = roc_modules.createModuleTests(b, target, optimize, zstd, test_filters);
     for (module_tests) |module_test| {
-        // Add compiled builtins to check module tests
-        if (std.mem.eql(u8, module_test.test_step.name, "check")) {
+        // Add compiled builtins to check, repl, and eval module tests
+        if (std.mem.eql(u8, module_test.test_step.name, "check") or std.mem.eql(u8, module_test.test_step.name, "repl") or std.mem.eql(u8, module_test.test_step.name, "eval")) {
             module_test.test_step.root_module.addImport("compiled_builtins", compiled_builtins_module);
             module_test.test_step.step.dependOn(&write_compiled_builtins.step);
         }
@@ -496,6 +509,8 @@ pub fn build(b: *std.Build) void {
         roc_modules.addAll(cli_test);
         cli_test.linkLibrary(zstd.artifact("zstd"));
         add_tracy(b, roc_modules.build_options, cli_test, target, false, flag_enable_tracy);
+        cli_test.root_module.addImport("compiled_builtins", compiled_builtins_module);
+        cli_test.step.dependOn(&write_compiled_builtins.step);
 
         const run_cli_test = b.addRunArtifact(cli_test);
         if (run_args.len != 0) {
@@ -633,6 +648,9 @@ fn generateCompiledBuiltinsSource(b: *std.Build, roc_files: []const []const u8)
         });
     }
 
+    // Also embed builtin_indices.bin
+    try writer.writeAll("pub const builtin_indices_bin = @embedFile(\"builtin_indices.bin\");\n");
+
     return builtins_source.toOwnedSlice();
 }
 

src/base/CommonEnv.zig

@@ -84,13 +84,6 @@ pub fn serialize(
     return @constCast(offset_self);
 }
 
-/// Freezes the identifier and string interners, preventing further modifications.
-/// This is used to ensure thread safety when sharing the environment across threads.
-pub fn freezeInterners(self: *CommonEnv) void {
-    self.idents.freeze();
-    self.strings.freeze();
-}
-
 /// Serialized representation of ModuleEnv
 pub const Serialized = struct {
     idents: Ident.Store.Serialized,
@@ -123,10 +116,10 @@ pub const Serialized = struct {
         offset: i64,
         source: []const u8,
     ) *CommonEnv {
-        // CommonEnv.Serialized should be at least as big as CommonEnv
-        std.debug.assert(@sizeOf(Serialized) >= @sizeOf(CommonEnv));
-
-        // Overwrite ourself with the deserialized version, and return our pointer after casting it to CommonEnv.
+        // Note: Serialized may be smaller than the runtime struct because:
+        // - Uses i64 offsets instead of usize pointers (same size on 64-bit, but conceptually different)
+        // - May have different alignment/padding requirements
+        // We deserialize by overwriting the Serialized memory with the runtime struct.
         const env = @as(*CommonEnv, @ptrFromInt(@intFromPtr(self)));
 
         env.* = CommonEnv{

src/base/Ident.zig

@@ -115,10 +115,8 @@ pub const Store = struct {
 
         /// Deserialize this Serialized struct into a Store
         pub fn deserialize(self: *Serialized, offset: i64) *Store {
-            // Ident.Store.Serialized should be at least as big as Ident.Store
-            std.debug.assert(@sizeOf(Serialized) >= @sizeOf(Store));
-
-            // Overwrite ourself with the deserialized version, and return our pointer after casting it to Self.
+            // Note: Serialized may be smaller than the runtime struct.
+            // We deserialize by overwriting the Serialized memory with the runtime struct.
             const store = @as(*Store, @ptrFromInt(@intFromPtr(self)));
 
             store.* = Store{
@@ -226,10 +224,6 @@ pub const Store = struct {
         };
     }
 
-    /// Freeze the identifier store, preventing any new entries from being added.
-    pub fn freeze(self: *Store) void {
-        self.interner.freeze();
-    }
     /// Calculate the size needed to serialize this Ident.Store
     pub fn serializedSize(self: *const Store) usize {
         var size: usize = 0;
@@ -535,7 +529,7 @@ test "Ident.Store with genUnique CompactWriter roundtrip" {
     try std.testing.expectEqual(@as(u32, 3), deserialized.next_unique_name);
 }
 
-test "Ident.Store frozen state CompactWriter roundtrip" {
+test "Ident.Store CompactWriter roundtrip" {
     const gpa = std.testing.allocator;
 
     // Create and populate store
@@ -545,14 +539,6 @@ test "Ident.Store frozen state CompactWriter roundtrip" {
     _ = try original.insert(gpa, Ident.for_text("test1"));
     _ = try original.insert(gpa, Ident.for_text("test2"));
 
-    // Freeze the store
-    original.freeze();
-
-    // Verify interner is frozen
-    if (std.debug.runtime_safety) {
-        try std.testing.expect(original.interner.frozen);
-    }
-
     // Create a temp file
     var tmp_dir = std.testing.tmpDir(.{});
     defer tmp_dir.cleanup();
@@ -591,11 +577,6 @@ test "Ident.Store frozen state CompactWriter roundtrip" {
     const deserialized = @as(*Ident.Store, @ptrCast(@alignCast(buffer.ptr)));
 
     deserialized.relocate(@as(isize, @intCast(@intFromPtr(buffer.ptr))));
-
-    // Verify frozen state is preserved
-    if (std.debug.runtime_safety) {
-        try std.testing.expect(deserialized.interner.frozen);
-    }
 }
 
 test "Ident.Store comprehensive CompactWriter roundtrip" {

src/base/SmallStringInterner.zig

@@ -25,9 +25,6 @@ bytes: collections.SafeList(u8) = .{},
 hash_table: collections.SafeList(Idx) = .{},
 /// The current number of entries in the hash table.
 entry_count: u32 = 0,
-/// When true, no new entries can be added to the interner.
-/// This is set after parsing is complete.
-frozen: if (std.debug.runtime_safety) bool else void = if (std.debug.runtime_safety) false else {},
 
 /// A unique index for a deduped string in this interner.
 pub const Idx = enum(u32) {
@@ -134,10 +131,6 @@ fn resizeHashTable(self: *SmallStringInterner, gpa: std.mem.Allocator) std.mem.A
 
 /// Add a string to this interner, returning a unique, serial index.
 pub fn insert(self: *SmallStringInterner, gpa: std.mem.Allocator, string: []const u8) std.mem.Allocator.Error!Idx {
-    if (std.debug.runtime_safety) {
-        std.debug.assert(!self.frozen); // Should not insert into a frozen interner
-    }
-
     // Check if we need to resize the hash table (when 80% full = entry_count * 5 >= hash_table.len() * 4)
     if (self.entry_count * 5 >= self.hash_table.len() * 4) {
         try self.resizeHashTable(gpa);
@@ -178,13 +171,6 @@ pub fn getText(self: *const SmallStringInterner, idx: Idx) []u8 {
     return std.mem.sliceTo(bytes_slice[start..], 0);
 }
 
-/// Freeze the interner, preventing any new entries from being added.
-pub fn freeze(self: *SmallStringInterner) void {
-    if (std.debug.runtime_safety) {
-        self.frozen = true;
-    }
-}
-
 /// Serialize this interner to the given CompactWriter. The resulting interner
 /// in the writer's buffer will have offsets instead of pointers. Calling any
 /// methods on it or dereferencing its internal "pointers" (which are now
@@ -205,7 +191,6 @@ pub fn serialize(
         .bytes = serialized_bytes.*,
         .hash_table = serialized_hash_table.*,
         .entry_count = self.entry_count,
-        .frozen = self.frozen,
     };
 
     // Return the version of Self that's in the writer's buffer
@@ -223,7 +208,6 @@ pub const Serialized = struct {
     bytes: collections.SafeList(u8).Serialized,
     hash_table: collections.SafeList(Idx).Serialized,
     entry_count: u32,
-    frozen: if (std.debug.runtime_safety) bool else void,
 
     /// Serialize a SmallStringInterner into this Serialized struct, appending data to the writer
     pub fn serialize(
@@ -238,22 +222,17 @@ pub const Serialized = struct {
         try self.hash_table.serialize(&interner.hash_table, allocator, writer);
         // Copy simple values directly
         self.entry_count = interner.entry_count;
-        self.frozen = interner.frozen;
     }
 
     /// Deserialize this Serialized struct into a SmallStringInterner
     pub fn deserialize(self: *Serialized, offset: i64) *SmallStringInterner {
-        // Self.Serialized should be at least as big as Self
-        std.debug.assert(@sizeOf(Serialized) >= @sizeOf(SmallStringInterner));
-
         // Overwrite ourself with the deserialized version, and return our pointer after casting it to Self.
         const interner = @as(*SmallStringInterner, @ptrCast(self));
 
         interner.* = .{
             .bytes = self.bytes.deserialize(offset).*,
             .hash_table = self.hash_table.deserialize(offset).*,
             .entry_count = self.entry_count,
-            .frozen = self.frozen,
         };
 
         return interner;
@@ -460,7 +439,7 @@ test "SmallStringInterner with populated hashmap CompactWriter roundtrip" {
     try std.testing.expect(original_entry_count > 0);
 }
 
-test "SmallStringInterner frozen state CompactWriter roundtrip" {
+test "SmallStringInterner CompactWriter roundtrip" {
     const gpa = std.testing.allocator;
 
     // Create and populate interner
@@ -470,14 +449,6 @@ test "SmallStringInterner frozen state CompactWriter roundtrip" {
     _ = try original.insert(gpa, "test1");
     _ = try original.insert(gpa, "test2");
 
-    // Freeze the interner
-    original.freeze();
-
-    // Verify it's frozen
-    if (std.debug.runtime_safety) {
-        try std.testing.expect(original.frozen);
-    }
-
     // Create a temp file
     var tmp_dir = std.testing.tmpDir(.{});
     defer tmp_dir.cleanup();
@@ -510,11 +481,6 @@ test "SmallStringInterner frozen state CompactWriter roundtrip" {
     const deserialized = @as(*SmallStringInterner, @ptrCast(@alignCast(buffer.ptr)));
     deserialized.relocate(@as(isize, @intCast(@intFromPtr(buffer.ptr))));
 
-    // Verify frozen state is preserved
-    if (std.debug.runtime_safety) {
-        try std.testing.expect(deserialized.frozen);
-    }
-
     // Verify strings are still accessible
     // Note: Index 0 is reserved for the unused marker, so strings start at index 1
     try std.testing.expectEqualStrings("test1", deserialized.getText(@enumFromInt(1)));
@@ -682,14 +648,11 @@ test "SmallStringInterner edge cases CompactWriter roundtrip" {
 //     try std.testing.expectEqualStrings("interner1_string2", deserialized1.getText(idx1_2));
 //     try std.testing.expectEqual(@as(u32, 2), deserialized1.entry_count);
 
-//     // Verify interner 2 (frozen)
+//     // Verify interner 2
 //     try std.testing.expectEqualStrings("interner2_string1", deserialized2.getText(idx2_1));
 //     try std.testing.expectEqualStrings("interner2_string2", deserialized2.getText(idx2_2));
 //     try std.testing.expectEqualStrings("interner2_string3", deserialized2.getText(idx2_3));
 //     try std.testing.expectEqual(@as(u32, 3), deserialized2.entry_count);
-//     if (std.debug.runtime_safety) {
-//         try std.testing.expect(deserialized2.frozen);
-//     }
 
 //     // Verify interner 3
 //     try std.testing.expectEqualStrings("interner3_string1", deserialized3.getText(idx3_1));