add some more comments

Author: vchuravy

src/driver.jl

@@ -191,7 +191,9 @@ const __llvm_initialized = Ref(false)
     entry = finish_module!(job, ir, entry)
 
     # deferred code generation
+    run_optimization_for_deferred = false
     if haskey(functions(ir), "gpuc.lookup")
+        run_optimization_for_deferred = true
         dyn_marker = functions(ir)["gpuc.lookup"]
 
         # gpuc.deferred is lowered to a gpuc.lookup foreigncall, so we need to extract the
@@ -262,9 +264,9 @@ const __llvm_initialized = Ref(false)
         unsafe_delete!(ir, dyn_marker)
     end
     ## old, deprecated implementation
-    has_deferred_jobs = toplevel && !only_entry && haskey(functions(ir), "deferred_codegen")
     jobs = Dict{CompilerJob, String}(job => entry_fn)
-    if has_deferred_jobs
+    if toplevel && !only_entry && haskey(functions(ir), "deferred_codegen")
+        run_optimization_for_deferred = true
         dyn_marker = functions(ir)["deferred_codegen"]
 
         # iterative compilation (non-recursive)
@@ -395,7 +397,7 @@ const __llvm_initialized = Ref(false)
                 # deferred codegen has some special optimization requirements,
                 # which also need to happen _after_ regular optimization.
                 # XXX: make these part of the optimizer pipeline?
-                if has_deferred_jobs
+                if run_optimization_for_deferred
                     @dispose pb=NewPMPassBuilder() begin
                         add!(pb, NewPMFunctionPassManager()) do fpm
                             add!(fpm, InstCombinePass())

src/irgen.jl

@@ -80,6 +80,11 @@ function irgen(@nospecialize(job::CompilerJob))
     compiled[job.source] =
         (; compiled[job.source].ci, func, specfunc)
 
+    # Earlier we sanitize global names, this invalidates the
+    # func, specfunc names safed in compiled. Update the names now,
+    # such that when when use the compiled mappings to lookup the
+    # llvm function for a methodinstance (deferred codegen) we have 
+    # valid targets.
     for mi in keys(compiled)
         mi == job.source && continue
         ci, func, specfunc = compiled[mi]

src/jlgen.jl

@@ -692,16 +692,22 @@ function compile_method_instance(@nospecialize(job::CompilerJob))
         error("Cannot compile $(job.source) for world $(job.world); method is only valid in worlds $(job.source.def.primary_world) to $(job.source.def.deleted_world)")
     end
 
+    # A poor man's worklist implementation.
+    # `compiled` contains a mapping from `mi->ci, func, specfunc`
+    # FIXME: Since we are disabling Julia internal caching we might
+    #        generate for the same mi multiple LLVM functions. 
+    # `outstanding` are the missing edges that were not compiled by `compile_method_instance`
+    # Currently these edges are generated through deferred codegen.
     compiled = IdDict()
     llvm_mod, outstanding = compile_method_instance(job, compiled)
     worklist = outstanding
     while !isempty(worklist)
         source = pop!(worklist)
-        haskey(compiled, source) && continue
+        haskey(compiled, source) && continue # We have fulfilled the request already
+        # Create a new compiler job for this edge, reusing the config settings from the inital one
         job2 = CompilerJob(source, job.config)
-        @debug "Processing..." job2
         llvm_mod2, outstanding = compile_method_instance(job2, compiled)
-        append!(worklist, outstanding)
+        append!(worklist, outstanding) # merge worklist with new outstanding edges
         @assert context(llvm_mod) == context(llvm_mod2)
         link!(llvm_mod, llvm_mod2)
     end
@@ -818,6 +824,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob), compiled::IdDi
 
         # NOTE: it's not safe to store raw LLVM functions here, since those may get
         #       removed or renamed during optimization, so we store their name instead.
+        # FIXME: Enable this assert when we have a fully featured worklist
+        # @assert !haskey(compiled, mi)
         compiled[mi] = (; ci, func=llvm_func, specfunc=llvm_specfunc)
     end