Code loading: add support for "portable scripts".

These are jl files that have project and manifest embedded inside them, similar to how Pluto notebooks have done for quite a while.

The delimiters are `#!manifest begin` and `#!manifest end` (analogous for project data) and the content can either have single line comments or be inside a multi line comment.

When the active project is set to a portable script the project and manifest data will be read from the inlined toml data.

Starting julia with a file (`julia file.jl`) will set the file as the active project if julia detects that it is a portable script.

Author: KristofferC

base/client.jl

@@ -275,6 +275,11 @@ function exec_options(opts)
     # remove filename from ARGS
     global PROGRAM_FILE = arg_is_program ? popfirst!(ARGS) : ""
 
+    if arg_is_program && PROGRAM_FILE != "-" && Base.active_project(false) === nothing
+        script_path = abspath(PROGRAM_FILE)
+        Base.has_inline_project(script_path) && Base.set_active_project(script_path)
+    end
+
     # Load Distributed module only if any of the Distributed options have been specified.
     distributed_mode = (opts.worker == 1) || (opts.nprocs > 0) || (opts.machine_file != C_NULL)
     if distributed_mode

base/initdefs.jl

@@ -291,7 +291,11 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
             # Expand trailing relative path
             dir = dirname(program_file)
             dir = env != "@script" ? (dir * env[length("@script")+1:end]) : dir
-            return current_project(dir)
+            project = current_project(dir)
+            if project === nothing && env == "@script" && Base.has_inline_project(program_file)
+                return abspath(program_file)
+            end
+            return project
         end
         env = replace(env, '#' => VERSION.major, count=1)
         env = replace(env, '#' => VERSION.minor, count=1)
@@ -326,7 +330,9 @@ load_path_expand(::Nothing) = nothing
 """
     active_project()
 
-Return the path of the active `Project.toml` file. See also [`Base.set_active_project`](@ref).
+Return the path of the active project (either a `Project.toml` file or a julia
+file when using a [portable script](@ref portable-scripts)).
+See also [`Base.set_active_project`](@ref).
 """
 function active_project(search_load_path::Bool=true)
     for project in (ACTIVE_PROJECT[],)
@@ -355,7 +361,9 @@ end
 """
     set_active_project(projfile::Union{AbstractString,Nothing})
 
-Set the active `Project.toml` file to `projfile`. See also [`Base.active_project`](@ref).
+Set the active `Project.toml` file to `projfile`. The `projfile` can be a path to a traditional
+`Project.toml` file, a [portable script](@ref portable-scripts) with inline metadata, or `nothing`
+to clear the active project. See also [`Base.active_project`](@ref).
 
 !!! compat "Julia 1.8"
     This function requires at least Julia 1.8.
@@ -376,6 +384,7 @@ end
     active_manifest(project_file::AbstractString)
 
 Return the path of the active manifest file, or the manifest file that would be used for a given `project_file`.
+When a [portable script](@ref portable-scripts) is active, this returns the script path itself.
 
 In a stacked environment (where multiple environments exist in the load path), this returns the manifest
 file for the primary (active) environment only, not the manifests from other environments in the stack.

base/loading.jl

@@ -212,22 +212,22 @@ mutable struct CachedTOMLDict
     size::Int64
     hash::UInt32
     d::Dict{String, Any}
+    kind::Symbol  # :full (regular TOML), :project, :manifest (inline)
 end
 
-function CachedTOMLDict(p::TOML.Parser, path::String)
+function CachedTOMLDict(p::TOML.Parser, path::String, kind)
     s = stat(path)
-    content = read(path)
+    content = if !isfile(path)
+        ""  # A bit fishy maybe...
+    elseif kind === :full
+        String(read(path))
+    else
+        String(extract_inline_section(path, kind))
+    end
     crc32 = _crc32c(content)
-    TOML.reinit!(p, String(content); filepath=path)
+    TOML.reinit!(p, content; filepath=path)
     d = TOML.parse(p)
-    return CachedTOMLDict(
-        path,
-        s.inode,
-        s.mtime,
-        s.size,
-        crc32,
-        d,
-   )
+    return CachedTOMLDict(path, s.inode, s.mtime, s.size, crc32, d, kind)
 end
 
 function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict)
@@ -236,20 +236,126 @@ function get_updated_dict(p::TOML.Parser, f::CachedTOMLDict)
     # identical but that is solvable by not doing in-place updates, and not
     # rapidly changing these files
     if s.inode != f.inode || s.mtime != f.mtime || f.size != s.size
-        content = read(f.path)
-        new_hash = _crc32c(content)
+        file_content = read(f.path)
+        new_hash = _crc32c(file_content)
         if new_hash != f.hash
             f.inode = s.inode
             f.mtime = s.mtime
             f.size = s.size
             f.hash = new_hash
-            TOML.reinit!(p, String(content); filepath=f.path)
+
+            # Extract the appropriate TOML content based on kind
+            toml_content = if f.kind == :full
+                String(file_content)
+            else
+                String(extract_inline_section(f.path, f.kind))
+            end
+
+            TOML.reinit!(p, toml_content; filepath=f.path)
             return f.d = TOML.parse(p)
         end
     end
     return f.d
 end
 
+
+function extract_inline_section(path::String, type::Symbol)
+    buf = IOBuffer()
+    start_fence = "#!$type begin"
+    end_fence = "#!$type end"
+    state = :none
+    multiline_mode = false
+    in_multiline = false
+
+    for (lineno, line) in enumerate(eachline(path))
+        stripped = lstrip(line)
+        state == :done && break
+
+        if startswith(stripped, start_fence)
+            state = :reading_first
+            continue
+        elseif startswith(stripped, end_fence)
+            state = :done
+            continue
+        elseif state === :reading_first
+            # First line determines the format
+            if startswith(stripped, "#=")
+                multiline_mode = true
+                state = :reading
+                # Check if the opening #= and closing =# are on the same line
+                if endswith(rstrip(stripped), "=#")
+                    # Single-line multi-line comment
+                    content = rstrip(stripped)[3:end-2]
+                    write(buf, content)
+                    in_multiline = false
+                else
+                    # Multi-line comment continues
+                    in_multiline = true
+                    content = stripped[3:end]  # Remove #= from start
+                    write(buf, content)
+                    write(buf, '\n')
+                end
+            else
+                # Line-by-line format
+                multiline_mode = false
+                state = :reading
+                # Process this first line
+                if startswith(stripped, '#')
+                    toml_line = lstrip(chop(stripped, head=1, tail=0))
+                    write(buf, toml_line)
+                else
+                    write(buf, line)
+                end
+                write(buf, '\n')
+            end
+        elseif state === :reading
+            if multiline_mode && in_multiline
+                # In multi-line comment mode, look for closing =#
+                if endswith(rstrip(stripped), "=#")
+                    # Found closing delimiter
+                    content = rstrip(stripped)[1:end-2]  # Remove =# from end
+                    write(buf, content)
+                    in_multiline = false
+                else
+                    # Still inside multi-line comment
+                    write(buf, line)
+                    write(buf, '\n')
+                end
+            elseif !multiline_mode
+                # Line-by-line comment mode, strip # from each line
+                if startswith(stripped, '#')
+                    toml_line = lstrip(chop(stripped, head=1, tail=0))
+                    write(buf, toml_line)
+                else
+                    write(buf, line)
+                end
+                write(buf, '\n')
+            end
+            # If multiline_mode && !in_multiline, the multiline comment has ended.
+            # Don't accumulate any more content; just wait for the end fence.
+        end
+    end
+
+    if state === :done
+        return strip(String(take!(buf)))
+    elseif state === :none
+        return ""
+    else
+        error("incomplete inline $type block in $path (missing #!$type end)")
+    end
+end
+
+function has_inline_project(path::String)::Bool
+    for line in eachline(path)
+        stripped = lstrip(line)
+        if startswith(stripped, "#!project begin")
+            return true
+        end
+    end
+    return false
+end
+
+
 struct LoadingCache
     load_path::Vector{String}
     dummy_uuid::Dict{String, UUID}
@@ -273,26 +379,38 @@ TOMLCache(p::TOML.Parser, d::Dict{String, Dict{String, Any}}) = TOMLCache(p, con
 
 const TOML_CACHE = TOMLCache(TOML.Parser{nothing}())
 
-parsed_toml(project_file::AbstractString) = parsed_toml(project_file, TOML_CACHE, require_lock)
-function parsed_toml(project_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock)
+parsed_toml(toml_file::AbstractString; manifest::Bool=false, project::Bool=!manifest) =
+    parsed_toml(toml_file, TOML_CACHE, require_lock; manifest=manifest, project=project)
+function parsed_toml(toml_file::AbstractString, toml_cache::TOMLCache, toml_lock::ReentrantLock;
+                     manifest::Bool=false, project::Bool=!manifest)
+    manifest && project && throw(ArgumentError("cannot request both project and manifest TOML"))
     lock(toml_lock) do
+        # Portable script?
+        if endswith(toml_file, ".jl") && isfile_casesensitive(toml_file)
+            kind = manifest ? :manifest : :project
+            cache_key = "$(toml_file)::$(kind)"
+        else
+            kind = :full
+            cache_key = toml_file
+        end
+
         cache = LOADING_CACHE[]
-        dd = if !haskey(toml_cache.d, project_file)
-            d = CachedTOMLDict(toml_cache.p, project_file)
-            toml_cache.d[project_file] = d
+        dd = if !haskey(toml_cache.d, cache_key)
+            d = CachedTOMLDict(toml_cache.p, toml_file, kind)
+            toml_cache.d[cache_key] = d
             d.d
         else
-            d = toml_cache.d[project_file]
+            d = toml_cache.d[cache_key]
             # We are in a require call and have already parsed this TOML file
             # assume that it is unchanged to avoid hitting disk
-            if cache !== nothing && project_file in cache.require_parsed
+            if cache !== nothing && cache_key in cache.require_parsed
                 d.d
             else
                 get_updated_dict(toml_cache.p, d)
             end
         end
         if cache !== nothing
-            push!(cache.require_parsed, project_file)
+            push!(cache.require_parsed, cache_key)
         end
         return dd
     end
@@ -663,6 +781,8 @@ function env_project_file(env::String)::Union{Bool,String}
         project_file = locate_project_file(env)
     elseif basename(env) in project_names && isfile_casesensitive(env)
         project_file = env
+    elseif endswith(env, ".jl") && isfile_casesensitive(env)
+        project_file = has_inline_project(env) ? env : false
     else
         project_file = false
     end
@@ -885,7 +1005,8 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
         manifest_path === missing || return manifest_path
     end
     dir = abspath(dirname(project_file))
-    isfile_casesensitive(project_file) || return nothing
+    has_file = isfile_casesensitive(project_file)
+    has_file || return nothing
     d = parsed_toml(project_file)
     base_manifest = workspace_manifest(project_file)
     if base_manifest !== nothing
@@ -894,10 +1015,11 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String}
     explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing}
     manifest_path = nothing
     if explicit_manifest !== nothing
-        manifest_file = normpath(joinpath(dir, explicit_manifest))
-        if isfile_casesensitive(manifest_file)
-            manifest_path = manifest_file
-        end
+        manifest_path = normpath(joinpath(dir, explicit_manifest))
+    end
+    if manifest_path === nothing && endswith(project_file, ".jl") && has_file
+        # portable script: manifest is the same file as the project file
+        manifest_path = project_file
     end
     if manifest_path === nothing
         for mfst in manifest_names
@@ -1026,7 +1148,7 @@ dep_stanza_get(stanza::Nothing, name::String) = nothing
 function explicit_manifest_deps_get(project_file::String, where::PkgId, name::String)::Union{Nothing,PkgId}
     manifest_file = project_file_manifest_path(project_file)
     manifest_file === nothing && return nothing # manifest not found--keep searching LOAD_PATH
-    d = get_deps(parsed_toml(manifest_file))
+    d = get_deps(parsed_toml(manifest_file; manifest=true))
     for (dep_name, entries) in d
         entries::Vector{Any}
         for entry in entries
@@ -1099,7 +1221,7 @@ function explicit_manifest_uuid_path(project_file::String, pkg::PkgId)::Union{No
     manifest_file = project_file_manifest_path(project_file)
     manifest_file === nothing && return nothing # no manifest, skip env
 
-    d = get_deps(parsed_toml(manifest_file))
+    d = get_deps(parsed_toml(manifest_file; manifest=true))
     entries = get(d, pkg.name, nothing)::Union{Nothing, Vector{Any}}
     if entries !== nothing
         for entry in entries
@@ -1531,7 +1653,7 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi
         project_file isa String || return nothing
         manifest_file = project_file_manifest_path(project_file)
         manifest_file === nothing && return
-        d = get_deps(parsed_toml(manifest_file))
+        d = get_deps(parsed_toml(manifest_file; manifest=true))
         for (dep_name, entries) in d
             entries::Vector{Any}
             for entry in entries
@@ -2507,7 +2629,7 @@ function find_unsuitable_manifests_versions()
         project_file isa String || continue # no project file
         manifest_file = project_file_manifest_path(project_file)
         manifest_file isa String || continue # no manifest file
-        m = parsed_toml(manifest_file)
+        m = parsed_toml(manifest_file; manifest=true)
         man_julia_version = get(m, "julia_version", nothing)
         man_julia_version isa String || @goto mark
         man_julia_version = VersionNumber(man_julia_version)

base/precompilation.jl

@@ -108,7 +108,7 @@ function ExplicitEnv(envpath::String)
     end
 
     manifest = project_file_manifest_path(envpath)
-    manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest)
+    manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest; manifest=true)
 
     # Dependencies in a manifest can either be stored compressed (when name is unique among all packages)
     # in which case it is a `Vector{String}` or expanded where it is a `name => uuid` mapping.

doc/src/manual/code-loading.md

@@ -397,6 +397,47 @@ are stored in the manifest file in the section for that package. The dependency
 a package are the same as for its "parent" except that the listed triggers are also considered as
 dependencies.
 
+### [Portable scripts](@id portable-scripts)
+
+Julia also understands *portable scripts*: scripts that embed their own `Project.toml` (and optionally `Manifest.toml`) so they can be executed as self-contained environments. To do this, place TOML data inside comment fences named `#!project` and `#!manifest`:
+
+```julia
+#!project begin
+# name = "HelloApp"
+# uuid = "9c5fa7d8-7220-48e8-b2f7-0042191c5f6d"
+# version = "0.1.0"
+# [deps]
+# Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+#!project end
+
+#!manifest begin
+# [[deps]]
+# name = "Markdown"
+# uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+# version = "1.0.0"
+#!manifest end
+
+using Markdown
+println(md"# Hello, single-file world!")
+```
+
+Lines inside the fenced blocks may either start with `#` (as in the example), be plain TOML, or be wrapped in multi-line comment delimiters `#= ... =#`:
+
+```julia
+#!project begin
+#=
+name = "HelloApp"
+uuid = "9c5fa7d8-7220-48e8-b2f7-0042191c5f6d"
+version = "0.1.0"
+[deps]
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+=#
+#!project end
+```
+
+
+Running `julia hello.jl` automatically activates the embedded project. The script path becomes the active project entry in `LOAD_PATH`, so package loading works exactly as if `Project.toml` and `Manifest.toml` lived next to the script. The `--project=@script` flag also expands to the script itself when no on-disk project exists but inline metadata is present.
+
 ### [Workspaces](@id workspaces)
 
 A project file can define a workspace by giving a set of projects that is part of that workspace: