Docsystem Improvements

Moves a small docsystem component into coreimg which helps simplify the docs bootstrapping.
`@doc`, `@__doc__`, and `@doc_str` are now defined in `Core` rather than in both `Core` and
`Base`. From some adhoc timings compiling base it seems to have reduced it slightly somehow.

Bare docstrings are parsed as

    Core.@doc "..." f

rather than

    @doc "..." f

This avoids the unlikely case of a user redefining `@doc` in their own module. Seems like
something that could be classed as surprising behaviour and should be disallowed.

Adjusts the keyword docs in `basedocs.jl` to use `@doc` rather than side-stepping it entirely.

Improves the file and line info captured by `@doc` so that docs can be located correctly in
both `Base` and packages.

Interpolation in docstrings is now lazy and only occurs during display; though cached for
repeated calls. Instead of storing the raw string it is stored as a `SimpleVector`
containing the string parts and interpolated values. This allows for context dependent
autogenerated content in docstrings, such as method signatures, etc. by defining methods of
`Docs.formatdoc` as follows:

    immutable __SIGNATURE__ end

    function Docs.formatdoc(buffer, d, part::Type{__SIGNATURE__})
        # inspect the source code stored in docstring `d` and generate a user-friendly
        # signature to replace `part` in the final docstring.
        # ...
    end

and used as

    """
    $__SIGNATURE__

    ...
    """
    function foo(x, y, z = 1)
        # ...
    end

Other possibilities aside from `__SIGNATURE__` might be `__FIELDS__` for embedding the field
docs of a type into the main docstring. Neither of these are included in this commit.

Author: MichaelHatherly

base/boot.jl

@@ -322,6 +322,19 @@ Array{T}(::Type{T}, m::Int)               = Array{T,1}(m)
 Array{T}(::Type{T}, m::Int,n::Int)        = Array{T,2}(m,n)
 Array{T}(::Type{T}, m::Int,n::Int,o::Int) = Array{T,3}(m,n,o)
 
+# docsystem basics
+macro doc(x...)
+    atdoc(x...)
+end
+macro __doc__(x)
+    Expr(:escape, Expr(:block, Expr(:meta, :doc), x))
+end
+macro doc_str(s)
+    Expr(:escape, s)
+end
+atdoc     = (str, expr) -> Expr(:escape, expr)
+atdoc!(λ) = global atdoc = λ
+
 module TopModule
     # this defines the types that lowering expects to be defined in a (top) module
     # that are usually inherited from Core, but could be defined custom for a module

base/coreimg.jl

@@ -16,9 +16,6 @@ print(x::ANY) = show(x)
 println(x::ANY) = ccall(:jl_, Void, (Any,), x) # includes a newline
 print(a::ANY...) = for x=a; print(x); end
 
-# Doc macro shim.
-macro doc(str, def) Expr(:escape, def) end
-
 ## Load essential files and libraries
 include("essentials.jl")
 include("generator.jl")
@@ -67,6 +64,9 @@ include("intset.jl")
 include("dict.jl")
 include("iterator.jl")
 
+# core docsystem
+include("docs/core.jl")
+
 # compiler
 include("inference.jl")
 

base/docs/Docs.jl

@@ -52,13 +52,14 @@ will cause that specific method to be documented, as opposed to the whole functi
 docs are concatenated together in the order they were defined to provide docs for the
 function.
 """
-:(Base.DocBootstrap.@doc)
+:(Core.@doc)
 
 include("bindings.jl")
 
 import Base.Markdown: @doc_str, MD
 import Base.Meta: quot, isexpr
 import Base: Callable
+import Core.Inference.CoreDocs: lazy_iterpolate
 
 export doc
 
@@ -99,7 +100,7 @@ function argtype(expr::Expr)
     isexpr(expr, :(...)) && return :(Vararg{$(argtype(expr.args[1]))})
     argtype(expr.args[1])
 end
-argtype(::Symbol) = :Any
+argtype(other) = :Any
 
 function typevars(expr::Expr)
     isexpr(expr, :curly) && return [tvar(x) for x in expr.args[2:end]]
@@ -126,13 +127,9 @@ The `.data` fields stores several values related to the docstring, such as: path
 linenumber, source code, and fielddocs.
 """
 type DocStr
-    text   :: UTF8String
+    text   :: Core.SimpleVector
     object :: Nullable
     data   :: Dict{Symbol, Any}
-
-    DocStr(text::AbstractString, data) = new(text, Nullable(), data)
-    DocStr(object,               data) = new("", Nullable(object), data)
-    DocStr(docstr::DocStr,       data) = docstr
 end
 
 function docstr(binding::Binding, typesig::ANY = Union{})
@@ -147,7 +144,13 @@ function docstr(binding::Binding, typesig::ANY = Union{})
     end
     error("could not find matching docstring for '$binding :: $typesig'.")
 end
-docstr(object, data = Dict()) = DocStr(object, data)
+docstr(object, data = Dict()) = _docstr(object, data)
+
+_docstr(vec::Core.SimpleVector, data) = DocStr(vec,            Nullable(),       data)
+_docstr(str::AbstractString,    data) = DocStr(Core.svec(str), Nullable(),       data)
+_docstr(object,                 data) = DocStr(Core.svec(),    Nullable(object), data)
+
+_docstr(doc::DocStr, data) = (doc.data = merge(data, doc.data); doc)
 
 macro ref(x)
     binding = bindingexpr(namify(x))
@@ -157,9 +160,18 @@ end
 
 docexpr(args...) = Expr(:call, docstr, args...)
 
+function formatdoc(d::DocStr)
+    buffer = IOBuffer()
+    for part in d.text
+        formatdoc(buffer, d, part)
+    end
+    Markdown.parse(seekstart(buffer))
+end
+@noinline formatdoc(buffer, d, part) = print(buffer, part)
+
 function parsedoc(d::DocStr)
     if isnull(d.object)
-        md = Markdown.parse(d.text)
+        md = formatdoc(d)
         md.meta[:module] = d.data[:module]
         md.meta[:path]   = d.data[:path]
         d.object = Nullable(md)
@@ -282,7 +294,10 @@ function fielddoc(binding::Binding, field::Symbol)
             multidoc = dict[binding]
             if haskey(multidoc.docs, Union{})
                 fields = multidoc.docs[Union{}].data[:fields]
-                haskey(fields, field) && return Markdown.parse(fields[field])
+                if haskey(fields, field)
+                    doc = fields[field]
+                    return isa(doc, Markdown.MD) ? doc : Markdown.parse(doc)
+                end
             end
         end
     end
@@ -377,7 +392,7 @@ isdoc(s::AbstractString) = true
 
 isdoc(x) = isexpr(x, :string) ||
     (isexpr(x, :macrocall) && x.args[1] == symbol("@doc_str")) ||
-    (isexpr(x, :call) && x.args[1] == Expr(:., Base.Markdown, QuoteNode(:doc_str)))
+    (isexpr(x, :call) && x.args[1] == Base.Markdown.doc_str)
 
 function unblock(ex)
     isexpr(ex, :block) || return ex
@@ -430,7 +445,8 @@ function metadata(expr)
     # Source code for the object being documented.
     push!(args, :($(Pair)(:source, $(quot(expr)))))
     # Filename and linenumber of the docstring.
-    push!(args, :($(Pair)(:path, $(Base).@__FILE__)), :($(Pair)(:linenumber, @__LINE__)))
+    push!(args, :($(Pair)(:path, $(Base).@__FILE__)))
+    push!(args, :($(Pair)(:linenumber, $(unsafe_load(cglobal(:jl_lineno, Int))))))
     # Module in which the docstring is defined.
     push!(args, :($(Pair)(:module, $(current_module)())))
     # Field docs for concrete types.
@@ -451,9 +467,14 @@ function metadata(expr)
     :($(Dict)($(args...)))
 end
 
+function keyworddoc(str, def)
+    docstr = esc(docexpr(lazy_iterpolate(str), metadata(def)))
+    :($(keywords)[$(esc(quot(def.name)))] = $docstr)
+end
+
 function objectdoc(str, def, expr, sig = :(Union{}))
     binding = esc(bindingexpr(namify(expr)))
-    docstr  = esc(docexpr(str, metadata(expr)))
+    docstr  = esc(docexpr(lazy_iterpolate(str), metadata(expr)))
     quote
         $(esc(def))
         $(doc!)($binding, $docstr, $(esc(sig)))
@@ -480,7 +501,7 @@ end
 # Shares a single doc, `meta`, between several expressions from the tuple expression `ex`.
 function multidoc(meta, ex, define)
     out = Expr(:toplevel)
-    str = docexpr(meta, metadata(ex))
+    str = docexpr(lazy_iterpolate(meta), metadata(ex))
     ref = Ref{DocStr}()
     for (n, arg) in enumerate(ex.args)
         # The first `arg` to be documented needs to also create the docstring for the group.
@@ -508,7 +529,7 @@ more than one expression is marked then the same docstring is applied to each ex
 
 `@__doc__` has no effect when a macro that uses it is not documented.
 """
-:(Base.@__doc__)
+:(Core.@__doc__)
 
 function __doc__!(meta, def, define)
     # Two cases must be handled here to avoid redefining all definitions contained in `def`:
@@ -575,6 +596,13 @@ function docm(meta, ex, define = true)
     # Initalise the module's docstring storage.
     initmeta()
 
+    # Keywords using the `@kw_str` macro in `base/docs/basedocs.jl`.
+    #
+    #   "..."
+    #   kw"if", kw"else"
+    #
+    isa(x, Base.BaseDocs.Keyword) ? keyworddoc(meta, x) :
+
     # Method / macro definitions and "call" syntax.
     #
     #   function f(...) ... end
@@ -631,7 +659,7 @@ function docerror(ex)
     if isexpr(ex, :macrocall)
         txt *= "\n\n'$(ex.args[1])' not documentable. See 'Base.@__doc__' docs for details."
     end
-    :(error($txt, "\n"))
+    :($(error)($txt, "\n"))
 end
 
 function docm(ex)
@@ -652,20 +680,21 @@ function docm(ex)
     end
 end
 
-# Swap out the bootstrap macro with the real one
-
-Base.DocBootstrap.setexpand!(docm)
-
-# Names are resolved relative to the Base module, so inject the ones we need there.
-
-eval(Base, :(import .Docs: @doc_str))
-
-Base.DocBootstrap.loaddocs()
-
 # MD support
-
 catdoc(md::MD...) = MD(md...)
 
 include("utils.jl")
 
+# Swap out the bootstrap macro with the real one.
+Core.atdoc!(docm)
+
+function loaddocs(docs)
+    for (mod, ex, str, file, line) in docs
+        data = Dict(:path => string(file), :linenumber => line)
+        doc = docstr(str, data)
+        eval(mod, :(@doc($doc, $ex, false)))
+    end
+    empty!(docs)
+end
+
 end