Literate examples

docs/Project.toml

@@ -1,2 +1,6 @@
 [deps]
+Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54"
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

docs/make.jl

@@ -1,5 +1,42 @@
 using Dagger
 using Documenter
+using Literate
+using Test
+
+const EXAMPLE_DIR = joinpath(@__DIR__, "src", "examples")
+
+function link_example(content)
+    edit_url = match(r"EditURL = \"(.+?)\"", content)[1]
+    footer = match(r"^(---\n\n\*This page was generated using)"m, content)[1]
+    content = replace(
+        content, footer => "[View this file on Github]($(edit_url)).\n\n" * footer
+    )
+    return content
+end
+
+function literate_examples()
+    for file in readdir(EXAMPLE_DIR)
+        if !endswith(file, ".jl")
+            continue
+        end
+        filename = joinpath(EXAMPLE_DIR, file)
+        # # `include` the file to test it before `#src` lines are removed. It is
+        # # in a testset to isolate local variables between files.
+        # @testset "$(file)" begin
+        #     include(filename)
+        # end
+        Literate.markdown(
+            filename,
+            EXAMPLE_DIR;
+            documenter = true,
+            postprocess = link_example,
+            execute=false,
+        )
+    end
+    return nothing
+end
+
+literate_examples()
 
 makedocs(;
     modules = [Dagger],
@@ -17,6 +54,13 @@ makedocs(;
         "Scheduler Internals" => "scheduler-internals.md",
         "Logging and Graphing" => "logging.md",
         "Dynamic Scheduler Control" => "dynamic.md",
+        "Examples" => map(
+            file -> joinpath("examples", file),
+            filter(
+                file -> endswith(file, ".md"),
+                sort(readdir(EXAMPLE_DIR)),
+            )
+        ),
     ]
 )
 

docs/src/examples/nnmf.jl

@@ -0,0 +1,46 @@
+# #
+
+# ## Setting up the cluster
+
+using Distributed
+addprocs(4, exeflags=["--project=$(Base.active_project())"])
+
+# ## Loading Dagger
+
+@everywhere using Dagger
+import Dagger: Computation, reduceblock
+
+# ## Hierachical parallelism
+# 
+# Each Julia worker is modeled as a `OSProc` which can have multiple `ThreadProc`
+# assigned to it. Let's walk all workers and discover how many `ThreadProc`s there are.
+
+p = sum(length(Dagger.get_processors(OSProc(id))) for id in workers())
+
+# ## NNMF
+
+function nnmf(X, W, H)
+    # H update
+    H = (H .* (W' * (X ./ (W * H))) ./ (sum(W; dims=1))')
+    # W update
+    W = (W .* ((X ./ (W * H)) * (H')) ./ (sum(H; dims=2)'))
+    # error estimate
+    (X - W * H, W, H)
+end
+
+# ## Sample data
+
+ncol = 2000
+nrow = 10000
+nfeatures = 12
+
+# Eagerly allocating 
+
+X = compute(rand(Blocks(nrow, ncol÷p), nrow, ncol))
+W = compute(rand(Blocks(nrow, ncol÷p), nrow, nfeatures))
+H = compute(rand(Blocks(nrow, ncol÷p), nfeatures, ncol))
+
+# ## Run a NMF step
+
+err, W, H = nnmf(X, W, H)
+compute(err)