diff --git a/docs/Project.toml b/docs/Project.toml index dfa65cd10..62b43fa39 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,2 +1,6 @@ [deps] +Dagger = "d58978e5-989f-55fb-8d15-ea34adc7bf54" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/docs/make.jl b/docs/make.jl index 8b9f9c348..663ab002f 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,42 @@ using Dagger using Documenter +using Literate +using Test + +const EXAMPLE_DIR = joinpath(@__DIR__, "src", "examples") + +function link_example(content) + edit_url = match(r"EditURL = \"(.+?)\"", content)[1] + footer = match(r"^(---\n\n\*This page was generated using)"m, content)[1] + content = replace( + content, footer => "[View this file on Github]($(edit_url)).\n\n" * footer + ) + return content +end + +function literate_examples() + for file in readdir(EXAMPLE_DIR) + if !endswith(file, ".jl") + continue + end + filename = joinpath(EXAMPLE_DIR, file) + # # `include` the file to test it before `#src` lines are removed. It is + # # in a testset to isolate local variables between files. + # @testset "$(file)" begin + # include(filename) + # end + Literate.markdown( + filename, + EXAMPLE_DIR; + documenter = true, + postprocess = link_example, + execute=false, + ) + end + return nothing +end + +literate_examples() makedocs(; modules = [Dagger], @@ -17,6 +54,13 @@ makedocs(; "Scheduler Internals" => "scheduler-internals.md", "Logging and Graphing" => "logging.md", "Dynamic Scheduler Control" => "dynamic.md", + "Examples" => map( + file -> joinpath("examples", file), + filter( + file -> endswith(file, ".md"), + sort(readdir(EXAMPLE_DIR)), + ) + ), ] ) diff --git a/docs/src/examples/nnmf.jl b/docs/src/examples/nnmf.jl new file mode 100644 index 000000000..a7b6b7284 --- /dev/null +++ b/docs/src/examples/nnmf.jl @@ -0,0 +1,46 @@ +# # + +# ## Setting up the cluster + +using Distributed +addprocs(4, exeflags=["--project=$(Base.active_project())"]) + +# ## Loading Dagger + +@everywhere using Dagger +import Dagger: Computation, reduceblock + +# ## Hierachical parallelism +# +# Each Julia worker is modeled as a `OSProc` which can have multiple `ThreadProc` +# assigned to it. Let's walk all workers and discover how many `ThreadProc`s there are. + +p = sum(length(Dagger.get_processors(OSProc(id))) for id in workers()) + +# ## NNMF + +function nnmf(X, W, H) + # H update + H = (H .* (W' * (X ./ (W * H))) ./ (sum(W; dims=1))') + # W update + W = (W .* ((X ./ (W * H)) * (H')) ./ (sum(H; dims=2)')) + # error estimate + (X - W * H, W, H) +end + +# ## Sample data + +ncol = 2000 +nrow = 10000 +nfeatures = 12 + +# Eagerly allocating + +X = compute(rand(Blocks(nrow, ncol÷p), nrow, ncol)) +W = compute(rand(Blocks(nrow, ncol÷p), nrow, nfeatures)) +H = compute(rand(Blocks(nrow, ncol÷p), nfeatures, ncol)) + +# ## Run a NMF step + +err, W, H = nnmf(X, W, H) +compute(err)