Full bundling for small slider spaces

Author: ctrekker

src/Actions.jl

@@ -71,8 +71,7 @@ function process(
         !is_glob_match(path, settings.SliderServer.exclude)
     skip_cache =
         keep_running ||
-        is_glob_match(path, settings.Export.ignore_cache) ||
-        path ∈ settings.Export.ignore_cache
+        is_glob_match(path, settings.Export.ignore_cache)
 
     cached_state = skip_cache ? nothing : try_fromcache(settings.Export.cache_dir, new_hash)
 
@@ -136,6 +135,7 @@ function process(
             output_dir,
         )
         # TODO shutdown
+        # TODO cache
     end
 
     @info "### ✓ $(progress) Ready" s.path new_hash

src/Configuration.jl

@@ -32,6 +32,11 @@ end
     "List of notebook files to skip precomputation. Provide paths relative to `start_dir`."
     exclude::Vector{String} = String[]
     max_filesize_per_group::Integer = 1_000_000
+
+    "Combines multiple bind state precomputations into a single file"
+    bundling_enabled::Bool = false
+    "Maximum filesize of a full bundle including all group bond updates. Since these files are sent in full over the internet, their size must be small"
+    max_full_bundle_size::Integer = 500_000
 end
 
 @extract_docs @option struct ExportSettings

src/precomputed/index.jl

@@ -130,10 +130,14 @@ function generate_precomputed_staterequests(
     connections = run.bond_connections
     current_hash = sesh.current_hash
 
+    should_bundle = settings.Precompute.bundling_enabled
+
     @assert run isa RunningNotebook
 
     mkpath(joinpath(output_dir, "bondconnections"))
+    mkpath(joinpath(output_dir, "bundles"))
     mkpath(joinpath(output_dir, "staterequest", URIs.escapeuri(current_hash)))
+    mkpath(joinpath(output_dir, "staterequest-bundled", URIs.escapeuri(current_hash)))
 
     bondconnections_path =
         joinpath(output_dir, "bondconnections", URIs.escapeuri(current_hash))
@@ -156,8 +160,12 @@ function generate_precomputed_staterequests(
         @warn "Notebook cannot be (fully) precomputed" report
     end
 
+    bundle_index = String[]
     foreach(groups) do group::VariableGroupPossibilities
         if group.judgement.should_precompute_all
+            should_bundle_group = should_bundle && group.judgement.can_fully_bundle
+
+            bundle = Dict{String,Any}()
             for (combination, bonds_dict) in combination_iterator(group)
                 filename = Pluto.pack(bonds_dict) |> base64urlencode
                 if length(filename) > 255
@@ -176,10 +184,32 @@ function generate_precomputed_staterequests(
 
                     write(write_path, Pluto.pack(result))
 
+                    if should_bundle_group
+                        bundle[filename] = result
+                    end
+
                     @debug "Written state request to " write_path values =
                         (; (zip(group.names, combination))...)
                 end
             end
+
+            if should_bundle_group
+                bundle_signature = Pluto.pack(sort(group.names)) |> base64urlencode
+                push!(bundle_index, bundle_signature)
+                bundle_path = joinpath(
+                    output_dir,
+                    "staterequest-bundled",
+                    URIs.escapeuri(current_hash),
+                    bundle_signature,
+                )
+                write(bundle_path, Pluto.pack(bundle))
+
+                @debug "Written bundled states to " bundle_path bundle_signature
+            end
         end
     end
+    write(
+        joinpath(output_dir, "bundles", URIs.escapeuri(current_hash)),
+        Pluto.pack(bundle_index),
+    )
 end

src/precomputed/types.jl

@@ -11,6 +11,7 @@ const Reason = Symbol
 
 Base.@kwdef struct Judgement
     should_precompute_all::Bool = false
+    can_fully_bundle::Bool = false
     not_available::Bool = false
     close_to_filesize_limit::Bool = false
     exceeds_filesize_limit::Bool = false
@@ -42,17 +43,21 @@ function VariableGroupPossibilities(;
         @assert settings isa PlutoDeploySettings
 
         limit = settings.Precompute.max_filesize_per_group
+        bundle_limit = settings.Precompute.max_full_bundle_size
         current = mean(file_size_sample_distribution)
 
         exceeds_filesize_limit = current > limit
+        exceeds_bundle_size_limit = current > bundle_limit
         close_to_filesize_limit = current > limit * 0.7
     else
-        exceeds_filesize_limit = close_to_filesize_limit = false
+
+        exceeds_filesize_limit = exceeds_bundle_size_limit = close_to_filesize_limit = false
+
     end
 
     j = Judgement(;
         should_precompute_all=!is_not_available && !exceeds_filesize_limit,
-        exceeds_filesize_limit,
+        can_fully_bundle=!exceeds_bundle_size_limit,
         close_to_filesize_limit,
         not_available=is_not_available,
     )
@@ -203,9 +208,15 @@ sum_distributions(ds; init=Normal(0, 0)) =
     any(isnothing, ds) ? nothing : reduce(convolve, ds; init=init)
 
 
-pretty(j::Judgement) =
-    if j.should_precompute_all
+pretty(j::Judgement) = begin
+    s = ""
+    s *= if j.should_precompute_all
         j.close_to_filesize_limit ? "⚠️" : "✓"
     else
         "❌"
     end
+    if j.can_fully_bundle
+        s *= " 📦"
+    end
+    s
+end