Adjusts site to accommodate multiple user-doc branches

* Build directories for each branch
* Path changes for multiple ref branches, testing and cleanup
* Add minitest-reporters, sort gems
* Adjust menus for branches
* Better default path handling in Miq module

Author: hayesr

.gitignore

@@ -6,6 +6,9 @@ _site
 dest/
 site/docs/guides
 site/docs/reference/latest/
+site/docs/reference/fine/
+site/docs/reference/euwe/
+site/docs/reference/darga/
 site/.asset-cache
 
 tmp/

Dockerfile

@@ -37,7 +37,7 @@ ENV MIQ_REF_REPO=https://github.com/manageiq/manageiq_docs
 ENV MIQ_REF_BRANCH=master
 # ENV MIQ_REF_TMP=/tmp/manageiq_docs
 # ENV MIQ_REF_SRC=_preview/manageiq/${MIQ_REF_BRANCH}
-# ENV MIQ_REF_DST=${MIQ_DOCS_DIR}/reference/latest
+# ENV MIQ_REF_DIR=${MIQ_DOCS_DIR}/reference/latest
 
 # Nginx configs
 RUN mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.dist

Gemfile

@@ -1,25 +1,26 @@
 source 'https://rubygems.org'
 
-gem "thor"
+gem "activesupport", :require => false
+gem "bootstrap-sass"
+gem "font-awesome-sass"
 gem "jekyll"
-gem "jekyll-paginate"
 gem "jekyll-assets"
 gem "jekyll-coffeescript"
+gem "jekyll-paginate"
 gem "jekyll-sitemap"
-gem "therubyracer"
-gem "bootstrap-sass"
-gem "font-awesome-sass"
-gem "activesupport", :require => false
 gem "jemoji"
+gem "therubyracer"
+gem "thor"
 
 group :test, :development do
   gem "dotenv"
-  gem "rake"
-  gem "pry"
-  gem "pry-byebug"
   gem "guard"
   gem "guard-livereload"
   gem "guard-minitest"
+  gem "minitest-reporters"
+  gem "pry"
+  gem "pry-byebug"
+  gem "rake"
 end
 
 # source 'https://rails-assets.org' do

Gemfile.lock

@@ -9,11 +9,13 @@ GEM
       tzinfo (~> 1.1)
     addressable (2.5.0)
       public_suffix (~> 2.0, >= 2.0.2)
+    ansi (1.5.0)
     autoprefixer-rails (6.7.2)
       execjs
     bootstrap-sass (3.3.7)
       autoprefixer-rails (>= 5.2.1)
       sass (>= 3.3.4)
+    builder (3.2.3)
     byebug (9.0.6)
     coderay (1.1.1)
     coffee-script (2.4.1)
@@ -106,6 +108,11 @@ GEM
     method_source (0.8.2)
     mini_portile2 (2.1.0)
     minitest (5.10.1)
+    minitest-reporters (1.1.14)
+      ansi
+      builder
+      minitest (>= 5.0)
+      ruby-progressbar
     multi_json (1.12.1)
     nenv (0.3.0)
     nokogiri (1.7.0.1)
@@ -130,6 +137,7 @@ GEM
       ffi (>= 0.5.0)
     ref (2.0.0)
     rouge (1.11.1)
+    ruby-progressbar (1.8.1)
     safe_yaml (1.0.4)
     sass (3.4.23)
     shellany (0.0.1)
@@ -162,11 +170,12 @@ DEPENDENCIES
   jekyll-paginate
   jekyll-sitemap
   jemoji
+  minitest-reporters
   pry
   pry-byebug
   rake
   therubyracer
   thor
 
 BUNDLED WITH
-   1.13.6
+   1.14.6

Guardfile

@@ -21,7 +21,8 @@ end
 
 guard :minitest do
   watch(%r{^test/(.*)\/?(.*)_test[s]?\.rb$})
-  watch(%r{^lib/(.*/)?([^/]+)\.rb$}) { |m| "test/#{m[1]}test_#{m[2]}.rb" }
+  # watch('lib/miq.rb') { |_| "test/lib/miq_test.rb" }
+  watch(%r{^lib\/(.*\/)?([^\/]+)\.rb$}) { |m| "test/lib/#{m[1]}#{m[2]}_test.rb" }
   watch(%r{^site/_plugins/(.*/)?([^/]+)\.rb$}) { "test/plugin_tests.rb" }
   watch(%r{^test/test_helper\.rb$})  { "test" }
 end

dot-env-example

@@ -8,14 +8,15 @@ MIQ_ENV=development
 
 # MIQ_BASE_DIR
 # MIQ_DOCS_DIR
+# MIQ_TMP_DIR
 # MIQ_GUIDES_DIR
 
 # MIQ_SITE_DEST
 # MIQ_SITE_SOURCE
-# MIQ_SITE_BRANCH
 
+# MIQ_REF_BRANCHES=master,fine,euwe
 # MIQ_REF_REPO
 # MIQ_REF_BRANCH
 # MIQ_REF_TMP
 # MIQ_REF_SRC
-# MIQ_REF_DST
+# MIQ_REF_DIR

lib/miq.rb

@@ -17,6 +17,7 @@
 
 require_relative "miq/ref_docs"
 require_relative "miq/ref_menu"
+require_relative "miq/ref_page"
 
 require_relative "miq/md_link_converter"
 require_relative "miq/dir_index"
@@ -28,35 +29,37 @@
 
 module Miq
   def self.working_dir
-    if ENV["MIQ_BASE_DIR"]
-      Pathname.new(ENV["MIQ_BASE_DIR"])
-    else
-      Pathname.new(Dir.pwd)
-    end
+    Pathname.new(ENV.fetch("MIQ_BASE_DIR", Dir.pwd))
   end
 
+  # Main directories
+
   def self.dest_dir
-    working_dir.join( (ENV["MIQ_SITE_DEST"] || "dest") )
+    Pathname.new ENV.fetch("MIQ_SITE_DEST", working_dir.join("dest"))
   end
 
   def self.site_dir
-    working_dir.join( (ENV["MIQ_SITE_SOURCE"] || "site") )
+    Pathname.new ENV.fetch("MIQ_SITE_SOURCE", working_dir.join("site"))
   end
 
   def self.docs_dir
-    if ENV["MIQ_DOCS_DIR"]
-      working_dir.join(ENV["MIQ_DOCS_DIR"])
-    else
-      working_dir.join("site", "docs")
-    end
+    Pathname.new ENV.fetch("MIQ_DOCS_DIR", site_dir.join("docs"))
+  end
+
+  def self.tmp_dir
+    working_dir.join(ENV["MIQ_TMP_DIR"] || "tmp")
   end
 
+  # Docs subdirectories
+
+  # Where Jekyll will look for guides
   def self.guides_dir
-    site_dir.join(guides_relative)
+    docs_dir.join(ENV["MIQ_GUIDES_DIR"] || "guides")
   end
 
-  def self.guides_relative
-    (ENV["MIQ_GUIDES_DIR"] || "docs/guides")
+  # The reference doc destination that Jekyll will use
+  def self.reference_dir
+    docs_dir.join(ENV["MIQ_REF_DIR"] || "reference")
   end
 
   # Might want to make these override-able in the future:
@@ -76,4 +79,8 @@ def self.blog_year_template
   def self.blog_month_template
     "blog_month.html"
   end
+
+  def self.doc_branches
+    ENV.fetch('MIQ_DOC_BRANCHES', 'euwe,fine,latest').split(',')
+  end
 end

lib/miq/guide_page.rb

@@ -6,6 +6,8 @@ def self.new_from(page)
       new(page.site, Miq.site_dir.to_s, page.dir, page.name)
     end
 
+    # Don't render guides with liquid because some
+    # pages have conflicting syntax
     def render_with_liquid?
       false
     end

lib/miq/ref_docs.rb

@@ -14,25 +14,23 @@ def self.update
       new.update
     end
 
-    attr_reader :repo, :branch, :tmp_dir, :src_dir, :dst_dir
+    attr_reader :repo, :branches, :tmp_dir, :src_dir, :dst_dir
 
     def initialize
       # Where do the docs live?
       @repo     = ENV["MIQ_REF_REPO"] || "https://github.com/ManageIQ/manageiq_docs.git"
 
-      # What branch to checkout?
-      @branch   = ENV["MIQ_REF_BRANCH"] || "master"
+      # What branches to copy?
+      @branches = ENV["MIQ_REF_BRANCHES"]&.split(',') || %w(master fine euwe)
 
       # Where should we cache and build?
       @tmp_dir  = ENV["MIQ_REF_TMP"]  || "/tmp/manageiq_docs"
 
-      # TODO: src & dst will need to be changed to support multiple versions
-
       # Where are the built files that we want?
-      @src_dir  = ENV["MIQ_REF_SRC"]  || "_preview/manageiq/#{@branch}"
+      @src_dir  = ENV["MIQ_REF_SRC"]  || "_package/community"
 
       # Where should the files end up?
-      @dst_dir  = ENV["MIQ_REF_DST"]  || Miq.docs_dir.join("reference", "latest")
+      @dst_dir  = ENV["MIQ_REF_DIR"]  || Miq.docs_dir.join("reference")
     end
 
     def reset
@@ -61,39 +59,35 @@ def update
 
     def clone_or_update_repo
       if File.directory?(tmp_dir)
-        logger.info "Updating local ref docs repo"
-        shell "cd #{tmp_dir} && git pull origin"
+        update_repo
       else
-        logger.info "Cloning #{repo} to #{tmp_dir}"
-        shell "git clone #{repo} --depth 1 #{tmp_dir}"
+        clone_repo
       end
     end
 
     def setup_ascii_binder
       logger.info "Installing Ascii Binder"
       shell [
         "cd #{tmp_dir}",
-        "git checkout #{branch}",
+        "git checkout master",
         "#{bundler} install"
       ].join(" && ")
     end
 
     def build_ref_docs
       logger.info "Building ref docs"
-      shell [
-        "cd #{tmp_dir}",
-        "#{bundler} exec asciibinder build"
-      ].join(" && ")
+      shell "cd #{tmp_dir} && #{bundler} exec asciibinder package"
     end
 
     def sync_files
       if File.directory?("#{tmp_dir}/#{src_dir}") || debug?
         prep dst_dir
-        logger.info "Syncing files to #{dst_dir}"
-        cmd = "rsync -av "
-        cmd << exclude_files.map{|x| "--exclude '#{x}'"}.join(' ')
-        cmd << " #{tmp_dir}/#{src_dir}/* #{dst_dir}/"
-        shell cmd
+
+        branches.each do |branch|
+          rsync_copy(branch)
+        end
+
+        shell "mv #{dst_dir}/master #{dst_dir}/latest"
       else
         logger.error "Reference docs source directory not present."
       end
@@ -105,5 +99,42 @@ def sync_files
     def exclude_files
       ["_stylesheets", "/index.html", "sitemap.xml"]
     end
+
+    def rsync_copy(branch)
+      source = "#{tmp_dir}/#{src_dir}/#{branch}/*"
+      dest   = "#{dst_dir}/#{branch}"
+
+      logger.info "Syncing files to #{dest}"
+
+      cmd = ["rsync -av ", rsync_excludes, source, dest].join(' ')
+
+      shell cmd
+    end
+
+    def branch_paths
+      branches.map { |b| "#{tmp_dir}/#{src_dir}/#{b}/*" }.join(' ')
+    end
+
+    def rsync_excludes
+      exclude_files.map { |x| "--exclude '#{x}'" }.join(' ')
+    end
+
+    def update_repo
+      logger.info "Updating local ref docs repo"
+      shell "cd #{tmp_dir} && git fetch origin"
+    end
+
+    def clone_repo
+      logger.info "Cloning #{repo} to #{tmp_dir}"
+      shell "git clone #{repo} #{tmp_dir} && cd #{tmp_dir} && git fetch origin #{branch_fetch_segments}"
+    end
+
+    def branch_fetch_segments
+      branches_not_master.map { |b| "+#{b}:#{b}" }.join(' ')
+    end
+
+    def branches_not_master
+      branches.reject { |b| b == "master" }
+    end
   end
 end

lib/miq/ref_menu.rb

@@ -4,14 +4,20 @@
 module Miq
   class RefMenu < Executor
     def self.build
-      new.build
+      # build for each dir in MIQ_REF_DIR
+      Dir["#{Miq.reference_dir}/*"].each do |path|
+        new(source_dir: path).build if File.directory?(path)
+      end
     end
 
-    attr_reader :source_dir, :output_dir
+    attr_reader :source_dir, :output_dir, :output_file
 
-    def initialize
-      @source_dir = ENV["MIQ_REF_DST"] || Miq.docs_dir.join("reference", "latest")
-      @output_dir = Miq.working_dir.join("tmp", "menus")
+    def initialize(source_dir: nil, output_dir: nil)
+      # @source_dir = Miq.default_path(source_dir, Miq.reference_dir, "latest")
+      @source_dir = Pathname.new(source_dir || Miq.reference_dir.join("latest"))
+      # @output_dir = Miq.default_path(output_dir, Miq.tmp_dir, "menus")
+      @output_dir = Pathname.new(output_dir || Miq.tmp_dir.join("menus"))
+      @output_file = @source_dir.basename
     end
 
     def menu_data
@@ -25,7 +31,7 @@ def to_yaml
     def build
       prep(output_dir)
 
-      File.open("#{output_dir}/ref_menu.yml", "w") do |f|
+      File.open("#{output_dir}/ref_menu_#{output_file}.yml", "w") do |f|
         f.puts to_yaml
       end
     end
@@ -137,7 +143,7 @@ def replace_path(path)
     end
 
     def exclusions
-      ["." ".."]
+      [".", ".."]
     end
 
     def inclusions