Maintenance: Lock stack job runs

.rubocop.yml

@@ -1,6 +1,6 @@
 ---
 # Overwrite or add rules to create your own house style
-require:
+plugins:
   - rubocop-factory_bot
   - rubocop-faker
   - rubocop-obsession

app/controllers/stacks_controller/streams_logs.rb

@@ -28,8 +28,13 @@ def stream_logs(sse)
       def open_log_file!(sse)
         log_file = @stack.assets.log_file.to_s
 
-        unless File.exist?(log_file)
-          send_close_message(sse, 'No log file found')
+        timeout = 2.minutes
+        start_time = Time.zone.now
+        until File.exist?(log_file)
+          sleep 0.1
+          next if Time.zone.now - start_time < timeout
+
+          send_close_message(sse, 'Timeout waiting for log file')
           return nil
         end
 

app/jobs/stack_destroy_job.rb

@@ -1,9 +1,4 @@
 class StackDestroyJob < StackJob
   queue_as :deploy
   script_template :destroy
-
-  def execute
-    script = render_script(@stack, @assets)
-    run_script(script)
-  end
 end

app/jobs/stack_job.rb

@@ -5,18 +5,35 @@ class StackJob < ApplicationJob
   STACKS_ROOT = ENV.fetch('SUPERVISOR_STACKS_ROOT', Rails.root.join('storage/stack'))
 
   def perform(stack)
-    # On destroy, the relevant stack attributes are passed as a hash
     @stack = stack.is_a?(Hash) ? Hashie::Mash.new(stack) : stack
 
     include_files = @stack.compose_includes.join(' ')
-    @assets = @stack.assets
-                    .merge(include_files:)
+    @assets = @stack.assets.merge(include_files:)
+
+    return if !instance_of?(StackDestroyJob) && !lock_for_stack!
 
     execute
   end
 
   private
 
+  def lock_for_stack!
+    lock_file = File.join(@assets.base_dir.to_s, '.lock')
+    FileUtils.mkdir_p(@assets.base_dir)
+
+    File.open(lock_file, File::RDWR | File::CREAT) do |f|
+      unless f.flock(File::LOCK_EX | File::LOCK_NB)
+        Rails.logger.info("StackJob: lock held for #{@stack.uuid}, exiting early")
+        return false
+      end
+      f.flock(File::LOCK_UN)
+    end
+
+    true
+  rescue StandardError => e
+    Rails.logger.error("StackJob: lock check failed: #{e.message}")
+  end
+
   def execute
     script = ERB.new(
       Rails.root.join("app/jobs/stack_job/templates/#{self.class.script_template}.sh.tt").read,

app/jobs/stack_job/templates/control.sh.tt

@@ -1,15 +1,19 @@
 #!/bin/sh
 set -e
 
+BASE_DIR="<%= @assets.base_dir %>"
+LOCK_FILE="$BASE_DIR/.lock"
+mkdir -p "$BASE_DIR"
+exec 9>"$LOCK_FILE"
+flock -n 9 || { echo "Another operation is already in progress for this stack." >&2; exit 1; }
+
 __compose_includes() {
     local include_files="<%= @assets.include_files %>"
     if [ -z "$include_files" ]; then
         return
     fi
 
-    local includes
-    includes=$(echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' ')
-    printf "%s" "${includes}"
+    echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' '
 }
 
 

app/jobs/stack_job/templates/deploy.sh.tt

@@ -1,6 +1,12 @@
 #!/bin/sh
 set -e
 
+BASE_DIR="<%= @assets.base_dir %>"
+LOCK_FILE="$BASE_DIR/.lock"
+mkdir -p "$BASE_DIR"
+exec 9>"$LOCK_FILE"
+flock -n 9 || { echo "Another operation is already in progress for this stack." >&2; exit 1; }
+
 __compose_includes() {
     local include_files="<%= @assets.include_files %>"
     if [ -z "$include_files" ]; then

app/jobs/stack_job/templates/destroy.sh.tt

@@ -1,15 +1,19 @@
 #!/bin/sh
 set -e
 
+BASE_DIR="<%= @assets.base_dir %>"
+LOCK_FILE="$BASE_DIR/.lock"
+mkdir -p "$BASE_DIR"
+exec 9>"$LOCK_FILE"
+flock -n 9 || echo "Warning: overriding lock, proceeding with destroy" >&2
+
 __compose_includes() {
     local include_files="<%= @assets.include_files %>"
     if [ -z "$include_files" ]; then
         return
     fi
 
-    local includes
-    includes=$(echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' ')
-    printf "%s" "${includes}"
+    echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' '
 }
 
 remove_stop_stack() {

app/jobs/stack_job/templates/update.sh.tt

@@ -1,15 +1,19 @@
 #!/bin/sh
 set -e
 
+BASE_DIR="<%= @assets.base_dir %>"
+LOCK_FILE="$BASE_DIR/.lock"
+mkdir -p "$BASE_DIR"
+exec 9>"$LOCK_FILE"
+flock -n 9 || { echo "Another operation is already in progress for this stack." >&2; exit 1; }
+
 __compose_includes() {
     local include_files="<%= @assets.include_files %>"
     if [ -z "$include_files" ]; then
         return
     fi
 
-    local includes
-    includes=$(echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' ')
-    printf "%s" "${includes}"
+    echo "<%= @assets.include_files %>" | tr ' ' '\n' | sed 's/^/--file /' | tr '\n' ' '
 }
 
 __resolve_path() {