Adds a Slack integration, and fixes import error in main. (#1186)

* First commit for Slack integration.

updated code

Added fake error

* Fixed import error

* Undid changes to script

* update error message

* Removed fake error

* Handle Slack alert failures more gracefully (#1189)

* Cleaned up PR.

* Added back functions

* Now, we disable slack alerts by default.

* Added test

Author: finbarrtimbers

open_instruct/grpo_fast.py

@@ -419,6 +419,8 @@ class Args:
     """multiply the gpus used for each oe-eval task"""
     eval_priority: Literal["low", "normal", "high", "urgent"] = "normal"
     """the priority of auto-launched evaluation jobs"""
+    send_slack_alerts: bool = False
+    """Whether to send Slack alerts on training failures"""
 
     # Evaluation behavior
     eval_on_step_0: bool = False
@@ -3249,6 +3251,10 @@ def main(args: Args, tc: TokenizerConfig, model_config: ModelConfig):
             model_dims,
             checkpoint_state,
         )
+    except Exception as e:
+        if args.send_slack_alerts:
+            utils.send_slack_alert(e)
+        raise
     finally:
         cleanup_training_resources(
             stop_event, executor, [inference_results_Q, param_prompt_Q, evaluation_inference_results_Q], actor_manager

open_instruct/test_utils.py

@@ -19,6 +19,7 @@
 from unittest import mock
 
 import pytest
+import responses
 import torch
 import vllm
 from dateutil import parser
@@ -240,6 +241,27 @@ def test_description_without_progress(self, mock_is_beaker_job, mock_beaker_from
         self.assertNotIn("% complete", desc)
 
 
+class TestSlackAlert(unittest.TestCase):
+    @responses.activate
+    @mock.patch("open_instruct.utils.get_beaker_experiment_url")
+    @mock.patch("os.environ.get")
+    def test_send_slack_alert_with_beaker_url(self, mock_environ_get, mock_get_beaker_url):
+        webhook_url = "https://hooks.slack.com/services/test"
+        mock_environ_get.return_value = webhook_url
+        mock_get_beaker_url.return_value = "https://beaker.org/ex/test-123"
+
+        responses.add(responses.POST, webhook_url, json={"ok": True}, status=200)
+
+        test_error = ValueError("Test error message")
+        utils.send_slack_alert(test_error)
+
+        self.assertEqual(len(responses.calls), 1)
+        request_body = json.loads(responses.calls[0].request.body)
+        self.assertIn("<!here> A RL job has died.", request_body["text"])
+        self.assertIn("https://beaker.org/ex/test-123", request_body["text"])
+        self.assertIn("Test error message", request_body["text"])
+
+
 class TestUtilityFunctions(unittest.TestCase):
     """Test utility functions in utils module."""
 

open_instruct/utils.py

@@ -2433,3 +2433,29 @@ def combine_reward_metrics(reward_metrics: list[dict[str, Any]]) -> dict[str, An
             # Fallback: keep the latest value if aggregation strategy is unclear.
             combined[key] = records[-1]
     return combined
+
+
+def send_slack_alert(error: Exception) -> None:
+    """Sends an alert about a training failure to a Slack webhook (if the env var SLACK_WEBHOOK is set)."""
+    slack_webhook_url = os.environ.get("SLACK_WEBHOOK")
+    if not slack_webhook_url:
+        logger.warning("SLACK_WEBHOOK environment variable not set. Skipping Slack alert.")
+        return
+    beaker_url = get_beaker_experiment_url()
+    beaker_message = f"Check it out: {beaker_url}. " if beaker_url else ""
+    message = f"<!here> A RL job has died. {beaker_message}Error message: {str(error)}."
+    payload = {"text": message}
+    response = requests.post(slack_webhook_url, json=payload)
+    if not response.ok:
+        logger.warning("Failed to send Slack alert with status %s: %s", response.status_code, response.text)
+
+
+def get_beaker_experiment_url() -> str | None:
+    """If the env var BEAKER_WORKLOAD_ID is set, gets the current experiment URL."""
+    try:
+        beaker_client = beaker.Beaker.from_env()
+        workload = beaker_client.workload.get(os.environ["BEAKER_WORKLOAD_ID"])
+        url = beaker_client.experiment.url(workload.experiment)
+        return url
+    except Exception:
+        return None

pyproject.toml

@@ -83,6 +83,7 @@ dev = [
     "ruff>=0.11.13",
     "parameterized>=0.9.0",
     "rich>=13.7.0",
+    "responses>=0.25.8",
 ]
 
 [tool.pytest.ini_options]