feat: Only annotate failing heartbeat calls (#399)

This will reduce the amount of APM data sent (heartbeats make up a
substantial portion of requests) and also include some additional details.

Author: timmc-edx

openedx/core/djangoapps/heartbeat/runchecks.py

@@ -35,7 +35,8 @@ def runchecks(include_extended=False):
                 'status': is_ok,
                 'message': message
             }
-            set_custom_attribute(f"heartbeat.status.{path}", is_ok)
+            if not is_ok:
+                set_custom_attribute(f"heartbeat.failure.{path}", message)
         except ImportError as e:
             raise ImproperlyConfigured(f'Error importing module {module}: "{e}"')  # lint-amnesty, pylint: disable=raise-missing-from
         except AttributeError:

openedx/core/djangoapps/heartbeat/tests/test_heartbeat.py

@@ -29,10 +29,8 @@ def test_success(self, mock_set_attribute):
         response = self.client.get(self.heartbeat_url + '?extended')
 
         assert response.status_code == 200
-        # Spot-checking a success
-        mock_set_attribute.assert_any_call(
-            'heartbeat.status.openedx.core.djangoapps.heartbeat.default_checks.check_database', True
-        )
+        # We only annotate failing requests
+        mock_set_attribute.assert_not_called()
 
     def test_sql_fail(self):
         with patch('openedx.core.djangoapps.heartbeat.default_checks.connection') as mock_connection:
@@ -50,5 +48,5 @@ def test_modulestore_fail(self, mock_set_attribute):
             assert response.status_code == 503
         # Spot-checking a failure
         mock_set_attribute.assert_any_call(
-            'heartbeat.status.openedx.core.djangoapps.heartbeat.default_checks.check_modulestore', False
+            'heartbeat.failure.openedx.core.djangoapps.heartbeat.default_checks.check_modulestore', 'msg'
         )