Update the Splunk On-Call integration to not retry HTTP 404 responses

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project will be documented in this file.
 
 ### Improvements
 - Update logging configuration to write logs to database (to table `logs_record`)
+- Update the Splunk On-Call integration to not retry HTTP 404 responses
 
 ## v3.0.1 - 2023-10-30
 

hc/api/tests/test_notify_victorops.py

@@ -58,3 +58,21 @@ class NotifyVictorOpsTestCase(BaseTestCase):
         self.assertEqual(
             payload["state_message"], "Foo & Bar received a ping and is now UP"
         )
+
+    @patch("hc.api.transports.curl.request", autospec=True)
+    def test_it_does_not_retry_404(self, mock_post: Mock) -> None:
+        mock_post.return_value.status_code = 404
+
+        self.channel.notify(self.check)
+
+        n = Notification.objects.get()
+        self.assertEqual(n.error, "Received status code 404")
+        self.assertEqual(mock_post.call_count, 1)
+
+    @patch("hc.api.transports.curl.request", autospec=True)
+    def test_it_disables_channel_on_404(self, mock_post: Mock) -> None:
+        mock_post.return_value.status_code = 404
+
+        self.channel.notify(self.check)
+        self.channel.refresh_from_db()
+        self.assertTrue(self.channel.disabled)

hc/api/transports.py

@@ -717,6 +717,13 @@ class RocketChat(HttpTransport):
 
 
 class VictorOps(HttpTransport):
+    @classmethod
+    def raise_for_response(cls, response: curl.Response) -> NoReturn:
+        message = f"Received status code {response.status_code}"
+        # If the endpoint returns 404, this endpoint is unlikely to ever work again
+        permanent = response.status_code == 404
+        raise TransportError(message, permanent=permanent)
+
     def notify(self, check: Check, notification: Notification) -> None:
         if not settings.VICTOROPS_ENABLED:
             raise TransportError("Splunk On-Call notifications are not enabled.")