Update Slack integration to not retry when it hits 400 "invalid_token"

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 ### Improvements
 - Update logging configuration to write logs to database (to table `logs_record`)
 - Update the Splunk On-Call integration to not retry HTTP 404 responses
+- Update the Slack integration to not retry when Slack returns 400 "invalid_token"
 
 ## v3.0.1 - 2023-10-30
 

hc/api/tests/test_notify_slack.py

@@ -147,9 +147,24 @@ class NotifySlackTestCase(BaseTestCase):
         mock_post.return_value.status_code = 404
 
         self.channel.notify(self.check)
+        # Make sure the HTTP request was made only once (no retries):
+        self.assertEqual(mock_post.call_count, 1)
         self.channel.refresh_from_db()
         self.assertTrue(self.channel.disabled)
 
+    @patch("hc.api.transports.curl.request", autospec=True)
+    def test_it_disables_channel_on_400_invalid_token(self, mock_post: Mock) -> None:
+        self._setup_data("123")
+        mock_post.return_value.status_code = 400
+        mock_post.return_value.content = b"invalid_token"
+
+        self.channel.notify(self.check)
+        # Make sure the HTTP request was made only once (no retries):
+        self.channel.refresh_from_db()
+        self.assertTrue(self.channel.disabled)
+
+        self.assertEqual(mock_post.call_count, 1)
+
     @override_settings(SITE_ROOT="http://testserver")
     @patch("hc.api.transports.curl.request", autospec=True)
     def test_it_handles_last_ping_fail(self, mock_post: Mock) -> None:

hc/api/transports.py

@@ -469,9 +469,19 @@ class Slack(Slackalike):
             logger.debug("Slack returned HTTP 400 with body: %s", response.content)
 
         message = f"Received status code {response.status_code}"
-        # If Slack returns 404, this endpoint is unlikely to ever work again
-        # https://api.slack.com/messaging/webhooks#handling_errors
-        permanent = response.status_code == 404
+
+        permanent = False
+        if response.status_code == 404:
+            # If Slack returns 404, this endpoint is unlikely to ever work again
+            # https://api.slack.com/messaging/webhooks#handling_errors
+            permanent = True
+        elif response.status_code == 400 and response.content == b"invalid_token":
+            # If Slack returns 400 with "invalid_token" in response body,
+            # we're using a deactivated user's token to post to a private channel.
+            # In theory this condition can recover (a deactivated user can be
+            # activated), but in practice it is unlikely to happen.
+            permanent = True
+
         raise TransportError(message, permanent=permanent)
 
     def notify(self, check: Check, notification: Notification) -> None: