Adding retries to Google Chat Notifications #6242 (#6245)

Co-authored-by: Frank Elsinga <frank@elsinga.de>

server/notification-providers/google-chat.js

@@ -12,6 +12,29 @@ class GoogleChat extends NotificationProvider {
     async send(notification, msg, monitorJSON = null, heartbeatJSON = null) {
         const okMsg = "Sent Successfully.";
 
+        // If Google Chat Webhook rate limit is reached, retry to configured max retries defaults to 3, delay between 60-180 seconds
+        const post = async (url, data, config) => {
+            let retries = notification.googleChatMaxRetries || 1; // Default to 1 retries
+            retries = (retries > 10) ? 10 : retries; // Enforce maximum retries in backend
+            while (retries > 0) {
+                try {
+                    await axios.post(url, data, config);
+                    return;
+                } catch (error) {
+                    if (error.response && error.response.status === 429) {
+                        retries--;
+                        if (retries === 0) {
+                            throw error;
+                        }
+                        const delay = 60000 + Math.random() * 120000;
+                        await new Promise(resolve => setTimeout(resolve, delay));
+                    } else {
+                        throw error;
+                    }
+                }
+            }
+        };
+
         try {
             let config = this.getAxiosConfigWithProxy({});
             // Google Chat message formatting: https://developers.google.com/chat/api/guides/message-formats/basic
@@ -24,7 +47,7 @@ class GoogleChat extends NotificationProvider {
                     heartbeatJSON
                 );
                 const data = { "text": renderedText };
-                await axios.post(notification.googleChatWebhookURL, data, config);
+                await post(notification.googleChatWebhookURL, data, config);
                 return okMsg;
             }
 
@@ -96,7 +119,7 @@ class GoogleChat extends NotificationProvider {
                 ],
             };
 
-            await axios.post(notification.googleChatWebhookURL, data, config);
+            await post(notification.googleChatWebhookURL, data, config);
             return okMsg;
         } catch (error) {
             this.throwGeneralAxiosError(error);

src/components/notifications/GoogleChat.vue

@@ -11,6 +11,14 @@
         </div>
     </div>
 
+    <div class="mb-3">
+        <label for="google-chat-max-retries" class="form-label">{{ $t("Maximum Retries") }}<span style="color: red;"><sup>*</sup></span></label>
+        <input id="google-chat-max-retries" v-model.number="$parent.notification.googleChatMaxRetries" type="number" class="form-control" min="1" max="10" step="1" required>
+        <div class="form-text">
+            {{ $t("Number of retry attempts if webhook fails") }}
+        </div>
+    </div>
+
     <div class="mb-3">
         <div class="form-check form-switch">
             <input id="google-chat-use-template" v-model="$parent.notification.googleChatUseTemplate" type="checkbox" class="form-check-input">
@@ -45,5 +53,11 @@ export default {
             ]);
         }
     },
+    mounted() {
+        // Initialize default if needed
+        if (!this.$parent.notification.googleChatMaxRetries) {
+            this.$parent.notification.googleChatMaxRetries ||= 1;
+        }
+    },
 };
 </script>

src/lang/en.json

@@ -1170,5 +1170,7 @@
     "Bot secret": "Bot secret",
     "Send UP silently": "Send UP silently",
     "Send DOWN silently": "Send DOWN silently",
-    "Installing a Nextcloud Talk bot requires administrative access to the server.": "Installing a Nextcloud Talk bot requires administrative access to the server."
+    "Installing a Nextcloud Talk bot requires administrative access to the server.": "Installing a Nextcloud Talk bot requires administrative access to the server.",
+    "Number of retry attempts if webhook fails": "Number of retry attempts (every 60-180 seconds) if the webhook fails.",
+    "Maximum Retries": "Maximum Retries"
 }