From 9bbc452ee90933e4108d7d32a415c027715be1bc Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Sat, 17 Feb 2018 20:45:37 -0800
Subject: [PATCH] Add exponential backoff

---
 minitor/main.py   | 20 +++++++++++++++-----
 sample-config.yml |  2 +-
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/minitor/main.py b/minitor/main.py
index 8b21e68..2298fab 100644
--- a/minitor/main.py
+++ b/minitor/main.py
@@ -41,8 +41,8 @@ class Monitor(object):
         settings = {
             'alerts': [ 'log' ],
             'check_interval': 30,
-            'alert_after': 3,
-            'alert_every': 4,
+            'alert_after': 4,
+            'alert_every': -1,
         }
         settings.update(config)
         self.validate_settings(settings)
@@ -56,6 +56,7 @@ class Monitor(object):
 
         self.last_check = None
         self.failure_count = 0
+        self.alert_count = 0
 
     def validate_settings(self, settings):
         """Validates that settings for this Monitor are valid
@@ -112,13 +113,22 @@ class Monitor(object):
     def success(self):
         """Handles success tasks"""
         self.failure_count = 0
+        self.alert_count = 0
 
     def failure(self):
         """Handles failure tasks and possibly raises MinitorAlert"""
         self.failure_count += 1
-        min_failures = self.failure_count >= self.alert_after
-        failure_interval = (self.failure_count % self.alert_every) == 0
-        if min_failures and failure_interval:
+        if self.failure_count < self.alert_after:
+            return
+        if self.alert_every >= 0:
+            failure_interval = (self.failure_count % self.alert_every) == 0
+        else:
+            failure_interval = (
+                (self.failure_count - self.alert_after) >=
+                (2 ** self.alert_count)
+            )
+        if failure_interval:
+            self.alert_count += 1
             raise MinitorAlert('{} check has failed {} times'.format(
                 self.name, self.failure_count
             ))
diff --git a/sample-config.yml b/sample-config.yml
index 6309202..e9f2cf0 100644
--- a/sample-config.yml
+++ b/sample-config.yml
@@ -6,7 +6,7 @@ monitors:
     alerts: [ log, email, sms ]
     check_interval: 30
     alert_after: 3
-    alert_every: 4
+    alert_every: -1 # Defaults to -1 for exponential backoff
 
 alerts:
   email: