diff --git a/config.go b/config.go
index 34b870b..ad348aa 100644
--- a/config.go
+++ b/config.go
@@ -29,6 +29,18 @@ func (config Config) IsValid() (isValid bool) {
 			log.Printf("ERROR: Invalid monitor configuration: %s", monitor.Name)
 			isValid = false
 		}
+		// Check that all Monitor alerts actually exist
+		for _, isUp := range []bool{true, false} {
+			for _, alertName := range monitor.GetAlertNames(isUp) {
+				if _, ok := config.Alerts[alertName]; !ok {
+					log.Printf(
+						"ERROR: Invalid monitor configuration: %s. Unknown alert %s",
+						monitor.Name, alertName,
+					)
+					isValid = false
+				}
+			}
+		}
 	}
 
 	// Validate alerts
diff --git a/config_test.go b/config_test.go
index be1bdc0..7143332 100644
--- a/config_test.go
+++ b/config_test.go
@@ -15,6 +15,7 @@ func TestLoadConfig(t *testing.T) {
 		{"./test/does-not-exist", true, "Invalid config path"},
 		{"./test/invalid-config-type.yml", true, "Invalid config type for key"},
 		{"./test/invalid-config-missing-alerts.yml", true, "Invalid config missing alerts"},
+		{"./test/invalid-config-unknown-alert.yml", true, "Invalid config unknown alert"},
 	}
 
 	for _, c := range cases {
diff --git a/main.go b/main.go
index a35bc68..d096fd6 100644
--- a/main.go
+++ b/main.go
@@ -27,7 +27,7 @@ func checkMonitors(config *Config) error {
 				}
 				alertNames := monitor.GetAlertNames(alertNotice.IsUp)
 				if alertNames == nil {
-					// TODO: Should this be a panic? Should this be validated against? Probably
+					// This should only happen for a recovery alert. AlertDown is validated not empty
 					log.Printf(
 						"WARNING: Recieved alert, but no alert mechanisms exist. MonitorName=%s IsUp=%t",
 						alertNotice.MonitorName, alertNotice.IsUp,
@@ -44,15 +44,16 @@ func checkMonitors(config *Config) error {
 								output,
 							)
 							return fmt.Errorf(
-								"ERROR: Unsuccessfully triggered alert '%s'. "+
+								"Unsuccessfully triggered alert '%s'. "+
 									"Crashing to avoid false negatives: %v",
 								alert.Name,
 								err,
 							)
 						}
 					} else {
-						// TODO: Maybe panic here. Also, probably validate up front
-						log.Printf("ERROR: Alert with name '%s' not found", alertName)
+						// This case should never actually happen since we validate against it
+						log.Printf("ERROR: Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
+						return fmt.Errorf("Unknown alert for monitor %s: %s", alertNotice.MonitorName, alertName)
 					}
 				}
 			}
diff --git a/test/invalid-config-unknown-alert.yml b/test/invalid-config-unknown-alert.yml
new file mode 100644
index 0000000..9991a6c
--- /dev/null
+++ b/test/invalid-config-unknown-alert.yml
@@ -0,0 +1,13 @@
+check_interval: 1
+
+monitors:
+  - name: Command
+    command: ['echo', '$PATH']
+    alert_down: [ 'not_log']
+    # alert_every: -1
+    alert_every: 0
+
+
+alerts:
+  log:
+    command: ['true']