2019-09-21 22:03:26 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2019-10-01 15:26:07 +00:00
|
|
|
"bytes"
|
2021-05-11 04:00:58 +00:00
|
|
|
"errors"
|
2019-10-03 23:30:49 +00:00
|
|
|
"fmt"
|
2019-09-21 22:03:26 +00:00
|
|
|
"os/exec"
|
|
|
|
"text/template"
|
|
|
|
"time"
|
2021-05-11 03:12:18 +00:00
|
|
|
|
|
|
|
"git.iamthefij.com/iamthefij/slog"
|
2019-09-21 22:03:26 +00:00
|
|
|
)
|
|
|
|
|
2021-05-11 04:00:58 +00:00
|
|
|
var (
|
|
|
|
errNoTemplate = errors.New("no template")
|
|
|
|
|
|
|
|
// ErrAlertFailed indicates that an alert failed to send
|
|
|
|
ErrAlertFailed = errors.New("alert failed")
|
|
|
|
)
|
|
|
|
|
2019-10-02 23:09:11 +00:00
|
|
|
// Alert is a config driven mechanism for sending a notice
|
2019-09-21 22:03:26 +00:00
|
|
|
type Alert struct {
|
2022-01-27 00:34:31 +00:00
|
|
|
Name string `hcl:"name,label"`
|
|
|
|
Command []string `hcl:"command,optional"`
|
|
|
|
ShellCommand string `hcl:"shell_command,optional"`
|
2019-10-02 16:37:29 +00:00
|
|
|
commandTemplate []*template.Template
|
2019-10-01 15:26:07 +00:00
|
|
|
commandShellTemplate *template.Template
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 23:09:11 +00:00
|
|
|
// AlertNotice captures the context for an alert to be sent
|
|
|
|
type AlertNotice struct {
|
2022-01-27 00:34:31 +00:00
|
|
|
AlertCount int
|
|
|
|
FailureCount int
|
2019-10-02 23:09:11 +00:00
|
|
|
IsUp bool
|
2021-05-11 04:00:58 +00:00
|
|
|
LastSuccess time.Time
|
|
|
|
MonitorName string
|
|
|
|
LastCheckOutput string
|
2019-10-02 23:09:11 +00:00
|
|
|
}
|
|
|
|
|
2024-11-16 00:36:45 +00:00
|
|
|
// Validate checks that the Alert is properly configured and returns errors if not
|
|
|
|
func (alert Alert) Validate() error {
|
|
|
|
hasCommand := len(alert.Command) > 0
|
|
|
|
hasShellCommand := alert.ShellCommand != ""
|
2022-01-27 00:34:31 +00:00
|
|
|
|
2024-11-16 00:36:45 +00:00
|
|
|
var err error
|
|
|
|
|
|
|
|
hasAtLeastOneCommand := hasCommand || hasShellCommand
|
|
|
|
if !hasAtLeastOneCommand {
|
|
|
|
err = errors.Join(err, fmt.Errorf(
|
|
|
|
"%w: alert %s has no command or shell_command configured",
|
|
|
|
ErrInvalidAlert,
|
|
|
|
alert.Name,
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
|
|
|
hasAtMostOneCommand := !(hasCommand && hasShellCommand)
|
|
|
|
if !hasAtMostOneCommand {
|
|
|
|
err = errors.Join(err, fmt.Errorf(
|
|
|
|
"%w: alert %s has both command and shell_command configured",
|
|
|
|
ErrInvalidAlert,
|
|
|
|
alert.Name,
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
|
|
|
return err
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 23:09:11 +00:00
|
|
|
// BuildTemplates compiles command templates for the Alert
|
2019-10-03 23:30:49 +00:00
|
|
|
func (alert *Alert) BuildTemplates() error {
|
2021-05-11 03:12:18 +00:00
|
|
|
slog.Debugf("Building template for alert %s", alert.Name)
|
|
|
|
|
2023-08-10 20:22:30 +00:00
|
|
|
// Time format func factory
|
|
|
|
tff := func(formatString string) func(time.Time) string {
|
|
|
|
return func(t time.Time) string {
|
|
|
|
return t.Format(formatString)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Create some functions for formatting datetimes in popular formats
|
|
|
|
timeFormatFuncs := template.FuncMap{
|
|
|
|
"ANSIC": tff(time.ANSIC),
|
|
|
|
"UnixDate": tff(time.UnixDate),
|
|
|
|
"RubyDate": tff(time.RubyDate),
|
|
|
|
"RFC822Z": tff(time.RFC822Z),
|
|
|
|
"RFC850": tff(time.RFC850),
|
|
|
|
"RFC1123": tff(time.RFC1123),
|
|
|
|
"RFC1123Z": tff(time.RFC1123Z),
|
|
|
|
"RFC3339": tff(time.RFC3339),
|
|
|
|
"RFC3339Nano": tff(time.RFC3339Nano),
|
|
|
|
"FormatTime": func(t time.Time, timeFormat string) string {
|
|
|
|
return t.Format(timeFormat)
|
|
|
|
},
|
|
|
|
"InTZ": func(t time.Time, tzName string) (time.Time, error) {
|
|
|
|
tz, err := time.LoadLocation(tzName)
|
|
|
|
if err != nil {
|
|
|
|
return t, fmt.Errorf("failed to convert time to specified tz: %w", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return t.In(tz), nil
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2021-05-11 04:00:58 +00:00
|
|
|
switch {
|
2022-01-27 00:34:31 +00:00
|
|
|
case alert.commandTemplate == nil && alert.Command != nil:
|
2019-10-02 16:37:29 +00:00
|
|
|
alert.commandTemplate = []*template.Template{}
|
2022-01-27 00:34:31 +00:00
|
|
|
for i, cmdPart := range alert.Command {
|
2019-10-02 16:37:29 +00:00
|
|
|
alert.commandTemplate = append(alert.commandTemplate, template.Must(
|
2023-08-10 20:22:30 +00:00
|
|
|
template.New(alert.Name+fmt.Sprint(i)).Funcs(timeFormatFuncs).Parse(cmdPart),
|
2019-10-02 16:37:29 +00:00
|
|
|
))
|
|
|
|
}
|
2022-01-27 00:34:31 +00:00
|
|
|
case alert.commandShellTemplate == nil && alert.ShellCommand != "":
|
|
|
|
shellCmd := alert.ShellCommand
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-09-21 22:03:26 +00:00
|
|
|
alert.commandShellTemplate = template.Must(
|
2023-08-10 20:22:30 +00:00
|
|
|
template.New(alert.Name).Funcs(timeFormatFuncs).Parse(shellCmd),
|
2019-09-21 22:03:26 +00:00
|
|
|
)
|
2021-05-11 04:00:58 +00:00
|
|
|
default:
|
|
|
|
return fmt.Errorf("No template provided for alert %s: %w", alert.Name, errNoTemplate)
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
2019-10-03 23:30:49 +00:00
|
|
|
|
|
|
|
return nil
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
|
|
|
|
2019-10-02 23:09:11 +00:00
|
|
|
// Send will send an alert notice by executing the command template
|
2020-02-18 00:46:56 +00:00
|
|
|
func (alert Alert) Send(notice AlertNotice) (outputStr string, err error) {
|
2021-05-11 03:12:18 +00:00
|
|
|
slog.Infof("Sending alert %s for %s", alert.Name, notice.MonitorName)
|
|
|
|
|
2019-09-21 22:03:26 +00:00
|
|
|
var cmd *exec.Cmd
|
2021-05-11 04:00:58 +00:00
|
|
|
|
|
|
|
switch {
|
|
|
|
case alert.commandTemplate != nil:
|
2019-10-02 16:37:29 +00:00
|
|
|
command := []string{}
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
for _, cmdTmp := range alert.commandTemplate {
|
|
|
|
var commandBuffer bytes.Buffer
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-03 23:30:49 +00:00
|
|
|
err = cmdTmp.Execute(&commandBuffer, notice)
|
2019-10-02 16:37:29 +00:00
|
|
|
if err != nil {
|
2019-10-03 23:30:49 +00:00
|
|
|
return
|
2019-10-02 16:37:29 +00:00
|
|
|
}
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
command = append(command, commandBuffer.String())
|
|
|
|
}
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
cmd = exec.Command(command[0], command[1:]...)
|
2021-05-11 04:00:58 +00:00
|
|
|
case alert.commandShellTemplate != nil:
|
2019-09-21 22:03:26 +00:00
|
|
|
var commandBuffer bytes.Buffer
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-03 23:30:49 +00:00
|
|
|
err = alert.commandShellTemplate.Execute(&commandBuffer, notice)
|
2019-10-01 15:26:07 +00:00
|
|
|
if err != nil {
|
2019-10-03 23:30:49 +00:00
|
|
|
return
|
2019-10-01 15:26:07 +00:00
|
|
|
}
|
2021-05-11 04:00:58 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
shellCommand := commandBuffer.String()
|
2019-10-01 15:26:07 +00:00
|
|
|
|
2019-10-02 16:37:29 +00:00
|
|
|
cmd = ShellCommand(shellCommand)
|
2021-05-11 04:00:58 +00:00
|
|
|
default:
|
|
|
|
err = fmt.Errorf("No templates compiled for alert %s: %w", alert.Name, errNoTemplate)
|
|
|
|
|
2019-10-03 23:30:49 +00:00
|
|
|
return
|
2019-10-02 16:37:29 +00:00
|
|
|
}
|
|
|
|
|
2019-10-03 23:30:49 +00:00
|
|
|
// Exit if we're not ready to run the command
|
|
|
|
if cmd == nil || err != nil {
|
|
|
|
return
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|
2019-10-03 23:30:49 +00:00
|
|
|
|
|
|
|
var output []byte
|
|
|
|
output, err = cmd.CombinedOutput()
|
2020-02-18 00:46:56 +00:00
|
|
|
outputStr = string(output)
|
2021-05-11 03:12:18 +00:00
|
|
|
slog.Debugf("Alert output for: %s\n---\n%s\n---", alert.Name, outputStr)
|
2019-10-03 23:30:49 +00:00
|
|
|
|
2021-05-11 04:00:58 +00:00
|
|
|
if err != nil {
|
|
|
|
err = fmt.Errorf(
|
2023-08-10 20:23:02 +00:00
|
|
|
"Alert %s failed to send. Returned %w: %w",
|
2021-05-11 04:00:58 +00:00
|
|
|
alert.Name,
|
|
|
|
err,
|
|
|
|
ErrAlertFailed,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2020-02-18 00:46:56 +00:00
|
|
|
return outputStr, err
|
2019-09-21 22:03:26 +00:00
|
|
|
}
|