Use buildx

This is helpful to determine if your alerts are valid before an actual failure

.drone.yml

@@ -60,51 +60,15 @@ steps:
     when:
       event: tag
 
-  - name: push image - arm
-    image: plugins/docker
+  - name: Build and publish docker images
+    image: thegeeklab/drone-docker-buildx
     settings:
       repo: iamthefij/minitor-go
       auto_tag: true
-      auto_tag_suffix: linux-arm
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      build_args:
-        - ARCH=arm
-        - REPO=arm32v7
-
-  - name: push image - arm64
-    image: plugins/docker
-    settings:
-      repo: iamthefij/minitor-go
-      auto_tag: true
-      auto_tag_suffix: linux-arm64
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      build_args:
-        - ARCH=arm64
-        - REPO=arm64v8
-
-  - name: push image - amd64
-    image: plugins/docker
-    settings:
-      repo: iamthefij/minitor-go
-      auto_tag: true
-      auto_tag_suffix: linux-amd64
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-
-  - name: publish manifest
-    image: plugins/manifest
-    settings:
-      spec: manifest.tmpl
-      auto_tag: true
-      ignore_missing: true
+      platforms:
+        - linux/amd64
+        - linux/arm64
+        - linux/arm
       username:
         from_secret: docker_username
       password:

Dockerfile

@@ -1,11 +1,10 @@
-ARG REPO=library
-FROM ${REPO}/alpine:3.18
+FROM alpine:3.18
 
 RUN mkdir /app
 WORKDIR /app/
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5 curl=~8 jq=~1.6 bind-tools~=9
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor
@@ -15,8 +14,9 @@ COPY ./scripts /app/scripts
 RUN chmod -R 755 /app/scripts
 
 # Copy minitor in
-ARG ARCH=amd64
-COPY ./dist/minitor-linux-${ARCH} ./minitor
+ARG TARGETOS
+ARG TARGETARCH
+COPY ./dist/minitor-${TARGETOS}-${TARGETARCH} ./minitor
 
 # Drop to non-root user
 USER minitor

Dockerfile.multi-stage

@@ -1,4 +1,3 @@
-ARG REPO=library
 FROM golang:1.20 AS builder
 
 RUN mkdir /app
@@ -9,12 +8,13 @@ RUN go mod download
 
 COPY ./*.go /app/
 
-ARG ARCH=amd64
+ARG TARGETOS
+ARG TARGETARCH
 ARG VERSION=dev
-ENV CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH}
+ENV CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=${TARGETARCH}
 RUN go build -ldflags "-X main.version=${VERSION}" -a -installsuffix nocgo -o minitor .
 
-FROM ${REPO}/alpine:3.18
+FROM alpine:3.18
 RUN mkdir /app
 WORKDIR /app/
 
@@ -22,7 +22,7 @@ WORKDIR /app/
 COPY --from=builder /app/minitor .
 
 # Add common checking tools
-RUN apk --no-cache add bash=~5 curl=~8 jq=~1.6 bind-tools~=9
+RUN apk --no-cache add bash=~5 curl=~8 jq=~1 bind-tools=~9 tzdata~=2024a
 
 # Add minitor user for running as non-root
 RUN addgroup -S minitor && adduser -S minitor -G minitor

Makefile

@@ -79,11 +79,11 @@ $(TARGET_ALIAS):
 # Arch specific docker build targets
 .PHONY: docker-build-arm
 docker-build-arm: dist/minitor-linux-arm
-	docker build --build-arg REPO=arm32v7 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
+	docker build --platform linux/arm . -t ${DOCKER_TAG}-linux-arm
 
 .PHONY: docker-build-arm64
 docker-build-arm64: dist/minitor-linux-arm64
-	docker build --build-arg REPO=arm64v8 --build-arg ARCH=arm64 . -t ${DOCKER_TAG}-linux-arm64
+	docker build  --platform linux/arm64 . -t ${DOCKER_TAG}-linux-arm64
 
 # Cross run on host architechture
 .PHONY: docker-run-arm

README.md

@@ -46,6 +46,8 @@ docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor-go:latest
 
 Images are provided for `amd64`, `arm`, and `arm64` architechtures.
 
+Timezone configuration for the container is set by passing the `TZ` env variable. Eg. `TZ=America/Los_Angeles`.
+
 ## Configuring
 
 In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
@@ -116,6 +118,16 @@ To provide flexible formatting, the following non-standard functions are availab
 
 For more information, check out the [Go documentation for the time module](https://pkg.go.dev/time@go1.20.7#pkg-constants).
 
+#### Running alerts on startup
+
+It's not the best feeling to find out your alerts are broken when you're expecting to be alerted about another failure. To avoid this and provide early insight into broken alerts, it is possible to specify a list of alerts to run when Minitor starts up. This can be done using the command line flag `-startup-alerts`. This flag accepts a comma separated list of strings and will run a test of each of those alerts. Minitor will then respond as it typically does for any failed alert. This can be used to allow you time to correct when initially launching, and to allow schedulers to more easily detect a failed deployment of Minitor.
+
+Eg.
+
+```bash
+minitor -startup-alerts=log_down,log_up -config ./config.yml
+```
+
 ### Metrics
 
 Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	git.iamthefij.com/iamthefij/slog v1.3.0
-	github.com/prometheus/client_golang v1.15.0
+	github.com/prometheus/client_golang v1.19.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -14,10 +14,10 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 )

go.sum

@@ -182,12 +182,16 @@ github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrb
 github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y=
 github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
 github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
@@ -195,6 +199,8 @@ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+
 github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
 github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
@@ -203,6 +209,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -365,6 +373,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -504,6 +514,10 @@ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

main.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"strings"
 	"time"
 
 	"git.iamthefij.com/iamthefij/slog"
@@ -91,9 +92,38 @@ func checkMonitors(config *Config) error {
 	return nil
 }
 
+func sendStartupAlerts(config *Config, alertNames []string) error {
+	for _, alertName := range alertNames {
+		var err error
+
+		alert, ok := config.Alerts[alertName]
+		if !ok {
+			err = fmt.Errorf("unknown alert %s: %w", alertName, errUnknownAlert)
+		}
+
+		if err == nil {
+			_, err = alert.Send(AlertNotice{
+				AlertCount:      0,
+				FailureCount:    0,
+				IsUp:            true,
+				LastSuccess:     time.Now(),
+				MonitorName:     fmt.Sprintf("First Run Alert Test: %s", alert.Name),
+				LastCheckOutput: "",
+			})
+		}
+
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func main() {
 	showVersion := flag.Bool("version", false, "Display the version of minitor and exit")
 	configPath := flag.String("config", "config.yml", "Alternate configuration path (default: config.yml)")
+	startupAlerts := flag.String("startup-alerts", "", "List of alerts to run on startup. This can help determine unhealthy alerts early on. (default \"\")")
 
 	flag.BoolVar(&slog.DebugLevel, "debug", false, "Enables debug logs (default: false)")
 	flag.BoolVar(&ExportMetrics, "metrics", false, "Enables prometheus metrics exporting (default: false)")
@@ -119,6 +149,14 @@ func main() {
 		go ServeMetrics()
 	}
 
+	if *startupAlerts != "" {
+		alertNames := strings.Split(*startupAlerts, ",")
+
+		err = sendStartupAlerts(&config, alertNames)
+
+		slog.OnErrPanicf(err, "Error running startup alerts")
+	}
+
 	// Start main loop
 	for {
 		err = checkMonitors(&config)

main_test.go

@@ -134,3 +134,64 @@ func TestCheckMonitors(t *testing.T) {
 		}
 	}
 }
+
+func TestFirstRunAlerts(t *testing.T) {
+	cases := []struct {
+		config        Config
+		expectErr     bool
+		startupAlerts []string
+		name          string
+	}{
+		{
+			config:        Config{},
+			expectErr:     false,
+			startupAlerts: []string{},
+			name:          "Empty",
+		},
+		{
+			config:        Config{},
+			expectErr:     true,
+			startupAlerts: []string{"missing"},
+			name:          "Unknown",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"good": {
+						Command: CommandOrShell{Command: []string{"true"}},
+					},
+				},
+			},
+			expectErr:     false,
+			startupAlerts: []string{"good"},
+			name:          "Successful alert",
+		},
+		{
+			config: Config{
+				Alerts: map[string]*Alert{
+					"bad": {
+						Name:    "bad",
+						Command: CommandOrShell{Command: []string{"false"}},
+					},
+				},
+			},
+			expectErr:     true,
+			startupAlerts: []string{"bad"},
+			name:          "Failed alert",
+		},
+	}
+
+	for _, c := range cases {
+		err := c.config.Init()
+		if err != nil {
+			t.Errorf("sendFirstRunAlerts(%s): unexpected error reading config: %v", c.name, err)
+		}
+
+		err = sendStartupAlerts(&c.config, c.startupAlerts)
+		if err == nil && c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Expected error, the code did not error", c.name)
+		} else if err != nil && !c.expectErr {
+			t.Errorf("sendFirstRunAlerts(%s): Did not expect an error, but we got one anyway: %v", c.name, err)
+		}
+	}
+}

manifest.tmpl

@@ -1,25 +0,0 @@
-image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
-{{#if build.tags}}
-tags:
-{{#each build.tags}}
-  - {{this}}
-{{/each}}
-{{/if}}
-manifests:
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
-    platform:
-      architecture: amd64
-      os: linux
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
-    platform:
-      architecture: arm64
-      os: linux
-      variant: v8
-  -
-    image: iamthefij/minitor-go:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
-    platform:
-      architecture: arm
-      os: linux
-      variant: v7