Compare commits

..

1 Commits

Author SHA1 Message Date
82bbf75e3b Makefile now checks for venv and falls back to virtualenv
This can be handy for environments that don't have venv installed, but
could also result in some unreliable tests.
2018-04-11 14:33:10 -07:00
29 changed files with 245 additions and 1193 deletions

View File

@ -1,9 +0,0 @@
*$py.class
*.egg
*.egg-info/
*.py[cod]
*.so
**/__pycache__/
scripts/README.md
examples/
env/

View File

@ -1,201 +0,0 @@
# Build pipelines
def main(ctx):
pipelines = []
# Run tests
pipelines += run_tests()
# Add pypi push pipeline
pipelines += push_to_pypi(ctx)
# Add docker push pipelines
pipelines += push_to_docker(ctx)
return pipelines
# Return workspace in the container
def get_workspace():
return {
"base": "/app",
"path": ".",
}
# Builds a list of all test pipelines to be executed
def run_tests():
return [{
"kind": "pipeline",
"name": "tests",
"workspace": get_workspace(),
"steps": [
tox_step("python:3.7"),
tox_step("python:3.8"),
tox_step("python:3.9"),
tox_step("python:3.10"),
tox_step("python:3"),
# tox_step("pypy:3.9", "pypy3", "pypy3"),
# tox_step("pypy:3", "pypy3", "pypy3"),
notify_step(),
],
}]
# Builds a single python test step
def tox_step(docker_tag, python_cmd="python", tox_env="py3"):
return {
"name": "test {}".format(docker_tag.replace(":", "")),
"image": docker_tag,
"environment": {
"TOXENV": tox_env,
},
"commands": [
"{} -V".format(python_cmd),
"pip install tox",
"tox",
],
}
# Builds a notify step that will notify when the previous step changes
def notify_step():
return {
"name": "notify",
"image": "drillster/drone-email",
"settings": {
"host": {
"from_secret": "SMTP_HOST",
},
"username": {
"from_secret": "SMTP_USER",
},
"password": {
"from_secret": "SMTP_PASS",
},
"from": "drone@iamthefij.com",
},
"when": {
"status": [
"changed",
"failure",
],
},
}
# Push package to pypi
def push_to_pypi(ctx):
return [{
"kind": "pipeline",
"name": "deploy to pypi",
"depends_on": ["tests"],
"workspace": get_workspace(),
"trigger": {
"event": ["tag"],
"ref": [
"refs/heads/master",
"refs/tags/v*",
],
},
"steps": [
{
"name": "push to test pypi",
"image": "python:3",
"environment": {
"TWINE_USERNAME": {
"from_secret": "PYPI_USERNAME",
},
"TWINE_PASSWORD": {
"from_secret": "TEST_PYPI_PASSWORD",
},
},
"commands": ["make upload-test"],
},
{
"name": "push to pypi",
"image": "python:3",
"environment": {
"TWINE_USERNAME": {
"from_secret": "PYPI_USERNAME",
},
"TWINE_PASSWORD": {
"from_secret": "PYPI_PASSWORD",
},
},
"commands": ["make upload"],
"when": {
"event": ["tag"],
},
},
notify_step(),
]
}]
# Build and push docker image
def push_docker_step(tag_suffix, arch, repo):
return {
"name": "push {}".format(tag_suffix),
"image": "plugins/docker",
"settings": {
"repo": "iamthefij/minitor",
"auto_tag": True,
"auto_tag_suffix": tag_suffix,
"username": {
"from_secret": "docker_username",
},
"password": {
"from_secret": "docker_password",
},
"build_args": [
"ARCH={}".format(arch),
"REPO={}".format(repo),
],
},
}
# Builds a pipeline to push to docker
def push_to_docker(ctx):
return [{
"kind": "pipeline",
"name": "push to docker",
"depends_on": ["tests"],
"workspace": get_workspace(),
"trigger": {
"event": ["tag", "push"],
"ref": [
"refs/heads/master",
"refs/tags/v*",
],
},
"steps": [
{
"name": "get qemu",
"image": "busybox",
"commands": ["sh ./get_qemu.sh x86_64 arm aarch64"],
},
push_docker_step("linux-amd64", "x86_64", "library"),
push_docker_step("linux-arm", "arm", "arm32v6"),
push_docker_step("linux-arm64", "aarch64", "arm64v8"),
{
"name": "publish manifest",
"image": "plugins/manifest",
"settings": {
"spec": "manifest.tmpl",
"auto_tag": True,
"ignore_missing": True,
"username": {
"from_secret": "docker_username",
},
"password": {
"from_secret": "docker_password",
},
}
},
notify_step(),
],
}]
# vim: ft=python

10
.drone.yml Normal file
View File

@ -0,0 +1,10 @@
workspace:
base: /app
path: .
pipeline:
build:
image: python:3
commands:
- make test
- make build

1
.gitignore vendored
View File

@ -60,4 +60,3 @@ docs/_build/
target/ target/
config.yml config.yml
.mypy_cache/

View File

@ -1,23 +1,23 @@
repos: repos:
- repo: https://github.com/psf/black
rev: 22.3.0
hooks:
- id: black
- repo: https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks
rev: v1.2.3 rev: v1.2.3
hooks: hooks:
- id: trailing-whitespace - id: trailing-whitespace
- id: end-of-file-fixer - id: end-of-file-fixer
- id: autopep8-wrapper
args:
- -i
- --ignore=E265,E309,E501
- id: debug-statements - id: debug-statements
language_version: python3 language_version: python3
- id: flake8
language_version: python3
- id: check-yaml - id: check-yaml
args:
- --allow-multiple-documents
- id: check-merge-conflict - id: check-merge-conflict
- id: name-tests-test - id: name-tests-test
exclude: tests/(common.py|util.py|(helpers|integration/factories)/(.+).py) exclude: tests/(common.py|util.py|(helpers|integration/factories)/(.+).py)
- repo: https://github.com/asottile/reorder_python_imports - repo: https://github.com/asottile/reorder_python_imports
rev: v1.0.1 sha: v1.0.1
hooks: hooks:
- id: reorder-python-imports - id: reorder-python-imports
args: args:

View File

@ -1,18 +0,0 @@
---
dist: xenial
language: python
matrix:
include:
- python: "3.5"
env: TOXENV=py3
- python: "3.6"
env: TOXENV=py3
- python: "3.7"
env: TOXENV=py3
- python: "pypy3.5"
env: TOXENV=pypy3
install:
- pip install tox
script:
- tox

View File

@ -1,38 +0,0 @@
ARG REPO=library
FROM ${REPO}/python:3-alpine
LABEL maintainer="ian@iamthefij.com"
# Minitor: https://git.iamthefij.com/iamthefij/minitor
# This should be the target qemu arch
ARG ARCH=x86_64
COPY ./build/qemu-${ARCH}-static /usr/bin/
# Add common checking tools
RUN apk --no-cache add bash=~5.1 curl=~7.80 jq=~1.6
WORKDIR /app
# Add minitor user for running as non-root
RUN addgroup -S minitor && adduser -S minitor -G minitor
# Expose default metrics port
EXPOSE 8080
# Copy default sample config
COPY ./sample-config.yml /app/config.yml
# Copy Python package to container
COPY ./README.md /app/
COPY ./setup.py /app/
COPY ./minitor /app/minitor
RUN pip install --no-cache-dir -e .
# Copy scripts
COPY ./scripts /app/scripts
# Allow all users to execute minitor and scripts
RUN chmod -R 755 /app
# Drop to non-root user
USER minitor
ENTRYPOINT [ "python3", "-m", "minitor.main" ]

142
Makefile
View File

@ -1,149 +1,51 @@
DOCKER_TAG := minitor-dev # Try to use python3 -m venv, but fallback to virtualenv just in case
OPEN_CMD := $(shell type xdg-open &> /dev/null && echo 'xdg-open' || echo 'open') VENV := $(shell python3 -m venv --help &> /dev/null && echo 'python3 -m venv' || echo 'virtualenv -p python3')
ENV := env
.PHONY: default env:
default: test $(VENV) env
./env/bin/pip install -r requirements-dev.txt
# Create sample config
config.yml:
cp sample-config.yml config.yml
# Creates virtualenv
$(ENV):
python3 -m venv $(ENV)
# Install minitor and dependencies in virtualenv
$(ENV)/bin/minitor: $(ENV)
$(ENV)/bin/pip install -r requirements-dev.txt
# Install tox into virtualenv for running tests
$(ENV)/bin/tox: $(ENV)
$(ENV)/bin/pip install tox
# Install wheel for building packages
$(ENV)/bin/wheel: $(ENV)
$(ENV)/bin/pip install wheel
# Install twine for uploading packages
$(ENV)/bin/twine: $(ENV)
$(ENV)/bin/pip install twine
# Installs dev requirements to virtualenv
.PHONY: devenv
devenv: $(ENV)/bin/minitor
# Generates a smaller env for running tox, which builds it's own env
.PHONY: test-env
test-env: $(ENV)/bin/tox
# Generates a small build env for building and uploading dists
.PHONY: build-env
build-env: $(ENV)/bin/twine $(ENV)/bin/wheel
# Runs Minitor
.PHONY: run .PHONY: run
run: $(ENV)/bin/minitor config.yml run: env
$(ENV)/bin/minitor -vvv ./env/bin/python -m minitor.main
# Runs Minitor with metrics
.PHONY: run-metrics
run-metrics: $(ENV)/bin/minitor config.yml
$(ENV)/bin/minitor -vvv --metrics
# Runs tests with tox
.PHONY: test .PHONY: test
test: $(ENV)/bin/tox test: env
$(ENV)/bin/tox -e py3 ./env/bin/tox
# Builds wheel for package to upload
.PHONY: build .PHONY: build
build: $(ENV)/bin/wheel build: env
$(ENV)/bin/python setup.py sdist ./env/bin/python setup.py sdist
$(ENV)/bin/python setup.py bdist_wheel ./env/bin/python setup.py bdist_wheel
# Verify that the python version matches the git tag so we don't push bad shas
.PHONY: verify-tag-version
verify-tag-version:
$(eval TAG_NAME = $(shell [ -n "$(DRONE_TAG)" ] && echo $(DRONE_TAG) || git describe --tags --exact-match))
test "v$(shell python setup.py -V)" = "$(TAG_NAME)"
# Uses twine to upload to pypi
.PHONY: upload .PHONY: upload
upload: verify-tag-version build $(ENV)/bin/twine upload: env
$(ENV)/bin/twine upload dist/* ./env/bin/twine upload dist/*
# Uses twine to upload to test pypi
.PHONY: upload-test .PHONY: upload-test
upload-test: verify-tag-version build $(ENV)/bin/twine upload-test: env
$(ENV)/bin/twine upload --repository-url https://test.pypi.org/legacy/ dist/* ./env/bin/twine upload --repository-url https://test.pypi.org/legacy/ dist/*
# Cleans all build, runtime, and test artifacts
.PHONY: clean .PHONY: clean
clean: clean:
rm -fr ./build ./minitor.egg-info ./htmlcov ./.coverage ./.pytest_cache ./.tox rm -fr ./build ./minitor.egg-info ./htmlcov ./.coverage ./.pytest_cache ./.tox
find . -name '*.pyc' -delete find . -name '*.pyc' -delete
find . -name '__pycache__' -delete find . -name '__pycache__' -delete
# Cleans dist and env
.PHONY: dist-clean .PHONY: dist-clean
dist-clean: clean dist-clean: clean
rm -fr ./dist $(ENV) rm -fr ./dist ./env
# Install pre-commit hooks
.PHONY: install-hooks .PHONY: install-hooks
install-hooks: $(ENV) install-hooks:
$(ENV)/bin/tox -e pre-commit -- install -f --install-hooks ./env/bin/tox -e pre-commit -- install -f --install-hooks
# Generates test coverage
.coverage: .coverage:
$(ENV)/bin/tox ./env/bin/tox
# Builds coverage html
htmlcov/index.html: .coverage htmlcov/index.html: .coverage
$(ENV)/bin/coverage html ./env/bin/coverage html
# Opens coverage html in browser (on macOS and some Linux systems)
.PHONY: open-coverage .PHONY: open-coverage
open-coverage: htmlcov/index.html open-coverage: htmlcov/index.html
$(OPEN_CMD) htmlcov/index.html open htmlcov/index.html
# Docker targets
# Targets to download required qemu binaries for running on an amd64 machine
build/qemu-x86_64-static:
./get_qemu.sh x86_64
build/qemu-arm-static:
./get_qemu.sh arm
build/qemu-aarch64-static:
./get_qemu.sh aarch64
# Build Docker image for host architechture (amd64)
.PHONY: docker-build
docker-build: build/qemu-x86_64-static
docker build . -t ${DOCKER_TAG}-linux-amd64
# Cross build for arm architechtures
.PHONY: docker-cross-build-arm
docker-cross-build-arm: build/qemu-arm-static
docker build --build-arg REPO=arm32v6 --build-arg ARCH=arm . -t ${DOCKER_TAG}-linux-arm
.PHONY: docker-cross-build-arm64
docker-cross-build-arm64: build/qemu-aarch64-static
docker build --build-arg REPO=arm64v8 --build-arg ARCH=aarch64 . -t ${DOCKER_TAG}-linux-arm64
# Run on host architechture
.PHONY: docker-run
docker-run: docker-build config.yml
docker run --rm -v $(shell pwd)/config.yml:/app/config.yml ${DOCKER_TAG}-linux-amd64
# Cross run on host architechture
.PHONY: docker-cross-run-arm
docker-cross-run-arm: docker-cross-build-arm config.yml
docker run --rm -v $(shell pwd)/config.yml:/app/config.yml ${DOCKER_TAG}-linux-arm
.PHONY: docker-cross-run-arm64
docker-cross-run-arm64: docker-cross-build-arm64 config.yml
docker run --rm -v $(shell pwd)/config.yml:/app/config.yml ${DOCKER_TAG}-linux-arm64

View File

@ -2,10 +2,6 @@
A minimal monitoring system A minimal monitoring system
## Important
*This has been more or less replaced by a version written in Go. Check out [minitor-go](/iamthefij/minitor-go)*. There are no known issues with this version, but it is not really maintained anymore as I've migrated to the Go version since it uses fewer system resources.
## What does it do? ## What does it do?
Minitor accepts a YAML configuration file with a set of commands to run and a set of alerts to execute when those commands fail. It is designed to be as simple as possible and relies on other command line tools to do checks and issue alerts. Minitor accepts a YAML configuration file with a set of commands to run and a set of alerts to execute when those commands fail. It is designed to be as simple as possible and relies on other command line tools to do checks and issue alerts.
@ -20,103 +16,26 @@ I'm running a few small services and found Sensu, Consul, Nagios, etc. to all be
Install and execute with: Install and execute with:
```bash ```
pip install minitor pip install minitor
minitor minitor
``` ```
If locally developing you can use: If locally developing you can use:
```bash ```
make run make run
``` ```
It will read the contents of `config.yml` and begin its loop. You could also run it directly and provide a new config file via the `--config` argument. It will read the contents of `config.yml` and begin its loop. You could also run it directly and provide a new config file via the `--config` argument.
### Configuring
#### Docker In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file. Also, when alerts are executed, they will be passed through Python's format function with arguments for some attributes of the Monitor. Currently this is limited to `{monitor_name}`.
You can pull this repository directly from Docker:
```bash
docker pull iamthefij/minitor
```
The Docker image uses a default `config.yml` that is copied from `sample-config.yml`. This won't really do anything for you, so when you run the Docker image, you should supply your own `config.yml` file:
```bash
docker run -v $PWD/config.yml:/app/config.yml iamthefij/minitor
```
Images are provided for `amd64`, `arm`, and `arm64` architechtures, but the Python package should be compatible with anything that supports Python.
## Configuring
In this repo, you can explore the `sample-config.yml` file for an example, but the general structure is as follows. It should be noted that environment variable interpolation happens on load of the YAML file.
The global configurations are:
|key|value|
|---|---|
|`check_interval`|Maximum frequency to run checks for each monitor|
|`monitors`|List of all monitors. Detailed description below|
|`alerts`|List of all alerts. Detailed description below|
### Monitors
All monitors should be listed under `monitors`.
Each monitor allows the following configuration:
|key|value|
|---|---|
|`name`|Name of the monitor running. This will show up in messages and logs.|
|`command`|Specifies the command that should be executed, either in exec or shell form. This command's exit value will determine whether the check is successful|
|`alert_down`|A list of Alerts to be triggered when the monitor is in a "down" state|
|`alert_up`|A list of Alerts to be triggered when the monitor moves to an "up" state|
|`check_interval`|The interval at which this monitor should be checked. This must be greater than the global `check_interval` value|
|`alert_after`|Allows specifying the number of failed checks before an alert should be triggered|
|`alert_every`|Allows specifying how often an alert should be retriggered. There are a few magic numbers here. Defaults to `-1` for an exponential backoff. Setting to `0` disables re-alerting. Positive values will allow retriggering after the specified number of checks|
### Alerts
Alerts exist as objects keyed under `alerts`. Their key should be the name of the Alert. This is used in your monitor setup in `alert_down` and `alert_up`.
Eachy alert allows the following configuration:
|key|value|
|---|---|
|`command`|Specifies the command that should be executed, either in exec or shell form. This is the command that will be run when the alert is executed. This can be templated with environment variables or the variables shown in the table below|
Also, when alerts are executed, they will be passed through Python's format function with arguments for some attributes of the Monitor. The following monitor specific variables can be referenced using Python formatting syntax:
|token|value|
|---|---|
|`{alert_count}`|Number of times this monitor has alerted|
|`{alert_message}`|The exception message that was raised|
|`{failure_count}`|The total number of sequential failed checks for this monitor|
|`{last_output}`|The last returned value from the check command to either stderr or stdout|
|`{last_success}`|The ISO datetime of the last successful check|
|`{monitor_name}`|The name of the monitor that failed and triggered the alert|
### Metrics
As of v0.3.0, Minitor supports exporting metrics for [Prometheus](https://prometheus.io/). Prometheus is an open source tool for reading and querying metrics from different sources. Combined with another tool, [Grafana](https://grafana.com/), it allows building of charts and dashboards. You could also opt to just use Minitor to log check results, and instead do your alerting with Grafana.
It is also possible to use the metrics endpoint for monitoring Minitor itself! This allows setting up multiple instances of Minitor on different servers and have them monitor each-other so that you can detect a minitor outage.
To run minitor with metrics, use the `--metrics` (or `-m`) flag. The metrics will be served on port `8080` by default, though it can be overriden using `--metrics-port` (or `-p`)
```bash
minitor --metrics
# or
minitor --metrics --metrics-port 3000
```
## Contributing ## Contributing
Whether you're looking to submit a patch or just tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository. Whether you're looking to submit a patch or just tell me I broke something, you can contribute through the Github mirror and I can merge PRs back to the source repository.
Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git Primary Repo: https://git.iamthefij.com/iamthefij/minitor.git
Github Mirror: https://github.com/IamTheFij/minitor.git Github Mirror: https://github.com/IamTheFij/minitor.git

View File

@ -1,7 +0,0 @@
# Minitor Examples
A few examples of advanced Minitor configuration using Docker Compose.
These are all built against the local version and not Docker Hub. Before you try anything here, run `make docker-build` from the root `minitor` directory so that the `minitor-dev` image is built on your local host.
If you are trying to deploy these configs and you would like to use the public image instead, change `minitor-dev` to `iamthefij/minitor`.

View File

@ -1,7 +0,0 @@
# Docker Checks
A sample docker-compose example that uses the bundled shell scripts to monitor the health of other Docker containers.
## Security note
Exposing `/var/run/docker.sock` comes at a risk. Please be careful when doing this. If someone is able to take over your Minitor container, they will then essentially have root access to your whole host. To minimize risk, be wary of exposing Minitor to the public internet when using a configuration like this.

View File

@ -1,17 +0,0 @@
version: '2'
services:
minitor:
build: ../..
volumes:
- ./config.yml:/app/config.yml
- /var/run/docker.sock:/var/run/docker.sock:ro
failure:
container_name: failure
image: busybox
command: "sh -c 'exit 1'"
success:
container_name: success
image: busybox
command: "sh -c 'exit 0'"

View File

@ -1,3 +0,0 @@
# Example Prometheus Exporter
An example configuration exporting Minitor stats to Prometheus

View File

@ -1,33 +0,0 @@
version: '2'
services:
prom:
image: prom/prometheus
ports:
- "9090:9090"
volumes:
- ./prometheus.yml:/etc/prometheus/prometheus.yml
grafana:
image: grafana/grafana
ports:
- "8000:3000"
volumes:
- /var/lib/grafana
cadvisor:
image: google/cadvisor
ports:
- "8088:8080"
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
minitor:
build: ../../
ports:
- "8080:8080"
command: [ "--metrics", "-v" ]
volumes:
- ../../config.yml:/app/config.yml

View File

@ -1,16 +0,0 @@
---
global:
scrape_interval: 30s
scrape_configs:
- job_name: prometheus
scrape_interval: 5s
static_configs:
- targets: ['localhost:9090']
- job_name: cadvisor
static_configs:
- targets: ['cadvisor:8080']
- job_name: minitor
metrics_path: /
static_configs:
- targets: ['minitor:8080']

View File

@ -1,14 +0,0 @@
#! /bin/bash
HOST_ARCH=x86_64
VERSION=v2.9.1-1
mkdir -p build
cd build
# Multiple args can be passed in, but in most cases (Makefile and .drone.yml) we only use one at a time
for target_arch in $*; do
wget https://github.com/multiarch/qemu-user-static/releases/download/$VERSION/${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
tar -xvf ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
rm ${HOST_ARCH}_qemu-${target_arch}-static.tar.gz
done

View File

@ -1,25 +0,0 @@
image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
-
image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
platform:
architecture: amd64
os: linux
-
image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
platform:
architecture: arm64
os: linux
variant: v8
-
image: iamthefij/minitor:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
platform:
architecture: arm
os: linux
variant: v7

View File

@ -3,27 +3,23 @@ import subprocess
import sys import sys
from argparse import ArgumentParser from argparse import ArgumentParser
from datetime import datetime from datetime import datetime
from itertools import chain
from subprocess import CalledProcessError from subprocess import CalledProcessError
from subprocess import check_output from subprocess import check_output
from time import sleep from time import sleep
import yamlenv import yamlenv
from prometheus_client import Counter
from prometheus_client import Gauge
from prometheus_client import start_http_server
DEFAULT_METRICS_PORT = 8080
logging.basicConfig( logging.basicConfig(
level=logging.ERROR, format="%(asctime)s %(levelname)s %(name)s %(message)s" level=logging.INFO,
format='%(asctime)s %(levelname)s %(name)s %(message)s'
) )
logging.getLogger(__name__).addHandler(logging.NullHandler()) logging.getLogger(__name__).addHandler(logging.NullHandler())
def read_yaml(path): def read_yaml(path):
"""Loads config from a YAML file with env interpolation""" """Loads config from a YAML file with env interpolation"""
with open(path, "r") as yaml: with open(path, 'r') as yaml:
contents = yaml.read() contents = yaml.read()
return yamlenv.load(contents) return yamlenv.load(contents)
@ -34,40 +30,45 @@ def validate_monitor_settings(settings):
Note: Cannot yet validate the Alerts exist from within this class. Note: Cannot yet validate the Alerts exist from within this class.
That will be done by Minitor later That will be done by Minitor later
""" """
name = settings.get("name") name = settings.get('name')
if not name: if not name:
raise InvalidMonitorException("Invalid name for monitor") raise InvalidMonitorException('Invalid name for monitor')
if not settings.get("command"): if not settings.get('command'):
raise InvalidMonitorException("Invalid command for monitor {}".format(name)) raise InvalidMonitorException(
'Invalid command for monitor {}'.format(name)
)
type_assertions = ( type_assertions = (
("check_interval", int), ('check_interval', int),
("alert_after", int), ('alert_after', int),
("alert_every", int), ('alert_every', int),
) )
for key, val_type in type_assertions: for key, val_type in type_assertions:
val = settings.get(key) val = settings.get(key)
if not isinstance(val, val_type): if not isinstance(val, val_type):
raise InvalidMonitorException( raise InvalidMonitorException(
"Invalid type on {}: {}. Expected {} and found {}".format( 'Invalid type on {}: {}. Expected {} and found {}'.format(
name, key, val_type.__name__, type(val).__name__ name, key, val_type.__name__, type(val).__name__
) )
) )
non_zero = ( non_zero = (
"check_interval", 'check_interval',
"alert_after", 'alert_after',
'alert_every',
) )
for key in non_zero: for key in non_zero:
if settings.get(key) == 0: if settings.get(key) == 0:
raise InvalidMonitorException( raise InvalidMonitorException(
"Invalid value for {}: {}. Value cannot be 0".format(name, key) 'Invalid value for {}: {}. Value cannot be 0'.format(
name, key
)
) )
def maybe_decode(bstr, encoding="utf-8"): def maybe_decode(bstr, encoding='utf-8'):
try: try:
return bstr.decode(encoding) return bstr.decode(encoding)
except TypeError: except TypeError:
@ -77,14 +78,14 @@ def maybe_decode(bstr, encoding="utf-8"):
def call_output(*popenargs, **kwargs): def call_output(*popenargs, **kwargs):
"""Similar to check_output, but instead returns output and exception""" """Similar to check_output, but instead returns output and exception"""
# So we can capture complete output, redirect sderr to stdout # So we can capture complete output, redirect sderr to stdout
kwargs.setdefault("stderr", subprocess.STDOUT) kwargs.setdefault('stderr', subprocess.STDOUT)
output, ex = None, None output, ex = None, None
try: try:
output = check_output(*popenargs, **kwargs) output = check_output(*popenargs, **kwargs)
except CalledProcessError as e: except CalledProcessError as e:
output, ex = e.output, e output, ex = e.output, e
output = output.rstrip(b"\n") output = output.rstrip(b'\n')
return output, ex return output, ex
@ -97,58 +98,37 @@ class InvalidMonitorException(Exception):
class MinitorAlert(Exception): class MinitorAlert(Exception):
def __init__(self, message, monitor): pass
super().__init__(message)
self.monitor = monitor
class Monitor(object): class Monitor(object):
"""Primary configuration item for Minitor""" """Primary configuration item for Minitor"""
def __init__(self, config, counter=None, logger=None): def __init__(self, config):
"""Accepts a dictionary of configuration items to override defaults""" """Accepts a dictionary of configuration items to override defaults"""
settings = { settings = {
"alerts": ["log"], 'alerts': ['log'],
"check_interval": 30, 'check_interval': 30,
"alert_after": 4, 'alert_after': 4,
"alert_every": -1, 'alert_every': -1,
} }
settings.update(config) settings.update(config)
validate_monitor_settings(settings) validate_monitor_settings(settings)
self.name = settings["name"] self.name = settings['name']
self.command = settings["command"] self.command = settings['command']
self.alert_down = settings.get("alert_down", []) self.alerts = settings.get('alerts', [])
if not self.alert_down: self.check_interval = settings.get('check_interval')
self.alert_down = settings.get("alerts", []) self.alert_after = settings.get('alert_after')
self.alert_up = settings.get("alert_up", []) self.alert_every = settings.get('alert_every')
self.check_interval = settings.get("check_interval")
self.alert_after = settings.get("alert_after")
self.alert_every = settings.get("alert_every")
self.alert_count = 0
self.last_check = None self.last_check = None
self.last_output = None
self.last_success = None
self.total_failure_count = 0 self.total_failure_count = 0
self.alert_count = 0
self._counter = counter self.logger = logging.getLogger(
if logger is None: '{}({})'.format(self.__class__.__name__, self.name)
self._logger = logging.getLogger(
"{}({})".format(self.__class__.__name__, self.name)
) )
else:
self._logger = logger.getChild(
"{}({})".format(self.__class__.__name__, self.name)
)
def _count_check(self, is_success=True, is_alert=False):
if self._counter is not None:
self._counter.labels(
monitor=self.name,
status=("success" if is_success else "failure"),
is_alert=is_alert,
).inc()
def should_check(self): def should_check(self):
"""Determines if this Monitor should run it's check command""" """Determines if this Monitor should run it's check command"""
@ -169,39 +149,20 @@ class Monitor(object):
self.command, self.command,
shell=isinstance(self.command, str), shell=isinstance(self.command, str),
) )
output = maybe_decode(output) self.logger.debug(output)
self._logger.debug(output)
self.last_check = datetime.now() self.last_check = datetime.now()
self.last_output = output
is_success = None
try:
if ex is None: if ex is None:
is_success = True
self.success() self.success()
return True
else: else:
is_success = False
self.failure() self.failure()
except MinitorAlert: return False
self._count_check(is_success=is_success, is_alert=True)
raise
self._count_check(is_success=is_success)
return is_success
def success(self): def success(self):
"""Handles success tasks""" """Handles success tasks"""
back_up = None
if not self.is_up():
back_up = MinitorAlert(
"{} check is up again!".format(self.name),
self,
)
self.total_failure_count = 0 self.total_failure_count = 0
self.alert_count = 0 self.alert_count = 0
self.last_success = datetime.now()
if back_up:
raise back_up
def failure(self): def failure(self):
"""Handles failure tasks and possibly raises MinitorAlert""" """Handles failure tasks and possibly raises MinitorAlert"""
@ -210,55 +171,31 @@ class Monitor(object):
if self.total_failure_count < self.alert_after: if self.total_failure_count < self.alert_after:
return return
failure_count = self.total_failure_count - self.alert_after failure_count = (self.total_failure_count - self.alert_after)
if self.alert_every > 0: if self.alert_every > 0:
# Otherwise, we should check against our alert_every # Otherwise, we should check against our alert_every
should_alert = (failure_count % self.alert_every) == 0 should_alert = (failure_count % self.alert_every) == 0
elif self.alert_every == 0:
# Only alert on the first failure
should_alert = failure_count == 1
else: else:
should_alert = failure_count >= (2**self.alert_count) - 1 should_alert = (failure_count >= (2 ** self.alert_count) - 1)
if should_alert: if should_alert:
self.alert_count += 1 self.alert_count += 1
raise MinitorAlert( raise MinitorAlert('{} check has failed {} times'.format(
"{} check has failed {} times".format(
self.name, self.total_failure_count self.name, self.total_failure_count
), ))
self,
)
def is_up(self):
"""Indicates if the monitor is already alerting failures"""
return self.alert_count == 0
class Alert(object): class Alert(object):
def __init__(self, name, config, counter=None, logger=None): def __init__(self, name, config):
"""An alert must be named and have a config dict""" """An alert must be named and have a config dict"""
self.name = name self.name = name
self.command = config.get("command") self.command = config.get('command')
if not self.command: if not self.command:
raise InvalidAlertException("Invalid alert {}".format(self.name)) raise InvalidAlertException('Invalid alert {}'.format(self.name))
self._counter = counter self.logger = logging.getLogger(
if logger is None: '{}({})'.format(self.__class__.__name__, self.name)
self._logger = logging.getLogger(
"{}({})".format(self.__class__.__name__, self.name)
) )
else:
self._logger = logger.getChild(
"{}({})".format(self.__class__.__name__, self.name)
)
def _count_alert(self, monitor):
"""Increments the alert counter"""
if self._counter is not None:
self._counter.labels(
alert=self.name,
monitor=monitor,
).inc()
def _formated_command(self, **kwargs): def _formated_command(self, **kwargs):
"""Formats command array or string with kwargs from Monitor""" """Formats command array or string with kwargs from Monitor"""
@ -269,27 +206,17 @@ class Alert(object):
args.append(arg.format(**kwargs)) args.append(arg.format(**kwargs))
return args return args
def _format_datetime(self, dt): def alert(self, monitor):
"""Formats a datetime for an alert"""
if dt is None:
return "Never"
return dt.isoformat()
def alert(self, message, monitor):
"""Calls the alert command for the provided monitor""" """Calls the alert command for the provided monitor"""
self._count_alert(monitor.name)
output, ex = call_output( output, ex = call_output(
self._formated_command( self._formated_command(
alert_count=monitor.alert_count, alert_count=monitor.alert_count,
alert_message=message,
failure_count=monitor.total_failure_count,
last_output=monitor.last_output,
last_success=self._format_datetime(monitor.last_success),
monitor_name=monitor.name, monitor_name=monitor.name,
failure_count=monitor.total_failure_count,
), ),
shell=isinstance(self.command, str), shell=isinstance(self.command, str),
) )
self._logger.error(maybe_decode(output)) self.logger.error(maybe_decode(output))
if ex is not None: if ex is not None:
raise ex raise ex
@ -301,180 +228,90 @@ class Minitor(object):
check_interval = None check_interval = None
def __init__(self): def __init__(self):
self._logger = logging.getLogger(self.__class__.__name__) self.logger = logging.getLogger(self.__class__.__name__)
self._alert_counter = None
self._monitor_counter = None
self._monitor_status_gauge = None
def _parse_args(self, args=None): def setup(self, config_path):
"""Parses command line arguments and returns them"""
parser = ArgumentParser(description="Minimal monitoring")
parser.add_argument(
"--config",
"-c",
dest="config_path",
default="config.yml",
help="Path to the config YAML file to use",
)
parser.add_argument(
"--metrics",
"-m",
dest="metrics",
action="store_true",
help="Start webserver with metrics",
)
parser.add_argument(
"--metrics-port",
"-p",
dest="metrics_port",
type=int,
default=DEFAULT_METRICS_PORT,
help="Port to use when serving metrics",
)
parser.add_argument(
"--verbose",
"-v",
action="count",
help=(
"Adjust log verbosity by increasing arg count. Default log",
"level is ERROR. Level increases with each `v`",
),
)
return parser.parse_args(args)
def _setup(self, config_path):
"""Load all setup from YAML file at provided path""" """Load all setup from YAML file at provided path"""
config = read_yaml(config_path) config = read_yaml(config_path)
self.check_interval = config.get("check_interval", 30) self.check_interval = config.get('check_interval', 30)
self.monitors = [ self.monitors = [Monitor(mon) for mon in config.get('monitors', [])]
Monitor(
mon,
counter=self._monitor_counter,
logger=self._logger,
)
for mon in config.get("monitors", [])
]
# Add default alert for logging # Add default alert for logging
self.alerts = { self.alerts = {
"log": Alert( 'log': Alert(
"log", 'log',
{"command": ["echo", "{alert_message}!"]}, {'command': ['echo', '{monitor_name} has failed!']}
counter=self._alert_counter,
logger=self._logger,
) )
} }
self.alerts.update( self.alerts.update({
{ alert_name: Alert(alert_name, alert)
alert_name: Alert( for alert_name, alert in config.get('alerts', {}).items()
alert_name, })
alert,
counter=self._alert_counter,
logger=self._logger,
)
for alert_name, alert in config.get("alerts", {}).items()
}
)
def _validate_monitors(self): def validate_monitors(self):
"""Validates monitors are valid against other config values""" """Validates monitors are valid against other config values"""
for monitor in self.monitors: for monitor in self.monitors:
# Validate that the interval is valid # Validate that the interval is valid
if monitor.check_interval < self.check_interval: if monitor.check_interval < self.check_interval:
raise InvalidMonitorException( raise InvalidMonitorException(
"Monitor {} check interval is lower global value {}".format( 'Monitor {} check interval is lower global value {}'.format(
monitor.name, self.check_interval monitor.name, self.check_interval
) )
) )
# Validate that the the alerts for the monitor exist # Validate that the the alerts for the monitor exist
for alert in chain(monitor.alert_down, monitor.alert_up): for alert in monitor.alerts:
if alert not in self.alerts: if alert not in self.alerts:
raise InvalidMonitorException( raise InvalidMonitorException(
"Monitor {} contains an unknown alert: {}".format( 'Monitor {} contains an unknown alert: {}'.format(
monitor.name, alert monitor.name, alert
) )
) )
def _init_metrics(self): def alert_for_monitor(self, monitor):
self._alert_counter = Counter( """Issues all alerts for a provided monitor"""
"minitor_alert_total", for alert in monitor.alerts:
"Number of Minitor alerts", self.alerts[alert].alert(monitor)
["alert", "monitor"],
) def parse_args(self):
self._monitor_counter = Counter( """Parses command line arguments and returns them"""
"minitor_check_total", parser = ArgumentParser(description='Minimal monitoring')
"Number of Minitor checks", parser.add_argument(
["monitor", "status", "is_alert"], '--config', '-c',
) dest='config_path',
self._monitor_status_gauge = Gauge( default='config.yml',
"minitor_monitor_up_count", help='Path to the config YAML file to use',
"Currently responsive monitors", )
["monitor"], return parser.parse_args()
)
def run(self):
"""Runs Minitor in a loop"""
args = self.parse_args()
self.setup(args.config_path)
self.validate_monitors()
def _loop(self):
while True: while True:
self._check()
sleep(self.check_interval)
def _check(self):
"""The main run loop"""
for monitor in self.monitors: for monitor in self.monitors:
try: try:
result = monitor.check() result = monitor.check()
if result is not None: if result is not None:
self._logger.info( self.logger.info(
"%s: %s", monitor.name, "SUCCESS" if result else "FAILURE" '%s: %s',
monitor.name,
'SUCCESS' if result else 'FAILURE'
) )
except MinitorAlert as minitor_alert: except MinitorAlert as minitor_alert:
self._logger.warning(minitor_alert) self.logger.warn(minitor_alert)
self._handle_minitor_alert(minitor_alert) self.alert_for_monitor(monitor)
# Track the status of the Monitor sleep(self.check_interval)
if self._monitor_status_gauge:
self._monitor_status_gauge.labels(
monitor=monitor.name,
).set(int(monitor.is_up()))
def _handle_minitor_alert(self, minitor_alert):
"""Issues all alerts for a provided monitor"""
monitor = minitor_alert.monitor
alerts = monitor.alert_up if monitor.is_up() else monitor.alert_down
for alert in alerts:
self.alerts[alert].alert(str(minitor_alert), monitor)
def _set_log_level(self, verbose):
"""Sets the log level for the class using the provided verbose count"""
if verbose == 1:
self._logger.setLevel(logging.WARNING)
elif verbose == 2:
self._logger.setLevel(logging.INFO)
elif verbose >= 3:
self._logger.setLevel(logging.DEBUG)
def run(self, args=None):
"""Runs Minitor in a loop"""
args = self._parse_args(args)
if args.verbose:
self._set_log_level(args.verbose)
if args.metrics:
self._init_metrics()
start_http_server(args.metrics_port)
self._setup(args.config_path)
self._validate_monitors()
self._loop()
def main(args=None): def main():
try: try:
Minitor().run(args) Minitor().run()
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
return 0 return 0
if __name__ == "__main__": if __name__ == '__main__':
sys.exit(main()) sys.exit(main())

View File

@ -2,17 +2,16 @@ check_interval: 30
monitors: monitors:
- name: My Website - name: My Website
command: [ 'curl', '-s', '-o', '/dev/null', 'https://minitor.mon' ] command: [ 'curl', 'https://minitor.mon' ]
alert_down: [ log, mailgun_down, sms_down ] alerts: [ log, email, sms ]
alert_up: [ log, email_up ] check_interval: 30
check_interval: 30 # Must be at minimum the global `check_interval`
alert_after: 3 alert_after: 3
alert_every: -1 # Defaults to -1 for exponential backoff. 0 to disable repeating alert_every: -1 # Defaults to -1 for exponential backoff
alerts: alerts:
email_up: email:
command: [ sendmail, "me@minitor.mon", "Recovered: {monitor_name}", "We're back!" ] command: [ sendmail, "me@minitor.mon", "Failure: {monitor_name}", "This thing failed!" ]
mailgun_down: mailgun:
command: > command: >
curl -s -X POST curl -s -X POST
-F subject="Alert! {monitor_name} failed" -F subject="Alert! {monitor_name} failed"
@ -21,9 +20,15 @@ alerts:
-F text="Our monitor failed" -F text="Our monitor failed"
https://api.mailgun.net/v3/minitor.mon/messages https://api.mailgun.net/v3/minitor.mon/messages
-u "api:${MAILGUN_API_KEY}" -u "api:${MAILGUN_API_KEY}"
sms_down: sms:
command: > command: >
curl -s -X POST -F "Body=Failure! {monitor_name} has failed" curl -s -X POST -F "Body=Failure: {monitor_name} has failed"
-F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}" -F "From=${AVAILABLE_NUMBER}" -F "To=${MY_PHONE}"
"https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages" "https://api.twilio.com/2010-04-01/Accounts/${ACCOUNT_SID}/Messages"
-u "${ACCOUNT_SID}:${AUTH_TOKEN}" -u "${ACCOUNT_SID}:${AUTH_TOKEN}"
# federation:
# - location: https://host1.com
# client_key: keyfromhost1
# server_key: keyhost1uses
# alerts: [ sms ]

View File

@ -1,5 +0,0 @@
# Minitor Scripts
A collection of some handy scripts to use with Minitor
These are not included with the Python package, but they are included in the Docker image in `/app/scripts`.

View File

@ -1,51 +0,0 @@
#! /bin/bash
set -e
#################
# docker_check.sh
#
# Checks the most recent state exit code of a Docker container
#################
# Docker host will default to a socket
# To override, export DOCKER_HOST to a new hostname
DOCKER_HOST="${DOCKER_HOST:=socket}"
container_name="$1"
# Curls Docker either using a socket or URL
function curl_docker {
local path="$1"
if [ "$DOCKER_HOST" == "socket" ]; then
curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
else
curl "http://${DOCKER_HOST}/$path" 2>/dev/null
fi
}
# Returns caintainer ID for a given container name
function get_container_id {
local container_name="$1"
curl_docker 'containers/json?all=1' \
| jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
}
# Returns container JSON
function inspect_container {
local container_id=$1
curl_docker "containers/$container_id/json"
}
if [ -z "$container_name" ]; then
echo "Usage: $0 container_name"
echo "Will exit with the last status code of continer with provided name"
exit 1
fi
container_id=$(get_container_id $container_name)
if [ -z "$container_id" ]; then
echo "ERROR: Could not find container with name: $container_name"
exit 1
fi
exit_code=$(inspect_container "$container_id" | jq -r .State.ExitCode)
exit "$exit_code"

View File

@ -1,61 +0,0 @@
#! /bin/bash
set -e
#################
# docker_healthcheck.sh
#
# Returns the results of a Docker Healthcheck for a container
#################
# Docker host will default to a socket
# To override, export DOCKER_HOST to a new hostname
DOCKER_HOST="${DOCKER_HOST:=socket}"
container_name="$1"
# Curls Docker either using a socket or URL
function curl_docker {
local path="$1"
if [ "$DOCKER_HOST" == "socket" ]; then
curl --unix-socket /var/run/docker.sock "http://localhost/$path" 2>/dev/null
else
curl "http://${DOCKER_HOST}/$path" 2>/dev/null
fi
}
# Returns caintainer ID for a given container name
function get_container_id {
local container_name="$1"
curl_docker 'containers/json?all=1' \
| jq -r ".[] | {Id, Name: .Names[]} | select(.Name == \"/${container_name}\") | .Id"
}
# Returns container JSON
function inspect_container {
local container_id="$1"
curl_docker "containers/$container_id/json"
}
if [ -z "$container_name" ]; then
echo "Usage: $0 container_name"
echo "Will return results of healthcheck for continer with provided name"
exit 1
fi
container_id=$(get_container_id "$container_name")
if [ -z "$container_id" ]; then
echo "ERROR: Could not find container with name: $container_name"
exit 1
fi
health=$(inspect_container "$container_id" | jq -r '.State.Health.Status')
case "$health" in
null)
echo "No healthcheck results"
;;
starting|healthy)
echo "Status: '$health'"
;;
*)
echo "Status: '$health'"
exit 1
esac

View File

@ -7,49 +7,42 @@ from setuptools import setup
here = path.abspath(path.dirname(__file__)) here = path.abspath(path.dirname(__file__))
# Get the long description from the README file # Get the long description from the README file
with open(path.join(here, "README.md"), encoding="utf-8") as f: with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read() long_description = f.read()
setup( setup(
name="minitor", name='minitor',
version="1.0.3", version='0.1.2',
description="A minimal monitoring tool", description='A minimal monitoring tool',
long_description=long_description, long_description=long_description,
long_description_content_type="text/markdown", url='https://git.iamthefij.com/iamthefij/minitor',
url="https://git.iamthefij.com/iamthefij/minitor", download_url=(
download_url=("https://git.iamthefij.com/iamthefij/minitor/archive/master.tar.gz"), 'https://git.iamthefij.com/iamthefij/minitor/archive/master.tar.gz'
author="Ian Fijolek",
author_email="ian@iamthefij.com",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"Topic :: System :: Monitoring",
"License :: OSI Approved :: Apache Software License",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3.9",
"Programming Language :: Python :: 3.10",
],
keywords="minitor monitoring alerting",
packages=find_packages(
exclude=[
"contrib",
"docs",
"examples",
"scripts",
"tests",
]
), ),
author='Ian Fijolek',
author_email='ian@iamthefij.com',
classifiers=[
# How mature is this project? Common values are
# 4 - Beta
# 5 - Production/Stable
'Development Status :: 4 - Beta',
'Intended Audience :: Developers',
'Intended Audience :: System Administrators',
'Topic :: System :: Monitoring',
'License :: OSI Approved :: Apache Software License',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
],
keywords='minitor monitoring alerting',
packages=find_packages(exclude=['contrib', 'docs', 'tests']),
install_requires=[ install_requires=[
"prometheus_client", 'yamlenv',
"yamlenv",
], ],
entry_points={ entry_points={
"console_scripts": [ 'console_scripts': [
"minitor=minitor.main:main", 'minitor=minitor.main:main',
], ],
}, },
) )

View File

View File

@ -1,55 +1,40 @@
from datetime import datetime
from unittest.mock import patch from unittest.mock import patch
import pytest import pytest
from minitor.main import Alert from minitor.main import Alert
from minitor.main import Monitor from minitor.main import Monitor
from tests.util import assert_called_once_with
class TestAlert(object): class TestAlert(object):
@pytest.fixture @pytest.fixture
def monitor(self): def monitor(self):
return Monitor( return Monitor({
{ 'name': 'Dummy Monitor',
"name": "Dummy Monitor", 'command': ['echo', 'foo'],
"command": ["echo", "foo"], })
}
)
@pytest.fixture @pytest.fixture
def echo_alert(self): def echo_alert(self):
return Alert( return Alert(
"log", 'log',
{ {
"command": [ 'command': [
"echo", 'echo', (
( '{monitor_name} has failed {failure_count} time(s)!\n'
"{monitor_name} has failed {failure_count} time(s)!\n" 'We have alerted {alert_count} time(s)'
"We have alerted {alert_count} time(s)\n" )
"Last success was {last_success}\n"
"Last output was: {last_output}"
),
] ]
}, }
) )
@pytest.mark.parametrize( def test_simple_alert(self, monitor, echo_alert):
"last_success,expected_success",
[(None, "Never"), (datetime(2018, 4, 10), "2018-04-10T00:00:00")],
)
def test_simple_alert(self, monitor, echo_alert, last_success, expected_success):
monitor.alert_count = 1
monitor.last_output = "beep boop"
monitor.last_success = last_success
monitor.total_failure_count = 1 monitor.total_failure_count = 1
with patch.object(echo_alert._logger, "error") as mock_error: monitor.alert_count = 1
echo_alert.alert("Exception message", monitor) with patch.object(echo_alert.logger, 'error') as mock_error:
assert_called_once_with( echo_alert.alert(monitor)
mock_error, mock_error.assert_called_once_with(
"Dummy Monitor has failed 1 time(s)!\n" 'Dummy Monitor has failed 1 time(s)!\n'
"We have alerted 1 time(s)\n" 'We have alerted 1 time(s)'
"Last success was " + expected_success + "\n"
"Last output was: beep boop",
) )

View File

@ -1,37 +1,14 @@
import os
from unittest.mock import patch
from minitor.main import call_output from minitor.main import call_output
from minitor.main import Minitor
class TestMinitor(object): class TestMinitor(object):
def test_call_output(self): def test_call_output(self):
# valid command should have result and no exception # valid command should have result and no exception
output, ex = call_output(["echo", "test"]) output, ex = call_output(['echo', 'test'])
assert output == b"test" assert output == b'test'
assert ex is None assert ex is None
output, ex = call_output(["ls", "--not-a-real-flag"]) output, ex = call_output(['ls', '--not-a-real-flag'])
assert output.startswith(b"ls: ") assert output.startswith(b'ls: ')
assert ex is not None assert ex is not None
def test_run(self):
"""Doesn't really check much, but a simple integration sanity test"""
test_loop_count = 5
os.environ.update(
{
"MAILGUN_API_KEY": "test-mg-key",
"AVAILABLE_NUMBER": "555-555-5050",
"MY_PHONE": "555-555-0505",
"ACCOUNT_SID": "test-account-id",
"AUTH_TOKEN": "test-account-token",
}
)
args = "--config ./sample-config.yml".split(" ")
minitor = Minitor()
with patch.object(minitor, "_loop"):
minitor.run(args)
# Skip the loop, but run a single check
for _ in range(test_loop_count):
minitor._check()

View File

@ -1,54 +1,41 @@
from datetime import datetime
from unittest.mock import patch
import pytest import pytest
from minitor.main import InvalidMonitorException from minitor.main import InvalidMonitorException
from minitor.main import MinitorAlert from minitor.main import MinitorAlert
from minitor.main import Monitor from minitor.main import Monitor
from minitor.main import validate_monitor_settings from minitor.main import validate_monitor_settings
from tests.util import assert_called_once
class TestMonitor(object): class TestMonitor(object):
@pytest.fixture @pytest.fixture
def monitor(self): def monitor(self):
return Monitor( return Monitor({
{ 'name': 'Sample Monitor',
"name": "Sample Monitor", 'command': ['echo', 'foo'],
"command": ["echo", "foo"], })
"alert_down": ["log"],
"alert_up": ["log"],
"check_interval": 1,
"alert_after": 1,
"alert_every": 1,
}
)
@pytest.mark.parametrize( @pytest.mark.parametrize('settings', [
"settings", {'alert_after': 0},
[ {'alert_every': 0},
{"alert_after": 0}, {'check_interval': 0},
{"alert_every": 0}, {'alert_after': 'invalid'},
{"check_interval": 0}, {'alert_every': 'invalid'},
{"alert_after": "invalid"}, {'check_interval': 'invalid'},
{"alert_every": "invalid"}, ])
{"check_interval": "invalid"},
],
)
def test_monitor_invalid_configuration(self, settings): def test_monitor_invalid_configuration(self, settings):
with pytest.raises(InvalidMonitorException): with pytest.raises(InvalidMonitorException):
validate_monitor_settings(settings) validate_monitor_settings(settings)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"alert_after", 'alert_after',
[1, 20], [1, 20],
ids=lambda arg: "alert_after({})".format(arg), ids=lambda arg: 'alert_after({})'.format(arg),
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"alert_every", 'alert_every',
[-1, 1, 2, 1440], [-1, 1, 2, 1440],
ids=lambda arg: "alert_every({})".format(arg), ids=lambda arg: 'alert_every({})'.format(arg),
) )
def test_monitor_alert_after(self, monitor, alert_after, alert_every): def test_monitor_alert_after(self, monitor, alert_after, alert_every):
monitor.alert_after = alert_after monitor.alert_after = alert_after
@ -63,14 +50,14 @@ class TestMonitor(object):
monitor.failure() monitor.failure()
@pytest.mark.parametrize( @pytest.mark.parametrize(
"alert_after", 'alert_after',
[1, 20], [1, 20],
ids=lambda arg: "alert_after({})".format(arg), ids=lambda arg: 'alert_after({})'.format(arg),
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"alert_every", 'alert_every',
[1, 2, 1440], [1, 2, 1440],
ids=lambda arg: "alert_every({})".format(arg), ids=lambda arg: 'alert_every({})'.format(arg),
) )
def test_monitor_alert_every(self, monitor, alert_after, alert_every): def test_monitor_alert_every(self, monitor, alert_after, alert_every):
monitor.alert_after = alert_after monitor.alert_after = alert_after
@ -105,46 +92,3 @@ class TestMonitor(object):
monitor.failure() monitor.failure()
else: else:
monitor.failure() monitor.failure()
@pytest.mark.parametrize("last_check", [None, datetime(2018, 4, 10)])
def test_monitor_should_check(self, monitor, last_check):
monitor.last_check = last_check
assert monitor.should_check()
def test_monitor_check_fail(self, monitor):
assert monitor.last_output is None
with patch.object(monitor, "failure") as mock_failure:
monitor.command = ["ls", "--not-real"]
assert not monitor.check()
assert_called_once(mock_failure)
assert monitor.last_output is not None
def test_monitor_check_success(self, monitor):
assert monitor.last_output is None
with patch.object(monitor, "success") as mock_success:
assert monitor.check()
assert_called_once(mock_success)
assert monitor.last_output is not None
@pytest.mark.parametrize("failure_count", [0, 1])
def test_monitor_success(self, monitor, failure_count):
monitor.alert_count = 0
monitor.total_failure_count = failure_count
assert monitor.last_success is None
monitor.success()
assert monitor.alert_count == 0
assert monitor.last_success is not None
assert monitor.total_failure_count == 0
def test_monitor_success_back_up(self, monitor):
monitor.total_failure_count = 1
monitor.alert_count = 1
with pytest.raises(MinitorAlert):
monitor.success()
assert monitor.alert_count == 0
assert monitor.last_success is not None
assert monitor.total_failure_count == 0

View File

@ -1,12 +0,0 @@
from unittest import mock
def assert_called_once(mocked):
"""Safe convenient methods for mock asserts"""
assert mocked.call_count == 1
def assert_called_once_with(mocked, *args, **kwargs):
"""Safe convenient methods for mock asserts"""
assert_called_once(mocked)
assert mocked.call_args == mock.call(*args, **kwargs)

View File

@ -1,5 +1,5 @@
[tox] [tox]
envlist = py3,pypy3 envlist = py3
[testenv] [testenv]
deps = deps =
@ -7,12 +7,10 @@ deps =
commands = commands =
coverage erase coverage erase
coverage run --source=minitor/ -m pytest --capture=no -vv {posargs:tests} coverage run --source=minitor/ -m pytest --capture=no -vv {posargs:tests}
coverage report -m --fail-under 70 coverage report -m --fail-under 50
pre-commit run --all-files pre-commit run --all-files
[testenv:pre-commit] [testenv:pre-commit]
deps =
pre-commit
commands = commands =
pre-commit {posargs} pre-commit {posargs}