iamthefij
/
email-assistant
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: iamthefij:master
Branches Tags
iamthefij:master
iamthefij:imbox
..
compare: iamthefij:imbox
Branches Tags
iamthefij:master
iamthefij:imbox

No commits in common. "master" and "imbox" have entirely different histories.
master ... imbox

23 changed files with 144 additions and 434 deletions
Show Stats Expand all files Collapse all files

2
.gitignore vendored
View File

@ -125,5 +125,3 @@ _testmain.go
.DS_Store
.env
docker-compose-prod.yml
.mypy_cache

39
.pre-commit-config.yaml
View File

@ -1,39 +0,0 @@
---
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.3.0
hooks:
- id: check-added-large-files
- id: check-yaml
args:
- --allow-multiple-documents
- id: check-json
- id: requirements-txt-fixer
- id: trailing-whitespace
- id: end-of-file-fixer
# Python
- id: debug-statements
- id: check-merge-conflict
- id: name-tests-test
# exclude: tests/(common.py|util.py|(helpers)/(.+).py)
- repo: https://github.com/psf/black
rev: 20.8b1
hooks:
- id: black
- repo: git://github.com/asottile/reorder_python_imports
rev: v2.3.6
hooks:
- id: reorder-python-imports
args:
- --py3-plus
- repo: git://github.com/dnephin/pre-commit-golang
rev: v0.3.5
hooks:
- id: go-fmt
- id: go-imports
- id: golangci-lint
# - repo: git://github.com/jumanjihouse/pre-commit-hooks
# rev: 1.11.0
# hooks:
# - id: rubocop
# - id: fasterer

21
Makefile
View File

@ -1,21 +0,0 @@
SUBDIRS := $(wildcard */.)
.PHONY: default test clean
default: check
.PHONY: all
all: $(SUBDIRS)
.PHONY: $(SUBDIRS)
$(SUBDIRS):
$(MAKE) -C $@
# Installs pre-commit hooks
.PHONY: install-hooks
install-hooks:
pre-commit install -f --install-hooks
# Checks files for encryption
.PHONY: check
check:
pre-commit run --all-files

2
crawler/Dockerfile
View File

@ -1,3 +1,3 @@
FROM python:3.6-onbuild
CMD python -m crawler.main -v
CMD python -m crawler.main

4
crawler/Makefile
View File

@ -1,4 +0,0 @@
.PHONY: default all test clean
default:
@echo ok

187
crawler/crawler/main.py
View File

@ -1,46 +1,35 @@
import logging
import os
from argparse import ArgumentParser
from datetime import date
from datetime import datetime
from datetime import timedelta
from imaplib import IMAP4
from getpass import getpass
from time import sleep
import email
import json
import os
import requests
from dateutil import parser
from dateutil.tz import tzutc
from imbox import Imbox
import requests
logging.basicConfig(
level=logging.WARNING, format="%(asctime)s %(levelname)s %(name)s %(message)s"
)
logging.getLogger(__name__).addHandler(logging.NullHandler())
VALID_CONTENT_TYPES = ["text/plain", "text/html"]
def get_message_subject(message):
"""Returns message subject or a placeholder text"""
return getattr(message, "subject", "NO SUBJECT")
VALID_CONTENT_TYPES = [ 'text/plain', 'text/html' ]
class MailCrawler(object):
parser_hosts = None
indexer_host = os.environ['INDEXER']
def __init__(self):
self._logger = logging.getLogger(self.__class__.__name__)
self.imap_url = os.environ["IMAP_URL"]
self.imap_user = os.environ["IMAP_USER"]
self.imap_pass = os.environ["IMAP_PASS"]
self.parser_hosts = None
self.indexer_host = os.environ.get("INDEXER")
self.debug_mode = os.environ.get("DEBUG", False)
self.imap_url = os.environ['IMAP_URL']
self.imap_user = os.environ['IMAP_USER']
self.imap_pass = os.environ['IMAP_PASS']
def get_parsers(self):
"""Retrieves a list of parser hosts"""
if self.parser_hosts is None:
self.parser_hosts = []
parser_format = "PARSER_{}"
parser_format = 'PARSER_{}'
parser_index = 1
parser_host = os.environ.get(parser_format.format(parser_index))
while parser_host is not None:
@ -54,21 +43,21 @@ class MailCrawler(object):
"""Parses tokens from an email message"""
text = self.get_email_text(message)
if not text:
print("No email text returned")
print('No email text returned')
return []
results = []
for parser_host in self.get_parsers():
# print('Parsing email text... ', text)
response = requests.post(
parser_host + "/parse",
parser_host+'/parse',
json={
"subject": get_message_subject(message),
"message": text,
'subject': message.subject,
'message': text,
},
)
response.raise_for_status()
print("Got response", response.text)
print('Got response', response.text)
results += response.json()
return results
@ -83,18 +72,16 @@ class MailCrawler(object):
def get_email_text(self, message):
"""Retrieves the text body of an email message"""
body = message.body.get("plain") or message.body.get("html")
body = message.body.get('plain') or message.body.get('html')
if not body:
return None
# Concat all known body content together since it doesn't really matter
return "".join([text for text in body if isinstance(text, str)])
return ''.join([text for text in body if isinstance(text, str)])
def index_token(self, message):
"""Sends a token from the parser to the indexer"""
if self.indexer_host is None and self.debug_mode:
print("DDB No indexer host, but OK for debugging")
response = requests.post(
self.indexer_host + "/token",
self.indexer_host+'/token',
json=message,
)
response.raise_for_status()
@ -103,115 +90,59 @@ class MailCrawler(object):
def process_message(self, message):
"""Process a single email message"""
for result in self.parse_message(message):
result.update(
{
"subject": message.subject,
}
)
print("Parsed result: ", result)
print("Indexed result: ", self.index_token(result))
result.update({
'subject': message.subject,
})
print('Parsed result: ', result)
print('Indexed result: ', self.index_token(result))
def process_messages(self, server, since_date, last_uid=0):
kwargs = {}
if last_uid > 0:
kwargs["uid__range"] = "{}:*".format(last_uid)
else:
kwargs["date__gt"] = since_date
self._logger.info("Mailbox search kwargs %s", kwargs)
for uid, message in server.messages(**kwargs):
def process_messages(self, server, since_date, last_message=0):
for uid, message in server.messages(date__gt=since_date):
uid = int(uid)
if uid <= last_uid:
self._logger.debug(
"DDB Already seen message with uid {}. Skipping".format(uid)
)
if uid <= last_message:
print('DDB Already seen message with uid {}. Skipping'.format(uid))
continue
self._logger.info(
"Processing message uid %s message_id %s with subject '%s'",
uid,
message.message_id,
get_message_subject(message),
print(
'Processing message uid {} message_id {} '
'with subject "{}"'.format(
uid, message.message_id, message.subject
)
)
self.process_message(message)
# Update since_date
message_date = parser.parse(message.date)
self._logger.debug(
"DDB Processed message. Message date: %s Old date: %s",
message_date,
since_date,
)
try:
since_date = max(since_date, message_date)
except TypeError:
self._logger.error("Error comparing dates. We'll just use the last one")
self._logger.debug("DDB Since date is now %s", since_date)
last_uid = max(uid, last_uid)
print('DDB Processed message. Message date: {} Old date: {}'.format(
message_date, since_date
))
since_date = max(since_date, message_date)
print('DDB Since date is now ', since_date)
last_message = max(uid, last_message)
return since_date, last_uid
return since_date, last_message
def _parse_args(self, args=None):
"""Parses command line arguments and returns them"""
parser = ArgumentParser(description="Inbox crawler")
parser.add_argument(
"--sleep",
"-s",
default=10 * 60,
help=(
"Number of seconds to wait between polling IMAP server."
"Default 10 min"
),
)
parser.add_argument(
"--verbosity",
"-v",
action="count",
help=(
"Adjust log verbosity by increasing arg count. Default log",
"level is ERROR. Level increases with each `v`",
),
)
return parser.parse_args(args)
def _set_log_level(self, verbosity):
"""Sets the log level for the class using the provided verbosity count"""
if verbosity == 1:
# Set this logger to info
self._logger.setLevel(logging.INFO)
elif verbosity == 2:
# Set this logger to debug and root to info
logging.getLogger().setLevel(logging.INFO)
self._logger.setLevel(logging.DEBUG)
elif verbosity >= 3:
# Set the root logger to debug
logging.getLogger().setLevel(logging.DEBUG)
def run(self, args=None):
args = self._parse_args(args=args)
if args.verbosity:
self._set_log_level(args.verbosity)
self._logger.info("Starting crawler")
def run(self):
print('Starting crawler')
# TODO: Put server into some kind of context manager and property
with self.get_server() as server:
# TODO: parameterize startup date, maybe relative
since_date = datetime.now(tzutc()) - timedelta(days=16)
last_uid = 0
since_date = datetime.now(tzutc()) - timedelta(days=1)
last_message = 0
while True:
print("Processing messages")
since_date, last_uid = self.process_messages(
server, since_date, last_uid=last_uid
)
self._logger.info(
"DDB Processed all. New since_date %s",
print('Lets process')
since_date, last_message = self.process_messages(
server,
since_date,
last_message=last_message
)
sleep(args.sleep)
print('DDB Processed all. New since_date', since_date)
# TODO: parameterize sleep
# Sleep for 10 min
sleep(10 * 60)
if __name__ == "__main__":
while True:
try:
MailCrawler().run()
except IMAP4.abort:
print("Imap abort. We will try to reconnect")
pass
if __name__ == '__main__':
MailCrawler().run()

2
crawler/docker-compose.yml
View File

@ -8,5 +8,3 @@ services:
IMAP_URL: my.iamthefij.com
IMAP_USER: iamthefij@iamthefij.com
IMAP_PASS: "${IMAP_PASS}"
INDEXER: http://indexer:5000
PARSER_1: http://parser_package_tracking:3000

2
crawler/requirements.txt
View File

@ -1,3 +1,3 @@
imbox
python-dateutil
imbox
requests

4
indexer/Makefile
View File

@ -1,4 +0,0 @@
.PHONY: default all test clean
default:
@echo ok

96
indexer/indexer/main.py
View File

@ -2,25 +2,24 @@ import json
import os
import sys
import flask
from flask import jsonify
from flask import request
from flask.ext.sqlalchemy import SQLAlchemy
import flask
app = flask.Flask(__name__)
app.config["SQLALCHEMY_DATABASE_URI"] = os.environ.get(
"SQLALCHEMY_DATABASE_URI", "sqlite:///../tokens.db"
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get(
'SQLALCHEMY_DATABASE_URI',
'sqlite:///../tokens.db'
)
app.config["SQLALCHEMY_ECHO"] = True
app.config["DEBUG"] = True
app.config['SQLALCHEMY_ECHO'] = True
app.config['DEBUG'] = True
db = SQLAlchemy(app)
class EmailToken(db.Model):
"""Model to store the indexed tokens"""
id = db.Column(db.Integer, primary_key=True)
subject = db.Column(db.String(1024))
token = db.Column(db.String(1024))
@ -35,47 +34,46 @@ class EmailToken(db.Model):
def as_dict(self):
return {
"id": self.id,
"subject": self.subject,
"token": self.token,
"type": self.token_type,
"metadata": self.get_token_metadata(),
"disabled": self.disabled,
'id': self.id,
'subject': self.subject,
'token': self.token,
'type': self.token_type,
'metadata': self.get_token_metadata(),
'disabled': self.disabled,
}
@classmethod
def from_json(cls, data):
metadata = data.get("metadata")
metadata = data.get('metadata')
try:
metadata = json.dumps(metadata)
except TypeError as err:
print("Error dumping metadata", err, file=sys.stderr)
print('Error dumping metadata', err, file=sys.stderr)
return cls(
subject=data.get("subject"),
token=data.get("token"),
token_type=data.get("type"),
subject=data.get('subject'),
token=data.get('token'),
token_type=data.get('type'),
token_metadata=metadata,
disabled=data.get("disabled", False),
disabled=data.get('disabled', False),
)
@classmethod
def jsonify_all(cls, token_type=None, desc=False):
query = cls.query
def jsonify_all(cls, token_type=None):
if token_type:
print("Filtering query by token type", file=sys.stderr)
query = query.filter_by(token_type=token_type)
if desc:
query = query.order_by(cls.id.desc())
return jsonify(tokens=[token.as_dict() for token in query.all()])
print('Filtering query by token type', file=sys.stderr)
results = cls.query.filter_by(token_type=token_type).all()
else:
results = cls.query.all()
return jsonify(tokens=[token.as_dict() for token in results])
@app.route("/")
@app.route('/')
def check():
return "OK"
return 'OK'
@app.route("/token", methods=["POST"])
@app.route('/token', methods=['POST'])
def create_tokens():
"""Creates a token from posted JSON request"""
new_token = EmailToken.from_json(request.get_json(force=True))
@ -86,41 +84,47 @@ def create_tokens():
).first()
print(
"Received token with value {} and type {}".format(
new_token.token, new_token.token_type
),
file=sys.stderr,
'Received token with value {} and type {}'.format(
new_token.token, new_token.token_type
), file=sys.stderr
)
print("Existing token? ", existing_token, file=sys.stderr)
print('Existing token? ', existing_token, file=sys.stderr)
if not existing_token:
print("No existing token, creating a new one", file=sys.stderr)
print('No existing token, creating a new one', file=sys.stderr)
db.session.add(new_token)
db.session.commit()
db.session.refresh(new_token)
return jsonify(success=True, created=True, record=new_token.as_dict())
return jsonify(
success=True,
created=True,
record=new_token.as_dict()
)
else:
print("Found an existing token", file=sys.stderr)
return jsonify(success=True, created=False, record=existing_token.as_dict())
print('Found an existing token', file=sys.stderr)
return jsonify(
success=True,
created=False,
record=existing_token.as_dict()
)
@app.route("/token", methods=["GET"])
@app.route('/token', methods=['GET'])
def list_all_tokens():
"""Lists all tokens with an optional type filter"""
token_type = request.args.get("filter_type")
desc = request.args.get("desc", False)
print("Asked to filter by ", token_type, file=sys.stderr)
return EmailToken.jsonify_all(token_type=token_type, desc=desc)
token_type = request.args.get('filter_type')
print('Asked to filter by ', token_type, file=sys.stderr)
return EmailToken.jsonify_all(token_type=token_type)
@app.route("/token/<int:token_id>", methods=["GET"])
@app.route('/token/<int:token_id>', methods=['GET'])
def get_token(token_id):
"""Gets a token by its primary key id"""
token = EmailToken.query.get(token_id)
return jsonify(token.as_dict())
if __name__ == "__main__":
if __name__ == '__main__':
db.create_all()
app.run(host="0.0.0.0", port=5000)
app.run(host='0.0.0.0', port=5000)

2
indexer/requirements.txt
View File

@ -1,3 +1,3 @@
flask==0.12.2
flask-sqlalchemy==2.3.2
sqlalchemy==1.2.2
flask-sqlalchemy==2.3.2

4
parsers/Makefile
View File

@ -1,4 +0,0 @@
.PHONY: default all test clean
default:
@echo ok

12
parsers/package-tracking/Dockerfile
View File

@ -1,14 +1,14 @@
FROM ruby:2.5.0
# TODO: Move to Gemfile
RUN gem install tracking_number -v 1.0.3
RUN gem install sinatra -v 2.0
EXPOSE 3000
RUN mkdir -p /src
WORKDIR /src
EXPOSE 3000
COPY Gemfile /src/
COPY Gemfile.lock /src/
RUN bundle install
COPY main.rb /src/
CMD ruby main.rb

5
parsers/package-tracking/Gemfile
View File

@ -1,5 +0,0 @@
source 'https://rubygems.org'
ruby '2.5'
gem 'tracking_number', '1.0.3'
gem 'sinatra', '2.0.1'

40
parsers/package-tracking/Gemfile.lock
View File

@ -1,40 +0,0 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (5.1.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
concurrent-ruby (1.0.5)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
minitest (5.11.3)
mustermann (1.0.2)
rack (2.0.4)
rack-protection (2.0.1)
rack
sinatra (2.0.1)
mustermann (~> 1.0)
rack (~> 2.0)
rack-protection (= 2.0.1)
tilt (~> 2.0)
thread_safe (0.3.6)
tilt (2.0.8)
tracking_number (1.0.3)
activesupport (>= 4.2.5)
tzinfo (1.2.5)
thread_safe (~> 0.1)
PLATFORMS
ruby
DEPENDENCIES
sinatra (= 2.0.1)
tracking_number (= 1.0.3)
RUBY VERSION
ruby 2.5.0p0
BUNDLED WITH
1.16.1

4
viewers/Makefile
View File

@ -1,4 +0,0 @@
.PHONY: default all test clean
default:
@echo ok

1
viewers/main/requirements.txt
View File

@ -1,2 +1,3 @@
flask==0.12.2
requests
flask_bootstrap

42
viewers/main/viewer/main.py
View File

@ -1,51 +1,45 @@
import os
import sys
from flask_bootstrap import Bootstrap
import flask
import requests
app = flask.Flask(__name__)
app.config["DEBUG"] = True
app.config['DEBUG'] = True
Bootstrap(app)
indexer_url = os.environ.get("INDEXER_URL", "http://indexer")
indexer_url = os.environ.get('INDEXER_URL', 'http://indexer')
@app.route("/healthcheck")
def healthcheck():
return "OK"
@app.route('/')
def check():
return 'OK'
@app.route("/")
def home():
return flask.render_template("home.html")
@app.route("/shipping")
@app.route('/shipping')
def get_tokens():
resp = requests.get(
indexer_url + "/token",
params={
"filter_type": "SHIPPING",
"desc": True,
},
indexer_url+'/token',
params={'filter_type': 'SHIPPING'},
)
resp.raise_for_status()
tokens = resp.json().get("tokens")
tokens = resp.json().get('tokens')
for token in tokens:
try:
resp = requests.get(
"http://viewer_package_tracking:3000/info/" + token["token"]
'http://viewer_package_tracking:3000/info/'+token['token']
)
resp.raise_for_status()
print("Response: ", resp.text, file=sys.stderr)
print('Response: ', resp.text, file=sys.stderr)
info = resp.json()
token["metadata"].update(info)
token['metadata'].update(info)
except Exception as e:
print("Error", e, file=sys.stderr)
print('Error', e, file=sys.stderr)
pass
return flask.render_template("shipping.html", trackers=tokens)
return flask.render_template('shipping.html', trackers=tokens)
if __name__ == "__main__":
app.run(host="0.0.0.0", port=5000)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)

17
viewers/main/viewer/templates/home.html
View File

@ -1,17 +0,0 @@
{% extends "base.html" %}
{% block title %}Home{% endblock %}
{% block navbar_nav %}
<li class="nav-item">
<a class="nav-link" href="/shipping">Shipping</a>
</li>
{% endblock %}
{% block container %}
<div class="row mt-3">
<div class="col">
<h2>Email Assistant</h2>
<p>Let's get trackin!</p>
</div>
</div>
{% endblock %}

2
viewers/main/viewer/templates/shipping.html
View File

@ -16,7 +16,7 @@
</div>
<div class="row mt-2">
<div class="col">
<table class="table table-responsive">
<table class="table">
<thead>
<tr>
<th scope="col">#</th>

12
viewers/package-tracking/Dockerfile
View File

@ -1,14 +1,14 @@
FROM ruby:2.5.0
# TODO: Move to Gemfile
RUN gem install trackerific -v 0.8.0
RUN gem install sinatra -v 2.0
EXPOSE 3000
RUN mkdir -p /src
WORKDIR /src
EXPOSE 3000
COPY Gemfile /src/
COPY Gemfile.lock /src/
RUN bundle install
COPY main.rb /src/
CMD ruby main.rb

5
viewers/package-tracking/Gemfile
View File

@ -1,5 +0,0 @@
source 'https://rubygems.org'
ruby '2.5'
gem 'trackerific', '0.8.0'
gem 'sinatra', '2.0.1'

73
viewers/package-tracking/Gemfile.lock
View File

@ -1,73 +0,0 @@
GEM
remote: https://rubygems.org/
specs:
activesupport (5.1.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (~> 0.7)
minitest (~> 5.1)
tzinfo (~> 1.1)
akami (1.2.2)
gyoku (>= 0.4.0)
nokogiri
builder (3.2.3)
concurrent-ruby (1.0.5)
gyoku (1.1.1)
builder (>= 2.1.2)
httparty (0.16.0)
multi_xml (>= 0.5.2)
httpi (2.1.1)
rack
rubyntlm (~> 0.3.2)
i18n (0.9.5)
concurrent-ruby (~> 1.0)
mime-types (1.25.1)
mini_portile2 (2.3.0)
minitest (5.11.3)
multi_xml (0.6.0)
mustermann (1.0.2)
nokogiri (1.8.2)
mini_portile2 (~> 2.3.0)
nori (2.3.0)
rack (2.0.4)
rack-protection (2.0.1)
rack
rubyntlm (0.3.4)
savon (2.3.3)
akami (~> 1.2.0)
builder (>= 2.1.2)
gyoku (~> 1.1.0)
httpi (~> 2.1.0)
nokogiri (>= 1.4.0)
nori (~> 2.3.0)
wasabi (~> 3.2.2)
sinatra (2.0.1)
mustermann (~> 1.0)
rack (~> 2.0)
rack-protection (= 2.0.1)
tilt (~> 2.0)
thread_safe (0.3.6)
tilt (2.0.8)
trackerific (0.8.0)
activesupport
builder
httparty (>= 0.12.0)
savon (~> 2.3.0)
tzinfo (1.2.5)
thread_safe (~> 0.1)
wasabi (3.2.3)
httpi (~> 2.0)
mime-types (< 2.0.0)
nokogiri (>= 1.4.0)
PLATFORMS
ruby
DEPENDENCIES
sinatra (= 2.0.1)
trackerific (= 0.8.0)
RUBY VERSION
ruby 2.5.0p0
BUNDLED WITH
1.16.1