From 52883ff5c4f5373055a19a4483f41dd4f22c2cc9 Mon Sep 17 00:00:00 2001 From: Ian Fijolek Date: Sat, 3 Feb 2018 00:06:55 -0800 Subject: [PATCH] Add initial indexer apis --- docker-compose.yml | 5 + indexer/.gitignore | 1 + indexer/Dockerfile | 9 ++ indexer/docker-compose.yml | 8 ++ indexer/indexer/__init__.py | 0 indexer/indexer/main.py | 108 ++++++++++++++++++++ indexer/requirements-dev.txt | 2 + indexer/requirements.txt | 3 + parsers/Readme.md | 3 +- parsers/package-tracking/docker-compose.yml | 2 +- 10 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 indexer/.gitignore create mode 100644 indexer/Dockerfile create mode 100644 indexer/docker-compose.yml create mode 100644 indexer/indexer/__init__.py create mode 100644 indexer/indexer/main.py create mode 100644 indexer/requirements-dev.txt create mode 100644 indexer/requirements.txt diff --git a/docker-compose.yml b/docker-compose.yml index c3597bd..8bf4db2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,16 @@ services: build: ./crawler links: - parser_package_tracking + - indexer environment: IMAP_URL: my.iamthefij.com IMAP_USER: iamthefij@iamthefij.com IMAP_PASS: "${IMAP_PASS}" PARSER_1: http://parser_package_tracking:3000 + indexer: + build: ./indexer + ports: + - "8181:5000" parser_package_tracking: build: ./parsers/package-tracking ports: diff --git a/indexer/.gitignore b/indexer/.gitignore new file mode 100644 index 0000000..c053d8b --- /dev/null +++ b/indexer/.gitignore @@ -0,0 +1 @@ +tokens.db diff --git a/indexer/Dockerfile b/indexer/Dockerfile new file mode 100644 index 0000000..1696cc1 --- /dev/null +++ b/indexer/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.6-onbuild + +# ENV FLASK_APP=indexer/app.py +ENV FLASK_DEBUG=1 + +EXPOSE 5000 + +# TODO: Track debug in env and use threads +CMD python -m indexer.main diff --git a/indexer/docker-compose.yml b/indexer/docker-compose.yml new file mode 100644 index 0000000..12c36a2 --- /dev/null +++ b/indexer/docker-compose.yml @@ -0,0 +1,8 @@ +version: '2' +services: + main: + build: . + volumes: + - .:/usr/src/app + ports: + - "8181:5000" diff --git a/indexer/indexer/__init__.py b/indexer/indexer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/indexer/indexer/main.py b/indexer/indexer/main.py new file mode 100644 index 0000000..d180108 --- /dev/null +++ b/indexer/indexer/main.py @@ -0,0 +1,108 @@ +import json +import os +import sys + +from flask import jsonify +from flask import request +from flask.ext.sqlalchemy import SQLAlchemy +import flask + + +app = flask.Flask(__name__) +# TODO: use a real database or something +app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///tokens.db') +app.config['SQLALCHEMY_ECHO'] = True +app.config['DEBUG'] = True + +db = SQLAlchemy(app) + +class EmailToken(db.Model): + """Model to store the indexed tokens""" + id = db.Column(db.Integer, primary_key=True) + subject = db.Column(db.String(1024)) + token = db.Column(db.String(1024)) + token_type = db.Column(db.String(1024)) + token_metadata = db.Column(db.String(2048)) + disabled = db.Column(db.Boolean, default=False) + + def get_token_metadata(self): + if self.token_metadata: + return json.loads(self.token_metadata) + return None + + def as_dict(self): + return { + 'id': self.id, + 'subject': self.subject, + 'token': self.token, + 'type': self.token_type, + 'metadata': self.get_token_metadata(), + 'disabled': self.disabled, + } + + @classmethod + def from_json(cls, data): + metadata = data.get('metadata') + if metadata: + try: + metadata = json.dumps(metadata) + except TypeError: + pass + + return cls( + subject=data.get('subject'), + token=data.get('token'), + token_type=data.get('type'), + token_metadata=metadata, + disabled=data.get('disabled', False), + ) + + @classmethod + def jsonify_all(cls, token_type=None): + if token_type: + print('Filtering query by token type', file=sys.stderr) + results = cls.query.filter_by(token_type=token_type).all() + else: + results = cls.query.all() + return jsonify(tokens=[token.as_dict() for token in results]) + + +@app.route("/") +def check(): + return "OK" + + +@app.route('/token', methods=['POST']) +def create_tokens(): + """Creates a token from posted JSON request""" + if request.is_json: + print(request.get_json(), file=sys.stderr) + else: + print('Not a json request', file=sys.stderr) + print(request.get_json(force=True), file=sys.stderr) + + token = EmailToken.from_json(request.get_json(force=True)) + db.session.add(token) + db.session.commit() + db.session.refresh(token) + return jsonify(success=True, record=token.as_dict()) + + +@app.route('/token', methods=['GET']) +def list_all_tokens(): + """Lists all tokens with an optional type filter""" + token_type = request.args.get('filter_type') + print('Asked to filter by ', token_type, file=sys.stderr) + return EmailToken.jsonify_all(token_type=token_type) + + +@app.route('/token/', methods=['GET']) +def get_token(token_id): + """Gets a token by its primary key id""" + token = EmailToken.query.get(token_id) + return jsonify(token.as_dict()) + + +if __name__ == "__main__": + db.create_all() + app.run(host='0.0.0.0', port=5000) diff --git a/indexer/requirements-dev.txt b/indexer/requirements-dev.txt new file mode 100644 index 0000000..988613e --- /dev/null +++ b/indexer/requirements-dev.txt @@ -0,0 +1,2 @@ +ipdb +ipython diff --git a/indexer/requirements.txt b/indexer/requirements.txt new file mode 100644 index 0000000..2d23893 --- /dev/null +++ b/indexer/requirements.txt @@ -0,0 +1,3 @@ +flask +sqlalchemy +flask-sqlalchemy diff --git a/parsers/Readme.md b/parsers/Readme.md index 3e5a11d..a15e1e8 100644 --- a/parsers/Readme.md +++ b/parsers/Readme.md @@ -24,7 +24,8 @@ Request: |Key |Example Value |Description| |--------|-----------------------------------------------------|-----------| -|`"message"` |`"Here's your tracking number: 1Z879E930346834440"`|Full contents of the email message| +|`"message"`|`"Here's your tracking number: 1Z879E930346834440"`|Full contents of the email message| +|`"subject"`|`"Your email is here"`|Full contents of the email message| Response: diff --git a/parsers/package-tracking/docker-compose.yml b/parsers/package-tracking/docker-compose.yml index dbe797e..bce1120 100644 --- a/parsers/package-tracking/docker-compose.yml +++ b/parsers/package-tracking/docker-compose.yml @@ -5,4 +5,4 @@ services: volumes: - .:/src ports: - - "127.0.0.1:8183:3000" + - "8183:3000"