Add initial indexer apis

This commit is contained in:
IamTheFij 2018-02-03 00:06:55 -08:00
parent 9d88bcf1b2
commit 52883ff5c4
10 changed files with 139 additions and 2 deletions

View File

@ -4,11 +4,16 @@ services:
build: ./crawler build: ./crawler
links: links:
- parser_package_tracking - parser_package_tracking
- indexer
environment: environment:
IMAP_URL: my.iamthefij.com IMAP_URL: my.iamthefij.com
IMAP_USER: iamthefij@iamthefij.com IMAP_USER: iamthefij@iamthefij.com
IMAP_PASS: "${IMAP_PASS}" IMAP_PASS: "${IMAP_PASS}"
PARSER_1: http://parser_package_tracking:3000 PARSER_1: http://parser_package_tracking:3000
indexer:
build: ./indexer
ports:
- "8181:5000"
parser_package_tracking: parser_package_tracking:
build: ./parsers/package-tracking build: ./parsers/package-tracking
ports: ports:

1
indexer/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
tokens.db

9
indexer/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM python:3.6-onbuild
# ENV FLASK_APP=indexer/app.py
ENV FLASK_DEBUG=1
EXPOSE 5000
# TODO: Track debug in env and use threads
CMD python -m indexer.main

View File

@ -0,0 +1,8 @@
version: '2'
services:
main:
build: .
volumes:
- .:/usr/src/app
ports:
- "8181:5000"

View File

108
indexer/indexer/main.py Normal file
View File

@ -0,0 +1,108 @@
import json
import os
import sys
from flask import jsonify
from flask import request
from flask.ext.sqlalchemy import SQLAlchemy
import flask
app = flask.Flask(__name__)
# TODO: use a real database or something
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///tokens.db')
app.config['SQLALCHEMY_ECHO'] = True
app.config['DEBUG'] = True
db = SQLAlchemy(app)
class EmailToken(db.Model):
"""Model to store the indexed tokens"""
id = db.Column(db.Integer, primary_key=True)
subject = db.Column(db.String(1024))
token = db.Column(db.String(1024))
token_type = db.Column(db.String(1024))
token_metadata = db.Column(db.String(2048))
disabled = db.Column(db.Boolean, default=False)
def get_token_metadata(self):
if self.token_metadata:
return json.loads(self.token_metadata)
return None
def as_dict(self):
return {
'id': self.id,
'subject': self.subject,
'token': self.token,
'type': self.token_type,
'metadata': self.get_token_metadata(),
'disabled': self.disabled,
}
@classmethod
def from_json(cls, data):
metadata = data.get('metadata')
if metadata:
try:
metadata = json.dumps(metadata)
except TypeError:
pass
return cls(
subject=data.get('subject'),
token=data.get('token'),
token_type=data.get('type'),
token_metadata=metadata,
disabled=data.get('disabled', False),
)
@classmethod
def jsonify_all(cls, token_type=None):
if token_type:
print('Filtering query by token type', file=sys.stderr)
results = cls.query.filter_by(token_type=token_type).all()
else:
results = cls.query.all()
return jsonify(tokens=[token.as_dict() for token in results])
@app.route("/")
def check():
return "OK"
@app.route('/token', methods=['POST'])
def create_tokens():
"""Creates a token from posted JSON request"""
if request.is_json:
print(request.get_json(), file=sys.stderr)
else:
print('Not a json request', file=sys.stderr)
print(request.get_json(force=True), file=sys.stderr)
token = EmailToken.from_json(request.get_json(force=True))
db.session.add(token)
db.session.commit()
db.session.refresh(token)
return jsonify(success=True, record=token.as_dict())
@app.route('/token', methods=['GET'])
def list_all_tokens():
"""Lists all tokens with an optional type filter"""
token_type = request.args.get('filter_type')
print('Asked to filter by ', token_type, file=sys.stderr)
return EmailToken.jsonify_all(token_type=token_type)
@app.route('/token/<int:token_id>', methods=['GET'])
def get_token(token_id):
"""Gets a token by its primary key id"""
token = EmailToken.query.get(token_id)
return jsonify(token.as_dict())
if __name__ == "__main__":
db.create_all()
app.run(host='0.0.0.0', port=5000)

View File

@ -0,0 +1,2 @@
ipdb
ipython

3
indexer/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
flask
sqlalchemy
flask-sqlalchemy

View File

@ -25,6 +25,7 @@ Request:
|Key |Example Value |Description| |Key |Example Value |Description|
|--------|-----------------------------------------------------|-----------| |--------|-----------------------------------------------------|-----------|
|`"message"`|`"Here's your tracking number: 1Z879E930346834440"`|Full contents of the email message| |`"message"`|`"Here's your tracking number: 1Z879E930346834440"`|Full contents of the email message|
|`"subject"`|`"Your email is here"`|Full contents of the email message|
Response: Response:

View File

@ -5,4 +5,4 @@ services:
volumes: volumes:
- .:/src - .:/src
ports: ports:
- "127.0.0.1:8183:3000" - "8183:3000"