Add initial indexer apis

docker-compose.yml

@@ -4,11 +4,16 @@ services:
     build: ./crawler
     links:
       - parser_package_tracking
+      - indexer
     environment:
       IMAP_URL: my.iamthefij.com
       IMAP_USER: iamthefij@iamthefij.com
       IMAP_PASS: "${IMAP_PASS}"
       PARSER_1: http://parser_package_tracking:3000
+  indexer:
+    build: ./indexer
+    ports:
+      - "8181:5000"
   parser_package_tracking:
     build: ./parsers/package-tracking
     ports:

indexer/.gitignore

@@ -0,0 +1 @@
+tokens.db

indexer/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.6-onbuild
+
+# ENV FLASK_APP=indexer/app.py
+ENV FLASK_DEBUG=1
+
+EXPOSE 5000
+
+# TODO: Track debug in env and use threads
+CMD python -m indexer.main

indexer/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2'
+services:
+  main:
+    build: .
+    volumes:
+      - .:/usr/src/app
+    ports:
+      - "8181:5000"

indexer/indexer/main.py

@@ -0,0 +1,108 @@
+import json
+import os
+import sys
+
+from flask import jsonify
+from flask import request
+from flask.ext.sqlalchemy import SQLAlchemy
+import flask
+
+
+app = flask.Flask(__name__)
+# TODO: use a real database or something
+app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URI', 'sqlite:///tokens.db')
+app.config['SQLALCHEMY_ECHO'] = True
+app.config['DEBUG'] = True
+
+db = SQLAlchemy(app)
+
+class EmailToken(db.Model):
+    """Model to store the indexed tokens"""
+    id = db.Column(db.Integer, primary_key=True)
+    subject = db.Column(db.String(1024))
+    token = db.Column(db.String(1024))
+    token_type = db.Column(db.String(1024))
+    token_metadata = db.Column(db.String(2048))
+    disabled = db.Column(db.Boolean, default=False)
+
+    def get_token_metadata(self):
+        if self.token_metadata:
+            return json.loads(self.token_metadata)
+        return None
+
+    def as_dict(self):
+        return {
+            'id': self.id,
+            'subject': self.subject,
+            'token': self.token,
+            'type': self.token_type,
+            'metadata': self.get_token_metadata(),
+            'disabled': self.disabled,
+        }
+
+    @classmethod
+    def from_json(cls, data):
+        metadata = data.get('metadata')
+        if metadata:
+            try:
+                metadata = json.dumps(metadata)
+            except TypeError:
+                pass
+
+        return cls(
+            subject=data.get('subject'),
+            token=data.get('token'),
+            token_type=data.get('type'),
+            token_metadata=metadata,
+            disabled=data.get('disabled', False),
+        )
+
+    @classmethod
+    def jsonify_all(cls, token_type=None):
+        if token_type:
+            print('Filtering query by token type', file=sys.stderr)
+            results = cls.query.filter_by(token_type=token_type).all()
+        else:
+            results = cls.query.all()
+        return jsonify(tokens=[token.as_dict() for token in results])
+
+
+@app.route("/")
+def check():
+    return "OK"
+
+
+@app.route('/token', methods=['POST'])
+def create_tokens():
+    """Creates a token from posted JSON request"""
+    if request.is_json:
+        print(request.get_json(), file=sys.stderr)
+    else:
+        print('Not a json request', file=sys.stderr)
+        print(request.get_json(force=True), file=sys.stderr)
+
+    token = EmailToken.from_json(request.get_json(force=True))
+    db.session.add(token)
+    db.session.commit()
+    db.session.refresh(token)
+    return jsonify(success=True, record=token.as_dict())
+
+
+@app.route('/token', methods=['GET'])
+def list_all_tokens():
+    """Lists all tokens with an optional type filter"""
+    token_type = request.args.get('filter_type')
+    print('Asked to filter by ', token_type, file=sys.stderr)
+    return EmailToken.jsonify_all(token_type=token_type)
+
+
+@app.route('/token/<int:token_id>', methods=['GET'])
+def get_token(token_id):
+    """Gets a token by its primary key id"""
+    token = EmailToken.query.get(token_id)
+    return jsonify(token.as_dict())
+
+
+if __name__ == "__main__":
+    db.create_all()
+    app.run(host='0.0.0.0', port=5000)

indexer/requirements-dev.txt

@@ -0,0 +1,2 @@
+ipdb
+ipython

indexer/requirements.txt

@@ -0,0 +1,3 @@
+flask
+sqlalchemy
+flask-sqlalchemy

parsers/Readme.md

@@ -25,6 +25,7 @@ Request:
 |Key     |Example Value                                        |Description|
 |--------|-----------------------------------------------------|-----------|
 |`"message"`|`"Here's your tracking number: 1Z879E930346834440"`|Full contents of the email message|
+|`"subject"`|`"Your email is here"`|Full contents of the email message|
 
 Response:
 

parsers/package-tracking/docker-compose.yml

@@ -5,4 +5,4 @@ services:
     volumes:
       - .:/src
     ports:
-      - "127.0.0.1:8183:3000"
+      - "8183:3000"