From b5358b28781e04b65f45048e6fad1bd509387b06 Mon Sep 17 00:00:00 2001
From: Ian Fijolek <ian@iamthefij.com>
Date: Sat, 3 Feb 2018 00:21:31 -0800
Subject: [PATCH] Now indexing metadata

---
 crawler/crawler/main.py | 12 +++++++++++-
 docker-compose.yml      |  1 +
 indexer/indexer/main.py | 10 +++++-----
 3 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/crawler/crawler/main.py b/crawler/crawler/main.py
index afad10f..16086ed 100644
--- a/crawler/crawler/main.py
+++ b/crawler/crawler/main.py
@@ -13,6 +13,7 @@ VALID_CONTENT_TYPES = [ 'text/plain', 'text/html' ]
 
 class MailCrawler(object):
     parser_hosts = None
+    indexer_host = os.environ["INDEXER"]
 
     def __init__(self):
         self.imap_url = os.environ['IMAP_URL']
@@ -71,6 +72,14 @@ class MailCrawler(object):
                     return text
         return None
 
+    def index_message(self, message):
+        response = requests.post(
+            self.indexer_host+'/token',
+            json=message,
+        )
+        response.raise_for_status()
+        return response.json()
+
     def run(self):
         server = self.get_server()
         server.select_folder('INBOX')
@@ -81,7 +90,8 @@ class MailCrawler(object):
                 result.update({
                     'subject': email_message['SUBJECT'],
                 })
-                print(result)
+                print("Parsed result: ", result)
+                print("Indexed result: ", self.index_message(result))
 
 
 if __name__ == '__main__':
diff --git a/docker-compose.yml b/docker-compose.yml
index 8bf4db2..969fab8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,6 +9,7 @@ services:
       IMAP_URL: my.iamthefij.com
       IMAP_USER: iamthefij@iamthefij.com
       IMAP_PASS: "${IMAP_PASS}"
+      INDEXER: http://indexer:5000
       PARSER_1: http://parser_package_tracking:3000
   indexer:
     build: ./indexer
diff --git a/indexer/indexer/main.py b/indexer/indexer/main.py
index d180108..c3b98ce 100644
--- a/indexer/indexer/main.py
+++ b/indexer/indexer/main.py
@@ -43,11 +43,10 @@ class EmailToken(db.Model):
     @classmethod
     def from_json(cls, data):
         metadata = data.get('metadata')
-        if metadata:
-            try:
-                metadata = json.dumps(metadata)
-            except TypeError:
-                pass
+        try:
+            metadata = json.dumps(metadata)
+        except TypeError as err:
+            print('Error dumping metadata', err, file=sys.stderr)
 
         return cls(
             subject=data.get('subject'),
@@ -76,6 +75,7 @@ def check():
 def create_tokens():
     """Creates a token from posted JSON request"""
     if request.is_json:
+        print('Got a json request', file=sys.stderr)
         print(request.get_json(), file=sys.stderr)
     else:
         print('Not a json request', file=sys.stderr)