Now indexing metadata

crawler/crawler/main.py

@@ -13,6 +13,7 @@ VALID_CONTENT_TYPES = [ 'text/plain', 'text/html' ]
 
 class MailCrawler(object):
     parser_hosts = None
+    indexer_host = os.environ["INDEXER"]
 
     def __init__(self):
         self.imap_url = os.environ['IMAP_URL']
@@ -71,6 +72,14 @@ class MailCrawler(object):
                     return text
         return None
 
+    def index_message(self, message):
+        response = requests.post(
+            self.indexer_host+'/token',
+            json=message,
+        )
+        response.raise_for_status()
+        return response.json()
+
     def run(self):
         server = self.get_server()
         server.select_folder('INBOX')
@@ -81,7 +90,8 @@ class MailCrawler(object):
                 result.update({
                     'subject': email_message['SUBJECT'],
                 })
-                print(result)
+                print("Parsed result: ", result)
+                print("Indexed result: ", self.index_message(result))
 
 
 if __name__ == '__main__':

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       IMAP_URL: my.iamthefij.com
       IMAP_USER: iamthefij@iamthefij.com
       IMAP_PASS: "${IMAP_PASS}"
+      INDEXER: http://indexer:5000
       PARSER_1: http://parser_package_tracking:3000
   indexer:
     build: ./indexer

indexer/indexer/main.py

@@ -43,11 +43,10 @@ class EmailToken(db.Model):
     @classmethod
     def from_json(cls, data):
         metadata = data.get('metadata')
-        if metadata:
-            try:
-                metadata = json.dumps(metadata)
-            except TypeError:
-                pass
+        try:
+            metadata = json.dumps(metadata)
+        except TypeError as err:
+            print('Error dumping metadata', err, file=sys.stderr)
 
         return cls(
             subject=data.get('subject'),
@@ -76,6 +75,7 @@ def check():
 def create_tokens():
     """Creates a token from posted JSON request"""
     if request.is_json:
+        print('Got a json request', file=sys.stderr)
         print(request.get_json(), file=sys.stderr)
     else:
         print('Not a json request', file=sys.stderr)