Now indexing metadata

This commit is contained in:
IamTheFij 2018-02-03 00:21:31 -08:00
parent 52883ff5c4
commit b5358b2878
3 changed files with 17 additions and 6 deletions

View File

@ -13,6 +13,7 @@ VALID_CONTENT_TYPES = [ 'text/plain', 'text/html' ]
class MailCrawler(object):
parser_hosts = None
indexer_host = os.environ["INDEXER"]
def __init__(self):
self.imap_url = os.environ['IMAP_URL']
@ -71,6 +72,14 @@ class MailCrawler(object):
return text
return None
def index_message(self, message):
response = requests.post(
self.indexer_host+'/token',
json=message,
)
response.raise_for_status()
return response.json()
def run(self):
server = self.get_server()
server.select_folder('INBOX')
@ -81,7 +90,8 @@ class MailCrawler(object):
result.update({
'subject': email_message['SUBJECT'],
})
print(result)
print("Parsed result: ", result)
print("Indexed result: ", self.index_message(result))
if __name__ == '__main__':

View File

@ -9,6 +9,7 @@ services:
IMAP_URL: my.iamthefij.com
IMAP_USER: iamthefij@iamthefij.com
IMAP_PASS: "${IMAP_PASS}"
INDEXER: http://indexer:5000
PARSER_1: http://parser_package_tracking:3000
indexer:
build: ./indexer

View File

@ -43,11 +43,10 @@ class EmailToken(db.Model):
@classmethod
def from_json(cls, data):
metadata = data.get('metadata')
if metadata:
try:
metadata = json.dumps(metadata)
except TypeError:
pass
except TypeError as err:
print('Error dumping metadata', err, file=sys.stderr)
return cls(
subject=data.get('subject'),
@ -76,6 +75,7 @@ def check():
def create_tokens():
"""Creates a token from posted JSON request"""
if request.is_json:
print('Got a json request', file=sys.stderr)
print(request.get_json(), file=sys.stderr)
else:
print('Not a json request', file=sys.stderr)