iamthefij/email-assistant
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Now indexing metadata

Browse Source
This commit is contained in:
IamTheFij 2018-02-03 00:21:31 -08:00
parent 52883ff5c4
commit b5358b2878
3 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
crawler/crawler/main.py
View File

@ -13,6 +13,7 @@ VALID_CONTENT_TYPES = [ 'text/plain', 'text/html' ]
class MailCrawler(object): class MailCrawler(object):
parser_hosts = None parser_hosts = None
indexer_host = os.environ["INDEXER"]
def __init__(self): def __init__(self):
self.imap_url = os.environ['IMAP_URL'] self.imap_url = os.environ['IMAP_URL']
@ -71,6 +72,14 @@ class MailCrawler(object):
return text return text
return None return None
def index_message(self, message):
response = requests.post(
self.indexer_host+'/token',
json=message,
)
response.raise_for_status()
return response.json()
def run(self): def run(self):
server = self.get_server() server = self.get_server()
server.select_folder('INBOX') server.select_folder('INBOX')
@ -81,7 +90,8 @@ class MailCrawler(object):
result.update({ result.update({
'subject': email_message['SUBJECT'], 'subject': email_message['SUBJECT'],
}) })
print(result) print("Parsed result: ", result)
print("Indexed result: ", self.index_message(result))
if __name__ == '__main__': if __name__ == '__main__':

1
docker-compose.yml
View File

@ -9,6 +9,7 @@ services:
IMAP_URL: my.iamthefij.com IMAP_URL: my.iamthefij.com
IMAP_USER: iamthefij@iamthefij.com IMAP_USER: iamthefij@iamthefij.com
IMAP_PASS: "${IMAP_PASS}" IMAP_PASS: "${IMAP_PASS}"
INDEXER: http://indexer:5000
PARSER_1: http://parser_package_tracking:3000 PARSER_1: http://parser_package_tracking:3000
indexer: indexer:
build: ./indexer build: ./indexer

10
indexer/indexer/main.py
View File

@ -43,11 +43,10 @@ class EmailToken(db.Model):
@classmethod @classmethod
def from_json(cls, data): def from_json(cls, data):
metadata = data.get('metadata') metadata = data.get('metadata')
if metadata: try:
try: metadata = json.dumps(metadata)
metadata = json.dumps(metadata) except TypeError as err:
except TypeError: print('Error dumping metadata', err, file=sys.stderr)
pass
return cls( return cls(
subject=data.get('subject'), subject=data.get('subject'),
@ -76,6 +75,7 @@ def check():
def create_tokens(): def create_tokens():
"""Creates a token from posted JSON request""" """Creates a token from posted JSON request"""
if request.is_json: if request.is_json:
print('Got a json request', file=sys.stderr)
print(request.get_json(), file=sys.stderr) print(request.get_json(), file=sys.stderr)
else: else:
print('Not a json request', file=sys.stderr) print('Not a json request', file=sys.stderr)