Add IMAP support and refactor environment configuration

- Introduced `imap_connect.py` for handling IMAP email interactions.
- Created `run.py` to manage execution based on the selected mode (IMAP or EWS).
- Updated Dockerfile and .env.dev to include necessary environment variables.
- Enhanced logging and email processing in `exchange_connect.py`.
- Added `weasyprint` and `imapclient` to requirements.

.env.dev

@@ -1,3 +1,4 @@
+mode="EWS"
 username=""
 password=""
 server=""

Dockerfile

@@ -1,5 +1,6 @@
 FROM python:3.10-bullseye
 
+ENV mode="EWS"
 ENV username=
 ENV password=
 ENV server="exchange.sankt-wendel.de"
@@ -61,4 +62,4 @@ COPY ./app .
 COPY *.crt /usr/local/share/ca-certificates/
 RUN update-ca-certificates
 
-CMD [ "sh","-c","/etc/init.d/cups start && python3 /usr/src/app/exchange_connect.py" ]
+CMD [ "sh","-c","/etc/init.d/cups start && python3 /usr/src/app/run.py" ]

app/exchange_connect.py

@@ -21,65 +21,65 @@ Message.register("alarmfax_parser_verarbeitet", alarmfax_parser_verarbeitet)
 Message.register("alarmfax_parser_id", alarmfax_parser_id)
 #print([f.name for f in Message.FIELDS if f.is_searchable])
 
-threads = {}
+def run():
+    threads = {}
 
-format = "%(asctime)s|%(threadName)s: %(message)s"
+    format = "%(asctime)s|%(threadName)s: %(message)s"
-logging.basicConfig(format=format, level=logging.INFO,
+    logging.basicConfig(format=format, level=logging.INFO,
-                    datefmt="%Y-%m-%d %H:%M:%S")
+                        datefmt="%Y-%m-%d %H:%M:%S")
 
-def eventHandler(ELEMENT_NAME, item_id, item_changekey):
+    def eventHandler(ELEMENT_NAME, item_id, item_changekey):
-    if (ELEMENT_NAME == 'ModifiedEvent' and IS_DEV) or ELEMENT_NAME == 'NewMailEvent' or ELEMENT_NAME == 'SearchFolderEvent':
+        if (ELEMENT_NAME == 'ModifiedEvent' and IS_DEV) or ELEMENT_NAME == 'NewMailEvent' or ELEMENT_NAME == 'SearchFolderEvent':
-        logging.info(ELEMENT_NAME + " - get Mail")
+            logging.info(ELEMENT_NAME + " - get Mail")
-        m: Message = a.inbox.get(id=item_id, changekey=item_changekey)
+            m: Message = a.inbox.get(id=item_id, changekey=item_changekey)
-        if m.alarmfax_parser_verarbeitet and parser_id in ("" if m.alarmfax_parser_id == None else m.alarmfax_parser_id):
+            if m.alarmfax_parser_verarbeitet and parser_id in ("" if m.alarmfax_parser_id == None else m.alarmfax_parser_id):
-            logging.info("Mail {} bereits verarbeitet.. ignoriere".format(m.id))
+                logging.info("Mail {} bereits verarbeitet.. ignoriere".format(m.id))
-            if not IS_DEV: 
+                if not IS_DEV: 
-                return
+                    return
-        else:
+            else:
-            m.alarmfax_parser_verarbeitet = True
+                m.alarmfax_parser_verarbeitet = True
-            m.alarmfax_parser_id = ("" if m.alarmfax_parser_id == None else m.alarmfax_parser_id) + parser_id
+                m.alarmfax_parser_id = ("" if m.alarmfax_parser_id == None else m.alarmfax_parser_id) + parser_id
-            m.save(update_fields=["alarmfax_parser_verarbeitet","alarmfax_parser_id"])
+                m.save(update_fields=["alarmfax_parser_verarbeitet","alarmfax_parser_id"])
-        logging.info("got Mail {} von {}".format(m.subject, m.sender.email_address))
+            logging.info("got Mail {} von {}".format(m.subject, m.sender.email_address))
-        if m.sender.email_address in filter_from:
+            if m.sender.email_address in filter_from:
-            parsed_body = parse_securecad_message(m.body)
+                parsed_body = parse_securecad_message(m.body)
-            logging.debug(parsed_body)
+                logging.debug(parsed_body)
-            if parsed_body != None:
+                if parsed_body != None:
-                if 'ALARMDEPESCHE' in parsed_body:
+                    if 'ALARMDEPESCHE' in parsed_body:
-                    logging.info("Alarm für: {}".format(parsed_body['ALARMDEPESCHE']))
+                        logging.info("Alarm für: {}".format(parsed_body['ALARMDEPESCHE']))
-                webhook(parsed_body)
+                    webhook(parsed_body)
-                alarminator_api(parsed_body)
+                    alarminator_api(parsed_body)
-                cups_print(parsed_body,m.body)
+                    cups_print(parsed_body,m.body)
-        pass
+            pass
 
-def folder_event_subscriber(folder: Folder):
+    def folder_event_subscriber(folder: Folder):
-    logging.info('folder_event_subscriber startet for Folder: {}'.format(folder.name))
+        logging.info('folder_event_subscriber startet for Folder: {}'.format(folder.name))
-    while True:
+        while True:
-        # filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
+            # filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
-        now = datetime.datetime.now(a.default_timezone)
+            now = datetime.datetime.now(a.default_timezone)
-        folder.all()
+            folder.all()
-        folder.all()
+            folder.all()
-        filtered_items = folder.filter(
+            filtered_items = folder.filter(
-            datetime_received__range=(now - datetime.timedelta(days=1), now + datetime.timedelta(days=1))
+                datetime_received__range=(now - datetime.timedelta(days=1), now + datetime.timedelta(days=1))
-        ).exclude(
+            ).exclude(
-            alarmfax_parser_verarbeitet=True,
+                alarmfax_parser_verarbeitet=True,
-            alarmfax_parser_id__contains=parser_id
+                alarmfax_parser_id__contains=parser_id
-        )
+            )
-        cnt = filtered_items.count()
+            cnt = filtered_items.count()
-        if cnt > 0:
+            if cnt > 0:
-            logging.info("{} Mails nicht verarbeitet in den letzten 2 Tagen in ordner: {}".format(cnt, folder.name))
+                logging.info("{} Mails nicht verarbeitet in den letzten 2 Tagen in ordner: {}".format(cnt, folder.name))
-            filtered_items = filtered_items.values("id", "changekey")
+                filtered_items = filtered_items.values("id", "changekey")
-            for m in filtered_items:
+                for m in filtered_items:
-                t = Thread(target=eventHandler, args=('SearchFolderEvent',m["id"],m["changekey"],),name="eventHandler: SearchFolderEvent ({})".format(m["id"]))
+                    t = Thread(target=eventHandler, args=('SearchFolderEvent',m["id"],m["changekey"],),name="eventHandler: SearchFolderEvent ({})".format(m["id"]))
-                t.start()
-        # aktives warten auf streaming_events. maximal eine minute lang, dann wird nochmal der ordner durchsucht, falls mails angekommen sind während eines timeout/cooldown.
-        subscription_id = folder.subscribe_to_streaming()
-        for notification in folder.get_streaming_events(subscription_id, connection_timeout=1):
-            for event in notification.events:
-                if event.item_id != None:
-                    t = Thread(target=eventHandler, args=(event.ELEMENT_NAME,event.item_id.id,event.item_id.changekey,),name="eventHandler: {} ({})".format(event.ELEMENT_NAME, event.item_id.id))
                     t.start()
+            # aktives warten auf streaming_events. maximal eine minute lang, dann wird nochmal der ordner durchsucht, falls mails angekommen sind während eines timeout/cooldown.
+            subscription_id = folder.subscribe_to_streaming()
+            for notification in folder.get_streaming_events(subscription_id, connection_timeout=1):
+                for event in notification.events:
+                    if event.item_id != None:
+                        t = Thread(target=eventHandler, args=(event.ELEMENT_NAME,event.item_id.id,event.item_id.changekey,),name="eventHandler: {} ({})".format(event.ELEMENT_NAME, event.item_id.id))
+                        t.start()
 
-if __name__ == "__main__":
     try:
         username = os.environ.get('username')
         password = os.environ.get('password')

app/imap_connect.py

@@ -0,0 +1,124 @@
+
+from datetime import timedelta, datetime
+import os
+import ssl
+import email
+import logging
+from imapclient import IMAPClient
+from threading import Thread
+from securecad_parser import parse_securecad_message
+from hooks import webhook, alarminator_api, cups_print
+import re
+
+def run():
+    threads = {}
+
+    format = "%(asctime)s|%(threadName)s: %(message)s"
+    logging.basicConfig(format=format, level=logging.INFO,
+                        datefmt="%Y-%m-%d %H:%M:%S")
+
+    def eventHandler(ELEMENT_NAME, uid, message_data, _server: IMAPClient = None):
+        email_message = email.message_from_bytes(message_data[b'RFC822'])
+        email_from: list[str] = re.findall(r'([\w\.-]+@[\w\.-]+)', email_message.get('From'))
+        flags = _server.get_flags(uid)
+        logging.info(ELEMENT_NAME + " - get Mail")
+        if 'Processed_{}'.format(parser_id).encode() in flags[uid]:
+            logging.info("Mail {} bereits verarbeitet.. ignoriere".format(uid))
+            if not IS_DEV: 
+                return
+        else:
+           _server.set_flags(uid, ['\\SEEN','Processed_{}'.format(parser_id)])
+        logging.info("got Mail {} von {}".format(email_message.get('Subject'), email_from))
+        if any(mail in filter_from for mail in email_from):
+            # Get HTML body
+            html_body = ""
+            if email_message.is_multipart():
+                for part in email_message.walk():
+                    if part.get_content_type() == 'text/html':
+                        html_body = part.get_payload(decode=True).decode(part.get_content_charset() or 'utf-8', errors='replace')
+                        break
+            else:
+                if email_message.get_content_type() == 'text/html':
+                    html_body = email_message.get_payload(decode=True).decode(email_message.get_content_charset() or 'utf-8', errors='replace')
+    
+            
+            parsed_body = parse_securecad_message(html_body)
+            logging.debug(parsed_body)
+            if parsed_body != None:
+                if 'ALARMDEPESCHE' in parsed_body:
+                    logging.info("Alarm für: {}".format(parsed_body['ALARMDEPESCHE']))
+                webhook(parsed_body)
+                alarminator_api(parsed_body)
+                cups_print(parsed_body,html_body)
+            pass
+
+    def folder_event_subscriber(folder: str):
+        logging.info('folder_event_subscriber startet for Folder: {}'.format(folder))
+        with IMAPClient(server, ssl_context=ssl_context) as _server:
+            _server.login(username, password)
+            while True:
+                # filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
+                now = datetime.now()
+                _server.select_folder(folder, readonly=False)
+                q = ['SENTSINCE', now - timedelta(days=1),'NOT','KEYWORD', 'Processed_{}'.format(parser_id)]
+                if IS_DEV:
+                    q = ['SENTSINCE', now - timedelta(days=1),'UNSEEN']
+                    q = ['UNSEEN']
+                messages = _server.search(q)
+                cnt = messages.__len__()
+                if cnt > 0:
+                    logging.info("{} Mails nicht verarbeitet in den letzten 2 Tagen in ordner: {}".format(cnt, folder))
+                    for uid, message_data in _server.fetch(messages, 'RFC822').items():
+                        # IMAPClient ist nicht thread-safe, daher wird hier der _server übergeben und kein Thread verwendet.
+                        eventHandler('SearchFolderEvent', uid, message_data, _server) 
+                        # t = Thread(target=eventHandler, args=('SearchFolderEvent',uid,message_data,),name="eventHandler: SearchFolderEvent ({})".format(uid))
+                        # t.start()
+                # aktives warten auf streaming_events. maximal eine minute lang, dann wird nochmal der ordner durchsucht, falls mails angekommen sind während eines timeout/cooldown.
+                _server.idle()
+                try:
+                    logging.debug("Idle check for folder: {}".format(folder))
+                    messages = _server.idle_check(timeout=60)  # Timeout after 60 seconds
+                    # if not messages:
+                    #     logging.info("No new messages in folder: {}".format(folder))
+                    #     continue
+                    for item in messages:
+                        if item[1] in (b'EXISTS'):
+                            logging.info("New messages in folder: {}".format(folder))
+                            for uid, message_data in _server.fetch([item[0]], 'RFC822').items():
+                                if uid:
+                                    eventHandler('NewMailEvent', uid, message_data, _server)                    
+                except Exception as e:
+                    logging.error("Error during idle check: {}".format(e))
+                _server.idle_done()
+
+    username = os.environ.get('username')
+    password = os.environ.get('password')
+    server = os.environ.get('server')
+    folders = os.environ.get('folders',"")
+    parser_id = os.environ.get('alarmfax_parser_id',"")
+    primary_smtp_address = os.environ.get('primary_smtp_address')
+    filter_from = os.environ.get('filter_from').split(";") if os.environ.get('filter_from') else []
+    IS_DEV = True if os.environ.get('IS_DEV') and os.environ.get('IS_DEV') == "True" else False
+    if IS_DEV:
+        logging.getLogger().setLevel(logging.INFO)
+
+    ssl_context = ssl.create_default_context()
+
+
+    with IMAPClient(server, ssl_context=ssl_context) as _server:
+        _server.login(username, password)
+        _server.logout()
+        folders_to_subscribe = []
+        for f in folders.split(";"):
+            if f == "":
+                folders_to_subscribe.append('INBOX')
+            else:
+                folders_to_subscribe.append(f)
+
+        while True:
+            for f in folders_to_subscribe:
+                if not f in threads or not threads[f].is_alive():
+                    logging.info("folder_event_subscriber for folder \"{}\" not alive, starting".format(f))
+                    t = Thread(target=folder_event_subscriber, args=(f,), daemon=True, name="folder_event_subscriber {}".format(f))
+                    threads[f] = t
+                    t.start()

app/run.py

@@ -0,0 +1,10 @@
+import os
+from imap_connect import run as run_imap
+from exchange_connect import run as run_ews
+mode = os.environ.get('mode')
+if mode == 'IMAP':
+    run_imap()
+elif mode == 'EWS':
+    run_ews()
+else:
+    raise ValueError("Invalid mode specified. Use 'IMAP' or 'EWS'.")

docker-compose.yaml

@@ -6,6 +6,7 @@ services:
     restart: always
     privileged: true
     environment:
+      - mode=${mode}
       - username=${username}
       - password=${password}
       - server=${server}

requirements.txt

@@ -29,3 +29,4 @@ xmltodict==0.12.0
 xmltojson==2.0.1
 pycups==2.0.1
 weasyprint
+imapclient==3.0.1