simon/wnd_ils_alarmfax_parser
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: simon:jfw
Branches Tags
simon:master
simon:jfw
..
compare: simon:master
Branches Tags
simon:master
simon:jfw

6 Commits
jfw ... master

Author SHA1 Message Date
Simon Zeyer
f64879398f Add sleep delay in run function to prevent tight loop execution 2025-06-16 21:22:55 +00:00
Simon Zeyer
3e33de4606 Fix flag setting method in IMAP message handling 2025-06-09 12:01:03 +00:00
Simon Zeyer
8b4bc74b1d Add PDF generation and printing functionality using Google Chrome 2025-06-09 11:52:44 +00:00
Simon Zeyer
2b331fbd1f Remove redundant idle check logic for existing messages in IMAP connection 2025-06-09 11:37:44 +00:00
Simon Zeyer
b528b0a748 Refactor idle check logic in IMAP connection to improve message handling 2025-06-09 11:12:18 +00:00
Simon Zeyer
5127caed03 Add IMAP support and refactor environment configuration 
- Introduced `imap_connect.py` for handling IMAP email interactions.
- Created `run.py` to manage execution based on the selected mode (IMAP or EWS).
- Updated Dockerfile and .env.dev to include necessary environment variables.
- Enhanced logging and email processing in `exchange_connect.py`.
- Added `weasyprint` and `imapclient` to requirements.
 2025-06-09 10:53:08 +00:00
9 changed files with 210 additions and 230 deletions
Show Stats Expand all files Collapse all files

2
.env.dev
View File

@ -1,3 +1,4 @@
mode="EWS"
username="" username=""
password="" password=""
server="" server=""
@ -12,4 +13,3 @@ alarminator_token=""
alarminator_zvies_use_PEALGRP="False" alarminator_zvies_use_PEALGRP="False"
printer=DEFAULT printer=DEFAULT
print_num=0 print_num=0
BASIC_AUTH_PASSWORD=""

8
Dockerfile
View File

@ -1,5 +1,6 @@
FROM python:3.10-bullseye FROM python:3.10-bullseye
ENV mode="EWS"
ENV username= ENV username=
ENV password= ENV password=
ENV server="exchange.sankt-wendel.de" ENV server="exchange.sankt-wendel.de"
@ -13,8 +14,6 @@ ENV alarminator_zvies_use_PEALGRP="False"
ENV print_num=0 ENV print_num=0
ENV printer="DEFAULT" ENV printer="DEFAULT"
ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
ENV MAPS_API_KEY=""
ENV BASIC_AUTH_PASSWORD=""
COPY *.deb / COPY *.deb /
@ -46,7 +45,6 @@ RUN apt-get --yes --force-yes install ca-certificates cups cups-filters libcups
WORKDIR /usr/src/app WORKDIR /usr/src/app
EXPOSE 631 EXPOSE 631
EXPOSE 5000
# Add user and disable sudo password checking # Add user and disable sudo password checking
RUN useradd \ RUN useradd \
--groups=sudo,lp,lpadmin \ --groups=sudo,lp,lpadmin \
@ -57,6 +55,7 @@ RUN useradd \
print \ print \
&& sed -i '/%sudo[[:space:]]/ s/ALL[[:space:]]*$/NOPASSWD:ALL/' /etc/sudoers && sed -i '/%sudo[[:space:]]/ s/ALL[[:space:]]*$/NOPASSWD:ALL/' /etc/sudoers
# Print PDF # Print PDF
RUN apt-get update && apt-get install -y \ RUN apt-get update && apt-get install -y \
apt-transport-https \ apt-transport-https \
@ -76,10 +75,9 @@ RUN groupadd chrome && useradd -g chrome -s /bin/bash -G audio,video chrome \
COPY requirements.txt ./ COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
RUN pip install --no-cache-dir waitress
COPY ./app . COPY ./app .
COPY *.crt /usr/local/share/ca-certificates/ COPY *.crt /usr/local/share/ca-certificates/
RUN update-ca-certificates RUN update-ca-certificates
CMD [ "sh","-c","/etc/init.d/cups start && waitress-serve --port=5000 app:app && python3 exchange_connect.py" ] CMD [ "sh","-c","/etc/init.d/cups start && python3 /usr/src/app/run.py" ]

136
app/app.py
View File

@ -1,136 +0,0 @@
from flask import Flask, flash, request, redirect, send_from_directory, get_flashed_messages
import os
import glob
import securecad_parser
import datetime
from hooks import webhook, alarminator_api, cups_print
from pathlib import Path
from flask_basicauth import BasicAuth
app = Flask(__name__)
app.secret_key = 'super secret key'
app.config['SESSION_TYPE'] = 'filesystem'
app.config['BASIC_AUTH_USERNAME'] = 'admin'
app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('BASIC_AUTH_PASSWORD','password')
basic_auth = BasicAuth(app)
def wrapHtml(innerHtml):
return '''<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>JFW Alarm</title>
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-QWTKZyjpPEjISv5WaRU9OFeRpok6YctnYmDr5pNlyT2bRjXh0JMhjY6hW+ALEwIH" crossorigin="anonymous">
</head>
<body><div class="container text-center">
''' + flashMessages() +'''
''' + innerHtml + '''
</div>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.3/dist/js/bootstrap.bundle.min.js" integrity="sha384-YvpcrYf0tY3lHB60NNkmXc5s9fDVZLESaAA55NDzOxhy9GkcIdslK1eN7N6jIeHz" crossorigin="anonymous"></script>
</body>
</html>'''
def flashMessages():
messages = get_flashed_messages(with_categories=True)
html = ""
if messages:
for category, message in messages:
html = html + category + ": "+message+"<br>"
return html
def alarmsPath():
dir_path = os.path.dirname(os.path.realpath(__file__))
return dir_path + "/alarms" + os.sep
def getAlarmFiles():
a_list = glob.glob(alarmsPath()+"*.html")
a_list.sort()
return [os.path.basename(f) for f in a_list]
def parseAlarm(f):
f = alarmsPath() + os.sep + f
with open(f, "r") as o:
_r: str = o.read()
now = datetime.datetime.now()
_r = _r.replace('%DATUM%',now.strftime("%d.%m.%Y"))
_r = _r.replace('%UHRZEIT%',now.strftime("%H:%M"))
if _r != '':
return [securecad_parser.parse_securecad_message(_r),_r]
def allowed_file(filename:str):
print(filename.rsplit('.', 1)[1].lower())
return '.' in filename and \
filename.rsplit('.', 1)[1].lower() in ["html"]
@app.route("/")
@basic_auth.required
def root():
html = ""
file = request.args.get('sendalarm')
if file is not None:
parsed_body, raw = parseAlarm(file)
if parsed_body != None:
if 'ALARMDEPESCHE' in parsed_body:
webhook(parsed_body)
alarminator_api(parsed_body)
cups_print(parsed_body,raw)
flash("Alarm gesendet!")
return redirect('/')
html = html + '<div class="row align-items-start p-3 ">'
i: str
files = getAlarmFiles()
for f in files:
html = html + "<div class=\"col-6\" style='padding: 5px;'><a style='display:block; padding:20px' class=\"btn btn-block btn-danger\" href=\"/?sendalarm="+f+"\">" + f + "</a></div>"
html = html + "</div>"
return wrapHtml(html)
@app.route('/files', methods=['POST'])
@basic_auth.required
def upload_file():
# check if the post request has the file part
print(request.files)
if 'file' not in request.files:
flash('No file part')
return redirect(request.url)
file = request.files['file']
# If the user does not select a file, the browser submits an
# empty file without a filename.
print(file.filename)
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
file.save(os.path.join(alarmsPath(), file.filename))
return redirect('/files')
@app.route('/files/<path:filename>', methods=['GET'])
@basic_auth.required
def download(filename):
uploads = alarmsPath()
return send_from_directory(uploads, filename, as_attachment=True)
@app.route("/files", methods=['GET'])
@basic_auth.required
def files():
Path(alarmsPath()).mkdir(parents=True, exist_ok=True)
file = request.args.get('del')
if file is not None:
try:
os.remove(alarmsPath()+os.sep+file)
except:
pass
flash("Datei gelöscht!")
return redirect('/files')
files = getAlarmFiles()
html = '<ul class="list-group">'
for f in files:
html = html + '<li class="list-group-item"><a href="/files/'+f+'">'+f+'</a> <a href="/files?del='+f+'">löschen</a></li>'
html = html + '</ul><form method="post" enctype="multipart/form-data"><input type="file" name="file"><input type=submit value=Upload></form>'
return html
# if __name__ == "__main__":
# app.run(port=5000, host='0.0.0.0')

12
app/exchange_connect.py
View File

@ -21,13 +21,14 @@ Message.register("alarmfax_parser_verarbeitet", alarmfax_parser_verarbeitet)
Message.register("alarmfax_parser_id", alarmfax_parser_id) Message.register("alarmfax_parser_id", alarmfax_parser_id)
#print([f.name for f in Message.FIELDS if f.is_searchable]) #print([f.name for f in Message.FIELDS if f.is_searchable])
threads = {} def run():
threads = {}
format = "%(asctime)s|%(threadName)s: %(message)s" format = "%(asctime)s|%(threadName)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO, logging.basicConfig(format=format, level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S") datefmt="%Y-%m-%d %H:%M:%S")
def eventHandler(ELEMENT_NAME, item_id, item_changekey): def eventHandler(ELEMENT_NAME, item_id, item_changekey):
if (ELEMENT_NAME == 'ModifiedEvent' and IS_DEV) or ELEMENT_NAME == 'NewMailEvent' or ELEMENT_NAME == 'SearchFolderEvent': if (ELEMENT_NAME == 'ModifiedEvent' and IS_DEV) or ELEMENT_NAME == 'NewMailEvent' or ELEMENT_NAME == 'SearchFolderEvent':
logging.info(ELEMENT_NAME + " - get Mail") logging.info(ELEMENT_NAME + " - get Mail")
m: Message = a.inbox.get(id=item_id, changekey=item_changekey) m: Message = a.inbox.get(id=item_id, changekey=item_changekey)
@ -51,7 +52,7 @@ def eventHandler(ELEMENT_NAME, item_id, item_changekey):
cups_print(parsed_body,m.body) cups_print(parsed_body,m.body)
pass pass
def folder_event_subscriber(folder: Folder): def folder_event_subscriber(folder: Folder):
logging.info('folder_event_subscriber startet for Folder: {}'.format(folder.name)) logging.info('folder_event_subscriber startet for Folder: {}'.format(folder.name))
while True: while True:
# filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden # filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
@ -79,7 +80,6 @@ def folder_event_subscriber(folder: Folder):
t = Thread(target=eventHandler, args=(event.ELEMENT_NAME,event.item_id.id,event.item_id.changekey,),name="eventHandler: {} ({})".format(event.ELEMENT_NAME, event.item_id.id)) t = Thread(target=eventHandler, args=(event.ELEMENT_NAME,event.item_id.id,event.item_id.changekey,),name="eventHandler: {} ({})".format(event.ELEMENT_NAME, event.item_id.id))
t.start() t.start()
if __name__ == "__main__":
try: try:
username = os.environ.get('username') username = os.environ.get('username')
password = os.environ.get('password') password = os.environ.get('password')

18
app/hooks.py
View File

@ -2,6 +2,7 @@ import os
import logging import logging
import requests import requests
import cups import cups
from weasyprint import HTML
from requests.adapters import Retry from requests.adapters import Retry
import uuid import uuid
import subprocess import subprocess
@ -68,9 +69,9 @@ def alarminator_api(parsed_body: dict):
gear.append(r['Ressourcen']) gear.append(r['Ressourcen'])
req_string +="&gear={}".format(';'.join(gear)) req_string +="&gear={}".format(';'.join(gear))
# req_string +="&district={}".format('district') req_string +="&district={}".format('district')
# req_string +="&floor={}".format('floor') req_string +="&floor={}".format('floor')
# req_string +="&section={}".format('section') req_string +="&section={}".format('section')
req_string +="&keywordRaw={}".format(parsed_body['Einsatzstichwort']) req_string +="&keywordRaw={}".format(parsed_body['Einsatzstichwort'])
#req_string +="&keywordId={}".format('keywordId') #req_string +="&keywordId={}".format('keywordId')
req_string +="&keywordCategory={}".format(parsed_body['Einsatzstichwort'].split("(")[0]) req_string +="&keywordCategory={}".format(parsed_body['Einsatzstichwort'].split("(")[0])
@ -110,6 +111,8 @@ def alarminator_api(parsed_body: dict):
# if False # if False
# req_string +="&lon={}".format() if False # req_string +="&lon={}".format() if False
subject = "" subject = ""
if 'Notfallgeschehen' in parsed_body:
subject = parsed_body['Notfallgeschehen'] + "\n"
if 'Notfallgeschehen' in parsed_body: if 'Notfallgeschehen' in parsed_body:
subject = parsed_body['Notfallgeschehen'] + "\n" subject = parsed_body['Notfallgeschehen'] + "\n"
req_string +="&subject={}".format(subject) req_string +="&subject={}".format(subject)
@ -130,16 +133,17 @@ def generate_pdf(html_body, filename):
os.remove(f) os.remove(f)
def cups_print(parsed_body: dict, body: str): def cups_print(parsed_body: dict, body: str):
if os.environ.get('IS_DEV') and os.environ.get('IS_DEV') == "True": # if os.environ.get('IS_DEV') and os.environ.get('IS_DEV') == "True":
generate_pdf(body, "{}.pdf".format(uuid.uuid4())) # generate_pdf(body, "{}.pdf".format(uuid.uuid4()))
fname = "/tmp/{}.pdf".format(uuid.uuid4()) fname = "{}.pdf".format(uuid.uuid4())
try: try:
conn = cups.Connection () conn = cups.Connection ()
printer_arr = os.environ.get('printer',"DEFAULT").split(";") printer_arr = os.environ.get('printer',"DEFAULT").split(";")
print_num = int(os.environ.get('print_num',0)) print_num = int(os.environ.get('print_num',0))
if printer_arr.__len__() > 0: #if printer_arr.__len__() > 0:
generate_pdf(body, fname) generate_pdf(body, fname)
for printer in printer_arr: for printer in printer_arr:
print(printer)
if 'ALARMDEPESCHE' in parsed_body: if 'ALARMDEPESCHE' in parsed_body:
for i in range(0, print_num): for i in range(0, print_num):
conn.printFile (printer, fname, "Alarmfax", {}) conn.printFile (printer, fname, "Alarmfax", {})

124
app/imap_connect.py Normal file
View File

@ -0,0 +1,124 @@
from datetime import timedelta, datetime
import os
import ssl
import email
import logging
from imapclient import IMAPClient
from threading import Thread
from securecad_parser import parse_securecad_message
from hooks import webhook, alarminator_api, cups_print
import re
from time import sleep
def run():
threads = {}
format = "%(asctime)s|%(threadName)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S")
def eventHandler(ELEMENT_NAME, uid, message_data, _server: IMAPClient = None):
email_message = email.message_from_bytes(message_data[b'RFC822'])
email_from: list[str] = re.findall(r'([\w\.-]+@[\w\.-]+)', email_message.get('From'))
flags = _server.get_flags(uid)
logging.info(ELEMENT_NAME + " - get Mail")
if 'Processed_{}'.format(parser_id).encode() in flags[uid]:
logging.info("Mail {} bereits verarbeitet.. ignoriere".format(uid))
if not IS_DEV:
return
else:
_server.add_flags(uid, ['\\SEEN','Processed_{}'.format(parser_id)])
logging.info("got Mail {} von {}".format(email_message.get('Subject'), email_from))
if any(mail in filter_from for mail in email_from):
# Get HTML body
html_body = ""
if email_message.is_multipart():
for part in email_message.walk():
if part.get_content_type() == 'text/html':
html_body = part.get_payload(decode=True).decode(part.get_content_charset() or 'utf-8', errors='replace')
break
else:
if email_message.get_content_type() == 'text/html':
html_body = email_message.get_payload(decode=True).decode(email_message.get_content_charset() or 'utf-8', errors='replace')
parsed_body = parse_securecad_message(html_body)
logging.debug(parsed_body)
if parsed_body != None:
if 'ALARMDEPESCHE' in parsed_body:
logging.info("Alarm für: {}".format(parsed_body['ALARMDEPESCHE']))
webhook(parsed_body)
alarminator_api(parsed_body)
cups_print(parsed_body,html_body)
pass
def folder_event_subscriber(folder: str):
logging.info('folder_event_subscriber startet for Folder: {}'.format(folder))
with IMAPClient(server, ssl_context=ssl_context) as _server:
_server.login(username, password)
while True:
# filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
now = datetime.now()
_server.select_folder(folder, readonly=False)
q = ['SENTSINCE', now - timedelta(days=1),'NOT','KEYWORD', 'Processed_{}'.format(parser_id)]
if IS_DEV:
q = ['SENTSINCE', now - timedelta(days=1),'UNSEEN']
q = ['UNSEEN']
messages = _server.search(q)
cnt = messages.__len__()
if cnt > 0:
logging.info("{} Mails nicht verarbeitet in den letzten 2 Tagen in ordner: {}".format(cnt, folder))
for uid, message_data in _server.fetch(messages, 'RFC822').items():
# IMAPClient ist nicht thread-safe, daher wird hier der _server übergeben und kein Thread verwendet.
eventHandler('SearchFolderEvent', uid, message_data, _server)
# t = Thread(target=eventHandler, args=('SearchFolderEvent',uid,message_data,),name="eventHandler: SearchFolderEvent ({})".format(uid))
# t.start()
# aktives warten auf streaming_events. maximal eine minute lang, dann wird nochmal der ordner durchsucht, falls mails angekommen sind während eines timeout/cooldown.
_server.idle()
try:
logging.debug("Idle check for folder: {}".format(folder))
messages = _server.idle_check(timeout=60) # Timeout after 60 seconds
_server.idle_done()
# In den events stehen nur vorhandenen nachrichten. Exists ist nicht die neue Nachricht, sondern eine bereits vorhandene.
# for item in messages:
# if item[1] == b'EXISTS':
# logging.info("New messages in folder: {}".format(folder))
# for uid, message_data in _server.fetch([item[0]], 'RFC822').items():
# if uid:
# eventHandler('NewMailEvent', uid, message_data, _server)
except Exception as e:
logging.error("Error during idle check: {}".format(e))
username = os.environ.get('username')
password = os.environ.get('password')
server = os.environ.get('server')
folders = os.environ.get('folders',"")
parser_id = os.environ.get('alarmfax_parser_id',"")
primary_smtp_address = os.environ.get('primary_smtp_address')
filter_from = os.environ.get('filter_from').split(";") if os.environ.get('filter_from') else []
IS_DEV = True if os.environ.get('IS_DEV') and os.environ.get('IS_DEV') == "True" else False
if IS_DEV:
logging.getLogger().setLevel(logging.INFO)
ssl_context = ssl.create_default_context()
with IMAPClient(server, ssl_context=ssl_context) as _server:
_server.login(username, password)
_server.logout()
folders_to_subscribe = []
for f in folders.split(";"):
if f == "":
folders_to_subscribe.append('INBOX')
else:
folders_to_subscribe.append(f)
while True:
for f in folders_to_subscribe:
if not f in threads or not threads[f].is_alive():
logging.info("folder_event_subscriber for folder \"{}\" not alive, starting".format(f))
t = Thread(target=folder_event_subscriber, args=(f,), daemon=True, name="folder_event_subscriber {}".format(f))
threads[f] = t
t.start()
sleep(1)

10
app/run.py Normal file
View File

@ -0,0 +1,10 @@
import os
from imap_connect import run as run_imap
from exchange_connect import run as run_ews
mode = os.environ.get('mode')
if mode == 'IMAP':
run_imap()
elif mode == 'EWS':
run_ews()
else:
raise ValueError("Invalid mode specified. Use 'IMAP' or 'EWS'.")

8
docker-compose.yaml
View File

@ -2,13 +2,11 @@ version: "2"
services: services:
app: app:
build: ./ build: ./
#image: gitea.simonzeyer.de/simon/wnd_ils_alarmfax_parser:latest image: gitea.simonzeyer.de/simon/wnd_ils_alarmfax_parser:latest
restart: always restart: always
privileged: true privileged: true
# ports:
# - 631:631
# - 5000:5000
environment: environment:
- mode=${mode}
- username=${username} - username=${username}
- password=${password} - password=${password}
- server=${server} - server=${server}
@ -23,8 +21,6 @@ services:
- alarminator_zvies_use_PEALGRP=${alarminator_zvies_use_PEALGRP} - alarminator_zvies_use_PEALGRP=${alarminator_zvies_use_PEALGRP}
- printer=${printer} - printer=${printer}
- print_num=${print_num} - print_num=${print_num}
- MAPS_API_KEY=${MAPS_API_KEY}
- BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD}
volumes: volumes:
- ./cups:/etc/cups - ./cups:/etc/cups
- ./cupsd.conf.txt:/etc/cups/cupsd.conf - ./cupsd.conf.txt:/etc/cups/cupsd.conf

22
requirements.txt
View File

@ -1,35 +1,20 @@
blinker==1.8.2
Brotli==1.1.0
cached-property==1.5.2 cached-property==1.5.2
certifi==2022.12.7 certifi==2022.12.7
cffi==1.15.1 cffi==1.15.1
charset-normalizer==3.0.1 charset-normalizer==3.0.1
click==8.1.7
cryptography==39.0.0 cryptography==39.0.0
cssselect2==0.7.0
defusedxml==0.7.1 defusedxml==0.7.1
dnspython==2.3.0 dnspython==2.3.0
exchangelib==4.9.0 exchangelib==4.9.0
Flask==3.0.3
Flask-BasicAuth==0.2.0
fonttools==4.54.1
html5lib==1.1
idna==3.4 idna==3.4
isodate==0.6.1 isodate==0.6.1
itsdangerous==2.2.0
Jinja2==3.1.4
lxml==4.9.2 lxml==4.9.2
MarkupSafe==3.0.1
ntlm-auth==1.5.0 ntlm-auth==1.5.0
numpy==1.24.1 numpy==1.24.1
oauthlib==3.2.2 oauthlib==3.2.2
pandas==1.5.2 pandas==1.5.2
pillow==10.4.0
pycparser==2.21 pycparser==2.21
pycups==2.0.1
pydyf==0.11.0
Pygments==2.14.0 Pygments==2.14.0
pyphen==0.16.0
python-dateutil==2.8.2 python-dateutil==2.8.2
pytz==2022.7.1 pytz==2022.7.1
pytz-deprecation-shim==0.1.0.post0 pytz-deprecation-shim==0.1.0.post0
@ -37,12 +22,11 @@ requests==2.28.2
requests-ntlm==1.1.0 requests-ntlm==1.1.0
requests-oauthlib==1.3.1 requests-oauthlib==1.3.1
six==1.16.0 six==1.16.0
tinycss2==1.3.0
tzdata==2022.7 tzdata==2022.7
tzlocal==4.2 tzlocal==4.2
urllib3==1.26.14 urllib3==1.26.14
webencodings==0.5.1
Werkzeug==3.0.4
xmltodict==0.12.0 xmltodict==0.12.0
xmltojson==2.0.1 xmltojson==2.0.1
zopfli==0.2.3 pycups==2.0.1
weasyprint
imapclient==3.0.1