simon/wnd_ils_alarmfax_parser
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

commit message

Browse Source
This commit is contained in:
Simon Zeyer
2024-01-08 16:06:47 +00:00
commit 161e76bb5a
16 changed files with 2429 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/exchange_connect.py
+123
View File
@@ -0,0 +1,123 @@
from threading import Thread
from exchangelib import Credentials, Configuration, Account, DELEGATE, FaultTolerance, Folder, Message, ExtendedProperty
import exchangelib.errors
import logging, time, os
from securecad_parser import parse_securecad_message
import os
import datetime
from hooks import webhook, alarminator_api, cups_print
class alarmfax_parser_verarbeitet(ExtendedProperty):
property_set_id = "64901230-f5f2-4b07-a032-58fb3970a09e"
property_name = "vom Alarmfax Parser verarbeitet"
property_type = "Boolean"
class alarmfax_parser_id(ExtendedProperty):
property_set_id = "0c595b56-c79d-4ecb-9e9b-c94fcef86816"
property_name = "ID des Alarmfax Parser"
property_type = "String"
Message.register("alarmfax_parser_verarbeitet", alarmfax_parser_verarbeitet)
Message.register("alarmfax_parser_id", alarmfax_parser_id)
#print([f.name for f in Message.FIELDS if f.is_searchable])
threads = {}
format = "%(asctime)s|%(threadName)s: %(message)s"
logging.basicConfig(format=format, level=logging.INFO,
datefmt="%Y-%m-%d %H:%M:%S")
def eventHandler(ELEMENT_NAME, item_id, item_changekey):
if (ELEMENT_NAME == 'ModifiedEvent' and IS_DEV) or ELEMENT_NAME == 'NewMailEvent' or ELEMENT_NAME == 'SearchFolderEvent':
logging.info(ELEMENT_NAME + " - get Mail")
m: Message = a.inbox.get(id=item_id, changekey=item_changekey)
if m.alarmfax_parser_verarbeitet and parser_id in m.alarmfax_parser_id:
logging.info("Mail {} bereits verarbeitet.. ignoriere".format(m.id))
if not IS_DEV:
return
else:
m.alarmfax_parser_verarbeitet = True
m.alarmfax_parser_id = m.alarmfax_parser_id + parser_id
m.save(update_fields=["alarmfax_parser_verarbeitet","alarmfax_parser_id"])
logging.info("got Mail {} von {}".format(m.subject, m.sender.email_address))
if m.sender.email_address in filter_from:
parsed_body = parse_securecad_message(m.body)
logging.debug(parsed_body)
if parsed_body != None:
if 'ALARMDEPESCHE' in parsed_body:
logging.info("Alarm für: {}".format(parsed_body['ALARMDEPESCHE']))
webhook(parsed_body)
alarminator_api(parsed_body)
cups_print(parsed_body,m.body)
pass
def folder_event_subscriber(folder: Folder):
logging.info('folder_event_subscriber startet for Folder: {}'.format(folder.name))
while True:
# filtern des ordners nach mails der letzten 24h, die nicht verarbeitet wurden
now = datetime.datetime.now(a.default_timezone)
folder.all()
folder.all()
filtered_items = folder.filter(
datetime_received__range=(now - datetime.timedelta(days=1), now + datetime.timedelta(days=1))
).exclude(
alarmfax_parser_verarbeitet=True,
alarmfax_parser_id__contains=parser_id
)
cnt = filtered_items.count()
if cnt > 0:
logging.info("{} Mails nicht verarbeitet in den letzten 2 Tagen in ordner: {}".format(cnt, folder.name))
filtered_items = filtered_items.values("id", "changekey")
for m in filtered_items:
t = Thread(target=eventHandler, args=('SearchFolderEvent',m["id"],m["changekey"],),name="eventHandler: SearchFolderEvent ({})".format(m["id"]))
t.start()
# aktives warten auf streaming_events. maximal eine minute lang, dann wird nochmal der ordner durchsucht, falls mails angekommen sind während eines timeout/cooldown.
subscription_id = folder.subscribe_to_streaming()
for notification in folder.get_streaming_events(subscription_id, connection_timeout=1):
for event in notification.events:
if event.item_id != None:
t = Thread(target=eventHandler, args=(event.ELEMENT_NAME,event.item_id.id,event.item_id.changekey,),name="eventHandler: {} ({})".format(event.ELEMENT_NAME, event.item_id.id))
t.start()
if __name__ == "__main__":
try:
username = os.environ.get('username')
password = os.environ.get('password')
server = os.environ.get('server')
folders = os.environ.get('folders',"")
parser_id = os.environ.get('alarmfax_parser_id',"")
primary_smtp_address = os.environ.get('primary_smtp_address')
filter_from = os.environ.get('filter_from').split(";") if os.environ.get('filter_from') else []
IS_DEV = True if os.environ.get('IS_DEV') and os.environ.get('IS_DEV') == "True" else False
if IS_DEV:
logging.getLogger().setLevel(logging.DEBUG)
credentials = Credentials(username=username, password=password)
config = Configuration(server=server, credentials=credentials,retry_policy=FaultTolerance(max_wait=5), max_connections=10)
a = Account(primary_smtp_address=primary_smtp_address, config=config, autodiscover=False, access_type=DELEGATE)
folders_to_subscribe = []
for f in folders.split(";"):
if f == "":
folders_to_subscribe.append(a.inbox)
else:
folders_to_subscribe.append(a.inbox / f)
while True:
for f in folders_to_subscribe:
if not f.name in threads or not threads[f.name].is_alive():
logging.info("folder_event_subscriber for folder \"{}\" not alive, starting".format(f.name))
t = Thread(target=folder_event_subscriber, args=(f,), daemon=True, name="folder_event_subscriber {}".format(f.name))
threads[f.name] = t
t.start()
time.sleep(1)
except exchangelib.errors.RateLimitError as e:
logging.error('',exc_info=e)
if e.status_code == 401:
webhook({"ERROR":{
"status_code": e.status_code,
"url": e.url
}})
time.sleep(60*60*12)
else:
logging.info("rate limit!!!! sleep 60s")
time.sleep(60)
app/hooks.py
+110
View File
@@ -0,0 +1,110 @@
import os
import logging
import requests
import cups
from weasyprint import HTML
from requests.adapters import Retry
import uuid
retries = Retry(total=5,
backoff_factor=0.1,
status_forcelist=[ 500, 502, 503, 504 ])
def webhook(parsed_body: dict):
try:
webhooks = os.environ.get('webhooks').split(";") if os.environ.get('webhooks') else []
for webhook in webhooks:
logging.info("POST zu {}".format(webhook))
s = requests.Session()
s.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
s.post(webhook,json=parsed_body)
except Exception as e:
logging.error("alarminator_api", e)
def alarminator_api(parsed_body: dict):
try:
alarminator_api = os.environ.get('alarminator_api') if os.environ.get('alarminator_api') else ""
alarminator_token = os.environ.get('alarminator_token') if os.environ.get('alarminator_token') else ""
alarminator_zvies_use_PEALGRP = True if os.environ.get('alarminator_zvies_use_PEALGRP') == 'True' else False
if alarminator_api != "" and alarminator_token != "":
if 'ALARMDEPESCHE' in parsed_body: # sendAlarm triggern
logging.info("GET zu {}/operations/sendAlarm".format(alarminator_api))
s = requests.Session()
s.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
#&object=Kirmesplatz &district=Oberlinxweilerstrasse &subject=THK (TH klein &street=Oberlinxweilerstrasse &ils=\"secur.CAD\" <leitstelle@zrf-saar.de>&connector=mailParser&token=ea2110e1-11b9-421f-a53d-96cc0fc82c31
req_string = ""
req_string +="?token={}".format(alarminator_token)
if 'Einsatzbeginn(Soll)' in parsed_body:
req_string +="&alarmdate={}".format(parsed_body['Einsatzbeginn(Soll)'].split(" ")[0])
req_string +="&alarmtime={}".format(parsed_body['Einsatzbeginn(Soll)'].split(" ")[1])
if 'Auftragsnummer' in parsed_body:
req_string +="&operationnumber={}".format(parsed_body['Auftragsnummer'])
if 'Sachverhalt' in parsed_body:
req_string +="&message={}".format(parsed_body['Sachverhalt'])
if 'Einsatzziel' in parsed_body:
if 'Stadt' in parsed_body['Einsatzziel']:
req_string +="&community={}".format(parsed_body['Einsatzziel']['Stadt'])
if 'PLZ / Ort' in parsed_body['Einsatzziel']:
req_string +="&location={}".format(parsed_body['Einsatzziel']['PLZ / Ort'])
if 'Objekt' in parsed_body['Einsatzziel']:
req_string +="&object={}".format(parsed_body['Einsatzziel']['Objekt'])
if 'Strasse' in parsed_body['Einsatzziel']:
req_string +="&street={}".format(parsed_body['Einsatzziel']['Strasse']) + (("\n"+parsed_body['Einsatzziel']['Info ']) if 'Info ' in parsed_body['Einsatzziel'] else "" )
if 'Strasse / Hs.-Nr.' in parsed_body['Einsatzziel']:
req_string +="&street={}".format(parsed_body['Einsatzziel']['Strasse / Hs.-Nr.'] + (("\n"+parsed_body['Einsatzziel']['Info ']) if 'Info ' in parsed_body['Einsatzziel'] else "" ))
if 'Einsatzmittelliste' in parsed_body:
gear = []
for r in parsed_body['Einsatzmittelliste']:
if r['Typ'] != 'PEALGRP':
gear.append(r['Ressourcen'])
req_string +="&gear={}".format(';'.join(gear))
#req_string +="&district={}".format('district')
#req_string +="&floor={}".format('floor')
#req_string +="&section={}".format('section')
req_string +="&keywordRaw={}".format(parsed_body['Einsatzstichwort'])
#req_string +="&keywordId={}".format('keywordId')
req_string +="&keywordCategory={}".format(parsed_body['Einsatzstichwort'].split("(")[0])
req_string +="&keywordName={}".format(parsed_body['Einsatzstichwort'].split("(")[1].split(")")[0])
if alarminator_zvies_use_PEALGRP:
if 'Einsatzmittelliste' in parsed_body:
zveis = []
for r in parsed_body['Einsatzmittelliste']:
if r['Typ'] == 'PEALGRP':
zveis.append(r['Ressourcen'])
req_string +="&zveis={}".format(';'.join(zveis))
else:
req_string +="&zveis={}".format(parsed_body['ALARMDEPESCHE'])
# req_string +="&gkx={}".format() if False
# req_string +="&gky={}".format() if False
# req_string +="&lat={}".format() if False
# req_string +="&lon={}".format() if False
subject = ""
if 'Notfallgeschehen' in parsed_body:
subject = parsed_body['Notfallgeschehen'] + "\n"
if 'Notfallgeschehen' in parsed_body:
subject = parsed_body['Notfallgeschehen'] + "\n"
req_string +="&subject={}".format(subject)
req_string +="&ils={}".format("ILS Saar")
req_string +="&connector={}".format("MailParser")
s.get(alarminator_api+"/operations/sendAlarm/"+req_string)
except Exception as e:
logging.error("alarminator_api", e)
def cups_print(parsed_body: dict, body: str):
fname = "/tmp/{}.pdf".format(uuid.uuid4())
try:
conn = cups.Connection ()
printer = os.environ.get('printer',"DEFAULT")
print_num = int(os.environ.get('print_num',0))
if 'ALARMDEPESCHE' in parsed_body:
with open(fname,"wb") as f:
f.write(HTML(string=body, base_url="").write_pdf())
for i in range(0, print_num):
conn.printFile (printer, fname, "Alarmfax", {})
os.remove(fname)
except Exception as e:
if os.path.exists(fname):
os.remove(fname)
logging.error("cups_print", e)
app/securecad_parser.py
+95
View File
@@ -0,0 +1,95 @@
from glob import glob
import pandas as pd
from numpy import nan
tt_list_alarmdepesche = []
tt_list_infodepesche = []
tt_list_einsatzprotokoll = []
def parse_securecad_message(body_html: str):
body_html = body_html.replace(u'\xa0', u' ')
if 'ALARMDEPESCHE' in body_html:
t_list = {}
tables = pd.read_html(body_html) # Returns list of all tables on page
t_count = 0
for t in tables:
t_count = t_count + 1
table_dict = t.to_dict('index')
if t_count == 1 and 'ALARMDEPESCHE' in table_dict[0][0]:
k:str
v:str
k,v = table_dict[0][0].split('>>')
t_list[k.strip()] = v.strip()
elif t_count == 2:
continue
elif t_count == 3:
for r in table_dict:
if table_dict[r][0] is not nan:
t_list[table_dict[r][0].strip(' :')] = table_dict[r][1]
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Einsatzziel' and 'Einsatzziel' not in t_list:
t_list['Einsatzziel'] = {}
for r in table_dict:
if table_dict[r][0] is not nan and table_dict[r][0] != 'Einsatzziel':
t_list['Einsatzziel'][table_dict[r][0].strip(' :')] = table_dict[r][1]
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Einsatzziel' and 'Einsatzziel' in t_list:
continue
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Zusatztext zum Ort' and 'Zusatztext zum Ort' not in t_list:
t_list['Zusatztext zum Ort'] = []
for r in table_dict:
if table_dict[r][0] is not nan and table_dict[r][0] != 'Zusatztext zum Ort':
t_list['Zusatztext zum Ort'].append(table_dict[r][1])
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Zusatztext zum Ort' and 'Zusatztext zum Ort' in t_list:
continue
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Zusatztext zum Objekt' and 'Zusatztext zum Objekt' not in t_list:
t_list['Zusatztext zum Objekt'] = []
for r in table_dict:
if table_dict[r][0] is not nan and table_dict[r][0] != 'Zusatztext zum Objekt':
t_list['Zusatztext zum Objekt'].append(table_dict[r][1])
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Zusatztext zum Objekt' and 'Zusatztext zum Objekt' in t_list:
continue
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Einsatzmittelliste':
# es folgt die Einsatzmittelliste
continue
elif t_count > 3 and table_dict[0] is not nan and table_dict[0][0] == 'Ressourcen'and table_dict[0][1] == 'Typ' and table_dict[0][2] == 'Organisation':
t_list['Einsatzmittelliste'] = []
for r in table_dict:
if table_dict[r][0] != "Ressourcen":
t_list['Einsatzmittelliste'].append({
'Ressourcen': table_dict[r][0] if table_dict[r][0] is not nan else "",
'Typ': table_dict[r][1] if table_dict[r][1] is not nan else "",
'Organisation': table_dict[r][2] if table_dict[r][2] is not nan else "",
'Status': table_dict[r][3] if table_dict[r][3] is not nan else "",
'Alarm': table_dict[r][4] if table_dict[r][4] is not nan else "",
'aus': table_dict[r][5] if table_dict[r][5] is not nan else "",
'an': table_dict[r][6] if table_dict[r][6] is not nan else "",
'Auftrag': table_dict[r][7] if table_dict[r][7] is not nan else "",
})
else:
#print(table_dict)
pass
#print(t_list)
#tt_list_alarmdepesche.append(t_list)
# bereinigung
if 'Zusatztext zum Objekt' in t_list:
if t_list['Zusatztext zum Objekt'].__len__() == 1 and t_list['Zusatztext zum Objekt'][0].strip() == ".":
del t_list['Zusatztext zum Objekt']
else:
t_list['Zusatztext zum Objekt'] = "\n".join(t_list['Zusatztext zum Objekt'])
if 'Zusatztext zum Ort' in t_list:
if t_list['Zusatztext zum Ort'].__len__() == 1 and t_list['Zusatztext zum Ort'][0].strip() == ".":
del t_list['Zusatztext zum Ort']
else:
t_list['Zusatztext zum Ort'] = "\n".join(t_list['Zusatztext zum Ort'])
t_list["__HTML_BODY"] = body_html
return t_list
else:
pass
pass
if __name__ == "__main__":
for i in glob('*.html'):
with open(i,'r') as f:
parse_securecad_message(f.read())
pass