Merge pull request 'Turniermeldunen Struktur anlegen' (#32) from feat/turniermeldungen into develop

Reviewed-on: #32
This commit is contained in:
Christian Wolf 2024-01-14 19:46:53 +00:00
commit 2598f77156
31 changed files with 686 additions and 10 deletions

24
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,24 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: Remote Attach",
"type": "python",
"request": "attach",
"connect": {
"host": "localhost",
"port": 5678
},
"pathMappings": [
// {
// "localRoot": "${workspaceFolder}",
// "remoteRoot": "."
// }
],
"justMyCode": true
}
]
}

View File

@ -1,6 +0,0 @@
---
title: "{{ replace .Name "-" " " | title }}"
date: {{ .Date }}
draft: true
---

View File

@ -1,7 +1,8 @@
--- ---
title: "Turniermeldungen" title: "Turniermeldungen"
date: 2023-01-20T16:27:40+01:00 date: 2023-01-20T16:27:40+01:00
draft: true layout: turniermeldungen
draft: false
# type: home # type: home
menu: menu:
main: main:
@ -9,3 +10,17 @@ menu:
parent: aktuell parent: aktuell
--- ---
## Linkliste
- {{< tsc/link-external url="https://www.tbw.de/" >}}TBW{{< /tsc/link-external >}}
- {{< tsc/link-external url="http://www.tanzsport.de/" >}}Deutscher Tanzsportverband e.V. (DTV){{< /tsc/link-external >}}
- {{< tsc/link-external url="http://appsrv.tanzsport.de/dtv-webdbs/turnier/suche.spf" >}}Turnierdatenbank des DTV{{< /tsc/link-external >}}
- {{< tsc/link-external url="https://ev.tanzsport-portal.de/" >}}für Aktive: ESV-Anmeldung{{< /tsc/link-external >}}
- {{< tsc/link-external url="http://de.dancesportinfo.net/SearchCouples.aspx" >}}de.dancesportinfo.net{{< /tsc/link-external >}}

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-01-13
partner: "Westerhoff, Frank"
partnerin: "Westerhoff, Anja Dr."
verein: "GGC Clubheim"
ort: "Wuppertal"
telefon: "0202 712476"
gruppe: "Mas III"
klasse: "S"
sektion: "Std"
titel: "GGC Seniorentag Standard"
nummer: 113904
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-01-21
partner: "Kohler, Jürgen"
partnerin: "Kohler, Petra"
verein: "Saalbau Haus Nidda"
ort: "Frankfurt a.M."
telefon: "0176 61745268"
gruppe: "Mas III"
klasse: "B"
sektion: "Std"
titel: "Die Goldene Schuhbürste 2024"
nummer: 115126
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-03
partner: "Kieper, Alexander"
partnerin: "Hehl, Carolin"
verein: "Turnhalle Botnang"
ort: "Botnang"
telefon: "0170 8631320"
gruppe: "Mas II"
klasse: "A"
sektion: "Std"
titel: "Sportveranstaltung 2024"
nummer: 113800
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-03
partner: "Baal, Philipp"
partnerin: "Lis, Letizia"
verein: "Stadthalle"
ort: "Remseck"
telefon: "0173 3015671"
gruppe: "Jun II"
klasse: "B"
sektion: "Lat"
titel: "Landesmeisterschaft TBW Latein"
nummer: 115314
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-04
partner: "Mühlschein, Alexander"
partnerin: "Mühlschein, Maren"
verein: "Stadthalle"
ort: "Remseck"
telefon: "0173 3015671"
gruppe: "Hgr II"
klasse: "A"
sektion: "Lat"
titel: "Landesmeisterschaft TBW Latein"
nummer: 115316
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-17
partner: "Mühlschein, Alexander"
partnerin: "Mühlschein, Maren"
verein: "Stadthalle Holzgerlingen"
ort: "Holzgerlingen"
telefon: "0162 8202156"
gruppe: "Mas I"
klasse: "A"
sektion: "Lat"
titel: "Landesmeisterschaft TBW Latein der Mas I-III D-S"
nummer: 114741
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-24
partner: "Kohler, Jürgen"
partnerin: "Kohler, Petra"
verein: "Schlossfeldhalle"
ort: "Achern-Grooßweier"
telefon: "0157 35720521"
gruppe: "Mas III"
klasse: "B"
sektion: "Std"
titel: "ATaTa 2024"
nummer: 114270
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-25
partner: "Kohler, Jürgen"
partnerin: "Kohler, Petra"
verein: "Tanz Sport Zentrum Sinsheim"
ort: "Sinsheim"
telefon: "0160 97701166"
gruppe: "Mas III"
klasse: "B"
sektion: "Std"
titel: "Sinsheimer Tanzsporttage 2024"
nummer: 115291
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-02-25
partner: "Lehmann, Christopher"
partnerin: "Broschell, Silvia"
verein: "Tanz Sport Zentrum Sinsheim"
ort: "Sinsheim"
telefon: "0160 97701166"
gruppe: "Mas III"
klasse: "B"
sektion: "Std"
titel: "Sinsheimer Tanzsporttage 2024"
nummer: 115291
---

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-03-02
partner: "Kohler, Jürgen"
partnerin: "Kohler, Petra"
verein: "Sport- und Festhalle Mergelstetten"
ort: "Heidenheim-Mergelstetten"
telefon: "0162 6845232"
gruppe: "Mas III"
klasse: "B"
sektion: "Std"
titel: "Mergelpokal 2024"
nummer: 115204
---

View File

@ -0,0 +1 @@
__pycache__/

View File

@ -0,0 +1,50 @@
from . import cli
from . import mail
from . import headerExtractor
from . import mailParser
from . import competitionParser
from . import mboxReader
import logging
import debugpy
import os
def main():
args = cli.getArgs()
logging.basicConfig()
logger = logging.getLogger(__name__)
verbosityMap = {
0: logging.WARNING,
1: logging.INFO,
}
rootLogger = logging.getLogger()
rootLogger.setLevel(verbosityMap.get(args.verbose, logging.DEBUG))
if args.debug:
debugpy.listen(5678)
debugpy.wait_for_client()
mp = mailParser.MailParser()
cp = competitionParser.CompetitionParser()
if args.read_mbox is not None:
if args.output_folder is None:
logger.error('Cannot use batch mode without explicit output folder.')
exit(1)
reader = mboxReader.MBocReader()
mails = reader.parseMBoxFile(args.read_mbox[0])
for mail in mails:
body = mp.parseMail(mail)
cp.parseMail(body)
filename = cp.getFilename(args.output_folder[0])
logger.info('Using file %s to generate the output.', filename)
folder = os.path.dirname(filename)
os.makedirs(folder, exist_ok=True)
with open(filename, 'w') as fp:
fp.write(cp.getContent())
else:
raise Exception('Not yet implemented')

View File

@ -0,0 +1,3 @@
import competitionNotificationReader
competitionNotificationReader.main()

View File

@ -0,0 +1,11 @@
import argparse
def getArgs():
parser = argparse.ArgumentParser()
parser.add_argument('--read-mbox', nargs=1, help='Read mails from mbox file instead of stdin')
parser.add_argument('-o', '--output-folder', nargs=1, help='Set the output folder of the generated files.')
parser.add_argument('-v', '--verbose', action='count', default=0, help='Increase the verbosity')
parser.add_argument('--debug', action='store_true', help='Enable python debugger')
return parser.parse_args()

View File

@ -0,0 +1,132 @@
import bs4
import logging
import re
import os
import jinja2
class ParsingFailedEception(Exception):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
class CompetitionParser:
def __init__(self):
self._l = logging.getLogger(__name__)
self._partner = ''
self._partnerin = ''
self._date = ''
self._title = ''
self._number = ''
self._group = ''
self._class = ''
self._section = ''
self._ort = ''
self._verein = ''
self._telefon = ''
self._reName = re.compile('Neue Meldung für (.*) / (.*)!')
self._reDate = re.compile('([0-9]+)\.([0-9]+)\.([0-9]+)')
self._reNumber = re.compile('Turnier: ([0-9]+)')
self._rePhone = re.compile('Telefon: ([0-9 /]+)')
self._rePlace = re.compile('Ort: (.*), (.*)')
self._reCompetition = re.compile('(.*) ([A-ES]) ((?:Std)|(?:Lat)|(?:Kombi))')
self._reCleaningString = re.compile('[^a-z0-9-]')
self._reDashes = re.compile('-+')
def parseMail(self, body: str):
parser = bs4.BeautifulSoup(body, 'html.parser')
self._getNames(parser.h2)
self._parseTable(parser.table)
def _getNames(self, h2):
matcher = self._reName.match(h2.string)
if matcher is None:
self._l.error('Parsing of header "%s" failed.', h2)
raise ParsingFailedEception('Header could not be successfully parsed')
self._partner = matcher.group(1)
self._partnerin = matcher.group(2)
def _parseTable(self, table):
def parseDate(date):
match = self._reDate.fullmatch(date)
if match is None:
raise ParsingFailedEception('Cannot parse date %s in mail' % date)
self._date = f'{match.group(3)}-{match.group(2)}-{match.group(1)}'
def parseNumber(content):
match = self._reNumber.fullmatch(content)
if match is None:
raise ParsingFailedEception(f'Cannot parse the turnier number in field {content}')
self._number = match.group(1)
def parseCompetition(competition):
match = self._reCompetition.fullmatch(competition)
if match is None:
raise ParsingFailedEception(f'Cannot parse the competition line {competition}')
self._group = match.group(1)
self._class = match.group(2)
self._section = match.group(3)
def parsePlace(place):
match = self._rePlace.fullmatch(place)
if match is None:
raise ParsingFailedEception(f'Cannot parse the place entry {place}')
self._verein = match.group(1)
self._ort = match.group(2)
def parsePhone(phone):
match = self._rePhone.fullmatch(phone)
if match is None:
raise ParsingFailedEception(f'Cannot parse the phone line {phone}')
self._telefon = match.group(1)
tds = table('td')
parseDate(tds[0].string.strip())
self._title = tds[1].string.strip()
parseNumber(tds[2].string.strip())
parseCompetition(tds[3].string.strip())
parsePlace(tds[4].string.strip())
parsePhone(tds[5].string.strip())
def _cleanName(self, name: str) -> str:
cleanedName = name.lower()
cleanedName = re.sub('ä', 'ae', cleanedName)
cleanedName = re.sub('ö', 'oe', cleanedName)
cleanedName = re.sub('ü', 'ue', cleanedName)
cleanedName = re.sub('ß', 'ss', cleanedName)
cleanedName = re.sub(self._reCleaningString, '-', cleanedName)
cleanedName = re.sub(self._reDashes, '-', cleanedName)
return cleanedName.lower()
def getFilename(self, prefix: str) -> str:
namePartner = self._cleanName(self._partner)
namePartnerin = self._cleanName(self._partnerin)
competition = f'{self._group} {self._class} {self._section}'
competitionName = self._cleanName(competition)
return os.path.join(
prefix,
self._date[0:4],
f'{self._date}-{self._ort.lower()}-{namePartner}-{namePartnerin}-{competitionName}.md'
)
def getContent(self) -> str:
with open(os.path.join(os.path.dirname(__file__), 'contenttemplate.md.tmpl')) as fp:
tpl = fp.read()
j2 = jinja2.Template(tpl)
vars = {
'date': self._date,
'partner': self._partner,
'partnerin': self._partnerin,
'verein': self._verein,
'ort': self._ort,
'telefon': self._telefon,
'group': self._group,
'class': self._class,
'section': self._section,
'title': self._title,
'number': self._number,
}
return j2.render(**vars)

View File

@ -0,0 +1,13 @@
---
dateCompetition: {{ date }}
partner: "{{ partner }}"
partnerin: "{{ partnerin }}"
verein: "{{ verein }}"
ort: "{{ ort }}"
telefon: "{{ telefon }}"
gruppe: "{{ group }}"
klasse: "{{ class }}"
sektion: "{{ section }}"
titel: "{{ title }}"
nummer: {{ number }}
---

View File

@ -0,0 +1,30 @@
import competitionNotificationReader as cnr
import logging
def splitHeaders(lines: list[str]) -> cnr.mail.Mail:
l = logging.getLogger(__name__)
l.debug('Separating headers of an email')
def _getHeaders(lines: list[str]):
headerLines = []
for idx,l in enumerate(lines):
if l == '':
remainingLines = lines[idx+1:]
for j,rl in enumerate(remainingLines):
if rl.strip() != '':
return headerLines, remainingLines[j:]
return headerLines, []
if l.startswith('\t') or l.startswith(' '):
lastLine = headerLines.pop()
newLine = f'{lastLine[1]} {l.strip()}'
headerLines.append(tuple([lastLine[0], newLine]))
else:
parts = l.split(':', 1)
headerLines.append(tuple([parts[0].strip(), parts[1].strip()]))
headerLines, bodyLines = _getHeaders(lines)
mail = cnr.mail.Mail(headerLines, bodyLines)
return mail

View File

@ -0,0 +1,11 @@
import dataclasses
HeaderName_t = str
HeaderValue_t = str
HeaderEntry_t = tuple[HeaderName_t, HeaderValue_t]
@dataclasses.dataclass
class Mail:
headers: list[HeaderEntry_t]
body: list[str]

View File

@ -0,0 +1,113 @@
import competitionNotificationReader as cnr
import logging
import re
class MailParser:
def __init__(self):
self._l = logging.getLogger(__name__)
def parseMail(self, rawMail: cnr.mail.Mail):
# Look for the correct Mail encoding
contentType, boundary = self._getContentType(rawMail)
subMails = self._splitMultipartBody(rawMail.body, boundary)
def isCorrectContentType(mail):
for header in mail.headers:
if header[0].lower() != 'content-type':
continue
return header[1].startswith('text/html')
return False
subMails = list(filter(isCorrectContentType, subMails))
def isCorrectContentEncoding(mail):
for header in mail.headers:
if header[0].lower() != 'content-transfer-encoding':
continue
return header[1] == 'quoted-printable'
return False
subMails = list(filter(isCorrectContentEncoding, subMails))
if len(subMails) != 1:
raise Exception('Not implemented')
body = self._mapQuotedrintable(subMails[0].body)
return body
def _getContentType(self, rawMail: cnr.mail.Mail) -> str:
ctHeaders = list(filter(lambda x: x[0].lower() == 'content-type', rawMail.headers))
if len(ctHeaders) != 1:
self._l.error('No unique content type of the mail was found.')
exit(1)
ct = ctHeaders[0][1]
if not ct.startswith('multipart/alternative'):
raise Exception('Not yet implemented')
parser = re.compile('.*boundary="([^"]+)"')
matcher = parser.match(ct)
if matcher is None:
self._l.error('Cannot extract boundary from mail header.')
exit(1)
boundary = matcher.group(1)
return 'multipart/alternative', boundary
def _splitMultipartBody(self, bodyLines: list[str], boundary: str):
parts = []
subBody = []
for line in bodyLines:
if line.startswith(f'--{boundary}'):
if len(subBody) > 0:
parts.append(subBody)
subBody = []
else:
subBody.append(line)
return list(map(lambda x: cnr.headerExtractor.splitHeaders(x), parts))
def _mapQuotedrintable(self, lines: list[str]):
def mergeLines():
# Drop terminating newlines
ret = [l for l in lines]
r = list(range(len(ret)))
r.reverse()
for i in r:
currentLine = ret[i]
if currentLine.endswith('='):
currentLine = currentLine[:-1] + ret.pop(i+1)
ret[i] = currentLine
return ret
mergedLines = mergeLines()
def mapUnicodeChars():
ret = []
for line in mergedLines:
i = 0
chars = []
while i < len(line):
if line[i] != '=':
chars.extend(list(line[i].encode()))
else:
hexChars = line[i+1:i+3]
value = int(hexChars, 16)
# print(f'{hexChars} -> {value}')
chars.append(value)
i += 2
i += 1
ret.append(chars)
return ret
mappedLines = mapUnicodeChars()
def decodeLine(l):
bytes = [x.to_bytes(1, 'big') for x in l]
decodedLine = b''.join(bytes).decode()
return decodedLine
decodedLines = list(map(decodeLine, mappedLines))
return ''.join(decodedLines)

View File

@ -0,0 +1,49 @@
import logging
import re
import io
import competitionNotificationReader as cnr
class MBocReader:
def __init__(self):
self._l = logging.getLogger(__name__)
def parseMBoxFile(self, filename: str) -> list[cnr.mail.Mail]:
self._l.debug('Reading MBox file "%s"', filename)
mails = []
with open(filename) as fp:
return self._parseMails(fp)
def _isNewMailLine(self, line: str):
return line.startswith('From ')
def _fixSingleLine(self, line: str) -> str:
regex = re.compile('^>+From ')
matcher = regex.match(line)
if matcher is None:
return line
return line[1:]
def _parseMails(self, fp: io.FileIO) -> list[cnr.mail.Mail]:
lines = []
mails = []
while True:
line = fp.readline()
if line == '':
if len(lines) > 0:
mails.append(self._parseSingleMail(lines))
return mails
if self._isNewMailLine(line):
if len(lines) > 0:
mails.append(self._parseSingleMail(lines))
lines = []
else:
lines.append(self._fixSingleLine(line[0:-1]))
def _parseSingleMail(self, lines: list[str]) -> cnr.mail.Mail:
return cnr.headerExtractor.splitHeaders(lines)

View File

@ -0,0 +1,5 @@
beautifulsoup4==4.12.2
debugpy==1.8.0
Jinja2==3.1.3
MarkupSafe==2.1.3
soupsieve==2.5

View File

@ -1,6 +1,6 @@
--- ---
title: "" title: "{{ replace .Name "-" " " | title }}"
date: 2023-11-11T00:00:00+01:00 date: {{ .Date }}
summary: |- summary: |-
Hier kommt die Zusammenfassung hin Hier kommt die Zusammenfassung hin
draft: false draft: false

View File

@ -0,0 +1,13 @@
---
dateCompetition: 2024-01-24
partner: Kohler, Jürgen
partnerin: Kohler, Petra
verein: Saalbau Haus Nidda
ort: Frankfurt a.M.
telefon: 0176 61745268
gruppe: Mas III
klasse: B
sektion: Std
titel: Die Goldene Schuhbürste 2024
nummer: 115126
---

View File

@ -905,6 +905,44 @@ table.time {
} }
} }
.turniermeldung-list {
.turniermeldung {
display: flex;
.date {
font-weight: bold;
flex: auto 0 0;
}
.ort {
margin: 0 0 0 10px;
flex: auto 1 0;
a {
width: 100%;
height: 100%;
}
}
}
}
.turnier-details {
.title {
font-weight: 600;
}
.turnier {
font-weight: bold;
.nummer {
font-weight: normal;
font-style: italic;
}
}
.verein {
margin: 20px 0 0;
}
.contact {
font-style: italic;
}
}
.iframe-generic { .iframe-generic {
display: block; display: block;
width: 100%; width: 100%;

View File

@ -0,0 +1,19 @@
{{ define "main" }}
<a name="top">
<h1>{{ .Title }}</h1>
</a>
{{ $meldungen := where .Site.RegularPages "Section" "==" "turniermeldung" }}
<div class="turniermeldung-list">
{{ range (sort $meldungen ".Params.dateCompetition" "asc") }}
{{ $date := time.AsTime .Params.dateCompetition }}
{{ if ge $date (now.AddDate 0 0 -1) }}
<div class="turniermeldung">
<div class="date">{{ $date.Format "02.01.2006" }}</div>
<div class="ort"><a href="{{ .RelPermalink }}">{{ .Params.ort }}</a></div>
</div>
{{ end }}
{{ end }}
</div>
{{ .Content }}
{{ partial "totop" }}
{{ end }}

View File

@ -31,6 +31,7 @@
($.Page.IsMenuCurrent "main" .) ($.Page.IsMenuCurrent "main" .)
($.Page.HasMenuCurrent "main" .) ($.Page.HasMenuCurrent "main" .)
(and (eq $.Page.Type "news") (eq .Identifier "aktuell") ) (and (eq $.Page.Type "news") (eq .Identifier "aktuell") )
(and (eq $.Page.Type "turniermeldung") (eq .Identifier "aktuell") )
}} active{{ end }}" href="{{ .URL }}"> }} active{{ end }}" href="{{ .URL }}">
{{/*{{ if .Pre }} {{/*{{ if .Pre }}
{{ $icon := printf "<i data-feather=\"%s\"></i> " .Pre | safeHTML }} {{ $icon := printf "<i data-feather=\"%s\"></i> " .Pre | safeHTML }}

View File

@ -4,6 +4,7 @@
($currentPage.IsMenuCurrent "main" .) ($currentPage.IsMenuCurrent "main" .)
($currentPage.HasMenuCurrent "main" .) ($currentPage.HasMenuCurrent "main" .)
(and (eq $currentPage.Type "news") (eq .Identifier "aktuell") ) (and (eq $currentPage.Type "news") (eq .Identifier "aktuell") )
(and (eq $currentPage.Type "turniermeldung") (eq .Identifier "aktuell") )
}} }}
{{ if .HasChildren }} {{ if .HasChildren }}
{{ range .Children }} {{ range .Children }}

View File

@ -0,0 +1,23 @@
{{ define "main" }}
{{ $datum := time.AsTime .Params.dateCompetition }}
<a name="top">
<h1>{{ .Params.ort }} am {{ $datum.Format "02.01.2006" }}</h1>
</a>
<h2>{{ .Params.partner }} / {{ .Params.partnerin }}</h2>
<div class="turnier-details">
{{ with .Params.titel }}
<div class="title">{{ . }}</div>
{{ end }}
<div class="turnier">
{{ .Params.gruppe }} {{ .Params.klasse }} {{ .Params.sektion }}
{{ with .Params.nummer }}
<span class="nummer">({{ . }})</span>
{{ end }}
</div>
<div class="verein">{{ .Params.verein }}</div>
<div class="ort">{{ .Params.ort }}</div>
<div class="contact">Telefon am Turniertag: {{ .Params.telefon }}</div>
</div>
{{ .Content }}
{{ partial "totop" }}
{{ end }}