From 9795ca99c6d2695887604dff7f73a3d2c07ba4b9 Mon Sep 17 00:00:00 2001
From: OndraRehounek <ondra.rehounek@seznam.cz>
Date: Thu, 5 May 2022 15:24:29 +0200
Subject: [PATCH] calendar_utils: Replace ics library with icalevnt
---
.isort.cfg | 2 +-
...ove_calendar_source_calendar_event_hash.py | 22 +++++
.../migrations/0004_auto_20220505_1228.py | 42 +++++++++
calendar_utils/models.py | 33 ++++---
calendar_utils/parser.py | 89 ++++++++++---------
district/templates/district/base.html | 4 +-
.../elections2021_calendar_page.html | 4 +-
.../elections2021_home_page.html | 4 +-
region/templates/region/base.html | 4 +-
requirements/base.in | 1 +
requirements/base.txt | 25 ++++--
.../calendar_event_snippet.html | 6 +-
.../calendar_current_events_snippet.html | 10 +--
.../shared/small_calendar_snippet.html | 6 +-
.../uniweb/blocks/calendar_agenda.html | 6 +-
15 files changed, 175 insertions(+), 83 deletions(-)
create mode 100644 calendar_utils/migrations/0003_remove_calendar_source_calendar_event_hash.py
create mode 100644 calendar_utils/migrations/0004_auto_20220505_1228.py
diff --git a/.isort.cfg b/.isort.cfg
index 575531da..eadf7557 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -3,4 +3,4 @@
line_length = 88
multi_line_output = 3
include_trailing_comma = true
-known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,django,environ,faker,fastjsonschema,ics,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml
+known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,django,environ,faker,fastjsonschema,icalevnt,markdown,modelcluster,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,wagtail,wagtailmetadata,weasyprint,yaml,zoneinfo
diff --git a/calendar_utils/migrations/0003_remove_calendar_source_calendar_event_hash.py b/calendar_utils/migrations/0003_remove_calendar_source_calendar_event_hash.py
new file mode 100644
index 00000000..6c4c356d
--- /dev/null
+++ b/calendar_utils/migrations/0003_remove_calendar_source_calendar_event_hash.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.0.4 on 2022-05-05 10:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("calendar_utils", "0002_auto_20200523_0243"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="calendar",
+ name="source",
+ ),
+ migrations.AddField(
+ model_name="calendar",
+ name="event_hash",
+ field=models.CharField(max_length=256, null=True),
+ ),
+ ]
diff --git a/calendar_utils/migrations/0004_auto_20220505_1228.py b/calendar_utils/migrations/0004_auto_20220505_1228.py
new file mode 100644
index 00000000..f8ff43ff
--- /dev/null
+++ b/calendar_utils/migrations/0004_auto_20220505_1228.py
@@ -0,0 +1,42 @@
+# Generated by Django 4.0.4 on 2022-05-05 10:28
+from datetime import date, timedelta
+
+import arrow
+from django.db import migrations
+from icalevnt import icalevents
+
+from calendar_utils.parser import process_event_list
+
+
+def update_event_lists(apps, schema):
+ Calendar = apps.get_model("calendar_utils", "Calendar")
+
+ for calendar in Calendar.objects.all():
+ try:
+ event_list = icalevents.events(
+ url=calendar.url,
+ start=date.today() - timedelta(days=30),
+ end=date.today() + timedelta(days=60),
+ )
+ except ValueError:
+ print("Could not parse calendar from {}".format(calendar.url))
+
+ event_list_hash = str(hash(str(event_list)))
+ past, future = process_event_list(event_list)
+ calendar.past_events = past
+ calendar.future_events = list(reversed(future))
+ calendar.event_hash = event_list_hash
+
+ calendar.last_update = arrow.utcnow().datetime
+ calendar.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("calendar_utils", "0003_remove_calendar_source_calendar_event_hash"),
+ ]
+
+ operations = [
+ migrations.RunPython(update_event_lists, reverse_code=migrations.RunPython.noop)
+ ]
diff --git a/calendar_utils/models.py b/calendar_utils/models.py
index 53f92e8a..d0717399 100644
--- a/calendar_utils/models.py
+++ b/calendar_utils/models.py
@@ -1,15 +1,15 @@
-import re
+from datetime import date, timedelta
import arrow
-import requests
from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
+from icalevnt import icalevents
-from .parser import process_ical
+from .parser import process_event_list
def _convert_arrow_to_datetime(event):
- event["begin"] = event["begin"].datetime
+ event["start"] = event["start"].datetime
event["end"] = event["end"].datetime
return event
@@ -24,7 +24,7 @@ class EventsJSONField(models.JSONField):
value = super().from_db_value(value, expression, connection)
if value:
for event in value:
- event["begin"] = arrow.get(event["begin"]).datetime
+ event["start"] = arrow.get(event.get("start")).datetime
event["end"] = arrow.get(event["end"]).datetime
return value
@@ -33,20 +33,25 @@ class Calendar(models.Model):
CURRENT_NUM = 6
url = models.URLField()
- source = models.TextField(null=True)
+ event_hash = models.CharField(max_length=256, null=True)
last_update = models.DateTimeField(null=True)
past_events = EventsJSONField(encoder=DjangoJSONEncoder, null=True)
future_events = EventsJSONField(encoder=DjangoJSONEncoder, null=True)
def update_source(self):
- source = requests.get(self.url).text
- if self.source != source:
- self.source = source
- past, future = process_ical(source)
- self.past_events = list(map(_convert_arrow_to_datetime, past))
- self.future_events = list(
- reversed(list(map(_convert_arrow_to_datetime, future)))
- )
+ event_list = icalevents.events(
+ url=self.url,
+ start=date.today() - timedelta(days=30),
+ end=date.today() + timedelta(days=60),
+ )
+
+ event_list_hash = str(hash(str(event_list)))
+ if event_list_hash != self.event_hash:
+ past, future = process_event_list(event_list)
+ self.past_events = past
+ self.future_events = list(reversed(future))
+ self.event_hash = event_list_hash
+
self.last_update = arrow.utcnow().datetime
self.save()
diff --git a/calendar_utils/parser.py b/calendar_utils/parser.py
index c7c453c4..978ca57a 100644
--- a/calendar_utils/parser.py
+++ b/calendar_utils/parser.py
@@ -1,74 +1,81 @@
-import re
-from operator import attrgetter
+from operator import itemgetter
+from typing import TYPE_CHECKING
import arrow
import bleach
from django.conf import settings
-from ics import Calendar
+from zoneinfo import ZoneInfo
-EVENT_KEYS = ("begin", "end", "all_day", "name", "description", "location")
+if TYPE_CHECKING:
+ from icalevnt.icalparser import Event
+# FIXME "name" is "summary" and "begin" is "start" now
+EVENT_KEYS = ("start", "end", "all_day", "summary", "description", "location")
-def remove_alarms(source):
- """Removes VALARM blocks from iCal source."""
- return re.sub(r"(BEGIN:VALARM.*?END:VALARM\r?\n)", "", source, flags=re.S)
-
-def parse_ical(source):
- """Parses iCalendar source and returns events as list of dicts."""
- # there is a bug in parsing alarms, but we don't need them, so we remove them
- source = remove_alarms(source)
- cal = Calendar(source)
- events = []
- for event in sorted(cal.events, key=attrgetter("begin"), reverse=True):
- events.append({key: getattr(event, key) for key in EVENT_KEYS})
- return events
-
-
-def split_events(events):
+def split_event_dict_list(event_list: "list[dict]") -> tuple[list[dict], list[dict]]:
"""Splits events and returns list of past events and future events."""
singularity = arrow.utcnow().shift(hours=-2)
- past = [ev for ev in events if ev["end"] < singularity]
- future = [ev for ev in events if ev["end"] > singularity]
+
+ past = [ev for ev in event_list if ev["end"] < singularity]
+ future = [ev for ev in event_list if ev["end"] > singularity]
+
return past, future
-def set_event_description(event):
+def set_event_description(event: "Event") -> "Event":
"""Clears even description from unwanted tags."""
- description: str = event.get("description", "") or ""
- event["description"] = bleach.clean(description, tags=["a", "br"], strip=True)
+ description: str = event.description or ""
+ event.description = bleach.clean(description, tags=["a", "br"], strip=True)
return event
-def set_event_duration(event):
+def set_event_duration(event: "Event") -> "Event":
"""Sets duration for event."""
- if event["all_day"]:
- event["duration"] = "celý den"
+ if event.all_day:
+ event.duration = "celý den"
return event
- delta = event["end"] - event["begin"]
+ delta = event.end - event.start
if delta.days < 1:
- begin = event["begin"].to(settings.TIME_ZONE).format("H:mm")
- end = event["end"].to(settings.TIME_ZONE).format("H:mm")
- event["duration"] = f"{begin} - {end}"
+ start = event.start.strftime("%H:%M") # .to(settings.TIME_ZONE).format("H:mm")
+ end = event.end.strftime("%H:%M")
+ event.duration = f"{start} - {end}"
else:
- begin = event["begin"].to(settings.TIME_ZONE).format("H:mm")
- end = event["end"].to(settings.TIME_ZONE).format("H:mm (D.M.)")
- event["duration"] = f"{begin} - {end}"
+ start = event.start.strftime("%H:%M")
+ end = event.end.strftime("%H:%M")
+ event.duration = f"{start} - {end}"
return event
-def process_event(event):
+def set_event_timezone(event: "Event") -> "Event":
+ """Sets default project timezone for event if missing."""
+ if (
+ not event.start.tzinfo
+ or not event.start.tzinfo.utcoffset(event.start)
+ or not event.end.tzinfo
+ or not event.end.tzinfo.utcoffset(event.end)
+ ):
+ event.start = event.start.replace(tzinfo=ZoneInfo(settings.TIME_ZONE))
+ event.end = event.end.replace(tzinfo=ZoneInfo(settings.TIME_ZONE))
+ return event
+
+
+def process_event(event: "Event") -> dict:
"""Processes single event for use in Majak"""
+ event = set_event_timezone(event)
event = set_event_duration(event)
event = set_event_description(event)
- return event
+ # for event in sorted(cal.events, key=attrgetter("start"), reverse=True): TODO check
+ return {key: getattr(event, key) for key in EVENT_KEYS}
-def process_ical(source):
+def process_event_list(event_list: "list[Event]") -> tuple[list[dict], list[dict]]:
"""Parses iCalendar source and returns events as list of dicts. Returns
tuple of past and future events.
"""
- events = parse_ical(source)
- events = list(map(process_event, events))
- return split_events(events)
+ processed_event_list = list(map(process_event, event_list))
+ processed_event_list = sorted(
+ processed_event_list, key=itemgetter("start"), reverse=True
+ )
+ return split_event_dict_list(processed_event_list)
diff --git a/district/templates/district/base.html b/district/templates/district/base.html
index 0d5f9d45..ce022f9a 100644
--- a/district/templates/district/base.html
+++ b/district/templates/district/base.html
@@ -175,8 +175,8 @@
{% if first_event %}
<span class="btn text-sm max-w-full hidden lg:block" @click="toggleView('calendar')">
<div class="btn__body bg-grey-800 text-grey-200 flex divide-x">
- <span class="pr-4">{{ first_event.name }}</span>
- <span class="pl-4">{{ first_event.begin|date:"l j. E"|capfirst }}</span>
+ <span class="pr-4">{{ first_event.summary }}</span>
+ <span class="pl-4">{{ first_event.start|date:"l j. E"|capfirst }}</span>
</div>
</span>
{% endif %}
diff --git a/elections2021/templates/elections2021/elections2021_calendar_page.html b/elections2021/templates/elections2021/elections2021_calendar_page.html
index 25245da7..ca07a3e0 100644
--- a/elections2021/templates/elections2021/elections2021_calendar_page.html
+++ b/elections2021/templates/elections2021/elections2021_calendar_page.html
@@ -171,11 +171,11 @@
{% if not request.GET.region or request.GET.region == event.pir.region %}
<div class="grid grid-cols-12 items-center calendar-table-row my-1">
<div class="col-span-2 head-alt-md calendar-table-row__col" style="color: #92ac00">
- <span>{{ event.begin|date:"j." }}</span>
+ <span>{{ event.start|date:"j." }}</span>
</div>
<div class="col-span-8 grid grid-cols-3 calendar-table-row__col">
<div class="col-span-3 md:col-span-1">
- <strong class="block">{{ event.begin|date:"l j. E"|capfirst }}</strong>
+ <strong class="block">{{ event.start|date:"l j. E"|capfirst }}</strong>
<p class="font-light text-sm mt-1">{{ event.duration }}</p>
</div>
<div class="col-span-3 md:col-span-2 mt-4 md:mt-0">
diff --git a/elections2021/templates/elections2021/elections2021_home_page.html b/elections2021/templates/elections2021/elections2021_home_page.html
index 03f5b850..d524ae2c 100644
--- a/elections2021/templates/elections2021/elections2021_home_page.html
+++ b/elections2021/templates/elections2021/elections2021_home_page.html
@@ -114,11 +114,11 @@
{% for event in page.calendar.current_events %}
<div class="grid grid-cols-12 items-center my-1">
<div class="col-span-2 head-alt-md calendar-table-row__col text-acidgreen border-acidgreen">
- <span>{{ event.begin|date:"j." }}</span>
+ <span>{{ event.start|date:"j." }}</span>
</div>
<div class="col-span-8 grid grid-cols-3 calendar-table-row__col text-white border-black">
<div class="col-span-3 md:col-span-1">
- <strong class="block">{{ event.begin|date:"l j. E"|capfirst }}</strong>
+ <strong class="block">{{ event.start|date:"l j. E"|capfirst }}</strong>
<p class="font-light text-sm mt-1">{{ event.duration }}</p>
</div>
<div class="col-span-3 md:col-span-2 mt-4 md:mt-0">
diff --git a/region/templates/region/base.html b/region/templates/region/base.html
index 6ee50a4c..df54a1ac 100644
--- a/region/templates/region/base.html
+++ b/region/templates/region/base.html
@@ -160,8 +160,8 @@
{% if first_event %}
<span class="btn text-sm max-w-full hidden lg:block" @click="toggleView('calendar')">
<div class="btn__body bg-grey-800 text-grey-200 flex divide-x">
- <span class="pr-4">{{ first_event.name }}</span>
- <span class="pl-4">{{ first_event.begin|date:"l j. E"|capfirst }}</span>
+ <span class="pr-4">{{ first_event.summary }}</span>
+ <span class="pl-4">{{ first_event.start|date:"l j. E"|capfirst }}</span>
</div>
</span>
{% endif %}
diff --git a/requirements/base.in b/requirements/base.in
index 4dfa9baf..88cf58c9 100644
--- a/requirements/base.in
+++ b/requirements/base.in
@@ -11,6 +11,7 @@ pirates<=0.7
whitenoise
opencv-python
requests
+icalevnt
ics
arrow
sentry-sdk
diff --git a/requirements/base.txt b/requirements/base.txt
index 07f11e5d..c5f3caf1 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -8,8 +8,6 @@ amqp==5.1.1
# via kombu
anyascii==0.3.1
# via wagtail
-appnope==0.1.3
- # via ipython
arrow==0.14.7
# via
# -r base.in
@@ -63,6 +61,8 @@ cryptography==37.0.2
# pyopenssl
cssselect2==0.6.0
# via weasyprint
+datetime==4.3
+ # via icalevnt
decorator==5.1.1
# via ipython
deprecated==1.2.13
@@ -118,6 +118,12 @@ html5lib==1.1
# via
# wagtail
# weasyprint
+httplib2==0.20.1
+ # via icalevnt
+icalendar==4.0.8
+ # via icalevnt
+icalevnt==0.1.26
+ # via -r base.in
ics==0.7
# via -r base.in
idna==3.3
@@ -177,8 +183,10 @@ pygments==2.12.0
# via ipython
pyopenssl==22.0.0
# via josepy
-pyparsing==3.0.8
- # via packaging
+pyparsing==2.4.7
+ # via
+ # httplib2
+ # packaging
pypdf2==1.27.12
# via -r base.in
pyphen==0.12.0
@@ -186,12 +194,17 @@ pyphen==0.12.0
python-dateutil==2.8.2
# via
# arrow
+ # icalendar
+ # icalevnt
# ics
-pytz==2022.1
+pytz==2021.3
# via
# celery
+ # datetime
# django-modelcluster
# djangorestframework
+ # icalendar
+ # icalevnt
# l18n
pyyaml==6.0
# via -r base.in
@@ -270,6 +283,8 @@ xlsxwriter==3.0.3
# via wagtail
xlwt==1.3.0
# via tablib
+zope-interface==5.4.0
+ # via datetime
zopfli==0.2.1
# via fonttools
diff --git a/senat_campaign/templates/senat_campaign/calendar_event_snippet.html b/senat_campaign/templates/senat_campaign/calendar_event_snippet.html
index b8cee4a8..92d923b1 100644
--- a/senat_campaign/templates/senat_campaign/calendar_event_snippet.html
+++ b/senat_campaign/templates/senat_campaign/calendar_event_snippet.html
@@ -1,13 +1,13 @@
<div class="calendar__row">
<div class="calendar__row__date1">
- <h3>{{ event.begin|date:"d" }}.</h3>
+ <h3>{{ event.start|date:"d" }}.</h3>
</div>
<div class="calendar__row__date2">
- <h6>{{ event.begin|date:"j.n.Y" }}</h6>
+ <h6>{{ event.start|date:"j.n.Y" }}</h6>
<p>{{ event.duration }}</p>
</div>
<div class="calendar__row__content">
- <h6>{{ event.name }}</h6>
+ <h6>{{ event.summary }}</h6>
<p>{{ event.location }}</p>
</div>
</div><!-- /calendar__row -->
diff --git a/shared/templates/shared/calendar_current_events_snippet.html b/shared/templates/shared/calendar_current_events_snippet.html
index 0913caaa..0c95f7e7 100644
--- a/shared/templates/shared/calendar_current_events_snippet.html
+++ b/shared/templates/shared/calendar_current_events_snippet.html
@@ -5,6 +5,7 @@
<i class="ico--calendar banner__icon"></i>
<div class="banner__body"><h1 class="head-alt-md banner__cta">Kalendář</h1>
<button class="btn btn--white btn--fullwidth sm:btn--autowidth mt-8">
+ {{ event }}
<div class="btn__body">Zobrazit další</div>
</button>
</div>
@@ -14,15 +15,15 @@
{% for event in page.root_page.calendar.current_events %}
<div class="grid grid-cols-12 items-center calendar-table-row">
<div class="col-span-2 text-orange-300 head-alt-md calendar-table-row__col">
- <span>{{ event.begin|date:"j." }}</span>
+ <span>{{ event.start|date:"j." }}</span>
</div>
<div class="col-span-8 grid grid-cols-3 col-gap-4 calendar-table-row__col calendar-table-row__col--norborder">
<div class="col-span-3 md:col-span-1">
- <strong class="block">{{ event.begin|date:"l j. E"|capfirst }}</strong>
+ <strong class="block">{{ event.start|date:"l j. E"|capfirst }}</strong>
<p class="font-light text-sm mt-1">{{ event.duration }}</p></div>
<div class="col-span-3 md:col-span-2 mt-4 md:mt-0 overflow-hidden">
<span class="font-bold block">
- {{ event.name }}
+ {{ event.summary }}
</span>
{% if event.description %}
<p class="font-light text-sm mt-1">
@@ -34,8 +35,7 @@
<div class="col-span-2 text-center font-light calendar-table-row__col">
{% if event.location and 'jitsi.pirati' in event.location %}
<a href="{{ event.location }}" class="icon-link">
- <i aria-hidden="true" class="ico--link text-violet-300 mr-1"></i>
-{# <i aria-hidden="true" class="ico--jitsi text-violet-300 mr-1"></i> TODO requires latest styleguide version #}
+ <i aria-hidden="true" class="ico--jitsi text-violet-300 mr-1"></i>
<span>Jitsi URL</span>
</a>
{% elif event.location %}
diff --git a/shared/templates/shared/small_calendar_snippet.html b/shared/templates/shared/small_calendar_snippet.html
index ab10f212..dcb59eee 100644
--- a/shared/templates/shared/small_calendar_snippet.html
+++ b/shared/templates/shared/small_calendar_snippet.html
@@ -7,15 +7,15 @@
{% for event in events|default:page.root_page.calendar.current_events %}
<div class="grid grid-cols-12 items-center calendar-table-row calendar-table-row--standalone">
<div class="col-span-2 text-orange-300 head-alt-md calendar-table-row__col">
- <span>{{ event.begin|date:"j." }}</span>
+ <span>{{ event.start|date:"j." }}</span>
</div>
<div class="col-span-8 grid grid-cols-3 calendar-table-row__col">
<div class="col-span-3 md:col-span-1">
- <strong class="block">{{ event.begin|date:"l j. E"|capfirst }}</strong>
+ <strong class="block">{{ event.start|date:"l j. E"|capfirst }}</strong>
<p class="font-light text-sm mt-1">{{ event.duration }}</p></div>
<div class="col-span-3 md:col-span-2 mt-4 md:mt-0">
<span class="font-bold block">
- {{ event.name }}
+ {{ event.summary }}
</span>
{% if event.description %}
<p class="font-light text-sm mt-1">{{ event.description | safe }}</p>
diff --git a/uniweb/templates/uniweb/blocks/calendar_agenda.html b/uniweb/templates/uniweb/blocks/calendar_agenda.html
index ba8f6528..c45f1caf 100644
--- a/uniweb/templates/uniweb/blocks/calendar_agenda.html
+++ b/uniweb/templates/uniweb/blocks/calendar_agenda.html
@@ -2,15 +2,15 @@
{% for event in events %}
<div class="grid grid-cols-12 items-center calendar-table-row calendar-table-row--standalone">
<div class="col-span-2 text-blue-200 head-alt-md calendar-table-row__col">
- <span>{{ event.begin|date:"j." }}</span>
+ <span>{{ event.start|date:"j." }}</span>
</div>
<div class="col-span-8 grid grid-cols-3 calendar-table-row__col">
<div class="col-span-3 md:col-span-1">
- <strong class="block">{{ event.begin|date:"l j. E"|capfirst }}</strong>
+ <strong class="block">{{ event.start|date:"l j. E"|capfirst }}</strong>
<p class="font-light text-sm mt-1">{{ event.duration }}</p>
</div>
<div class="col-span-3 md:col-span-2 mt-4 md:mt-0">
- <strong class="block">{{ event.name }}</strong>
+ <strong class="block">{{ event.summary }}</strong>
{% if event.description %}
<p class="font-light text-sm mt-1">{{ event.description }}</p>
{% endif %}
--
GitLab