calendar utils: Model for iCalendar calendars

.isort.cfg

@@ -4,4 +4,4 @@ line_length = 88
 multi_line_output = 3
 default_sectiont = "THIRDPARTY"
 include_trailing_comma = true
-known_third_party = django,environ,pirates,wagtail
+known_third_party = arrow,django,environ,faker,ics,pirates,pytest,pytz,requests,snapshottest,wagtail

README.md

@@ -26,9 +26,11 @@ jako přehled pluginů a rozšíření pro Wagtail.
 
     .
     ├── home            = app na web úvodní stránky Majáku
+    ├── senat_campaign  = app na weby kandidátů na senátory
+    ...
     ├── majak           = Django projekt s konfigurací Majáku
+    ├── calendar_utils  = app s modelem a utilitami na iCal kalendáře
     ├── search          = app pro fulltext search (default, asi se k ničemu nepoužívá)
-    ├── senat_campaign  = app na weby kandidátů na senátory
     └── users           = app s custom user modelem a SSO, apod.
 
 Appky v sobě mají modely pro stránky a statické soubory a templaty. Momentálně se
@@ -55,6 +57,14 @@ V produkci musí být navíc nastaveno:
 | `DJANGO_SECRET_KEY` | | tajný šifrovací klíč |
 | `DJANGO_ALLOWED_HOSTS` | | allowed hosts (více hodnot odděleno čárkami) |
 
+### Management commands
+
+Přes CRON je třeba na pozadí spouštět Django `manage.py` commandy:
+
+* `clearsessions` - maže expirované sessions (denně až týdně)
+* `publish_scheduled_pages` - publikuje naplánované stránky (každou hodinu)
+* `update_callendars` - stáhne a aktualizuje kalendáře (několikrát denně)
+
 ### Přidání nového webu
 
 Doména či subdoména se musí nakonfigurovat v:

calendar_utils/apps.py

+from django.apps import AppConfig
+
+
+class CalendarUtilsConfig(AppConfig):
+    name = "calendar utils"

calendar_utils/management/commands/update_callendars.py

+from django.core.management.base import BaseCommand
+
+from ...models import Calendar
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        self.stdout.write("Updating calendars...")
+        for cal in Calendar.objects.all():
+            self.stdout.write(f"+ {cal.url}")
+            cal.update_source()
+        self.stdout.write("Updating calendars finished!")

calendar_utils/migrations/0001_initial.py

+# Generated by Django 3.0.6 on 2020-05-22 08:30
+
+import django.core.serializers.json
+from django.db import migrations, models
+
+import calendar_utils.models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="Calendar",
+            fields=[
+                (
+                    "id",
+                    models.AutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("url", models.URLField(unique=True)),
+                ("source", models.TextField(null=True)),
+                ("last_update", models.DateTimeField(null=True)),
+                (
+                    "past_events",
+                    calendar_utils.models.EventsJSONField(
+                        encoder=django.core.serializers.json.DjangoJSONEncoder,
+                        null=True,
+                    ),
+                ),
+                (
+                    "future_events",
+                    calendar_utils.models.EventsJSONField(
+                        encoder=django.core.serializers.json.DjangoJSONEncoder,
+                        null=True,
+                    ),
+                ),
+            ],
+        ),
+    ]

calendar_utils/models.py

+import arrow
+import requests
+from django.contrib.postgres.fields import JSONField
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import models
+
+from .parser import parse_ical, split_events
+
+
+def _convert_arrow_to_datetime(event):
+    event["begin"] = event["begin"].datetime
+    event["end"] = event["end"].datetime
+    return event
+
+
+class EventsJSONField(JSONField):
+    """
+    JSONField for lists of events which converts `begin` and `end` to datetime
+    on load from DB.
+    """
+
+    def from_db_value(self, value, expression, connection):
+        if value:
+            for event in value:
+                event["begin"] = arrow.get(event["begin"]).datetime
+                event["end"] = arrow.get(event["end"]).datetime
+        return value
+
+
+class Calendar(models.Model):
+    url = models.URLField(unique=True)
+    source = models.TextField(null=True)
+    last_update = models.DateTimeField(null=True)
+    past_events = EventsJSONField(encoder=DjangoJSONEncoder, null=True)
+    future_events = EventsJSONField(encoder=DjangoJSONEncoder, null=True)
+
+    def update_source(self):
+        source = requests.get(self.url).text
+        if self.source != source:
+            self.source = source
+            past, future = split_events(parse_ical(source))
+            self.past_events = list(map(_convert_arrow_to_datetime, past))
+            self.future_events = list(map(_convert_arrow_to_datetime, future))
+        self.last_update = arrow.utcnow().datetime
+        self.save()

calendar_utils/parser.py

+from operator import attrgetter
+
+import arrow
+from ics import Calendar
+
+EVENT_KEYS = ("begin", "end", "all_day", "name", "description", "location")
+
+
+def parse_ical(source):
+    """Parses iCalendar source and returns events as list of dicts"""
+    cal = Calendar(source)
+    events = []
+    for event in sorted(cal.events, key=attrgetter("begin"), reverse=True):
+        events.append({key: getattr(event, key) for key in EVENT_KEYS})
+    return events
+
+
+def split_events(events):
+    """Splits events and returns list of past events and future events"""
+    now = arrow.utcnow()
+    past = [ev for ev in events if ev["begin"] < now]
+    future = [ev for ev in events if ev["begin"] > now]
+    return past, future

majak/settings/base.py

@@ -33,6 +33,7 @@ DATABASES["default"]["ATOMIC_REQUESTS"] = True
 INSTALLED_APPS = [
     "senat_campaign",
     "home",
+    "calendar_utils",
     "users",
     "pirates",
     "search",

pytest.ini

 [pytest]
-addopts = --ds=majak.settings.dev
+addopts = --ds=majak.settings.dev -p no:warnings --nomigrations
 python_files = test_*.py

requirements/base.in

@@ -6,3 +6,6 @@ psycopg2-binary
 pirates<=0.4
 whitenoise
 opencv-python
+requests
+ics
+arrow

requirements/base.txt

@@ -4,6 +4,7 @@
 #
 #    pip-compile base.in
 #
+arrow==0.14.7             # via -r base.in, ics
 asgiref==3.2.7            # via django
 beautifulsoup4==4.8.2     # via wagtail
 certifi==2020.4.5.1       # via requests
@@ -20,6 +21,7 @@ django==3.0.6             # via django-taggit, django-treebeard, djangorestframe
 djangorestframework==3.11.0  # via wagtail
 draftjs-exporter==2.1.7   # via wagtail
 html5lib==1.0.1           # via wagtail
+ics==0.7                  # via -r base.in
 idna==2.9                 # via requests
 josepy==1.3.0             # via mozilla-django-oidc
 l18n==2018.5              # via wagtail
@@ -33,12 +35,14 @@ pyasn1-modules==0.2.8     # via python-ldap
 pyasn1==0.4.8             # via pyasn1-modules, python-ldap
 pycparser==2.20           # via cffi
 pyopenssl==19.1.0         # via josepy
+python-dateutil==2.8.1    # via arrow, ics
 python-ldap==3.2.0        # via pirates
 pytz==2020.1              # via django, django-modelcluster, l18n
-requests==2.23.0          # via mozilla-django-oidc, wagtail
-six==1.14.0               # via cryptography, django-extensions, html5lib, josepy, l18n, mozilla-django-oidc, pyopenssl
+requests==2.23.0          # via -r base.in, mozilla-django-oidc, wagtail
+six==1.14.0               # via cryptography, django-extensions, html5lib, ics, josepy, l18n, mozilla-django-oidc, pyopenssl, python-dateutil
 soupsieve==2.0            # via beautifulsoup4
 sqlparse==0.3.1           # via django
+tatsu==5.5.0              # via ics
 unidecode==1.1.1          # via wagtail
 urllib3==1.25.9           # via requests
 wagtail==2.9              # via -r base.in

requirements/dev.in

@@ -4,4 +4,5 @@ pytest-factoryboy
 pytest-cov
 pytest-django
 pytest-freezegun
+pytest-mock
 snapshottest

requirements/dev.txt

@@ -20,8 +20,9 @@ pytest-cov==2.8.1         # via -r dev.in
 pytest-django==3.9.0      # via -r dev.in
 pytest-factoryboy==2.0.3  # via -r dev.in
 pytest-freezegun==0.4.1   # via -r dev.in
+pytest-mock==3.1.0        # via -r dev.in
 pytest-sugar==0.9.3       # via -r dev.in
-pytest==5.4.2             # via -r dev.in, pytest-cov, pytest-django, pytest-factoryboy, pytest-freezegun, pytest-sugar
+pytest==5.4.2             # via -r dev.in, pytest-cov, pytest-django, pytest-factoryboy, pytest-freezegun, pytest-mock, pytest-sugar
 python-dateutil==2.8.1    # via faker, freezegun
 six==1.14.0               # via freezegun, packaging, python-dateutil, snapshottest
 snapshottest==0.5.1       # via -r dev.in

tests/calendar_utils/conftest.py

+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture(scope="session")
+def sample():
+    return (Path(__file__).parent / "sample.ics").read_text()