.gitignore

@@ -147,3 +147,9 @@ media_files/
 static_files/
 .python-version
 .idea/
+
+update_election_statics.sh
+download_static.sh
+matice.csv
+
+.vscode/

.gitlab-ci.yml

-image: docker:19.03.1
+stages:
+  - build
+
+image: docker:20.10.8
 
 variables:
   DOCKER_TLS_CERTDIR: "/certs"
@@ -6,7 +9,7 @@ variables:
   IMAGE_TAG_NGINX: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-nginx
 
 services:
-  - docker:19.03.1-dind
+  - docker:20.10.8-dind
 
 before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
@@ -14,14 +17,14 @@ before_script:
 build_app:
   stage: build
   script:
-    - docker pull $CI_REGISTRY_IMAGE:master || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:master -t $IMAGE_TAG_APP .
+    - docker pull $CI_REGISTRY_IMAGE:test || true
+    - docker build --cache-from $CI_REGISTRY_IMAGE:test -t $IMAGE_TAG_APP .
     - docker push $IMAGE_TAG_APP
 
 build_nginx:
   stage: build
   when: manual
   script:
-    - docker pull $CI_REGISTRY_IMAGE:master-nginx || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:master-nginx -t $IMAGE_TAG_NGINX . -f Dockerfile.nginx
+    - docker pull $CI_REGISTRY_IMAGE:test-nginx || true
+    - docker build --cache-from $CI_REGISTRY_IMAGE:test-nginx -t $IMAGE_TAG_NGINX . -f Dockerfile.nginx
     - docker push $IMAGE_TAG_NGINX

.isort.cfg

@@ -2,6 +2,5 @@
 # config compatible with Black
 line_length = 88
 multi_line_output = 3
-default_section = "THIRDPARTY"
 include_trailing_comma = true
-known_third_party = arrow,django,environ,faker,ics,modelcluster,nbconvert,pirates,pytest,pytz,requests,sentry_sdk,snapshottest,taggit,traitlets,wagtail,wagtailmetadata
+known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,clamd,dateutil,django,django_ratelimit,environ,faker,fastjsonschema,gql,httplib2,icalendar,instaloader,markdown,modelcluster,nh3,pirates,pytest,pytz,requests,sentry_sdk,taggit,wagtail,wagtailmetadata,weasyprint,willow,yaml

.pre-commit-config.yaml

 default_language_version:
-  python: python3.7
+  python: python3.11
 
 exclude: snapshots/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.3.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
         exclude: ^.*\.md$
@@ -21,11 +21,22 @@ repos:
       - id: seed-isort-config
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.12.0
     hooks:
       - id: isort
 
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+    -   id: autoflake
+
   - repo: https://github.com/psf/black
-    rev: 20.8b1
+    rev: 23.1.0
     hooks:
       - id: black
+        args: ["-t", "py310"]
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+    - id: autoflake
+      args: [--remove-all-unused-imports, --in-place]

Dockerfile

-FROM python:3.8
+FROM python:3.11
 
 RUN apt-get update && apt-get install -y \
     # requirements for OpenCV
     libgl1-mesa-dev \
-    # requirements to build python-ldap
-    build-essential python3-dev libldap2-dev libsasl2-dev \
+    # requirements for WeasyPrint
+    python3-cffi \
+    # requirements for Wand (GIF image resizing)
+    libmagickwand-dev \
     && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /app

Dockerfile.nginx

-FROM nginx:1.18
+FROM nginx:latest
 EXPOSE 8080
 ADD nginx.conf /etc/nginx/conf.d/majak.conf

Makefile

@@ -10,10 +10,12 @@ help:
 	@echo "  install        Install dependencies to venv"
 	@echo "  install-hooks  Install pre-commit hooks"
 	@echo "  hooks          Run pre-commit hooks manually"
+	@echo "  upgrade        Upgrade requirements"
 	@echo ""
 	@echo "Application:"
 	@echo "  run            Run the application on port ${PORT}"
 	@echo "  shell          Run Django shell"
+	@echo "  worker         Run Celery worker"
 	@echo ""
 	@echo "Database:"
 	@echo "  migrations     Generate migrations"
@@ -43,6 +45,9 @@ run: venv
 shell: venv
 	${VENV}/bin/python manage.py shell_plus
 
+worker: venv
+	${VENV}/bin/celery -A majak worker --pool solo -E -l INFO
+
 migrations: venv
 	${VENV}/bin/python manage.py makemigrations
 
@@ -55,7 +60,13 @@ test:
 coverage:
 	${VENV}/bin/pytest --cov --cov-report term-missing
 
-.PHONY: help venv install install-hooks hooks run shell
+upgrade:
+	(cd requirements && pip-compile -U base.in)
+	(cd requirements && pip-compile -U dev.in)
+	(cd requirements && pip-compile -U production.in)
+
+
+.PHONY: help venv install install-hooks hooks run shell upgrade
 .PHONY: migrations migrate test coverage
 
 # EOF

README.md

@@ -29,21 +29,41 @@ Rozšíření která používáme:
 ### Struktura projektu
 
     .
-    ├── home            = app na web úvodní stránky Majáku
     ├── donate          = app na web dary.pirati.cz
+    ├── elections2021   = app na količní web k volbám 2021
     ├── senate          = app na web senat.pirati.cz
     ├── senat_campaign  = app na weby kandidátů na senátory
     ├── districts       = app na web kraje.pirati.cz
-    ├── program2021     = app na prezentaci programu 2021
+    ├── district        = app na weby oblastních sdružení
     ├── uniweb          = app na univerzalni webove stranky
     ...
     ├── majak           = Django projekt s konfigurací Majáku
     ├── shared          = app se sdílenými static soubory a templaty pro weby
     ├── calendar_utils  = app s modelem a utilitami na iCal kalendáře
-    ├── search          = app pro fulltext search (default, asi se k ničemu nepoužívá)
     ├── tuning          = app na tuning administračního rozhraní Majáku
     └── users           = app s custom user modelem a SSO, apod.
 
+Poznámky k uspořádání:
+
+* nesdílíme a nemícháme modely zděděné z `Page` mezi jednotlivými appkami a weby,
+  abychom předešli rozbití cizího webu
+* části definice tříd stránek sdílíme jako mixin pattern (`shared.models.ArticleMixin`,
+  `share.models.SubpageMixin`, apod.)
+* sdílet se mohou obyčejné Django modely z `models.Model`, ale umístěné v nějaké
+  appce/knihovně určené pro sdílení (`shared`, `calendar_utils`, apod.), nikoliv
+  mezi weby (mezi appkami na weby by nikdy neměly vzniknout závislosti)
+
+### Konfigurace webu
+
+Konfigurace konkrétního webu (odkazy do patičky, Matomo ID, apod.) se definuje v
+kořenové `xxxHomePage` webu. Je to pro uživatele snažší na správu než
+[Site Settings](https://docs.wagtail.io/en/stable/reference/contrib/settings.html)
+Wagtailu pro konfigurace webů. A pro vývojáře je to skoro jedno.
+
+Z různých podstránek webu se k té konfiguraci dostaneme přes property `root_page`
+kterou přidává `shared.models.SubpageMixin`. V `xxxHomePage` webu je třeba
+definovat `root_page` tak, aby vedla na `self`.
+
 ### Styleguide
 
 Některé weby využívají [Pirátskou UI styleguide](https://gitlab.pirati.cz/to/weby/ui-styleguide)
@@ -61,7 +81,7 @@ obsahuje věci ze šablony verze `1.8.x`. Důvodem je snažší migrace na nově
 verze šablony, které mohou obsahovat nekompatibilní změny. Každý web tak může
 (a nemusí) migrovat nezávisle dle potřeby.
 
-Různé verze šablony jsou k vidění na [styleguide.pir-test.eu](https://styleguide.pir-test.eu/)
+Různé verze šablony jsou k vidění na [styleguide.pirati.cz](https://styleguide.pirati.cz/)
 
 ### Kalendáře
 
@@ -77,19 +97,34 @@ Kalendář se stáhne při uložení modelu obsahujícího `CalendarMixin`.
 Appka přidává management command `update_callendars`, který stahuje a updatuje
 kalendáře. Je třeba ho pravidelně volat na pozadí (přes CRON).
 
-### Jupyter notebooky
+### Celery (Import z Jekyllu)
+
+Import z Jekyll GitHub repozitářů pirátských webů je řešen asynchroně přes Celery.
+Celery využívá Redis, který může běžet např. lokálně, typicky 6379:
+V envu pak je pak potřeba mít nastavený Celery broker:
+
+```
+CELERY_BROKER_URL=redis://localhost:6379/6
+CELERY_RESULT_BACKEND=redis://localhost:6379/6
+```
 
-Appka Uniweb umí vložit do stránky Jupyter notebook a zobrazit jeho výstup.
+Aby se celery tasky vykonávaly, je potřeba pustit celery v terminálu:
 
-Pokud některé buňky nechceš generovat do výstupní stránky, nastav u nich tag
-"exclude".
+```
+celery -A majak worker
+```
 
-Pokud chceš generovat jen výstup dané buňky, použij tag "output"
+Pokud není zadán `CELERY_BROKER_URL`, tak se automaticky nastaví
+`CELERY_TASK_ALWAYS_EAGER = True`, čímž se tasky vykonají synchronně přímo v
+procesu webserveru.
 
-Pozor: u plotly grafů je nutno zadat tagem "output" výstup buňky s inicializací
-knihovny, tedy něco kde je "import plotly" apod. Pokud celou takovou buňku
-vynecháš tagem "exclude", žádné grafy se nezobrazí.
+### Stránka 404
 
+Pokud je třeba vlastní 404 pro web, stačí do kořenové `xxxHomePage` webu
+definovat metodu `get_404_response` vracející Django HTTP Reponse objekt.
+
+    def get_404_response(self, request):
+        return HttpResponse("Stránka nenalezena", status=404)
 
 ## Deployment
 
@@ -99,7 +134,7 @@ Je třeba nastavit environment proměnné:
 
 | proměnná | default | popis |
 | --- | --- | --- |
-| `DATABASE_URL` | | DSN k databázi (např. `postgres://user:pass@localhost:5342/majak`) |
+| `DATABASE_URL` | | DSN k databázi (např. `postgres://user:pass@localhost:5432/majak`) |
 | `OIDC_RP_REALM_URL` | | OpenID server realm URL (např. `http://localhost:8080/auth/realms/master/`) |
 | `OIDC_RP_CLIENT_ID` | | OpenID Client ID |
 | `OIDC_RP_CLIENT_SECRET` | | OpenID Client Secret |
@@ -111,6 +146,14 @@ V produkci musí být navíc nastaveno:
 | --- | --- | --- |
 | `DJANGO_SECRET_KEY` | | tajný šifrovací klíč |
 | `DJANGO_ALLOWED_HOSTS` | | allowed hosts (více hodnot odděleno čárkami) |
+| `CELERY_BROKER_URL` | | URL pro Celery Broker |
+| `CELERY_RESULT_BACKEND` | | URL pro Celery Result Backend |
+| `EMAIL_HOST` | SMTP pro odesílání přihlášek do kariér |
+| `EMAIL_HOST_USER` | --||-- Username |
+| `EMAIL_HOST_PASSWORD` | --||-- Heslo |
+| `EMAIL_PORT` | --||-- Port |
+| `CLAMD_TCP_ADDR` | ClamAV host (pro skenování virů v nahraných souborech) |
+| `CLAMD_TCP_SOCKET` | ClamAV socket |
 
 Různé:
 
@@ -118,6 +161,10 @@ Různé:
 | --- | --- | --- |
 | `MAJAK_ENV` | prod | `prod`/`test`/`dev` prostředí kde Maják běží |
 | `SENTRY_DSN` | | pokud je zadáno, pády se reportují do Sentry |
+| `SEARCH_CONFIG` | english | nastavení jazyka fulltextového vyhledávání, viz níže |
+| `DEBUG_TOOLBAR` | False | zobrazit Django Debug Toolbar (pro vývoj) |
+| `MAILTRAIN_API_URL` | | URL k API Mailtrain |
+| `MAILTRAIN_API_TOKEN` | | token k API Mailtrain |
 
 Settings pro appky na weby:
 
@@ -134,6 +181,33 @@ Přes CRON je třeba na pozadí spouštět Django `manage.py` commandy:
 * `clearsessions` - maže expirované sessions (denně až týdně)
 * `publish_scheduled_pages` - publikuje naplánované stránky (každou hodinu)
 * `update_callendars` - stáhne a aktualizuje kalendáře (několikrát denně)
+* `update_main_timeline_articles` - aktualizuje články na `pirati.cz` z `https://piratipracuji.cz/api/`
+* `update_redmine_issues` - aktualizuje programované body MS a KS stránek napojených na Redmine (několikrát denně)
+
+### Fulltextové vyhledávání v češtině
+
+Pro fulltextové vyhledávání je třeba do PostgreSQL přidat
+[slovníky](https://github.com/f00b4r/postgresql-czech) do adresáře
+`/usr/local/share/postgresql/tsearch_data/`.
+
+V databázi Majáku je třeba nakonfigurovat český fulltext:
+
+```sql
+CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile=czech, afffile=czech, stopwords=czech);
+CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
+ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;
+CREATE extension IF NOT EXISTS unaccent;
+```
+
+Otestovat funkčnost lze dotazem:
+
+```sql
+SELECT * FROM ts_debug('czech', 'Příliš žluťoučký kůň se napil žluté vody');
+```
+
+Dále nastavit environment proměnnou `SEARCH_CONFIG=czech`. A nakonec jednorázově
+spustit naplnění indexu pro vyhledávání `python manage.py update_index`.
+Aktualizace indexu poté probíhají automaticky při uložení stránek.
 
 ### Přidání nového webu
 
@@ -141,7 +215,6 @@ Doména či subdoména se musí nakonfigurovat v:
 
 * environment proměnné `DJANGO_ALLOWED_HOSTS`
 * proxy před Majákem
-* SSO Client redirect URIs
 
 ## Vývoj
 
@@ -208,6 +281,21 @@ Pro lokální vývoj obsahují settings tyto výchozí hodnoty:
     ALLOWED_HOSTS = ["*"]
     MAJAK_ENV = "dev"
 
+#### Autentifikace a autorizace
+
+Jako OIDC server můžete použít lokálně spuštěný Keycloak z [Dev Tools](https://gitlab.pirati.cz/to/dev-tools).
+
+Prvním přihlášením přes OIDC se vytvoří uživatel, ale bez oprávnění. Je třeba
+mu přes Django shell (viz níže) nastavit oprávnění pro přístup do administrace:
+
+```python
+from users.models import User
+u = User.objects.get()
+u.is_superuser = True
+u.is_staff = True
+u.save()
+```
+
 ### Management projektu
 
 #### Migrace databáze
@@ -228,12 +316,33 @@ Django development server na portu `8006` se spustí příkazem:
 
 Poté můžete otevřít web na adrese [http://localhost:8006](http://localhost:8006)
 
+##### Debug Toolbar
+
+Pro spuštění development serveru s Django Debug Toolbar nastavte environment
+proměnnou `DEBUG_TOOLBAR`. Např.:
+
+    $ DEBUG_TOOLBAR=1 make run
+
 #### Django shell
 
 Django shell používající `shell_plus` z Django extensions spustíte:
 
     $ make shell
 
+### Lokální weby
+
+Pro vývoj různých webů si nastavte v `/etc/hosts` (na Linux, Mac OS, apod.)
+pojmenované aliasy k lokální IP 127.0.0.1 jako např.:
+
+    127.0.0.1   majak.loc dalsiweb.loc
+
+K Majáku spuštěnému v development serveru na portu `8006` se pak dostanete na
+adrese [http://majak.loc:8006/admin/](http://majak.loc:8006/admin/).
+
+A pokud nějakému webu v nastavení v Majáku dáte adresu třeba `dalsiweb.loc` a
+port `8006`, dostanete se pak na něj přes adresu
+[http://dalsiweb.loc:8006](http://dalsiweb.loc:8006).
+
 ### Testy
 
 Používá se testovací framework [pytest](https://pytest.org). Spuštění testů:

article_import_utils/apps.py

+from django.apps import AppConfig
+
+
+class ArticleImportUtilsConfig(AppConfig):
+    name = "article_import_utils"

calendar_utils/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class CalendarUtilsConfig(AppConfig):
-    name = "calendar utils"
+    name = "calendar_utils"

calendar_utils/icalevents/__init__.py

+"""
+iCalEvents search all events occurring in a given time frame in an iCal file.
+"""
+
+__all__ = ["icaldownload", "icalparser", "icalevents"]

calendar_utils/icalevents/icaldownload.py

+"""
+Downloads an iCal url or reads an iCal file.
+"""
+import logging
+
+from httplib2 import Http
+
+
+def apple_data_fix(content):
+    """
+    Fix Apple tzdata bug.
+
+    :param content: content to fix
+    :return: fixed content
+    """
+    return content.replace("TZOFFSETFROM:+5328", "TZOFFSETFROM:+0053")
+
+
+def apple_url_fix(url):
+    """
+    Fix Apple URL.
+
+    :param url: URL to fix
+    :return: fixed URL
+    """
+    if url.startswith("webcal://"):
+        url = url.replace("webcal://", "http://", 1)
+    return url
+
+
+class ICalDownload:
+    """
+    Downloads or reads and decodes iCal sources.
+    """
+
+    def __init__(self, http=None, encoding="utf-8"):
+        # Get logger
+        logger = logging.getLogger()
+
+        # default http connection to use
+        if http is None:
+            try:
+                http = Http(".cache")
+            except (PermissionError, OSError) as e:
+                # Cache disabled if no write permission in working directory
+                logger.warning(
+                    (
+                        "Caching is disabled due to a read-only working directory: {}"
+                    ).format(e)
+                )
+                http = Http()
+
+        self.http = http
+        self.encoding = encoding
+
+    def data_from_url(self, url, apple_fix=False):
+        """
+        Download iCal data from URL.
+
+        :param url: URL to download
+        :param apple_fix: fix Apple bugs (protocol type and tzdata in iCal)
+        :return: decoded (and fixed) iCal data
+        """
+        if apple_fix:
+            url = apple_url_fix(url)
+
+        _, content = self.http.request(url)
+
+        if not content:
+            raise ConnectionError("Could not get data from %s!" % url)
+
+        return self.decode(content, apple_fix=apple_fix)
+
+    def data_from_file(self, file, apple_fix=False):
+        """
+        Read iCal data from file.
+
+        :param file: file to read
+        :param apple_fix: fix wrong Apple tzdata in iCal
+        :return: decoded (and fixed) iCal data
+        """
+        with open(file, mode="rb") as f:
+            content = f.read()
+
+        if not content:
+            raise IOError("File %s is not readable or is empty!" % file)
+
+        return self.decode(content, apple_fix=apple_fix)
+
+    def data_from_string(self, string_content, apple_fix=False):
+        if not string_content:
+            raise IOError("String content is not readable or is empty!")
+
+        return self.decode(string_content, apple_fix=apple_fix)
+
+    def decode(self, content, apple_fix=False):
+        """
+        Decode content using the set charset.
+
+        :param content: content do decode
+        :param apple_fix: fix Apple txdata bug
+        :return: decoded (and fixed) content
+        """
+        content = content.decode(self.encoding)
+        content = content.replace("\r", "")
+
+        if apple_fix:
+            content = apple_data_fix(content)
+
+        return content

calendar_utils/icalevents/icalevents.py

+from threading import Lock, Thread
+
+from .icaldownload import ICalDownload
+from .icalparser import Event, parse_events
+
+# Lock for event data
+event_lock = Lock()
+# Event data
+event_store = {}
+# Threads
+threads = {}
+
+
+def events(
+    url=None,
+    file=None,
+    string_content=None,
+    start=None,
+    end=None,
+    fix_apple=False,
+    http=None,
+    tzinfo=None,
+    sort=None,
+    strict=False,
+) -> list[Event]:
+    """
+    Get all events form the given iCal URL occurring in the given time range.
+
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date (see dateutils.date)
+    :param end: end date (see dateutils.date)
+    :param fix_apple: fix known Apple iCal issues
+    :param tzinfo: return values in specified tz
+    :param sort: sort return values
+    :param strict: return dates, datetimes and datetime with timezones as specified in ical
+    :sort sorts events by start time
+
+    :return events
+    """
+    found_events = []
+
+    content = None
+    ical_download = ICalDownload(http=http)
+
+    if url:
+        content = ical_download.data_from_url(url, apple_fix=fix_apple)
+
+    if not content and file:
+        content = ical_download.data_from_file(file, apple_fix=fix_apple)
+
+    if not content and string_content:
+        content = ical_download.data_from_string(string_content, apple_fix=fix_apple)
+
+    found_events += parse_events(
+        content, start=start, end=end, tzinfo=tzinfo, sort=sort, strict=strict
+    )
+
+    if found_events is not None and sort is True:
+        found_events.sort()
+
+    return found_events
+
+
+def request_data(key, url, file, string_content, start, end, fix_apple):
+    """
+    Request data, update local data cache and remove this Thread from queue.
+
+    :param key: key for data source to get result later
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date
+    :param end: end date
+    :param fix_apple: fix known Apple iCal issues
+    """
+    data = []
+
+    try:
+        data += events(
+            url=url,
+            file=file,
+            string_content=string_content,
+            start=start,
+            end=end,
+            fix_apple=fix_apple,
+        )
+    finally:
+        update_events(key, data)
+        request_finished(key)
+
+
+def events_async(
+    key, url=None, file=None, start=None, string_content=None, end=None, fix_apple=False
+):
+    """
+    Trigger an asynchronous data request.
+
+    :param key: key for data source to get result later
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date
+    :param end: end date
+    :param fix_apple: fix known Apple iCal issues
+    """
+    t = Thread(
+        target=request_data,
+        args=(key, url, file, string_content, start, end, fix_apple),
+    )
+
+    with event_lock:
+        if key not in threads:
+            threads[key] = []
+
+        threads[key].append(t)
+
+        if not threads[key][0].is_alive():
+            threads[key][0].start()
+
+
+def request_finished(key):
+    """
+    Remove finished Thread from queue.
+
+    :param key: data source key
+    """
+    with event_lock:
+        threads[key] = threads[key][1:]
+
+        if threads[key]:
+            threads[key][0].run()
+
+
+def update_events(key, data):
+    """
+    Set the latest events for a key.
+
+    :param key: key to set
+    :param data: events for key
+    """
+    with event_lock:
+        event_store[key] = data
+
+
+def latest_events(key):
+    """
+    Get the latest downloaded events for the given key.
+
+    :return: events for key
+    """
+    with event_lock:
+        # copy data
+        res = event_store[key][:]
+
+    return res
+
+
+def all_done(key):
+    """
+    Check if requests for the given key are active.
+
+    :param key: key for requests
+    :return: True if requests are pending or active
+    """
+    with event_lock:
+        if threads[key]:
+            return False
+        return True

calendar_utils/icalevents/icalparser.py

+"""
+Parse iCal data to Events.
+"""
+from datetime import date, datetime, timedelta
+
+# for UID generation
+from random import randint
+from typing import Optional
+from uuid import uuid4
+
+from dateutil.rrule import rrulestr
+from dateutil.tz import UTC, gettz
+from icalendar import Calendar
+from icalendar.prop import vDDDLists, vText
+from icalendar.timezone.windows_to_olson import WINDOWS_TO_OLSON
+from pytz import timezone
+
+
+def now():
+    """
+    Get current time.
+
+    :return: now as datetime with timezone
+    """
+    return datetime.now(UTC)
+
+
+class Attendee(str):
+    def __init__(self, address):
+        self.address = address
+
+    def __repr__(self):
+        return self.address.encode("utf-8").decode("ascii")
+
+    @property
+    def params(self):
+        return self.address.params
+
+
+class Event:
+    """
+    Represents one event (occurrence in case of reoccurring events).
+    """
+
+    def __init__(self):
+        """
+        Create a new event occurrence.
+        """
+        self.uid = -1
+        self.summary = None
+        self.description = None
+        self.start = None
+        self.end = None
+        self.all_day = True
+        self.transparent = False
+        self.recurring = False
+        self.location = None
+        self.private = False
+        self.created = None
+        self.last_modified = None
+        self.sequence = None
+        self.recurrence_id = None
+        self.attendee = None
+        self.organizer = None
+        self.categories = None
+        self.floating = None
+        self.status = None
+        self.url = None
+
+    def time_left(self, time=None):
+        """
+        timedelta form now to event.
+
+        :return: timedelta from now
+        """
+        time = time or now()
+        return self.start - time
+
+    def __lt__(self, other):
+        """
+        Events are sorted by start time by default.
+
+        :param other: other event
+        :return: True if start of this event is smaller than other
+        """
+        if not other or not isinstance(other, Event):
+            raise ValueError(
+                "Only events can be compared with each other! Other is %s" % type(other)
+            )
+        else:
+            # start and end can be dates, datetimes and datetimes with timezoneinfo
+            if type(self.start) is date and type(other.start) is date:
+                return self.start < other.start
+            elif type(self.start) is datetime and type(other.start) is datetime:
+                if self.start.tzinfo == other.start.tzinfo:
+                    return self.start < other.start
+                else:
+                    return self.start.astimezone(UTC) < other.start.astimezone(UTC)
+            elif type(self.start) is date and type(other.start) is datetime:
+                return self.start < other.start.date()
+            elif type(self.start) is datetime and type(other.start) is date:
+                return self.start.date() < other.start
+
+    def __str__(self):
+        return "%s: %s (%s)" % (self.start, self.summary, self.end - self.start)
+
+    def astimezone(self, tzinfo):
+        if type(self.start) is datetime:
+            self.start = self.start.astimezone(tzinfo)
+
+        if type(self.end) is datetime:
+            self.end = self.end.astimezone(tzinfo)
+
+        return self
+
+    def copy_to(self, new_start=None, uid=None):
+        """
+        Create a new event equal to this with new start date.
+
+        :param new_start: new start date
+        :param uid: UID of new event
+        :return: new event
+        """
+        if not new_start:
+            new_start = self.start
+
+        if not uid:
+            uid = "%s_%d" % (self.uid, randint(0, 1000000))
+
+        ne = Event()
+        ne.summary = self.summary
+        ne.description = self.description
+        ne.start = new_start
+
+        if self.end:
+            duration = self.end - self.start
+            ne.end = new_start + duration
+
+        ne.all_day = self.all_day
+        ne.recurring = self.recurring
+        ne.location = self.location
+        ne.attendee = self.attendee
+        ne.organizer = self.organizer
+        ne.private = self.private
+        ne.transparent = self.transparent
+        ne.uid = uid
+        ne.created = self.created
+        ne.last_modified = self.last_modified
+        ne.categories = self.categories
+        ne.floating = self.floating
+        ne.status = self.status
+        ne.url = self.url
+
+        return ne
+
+
+def encode(value: Optional[vText]) -> Optional[str]:
+    if value is None:
+        return None
+    try:
+        return str(value)
+    except UnicodeEncodeError:
+        return str(value.encode("utf-8"))
+
+
+def create_event(component, utc_default):
+    """
+    Create an event from its iCal representation.
+
+    :param component: iCal component
+    :return: event
+    """
+
+    event = Event()
+
+    event.start = component.get("dtstart").dt
+    # The RFC specifies that the TZID parameter must be specified for datetime or time
+    # Otherwise we set a default timezone (if only one is set with VTIMEZONE) or utc
+    event.floating = type(component.get("dtstart").dt) == date and utc_default
+
+    if component.get("dtend"):
+        event.end = component.get("dtend").dt
+    elif component.get("duration"):  # compute implicit end as start + duration
+        event.end = event.start + component.get("duration").dt
+    else:  # compute implicit end as start + 0
+        event.end = event.start
+
+    event.summary = encode(component.get("summary"))
+    event.description = encode(component.get("description"))
+    event.all_day = type(component.get("dtstart").dt) is date
+    if component.get("rrule"):
+        event.recurring = True
+    event.location = encode(component.get("location"))
+
+    if component.get("attendee"):
+        event.attendee = component.get("attendee")
+        if type(event.attendee) is list:
+            event.attendee = [Attendee(attendee) for attendee in event.attendee]
+        else:
+            event.attendee = Attendee(event.attendee)
+    else:
+        event.attendee = str(None)
+
+    if component.get("uid"):
+        event.uid = component.get("uid").encode("utf-8").decode("ascii")
+    else:
+        event.uid = str(uuid4())  # Be nice - treat every event as unique
+
+    if component.get("organizer"):
+        event.organizer = component.get("organizer").encode("utf-8").decode("ascii")
+    else:
+        event.organizer = str(None)
+
+    if component.get("class"):
+        event_class = component.get("class")
+        event.private = event_class == "PRIVATE" or event_class == "CONFIDENTIAL"
+
+    if component.get("transp"):
+        event.transparent = component.get("transp") == "TRANSPARENT"
+
+    if component.get("created"):
+        event.created = component.get("created").dt
+
+    if component.get("RECURRENCE-ID"):
+        rid = component.get("RECURRENCE-ID").dt
+
+        # Spec defines that if DTSTART is a date RECURRENCE-ID also is to be interpreted as a date
+        if type(component.get("dtstart").dt) is date:
+            event.recurrence_id = date(year=rid.year, month=rid.month, day=rid.day)
+        else:
+            event.recurrence_id = rid
+
+    if component.get("last-modified"):
+        event.last_modified = component.get("last-modified").dt
+    elif event.created:
+        event.last_modified = event.created
+
+    # sequence can be 0 - test for None instead
+    if not component.get("sequence") is None:
+        event.sequence = component.get("sequence")
+
+    if component.get("categories"):
+        categoriesval = component.get("categories")
+        categories = (
+            component.get("categories").cats
+            if hasattr(categoriesval, "cats")
+            else categoriesval
+        )
+        encoded_categories = list()
+        for category in categories:
+            encoded_categories.append(encode(category))
+        event.categories = encoded_categories
+
+    if component.get("status"):
+        event.status = encode(component.get("status"))
+
+    if component.get("url"):
+        event.url = encode(component.get("url"))
+
+    return event
+
+
+def parse_events(
+    content,
+    start=None,
+    end=None,
+    default_span=timedelta(days=7),
+    tzinfo=None,
+    sort=False,
+    strict=False,
+):
+    """
+    Query the events occurring in a given time range.
+
+    :param content: iCal URL/file content as String
+    :param start: start date for search, default today (in UTC format)
+    :param end: end date for search (in UTC format)
+    :param default_span: default query length (one week)
+    :return: events as list
+    """
+    if not start:
+        start = now()
+
+    if not end:
+        end = start + default_span
+
+    if not content:
+        raise ValueError("Content is invalid!")
+
+    calendar = Calendar.from_ical(content)
+
+    # > Will be deprecated ========================
+    # Calendar.from_ical already parses timezones as specified in the ical.
+    # We can specify a 'default' tz but this is not according to spec.
+    # Kept this here to verify tests and backward compatibility
+
+    # Keep track of the timezones defined in the calendar
+    timezones = {}
+
+    # Parse non standard timezone name
+    if "X-WR-TIMEZONE" in calendar:
+        x_wr_timezone = str(calendar["X-WR-TIMEZONE"])
+        timezones[x_wr_timezone] = get_timezone(x_wr_timezone)
+
+    for c in calendar.walk("VTIMEZONE"):
+        name = str(c["TZID"])
+        try:
+            timezones[name] = c.to_tz()
+        except IndexError:
+            # This happens if the VTIMEZONE doesn't
+            # contain start/end times for daylight
+            # saving time. Get the system pytz
+            # value from the name as a fallback.
+            timezones[name] = timezone(name)
+
+    # If there's exactly one timezone in the file,
+    # assume it applies globally, otherwise UTC
+    utc_default = False
+    if len(timezones) == 1:
+        cal_tz = get_timezone(list(timezones)[0])
+    else:
+        utc_default = True
+        cal_tz = UTC
+    # < ==========================================
+
+    found = []
+
+    def add_if_not_exception(event):
+        exdate = "%04d%02d%02d" % (
+            event.start.year,
+            event.start.month,
+            event.start.day,
+        )
+
+        if exdate not in exceptions:
+            found.append(event)
+
+    for component in calendar.walk():
+        exceptions = {}
+
+        if "EXDATE" in component:
+            # Deal with the fact that sometimes it's a list and
+            # sometimes it's a singleton
+            exlist = []
+            if isinstance(component["EXDATE"], vDDDLists):
+                exlist = component["EXDATE"].dts
+            else:
+                exlist = component["EXDATE"]
+            for ex in exlist:
+                exdate = ex.to_ical().decode("UTF-8")
+                exceptions[exdate[0:8]] = exdate
+
+        if component.name == "VEVENT":
+            e = create_event(component, utc_default)
+
+            # make rule.between happy and provide from, to points in time that have the same format as dtstart
+            s = component["dtstart"].dt
+            if type(s) is date and not e.recurring:
+                f, t = date(start.year, start.month, start.day), date(
+                    end.year, end.month, end.day
+                )
+            elif type(s) is datetime and s.tzinfo:
+                f, t = datetime(
+                    start.year, start.month, start.day, tzinfo=s.tzinfo
+                ), datetime(end.year, end.month, end.day, tzinfo=s.tzinfo)
+            else:
+                f, t = datetime(start.year, start.month, start.day), datetime(
+                    end.year, end.month, end.day
+                )
+
+            if e.recurring:
+                rule = parse_rrule(component)
+                for dt in rule.between(f, t, inc=True):
+                    # Recompute the start time in the current timezone *on* the
+                    # date of *this* occurrence. This handles the case where the
+                    # recurrence has crossed over the daylight savings time boundary.
+                    if type(dt) is datetime and dt.tzinfo:
+                        dtstart = dt.replace(tzinfo=get_timezone(str(dt.tzinfo)))
+                        ecopy = e.copy_to(
+                            dtstart.date() if type(s) is date else dtstart, e.uid
+                        )
+                    else:
+                        ecopy = e.copy_to(dt.date() if type(s) is date else dt, e.uid)
+                    add_if_not_exception(ecopy)
+
+            elif e.end >= f and e.start <= t:
+                add_if_not_exception(e)
+
+    result = found.copy()
+
+    # Remove events that are replaced in ical
+    for event in found:
+        if not event.recurrence_id and (event.uid, event.start) in [
+            (f.uid, f.recurrence_id) for f in found
+        ]:
+            result.remove(event)
+
+    # > Will be deprecated ========================
+    # We will apply default cal_tz as required by some tests.
+    # This is just here for backward-compatibility
+    if not strict:
+        for event in result:
+            if type(event.start) is date:
+                event.start = datetime(
+                    year=event.start.year,
+                    month=event.start.month,
+                    day=event.start.day,
+                    hour=0,
+                    minute=0,
+                    tzinfo=cal_tz,
+                )
+                event.end = datetime(
+                    year=event.end.year,
+                    month=event.end.month,
+                    day=event.end.day,
+                    hour=0,
+                    minute=0,
+                    tzinfo=cal_tz,
+                )
+            elif type(event.start) is datetime:
+                if event.start.tzinfo:
+                    event.start = event.start.astimezone(cal_tz)
+                    event.end = event.end.astimezone(cal_tz)
+                else:
+                    event.start = event.start.replace(tzinfo=cal_tz)
+                    event.end = event.end.replace(tzinfo=cal_tz)
+
+            if event.created:
+                if type(event.created) is date:
+                    event.created = datetime(
+                        year=event.created.year,
+                        month=event.created.month,
+                        day=event.created.day,
+                        hour=0,
+                        minute=0,
+                        tzinfo=cal_tz,
+                    )
+                if type(event.created) is datetime:
+                    if event.created.tzinfo:
+                        event.created = event.created.astimezone(cal_tz)
+                    else:
+                        event.created = event.created.replace(tzinfo=cal_tz)
+
+            if event.last_modified:
+                if type(event.last_modified) is date:
+                    event.last_modified = datetime(
+                        year=event.last_modified.year,
+                        month=event.last_modified.month,
+                        day=event.last_modified.day,
+                        hour=0,
+                        minute=0,
+                        tzinfo=cal_tz,
+                    )
+                if type(event.last_modified) is datetime:
+                    if event.last_modified.tzinfo:
+                        event.last_modified = event.last_modified.astimezone(cal_tz)
+                    else:
+                        event.last_modified = event.last_modified.replace(tzinfo=cal_tz)
+    # < ==========================================
+
+    if sort:
+        result.sort()
+
+    if tzinfo:
+        result = [event.astimezone(tzinfo) for event in result]
+
+    return result
+
+
+def parse_rrule(component):
+    """
+    Extract a dateutil.rrule object from an icalendar component. Also includes
+    the component's dtstart and exdate properties. The rdate and exrule
+    properties are not yet supported.
+
+    :param component: icalendar component
+    :return: extracted rrule or rruleset or None
+    """
+
+    dtstart = component.get("dtstart").dt
+
+    # component['rrule'] can be both a scalar and a list
+    rrules = component.get("rrule")
+    if not isinstance(rrules, list):
+        rrules = [rrules]
+
+    def conform_until(until, dtstart):
+        if type(dtstart) is datetime:
+            # If we have timezone defined adjust for daylight saving time
+            if type(until) is datetime:
+                return until + abs(
+                    (
+                        until.astimezone(dtstart.tzinfo).utcoffset()
+                        if until.tzinfo is not None and dtstart.tzinfo is not None
+                        else None
+                    )
+                    or timedelta()
+                )
+
+            return (
+                until.astimezone(UTC)
+                if type(until) is datetime
+                else datetime(
+                    year=until.year, month=until.month, day=until.day, tzinfo=UTC
+                )
+            ) + (
+                (dtstart.tzinfo.utcoffset(dtstart) if dtstart.tzinfo else None)
+                or timedelta()
+            )
+
+        return until.date() + timedelta(days=1) if type(until) is datetime else until
+
+    for index, rru in enumerate(rrules):
+        if "UNTIL" in rru:
+            rrules[index]["UNTIL"] = [
+                conform_until(until, dtstart) for until in rrules[index]["UNTIL"]
+            ]
+
+    rule = rrulestr(
+        "\n".join(x.to_ical().decode() for x in rrules),
+        dtstart=dtstart,
+        forceset=True,
+        unfold=True,
+    )
+
+    if component.get("exdate"):
+        # Add exdates to the rruleset
+        for exd in extract_exdates(component):
+            if type(dtstart) is date:
+                rule.exdate(
+                    datetime(
+                        year=exd.year,
+                        month=exd.month,
+                        day=exd.day,
+                        hour=0,
+                        minute=0,
+                        second=0,
+                    )
+                    if isinstance(exd, date)
+                    else exd
+                )
+            else:
+                rule.exdate(exd)
+
+    # TODO: What about rdates and exrules?
+    if component.get("exrule"):
+        pass
+
+    if component.get("rdate"):
+        pass
+
+    return rule
+
+
+def extract_exdates(component):
+    """
+    Compile a list of all exception dates stored with a component.
+
+    :param component: icalendar iCal component
+    :return: list of exception dates
+    """
+    dates = []
+    exd_prop = component.get("exdate")
+    if isinstance(exd_prop, list):
+        for exd_list in exd_prop:
+            dates.extend(exd.dt for exd in exd_list.dts)
+    else:  # it must be a vDDDLists
+        dates.extend(exd.dt for exd in exd_prop.dts)
+
+    return dates
+
+
+def get_timezone(tz_name):
+    if tz_name in WINDOWS_TO_OLSON:
+        return gettz(WINDOWS_TO_OLSON[tz_name])
+    else:
+        return gettz(tz_name)

calendar_utils/management/commands/update_callendars.py

+import logging
+
 from django.core.management.base import BaseCommand
 
 from ...models import Calendar
 
+logger = logging.getLogger(__name__)
+
 
 class Command(BaseCommand):
     def handle(self, *args, **options):
+        self.stdout.write("Removing orphaned calendars...")
+        for cal in Calendar.objects.filter(
+            districtcalendarpage=None,
+            districtcenterpage=None,
+            districthomepage=None,
+            districtpersonpage=None,
+            elections2021calendarpage=None,
+            mainpersonpage=None,
+            senatcampaignhomepage=None,
+            uniwebhomepage=None,
+            uniwebcalendarpage=None,
+        ):
+            try:
+                self.stdout.write(f"- {cal.id} | {cal.url}")
+                cal.delete()
+            except Exception as e:
+                logger.error("Calendar delete failed for %s", cal.url, exc_info=True)
+                self.stdout.write("  - failed")
+                self.stdout.write(str(e))
+
         self.stdout.write("Updating calendars...")
         for cal in Calendar.objects.all():
-            self.stdout.write(f"\n@ {cal.url}")
+            self.stdout.write(f"\n@ {cal.id} | {cal.url}")
+
             try:
                 cal.update_source()
                 self.stdout.write("+ ok")
             except Exception as e:
-                # TODO logging
+                logger.error("Calendar update failed for %s", cal.url, exc_info=True)
                 self.stdout.write("- failed")
                 self.stdout.write(str(e))
 

calendar_utils/migrations/0001_initial.py

@@ -7,7 +7,6 @@ import calendar_utils.models
 
 
 class Migration(migrations.Migration):
-
     initial = True
 
     dependencies = []

calendar_utils/migrations/0002_auto_20200523_0243.py

@@ -4,7 +4,6 @@ from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         ("calendar_utils", "0001_initial"),
     ]

calendar_utils/migrations/0003_remove_calendar_source_calendar_event_hash.py

+# Generated by Django 4.0.4 on 2022-05-05 10:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("calendar_utils", "0002_auto_20200523_0243"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="calendar",
+            name="source",
+        ),
+        migrations.AddField(
+            model_name="calendar",
+            name="event_hash",
+            field=models.CharField(max_length=256, null=True),
+        ),
+    ]