.gitignore

@@ -145,3 +145,11 @@ cython_debug/
 
 media_files/
 static_files/
+.python-version
+.idea/
+
+update_election_statics.sh
+download_static.sh
+matice.csv
+
+.vscode/

.gitlab-ci.yml

-image: docker:19.03.1
+stages:
+  - build
+
+image: docker:20.10.8
 
 variables:
   DOCKER_TLS_CERTDIR: "/certs"
@@ -6,15 +9,22 @@ variables:
   IMAGE_TAG_NGINX: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG-nginx
 
 services:
-  - docker:19.03.1-dind
+  - docker:20.10.8-dind
 
 before_script:
   - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
 
-build:
+build_app:
   stage: build
   script:
-    - docker build -t $IMAGE_TAG_APP .
-    - docker build -t $IMAGE_TAG_NGINX . -f Dockerfile.nginx
+    - docker pull $CI_REGISTRY_IMAGE:test || true
+    - docker build --cache-from $CI_REGISTRY_IMAGE:test -t $IMAGE_TAG_APP .
     - docker push $IMAGE_TAG_APP
+
+build_nginx:
+  stage: build
+  when: manual
+  script:
+    - docker pull $CI_REGISTRY_IMAGE:test-nginx || true
+    - docker build --cache-from $CI_REGISTRY_IMAGE:test-nginx -t $IMAGE_TAG_NGINX . -f Dockerfile.nginx
     - docker push $IMAGE_TAG_NGINX

.isort.cfg

@@ -2,6 +2,5 @@
 # config compatible with Black
 line_length = 88
 multi_line_output = 3
-default_sectiont = "THIRDPARTY"
 include_trailing_comma = true
-known_third_party = django,environ,pirates,wagtail
+known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,clamd,dateutil,django,django_ratelimit,environ,faker,fastjsonschema,gql,httplib2,icalendar,instaloader,markdown,modelcluster,nh3,pirates,pytest,pytz,requests,sentry_sdk,taggit,wagtail,wagtailmetadata,weasyprint,willow,yaml

.pre-commit-config.yaml

 default_language_version:
-  python: python3.7
+  python: python3.12
 
+exclude: snapshots/
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v4.4.0
     hooks:
       - id: trailing-whitespace
         exclude: ^.*\.md$
@@ -15,16 +16,27 @@ repos:
       - id: check-merge-conflict
 
   - repo: https://github.com/asottile/seed-isort-config
-    rev: v2.1.0
+    rev: v2.2.0
     hooks:
       - id: seed-isort-config
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 4.3.21
+    rev: 5.12.0
     hooks:
       - id: isort
 
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+    -   id: autoflake
+
   - repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 23.1.0
     hooks:
       - id: black
+        args: ["-t", "py310"]
+  - repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+    - id: autoflake
+      args: [--remove-all-unused-imports, --in-place]

Dockerfile

-FROM python:3.8-slim
+FROM python:3.11
+
+RUN apt-get update && apt-get install -y \
+    # requirements for OpenCV
+    libgl1-mesa-dev \
+    # requirements for WeasyPrint
+    python3-cffi \
+    # requirements for Wand (GIF image resizing)
+    libmagickwand-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 RUN mkdir /app
 WORKDIR /app
@@ -11,13 +20,18 @@ COPY . .
 RUN bash -c 'adduser --disabled-login --quiet --gecos app app &&  \
              chmod -R o+r /app/ && \
              mkdir /app/media_files && \
+             mkdir /app/static_files && \
              chown -R app:app /app/media_files && \
+             chown -R app:app /app/static_files && \
              chmod o+x /app/run.sh'
 USER app
 
-# TODO HACK!
-# ENV DJANGO_SETTINGS_MODULE "majak.settings.production"
-ENV DJANGO_SETTINGS_MODULE "majak.settings.dev"
+ENV DJANGO_SETTINGS_MODULE "majak.settings.production"
+
+# fake values for required env variables used to run collectstatic during build
+RUN DJANGO_SECRET_KEY=x DATABASE_URL=postgres://x/x DJANGO_ALLOWED_HOSTS=x \
+    OIDC_RP_CLIENT_ID=x OIDC_RP_CLIENT_SECRET=x OIDC_RP_REALM_URL=x \
+    python manage.py collectstatic
 
 EXPOSE 8000
 

Dockerfile.nginx

-FROM nginx
+FROM nginx:latest
 EXPOSE 8080
 ADD nginx.conf /etc/nginx/conf.d/majak.conf

Makefile

@@ -10,22 +10,28 @@ help:
 	@echo "  install        Install dependencies to venv"
 	@echo "  install-hooks  Install pre-commit hooks"
 	@echo "  hooks          Run pre-commit hooks manually"
+	@echo "  upgrade        Upgrade requirements"
 	@echo ""
 	@echo "Application:"
 	@echo "  run            Run the application on port ${PORT}"
 	@echo "  shell          Run Django shell"
+	@echo "  worker         Run Celery worker"
 	@echo ""
 	@echo "Database:"
 	@echo "  migrations     Generate migrations"
 	@echo "  migrate        Run migrations"
 	@echo ""
+	@echo "Testing:"
+	@echo "  test           Run tests"
+	@echo "  coverage       Coverage report"
+	@echo ""
 
 venv: .venv/bin/python
 .venv/bin/python:
 	${PYTHON} -m venv ${VENV}
 
 install: venv
-	${VENV}/bin/pip install -r requirements/base.txt
+	${VENV}/bin/pip install -r requirements/base.txt -r requirements/dev.txt
 
 install-hooks:
 	pre-commit install --install-hooks
@@ -39,13 +45,28 @@ run: venv
 shell: venv
 	${VENV}/bin/python manage.py shell_plus
 
+worker: venv
+	${VENV}/bin/celery -A majak worker --pool solo -E -l INFO
+
 migrations: venv
 	${VENV}/bin/python manage.py makemigrations
 
 migrate: venv
 	${VENV}/bin/python manage.py migrate
 
-.PHONY: help venv install install-hooks hooks run shell
-.PHONY: migrations migrate
+test:
+	${VENV}/bin/pytest
+
+coverage:
+	${VENV}/bin/pytest --cov --cov-report term-missing
+
+upgrade:
+	(cd requirements && pip-compile -U base.in)
+	(cd requirements && pip-compile -U dev.in)
+	(cd requirements && pip-compile -U production.in)
+
+
+.PHONY: help venv install install-hooks hooks run shell upgrade
+.PHONY: migrations migrate test coverage
 
 # EOF

README.md

@@ -5,14 +5,7 @@ Maják je CMS pro Pirátské weby. Postavený je na [Wagtail](https://wagtail.io
 [![code style: Black](https://img.shields.io/badge/code%20style-Black-000000)](https://github.com/psf/black)
 [![powered by: Wagtail](https://img.shields.io/badge/powered%20by-Wagtail-43b1b0)](https://wagtail.io)
 [![powered by: Django](https://img.shields.io/badge/powered%20by-Django-0C4B33)](https://www.djangoproject.com)
-
-## Konfigurace
-
-Je třeba nastavit environment proměnné:
-
-| proměnná | default | popis |
-| --- | --- | --- |
-| `DATABASE_URL` | | DSN k databázi (např. `postgres://user:pass@localhost:5342/majak`) |
+[![licence AGPLv3](https://img.shields.io/badge/licence-AGPLv3-brightgreen)](LICENSE)
 
 ## Pod pokličkou
 
@@ -24,11 +17,205 @@ Pár užitečných odkazů:
 
 * [docs Wagtail](https://docs.wagtail.io/)
 * [docs Django](https://docs.djangoproject.com/)
-* [docs Wagtailmenus](https://wagtailmenus.readthedocs.io/)
 
 A za zmínku stojí [Awesome Wagtail](https://github.com/springload/awesome-wagtail)
 jako přehled pluginů a rozšíření pro Wagtail.
 
+Rozšíření která používáme:
+
+* [wagtail-metadata](https://github.com/neon-jungle/wagtail-metadata)
+  - upravený template `shared/templates/wagtailmetadata/parts/tags.html`
+
+### Struktura projektu
+
+    .
+    ├── donate          = app na web dary.pirati.cz
+    ├── elections2021   = app na količní web k volbám 2021
+    ├── senate          = app na web senat.pirati.cz
+    ├── senat_campaign  = app na weby kandidátů na senátory
+    ├── districts       = app na web kraje.pirati.cz
+    ├── district        = app na weby oblastních sdružení
+    ├── uniweb          = app na univerzalni webove stranky
+    ...
+    ├── majak           = Django projekt s konfigurací Majáku
+    ├── shared          = app se sdílenými static soubory a templaty pro weby
+    ├── calendar_utils  = app s modelem a utilitami na iCal kalendáře
+    ├── tuning          = app na tuning administračního rozhraní Majáku
+    └── users           = app s custom user modelem a SSO, apod.
+
+Poznámky k uspořádání:
+
+* nesdílíme a nemícháme modely zděděné z `Page` mezi jednotlivými appkami a weby,
+  abychom předešli rozbití cizího webu
+* části definice tříd stránek sdílíme jako mixin pattern (`shared.models.ArticleMixin`,
+  `share.models.SubpageMixin`, apod.)
+* sdílet se mohou obyčejné Django modely z `models.Model`, ale umístěné v nějaké
+  appce/knihovně určené pro sdílení (`shared`, `calendar_utils`, apod.), nikoliv
+  mezi weby (mezi appkami na weby by nikdy neměly vzniknout závislosti)
+
+### Konfigurace webu
+
+Konfigurace konkrétního webu (odkazy do patičky, Matomo ID, apod.) se definuje v
+kořenové `xxxHomePage` webu. Je to pro uživatele snažší na správu než
+[Site Settings](https://docs.wagtail.io/en/stable/reference/contrib/settings.html)
+Wagtailu pro konfigurace webů. A pro vývojáře je to skoro jedno.
+
+Z různých podstránek webu se k té konfiguraci dostaneme přes property `root_page`
+kterou přidává `shared.models.SubpageMixin`. V `xxxHomePage` webu je třeba
+definovat `root_page` tak, aby vedla na `self`.
+
+### Styleguide
+
+Některé weby využívají [Pirátskou UI styleguide](https://gitlab.pirati.cz/to/weby/ui-styleguide)
+pro vzhled. Idea je, že se budou sdílet nejen statické assety (CSS, JS, ...),
+ale i univerzální kusy template, které se dají includovat v různých webech
+(patička webu, karta článku do přehledu, atp.).
+
+Sdílené části využívané na více webech se umísťují do:
+
+    shared/static/styleguideXX/     = statické assety (CSS, JS, obrázky, ...)
+    shared/templates/styleguideXX/  = snippety pro include v templatech
+
+`XX` v názvu adresáře je číslo major a minor verze šablony. Např. `styleguide18`
+obsahuje věci ze šablony verze `1.8.x`. Důvodem je snažší migrace na novější
+verze šablony, které mohou obsahovat nekompatibilní změny. Každý web tak může
+(a nemusí) migrovat nezávisle dle potřeby.
+
+Různé verze šablony jsou k vidění na [styleguide.pirati.cz](https://styleguide.pirati.cz/)
+
+### Kalendáře
+
+Pro práci s kalendáři v iCal formátu je připravena appka `calendar_utils`.
+
+Poskytuje `CalendarMixin` do modelu, který přidá fieldy `calendar_url` pro
+editaci a `calendar` pro vazbu na model `Calendar` (který se plní a automaticky
+spravuje na pozadí). Typicky se použije ve Wagtail settings pro web, kde stačí
+`calendar_url` zpřístupnit pro editaci.
+
+Kalendář se stáhne při uložení modelu obsahujícího `CalendarMixin`.
+
+Appka přidává management command `update_callendars`, který stahuje a updatuje
+kalendáře. Je třeba ho pravidelně volat na pozadí (přes CRON).
+
+### Celery (Import z Jekyllu)
+
+Import z Jekyll GitHub repozitářů pirátských webů je řešen asynchroně přes Celery.
+Celery využívá Redis, který může běžet např. lokálně, typicky 6379:
+V envu pak je pak potřeba mít nastavený Celery broker:
+
+```
+CELERY_BROKER_URL=redis://localhost:6379/6
+CELERY_RESULT_BACKEND=redis://localhost:6379/6
+```
+
+Aby se celery tasky vykonávaly, je potřeba pustit celery v terminálu:
+
+```
+celery -A majak worker
+```
+
+Pokud není zadán `CELERY_BROKER_URL`, tak se automaticky nastaví
+`CELERY_TASK_ALWAYS_EAGER = True`, čímž se tasky vykonají synchronně přímo v
+procesu webserveru.
+
+### Stránka 404
+
+Pokud je třeba vlastní 404 pro web, stačí do kořenové `xxxHomePage` webu
+definovat metodu `get_404_response` vracející Django HTTP Reponse objekt.
+
+    def get_404_response(self, request):
+        return HttpResponse("Stránka nenalezena", status=404)
+
+## Deployment
+
+### Konfigurace
+
+Je třeba nastavit environment proměnné:
+
+| proměnná | default | popis |
+| --- | --- | --- |
+| `DATABASE_URL` | | DSN k databázi (např. `postgres://user:pass@localhost:5432/majak`) |
+| `OIDC_RP_REALM_URL` | | OpenID server realm URL (např. `http://localhost:8080/auth/realms/master/`) |
+| `OIDC_RP_CLIENT_ID` | | OpenID Client ID |
+| `OIDC_RP_CLIENT_SECRET` | | OpenID Client Secret |
+| `BASE_URL` | https://majak.pirati.cz | základní URL pro notifikační emaily apod. |
+
+V produkci musí být navíc nastaveno:
+
+| proměnná | default | popis |
+| --- | --- | --- |
+| `DJANGO_SECRET_KEY` | | tajný šifrovací klíč |
+| `DJANGO_ALLOWED_HOSTS` | | allowed hosts (více hodnot odděleno čárkami) |
+| `CELERY_BROKER_URL` | | URL pro Celery Broker |
+| `CELERY_RESULT_BACKEND` | | URL pro Celery Result Backend |
+| `EMAIL_HOST` | SMTP pro odesílání přihlášek do kariér |
+| `EMAIL_HOST_USER` | --||-- Username |
+| `EMAIL_HOST_PASSWORD` | --||-- Heslo |
+| `EMAIL_PORT` | --||-- Port |
+| `CLAMD_TCP_ADDR` | ClamAV host (pro skenování virů v nahraných souborech) |
+| `CLAMD_TCP_SOCKET` | ClamAV socket |
+
+Různé:
+
+| proměnná | default | popis |
+| --- | --- | --- |
+| `MAJAK_ENV` | prod | `prod`/`test`/`dev` prostředí kde Maják běží |
+| `SENTRY_DSN` | | pokud je zadáno, pády se reportují do Sentry |
+| `SEARCH_CONFIG` | english | nastavení jazyka fulltextového vyhledávání, viz níže |
+| `DEBUG_TOOLBAR` | False | zobrazit Django Debug Toolbar (pro vývoj) |
+| `MAILTRAIN_API_URL` | | URL k API Mailtrain |
+| `MAILTRAIN_API_TOKEN` | | token k API Mailtrain |
+
+Settings pro appky na weby:
+
+| proměnná | default | popis |
+| --- | --- | --- |
+| `DONATE_PORTAL_REDIRECT_URL` | "" | URL pro přesměrování z darovacího formuláře |
+| `DONATE_PORTAL_REDIRECT_SOURCE` | dary.pirati.cz | identifikátor zdroje pro přesměrování na darovací portál |
+| `DONATE_PORTAL_API_URL` | "" | URL s API darovacího portálu |
+
+### Management commands
+
+Přes CRON je třeba na pozadí spouštět Django `manage.py` commandy:
+
+* `clearsessions` - maže expirované sessions (denně až týdně)
+* `publish_scheduled_pages` - publikuje naplánované stránky (každou hodinu)
+* `update_callendars` - stáhne a aktualizuje kalendáře (několikrát denně)
+* `update_main_timeline_articles` - aktualizuje články na `pirati.cz` z `https://piratipracuji.cz/api/`
+* `update_redmine_issues` - aktualizuje programované body MS a KS stránek napojených na Redmine (několikrát denně)
+
+### Fulltextové vyhledávání v češtině
+
+Pro fulltextové vyhledávání je třeba do PostgreSQL přidat
+[slovníky](https://github.com/f00b4r/postgresql-czech) do adresáře
+`/usr/local/share/postgresql/tsearch_data/`.
+
+V databázi Majáku je třeba nakonfigurovat český fulltext:
+
+```sql
+CREATE TEXT SEARCH DICTIONARY cspell (template=ispell, dictfile=czech, afffile=czech, stopwords=czech);
+CREATE TEXT SEARCH CONFIGURATION czech (copy=english);
+ALTER TEXT SEARCH CONFIGURATION czech ALTER MAPPING FOR word, asciiword WITH cspell, simple;
+CREATE extension IF NOT EXISTS unaccent;
+```
+
+Otestovat funkčnost lze dotazem:
+
+```sql
+SELECT * FROM ts_debug('czech', 'Příliš žluťoučký kůň se napil žluté vody');
+```
+
+Dále nastavit environment proměnnou `SEARCH_CONFIG=czech`. A nakonec jednorázově
+spustit naplnění indexu pro vyhledávání `python manage.py update_index`.
+Aktualizace indexu poté probíhají automaticky při uložení stránek.
+
+### Přidání nového webu
+
+Doména či subdoména se musí nakonfigurovat v:
+
+* environment proměnné `DJANGO_ALLOWED_HOSTS`
+* proxy před Majákem
+
 ## Vývoj
 
 Pro vývoj je definován pomocný `Makefile` pro časté akce. Pro nápovědu zavolej:
@@ -80,9 +267,34 @@ To nainstaluje Pythonní závislosti pro vývoj projektu na lokále.
 
 #### Nastavení environment proměnných
 
-Nastav environment proměnné (viz konfigurace výše). Pro jednoduchost doporučujeme
-použít [direnv](https://direnv.net/), který nastaví environment proměnné pro vývoj
-při změně adresáře na adresář s projektem. 
+Environment proměnné (viz konfigurace výše) se načítají ze souboru `.env`, který
+může vypadat takto:
+
+    DATABASE_URL=postgres://db:db@localhost:5432/majak
+    OIDC_RP_REALM_URL=http://localhost:8080/auth/realms/master/
+    OIDC_RP_CLIENT_ID=majak
+    OIDC_RP_CLIENT_SECRET=abcd
+
+Pro lokální vývoj obsahují settings tyto výchozí hodnoty:
+
+    DEBUG = True
+    ALLOWED_HOSTS = ["*"]
+    MAJAK_ENV = "dev"
+
+#### Autentifikace a autorizace
+
+Jako OIDC server můžete použít lokálně spuštěný Keycloak z [Dev Tools](https://gitlab.pirati.cz/to/dev-tools).
+
+Prvním přihlášením přes OIDC se vytvoří uživatel, ale bez oprávnění. Je třeba
+mu přes Django shell (viz níže) nastavit oprávnění pro přístup do administrace:
+
+```python
+from users.models import User
+u = User.objects.get()
+u.is_superuser = True
+u.is_staff = True
+u.save()
+```
 
 ### Management projektu
 
@@ -104,12 +316,47 @@ Django development server na portu `8006` se spustí příkazem:
 
 Poté můžete otevřít web na adrese [http://localhost:8006](http://localhost:8006)
 
+##### Debug Toolbar
+
+Pro spuštění development serveru s Django Debug Toolbar nastavte environment
+proměnnou `DEBUG_TOOLBAR`. Např.:
+
+    $ DEBUG_TOOLBAR=1 make run
+
 #### Django shell
 
 Django shell používající `shell_plus` z Django extensions spustíte:
 
     $ make shell
 
+### Lokální weby
+
+Pro vývoj různých webů si nastavte v `/etc/hosts` (na Linux, Mac OS, apod.)
+pojmenované aliasy k lokální IP 127.0.0.1 jako např.:
+
+    127.0.0.1   majak.loc dalsiweb.loc
+
+K Majáku spuštěnému v development serveru na portu `8006` se pak dostanete na
+adrese [http://majak.loc:8006/admin/](http://majak.loc:8006/admin/).
+
+A pokud nějakému webu v nastavení v Majáku dáte adresu třeba `dalsiweb.loc` a
+port `8006`, dostanete se pak na něj přes adresu
+[http://dalsiweb.loc:8006](http://dalsiweb.loc:8006).
+
+### Testy
+
+Používá se testovací framework [pytest](https://pytest.org). Spuštění testů:
+
+    $ pytest
+
+Případně přes `make`, ale bez možnosti parametrizovat spuštění testů:
+
+    $ make test
+
+Coverage report:
+
+    $ make coverage
+
 ### Code quality
 
 K formátování kódu se používá [black](https://github.com/psf/black). Doporučujeme
@@ -144,5 +391,6 @@ K upgrade se používají [pip-tools](https://github.com/jazzband/pip-tools) (`p
     $ cd requirements/
     $ pip-compile -U base.in
     $ pip-compile -U production.in
+    $ pip-compile -U dev.in
 
-Tím se vygenerují `base.txt` a `production.txt`.
+Tím se vygenerují `base.txt`, `production.txt` a `dev.txt`.

article_import_utils/apps.py

+from django.apps import AppConfig
+
+
+class ArticleImportUtilsConfig(AppConfig):
+    name = "article_import_utils"

calendar_utils/apps.py

+from django.apps import AppConfig
+
+
+class CalendarUtilsConfig(AppConfig):
+    name = "calendar_utils"

calendar_utils/icalevents/__init__.py

+"""
+iCalEvents search all events occurring in a given time frame in an iCal file.
+"""
+
+__all__ = ["icaldownload", "icalparser", "icalevents"]

calendar_utils/icalevents/icaldownload.py

+"""
+Downloads an iCal url or reads an iCal file.
+"""
+import logging
+
+from httplib2 import Http
+
+
+def apple_data_fix(content):
+    """
+    Fix Apple tzdata bug.
+
+    :param content: content to fix
+    :return: fixed content
+    """
+    return content.replace("TZOFFSETFROM:+5328", "TZOFFSETFROM:+0053")
+
+
+def apple_url_fix(url):
+    """
+    Fix Apple URL.
+
+    :param url: URL to fix
+    :return: fixed URL
+    """
+    if url.startswith("webcal://"):
+        url = url.replace("webcal://", "http://", 1)
+    return url
+
+
+class ICalDownload:
+    """
+    Downloads or reads and decodes iCal sources.
+    """
+
+    def __init__(self, http=None, encoding="utf-8"):
+        # Get logger
+        logger = logging.getLogger()
+
+        # default http connection to use
+        if http is None:
+            try:
+                http = Http(".cache")
+            except (PermissionError, OSError) as e:
+                # Cache disabled if no write permission in working directory
+                logger.warning(
+                    (
+                        "Caching is disabled due to a read-only working directory: {}"
+                    ).format(e)
+                )
+                http = Http()
+
+        self.http = http
+        self.encoding = encoding
+
+    def data_from_url(self, url, apple_fix=False):
+        """
+        Download iCal data from URL.
+
+        :param url: URL to download
+        :param apple_fix: fix Apple bugs (protocol type and tzdata in iCal)
+        :return: decoded (and fixed) iCal data
+        """
+        if apple_fix:
+            url = apple_url_fix(url)
+
+        _, content = self.http.request(url)
+
+        if not content:
+            raise ConnectionError("Could not get data from %s!" % url)
+
+        return self.decode(content, apple_fix=apple_fix)
+
+    def data_from_file(self, file, apple_fix=False):
+        """
+        Read iCal data from file.
+
+        :param file: file to read
+        :param apple_fix: fix wrong Apple tzdata in iCal
+        :return: decoded (and fixed) iCal data
+        """
+        with open(file, mode="rb") as f:
+            content = f.read()
+
+        if not content:
+            raise IOError("File %s is not readable or is empty!" % file)
+
+        return self.decode(content, apple_fix=apple_fix)
+
+    def data_from_string(self, string_content, apple_fix=False):
+        if not string_content:
+            raise IOError("String content is not readable or is empty!")
+
+        return self.decode(string_content, apple_fix=apple_fix)
+
+    def decode(self, content, apple_fix=False):
+        """
+        Decode content using the set charset.
+
+        :param content: content do decode
+        :param apple_fix: fix Apple txdata bug
+        :return: decoded (and fixed) content
+        """
+        content = content.decode(self.encoding)
+        content = content.replace("\r", "")
+
+        if apple_fix:
+            content = apple_data_fix(content)
+
+        return content

calendar_utils/icalevents/icalevents.py

+from threading import Lock, Thread
+
+from .icaldownload import ICalDownload
+from .icalparser import Event, parse_events
+
+# Lock for event data
+event_lock = Lock()
+# Event data
+event_store = {}
+# Threads
+threads = {}
+
+
+def events(
+    url=None,
+    file=None,
+    string_content=None,
+    start=None,
+    end=None,
+    fix_apple=False,
+    http=None,
+    tzinfo=None,
+    sort=None,
+    strict=False,
+) -> list[Event]:
+    """
+    Get all events form the given iCal URL occurring in the given time range.
+
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date (see dateutils.date)
+    :param end: end date (see dateutils.date)
+    :param fix_apple: fix known Apple iCal issues
+    :param tzinfo: return values in specified tz
+    :param sort: sort return values
+    :param strict: return dates, datetimes and datetime with timezones as specified in ical
+    :sort sorts events by start time
+
+    :return events
+    """
+    found_events = []
+
+    content = None
+    ical_download = ICalDownload(http=http)
+
+    if url:
+        content = ical_download.data_from_url(url, apple_fix=fix_apple)
+
+    if not content and file:
+        content = ical_download.data_from_file(file, apple_fix=fix_apple)
+
+    if not content and string_content:
+        content = ical_download.data_from_string(string_content, apple_fix=fix_apple)
+
+    found_events += parse_events(
+        content, start=start, end=end, tzinfo=tzinfo, sort=sort, strict=strict
+    )
+
+    if found_events is not None and sort is True:
+        found_events.sort()
+
+    return found_events
+
+
+def request_data(key, url, file, string_content, start, end, fix_apple):
+    """
+    Request data, update local data cache and remove this Thread from queue.
+
+    :param key: key for data source to get result later
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date
+    :param end: end date
+    :param fix_apple: fix known Apple iCal issues
+    """
+    data = []
+
+    try:
+        data += events(
+            url=url,
+            file=file,
+            string_content=string_content,
+            start=start,
+            end=end,
+            fix_apple=fix_apple,
+        )
+    finally:
+        update_events(key, data)
+        request_finished(key)
+
+
+def events_async(
+    key, url=None, file=None, start=None, string_content=None, end=None, fix_apple=False
+):
+    """
+    Trigger an asynchronous data request.
+
+    :param key: key for data source to get result later
+    :param url: iCal URL
+    :param file: iCal file path
+    :param string_content: iCal content as string
+    :param start: start date
+    :param end: end date
+    :param fix_apple: fix known Apple iCal issues
+    """
+    t = Thread(
+        target=request_data,
+        args=(key, url, file, string_content, start, end, fix_apple),
+    )
+
+    with event_lock:
+        if key not in threads:
+            threads[key] = []
+
+        threads[key].append(t)
+
+        if not threads[key][0].is_alive():
+            threads[key][0].start()
+
+
+def request_finished(key):
+    """
+    Remove finished Thread from queue.
+
+    :param key: data source key
+    """
+    with event_lock:
+        threads[key] = threads[key][1:]
+
+        if threads[key]:
+            threads[key][0].run()
+
+
+def update_events(key, data):
+    """
+    Set the latest events for a key.
+
+    :param key: key to set
+    :param data: events for key
+    """
+    with event_lock:
+        event_store[key] = data
+
+
+def latest_events(key):
+    """
+    Get the latest downloaded events for the given key.
+
+    :return: events for key
+    """
+    with event_lock:
+        # copy data
+        res = event_store[key][:]
+
+    return res
+
+
+def all_done(key):
+    """
+    Check if requests for the given key are active.
+
+    :param key: key for requests
+    :return: True if requests are pending or active
+    """
+    with event_lock:
+        if threads[key]:
+            return False
+        return True

calendar_utils/icalevents/icalparser.py

+"""
+Parse iCal data to Events.
+"""
+from datetime import date, datetime, timedelta
+
+# for UID generation
+from random import randint
+from typing import Optional
+from uuid import uuid4
+
+from dateutil.rrule import rrulestr
+from dateutil.tz import UTC, gettz
+from icalendar import Calendar
+from icalendar.prop import vDDDLists, vText
+from icalendar.timezone.windows_to_olson import WINDOWS_TO_OLSON
+from pytz import timezone
+
+
+def now():
+    """
+    Get current time.
+
+    :return: now as datetime with timezone
+    """
+    return datetime.now(UTC)
+
+
+class Attendee(str):
+    def __init__(self, address):
+        self.address = address
+
+    def __repr__(self):
+        return self.address.encode("utf-8").decode("ascii")
+
+    @property
+    def params(self):
+        return self.address.params
+
+
+class Event:
+    """
+    Represents one event (occurrence in case of reoccurring events).
+    """
+
+    def __init__(self):
+        """
+        Create a new event occurrence.
+        """
+        self.uid = -1
+        self.summary = None
+        self.description = None
+        self.start = None
+        self.end = None
+        self.all_day = True
+        self.transparent = False
+        self.recurring = False
+        self.location = None
+        self.private = False
+        self.created = None
+        self.last_modified = None
+        self.sequence = None
+        self.recurrence_id = None
+        self.attendee = None
+        self.organizer = None
+        self.categories = None
+        self.floating = None
+        self.status = None
+        self.url = None
+
+    def time_left(self, time=None):
+        """
+        timedelta form now to event.
+
+        :return: timedelta from now
+        """
+        time = time or now()
+        return self.start - time
+
+    def __lt__(self, other):
+        """
+        Events are sorted by start time by default.
+
+        :param other: other event
+        :return: True if start of this event is smaller than other
+        """
+        if not other or not isinstance(other, Event):
+            raise ValueError(
+                "Only events can be compared with each other! Other is %s" % type(other)
+            )
+        else:
+            # start and end can be dates, datetimes and datetimes with timezoneinfo
+            if type(self.start) is date and type(other.start) is date:
+                return self.start < other.start
+            elif type(self.start) is datetime and type(other.start) is datetime:
+                if self.start.tzinfo == other.start.tzinfo:
+                    return self.start < other.start
+                else:
+                    return self.start.astimezone(UTC) < other.start.astimezone(UTC)
+            elif type(self.start) is date and type(other.start) is datetime:
+                return self.start < other.start.date()
+            elif type(self.start) is datetime and type(other.start) is date:
+                return self.start.date() < other.start
+
+    def __str__(self):
+        return "%s: %s (%s)" % (self.start, self.summary, self.end - self.start)
+
+    def astimezone(self, tzinfo):
+        if type(self.start) is datetime:
+            self.start = self.start.astimezone(tzinfo)
+
+        if type(self.end) is datetime:
+            self.end = self.end.astimezone(tzinfo)
+
+        return self
+
+    def copy_to(self, new_start=None, uid=None):
+        """
+        Create a new event equal to this with new start date.
+
+        :param new_start: new start date
+        :param uid: UID of new event
+        :return: new event
+        """
+        if not new_start:
+            new_start = self.start
+
+        if not uid:
+            uid = "%s_%d" % (self.uid, randint(0, 1000000))
+
+        ne = Event()
+        ne.summary = self.summary
+        ne.description = self.description
+        ne.start = new_start
+
+        if self.end:
+            duration = self.end - self.start
+            ne.end = new_start + duration
+
+        ne.all_day = self.all_day
+        ne.recurring = self.recurring
+        ne.location = self.location
+        ne.attendee = self.attendee
+        ne.organizer = self.organizer
+        ne.private = self.private
+        ne.transparent = self.transparent
+        ne.uid = uid
+        ne.created = self.created
+        ne.last_modified = self.last_modified
+        ne.categories = self.categories
+        ne.floating = self.floating
+        ne.status = self.status
+        ne.url = self.url
+
+        return ne
+
+
+def encode(value: Optional[vText]) -> Optional[str]:
+    if value is None:
+        return None
+    try:
+        return str(value)
+    except UnicodeEncodeError:
+        return str(value.encode("utf-8"))
+
+
+def create_event(component, utc_default):
+    """
+    Create an event from its iCal representation.
+
+    :param component: iCal component
+    :return: event
+    """
+
+    event = Event()
+
+    event.start = component.get("dtstart").dt
+    # The RFC specifies that the TZID parameter must be specified for datetime or time
+    # Otherwise we set a default timezone (if only one is set with VTIMEZONE) or utc
+    event.floating = type(component.get("dtstart").dt) == date and utc_default
+
+    if component.get("dtend"):
+        event.end = component.get("dtend").dt
+    elif component.get("duration"):  # compute implicit end as start + duration
+        event.end = event.start + component.get("duration").dt
+    else:  # compute implicit end as start + 0
+        event.end = event.start
+
+    event.summary = encode(component.get("summary"))
+    event.description = encode(component.get("description"))
+    event.all_day = type(component.get("dtstart").dt) is date
+    if component.get("rrule"):
+        event.recurring = True
+    event.location = encode(component.get("location"))
+
+    if component.get("attendee"):
+        event.attendee = component.get("attendee")
+        if type(event.attendee) is list:
+            event.attendee = [Attendee(attendee) for attendee in event.attendee]
+        else:
+            event.attendee = Attendee(event.attendee)
+    else:
+        event.attendee = str(None)
+
+    if component.get("uid"):
+        event.uid = component.get("uid").encode("utf-8").decode("ascii")
+    else:
+        event.uid = str(uuid4())  # Be nice - treat every event as unique
+
+    if component.get("organizer"):
+        event.organizer = component.get("organizer").encode("utf-8").decode("ascii")
+    else:
+        event.organizer = str(None)
+
+    if component.get("class"):
+        event_class = component.get("class")
+        event.private = event_class == "PRIVATE" or event_class == "CONFIDENTIAL"
+
+    if component.get("transp"):
+        event.transparent = component.get("transp") == "TRANSPARENT"
+
+    if component.get("created"):
+        event.created = component.get("created").dt
+
+    if component.get("RECURRENCE-ID"):
+        rid = component.get("RECURRENCE-ID").dt
+
+        # Spec defines that if DTSTART is a date RECURRENCE-ID also is to be interpreted as a date
+        if type(component.get("dtstart").dt) is date:
+            event.recurrence_id = date(year=rid.year, month=rid.month, day=rid.day)
+        else:
+            event.recurrence_id = rid
+
+    if component.get("last-modified"):
+        event.last_modified = component.get("last-modified").dt
+    elif event.created:
+        event.last_modified = event.created
+
+    # sequence can be 0 - test for None instead
+    if not component.get("sequence") is None:
+        event.sequence = component.get("sequence")
+
+    if component.get("categories"):
+        categoriesval = component.get("categories")
+        categories = (
+            component.get("categories").cats
+            if hasattr(categoriesval, "cats")
+            else categoriesval
+        )
+        encoded_categories = list()
+        for category in categories:
+            encoded_categories.append(encode(category))
+        event.categories = encoded_categories
+
+    if component.get("status"):
+        event.status = encode(component.get("status"))
+
+    if component.get("url"):
+        event.url = encode(component.get("url"))
+
+    return event
+
+
+def parse_events(
+    content,
+    start=None,
+    end=None,
+    default_span=timedelta(days=7),
+    tzinfo=None,
+    sort=False,
+    strict=False,
+):
+    """
+    Query the events occurring in a given time range.
+
+    :param content: iCal URL/file content as String
+    :param start: start date for search, default today (in UTC format)
+    :param end: end date for search (in UTC format)
+    :param default_span: default query length (one week)
+    :return: events as list
+    """
+    if not start:
+        start = now()
+
+    if not end:
+        end = start + default_span
+
+    if not content:
+        raise ValueError("Content is invalid!")
+
+    calendar = Calendar.from_ical(content)
+
+    # > Will be deprecated ========================
+    # Calendar.from_ical already parses timezones as specified in the ical.
+    # We can specify a 'default' tz but this is not according to spec.
+    # Kept this here to verify tests and backward compatibility
+
+    # Keep track of the timezones defined in the calendar
+    timezones = {}
+
+    # Parse non standard timezone name
+    if "X-WR-TIMEZONE" in calendar:
+        x_wr_timezone = str(calendar["X-WR-TIMEZONE"])
+        timezones[x_wr_timezone] = get_timezone(x_wr_timezone)
+
+    for c in calendar.walk("VTIMEZONE"):
+        name = str(c["TZID"])
+        try:
+            timezones[name] = c.to_tz()
+        except IndexError:
+            # This happens if the VTIMEZONE doesn't
+            # contain start/end times for daylight
+            # saving time. Get the system pytz
+            # value from the name as a fallback.
+            timezones[name] = timezone(name)
+
+    # If there's exactly one timezone in the file,
+    # assume it applies globally, otherwise UTC
+    utc_default = False
+    if len(timezones) == 1:
+        cal_tz = get_timezone(list(timezones)[0])
+    else:
+        utc_default = True
+        cal_tz = UTC
+    # < ==========================================
+
+    found = []
+
+    def add_if_not_exception(event):
+        exdate = "%04d%02d%02d" % (
+            event.start.year,
+            event.start.month,
+            event.start.day,
+        )
+
+        if exdate not in exceptions:
+            found.append(event)
+
+    for component in calendar.walk():
+        exceptions = {}
+
+        if "EXDATE" in component:
+            # Deal with the fact that sometimes it's a list and
+            # sometimes it's a singleton
+            exlist = []
+            if isinstance(component["EXDATE"], vDDDLists):
+                exlist = component["EXDATE"].dts
+            else:
+                exlist = component["EXDATE"]
+            for ex in exlist:
+                exdate = ex.to_ical().decode("UTF-8")
+                exceptions[exdate[0:8]] = exdate
+
+        if component.name == "VEVENT":
+            e = create_event(component, utc_default)
+
+            # make rule.between happy and provide from, to points in time that have the same format as dtstart
+            s = component["dtstart"].dt
+            if type(s) is date and not e.recurring:
+                f, t = date(start.year, start.month, start.day), date(
+                    end.year, end.month, end.day
+                )
+            elif type(s) is datetime and s.tzinfo:
+                f, t = datetime(
+                    start.year, start.month, start.day, tzinfo=s.tzinfo
+                ), datetime(end.year, end.month, end.day, tzinfo=s.tzinfo)
+            else:
+                f, t = datetime(start.year, start.month, start.day), datetime(
+                    end.year, end.month, end.day
+                )
+
+            if e.recurring:
+                rule = parse_rrule(component)
+                for dt in rule.between(f, t, inc=True):
+                    # Recompute the start time in the current timezone *on* the
+                    # date of *this* occurrence. This handles the case where the
+                    # recurrence has crossed over the daylight savings time boundary.
+                    if type(dt) is datetime and dt.tzinfo:
+                        dtstart = dt.replace(tzinfo=get_timezone(str(dt.tzinfo)))
+                        ecopy = e.copy_to(
+                            dtstart.date() if type(s) is date else dtstart, e.uid
+                        )
+                    else:
+                        ecopy = e.copy_to(dt.date() if type(s) is date else dt, e.uid)
+                    add_if_not_exception(ecopy)
+
+            elif e.end >= f and e.start <= t:
+                add_if_not_exception(e)
+
+    result = found.copy()
+
+    # Remove events that are replaced in ical
+    for event in found:
+        if not event.recurrence_id and (event.uid, event.start) in [
+            (f.uid, f.recurrence_id) for f in found
+        ]:
+            result.remove(event)
+
+    # > Will be deprecated ========================
+    # We will apply default cal_tz as required by some tests.
+    # This is just here for backward-compatibility
+    if not strict:
+        for event in result:
+            if type(event.start) is date:
+                event.start = datetime(
+                    year=event.start.year,
+                    month=event.start.month,
+                    day=event.start.day,
+                    hour=0,
+                    minute=0,
+                    tzinfo=cal_tz,
+                )
+                event.end = datetime(
+                    year=event.end.year,
+                    month=event.end.month,
+                    day=event.end.day,
+                    hour=0,
+                    minute=0,
+                    tzinfo=cal_tz,
+                )
+            elif type(event.start) is datetime:
+                if event.start.tzinfo:
+                    event.start = event.start.astimezone(cal_tz)
+                    event.end = event.end.astimezone(cal_tz)
+                else:
+                    event.start = event.start.replace(tzinfo=cal_tz)
+                    event.end = event.end.replace(tzinfo=cal_tz)
+
+            if event.created:
+                if type(event.created) is date:
+                    event.created = datetime(
+                        year=event.created.year,
+                        month=event.created.month,
+                        day=event.created.day,
+                        hour=0,
+                        minute=0,
+                        tzinfo=cal_tz,
+                    )
+                if type(event.created) is datetime:
+                    if event.created.tzinfo:
+                        event.created = event.created.astimezone(cal_tz)
+                    else:
+                        event.created = event.created.replace(tzinfo=cal_tz)
+
+            if event.last_modified:
+                if type(event.last_modified) is date:
+                    event.last_modified = datetime(
+                        year=event.last_modified.year,
+                        month=event.last_modified.month,
+                        day=event.last_modified.day,
+                        hour=0,
+                        minute=0,
+                        tzinfo=cal_tz,
+                    )
+                if type(event.last_modified) is datetime:
+                    if event.last_modified.tzinfo:
+                        event.last_modified = event.last_modified.astimezone(cal_tz)
+                    else:
+                        event.last_modified = event.last_modified.replace(tzinfo=cal_tz)
+    # < ==========================================
+
+    if sort:
+        result.sort()
+
+    if tzinfo:
+        result = [event.astimezone(tzinfo) for event in result]
+
+    return result
+
+
+def parse_rrule(component):
+    """
+    Extract a dateutil.rrule object from an icalendar component. Also includes
+    the component's dtstart and exdate properties. The rdate and exrule
+    properties are not yet supported.
+
+    :param component: icalendar component
+    :return: extracted rrule or rruleset or None
+    """
+
+    dtstart = component.get("dtstart").dt
+
+    # component['rrule'] can be both a scalar and a list
+    rrules = component.get("rrule")
+    if not isinstance(rrules, list):
+        rrules = [rrules]
+
+    def conform_until(until, dtstart):
+        if type(dtstart) is datetime:
+            # If we have timezone defined adjust for daylight saving time
+            if type(until) is datetime:
+                return until + abs(
+                    (
+                        until.astimezone(dtstart.tzinfo).utcoffset()
+                        if until.tzinfo is not None and dtstart.tzinfo is not None
+                        else None
+                    )
+                    or timedelta()
+                )
+
+            return (
+                until.astimezone(UTC)
+                if type(until) is datetime
+                else datetime(
+                    year=until.year, month=until.month, day=until.day, tzinfo=UTC
+                )
+            ) + (
+                (dtstart.tzinfo.utcoffset(dtstart) if dtstart.tzinfo else None)
+                or timedelta()
+            )
+
+        return until.date() + timedelta(days=1) if type(until) is datetime else until
+
+    for index, rru in enumerate(rrules):
+        if "UNTIL" in rru:
+            rrules[index]["UNTIL"] = [
+                conform_until(until, dtstart) for until in rrules[index]["UNTIL"]
+            ]
+
+    rule = rrulestr(
+        "\n".join(x.to_ical().decode() for x in rrules),
+        dtstart=dtstart,
+        forceset=True,
+        unfold=True,
+    )
+
+    if component.get("exdate"):
+        # Add exdates to the rruleset
+        for exd in extract_exdates(component):
+            if type(dtstart) is date:
+                rule.exdate(
+                    datetime(
+                        year=exd.year,
+                        month=exd.month,
+                        day=exd.day,
+                        hour=0,
+                        minute=0,
+                        second=0,
+                    )
+                    if isinstance(exd, date)
+                    else exd
+                )
+            else:
+                rule.exdate(exd)
+
+    # TODO: What about rdates and exrules?
+    if component.get("exrule"):
+        pass
+
+    if component.get("rdate"):
+        pass
+
+    return rule
+
+
+def extract_exdates(component):
+    """
+    Compile a list of all exception dates stored with a component.
+
+    :param component: icalendar iCal component
+    :return: list of exception dates
+    """
+    dates = []
+    exd_prop = component.get("exdate")
+    if isinstance(exd_prop, list):
+        for exd_list in exd_prop:
+            dates.extend(exd.dt for exd in exd_list.dts)
+    else:  # it must be a vDDDLists
+        dates.extend(exd.dt for exd in exd_prop.dts)
+
+    return dates
+
+
+def get_timezone(tz_name):
+    if tz_name in WINDOWS_TO_OLSON:
+        return gettz(WINDOWS_TO_OLSON[tz_name])
+    else:
+        return gettz(tz_name)