From 8a0527e58addd366764bff094c1c95be90c8d0e2 Mon Sep 17 00:00:00 2001
From: "jindra12.underdark" <jindra12.underdark@gmail.com>
Date: Thu, 4 May 2023 01:26:18 +0200
Subject: [PATCH] Start building mastodon toots parser
#190
#184
---
.isort.cfg | 2 +-
shared/models.py | 129 ++++++++++++++++++
.../shared/mastodon_feed_snippet.html | 16 +--
.../shared/mastodon_feed_toots_snippet.html | 0
4 files changed, 132 insertions(+), 15 deletions(-)
create mode 100644 shared/templates/shared/mastodon_feed_toots_snippet.html
diff --git a/.isort.cfg b/.isort.cfg
index e59d6737..e2b23eaf 100644
--- a/.isort.cfg
+++ b/.isort.cfg
@@ -3,4 +3,4 @@
line_length = 88
multi_line_output = 3
include_trailing_comma = true
-known_third_party = PyPDF2,arrow,bleach,bs4,captcha,celery,dateutil,django,environ,faker,fastjsonschema,icalevents,markdown,modelcluster,pirates,pytest,pytz,requests,requests_cache,sentry_sdk,taggit,wagtail,wagtailmetadata,weasyprint,yaml
+known_third_party = PyPDF2,arrow,attr,bleach,bs4,captcha,celery,dateutil,django,environ,faker,fastjsonschema,icalevents,markdown,modelcluster,pirates,pytest,pytz,requests,requests_cache,sentry_sdk,taggit,wagtail,wagtailmetadata,weasyprint,yaml
diff --git a/shared/models.py b/shared/models.py
index 6ee08e0e..bb652dfd 100644
--- a/shared/models.py
+++ b/shared/models.py
@@ -1,7 +1,13 @@
import logging
+from time import time
+from attr import dataclass
+from django.core.files.temp import NamedTemporaryFile
+from django.core.serializers.json import DjangoJSONEncoder
from django.db import models
+from django.forms import ValidationError
from django.utils import timezone
+from requests import request
from wagtail.admin.panels import FieldPanel, MultiFieldPanel, PublishingPanel
from wagtail.fields import StreamField
from wagtail.models import Page
@@ -9,12 +15,104 @@ from wagtail.models import Page
from shared.blocks import (
DEFAULT_CONTENT_BLOCKS,
FooterLinksBlock,
+ ImageCreatorMixin,
MenuItemBlock,
MenuParentBlock,
)
logger = logging.getLogger(__name__)
+FIVE_MINUTES_IN_SECONDS = 300
+
+
+@dataclass
+class MastodonImage:
+ """
+ Partial type definition for mastodon API image link
+ """
+
+ url: str
+
+
+@dataclass
+class MastodonToot:
+ """
+ Partial type definition for mastodon API toot
+ """
+
+ id: str
+ content: str
+ published: str
+ attachment: list[MastodonImage]
+
+
+@dataclass
+class MastodonToots:
+ """
+ Partial type definition for mastodon API toots
+ """
+
+ orderedItems: list[MastodonToot]
+
+
+@dataclass
+class MastodonUser:
+ """
+ Partial type definition for mastodon API user
+ """
+
+ icon: MastodonImage
+ summary: str
+ preferredUsername: str
+
+
+class Mastodon(ImageCreatorMixin, models.Model):
+ url = models.URLField()
+ toots = models.JSONField(encoder=DjangoJSONEncoder, null=True)
+ summary = models.TextField(null=True, blank=True)
+ user_image = models.ImageField(null=True, blank=True, upload_to="mastodon_users/")
+ timestamp = models.FloatField()
+
+ def download_toots(self):
+ """
+ Downloads toots from mastodon feed
+ """
+ # https://mastodon.pirati.cz/users/ivanbartos/outbox?page=true
+ full_url = self.url + "/outbox?page=true"
+ return request("GET", full_url).json()
+
+ def download_user(self):
+ """
+ Downloads user page from mastodon feed
+ """
+ # https://mastodon.pirati.cz/users/ivanbartos.json
+ full_url = self.url + ".json"
+ return request("GET", full_url).json()
+
+ def parse_url(self, now: float):
+ toots: MastodonToots = self.download_toots()
+ user: MastodonUser = self.download_user()
+
+ # Taken from: https://gist.github.com/anderser/2172888
+ temporary_stored_image = NamedTemporaryFile(delete=True)
+ temporary_stored_image.write(user.icon.url)
+ temporary_stored_image.flush()
+ self.user_image.save(f"{user.preferredUsername}.jpg", temporary_stored_image)
+
+ self.toots = toots.orderedItems
+ self.summary = user.summary
+ self.timestamp = now
+
+ def refresh_toots(self):
+ now = time()
+ if self.timestamp != None and now - self.timestamp < FIVE_MINUTES_IN_SECONDS:
+ return
+ try:
+ self.parse_url(now)
+ self.save()
+ except:
+ logger.error("Mastodon refresh failed for %s", self.url, exc_info=True)
+
class MastodonFeedMixin(models.Model):
mastodon_feed = models.URLField(
@@ -22,6 +120,37 @@ class MastodonFeedMixin(models.Model):
blank=True,
help_text="Zadejte mastodon feed url v tomto formátu: https://mastodon.pirati.cz/users/ivanbartos",
)
+ mastodon = models.ForeignKey(
+ Mastodon, null=True, blank=True, on_delete=models.PROTECT
+ )
+
+ def clean(self):
+ super().clean()
+ try:
+ mastodon = (
+ self.mastodon
+ if self.mastodon is not None
+ else Mastodon.objects.create(url=self.calendar_url)
+ )
+ mastodon.parse_url(time())
+ except:
+ raise ValidationError("Update mastodonu se nepovedl")
+
+ def save(self, *args, **kwargs):
+ if self.mastodon_feed:
+ if self.mastodon:
+ if self.mastodon.url != self.mastodon_feed:
+ self.mastodon.url = self.mastodon_feed
+ self.mastodon.save()
+ else:
+ self.mastodon = Mastodon.objects.create(url=self.mastodon_feed)
+
+ self.mastodon.refresh_toots()
+
+ elif self.mastodon:
+ self.mastodon = None
+
+ super().save(*args, **kwargs)
class Meta:
abstract = True
diff --git a/shared/templates/shared/mastodon_feed_snippet.html b/shared/templates/shared/mastodon_feed_snippet.html
index 24a34a74..c135a0c8 100644
--- a/shared/templates/shared/mastodon_feed_snippet.html
+++ b/shared/templates/shared/mastodon_feed_snippet.html
@@ -2,24 +2,12 @@
{% if is_link %}
<a href="{{ page.mastodon_feed }}" class="social-icon ">{% include "shared/mastodon_icon_snippet.html" with size=link_size invert=True %}</a>
{% else %}
- <iframe
- allowfullscreen
- sandbox="allow-top-navigation allow-scripts allow-popups allow-popups-to-escape-sandbox"
- width="400"
- height="400"
- src="https://mastofeed.com/apiv2/feed?userurl={{ page.mastodon_feed | urlencode:'' }}&theme=dark&size=100&header=true&replies=false&boosts=false">
- </iframe>
+ {% include "shared/mastodon_feed_toots_snippet.html" with mastodon_feed=page.mastodon_feed %}
{% endif %}
{% elif page.root_page.mastodon_feed %}
{% if is_link %}
<a href="{{ page.root_page.mastodon_feed }}" class="social-icon ">{% include "shared/mastodon_icon_snippet.html" with size=link_size invert=True %}</a>
{% else %}
- <iframe
- allowfullscreen
- sandbox="allow-top-navigation allow-scripts allow-popups allow-popups-to-escape-sandbox"
- width="400"
- height="400"
- src="https://mastofeed.com/apiv2/feed?userurl={{ page.root_page.mastodon_feed | urlencode:'' }}&theme=dark&size=100&header=true&replies=false&boosts=false">
- </iframe>
+ {% include "shared/mastodon_feed_toots_snippet.html" with mastodon_feed=page.root_page.mastodon_feed %}
{% endif %}
{% endif %}
diff --git a/shared/templates/shared/mastodon_feed_toots_snippet.html b/shared/templates/shared/mastodon_feed_toots_snippet.html
new file mode 100644
index 00000000..e69de29b
--
GitLab