Skip to content
Snippets Groups Projects
Commit 96209edf authored by Tomi Valentová's avatar Tomi Valentová
Browse files

wip - import people from Octopus

parent 5292b2d3
No related branches found
No related tags found
2 merge requests!1097Release,!1095Add basic Octopus people sync
Pipeline #19274 passed
Stage: build
Show whitespace changes
Inline Side-by-side
Showing
with 339 additions and 40 deletions
district/forms.py
+ 21
− 1
View file @ 96209edf
import os
from shared.forms import JekyllImportForm as SharedJekyllImportForm
from shared.forms import OctopusPeopleImportForm as SharedOctopusPeopleImportForm
from .tasks import import_jekyll_articles
from .tasks import import_people_from_group
class JekyllImportForm(SharedJekyllImportForm):
def handle_import(self):
lock_file_name = f"/tmp/.{self.instance.id}.import-lock"
lock_file_name = f"/tmp/.{self.instance.id}.articles-import-lock"
if os.path.isfile(lock_file_name):
return
......@@ -21,3 +23,21 @@ class JekyllImportForm(SharedJekyllImportForm):
dry_run=self.cleaned_data["dry_run"],
use_git=True,
)
class OctopusImportForm(SharedOctopusPeopleImportForm):
def handle_import_from_group(self):
lock_file_name = f"/tmp/.{self.instance.id}.people-from-group-import-lock"
if os.path.isfile(lock_file_name):
return
open(lock_file_name, "w").close()
import_people_from_group.delay(
people_parent_page_id=self.instance.id,
collection_id=self.cleaned_data["collection"].id,
group_shortcut=self.cleaned_data["group_shortcut"],
dry_run=self.cleaned_data["dry_run"],
lock_file_name=lock_file_name
)
\ No newline at end of file
district/models.py
+ 23
− 1
View file @ 96209edf
......@@ -69,7 +69,7 @@ from shared.utils import (
)
from . import blocks
from .forms import JekyllImportForm
from .forms import JekyllImportForm, OctopusImportForm
CONTENT_BLOCKS = DEFAULT_CONTENT_BLOCKS + [
("chart", ChartBlock()),
......@@ -390,6 +390,8 @@ class DistrictPersonPage(MainPersonPageMixin):
class DistrictPeoplePage(MainPeoplePageMixin):
base_form_class = OctopusImportForm
content = StreamField(
[
("people_group", blocks.PeopleGroupBlock(label="Seznam osob")),
......@@ -403,6 +405,26 @@ class DistrictPeoplePage(MainPeoplePageMixin):
parent_page_types = ["district.DistrictHomePage"]
subpage_types = ["district.DistrictPersonPage"]
import_panels = [
MultiFieldPanel(
[
FieldPanel("do_import"),
FieldPanel("collection"),
FieldPanel("dry_run"),
FieldPanel("group_shortcut")
],
"import osob z Chobotnice",
),
]
edit_handler = TabbedInterface(
[
ObjectList(MainPeoplePageMixin.content_panels, heading="Obsah"),
ObjectList(MainPeoplePageMixin.promote_panels, heading="Metadata"),
ObjectList(import_panels, heading="Import"),
]
)
class DistrictCalendarPage(SubpageMixin, MetadataPageMixin, CalendarMixin, Page):
### PANELS
......
district/tasks.py
+ 22
− 0
View file @ 96209edf
......@@ -3,6 +3,7 @@ import logging
from celery import shared_task
from shared.jekyll_import import JekyllArticleImporter
from shared.people_import import PeopleGroupImporter
logger = logging.getLogger(__name__)
......@@ -26,3 +27,24 @@ def import_jekyll_articles(
use_git=use_git,
page_model=DistrictArticlePage,
).perform_import()
@shared_task()
def import_people_from_group(
people_parent_page_id,
collection_id,
group_shortcut,
dry_run,
lock_file_name
):
from .models import DistrictPeoplePage, DistrictPersonPage
return PeopleGroupImporter(
people_parent_page_id=people_parent_page_id,
people_parent_page_model=DistrictPeoplePage,
person_page_model=DistrictPersonPage,
collection_id=collection_id,
group_shortcut=group_shortcut,
dry_run=dry_run,
lock_file_name=lock_file_name,
).perform_import()
\ No newline at end of file
majak/settings/base.py
+ 1
− 0
View file @ 96209edf
......@@ -269,6 +269,7 @@ WAGTAILADMIN_BASE_URL = BASE_URL
# CUSTOM SETTINGS
# ------------------------------------------------------------------------------
STYLEGUIDE_URL = env.str("STYLEGUIDE_URL", "https://styleguide.pirati.cz/2.20.x/")
OCTOPUS_API_URL = env.str("OCTOPUS_API_URL", "https://chobotnice.pirati.cz/graphql/")
MAJAK_ENV = env.str("MAJAK_ENV", default="prod")
......
requirements/base.in
+ 1
− 0
View file @ 96209edf
......@@ -8,6 +8,7 @@ django-redis
django-settings-export
django-widget-tweaks
django-simple-captcha
gql[all]
psycopg2-binary
pirates
whitenoise==5.3.0
......
requirements/base.txt
+ 82
− 27
View file @ 96209edf
......@@ -4,10 +4,18 @@
#
# pip-compile base.in
#
aiohttp==3.9.5
# via gql
aiosignal==1.3.1
# via aiohttp
amqp==5.2.0
# via kombu
anyascii==0.3.2
# via wagtail
anyio==4.4.0
# via
# gql
# httpx
arrow==1.3.0
# via
# -r base.in
......@@ -18,9 +26,12 @@ asttokens==2.4.1
# via stack-data
attrs==23.2.0
# via
# aiohttp
# cattrs
# ics
# requests-cache
backoff==2.2.1
# via gql
beautifulsoup4==4.12.3
# via
# -r base.in
......@@ -29,14 +40,18 @@ billiard==4.2.0
# via celery
bleach==6.1.0
# via -r base.in
botocore==1.34.149
# via gql
brotli==1.1.0
# via fonttools
cattrs==23.2.3
# via requests-cache
celery==5.4.0
# via -r base.in
certifi==2024.6.2
certifi==2024.7.4
# via
# httpcore
# httpx
# requests
# sentry-sdk
cffi==1.16.0
......@@ -57,7 +72,7 @@ click-plugins==1.1.1
# via celery
click-repl==0.3.0
# via celery
cryptography==42.0.8
cryptography==43.0.0
# via
# josepy
# mozilla-django-oidc
......@@ -68,7 +83,7 @@ decorator==5.1.1
# via ipython
defusedxml==0.7.1
# via willow
django==5.0.6
django==5.0.7
# via
# -r base.in
# django-extensions
......@@ -109,7 +124,7 @@ django-treebeard==4.7.1
# via wagtail
django-widget-tweaks==1.5.0
# via -r base.in
djangorestframework==3.15.1
djangorestframework==3.15.2
# via wagtail
draftjs-exporter==5.0.0
# via wagtail
......@@ -117,26 +132,46 @@ et-xmlfile==1.1.0
# via openpyxl
executing==2.0.1
# via stack-data
fastjsonschema==2.19.1
fastjsonschema==2.20.0
# via -r base.in
filetype==1.2.0
# via willow
fonttools[woff]==4.53.0
fonttools[woff]==4.53.1
# via weasyprint
frozenlist==1.4.1
# via
# aiohttp
# aiosignal
gql[all]==3.5.0
# via -r base.in
graphql-core==3.2.3
# via gql
h11==0.14.0
# via httpcore
html5lib==1.1
# via weasyprint
httpcore==1.0.5
# via httpx
httplib2==0.22.0
# via -r base.in
icalendar==5.0.12
httpx==0.27.0
# via gql
icalendar==5.0.13
# via -r base.in
ics==0.7.2
# via -r base.in
idna==3.7
# via requests
ipython==8.25.0
# via
# anyio
# httpx
# requests
# yarl
ipython==8.26.0
# via -r base.in
jedi==0.19.1
# via ipython
jmespath==1.0.1
# via botocore
josepy==1.14.0
# via mozilla-django-oidc
kombu==5.3.7
......@@ -151,35 +186,39 @@ matplotlib-inline==0.1.7
# via ipython
mozilla-django-oidc==3.0.0
# via pirates
nh3==0.2.17
multidict==6.0.5
# via
# aiohttp
# yarl
nh3==0.2.18
# via -r base.in
numpy==1.26.4
numpy==2.0.1
# via opencv-python
oauthlib==3.2.2
# via
# requests-oauthlib
# tweepy
opencv-python==4.10.0.82
opencv-python==4.10.0.84
# via -r base.in
openpyxl==3.1.3
openpyxl==3.1.5
# via wagtail
parso==0.8.4
# via jedi
pexpect==4.9.0
# via ipython
pillow==10.3.0
pillow==10.4.0
# via
# django-simple-captcha
# pillow-heif
# wagtail
# weasyprint
pillow-heif==0.16.0
pillow-heif==0.18.0
# via willow
pirates==0.7.0
# via -r base.in
platformdirs==4.2.2
# via requests-cache
prompt-toolkit==3.0.46
prompt-toolkit==3.0.47
# via
# click-repl
# ipython
......@@ -187,15 +226,15 @@ psycopg2-binary==2.9.9
# via -r base.in
ptyprocess==0.7.0
# via pexpect
pure-eval==0.2.2
pure-eval==0.2.3
# via stack-data
pycparser==2.22
# via cffi
pydyf==0.10.0
pydyf==0.11.0
# via weasyprint
pygments==2.18.0
# via ipython
pyopenssl==24.1.0
pyopenssl==24.2.1
# via josepy
pyparsing==3.1.2
# via httplib2
......@@ -206,6 +245,7 @@ pyphen==0.15.0
python-dateutil==2.9.0.post0
# via
# arrow
# botocore
# celery
# icalendar
# ics
......@@ -217,21 +257,25 @@ pytz==2024.1
# l18n
pyyaml==6.0.1
# via -r base.in
redis==5.0.5
redis==5.0.7
# via django-redis
requests==2.32.3
# via
# -r base.in
# gql
# mozilla-django-oidc
# requests-cache
# requests-oauthlib
# requests-toolbelt
# tweepy
# wagtail
requests-cache==1.2.0
requests-cache==1.2.1
# via -r base.in
requests-oauthlib==1.3.1
# via tweepy
sentry-sdk==2.5.0
requests-toolbelt==1.0.0
# via gql
sentry-sdk==2.11.0
# via -r base.in
six==1.16.0
# via
......@@ -242,9 +286,13 @@ six==1.16.0
# l18n
# python-dateutil
# url-normalize
sniffio==1.3.1
# via
# anyio
# httpx
soupsieve==2.5
# via beautifulsoup4
sqlparse==0.5.0
sqlparse==0.5.1
# via django
stack-data==0.6.3
# via ipython
......@@ -264,14 +312,15 @@ tweepy==4.14.0
# via -r base.in
types-python-dateutil==2.9.0.20240316
# via arrow
typing-extensions==4.12.1
typing-extensions==4.12.2
# via ipython
tzdata==2024.1
# via celery
url-normalize==1.4.3
# via requests-cache
urllib3==2.2.1
urllib3==2.2.2
# via
# botocore
# requests
# requests-cache
# sentry-sdk
......@@ -280,7 +329,7 @@ vine==5.1.0
# amqp
# celery
# kombu
wagtail==6.1.2
wagtail==6.1.3
# via
# -r base.in
# wagtail-metadata
......@@ -296,7 +345,7 @@ wand==0.6.13
# via -r base.in
wcwidth==0.2.13
# via prompt-toolkit
weasyprint==62.2
weasyprint==62.3
# via -r base.in
webencodings==0.5.1
# via
......@@ -304,11 +353,17 @@ webencodings==0.5.1
# cssselect2
# html5lib
# tinycss2
websockets==11.0.3
# via gql
whitenoise==5.3.0
# via -r base.in
willow[heif]==1.8.0
# via
# wagtail
# willow
yarl==1.9.4
# via
# aiohttp
# gql
zopfli==0.2.3
# via fonttools
requirements/dev.txt
+ 8
− 8
View file @ 96209edf
......@@ -6,17 +6,17 @@
#
asgiref==3.8.1
# via django
coverage[toml]==7.5.3
coverage[toml]==7.6.0
# via pytest-cov
django==5.0.6
django==5.0.7
# via
# -r dev.in
# django-debug-toolbar
django-debug-toolbar==4.4.2
django-debug-toolbar==4.4.6
# via -r dev.in
factory-boy==3.3.0
# via pytest-factoryboy
faker==25.6.0
faker==26.0.0
# via factory-boy
fastdiff==0.3.0
# via snapshottest
......@@ -26,14 +26,14 @@ inflection==0.5.1
# via pytest-factoryboy
iniconfig==2.0.0
# via pytest
packaging==24.0
packaging==24.1
# via
# pytest
# pytest-factoryboy
# pytest-sugar
pluggy==1.5.0
# via pytest
pytest==8.2.2
pytest==8.3.2
# via
# -r dev.in
# pytest-cov
......@@ -64,7 +64,7 @@ six==1.16.0
# snapshottest
snapshottest==0.6.0
# via -r dev.in
sqlparse==0.5.0
sqlparse==0.5.1
# via
# django
# django-debug-toolbar
......@@ -72,7 +72,7 @@ termcolor==2.4.0
# via
# pytest-sugar
# snapshottest
typing-extensions==4.12.1
typing-extensions==4.12.2
# via pytest-factoryboy
wasmer==1.1.0
# via fastdiff
......
requirements/production.txt
+ 1
− 1
View file @ 96209edf
......@@ -6,5 +6,5 @@
#
gunicorn==22.0.0
# via -r production.in
packaging==24.0
packaging==24.1
# via gunicorn
shared/forms.py
+ 43
− 0
View file @ 96209edf
......@@ -9,6 +9,49 @@ class SubscribeForm(forms.Form):
return_page_id = forms.IntegerField()
class OctopusPeopleImportForm(WagtailAdminPageForm):
do_import = forms.BooleanField(
initial=False, required=False, label="Provést import osob z Chobotnice"
)
collection = forms.ModelChoiceField(
queryset=Collection.objects.all(), required=False, label="Kolekce obrázků"
)
dry_run = forms.BooleanField(
initial=True,
required=False,
label="Jenom na zkoušku",
)
group_shortcut = forms.CharField(
label="Zkratka skupiny osob",
required=False,
)
def clean(self):
cleaned_data = super().clean()
if not cleaned_data.get("do_import"):
return cleaned_data
if cleaned_data.get("do_import") and not self.instance.id:
self.add_error(
"do_import", "Import proveďte prosím až po vytvoření stránky"
)
if not cleaned_data.get("collection"):
self.add_error("collection", "Pro import je toto pole povinné")
if not cleaned_data.get("group_shortcut"):
self.add_error("group_shortcut", "Pro import je toto pole povinné")
return cleaned_data
def save(self, commit=True):
if self.cleaned_data.get("do_import"):
self.handle_import_from_group()
return super().save(commit=commit)
class JekyllImportForm(WagtailAdminPageForm):
do_import = forms.BooleanField(
initial=False, required=False, label="Provést import z Jekyllu"
......
shared/models/main.py
+ 0
− 2
View file @ 96209edf
......@@ -1841,8 +1841,6 @@ class MainPeoplePageMixin(
promote_panels = make_promote_panels()
settings_panels = []
### RELATIONS
# NOTE: Must be overridden
......
shared/people_import.py 0 → 100644
+ 137
− 0
View file @ 96209edf
from gql import gql, Client
from gql.transport.aiohttp import AIOHTTPTransport
from django.conf import settings
import os
class PeopleGroupImporter:
def __init__(
self,
people_parent_page_id,
people_parent_page_model,
person_page_model,
collection_id,
group_shortcut,
lock_file_name,
dry_run,
):
self.people_parent_page_id = people_parent_page_id
self.people_parent_page_model = people_parent_page_model
self.person_page_model = person_page_model
self.collection_id = collection_id
self.group_shortcut = group_shortcut
self.lock_file_name = lock_file_name
self.dry_run = dry_run
self.transport = AIOHTTPTransport(url=settings.OCTOPUS_API_URL)
self.client = Client(transport=self.transport, fetch_schema_from_transport=True)
def get_people_ids_from_group(self):
query = gql(
f"""
query {{
allGroups(
filters:
{{
shortcut: {{exact: "{self.group_shortcut}" }}
}}
) {{
edges {{
node {{
memberships {{
person {{
id
}}
}}
}}
}}
}}
}}
"""
)
result = self.client.execute(query)
user_ids = []
for node in result["allGroups"]["edges"]:
for membership in node["node"]["memberships"]:
user_ids.append(membership["person"]["id"])
return user_ids
def get_person_profile_from_id(self, id: str, kind: str):
query = gql(
f"""
query {{
allProfiles(
filters: {{
person: {{
id: "{id}"
}},
kind: {kind}
}}
) {{
edges {{
node {{
email
facebookUrl
flickrUrl
instagramUrl
kind
mastodonUrl
phone
photo
textLong
textShort
tiktokUrl
twitterUrl
url
webUrl
youtubeUrl
person {{
degreeAfterName
degreeBeforeName
displayName
profilePhoto
}}
}}
}}
}}
}}
"""
)
result = self.client.execute(query)
# Just return the first result, there should never be more than one in this case.
for node in result["allProfiles"]["edges"]:
return node["node"]
# If there are no results, return None.
return None
def perform_import(self):
people_ids = self.get_people_ids_from_group()
people_profiles = {}
for person_id in people_ids:
prirotizied_profiles = []
prirotizied_profiles.append(self.get_person_profile_from_id(person_id, "POLITICAL"))
prirotizied_profiles.append(self.get_person_profile_from_id(person_id, "PIRATE"))
for profile in prirotizied_profiles:
if profile is None:
continue
people_profiles[person_id] = profile
if person_id not in people_profiles:
people_profiles[person_id] = None
print(people_profiles)
os.remove(self.lock_file_name)
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

Pirátský GitLab - Home of the CopyLeft