From ffea67a280495ceca93729051fd6ae766f2dbcc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alexa=20Valentov=C3=A1?= <git@imaniti.org>
Date: Thu, 28 Nov 2024 12:28:43 +0100
Subject: [PATCH] attempt fixing Octopus import
---
district/blocks.py | 4 +-
district/forms.py | 5 +-
...uspersonpage_originating_group_and_more.py | 32 +++++
district/models.py | 22 ++--
.../templatetags/district_people_filters.py | 6 +-
..._octopuspersonoriginatinggroup_and_more.py | 27 ++++
shared/models/main.py | 14 +++
shared/people_import.py | 115 ++++++++++++++----
shared/static/styleguide2/pirati-ui.svg | 2 +-
uniweb/migrations/0114_auto_20241122_1238.py | 31 +++++
10 files changed, 211 insertions(+), 47 deletions(-)
create mode 100644 district/migrations/0304_remove_districtoctopuspersonpage_originating_group_and_more.py
create mode 100644 shared/migrations/0014_octopuspersonoriginatinggroup_and_more.py
create mode 100644 uniweb/migrations/0114_auto_20241122_1238.py
diff --git a/district/blocks.py b/district/blocks.py
index 2493fedf..6e92b929 100644
--- a/district/blocks.py
+++ b/district/blocks.py
@@ -150,7 +150,7 @@ class OctopusMixin(blocks.StructBlock):
context["person_list"] = (
DistrictOctopusPersonPage.objects.filter(
- originating_group=value["group_shortcut"]
+ originating_groups__name=value["group_shortcut"]
)
.order_by("person__order", "title")
.all()
@@ -208,7 +208,7 @@ class OctopusTeamBlock(OctopusMixin):
context = blocks.StructBlock.get_context(self, value, *args, **kwargs)
filter = Q(
- originating_team=value["team_shortcut"],
+ originating_teams__name=value["team_shortcut"],
)
if value["roles"]:
diff --git a/district/forms.py b/district/forms.py
index 7337a632..55bc2f17 100644
--- a/district/forms.py
+++ b/district/forms.py
@@ -1,13 +1,10 @@
import os
import tempfile
-
from shared.forms import ArticlesPageForm as SharedArticlesPageForm
from shared.forms import JekyllImportForm as SharedJekyllImportForm
-from .tasks import (
- import_jekyll_articles,
-)
+from .tasks import import_jekyll_articles
class JekyllImportForm(SharedJekyllImportForm):
diff --git a/district/migrations/0304_remove_districtoctopuspersonpage_originating_group_and_more.py b/district/migrations/0304_remove_districtoctopuspersonpage_originating_group_and_more.py
new file mode 100644
index 00000000..4696fa53
--- /dev/null
+++ b/district/migrations/0304_remove_districtoctopuspersonpage_originating_group_and_more.py
@@ -0,0 +1,32 @@
+# Generated by Django 5.0.7 on 2024-11-27 14:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('district', '0303_alter_districtcenterpage_content'),
+ ('shared', '0014_octopuspersonoriginatinggroup_and_more'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='districtoctopuspersonpage',
+ name='originating_group',
+ ),
+ migrations.RemoveField(
+ model_name='districtoctopuspersonpage',
+ name='originating_team',
+ ),
+ migrations.AddField(
+ model_name='districtoctopuspersonpage',
+ name='originating_groups',
+ field=models.ManyToManyField(help_text='Skupiny, ze kterých byla tato osba importována.', to='shared.octopuspersonoriginatinggroup', verbose_name='Skupiny'),
+ ),
+ migrations.AddField(
+ model_name='districtoctopuspersonpage',
+ name='originating_teams',
+ field=models.ManyToManyField(help_text='Týmy, ze kterých byla tato osba importována.', to='shared.octopuspersonoriginatingteam', verbose_name='Tým'),
+ ),
+ ]
diff --git a/district/models.py b/district/models.py
index 303745e5..b9908d0a 100644
--- a/district/models.py
+++ b/district/models.py
@@ -558,20 +558,16 @@ class DistrictOctopusPersonPage(
default=False,
)
- originating_group = models.CharField(
- verbose_name="Skupina",
- help_text="Skupina, ze které byla tato osba importována.",
- max_length=128,
- blank=True,
- null=True,
+ originating_groups = models.ManyToManyField(
+ "shared.OctopusPersonOriginatingGroup",
+ verbose_name="Skupiny",
+ help_text="Skupiny, ze kterých byla tato osba importována.",
)
- originating_team = models.CharField(
+ originating_teams = models.ManyToManyField(
+ "shared.OctopusPersonOriginatingTeam",
verbose_name="Tým",
- help_text="Tým, ze kterého byla tato osba importována.",
- max_length=128,
- blank=True,
- null=True,
+ help_text="Týmy, ze kterých byla tato osba importována.",
)
originating_role = models.CharField(
@@ -595,8 +591,8 @@ class DistrictOctopusPersonPage(
content_panels = Page.content_panels + [
FieldPanel("person"),
FieldPanel("is_automatically_created", read_only=True),
- FieldPanel("originating_group", read_only=True),
- FieldPanel("originating_team", read_only=True),
+ FieldPanel("originating_groups", read_only=True),
+ FieldPanel("originating_teams", read_only=True),
]
### RELATIONS
diff --git a/district/templatetags/district_people_filters.py b/district/templatetags/district_people_filters.py
index 42754599..11a8dad4 100644
--- a/district/templatetags/district_people_filters.py
+++ b/district/templatetags/district_people_filters.py
@@ -9,15 +9,15 @@ def get_block_octopus_person_list(block):
from district.models import DistrictOctopusPersonPage
filter = (
- models.Q(originating_group=block.value["group_shortcut"])
+ models.Q(originating_groups__name=block.value["group_shortcut"])
if "group_shortcut" in block.value
else (
models.Q(
- originating_team=block.value["team_shortcut"],
+ originating_teams__name=block.value["team_shortcut"],
originating_role__in=block.value["roles"].split(","),
)
if block.value["roles"]
- else models.Q(originating_team=block.value["team_shortcut"])
+ else models.Q(originating_teams__name=block.value["team_shortcut"])
)
)
diff --git a/shared/migrations/0014_octopuspersonoriginatinggroup_and_more.py b/shared/migrations/0014_octopuspersonoriginatinggroup_and_more.py
new file mode 100644
index 00000000..02c5c6e3
--- /dev/null
+++ b/shared/migrations/0014_octopuspersonoriginatinggroup_and_more.py
@@ -0,0 +1,27 @@
+# Generated by Django 5.0.7 on 2024-11-27 14:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('shared', '0013_alter_octopusperson_order'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='OctopusPersonOriginatingGroup',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=128)),
+ ],
+ ),
+ migrations.CreateModel(
+ name='OctopusPersonOriginatingTeam',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('name', models.CharField(max_length=128)),
+ ],
+ ),
+ ]
diff --git a/shared/models/main.py b/shared/models/main.py
index f2c2a027..43ef1817 100644
--- a/shared/models/main.py
+++ b/shared/models/main.py
@@ -2330,4 +2330,18 @@ class OctopusPerson(models.Model):
ordering = ["order"]
+class OctopusPersonOriginatingTeam(models.Model):
+ name = models.CharField(max_length=128, blank=False, null=False)
+
+ def __str__(self) -> str:
+ return self.name
+
+
+class OctopusPersonOriginatingGroup(models.Model):
+ name = models.CharField(max_length=128, blank=False, null=False)
+
+ def __str__(self) -> str:
+ return self.name
+
+
# --- END Django-only models ---
diff --git a/shared/people_import.py b/shared/people_import.py
index 35bb1ab5..d5454b0d 100644
--- a/shared/people_import.py
+++ b/shared/people_import.py
@@ -6,13 +6,16 @@ from io import BytesIO
import requests
from django.conf import settings
from django.core.files.images import ImageFile
-from django.db import models
from gql import Client, gql
from gql.transport.aiohttp import AIOHTTPTransport
from wagtail.images.models import Image
from wagtail.models.media import Collection
-from shared.models import OctopusPerson
+from shared.models import (
+ OctopusPerson,
+ OctopusPersonOriginatingGroup,
+ OctopusPersonOriginatingTeam,
+)
logger = logging.getLogger(__name__)
@@ -60,6 +63,17 @@ class ImporterMixin:
"display_name": profile["person"]["displayName"],
}
+ verifiably_inexsistent_person = OctopusPerson.objects.filter(
+ octopus_id=person_id
+ ).first()
+
+ print(verifiably_inexsistent_person)
+
+ # For some reason, probably due to integrity issues somewhere, the person
+ # can end up existing. FIXME
+ if person is None and verifiably_inexsistent_person is not None:
+ person = verifiably_inexsistent_person
+
if person is None:
if hasattr(self, "new_user_count"):
self.new_user_count += 1
@@ -332,6 +346,10 @@ class PeopleGroupImporter(ImporterMixin):
.first()
)
+ originating_group = OctopusPersonOriginatingGroup.objects.get_or_create(
+ name=self.group_shortcut
+ )[0]
+
if person_page is None:
if not isinstance(person_instance.display_name, str):
logger.warning(
@@ -344,7 +362,6 @@ class PeopleGroupImporter(ImporterMixin):
person_page = self.person_page_model(
person=person_instance,
is_automatically_created=True,
- originating_group=self.group_shortcut,
originating_display=self.group_display,
title=person_instance.display_name,
)
@@ -352,17 +369,24 @@ class PeopleGroupImporter(ImporterMixin):
self.people_parent_page.add_child(instance=person_page)
person_page.save_revision().publish()
+
+ person_page.originating_groups.add(originating_group)
else:
+ if not person_page.originating_groups.filter(
+ id=originating_group.id
+ ).exists():
+ person_page.originating_groups.add(originating_group)
+
person_page.originating_display = self.group_display
person_page.save_revision().publish()
# Delete old pages that correspond to profiles which aren't
# part of the group we are importing anymore.
- self.person_page_model.objects.filter(
- ~models.Q(person__in=people_instances),
- originating_group=self.group_shortcut,
- is_automatically_created=True,
- ).descendant_of(self.people_parent_page).delete()
+ # self.person_page_model.objects.filter(
+ # ~models.Q(person__in=people_instances),
+ # originating_groups__name=self.group_shortcut,
+ # is_automatically_created=True,
+ # ).descendant_of(self.people_parent_page).delete()
finally:
# No matter what happens, at least remove the lockfile.
@@ -520,13 +544,23 @@ class PeopleTeamImporter(ImporterMixin):
people_profiles = self.get_processed_people_profiles(people_ids)
people_instances = self.create_and_update_people_models(people_profiles)
+ print("People instances:", people_instances, "DONE")
+
for person_instance in people_instances:
+ print(f"Person instance:", person_instance, "DONE")
+
person_page = (
self.person_page_model.objects.filter(person=person_instance)
.descendant_of(self.people_parent_page)
.first()
)
+ originating_team = OctopusPersonOriginatingTeam.objects.get_or_create(
+ name=self.team_shortcut
+ )[0]
+
+ print("Person page:", person_page, "DONE")
+
if person_page is None:
if not isinstance(person_instance.display_name, str):
logger.warning(
@@ -539,20 +573,53 @@ class PeopleTeamImporter(ImporterMixin):
person_page = self.person_page_model(
person=person_instance,
is_automatically_created=True,
- originating_team=self.team_shortcut,
originating_display=self.team_display,
title=person_instance.display_name,
)
- if len(self.team_roles) != 0:
- person_page.originating_role = person_instance.position
+ last_child = self.people_parent_page.get_last_child()
+ if last_child is None:
+ # Handle empty parent case
+ new_path = (
+ self.people_parent_page.path + "0001"
+ ) # Example path increment
+ else:
+ new_path = last_child._inc_path()
- self.people_parent_page.add_child(instance=person_page)
+ person_page.path = new_path
+ self.people_parent_page.add_child(instance=person_page)
person_page.save_revision().publish()
+
+ person_page.originating_teams.add(originating_team)
+
+ if len(self.team_roles) != 0:
+ person_page.originating_role = person_instance.position
+
+ person_page.save()
else:
person_page.originating_display = self.team_display
+ currently_valid_teams = [
+ team["shortcut"]
+ for team in self.people_parent_page.get_syncable_octopus_teams()
+ ]
+
+ originating_teams = list(person_page.originating_teams.all())
+
+ for existing_originating_team in originating_teams:
+ if existing_originating_team.name not in currently_valid_teams:
+ person_page.originating_teams.remove(
+ existing_originating_team
+ )
+
+ if not person_page.originating_teams.filter(
+ id=originating_team.id
+ ).exists():
+ person_page.originating_teams.add(originating_team)
+
+ person_page.save()
+
if len(self.team_roles) != 0:
person_page.originating_role = person_instance.position
@@ -561,18 +628,18 @@ class PeopleTeamImporter(ImporterMixin):
# Delete old pages that correspond to profiles which aren't
# part of the group we are importing anymore.
- filter = models.Q(
- ~models.Q(person__in=people_instances),
- originating_team=self.team_shortcut,
- is_automatically_created=True,
- )
-
- if len(self.team_roles) != 0:
- filter = filter & models.Q(originating_role__in=self.team_roles)
-
- self.person_page_model.objects.filter(filter).descendant_of(
- self.people_parent_page
- ).delete()
+ # filter = models.Q(
+ # ~models.Q(person__in=people_instances),
+ # originating_team=self.team_shortcut,
+ # is_automatically_created=True,
+ # )
+ #
+ # if len(self.team_roles) != 0:
+ # filter = filter & models.Q(originating_role__in=self.team_roles)
+ #
+ # self.person_page_model.objects.filter(filter).descendant_of(
+ # self.people_parent_page
+ # ).delete()
finally:
# No matter what happens, at least remove the lockfile.
diff --git a/shared/static/styleguide2/pirati-ui.svg b/shared/static/styleguide2/pirati-ui.svg
index ffc20ecc..c6e3a71a 100644
--- a/shared/static/styleguide2/pirati-ui.svg
+++ b/shared/static/styleguide2/pirati-ui.svg
@@ -127,4 +127,4 @@
<glyph unicode="" glyph-name="price-tags" horiz-adv-x="1280" d="M1232 960h-384c-26.4 0-63.274-15.274-81.942-33.942l-476.116-476.116c-18.668-18.668-18.668-49.214 0-67.882l412.118-412.118c18.668-18.668 49.214-18.668 67.882 0l476.118 476.118c18.666 18.666 33.94 55.54 33.94 81.94v384c0 26.4-21.6 48-48 48zM992 576c-53.020 0-96 42.98-96 96s42.98 96 96 96 96-42.98 96-96-42.98-96-96-96zM128 416l544 544h-80c-26.4 0-63.274-15.274-81.942-33.942l-476.116-476.116c-18.668-18.668-18.668-49.214 0-67.882l412.118-412.118c18.668-18.668 49.214-18.668 67.882 0l30.058 30.058-416 416z" />
<glyph unicode="" glyph-name="twitter" horiz-adv-x="1001" d="M596.009 526.629l372.819 433.371h-88.346l-323.718-376.29-258.553 376.29h-298.21l390.983-569.018-390.983-454.457h88.351l341.855 397.375 273.051-397.375h298.21l-405.458 590.103zM475 385.969l-354.815 507.521h135.702l624.636-893.48h-135.702l-269.821 385.959z" />
<glyph unicode="" glyph-name="stats-dots" d="M128 64h896v-128h-1024v1024h128zM288 128c-53.020 0-96 42.98-96 96s42.98 96 96 96c2.828 0 5.622-0.148 8.388-0.386l103.192 171.986c-9.84 15.070-15.58 33.062-15.58 52.402 0 53.020 42.98 96 96 96s96-42.98 96-96c0-19.342-5.74-37.332-15.58-52.402l103.192-171.986c2.766 0.238 5.56 0.386 8.388 0.386 2.136 0 4.248-0.094 6.35-0.23l170.356 298.122c-10.536 15.408-16.706 34.036-16.706 54.11 0 53.020 42.98 96 96 96s96-42.98 96-96c0-53.020-42.98-96-96-96-2.14 0-4.248 0.094-6.35 0.232l-170.356-298.124c10.536-15.406 16.706-34.036 16.706-54.11 0-53.020-42.98-96-96-96s-96 42.98-96 96c0 19.34 5.74 37.332 15.578 52.402l-103.19 171.984c-2.766-0.238-5.56-0.386-8.388-0.386s-5.622 0.146-8.388 0.386l-103.192-171.986c9.84-15.068 15.58-33.060 15.58-52.4 0-53.020-42.98-96-96-96z" />
-</font></defs></svg>
\ No newline at end of file
+</font></defs></svg>
diff --git a/uniweb/migrations/0114_auto_20241122_1238.py b/uniweb/migrations/0114_auto_20241122_1238.py
new file mode 100644
index 00000000..ad98c190
--- /dev/null
+++ b/uniweb/migrations/0114_auto_20241122_1238.py
@@ -0,0 +1,31 @@
+# Generated by Django 5.0.7 on 2024-11-22 11:38
+
+from django.db import migrations
+
+
+def fix_po_people_page(apps, schema_editor):
+ # DistrictPeoplePage = apps.get_model("district", "DistrictPeoplePage")
+ # NOTE: Can't have this for the save_revision() method
+
+ from uniweb.models import UniwebPeoplePage
+
+ page = UniwebPeoplePage.objects.filter(id=30017).first()
+
+ if page is None:
+ return
+
+ page.content = page.content.get_prep_value()
+
+ page.save()
+ page.save_revision().publish()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('uniweb', '0113_alter_uniwebhomepage_dark_logo_and_more'),
+ ]
+
+ operations = [
+ migrations.RunPython(fix_po_people_page)
+ ]
--
GitLab