Implement periodic import of shifts

requirements.txt

@@ -3,22 +3,29 @@ asgiref==3.5.0
 async-timeout==4.0.2
 billiard==3.6.4.0
 celery==5.2.6
+certifi==2021.10.8
+charset-normalizer==2.0.12
 click==8.1.2
 click-didyoumean==0.3.0
 click-plugins==1.1.1
 click-repl==0.2.0
 Deprecated==1.2.13
 Django==4.0.4
+icalendar==4.0.9
+idna==3.3
 kombu==5.2.4
 librabbitmq==2.0.0
 packaging==21.3
 prompt-toolkit==3.0.29
 psycopg2-binary==2.9.3
 pyparsing==3.0.8
+python-dateutil==2.8.2
 pytz==2022.1
 redis==4.2.2
+requests==2.27.1
 six==1.16.0
 sqlparse==0.4.2
+urllib3==1.26.9
 vine==5.0.0
 wcwidth==0.2.5
 wrapt==1.14.0

shiftregister/app/admin.py

@@ -7,7 +7,7 @@ admin.site.register(Room)
 
 @admin.register(Shift)
 class ShiftAdmin(admin.ModelAdmin):
-    list_display = ("room_name", "start_at", "free_slots")
+    list_display = ("room_name", "start_at", "free_slots", "deleted")
 
     def room_name(self, object):
         return object.room.name

shiftregister/app/migrations/0006_shift_deleted.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.0.4 on 2022-04-23 00:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("app", "0005_alter_helper_phone"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="shift",
+            name="deleted",
+            field=models.BooleanField(default=False),
+        ),
+    ]

shiftregister/app/models.py

@@ -18,7 +18,9 @@ class Shift(models.Model):
     room = models.ForeignKey(Room, on_delete=models.RESTRICT)
     start_at = models.DateTimeField()
     duration = models.DurationField()
+    deleted = models.BooleanField(default=False)
     # todo: add helper amount override field
+
     def __str__(self):
         return f"{self.room.name}: {self.start_at}"
 

shiftregister/app/views.py

@@ -31,7 +31,11 @@ def index(request):
 
     free_shifts = (
         Shift.objects.annotate(reg_count=Count("shiftregistration"))
-        .filter(start_at__gt=timezone.now(), room__required_helpers__gt=F("reg_count"))
+        .filter(
+            start_at__gt=timezone.now(),
+            room__required_helpers__gt=F("reg_count"),
+            deleted=False,
+        )
         .order_by("start_at")
     )
 
@@ -39,7 +43,9 @@ def index(request):
         free_shifts = (
             Shift.objects.annotate(reg_count=Count("shiftregistration"))
             .filter(
-                start_at__gt=timezone.now(), room__required_helpers__gt=F("reg_count")
+                start_at__gt=timezone.now(),
+                room__required_helpers__gt=F("reg_count"),
+                deleted=False,
             )
             .filter(~Q(shiftregistration__helper=request.helper))
             .order_by("start_at")

shiftregister/importer/admin.py

@@ -0,0 +1,7 @@
+from django.contrib import admin
+from .models import Calendar
+
+
+@admin.register(Calendar)
+class CalendarAdmin(admin.ModelAdmin):
+    list_display = ("url", "has_errors")

shiftregister/importer/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ImporterConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "shiftregister.importer"

shiftregister/importer/importer.py

@@ -0,0 +1,83 @@
+from datetime import timezone
+from django.conf import settings
+from django.db import transaction
+from icalendar import Calendar
+from .models import Event, Room, Shift
+import requests
+
+
+def import_calendar(calendar):
+    try:
+        r = requests.get(calendar.url)
+        r.raise_for_status()
+    except:
+        if settings.DEBUG:
+            raise
+        return False
+
+    if not r.headers["content-type"].startswith("text/calendar"):
+        return False
+
+    try:
+        cal = Calendar.from_ical(r.text)
+
+        rooms = {}
+        events = {}
+        for event in cal.walk("vevent"):
+            uid = event.decoded("uid").decode()
+            room = (
+                event.decoded("location", None) or event.decoded("summary")
+            ).decode()
+            start = event.decoded("dtstart").astimezone(timezone.utc)
+            end = event.decoded("dtend").astimezone(timezone.utc)
+
+            if not uid or not room:
+                return False
+
+            rooms[room] = None
+            events[uid] = (
+                room,
+                {
+                    "start_at": start,
+                    "duration": end - start,
+                    "uuid": uid,
+                    "calendar": calendar,
+                },
+            )
+
+        with transaction.atomic():
+            for r in Room.objects.filter(name__in=rooms):
+                rooms[r.name] = r
+            for room, r in rooms.items():
+                if r == None:
+                    rooms[room] = Room(
+                        name=room, required_helpers=0
+                    )  # required_helpers=0 ensures a shift in a new room is not displayed until the correct number of required helpers is set
+                    rooms[room].save()
+
+            for e in Event.objects.filter(calendar=calendar, uuid__in=events):
+                uuid = str(e.uuid)
+                room, event = events[uuid]
+
+                e.room = rooms[room]
+                e.start_at = event["start_at"]
+                e.duration = event["duration"]
+                e.save()
+
+                events[uuid] = (room, e)
+            for event in events:
+                room, e = events[event]
+                if not isinstance(e, Event):
+                    Event(room=rooms[room], **e).save()
+
+            for event in Event.objects.filter(calendar=calendar).exclude(
+                uuid__in=events
+            ):
+                event.deleted = True
+                event.save()
+    except:
+        if settings.DEBUG:
+            raise
+        return False
+
+    return True

shiftregister/importer/migrations/0001_initial.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.0.4 on 2022-04-23 00:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("app", "0006_shift_deleted"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Calendar",
+            fields=[
+                ("url", models.URLField(primary_key=True, serialize=False)),
+                ("has_errors", models.BooleanField(default=False, editable=False)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                (
+                    "shift_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        to="app.shift",
+                    ),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(editable=False, primary_key=True, serialize=False),
+                ),
+                (
+                    "calendar",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="importer.calendar",
+                    ),
+                ),
+            ],
+            bases=("app.shift",),
+        ),
+    ]

shiftregister/importer/models.py

@@ -0,0 +1,12 @@
+from django.db import models
+from shiftregister.app.models import *
+
+
+class Calendar(models.Model):
+    url = models.URLField(primary_key=True)
+    has_errors = models.BooleanField(default=False, editable=False)
+
+
+class Event(Shift):
+    uuid = models.UUIDField(primary_key=True, editable=False)
+    calendar = models.ForeignKey(Calendar, on_delete=models.CASCADE)

shiftregister/importer/tasks.py

@@ -0,0 +1,10 @@
+from celery import shared_task
+from .importer import import_calendar
+from .models import Calendar
+
+
+@shared_task
+def import_shifts():
+    for calendar in Calendar.objects.all():
+        calendar.has_errors = not import_calendar(calendar)
+        calendar.save()

shiftregister/importer/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

shiftregister/importer/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

shiftregister/settings.py

@@ -35,6 +35,7 @@ ALLOWED_HOSTS = list(filter(lambda s: s != "", getenv("ALLOWED_HOSTS", "").split
 
 INSTALLED_APPS = [
     "shiftregister.app.apps.AppConfig",
+    "shiftregister.importer.apps.ImporterConfig",
     "shiftregister.team.apps.TeamConfig",
     "django.contrib.admin",
     "django.contrib.auth",
@@ -141,3 +142,10 @@ DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 CELERY_BROKER_URL = getenv("CELERY_BROKER_URL", "amqp://guest:guest@localhost:5672//")
 
 CELERY_RESULT_BACKEND = getenv("CELERY_RESULT_BACKEND", "redis://")
+
+CELERY_BEAT_SCHEDULE = {
+    "import-shifts-every-60-seconds": {
+        "task": "shiftregister.importer.tasks.import_shifts",
+        "schedule": float(getenv("SHIFT_IMPORT_INTERVAL", 60.0)),  # seconds
+    },
+}