CHANGELOG.rst

@@ -6,6 +6,47 @@ All notable changes to this project will be documented in this file.
 The format is based on `Keep a Changelog`_,
 and this project adheres to `Semantic Versioning`_.
 
+`2.0`_ - 2021-10-30
+-------------------
+
+Added
+~~~~~
+
+* Add script for moving all Chronos dates to the current (school) year (only for testing purposes).
+* Add demo data as Untis dump (also only for testing purposes).
+
+Changed
+~~~~~~~
+
+* Management commands can run the import in the foreground or in the background.
+* The management commands were merged to one with an argument to call the subcommands.
+
+`2.0rc3`_ - 2021-09-30
+----------------------
+
+Fixed
+~~~~~
+
+* Skip extra lessons without a subject.
+* Fix problems with lesson parts without a room and lesson parts with two courses and one teacher in a room.
+
+`2.0rc2`_ - 2021-07-30
+----------------------
+
+Fixed
+~~~~~
+
+* Get validity ranges by Untis ID and the corresponding school term.
+
+`2.0rc1`_ - 2021-06-23
+----------------------
+
+Fixed
+~~~~~
+
+* Preference section verbose names were displayed in server language and not
+  user language (fixed by using gettext_lazy).
+
 `2.0b0`_ - 2021-05-21
 ---------------------
 
@@ -81,3 +122,7 @@ Fixed
 .. _1.0a1: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/1.0a1
 .. _2.0a2: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0a2
 .. _2.0b0: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0b0
+.. _2.0rc1: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0rc1
+.. _2.0rc2: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0rc2
+.. _2.0rc3: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0rc3
+.. _2.0: https://edugit.org/Teckids/AlekSIS/AlekSIS-App-Untis/-/tags/2.0

aleksis/apps/untis/commands.py

+from typing import Optional
+
+from django.db.models import QuerySet
+from django.utils.functional import classproperty
+
+from aleksis.apps.untis.util.mysql.importers.terms import (
+    get_future_terms_for_date,
+    get_terms_for_date,
+)
+
+from .util.mysql.main import untis_import_mysql as _untis_import_mysql
+
+
+class ImportCommand:
+    """A generic UNTIS import command."""
+
+    name = None
+
+    @classproperty
+    def task_name(cls) -> str:  # noqa
+        """Get the name for the related Celery task."""
+        return f"untis_import_mysql_{cls.name}"
+
+    @classmethod
+    def get_terms(cls) -> Optional[QuerySet]:
+        """Return which terms should be imported."""
+        return None
+
+    @classmethod
+    def run(cls, background: bool = False):
+        """Run the import command (foreground/background)."""
+        if background:
+            from .tasks import TASKS
+
+            task = TASKS[cls]
+            task.delay()
+        else:
+            _untis_import_mysql(cls.get_terms())
+
+
+class CurrentImportCommand(ImportCommand):
+    """Import data of current term from UNTIS."""
+
+    name = "current"
+
+    @classmethod
+    def get_terms(cls) -> Optional[QuerySet]:
+        return get_terms_for_date()
+
+
+class FutureImportCommand(ImportCommand):
+    """Import data of future terms from UNTIS."""
+
+    name = "future"
+
+    @classmethod
+    def get_terms(cls) -> Optional[QuerySet]:
+        return get_future_terms_for_date()
+
+
+class AllImportCommand(ImportCommand):
+    name = "all"
+
+
+class CurrentNextImportCommand(ImportCommand):
+    """Import data of the current and next term from UNTIS."""
+
+    name = "current_next"
+
+    @classmethod
+    def get_terms(cls) -> Optional[QuerySet]:
+        terms = get_terms_for_date()
+        future_terms = get_future_terms_for_date()
+        if future_terms.exists():
+            terms = terms.union(future_terms[0:1])
+        return terms
+
+
+class CurrentFutureImportCommand(ImportCommand):
+    """Import data of the current and future terms from UNTIS."""
+
+    name = "current_future"
+
+    @classmethod
+    def get_terms(cls) -> Optional[QuerySet]:
+        terms = get_terms_for_date()
+        future_terms = get_future_terms_for_date()
+        terms = terms.union(future_terms)
+        return terms
+
+
+COMMANDS_BY_NAME = {c.name: c for c in ImportCommand.__subclasses__()}

aleksis/apps/untis/locale/ar/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

aleksis/apps/untis/locale/de_DE/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: 2020-12-05 20:52+0000\n"
 "Last-Translator: Jonathan Weth <teckids@jonathanweth.de>\n"
 "Language-Team: German <https://translate.edugit.org/projects/aleksis/aleksis-app-untis/de/>\n"

aleksis/apps/untis/locale/fr/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

aleksis/apps/untis/locale/la/LC_MESSAGES/django.po

@@ -7,7 +7,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: 2020-08-25 17:42+0000\n"
 "Last-Translator: Jonathan Weth <teckids@jonathanweth.de>\n"
 "Language-Team: Latin <https://translate.edugit.org/projects/aleksis/aleksis-app-untis/la/>\n"

aleksis/apps/untis/locale/nb_NO/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

aleksis/apps/untis/locale/tr_TR/LC_MESSAGES/django.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: PACKAGE VERSION\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-05-20 21:06+0200\n"
+"POT-Creation-Date: 2021-06-08 16:49+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"

aleksis/apps/untis/management/commands/move_dates_for_testing.py

+from datetime import date, timedelta
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from calendarweek import CalendarWeek
+from tqdm import tqdm
+
+from aleksis.apps.chronos.models import (
+    Absence,
+    Event,
+    Exam,
+    ExtraLesson,
+    Holiday,
+    LessonSubstitution,
+    ValidityRange,
+)
+from aleksis.core.models import SchoolTerm
+
+
+class Command(BaseCommand):
+    help = "Move all dates to current school year"  # noqa
+
+    def add_arguments(self, parser):
+        parser.add_argument("--dry", action="store_true")
+
+    def translate_date_start_end(self, name: str, qs):
+        self.stdout.write(name.upper())
+        for instance in tqdm(qs):
+            date_start = instance.date_start
+            date_end = instance.date_end
+            date_start_new = date_start + self.time_delta
+            date_end_new = date_end + self.time_delta
+            self.stdout.write(f"{date_start} → {date_start_new}; {date_end} → {date_end_new}")
+            instance.date_start = date_start_new
+            instance.date_end = date_end_new
+            if not self.dry:
+                instance.save()
+
+    def translate_date(self, name: str, qs):
+        self.stdout.write(name.upper())
+        for instance in tqdm(qs):
+            date_old = instance.date
+            date_new = date_old + self.time_delta
+            self.stdout.write(f"{date_old} → {date_new}")
+            instance.date = date_new
+            if not self.dry:
+                instance.save()
+
+    def translate_week_year(self, name: str, qs):
+        self.stdout.write(name.upper())
+        for instance in tqdm(qs):
+            date_old = instance.date
+            date_new = instance.date + self.time_delta
+            week_new = CalendarWeek.from_date(date_new)
+            self.stdout.write(f"{date_old}, {instance.week} → {date_new}, {week_new.week}")
+            instance.year = date_new.year
+            instance.week = week_new.week
+            if not self.dry:
+                instance.save()
+
+    def handle(self, *args, **options):
+        self.dry = options["dry"]
+
+        school_terms = SchoolTerm.objects.order_by("-date_end")
+
+        if not school_terms.exists():
+            raise RuntimeError("No validity range available.")
+
+        school_term = school_terms.first()
+        self.stdout.write(f"Used school term: {school_term}")
+
+        date_start = school_term.date_start
+        date_end = school_term.date_end
+        current_date = timezone.now().date()
+        current_year = current_date.year
+        if date_start.month <= current_date.month and date_start.day < current_date.day:
+            current_year -= 1
+        date_end = date(year=current_year, day=date_start.day, month=date_start.month)
+        self.stdout.write(f"{current_year}, {date_end}")
+        days = (date_end - date_start).days
+        self.weeks = round(days / 7)
+        self.time_delta = timedelta(days=self.weeks * 7)
+        self.stdout.write(f"{days}, {self.weeks}")
+
+        self.translate_date_start_end("SCHOOL TERM", [school_term])
+
+        self.translate_date_start_end(
+            "VALIDITY RANGES", ValidityRange.objects.filter(school_term=school_term)
+        )
+
+        if not self.weeks:
+            raise RuntimeError("You need at least one validity range to move data.")
+
+        self.translate_week_year(
+            "SUBSTITUTIONS",
+            LessonSubstitution.objects.filter(
+                lesson_period__lesson__validity__school_term=school_term
+            ),
+        )
+        self.translate_week_year(
+            "EXTRA LESSONS", ExtraLesson.objects.filter(school_term=school_term)
+        )
+        self.translate_date_start_end("ABSENCES", Absence.objects.filter(school_term=school_term))
+        self.translate_date("EXAMS", Exam.objects.filter(school_term=school_term))
+        self.translate_date_start_end(
+            "HOLIDAYS", Holiday.objects.within_dates(date_start, date_end)
+        )
+        self.translate_date_start_end("EVENTS", Event.objects.filter(school_term=school_term))

aleksis/apps/untis/management/commands/untis_import_mysql.py

 from django.core.management.base import BaseCommand
 
-from ...tasks import untis_import_mysql_current_term
+from ...commands import COMMANDS_BY_NAME
 
 
 class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "command", nargs="?", default="all", type=str, choices=list(COMMANDS_BY_NAME.keys())
+        )
+        parser.add_argument(
+            "--background", action="store_true", help="Run import job in background using Celery",
+        )
+
     def handle(self, *args, **options):
-        untis_import_mysql_current_term.delay()
+        command = COMMANDS_BY_NAME[options["command"]]
+        background = options["background"]
+        command.run(background=background)

aleksis/apps/untis/management/commands/untis_import_mysql_all.py

-from django.core.management.base import BaseCommand
-
-from ...tasks import untis_import_mysql_all_terms
-
-
-class Command(BaseCommand):
-    def handle(self, *args, **options):
-        untis_import_mysql_all_terms.delay()

aleksis/apps/untis/management/commands/untis_import_mysql_current_future.py

-from django.core.management.base import BaseCommand
-
-from ...tasks import untis_import_mysql_current_future_terms
-
-
-class Command(BaseCommand):
-    def handle(self, *args, **options):
-        untis_import_mysql_current_future_terms.delay()

aleksis/apps/untis/management/commands/untis_import_mysql_current_next.py

-from django.core.management.base import BaseCommand
-
-from ...tasks import untis_import_mysql_current_next_term
-
-
-class Command(BaseCommand):
-    def handle(self, *args, **options):
-        untis_import_mysql_current_next_term.delay()

aleksis/apps/untis/management/commands/untis_import_mysql_future.py

-from django.core.management.base import BaseCommand
-
-from ...tasks import untis_import_mysql_future_terms
-
-
-class Command(BaseCommand):
-    def handle(self, *args, **options):
-        untis_import_mysql_future_terms.delay()

aleksis/apps/untis/preferences.py

-from django.utils.translation import gettext as _
+from django.utils.translation import gettext_lazy as _
 
 from dynamic_preferences.preferences import Section
 from dynamic_preferences.types import BooleanPreference

aleksis/apps/untis/tasks.py

-from aleksis.apps.untis.util.mysql.importers.terms import (
-    get_future_terms_for_date,
-    get_terms_for_date,
-)
 from aleksis.core.celery import app
 
-from .util.mysql.main import untis_import_mysql as _untis_import_mysql
+from .commands import ImportCommand
 
+TASKS = {}
+for import_command in ImportCommand.__subclasses__():
 
-@app.task
-def untis_import_mysql_current_term():
-    """Celery task for import of UNTIS data from MySQL (current term)."""
-    terms = get_terms_for_date()
-    _untis_import_mysql(terms)
+    @app.task(name=import_command.task_name)
+    def _task():
+        import_command.run()
 
-
-@app.task
-def untis_import_mysql_future_terms():
-    """Celery task for import of UNTIS data from MySQL (all future terms)."""
-    terms = get_future_terms_for_date()
-    _untis_import_mysql(terms)
-
-
-@app.task
-def untis_import_mysql_all_terms():
-    """Celery task for import of UNTIS data from MySQL (all terms in DB)."""
-    _untis_import_mysql()
-
-
-@app.task
-def untis_import_mysql_current_next_term():
-    """Celery task for import of UNTIS data from MySQL (current and next term)."""
-    terms = get_terms_for_date()
-    future_terms = get_future_terms_for_date()
-    if future_terms.exists():
-        terms = terms.union(future_terms[0:1])
-    _untis_import_mysql(terms)
-
-
-@app.task
-def untis_import_mysql_current_future_terms():
-    """Celery task for import of UNTIS data from MySQL (current and future terms)."""
-    terms = get_terms_for_date()
-    future_terms = get_future_terms_for_date()
-    terms = terms.union(future_terms)
-    _untis_import_mysql(terms)
+    TASKS[import_command] = _task

aleksis/apps/untis/util/mysql/importers/lessons.py

@@ -63,12 +63,16 @@ def import_lessons(
         for el in raw_time_data_2:
             weekday = int(el[1]) - 1
             hour = int(el[2])
-            room_ids = untis_split_third(el[3], conv=int)
+            room_ids = list(filter(lambda r: r != "", el[3].split(";")))
 
             # Get rooms
             rooms = []
             for room_id in room_ids:
-                r = rooms_ref[room_id]
+                if room_id and room_id != "0":
+                    room_id = int(room_id)
+                    r = rooms_ref[room_id]
+                else:
+                    r = None
                 rooms.append(r)
 
             # Get time period
@@ -81,6 +85,8 @@ def import_lessons(
         for el in raw_lesson_data:
             raw_lesson_data_2.append(el.split("~"))
 
+        use_room_idx = 0
+        current_teacher_id = None
         # All part lessons (courses)
         for i, el in enumerate(raw_lesson_data_2):
             logger.info("  Lesson part {}".format(i))
@@ -105,6 +111,9 @@ def import_lessons(
                 logger.warning(_("    Skip because missing subject"))
                 continue
 
+            if current_teacher_id and current_teacher_id != teacher_id:
+                use_room_idx += 1
+
             # Get classes
             course_classes = []
             for class_id in class_ids:
@@ -235,8 +244,8 @@ def import_lessons(
 
                 # Get room if provided
                 rooms = rooms_per_periods[j]
-                if i < len(rooms):
-                    room = rooms[i]
+                if use_room_idx < len(rooms):
+                    room = rooms[use_room_idx]
                 else:
                     room = None
 
@@ -259,6 +268,8 @@ def import_lessons(
                     )
                     logger.info("      New time period added")
 
+            current_teacher_id = teacher_id
+
     for lesson in chronos_models.Lesson.objects.filter(validity=validity_range):
         if lesson.lesson_id_untis and lesson.lesson_id_untis not in existing_lessons:
             logger.info("Lesson {} deleted".format(lesson.id))

aleksis/apps/untis/util/mysql/importers/substitutions.py

@@ -180,6 +180,10 @@ def import_substitutions(
                 if not teacher_new and not teacher_old:
                     teachers = []
 
+                if not subject:
+                    logger.warning("  Skip because missing subject")
+                    continue
+
                 (extra_lesson, created,) = (
                     chronos_models.ExtraLesson.objects.select_related(None)
                     .prefetch_related(None)

aleksis/apps/untis/util/mysql/importers/terms.py

@@ -104,7 +104,9 @@ def import_terms(qs: Optional[QuerySet] = None,) -> Dict[int, chronos_models.Val
         school_term.save()
 
         try:
-            validity_range = chronos_models.ValidityRange.objects.get(import_ref_untis=term_id)
+            validity_range = chronos_models.ValidityRange.objects.get(
+                import_ref_untis=term_id, school_term=school_term
+            )
             logger.info("  Validity range found by import reference.")
         except chronos_models.ValidityRange.DoesNotExist:
             try:

fixtures/README.rst

+Example data for AlekSIS-App-Untis
+==================================
+
+In this directory, you can find MySQL dumps from Untis Multiuser databases
+with some anonymised demo data. You can restore them in your local MySQL
+database to test and develop the import procedure. Please pay attention that
+the data are partially stripped – all license data, for example, are just
+faked.
+
+If you just want to work with timetable data, you may use the demo data 
+already extracted to Django models provided in AlekSIS-App-Chronos.
+
+It could be pretty annoying to work with demo data with dates that are
+only in the past. This makes it nearly impossible to test views and
+functions that should show something for the current date, for example.
+The management command ``move_dates_for_testing`` move all Chronos and school term
+dates of the latest school term in the database as it would currently
+take place:
+
+::
+
+    aleksis-admin move_dates_for_testing
+
+If you want to test the command without directly changing your data,
+you could also do a dry run:
+
+::
+
+    aleksis-admin move_dates_for_testing --dry