Merge branch '52-importing-lesson-fails-with-in-subject-name' into 'master'

Resolve "Importing lesson fails with , in subject name"

Closes #52

See merge request AlekSIS/official/AlekSIS-App-Untis!142

CHANGELOG.rst

@@ -19,6 +19,7 @@ Fixed
 
 * Matching for groups while importing lessons was broken in some cases.
 * Import commands `current_next` and `current_future` imported all terms.
+* Untis objects with special characters like commas in their names broke the import.
 
 `2.2`_ - 2022-04-10
 -------------------

aleksis/apps/untis/tests/util/test_mysql_util.py

+from aleksis.apps.untis.util.mysql.util import (
+    untis_split_first,
+    untis_split_second,
+    untis_split_third,
+)
+
+test_lesson_element = """49~0~302~7r;~0;~0~0~175608~~~~~"Freiraum: Naturwissenschaften, Fokus Physik"~"Nawi"~~n~~;11;12;13;14;15;16;17;,49~0~302~7r;~0;~0~0~175608~~~~~"Freiraum: Naturwissenschaften, Fokus Physik"~"Nawi"~~n~~;11;12;13;14;15;16;17;"""
+
+test_lesson_element_partial = """49~0~302~7r;~0;~0~0~175608~~~~~"Freiraum: Naturwissenschaften, Fokus Physik"~"Nawi"~~n~~;11;12;13;14;15;16;17;"""
+test_lesson_element_partial_partial = ";11;12;13;14;15;16;17;"
+
+
+def test_untis_split_first():
+    assert untis_split_first("") == []
+
+    assert untis_split_first("a,b,c") == ["a", "b", "c"]
+
+    assert untis_split_first("1,2,3") == ["1", "2", "3"]
+    assert untis_split_first("1,2,3", int) == [1, 2, 3]
+
+    assert len(untis_split_first(test_lesson_element)) == 2
+    assert untis_split_first(test_lesson_element) == [
+        "~~n~~;11;12;13;14;15;16;17;",
+        "~~n~~;11;12;13;14;15;16;17;",
+    ]
+
+
+def test_untis_split_second():
+
+    assert untis_split_second("") == []
+
+    assert untis_split_second("a~b~c") == ["a", "b", "c"]
+
+    assert untis_split_second("1~2~3") == ["1", "2", "3"]
+    assert untis_split_second("1~2~3", int) == [1, 2, 3]
+
+    assert untis_split_second("1~~3", remove_empty=False) == ["1", None, "3"]
+    assert untis_split_second("1~~3", int, remove_empty=False) == [1, None, 3]
+    assert untis_split_second("1~~3", int) == [1, 3]
+
+    assert untis_split_second(""""asdf"~"asdf"~"asdf""") == ["asdf", "asdf", "asdf"]
+
+    assert len(untis_split_second(test_lesson_element_partial, remove_empty=False)) == 18
+    assert untis_split_second(test_lesson_element_partial, remove_empty=False) == [
+        "49",
+        "0",
+        "302",
+        "7r;",
+        "0;",
+        "0",
+        "0",
+        "175608",
+        None,
+        None,
+        None,
+        None,
+        "Freiraum: Naturwissenschaften, Fokus Physik",
+        "Nawi",
+        None,
+        "n",
+        None,
+        ";11;12;13;14;15;16;17;",
+    ]
+
+
+def test_untis_split_third():
+    assert untis_split_third("") == []
+
+    assert untis_split_third("a;b;c") == ["a", "b", "c"]
+
+    assert untis_split_third("1;2;3") == ["1", "2", "3"]
+    assert untis_split_third("1;2;3", int) == [1, 2, 3]
+
+    assert untis_split_third("1;;3", remove_empty=False) == ["1", None, "3"]
+    assert untis_split_third("1;;3", int, remove_empty=False) == [1, None, 3]
+    assert untis_split_third("1;;3", int) == [1, 3]
+
+    assert untis_split_third(""""asdf";"asdf";"asdf""") == ["asdf", "asdf", "asdf"]
+
+    assert len(untis_split_third(test_lesson_element_partial_partial)) == 7
+    assert untis_split_third(test_lesson_element_partial_partial) == [
+        "11",
+        "12",
+        "13",
+        "14",
+        "15",
+        "16",
+        "17",
+    ]

aleksis/apps/untis/util/mysql/importers/common_data.py

@@ -18,6 +18,7 @@ from ..util import (
     run_default_filter,
     untis_colour_to_hex,
     untis_split_first,
+    untis_split_second,
 )
 
 logger = logging.getLogger(__name__)
@@ -380,7 +381,7 @@ def import_supervision_areas(
         supervisions_ref = {}
         for raw_supervision in raw_supervisions:
             # Split more and get teacher id
-            raw_supervision_2 = raw_supervision.split("~")
+            raw_supervision_2 = untis_split_second(raw_supervision, remove_empty=False)
             teacher_id = int(raw_supervision_2[1])
             term_id = int(raw_supervision_2[0])
 

aleksis/apps/untis/util/mysql/importers/events.py

@@ -15,6 +15,7 @@ from ..util import (
     move_weekday_to_range,
     run_default_filter,
     untis_date_to_date,
+    untis_split_second,
 )
 
 logger = logging.getLogger(__name__)
@@ -78,7 +79,7 @@ def import_events(
         # Get groups, teachers and rooms
         raw_events = connect_untis_fields(event, "eventelement", 10)
         for raw_event in raw_events:
-            el = raw_event.split("~")
+            el = untis_split_second(raw_event, remove_empty=False)
 
             # Group
             if el[0] != "0" and el[0] != "":

aleksis/apps/untis/util/mysql/importers/lessons.py

@@ -19,6 +19,8 @@ from ..util import (
     compare_m2m,
     connect_untis_fields,
     run_default_filter,
+    untis_split_first,
+    untis_split_second,
     untis_split_third,
 )
 
@@ -51,12 +53,12 @@ def import_lessons(
 
         # Split data (,)
         raw_lesson_data = connect_untis_fields(lesson, "lessonelement", 10)
-        raw_time_data = lesson.lesson_tt.split(",")
+        raw_time_data = untis_split_first(lesson.lesson_tt, remove_empty=False)
 
         raw_time_data_2 = []
         for el in raw_time_data:
             # Split data (~)
-            raw_time_data_2.append(el.split("~"))
+            raw_time_data_2.append(untis_split_second(el, remove_empty=False))
 
         # Get time periods and rooms
         time_periods = []
@@ -64,16 +66,12 @@ def import_lessons(
         for el in raw_time_data_2:
             weekday = int(el[1]) - 1
             hour = int(el[2])
-            room_ids = list(filter(lambda r: r != "", el[3].split(";")))
+            room_ids = untis_split_third(el[3], conv=int)
 
             # Get rooms
             rooms = []
             for room_id in room_ids:
-                if room_id and room_id != "0":
-                    room_id = int(room_id)
-                    r = rooms_ref[room_id]
-                else:
-                    r = None
+                r = rooms_ref[room_id]
                 rooms.append(r)
 
             # Get time period
@@ -84,7 +82,7 @@ def import_lessons(
         # Split data more (~)
         raw_lesson_data_2 = []
         for el in raw_lesson_data:
-            raw_lesson_data_2.append(el.split("~"))
+            raw_lesson_data_2.append(untis_split_second(el, remove_empty=False))
 
         use_room_idx = 0
         current_teacher_id = None

aleksis/apps/untis/util/mysql/util.py

+import csv
 import logging
+import re
 from datetime import date, datetime
-from typing import Any, Callable, Optional, Sequence, Union
+from typing import Any, Callable, Optional, Sequence, Type, Union
 
 from django.db.models import Model, QuerySet
 
@@ -53,7 +55,9 @@ def run_default_filter(
     return qs
 
 
-def clean_array(seq: Sequence, conv: Callable[[Any], Any] = lambda el: el) -> Sequence:
+def clean_array(
+    seq: Sequence, conv: Callable[[Any], Any] = lambda el: el, remove_empty: bool = True
+) -> Sequence:
     """Clean array.
 
     Convert a sequence using a converter function, stripping all
@@ -65,20 +69,69 @@ def clean_array(seq: Sequence, conv: Callable[[Any], Any] = lambda el: el) -> Se
     >>> clean_array(["8", "", "12", "0"], int)
     [8, 12]
     """
-    filtered = filter(lambda el: bool(el), map(lambda el: conv(el) if el else None, seq))
+    filtered = filter(
+        lambda el: bool(el) or not remove_empty, map(lambda el: conv(el) if el else None, seq)
+    )
     return type(seq)(filtered)
 
 
-def untis_split_first(s: str, conv: Callable[[Any], Any] = lambda el: el) -> Sequence:
-    return clean_array(s.split(","), conv=conv)
+class UntisDialect(csv.Dialect):
+    """Custom CSV dialect for Untis."""
+
+    quoting = csv.QUOTE_MINIMAL
+    quotechar = '"'
+    doublequote = True
+    lineterminator = "\r\n"
+
+
+class UntisCommaDialect(UntisDialect):
+    """Custom CSV dialect for Untis with comma as delimiter."""
+
+    delimiter = ","
+
+
+class UntisTildeDialect(UntisDialect):
+    """Custom CSV dialect for Untis with tilde as delimiter."""
+
+    delimiter = "~"
+
 
+class UntisSemicolonDialect(UntisDialect):
+    """Custom CSV dialect for Untis with semicolon as delimiter."""
 
-def untis_split_second(s: str, conv: Callable[[Any], Any] = lambda el: el) -> Sequence:
-    return clean_array(s.split("~"), conv=conv)
+    delimiter = ";"
 
 
-def untis_split_third(s: str, conv: Callable[[Any], Any] = lambda el: el) -> Sequence:
-    return clean_array(s.split(";"), conv=conv)
+def split_with_csv_parser(s: str, dialect: Type[csv.Dialect]) -> Sequence:
+    """Split string with CSV parser."""
+    parsed = csv.reader([s], dialect=dialect)
+    return next(parsed)
+
+
+def untis_split_first(
+    s: str, conv: Callable[[Any], Any] = lambda el: el, remove_empty: bool = True
+) -> Sequence:
+    return clean_array(
+        re.findall(r"(\"[^\"]+\"|[^,\"]+)*,?", s)[:-1], conv=conv, remove_empty=remove_empty
+    )
+
+
+def untis_split_second(
+    s: str, conv: Callable[[Any], Any] = lambda el: el, remove_empty: bool = True
+) -> Sequence:
+    return clean_array(
+        split_with_csv_parser(s, dialect=UntisTildeDialect), conv=conv, remove_empty=remove_empty
+    )
+
+
+def untis_split_third(
+    s: str, conv: Callable[[Any], Any] = lambda el: el, remove_empty: bool = True
+) -> Sequence:
+    return clean_array(
+        split_with_csv_parser(s, dialect=UntisSemicolonDialect),
+        conv=conv,
+        remove_empty=remove_empty,
+    )
 
 
 def untis_date_to_date(untis: int) -> date: