Implement virtual fields with jq program generation

aleksis/apps/csv_import/field_types.py

@@ -27,6 +27,7 @@ class FieldType:
     db_field: str = ""
     converter: Optional[Union[str, Sequence[str]]] = None
     alternative_db_fields: Optional[str] = None
+    jq_prog: str = ""
     args: Optional[dict] = None
 
     @classmethod
@@ -76,10 +77,16 @@ class FieldType:
             return self.get_args()["column_name"]
         return self.column_name
 
-    def __init__(self, school_term: SchoolTerm, base_path: str):
+    def get_jq_prog(self) -> str:
+        if self.get_args().get("jq_prog"):
+            return self.get_args()["jq_prog"]
+        return self.jq_prog or "."
+
+    def __init__(self, school_term: SchoolTerm, base_path: str, **kwargs):
         self.school_term = school_term
         self.base_path = os.path.realpath(base_path)
         self.column_name = f"col_{uuid4()}"
+        self.args = kwargs
 
 
 class MatchFieldType(FieldType):
@@ -347,12 +354,12 @@ class ClassRangeFieldType(ProcessFieldType):
     name = "class_range"
     verbose_name = _("Class range (e. g. 7a-d)")
 
-    def __init__(self, school_term: SchoolTerm, base_path: str):
+    def __init__(self, school_term: SchoolTerm, base_path: str, **kwargs):
         # Prefetch class groups
         self.classes_per_short_name = get_classes_per_short_name(school_term)
         self.classes_per_grade = get_classes_per_grade(self.classes_per_short_name.keys())
 
-        super().__init__(school_term, base_path)
+        super().__init__(school_term, base_path, **kwargs)
 
     def process(self, instance: Model, value):
         classes = parse_class_range(

aleksis/apps/csv_import/models.py

@@ -62,16 +62,6 @@ class ImportTemplate(ExtensibleModel):
     def parsed_separator(self):
         return codecs.escape_decode(bytes(self.separator, "utf-8"))[0].decode("utf-8")
 
-    @property
-    def literal_fields(self) -> QuerySet:
-        """Get all literal (non-virtual) fields."""
-        return self.fields.filter(virtual=False)
-
-    @property
-    def virtual_fields(self) -> QuerySet:
-        """Get all virtual fields."""
-        return self.fields.filter(virtual=True)
-
     def save(self, *args, **kwargs):
         if not self.content_type.model == "person":
             self.group = None
@@ -179,7 +169,6 @@ class ImportTemplateField(ExtensibleModel):
     @property
     def field_type_class(self):
         field_type = field_type_registry.get_from_name(self.field_type)
-        field_type.args = self.args
         return field_type
 
     def clean(self):

aleksis/apps/csv_import/util/process.py

@@ -11,6 +11,7 @@ from django.db import transaction
 from django.utils.translation import gettext as _
 
 import chardet
+import jq
 import pandas
 from pandas.errors import ParserError
 from tqdm import tqdm
@@ -46,9 +47,10 @@ def import_csv(
         data_types = {}
         converters = {}
         field_types = {}
-        for field in template.literal_fields:
+        virtual_fields = []
+        for field in template.fields.all():
             # Get field type and prepare for import
-            field_type = field.field_type_class(school_term, temp_dir)
+            field_type = field.field_type_class(school_term, temp_dir, **field.args)
             column_name = field_type.get_column_name()
 
             # Get data type and conversion rules, if any,
@@ -57,6 +59,9 @@ def import_csv(
             if field_type.get_converter():
                 converters[column_name] = field_type.get_converter()
 
+            if field.virtual:
+                virtual_fields.append(column_name)
+
             field_types[column_name] = field_type
 
         # Determine whether the data file is a plain CSV or an archive
@@ -123,6 +128,22 @@ def import_csv(
 
                 iterator = recorder.iterate(data_as_dict) if recorder else tqdm(data_as_dict)
                 for row in iterator:
+                    # Generate virtual field data
+                    for column_name in virtual_fields:
+                        field_type = field_types[column_name]
+
+                        # Generate field using jq program and rest of row as input
+                        jq_prog = jq.compile(field_type.get_jq_prog())
+                        data = jq.input(row).first()
+                        if not isinstance(data, str):
+                            raise TypeError(f"The jq program for {column_name} generated another type than str.")
+
+                        # Post-process field using converter
+                        data = field_type.get_converter()(data)
+
+                        # Store
+                        row[column_name] = data
+
                     # Build dict with all fields that should be directly updated
                     update_dict = {}
                     for key, value in row.items():