now using bulk_update

core/TESTS/test_drawings.py

@@ -70,3 +70,16 @@ class DrawingsPathlibTests(TestCase):
             drawings.load_drawings_files()
 
             self.assertFalse(DrawingFile.objects.filter(dwgpath='.git/secret.txt').exists())
+
+    def test_bulk_create_chunks(self):
+        # Create more than chunk size files to ensure bulk_create is called in multiple chunks
+        count = 800
+        with tempfile.TemporaryDirectory() as td:
+            p = pathlib.Path(td)
+            for i in range(count):
+                (p / f'file{i}.txt').write_text('x')
+            settings.DRAWINGS_DATA = td
+
+            drawings.load_drawings_files()
+
+            self.assertEqual(DrawingFile.objects.count(), count)

parsers/drawings.py

@@ -339,6 +339,7 @@ def load_drawings_files():
     supported_extensions = {".txt", ".xml", ".th", ".th2", ".pdf", ".png", ".svg", ".jpg"}
 
     # Walk the tree with pathlib, skip hidden and backup files
+    files_meta = []  # list of tuples (ext, rel_path, dwgname, pathobj)
     for p in drawdatadir.rglob('*'):
         # Ignore anything under a .git directory
         if '.git' in p.parts:
@@ -358,9 +359,26 @@ def load_drawings_files():
                 dwgname = p.stem
                 ext = suffix[1:]
 
-            dwgfile = DrawingFile(dwgpath=rel, dwgname=dwgname)
-            dwgfile.save()
-            all_xml.append((ext, dwgfile, p))
+            files_meta.append((ext, rel, dwgname, p))
+
+    # Bulk create DrawingFile instances to avoid many individual DB saves
+    if files_meta:
+        objs_to_create = [DrawingFile(dwgpath=rel, dwgname=dwgname) for (_, rel, dwgname, _) in files_meta]
+        # Use chunks to avoid huge single queries
+        chunk_size = 700
+        for i in range(0, len(objs_to_create), chunk_size):
+            DrawingFile.objects.bulk_create(objs_to_create[i : i + chunk_size])
+
+        # Re-fetch created objects and map by dwgpath
+        rel_paths = [rel for (_, rel, _, _) in files_meta]
+        created_objs = DrawingFile.objects.filter(dwgpath__in=rel_paths)
+        mapping = {obj.dwgpath: obj for obj in created_objs}
+
+        # Reconstruct all_xml using the created model instances
+        for ext, rel, _, p in files_meta:
+            dwgfile = mapping.get(rel)
+            if dwgfile:
+                all_xml.append((ext, dwgfile, p))
 
     print(f" - {len(all_xml)} Drawings files found")