Merge branch 'develop' into 'fb-bros-352/copied'

bmartel · bmartel · commit f5c2fecce831 · 2025-08-22T21:27:25.000Z
Workflow run: https://github.com/HumanSignal/label-studio/actions/runs/17166365180
diff --git a/label_studio/io_storages/base_models.py b/label_studio/io_storages/base_models.py
@@ -791,15 +791,21 @@ def save_annotations(self, annotations: models.QuerySet[Annotation]):
         self.info_set_in_progress()
         self.cached_user = self.project.organization.created_by
 
+        # Calculate optimal batch size based on project data and worker count
+        project_batch_size = self.project.get_task_batch_size()
+        chunk_size = max(1, project_batch_size // self.max_workers)
+        logger.info(
+            f'Export storage {self.id}: using chunk_size={chunk_size} '
+            f'(project_batch_size={project_batch_size}, max_workers={self.max_workers})'
+        )
+
         with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
             # Batch annotations so that we update progress before having to submit every future.
             # Updating progress in thread requires coordinating on count and db writes, so just
             # batching to keep it simpler.
             for annotation_batch in _batched(
-                Annotation.objects.filter(project=self.project).iterator(
-                    chunk_size=settings.STORAGE_EXPORT_CHUNK_SIZE
-                ),
-                settings.STORAGE_EXPORT_CHUNK_SIZE,
+                Annotation.objects.filter(project=self.project).iterator(chunk_size=chunk_size),
+                chunk_size,
             ):
                 futures = []
                 for annotation in annotation_batch:
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -73,7 +73,7 @@ dependencies = [
     "djangorestframework-simplejwt[crypto] (>=5.4.0,<6.0.0)",
     "tldextract (>=5.1.3)",
     ## HumanSignal repo dependencies :start
-    "label-studio-sdk @ https://github.com/HumanSignal/label-studio-sdk/archive/a0a9c4c642a4bf13e10ded0d2ba105caf39349bb.zip",
+    "label-studio-sdk @ https://github.com/HumanSignal/label-studio-sdk/archive/a80479402d230f5e097a3052f4fe39647e05250a.zip",
     ## HumanSignal repo dependencies :end
 ]
 

Original file line number	Diff line number	Diff line change
`@@ -73,7 +73,7 @@ dependencies = [`
`73`	`73`	`"djangorestframework-simplejwt[crypto] (>=5.4.0,<6.0.0)",`
`74`	`74`	`"tldextract (>=5.1.3)",`
`75`	`75`	`## HumanSignal repo dependencies :start`
`76`		`- "label-studio-sdk @ https://github.com/HumanSignal/label-studio-sdk/archive/a0a9c4c642a4bf13e10ded0d2ba105caf39349bb.zip",`
	`76`	`+ "label-studio-sdk @ https://github.com/HumanSignal/label-studio-sdk/archive/a80479402d230f5e097a3052f4fe39647e05250a.zip",`
`77`	`77`	`## HumanSignal repo dependencies :end`
`78`	`78`	`]`
`79`	`79`