harbor-langsmith: persist run tags (top-level) and dataset metadata (under extra) (harbor-framework#1808)

nick-hollon-lc · web-flow · commit c978013a81af · 2026-06-07T10:41:20.000-07:00
diff --git a/packages/harbor-langsmith/src/harbor_langsmith/plugin.py b/packages/harbor-langsmith/src/harbor_langsmith/plugin.py
@@ -160,7 +160,9 @@ def _get_or_create_dataset(self, job: Any) -> str | None:
         payload = {
             "name": dataset_name,
             "description": f"Harbor dataset synced from job {job.config.job_name}",
-            "metadata": {"source": "harbor"},
+            # LangSmith stores dataset metadata under extra.metadata (matching the SDK);
+            # a top-level "metadata" key is silently dropped.
+            "extra": {"metadata": {"source": "harbor"}},
         }
         response = self._request(
             "POST", "/datasets", json=payload, ok_statuses={200, 201, 409}
@@ -239,9 +241,9 @@ def _create_root_run(self, event: TrialHookEvent) -> None:
             "start_time": self._format_time(event.timestamp),
             "session_id": self._experiment_id,
             "reference_example_id": reference_example_id,
+            "tags": ["harbor", "harbor-trial"],
             "extra": {
                 "metadata": self._trial_metadata(event),
-                "tags": ["harbor", "harbor-trial"],
             },
         }
         if payload["reference_example_id"] is None:
@@ -270,9 +272,9 @@ def _create_phase_run(self, event: TrialHookEvent) -> None:
             "start_time": self._format_time(event.timestamp),
             "session_id": self._experiment_id,
             "parent_run_id": parent_run_id,
+            "tags": ["harbor", "harbor-phase", event.event.value],
             "extra": {
                 "metadata": self._trial_metadata(event),
-                "tags": ["harbor", "harbor-phase", event.event.value],
             },
         }
         self._request("POST", "/runs", json=payload, ok_statuses={200, 201, 409})
diff --git a/packages/harbor-langsmith/tests/unit/test_plugin.py b/packages/harbor-langsmith/tests/unit/test_plugin.py
@@ -57,3 +57,47 @@ def test_stable_uuid_is_deterministic():
 
     assert first == second
     assert first != third
+
+
+@pytest.mark.unit
+def test_root_run_tags_are_top_level(monkeypatch):
+    plugin = LangSmithPlugin(api_key="test-key")
+    plugin._experiment_id = "exp"
+    monkeypatch.setattr(plugin, "_trial_metadata", lambda event: {})
+    with patch.object(plugin, "_request") as request:
+        plugin._create_root_run(MagicMock())
+
+    payload = request.call_args.kwargs["json"]
+    assert payload["tags"] == ["harbor", "harbor-trial"]
+    assert "tags" not in payload["extra"]
+
+
+@pytest.mark.unit
+def test_phase_run_tags_are_top_level(monkeypatch):
+    plugin = LangSmithPlugin(api_key="test-key")
+    plugin._experiment_id = "exp"
+    monkeypatch.setattr(plugin, "_trial_metadata", lambda event: {})
+    event = MagicMock()
+    event.event.value = "agent_start"
+    plugin._run_ids[event.config.trial_name] = "parent-run"
+    with patch.object(plugin, "_request") as request:
+        plugin._create_phase_run(event)
+
+    payload = request.call_args.kwargs["json"]
+    assert payload["tags"] == ["harbor", "harbor-phase", "agent_start"]
+    assert "tags" not in payload["extra"]
+
+
+@pytest.mark.unit
+def test_dataset_metadata_is_nested_under_extra(monkeypatch):
+    plugin = LangSmithPlugin(api_key="test-key")
+    plugin.dataset_name = "ds"
+    monkeypatch.setattr(plugin, "_find_dataset", lambda name: None)
+    response = MagicMock(status_code=201)
+    response.json.return_value = {"id": "d1"}
+    with patch.object(plugin, "_request", return_value=response) as request:
+        plugin._get_or_create_dataset(MagicMock())
+
+    payload = request.call_args.kwargs["json"]
+    assert payload["extra"]["metadata"] == {"source": "harbor"}
+    assert "metadata" not in payload
