feat(media): broaden content type support (#1011)

langfuse · Nov 28, 2024 · b677d0a · b677d0a
1 parent e5e3995
commit b677d0a
Show file tree

Hide file tree

Showing 120 changed files with 4,187 additions and 1,497 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -38,10 +38,10 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.8"
           - "3.9"
           - "3.10"
           - "3.11"
+
     name: Test on Python version ${{ matrix.python-version }}
     steps:
       - uses: actions/checkout@v3
@@ -90,7 +90,7 @@ jobs:
           rm -rf .env
 
           echo "::group::Run server"
-          TELEMETRY_ENABLED=false CLICKHOUSE_CLUSTER_ENABLED=false LANGFUSE_ASYNC_INGESTION_PROCESSING=false LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=false LANGFUSE_READ_FROM_POSTGRES_ONLY=true LANGFUSE_READ_FROM_CLICKHOUSE_ONLY=false docker compose -f docker-compose.v3preview.yml up -d
+          TELEMETRY_ENABLED=false CLICKHOUSE_CLUSTER_ENABLED=false LANGFUSE_ASYNC_INGESTION_PROCESSING=false LANGFUSE_ASYNC_CLICKHOUSE_INGESTION_PROCESSING=false LANGFUSE_READ_FROM_POSTGRES_ONLY=true LANGFUSE_RETURN_FROM_CLICKHOUSE=false docker compose -f docker-compose.v3preview.yml up -d
           echo "::endgroup::"
 
       # Add this step to check the health of the container

diff --git a/langfuse/_task_manager/media_manager.py b/langfuse/_task_manager/media_manager.py
@@ -127,6 +127,52 @@ def _process_data_recursively(data: Any, level: int):
 
                 return media
 
+            # Anthropic
+            if (
+                isinstance(data, dict)
+                and "type" in data
+                and data["type"] == "base64"
+                and "media_type" in data
+                and "data" in data
+            ):
+                media = LangfuseMedia(
+                    base64_data_uri=f"data:{data['media_type']};base64," + data["data"],
+                )
+
+                self._process_media(
+                    media=media,
+                    trace_id=trace_id,
+                    observation_id=observation_id,
+                    field=field,
+                )
+
+                data["data"] = media
+
+                return data
+
+            # Vertex
+            if (
+                isinstance(data, dict)
+                and "type" in data
+                and data["type"] == "media"
+                and "mime_type" in data
+                and "data" in data
+            ):
+                media = LangfuseMedia(
+                    base64_data_uri=f"data:{data['mime_type']};base64," + data["data"],
+                )
+
+                self._process_media(
+                    media=media,
+                    trace_id=trace_id,
+                    observation_id=observation_id,
+                    field=field,
+                )
+
+                data["data"] = media
+
+                return data
+
             if isinstance(data, list):
                 return [_process_data_recursively(item, level + 1) for item in data]
 

diff --git a/langfuse/api/README.md b/langfuse/api/README.md
@@ -16,8 +16,8 @@ pip install finto
 Instantiate and use the client with the following:
 
 ```python
-from finto import CreateCommentRequest
-from finto.client import FernLangfuse
+from langfuse.api import CreateCommentRequest
+from langfuse.api.client import FernLangfuse
 
 client = FernLangfuse(
     x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME",
@@ -45,8 +45,8 @@ The SDK also exports an `async` client so that you can make non-blocking calls t
 ```python
 import asyncio
 
-from finto import CreateCommentRequest
-from finto.client import AsyncFernLangfuse
+from langfuse.api import CreateCommentRequest
+from langfuse.api.client import AsyncFernLangfuse
 
 client = AsyncFernLangfuse(
     x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME",
@@ -116,7 +116,7 @@ The SDK defaults to a 60 second timeout. You can configure this with a timeout o
 
 ```python
 
-from finto.client import FernLangfuse
+from langfuse.api.client import FernLangfuse
 
 client = FernLangfuse(..., { timeout=20.0 }, )
 
@@ -131,9 +131,10 @@ client.comments.create(...,{
 
 You can override the `httpx` client to customize it for your use-case. Some common use-cases include support for proxies
 and transports.
+
 ```python
 import httpx
-from finto.client import FernLangfuse
+from langfuse.api.client import FernLangfuse
 
 client = FernLangfuse(
     ...,

diff --git a/langfuse/api/client.py b/langfuse/api/client.py
@@ -51,7 +51,7 @@ class FernLangfuse:
 
     Examples
     --------
-    from finto.client import FernLangfuse
+    from langfuse.api.client import FernLangfuse
 
     client = FernLangfuse(
         x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME",
@@ -140,7 +140,7 @@ class AsyncFernLangfuse:
 
     Examples
     --------
-    from finto.client import AsyncFernLangfuse
+    from langfuse.api.client import AsyncFernLangfuse
 
     client = AsyncFernLangfuse(
         x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME",

diff --git a/langfuse/api/core/api_error.py b/langfuse/api/core/api_error.py
@@ -7,7 +7,9 @@ class ApiError(Exception):
     status_code: typing.Optional[int]
     body: typing.Any
 
-    def __init__(self, *, status_code: typing.Optional[int] = None, body: typing.Any = None):
+    def __init__(
+        self, *, status_code: typing.Optional[int] = None, body: typing.Any = None
+    ):
         self.status_code = status_code
         self.body = body
 

diff --git a/langfuse/api/core/client_wrapper.py b/langfuse/api/core/client_wrapper.py
@@ -17,7 +17,7 @@ def __init__(
         username: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None,
         password: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None,
         base_url: str,
-        timeout: typing.Optional[float] = None
+        timeout: typing.Optional[float] = None,
     ):
         self._x_langfuse_sdk_name = x_langfuse_sdk_name
         self._x_langfuse_sdk_version = x_langfuse_sdk_version
@@ -71,7 +71,7 @@ def __init__(
         password: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None,
         base_url: str,
         timeout: typing.Optional[float] = None,
-        httpx_client: httpx.Client
+        httpx_client: httpx.Client,
     ):
         super().__init__(
             x_langfuse_sdk_name=x_langfuse_sdk_name,
@@ -101,7 +101,7 @@ def __init__(
         password: typing.Optional[typing.Union[str, typing.Callable[[], str]]] = None,
         base_url: str,
         timeout: typing.Optional[float] = None,
-        httpx_client: httpx.AsyncClient
+        httpx_client: httpx.AsyncClient,
     ):
         super().__init__(
             x_langfuse_sdk_name=x_langfuse_sdk_name,

diff --git a/langfuse/api/core/datetime_utils.py b/langfuse/api/core/datetime_utils.py
@@ -13,7 +13,9 @@ def serialize_datetime(v: dt.datetime) -> str:
     """
 
     def _serialize_zoned_datetime(v: dt.datetime) -> str:
-        if v.tzinfo is not None and v.tzinfo.tzname(None) == dt.timezone.utc.tzname(None):
+        if v.tzinfo is not None and v.tzinfo.tzname(None) == dt.timezone.utc.tzname(
+            None
+        ):
             # UTC is a special case where we use "Z" at the end instead of "+00:00"
             return v.isoformat().replace("+00:00", "Z")
         else:

diff --git a/langfuse/api/core/file.py b/langfuse/api/core/file.py
@@ -13,12 +13,17 @@
     # (filename, file (or bytes), content_type)
     typing.Tuple[typing.Optional[str], FileContent, typing.Optional[str]],
     # (filename, file (or bytes), content_type, headers)
-    typing.Tuple[typing.Optional[str], FileContent, typing.Optional[str], typing.Mapping[str, str]],
+    typing.Tuple[
+        typing.Optional[str],
+        FileContent,
+        typing.Optional[str],
+        typing.Mapping[str, str],
+    ],
 ]
 
 
 def convert_file_dict_to_httpx_tuples(
-    d: typing.Dict[str, typing.Union[File, typing.List[File]]]
+    d: typing.Dict[str, typing.Union[File, typing.List[File]]],
 ) -> typing.List[typing.Tuple[str, File]]:
     """
     The format we use is a list of tuples, where the first element is the