Skip to content
This repository has been archived by the owner on Nov 13, 2024. It is now read-only.

Commit

Permalink
add test for json schema validation
Browse files Browse the repository at this point in the history
  • Loading branch information
acatav committed Nov 6, 2023
1 parent 98d0248 commit 7ef1b54
Showing 1 changed file with 22 additions and 1 deletion.
23 changes: 22 additions & 1 deletion tests/system/llm/test_openai.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
from unittest.mock import patch
from unittest.mock import patch, MagicMock

import jsonschema
import pytest


Expand Down Expand Up @@ -184,3 +186,22 @@ def test_enforce_function_api_failure_populates(mock_api_call,
with pytest.raises(Exception, match="API call failed"):
openai_llm.enforced_function_call(messages=messages,
function=function_query_knowledgebase)

@staticmethod
@patch("openai.ChatCompletion")
def test_enforce_function_wrong_output_schema(chat_completion,
openai_llm,
messages,
function_query_knowledgebase):
chat_completion.create.return_value = MagicMock(
choices=[MagicMock(
message=MagicMock(
function_call={"arguments": "{\"key\": \"value\"}"}))])

with pytest.raises(jsonschema.ValidationError,
match="'queries' is a required property"):
openai_llm.enforced_function_call(messages=messages,
function=function_query_knowledgebase)

assert chat_completion.create.call_count == 3, \
"retry did not happen as expected"

0 comments on commit 7ef1b54

Please sign in to comment.