revert: removal of expected in dataset

empirical-run · May 22, 2024 · 07ff068 · 07ff068
1 parent b68545f
commit 07ff068
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/packages/scorer/src/provider/model-graded/llm.ts b/packages/scorer/src/provider/model-graded/llm.ts
@@ -63,6 +63,10 @@ export const checkLlmCriteria = async ({
   const scorerName = config.name || name;
   if (config.criteria) {
     let replacements: any = { ...sample.inputs };
+    if (sample.expected) {
+      // llm-critic supports {{expected}} as placeholder
+      replacements.expected = sample.expected;
+    }
     criteria = replacePlaceholders(config.criteria, replacements);
   }
   if (!criteria) {

diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
@@ -172,6 +172,7 @@ export type DatasetSampleInputs = any;
 export type DatasetSample = {
   id: string;
   inputs: DatasetSampleInputs;
+  expected?: string;
 };
 
 export type Dataset = {
@@ -182,6 +183,7 @@ export type Dataset = {
 export type DatasetSampleConfig = {
   id?: string;
   inputs: DatasetSampleInputs;
+  expected?: string;
 };
 
 export type DatasetConfig =
@@ -213,6 +215,9 @@ export type RunSampleOutput = {
   scores?: Score[];
   inputs: DatasetSampleInputs;
   output: RunOutput;
+  expected?: {
+    value: string;
+  };
   error?: {
     code?: string;
     message: string;