fix: review comments

empirical-run · May 22, 2024 · 9ee436a · 9ee436a
1 parent 0108f4a
commit 9ee436a
Show file tree

Hide file tree

Showing 6 changed files with 22 additions and 24 deletions.
diff --git a/.changeset/thin-phones-destroy.md b/.changeset/thin-phones-destroy.md
@@ -0,0 +1,8 @@
+---
+"empiricalrun": minor
+"@empiricalrun/scorer": minor
+"@empiricalrun/types": minor
+"web": patch
+---
+
+feat: add support for js/ts scripts as scorers
diff --git a/apps/web/components/run-config-view.tsx b/apps/web/components/run-config-view.tsx
@@ -16,7 +16,6 @@ enum RunConfigTab {
 const defaultTabMap: Record<RunConfigType, RunConfigTab> = {
   model: RunConfigTab.prompt,
   "py-script": RunConfigTab.parameters,
-  "js-script": RunConfigTab.parameters,
   assistant: RunConfigTab.instructions,
 };
 

diff --git a/examples/spider-using-ts/.gitignore b/examples/spider-using-ts/.gitignore
@@ -1,4 +1,4 @@
 
 # Ignore outputs from Empirical
 .empiricalrun
-db
+db_files
diff --git a/examples/spider-using-ts/db.ts b/examples/spider-using-ts/db.ts
@@ -12,8 +12,8 @@ async function createDatabase(
   con: sqlite3.Database,
   dbName: string,
 ): Promise<sqlite3.Database> {
-  return new Promise((resolve, reject) => {
-    const schemaScript = loadJson("./schema.json")[dbName];
+  return new Promise((resolve) => {
+    const schemaScript = dbSchemas[dbName];
     const createScript = loadJson("./create.json")[dbName];
 
     con.serialize(() => {
@@ -32,11 +32,12 @@ export async function getConnection (dbName:string): Promise<sqlite3.Database> {
     if (connectionCache.get(dbName)) {
         return await connectionCache.get(dbName)!
     }
+    const dbFilesDir = "db_files"
     const connectionPromise = new Promise<sqlite3.Database>(async (resolve) => {
-        if(!fs.existsSync("db")){
-            fs.mkdirSync("db");
+        if(!fs.existsSync(dbFilesDir)){
+            fs.mkdirSync(dbFilesDir);
         }
-        const dbfileName = `db/${dbName}.db`
+        const dbfileName = `${dbFilesDir}/${dbName}.db`
         const con = new sqlite3.Database(dbfileName);
         await createDatabase(con, dbName)
         resolve(con)

diff --git a/examples/using-js-config/package.json b/examples/using-js-config/package.json
@@ -10,6 +10,6 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
-    "empiricalrun": "workspace:*"
+    "empiricalrun": "^0.13.1"
   }
 }
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
@@ -41,21 +41,17 @@ export interface ScriptScorer extends ScorerBase {
 export interface JSScriptScorerParams {
   inputs: Record<string, any>;
   output: RunOutput;
-  options?: any;
 }
 
-export interface AsyncScoringFn {
+export interface JSScriptScorer {
   (
     args: JSScriptScorerParams,
-  ): Promise<Score[] | (Partial<Score> & { score: number })>;
+  ):
+    | Score[]
+    | (Partial<Score> & { score: number })
+    | Promise<Score[] | (Partial<Score> & { score: number })>;
 }
 
-export interface SyncScoringFn {
-  (args: JSScriptScorerParams): Score[] | (Partial<Score> & { score: number });
-}
-
-export type JSScriptScorer = SyncScoringFn | AsyncScoringFn;
-
 export type Scorer = LLMScorer | SyntaxScorer | ScriptScorer | JSScriptScorer;
 
 export type Prompt = string | ChatPrompt[];
@@ -106,7 +102,7 @@ export interface ModelRunConfig extends RunConfigBase {
     | "fireworks"
     | "azure-openai";
   model: string;
-  prompt?: Prompt;
+  prompt: Prompt;
   parameters?: ModelParameters;
 }
 
@@ -127,11 +123,6 @@ export interface AssistantsRunConfig extends RunConfigBase {
   };
 }
 
-export interface JSScriptRunConfig extends RunConfigBase {
-  type: "js-script";
-  path: string;
-}
-
 export interface PyScriptRunConfig extends RunConfigBase {
   type: "py-script";
   path: string;
@@ -144,7 +135,6 @@ export interface PyScriptRunConfig extends RunConfigBase {
 export type RunConfig =
   | ModelRunConfig
   | PyScriptRunConfig
-  | JSScriptRunConfig
   | AssistantsRunConfig;
 
 export interface ScoreStats {