diff --git a/src/gptscript.ts b/src/gptscript.ts index 7067a84..14695e7 100644 --- a/src/gptscript.ts +++ b/src/gptscript.ts @@ -13,6 +13,7 @@ export interface GlobalOpts { DefaultModel?: string DefaultModelProvider?: string DatasetToolRepo?: string + WorkspaceTool?: string Env?: string[] } @@ -140,9 +141,12 @@ export class GPTScript { if (!this.opts.URL) { this.opts.URL = GPTScript.serverURL } + if (this.opts.URL !== "" && !this.opts.URL.startsWith("http://") && !this.opts.URL.startsWith("https://")) { + this.opts.URL = "http://" + this.opts.URL + } if (!this.opts.Env) { - this.opts.Env = [] + this.opts.Env = Object.entries(process.env).map(([k, v]) => `${k}=${v}`) } if (this.opts.URL) { this.opts.Env.push(`GPTSCRIPT_URL=${this.opts.URL}`) @@ -362,113 +366,187 @@ export class GPTScript { } async createCredential(credential: Credential): Promise { - if (!this.opts.URL) { - await this.testGPTScriptURL(20) - } - - const r: Run = new RunSubcommand("credentials/create", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({content: credentialToJSON(credential)}) - await r.text() + await this.runBasicCommand("credentials/create", { + content: credentialToJSON(credential) + }) } async revealCredential(context: Array, name: string): Promise { - if (!this.opts.URL) { - await this.testGPTScriptURL(20) - } - - const r: Run = new RunSubcommand("credentials/reveal", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({context, name}) - return jsonToCredential(await r.text()) + const resp = await this.runBasicCommand("credentials/reveal", { + context, + name + }) + return jsonToCredential(resp) } async deleteCredential(context: string, name: string): Promise { - if (!this.opts.URL) { - await this.testGPTScriptURL(20) - } - - const r: Run = new RunSubcommand("credentials/delete", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({context: [context], name}) - await r.text() + await this.runBasicCommand("credentials/delete", { + context: [context], + name + }) } // Dataset methods - async listDatasets(workspace: string): Promise> { - if (workspace == "") { - workspace = process.env.GPTSCRIPT_WORKSPACE_DIR ?? "" + async listDatasets(workspaceID: string): Promise> { + if (workspaceID == "") { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" } - const r: Run = new RunSubcommand("datasets", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({input: "{}", workspace: workspace, datasetToolRepo: this.opts.DatasetToolRepo ?? ""}) - const result = await r.text() + const result = await this.runBasicCommand("datasets", { + workspaceID: workspaceID, + datasetToolRepo: this.opts.DatasetToolRepo ?? "", + env: this.opts.Env + }) return JSON.parse(result) as Array } - async createDataset(workspace: string, name: string, description: string): Promise { - if (workspace == "") { - workspace = process.env.GPTSCRIPT_WORKSPACE_DIR ?? "" + async createDataset(workspaceID: string, name: string, description: string): Promise { + if (workspaceID == "") { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" } - const r: Run = new RunSubcommand("datasets/create", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({ + const result = await this.runBasicCommand("datasets/create", { input: JSON.stringify({datasetName: name, datasetDescription: description}), - workspace: workspace, - datasetToolRepo: this.opts.DatasetToolRepo ?? "" + workspaceID: workspaceID, + datasetToolRepo: this.opts.DatasetToolRepo ?? "", + env: this.opts.Env }) - const result = await r.text() return JSON.parse(result) as Dataset } - async addDatasetElement(workspace: string, datasetID: string, elementName: string, elementDescription: string, elementContent: string): Promise { - if (workspace == "") { - workspace = process.env.GPTSCRIPT_WORKSPACE_DIR ?? "" + async addDatasetElement(workspaceID: string, datasetID: string, elementName: string, elementDescription: string, elementContent: string): Promise { + if (workspaceID == "") { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" } - const r: Run = new RunSubcommand("datasets/add-element", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({ + const result = await this.runBasicCommand("datasets/add-element", { input: JSON.stringify({ datasetID, - elementName, - elementDescription, - elementContent + elementName: elementName, + elementDescription: elementDescription, + elementContent: elementContent }), - workspace: workspace, - datasetToolRepo: this.opts.DatasetToolRepo ?? "" + workspaceID: workspaceID, + datasetToolRepo: this.opts.DatasetToolRepo ?? "", + env: this.opts.Env }) - const result = await r.text() return JSON.parse(result) as DatasetElementMeta } - async listDatasetElements(workspace: string, datasetID: string): Promise> { - if (workspace == "") { - workspace = process.env.GPTSCRIPT_WORKSPACE_DIR ?? "" + async listDatasetElements(workspaceID: string, datasetID: string): Promise> { + if (workspaceID == "") { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" } - const r: Run = new RunSubcommand("datasets/list-elements", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({ + + const result = await this.runBasicCommand("datasets/list-elements", { input: JSON.stringify({datasetID}), - workspace: workspace, - datasetToolRepo: this.opts.DatasetToolRepo ?? "" + workspaceID: workspaceID, + datasetToolRepo: this.opts.DatasetToolRepo ?? "", + env: this.opts.Env }) - const result = await r.text() return JSON.parse(result) as Array } - async getDatasetElement(workspace: string, datasetID: string, elementName: string): Promise { - if (workspace == "") { - workspace = process.env.GPTSCRIPT_WORKSPACE_DIR ?? "" + async getDatasetElement(workspaceID: string, datasetID: string, elementName: string): Promise { + if (workspaceID == "") { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" } - const r: Run = new RunSubcommand("datasets/get-element", "", {URL: this.opts.URL, Token: this.opts.Token}) - r.request({ + const result = await this.runBasicCommand("datasets/get-element", { input: JSON.stringify({datasetID, element: elementName}), - workspace: workspace, - datasetToolRepo: this.opts.DatasetToolRepo ?? "" + workspaceID: workspaceID, + datasetToolRepo: this.opts.DatasetToolRepo ?? "", + env: this.opts.Env }) - const result = await r.text() return JSON.parse(result) as DatasetElement } + async createWorkspace(providerType: string, ...fromWorkspaces: string[]): Promise { + const out = await this.runBasicCommand("workspaces/create", { + providerType: providerType, + fromWorkspaceIDs: fromWorkspaces, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + return out.trim() + } + + async deleteWorkspace(workspaceID?: string): Promise { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + await this.runBasicCommand("workspaces/delete", { + id: workspaceID, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + } + + async listFilesInWorkspace(prefix?: string, workspaceID?: string): Promise> { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + const out = await this.runBasicCommand("workspaces/list", { + id: workspaceID, + prefix: prefix, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + return JSON.parse(out) + } + + async removeAll(withPrefix?: string, workspaceID?: string): Promise { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + await this.runBasicCommand("workspaces/remove-all-with-prefix", { + id: workspaceID, + prefix: withPrefix, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + } + + async writeFileInWorkspace(filePath: string, content: ArrayBuffer, workspaceID?: string): Promise { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + await this.runBasicCommand("workspaces/write-file", { + id: workspaceID, + filePath: filePath, + contents: Buffer.from(content).toString("base64"), + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + } + + async deleteFileInWorkspace(filePath: string, workspaceID?: string): Promise { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + await this.runBasicCommand("workspaces/delete-file", { + id: workspaceID, + filePath: filePath, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + } + + async readFileInWorkspace(filePath: string, workspaceID?: string): Promise { + if (!workspaceID) { + workspaceID = process.env.GPTSCRIPT_WORKSPACE_ID ?? "" + } + const out = await this.runBasicCommand("workspaces/read-file", { + id: workspaceID, + filePath: filePath, + workspaceTool: this.opts.WorkspaceTool, + env: this.opts.Env, + }) + return Buffer.from(out.trim(), "base64") + } + /** * Helper method to handle the common logic for loading. * @@ -694,7 +772,10 @@ export class Run { fetch(req).then(resp => { return resp.json() }).then(res => { - resolve(res.stdout) + if (typeof res.stdout === "string") { + resolve(res.stdout) + } + resolve(JSON.stringify(res.stdout)) }).catch(e => { reject(new Error(e)) }) diff --git a/tests/gptscript.test.ts b/tests/gptscript.test.ts index 94cc0bd..b0ac6bd 100644 --- a/tests/gptscript.test.ts +++ b/tests/gptscript.test.ts @@ -1,7 +1,7 @@ import * as gptscript from "../src/gptscript" import { ArgumentSchemaType, - CredentialType, Dataset, + CredentialType, getEnv, PropertyType, RunEventType, @@ -13,7 +13,6 @@ import path from "path" import {fileURLToPath} from "url" import * as fs from "node:fs" import {randomBytes} from "node:crypto" -import {tmpdir} from "node:os"; let gFirst: gptscript.GPTScript let g: gptscript.GPTScript @@ -660,7 +659,7 @@ describe("gptscript module", () => { tools: ["sys.exec"] } - const commands = [`"ls"`, `"dir"`] + const commands = [`ls`, `dir`] let confirmCallCount = 0 const run = await g.evaluate(t, {confirm: true}) run.on(gptscript.RunEventType.CallConfirm, async (data: gptscript.CallFrame) => { @@ -683,7 +682,7 @@ describe("gptscript module", () => { } const run = await g.evaluate(t, {confirm: true}) run.on(gptscript.RunEventType.CallConfirm, async (data: gptscript.CallFrame) => { - expect(data.input).toContain(`"ls"`) + expect(data.input).toContain(`ls`) confirmFound = true await g.confirm({id: data.id, accept: false, message: "I will not allow it!"}) }) @@ -889,12 +888,12 @@ describe("gptscript module", () => { test("dataset operations", async () => { const datasetName = "test-" + randomBytes(10).toString("hex") - const workspace = fs.mkdtempSync(path.join(tmpdir(), "node-gptscript-")) + const workspaceID = await g.createWorkspace("directory") let datasetID: string // Create try { - const dataset = await g.createDataset(workspace, datasetName, "a test dataset") + const dataset = await g.createDataset(workspaceID, datasetName, "a test dataset") expect(dataset).toBeDefined() expect(dataset.name).toEqual(datasetName) expect(dataset.description).toEqual("a test dataset") @@ -908,21 +907,21 @@ describe("gptscript module", () => { // Add elements try { const e1 = await g.addDatasetElement( - workspace, - datasetID, - "element1", - "", - "this is element 1 contents" + workspaceID, + datasetID, + "element1", + "", + "this is element 1 contents" ) expect(e1.name).toEqual("element1") expect(e1.description).toEqual("") const e2 = await g.addDatasetElement( - workspace, - datasetID, - "element2", - "a description", - "this is element 2 contents" + workspaceID, + datasetID, + "element2", + "a description", + "this is element 2 contents" ) expect(e2.name).toEqual("element2") expect(e2.description).toEqual("a description") @@ -932,12 +931,12 @@ describe("gptscript module", () => { // Get elements try { - const e1 = await g.getDatasetElement(workspace, datasetID, "element1") + const e1 = await g.getDatasetElement(workspaceID, datasetID, "element1") expect(e1.name).toEqual("element1") expect(e1.description).toBeUndefined() expect(e1.contents).toEqual("this is element 1 contents") - const e2 = await g.getDatasetElement(workspace, datasetID, "element2") + const e2 = await g.getDatasetElement(workspaceID, datasetID, "element2") expect(e2.name).toEqual("element2") expect(e2.description).toEqual("a description") expect(e2.contents).toEqual("this is element 2 contents") @@ -947,7 +946,7 @@ describe("gptscript module", () => { // List the elements in the dataset try { - const elements = await g.listDatasetElements(workspace, datasetID) + const elements = await g.listDatasetElements(workspaceID, datasetID) expect(elements.length).toEqual(2) expect(elements.map(e => e.name)).toContain("element1") expect(elements.map(e => e.name)).toContain("element2") @@ -957,11 +956,120 @@ describe("gptscript module", () => { // List datasets try { - const datasets = await g.listDatasets(workspace) + const datasets = await g.listDatasets(workspaceID) expect(datasets.length).toBeGreaterThan(0) expect(datasets.map(d => d.name)).toContain(datasetName) } catch (e) { throw new Error("failed to list datasets: " + e) } - }, 20000) + }, 60000) + + test("create and delete workspace", async () => { + const workspaceID = await g.createWorkspace("directory") + expect(workspaceID).toBeDefined() + await g.deleteWorkspace(workspaceID) + }, 60000) + + test("write, read, and delete file", async () => { + const workspaceID = await g.createWorkspace("directory") + expect(workspaceID).toBeDefined() + + await g.writeFileInWorkspace("test.txt", Buffer.from("test"), workspaceID) + const content = await g.readFileInWorkspace("test.txt", workspaceID) + expect(content.toString()).toEqual("test") + await g.deleteWorkspace(workspaceID) + }, 60000) + + test("test complex ls", async () => { + const workspaceID = await g.createWorkspace("directory") + + // Write files in the workspace + await g.writeFileInWorkspace("test/test1.txt", Buffer.from("hello1"), workspaceID) + await g.writeFileInWorkspace("test1/test2.txt", Buffer.from("hello2"), workspaceID) + await g.writeFileInWorkspace("test1/test3.txt", Buffer.from("hello3"), workspaceID) + await g.writeFileInWorkspace(".hidden.txt", Buffer.from("hidden"), workspaceID) + + let content = await g.listFilesInWorkspace(undefined, workspaceID) + expect(content.length).toEqual(4) + expect(content).toContain("test1/test2.txt") + expect(content).toContain("test1/test3.txt") + expect(content).toContain("test/test1.txt") + expect(content).toContain(".hidden.txt") + + content = await g.listFilesInWorkspace("test1", workspaceID) + expect(content.length).toEqual(2) + expect(content).toContain("test1/test2.txt") + expect(content).toContain("test1/test3.txt") + + await g.removeAll("test1", workspaceID) + + content = await g.listFilesInWorkspace("", workspaceID) + expect(content.length).toEqual(2) + expect(content).toContain("test/test1.txt") + expect(content).toContain(".hidden.txt") + + await g.deleteWorkspace(workspaceID) + }, 60000) + + test("create and delete workspace in s3", async () => { + if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + console.log("AWS credentials not set, skipping test") + return + } + + const workspaceID = await g.createWorkspace("s3") + expect(workspaceID).toBeDefined() + await g.deleteWorkspace(workspaceID) + }, 60000) + + test("write, read, and delete file in s3", async () => { + if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + console.log("AWS credentials not set, skipping test") + return + } + + const workspaceID = await g.createWorkspace("s3") + expect(workspaceID).toBeDefined() + + await g.writeFileInWorkspace("test.txt", Buffer.from("test"), workspaceID) + const content = await g.readFileInWorkspace("test.txt", workspaceID) + expect(content.toString()).toEqual("test") + await g.deleteWorkspace(workspaceID) + }, 60000) + + test("test complex ls in s3", async () => { + if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + console.log("AWS credentials not set, skipping test") + return + } + + const workspaceID = await g.createWorkspace("s3") + + // Write files in the workspace + await g.writeFileInWorkspace("test/test1.txt", Buffer.from("hello1"), workspaceID) + await g.writeFileInWorkspace("test1/test2.txt", Buffer.from("hello2"), workspaceID) + await g.writeFileInWorkspace("test1/test3.txt", Buffer.from("hello3"), workspaceID) + await g.writeFileInWorkspace(".hidden.txt", Buffer.from("hidden"), workspaceID) + + let content = await g.listFilesInWorkspace(undefined, workspaceID) + expect(content.length).toEqual(4) + expect(content).toContain("test1/test2.txt") + expect(content).toContain("test1/test3.txt") + expect(content).toContain("test/test1.txt") + expect(content).toContain(".hidden.txt") + + content = await g.listFilesInWorkspace("test1", workspaceID) + expect(content.length).toEqual(2) + expect(content).toContain("test1/test2.txt") + expect(content).toContain("test1/test3.txt") + + await g.removeAll("test1", workspaceID) + + content = await g.listFilesInWorkspace("", workspaceID) + expect(content.length).toEqual(2) + expect(content).toContain("test/test1.txt") + expect(content).toContain(".hidden.txt") + + await g.deleteWorkspace(workspaceID) + }, 60000) })