Add playwright/cypress/puppeteer code dumping

examples/2048_recorder.ts

@@ -0,0 +1,106 @@
+// This is an example of how to use the recorder to record actions and then generate a test from the recorded actions.
+// Key Differences:
+// - It replaces a playwright call in the original 2048 with a stagehand act that gets recorded.
+// - It only runs the play loop one time.
+// - It logs the playwright code to the console at the end of the example.
+
+import { Stagehand } from "../lib";
+import { z } from "zod";
+async function example() {
+  console.log("🎮 Starting 2048 bot...");
+  const stagehand = new Stagehand({
+    env: "LOCAL",
+    verbose: 1,
+    debugDom: true,
+    domSettleTimeoutMs: 100,
+    enableRecording: true,
+  });
+  try {
+    console.log("🌟 Initializing Stagehand...");
+    await stagehand.init();
+    console.log("🌐 Navigating to 2048...");
+    await stagehand.page.goto("https://ovolve.github.io/2048-AI/");
+    console.log("⌛ Waiting for game to initialize...");
+    await stagehand.page.waitForSelector(".grid-container", { timeout: 10000 });
+    // Main game loop
+    while (true) {
+      console.log(`🔄 Game loop iteration...`);
+      // Add a small delay for UI updates
+      await new Promise((resolve) => setTimeout(resolve, 300));
+      // Get current game state
+      const gameState = await stagehand.page.extract({
+        instruction: `Extract the current game state:
+          1. Score from the score counter
+          2. All tile values in the 4x4 grid (empty spaces as 0)
+          3. Highest tile value present`,
+        schema: z.object({
+          score: z.number(),
+          highestTile: z.number(),
+          grid: z.array(z.array(z.number())),
+        }),
+      });
+      const transposedGrid = gameState.grid[0].map((_, colIndex) =>
+        gameState.grid.map((row) => row[colIndex]),
+      );
+      const grid = transposedGrid.map((row, rowIndex) => ({
+        [`row${rowIndex + 1}`]: row,
+      }));
+      console.log("Game State:", {
+        score: gameState.score,
+        highestTile: gameState.highestTile,
+        grid: grid,
+      });
+      // Analyze board and decide next move
+      const analysis = await stagehand.page.extract({
+        instruction: `Based on the current game state:
+          - Score: ${gameState.score}
+          - Highest tile: ${gameState.highestTile}
+          - Grid: This is a 4x4 matrix ordered by row (top to bottom) and column (left to right). The rows are stacked vertically, and tiles can move vertically between rows or horizontally between columns:\n${grid
+            .map((row) => {
+              const rowName = Object.keys(row)[0];
+              return `             ${rowName}: ${row[rowName].join(", ")}`;
+            })
+            .join("\n")}
+          What is the best move (up/down/left/right)? Consider:
+          1. Keeping high value tiles in corners (bottom left, bottom right, top left, top right)
+          2. Maintaining a clear path to merge tiles
+          3. Avoiding moves that could block merges
+          4. Only adjacent tiles of the same value can merge
+          5. Making a move will move all tiles in that direction until they hit a tile of a different value or the edge of the board
+          6. Tiles cannot move past the edge of the board
+          7. Each move must move at least one tile`,
+        schema: z.object({
+          move: z.enum(["up", "down", "left", "right"]),
+          confidence: z.number(),
+          reasoning: z.string(),
+        }),
+      });
+      console.log("Move Analysis:", analysis);
+      await stagehand.page.act({
+        action: `Press the ${analysis.move} key`,
+      });
+      console.log("🎯 Executed move:", analysis.move);
+      break;
+    }
+  } catch (error) {
+    console.error("❌ Error in game loop:", error);
+    const isGameOver = await stagehand.page.evaluate(() => {
+      return document.querySelector(".game-over") !== null;
+    });
+    if (isGameOver) {
+      console.log("🏁 Game Over!");
+      return;
+    }
+    throw error; // Re-throw non-game-over errors
+  }
+
+  const playwrightCode = await stagehand.dumpRecordedActionsCode({
+    testFramework: "playwright",
+    language: "typescript",
+  });
+  console.log("Playwright Code:\n\n", playwrightCode);
+  await stagehand.close();
+}
+(async () => {
+  await example();
+})();

lib/StagehandPage.ts

@@ -9,6 +9,8 @@ import { StagehandActHandler } from "./handlers/actHandler";
 import { StagehandContext } from "./StagehandContext";
 import { Page } from "../types/page";
 import {
+  DumpRecordedActionsCodeOptions,
+  DumpRecordedActionsCodeResult,
   ExtractOptions,
   ExtractResult,
   ObserveOptions,
@@ -65,6 +67,7 @@ export class StagehandPage {
         verbose: this.stagehand.verbose,
         llmProvider: this.stagehand.llmProvider,
         enableCaching: this.stagehand.enableCaching,
+        enableRecording: this.stagehand.enableRecording,
         logger: this.stagehand.logger,
         stagehandPage: this,
         stagehandContext: this.intContext,
@@ -361,6 +364,16 @@ export class StagehandPage {
       });
   }
 
+  async dumpRecordedActionsCode(
+    options: DumpRecordedActionsCodeOptions,
+  ): Promise<DumpRecordedActionsCodeResult> {
+    if (!this.actHandler) {
+      throw new Error("Act handler not initialized");
+    }
+
+    return this.actHandler.dumpRecordedActionsCode(options);
+  }
+
   async extract<T extends z.AnyZodObject>({
     instruction,
     schema,

lib/cache/ActionRecorder.ts

@@ -0,0 +1,214 @@
+import { LogLine } from "../../types/log";
+import { BaseCache, CacheEntry } from "./BaseCache";
+
+export interface PlaywrightCommand {
+  method: string;
+  args: string[];
+}
+
+export interface ActionRecorderEntry extends CacheEntry {
+  data: {
+    url: string;
+    playwrightCommand: PlaywrightCommand;
+    componentString: string;
+    xpaths: string[];
+    newStepString: string;
+    completed: boolean;
+    previousSelectors: string[];
+    action: string;
+  };
+}
+
+/**
+ * ActionRecorder handles logging and retrieving actions along with their Playwright commands for test framework code generation purposes.
+ */
+export class ActionRecorder extends BaseCache<ActionRecorderEntry> {
+  constructor(
+    logger: (message: LogLine) => void,
+    cacheDir?: string,
+    cacheFile?: string,
+  ) {
+    logger({
+      category: "action_recorder",
+      message:
+        "initializing action recorder at " +
+        cacheDir +
+        " with file " +
+        cacheFile,
+      level: 1,
+    });
+    super(logger, cacheDir, cacheFile || "action_recorder.json");
+    this.resetCache();
+  }
+
+  public async addActionStep({
+    url,
+    action,
+    previousSelectors,
+    playwrightCommand,
+    componentString,
+    xpaths,
+    newStepString,
+    completed,
+    requestId,
+  }: {
+    url: string;
+    action: string;
+    previousSelectors: string[];
+    playwrightCommand: PlaywrightCommand;
+    componentString: string;
+    requestId: string;
+    xpaths: string[];
+    newStepString: string;
+    completed: boolean;
+  }): Promise<void> {
+    this.logger({
+      category: "action_recorder",
+      message: "adding action step to recorder",
+      level: 1,
+      auxiliary: {
+        action: {
+          value: action,
+          type: "string",
+        },
+        requestId: {
+          value: requestId,
+          type: "string",
+        },
+        url: {
+          value: url,
+          type: "string",
+        },
+        previousSelectors: {
+          value: JSON.stringify(previousSelectors),
+          type: "object",
+        },
+        playwrightCommand: {
+          value: JSON.stringify(playwrightCommand),
+          type: "object",
+        },
+      },
+    });
+
+    await this.set(
+      { url, action, previousSelectors },
+      {
+        url,
+        playwrightCommand,
+        componentString,
+        xpaths,
+        newStepString,
+        completed,
+        previousSelectors,
+        action,
+      },
+      requestId,
+    );
+  }
+
+  /**
+   * Retrieves all actions for a specific trajectory.
+   * @param trajectoryId - Unique identifier for the trajectory.
+   * @param requestId - The identifier for the current request.
+   * @returns An array of TrajectoryEntry objects or null if not found.
+   */
+  public async getActionStep({
+    url,
+    action,
+    previousSelectors,
+    requestId,
+  }: {
+    url: string;
+    action: string;
+    previousSelectors: string[];
+    requestId: string;
+  }): Promise<ActionRecorderEntry["data"] | null> {
+    const data = await super.get({ url, action, previousSelectors }, requestId);
+    if (!data) {
+      return null;
+    }
+
+    return data;
+  }
+
+  public async removeActionStep(cacheHashObj: {
+    url: string;
+    action: string;
+    previousSelectors: string[];
+    requestId: string;
+  }): Promise<void> {
+    await super.delete(cacheHashObj);
+  }
+
+  /**
+   * Clears all actions for a specific trajectory.
+   * @param trajectoryId - Unique identifier for the trajectory.
+   * @param requestId - The identifier for the current request.
+   */
+  public async clearAction(requestId: string): Promise<void> {
+    await super.deleteCacheForRequestId(requestId);
+    this.logger({
+      category: "action_recorder",
+      message: "cleared action for ID",
+      level: 1,
+      auxiliary: {
+        requestId: {
+          value: requestId,
+          type: "string",
+        },
+      },
+    });
+  }
+
+  /**
+   * Gets all recorded actions sorted by timestamp.
+   * @returns An array of all recorded actions with their data.
+   */
+  public async getAllActions(): Promise<ActionRecorderEntry[]> {
+    if (!(await this.acquireLock())) {
+      this.logger({
+        category: "action_recorder",
+        message: "Failed to acquire lock for getting all actions",
+        level: 2,
+      });
+      return [];
+    }
+
+    try {
+      const cache = this.readCache();
+      const entries = Object.values(cache) as ActionRecorderEntry[];
+      return entries.sort((a, b) => a.timestamp - b.timestamp);
+    } catch (error) {
+      this.logger({
+        category: "action_recorder",
+        message: "Error getting all actions",
+        level: 2,
+        auxiliary: {
+          error: {
+            value: error.message,
+            type: "string",
+          },
+          trace: {
+            value: error.stack,
+            type: "string",
+          },
+        },
+      });
+      return [];
+    } finally {
+      this.releaseLock();
+    }
+  }
+
+  /**
+   * Resets the entire action cache.
+   */
+  public async resetCache(): Promise<void> {
+    await super.resetCache();
+    this.logger({
+      category: "action_recorder",
+      message: "Action recorder has been reset.",
+      level: 1,
+    });
+  }
+}