Annotator (#1861)

<!-- This is an auto-generated description by cubic. -->
## Summary by cubic
Adds an in-app screenshot annotator to the Preview panel for Pro users
so you can capture the current app view, draw or add text, and submit an
annotated image to chat.

- **New Features**
- Pen button in PreviewIframe to toggle annotator; captures a screenshot
via worker messaging and displays it in a Konva canvas.
- Tools: select, freehand draw, and draggable text; supports undo/redo,
delete, and resizing with Transformer. Canvas scales to the container.
Includes a color picker.
- Submit exports a PNG and attaches it to the chat via useAttachments;
prefills the chat input; annotator auto-closes after submit.
  - Pro-only: non-Pro users see an upsell screen.
- State atoms added: annotatorModeAtom, screenshotDataUrlAtom,
attachmentsAtom; PreviewIframe now handles dyad-screenshot-response
messages.

- **Dependencies**
  - Added konva, react-konva, perfect-freehand, and html-to-image.
- Proxy now injects html-to-image and the new dyad-screenshot-client.js
for screenshot capture.

<sup>Written for commit 580aca2.
Summary will update automatically on new commits.</sup>

<!-- End of auto-generated description by cubic. -->

Author: azizmejri1

e2e-tests/annotator.spec.ts

@@ -0,0 +1,75 @@
+import { testSkipIfWindows } from "./helpers/test_helper";
+import { expect } from "@playwright/test";
+import fs from "fs";
+
+testSkipIfWindows(
+  "annotator - capture and submit screenshot",
+  async ({ po }) => {
+    await po.setUpDyadPro({ autoApprove: true });
+
+    // Create a basic app
+    await po.sendPrompt("basic");
+
+    // Click the annotator button to activate annotator mode
+    await po.clickPreviewAnnotatorButton();
+
+    // Wait for annotator mode to be active
+    await po.waitForAnnotatorMode();
+
+    // Submit the screenshot to chat
+    await po.clickAnnotatorSubmit();
+
+    await expect(po.getChatInput()).toContainText(
+      "Please update the UI based on these screenshots",
+    );
+
+    // Verify the screenshot was attached to chat context
+    await po.sendPrompt("[dump]");
+
+    // Wait for the LLM response containing the dump path to appear in the UI
+    // before attempting to extract it from the messages list
+    await po.page.waitForSelector("text=/\\[\\[dyad-dump-path=.*\\]\\]/");
+
+    // Get the dump file path from the messages list
+    const messagesListText = await po.page
+      .getByTestId("messages-list")
+      .textContent();
+    const dumpPathMatch = messagesListText?.match(
+      /\[\[dyad-dump-path=([^\]]+)\]\]/,
+    );
+
+    if (!dumpPathMatch) {
+      throw new Error("No dump path found in messages list");
+    }
+
+    const dumpFilePath = dumpPathMatch[1];
+    const dumpContent = fs.readFileSync(dumpFilePath, "utf-8");
+    const parsedDump = JSON.parse(dumpContent);
+
+    // Get the last message from the dump
+    const messages = parsedDump.body.messages;
+    const lastMessage = messages[messages.length - 1];
+
+    expect(lastMessage).toBeTruthy();
+    expect(lastMessage.content).toBeTruthy();
+
+    // The content is an array with text and image parts
+    expect(Array.isArray(lastMessage.content)).toBe(true);
+
+    // Find the text part and verify it mentions the PNG attachment
+    const textPart = lastMessage.content.find(
+      (part: any) => part.type === "text",
+    );
+    expect(textPart).toBeTruthy();
+    expect(textPart.text).toMatch(/annotated-screenshot-.*\.png/);
+    expect(textPart.text).toMatch(/image\/png/);
+
+    // Find the image part and verify it has the correct structure
+    const imagePart = lastMessage.content.find(
+      (part: any) => part.type === "image_url",
+    );
+    expect(imagePart).toBeTruthy();
+    expect(imagePart.image_url).toBeTruthy();
+    expect(imagePart.image_url.url).toMatch(/^data:image\/png;base64,/);
+  },
+);

e2e-tests/helpers/test_helper.ts

@@ -553,6 +553,22 @@ export class PageObject {
     await this.page.getByTestId("preview-open-browser-button").click();
   }
 
+  async clickPreviewAnnotatorButton() {
+    await this.page
+      .getByTestId("preview-annotator-button")
+      .click({ timeout: Timeout.EXTRA_LONG });
+  }
+
+  async waitForAnnotatorMode() {
+    // Wait for the annotator toolbar to be visible
+    await expect(this.page.getByRole("button", { name: "Select" })).toBeVisible(
+      { timeout: Timeout.MEDIUM },
+    );
+  }
+
+  async clickAnnotatorSubmit() {
+    await this.page.getByRole("button", { name: "Add to Chat" }).click();
+  }
   locateLoadingAppPreview() {
     return this.page.getByText("Preparing app preview...");
   }

forge.config.ts

@@ -32,6 +32,9 @@ const ignore = (file: string) => {
   if (file.startsWith("/node_modules/stacktrace-js/dist")) {
     return false;
   }
+  if (file.startsWith("/node_modules/html-to-image")) {
+    return false;
+  }
   if (file.startsWith("/node_modules/better-sqlite3")) {
     return false;
   }

package-lock.json

@@ -145,17 +145,21 @@
     "framer-motion": "^12.6.3",
     "geist": "^1.3.1",
     "glob": "^11.0.2",
+    "html-to-image": "^1.11.13",
     "isomorphic-git": "^1.30.1",
     "jotai": "^2.12.2",
     "kill-port": "^2.0.1",
+    "konva": "^10.0.12",
     "lexical": "^0.33.1",
     "lexical-beautiful-mentions": "^0.1.47",
     "lucide-react": "^0.487.0",
     "monaco-editor": "^0.52.2",
     "openai": "^4.91.1",
+    "perfect-freehand": "^1.2.2",
     "posthog-js": "^1.236.3",
-    "react": "^19.2.1",
-    "react-dom": "^19.2.1",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "react-konva": "^19.2.1",
     "react-markdown": "^10.1.0",
     "react-resizable-panels": "^2.1.7",
     "react-shiki": "^0.9.0",

package.json

@@ -1,4 +1,4 @@
-import type { Message } from "@/ipc/ipc_types";
+import type { FileAttachment, Message } from "@/ipc/ipc_types";
 import { atom } from "jotai";
 import type { ChatSummary } from "@/lib/schemas";
 
@@ -20,3 +20,5 @@ export const chatsLoadingAtom = atom<boolean>(false);
 // Used for scrolling to the bottom of the chat messages (per chat)
 export const chatStreamCountByIdAtom = atom<Map<number, number>>(new Map());
 export const recentStreamChatIdsAtom = atom<Set<number>>(new Set<number>());
+
+export const attachmentsAtom = atom<FileAttachment[]>([]);

src/atoms/chatAtoms.ts

@@ -15,6 +15,9 @@ export const currentComponentCoordinatesAtom = atom<{
 
 export const previewIframeRefAtom = atom<HTMLIFrameElement | null>(null);
 
+export const annotatorModeAtom = atom<boolean>(false);
+
+export const screenshotDataUrlAtom = atom<string | null>(null);
 export const pendingVisualChangesAtom = atom<Map<string, VisualEditingChange>>(
   new Map(),
 );