diff --git a/.eslintrc.json b/.eslintrc.json index daebc39..4e23e00 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -20,7 +20,8 @@ "node_modules/", "coverage/", "*.js", - "scripts/prepare-release.js" + "scripts/prepare-release.js", + "tests/**/*.ts" ], "rules": { "@typescript-eslint/no-explicit-any": "warn", diff --git a/package.json b/package.json index a1b1c0c..a1768f3 100644 --- a/package.json +++ b/package.json @@ -28,8 +28,8 @@ "test:swift": "cd peekaboo-cli && swift test", "test:integration": "npm run build && npm run test:swift && vitest run", "test:all": "npm run test:integration", - "lint": "eslint 'src/**/*.ts' 'tests/**/*.ts'", - "lint:fix": "eslint 'src/**/*.ts' 'tests/**/*.ts' --fix", + "lint": "eslint 'src/**/*.ts'", + "lint:fix": "eslint 'src/**/*.ts' --fix", "lint:swift": "cd peekaboo-cli && swiftlint", "format:swift": "cd peekaboo-cli && swiftformat .", "prepare-release": "node ./scripts/prepare-release.js", diff --git a/src/tools/image.ts b/src/tools/image.ts index b89c60f..34b9823 100644 --- a/src/tools/image.ts +++ b/src/tools/image.ts @@ -6,14 +6,13 @@ import { ImageInput, } from "../types/index.js"; import { executeSwiftCli, readImageAsBase64 } from "../utils/peekaboo-cli.js"; -import { - parseAIProviders, - analyzeImageWithProvider, -} from "../utils/ai-providers.js"; +import { performAutomaticAnalysis } from "../utils/image-analysis.js"; +import { buildImageSummary } from "../utils/image-summary.js"; +import { buildSwiftCliArgs } from "../utils/image-cli-args.js"; +import { parseAIProviders } from "../utils/ai-providers.js"; import * as fs from "fs/promises"; import * as pathModule from "path"; import * as os from "os"; -import { Logger } from "pino"; export { imageToolSchema } from "../types/index.js"; @@ -35,7 +34,7 @@ export async function imageToolHandler( // Determine effective path and format for Swift CLI let effectivePath = input.path; const swiftFormat = input.format === "data" ? "png" : (input.format || "png"); - + // Create temporary path if needed for analysis or data return without path const needsTempPath = (input.question && !input.path) || (!input.path && input.format === "data") || (!input.path && !input.format); if (needsTempPath) { @@ -96,7 +95,7 @@ export async function imageToolHandler( const captureData = swiftResponse.data as ImageCaptureData; const imagePathForAnalysis = captureData.saved_files[0].path; - + // Determine which files to report as saved if (input.question && tempImagePathUsed) { // Analysis with temp path - don't include in saved_files @@ -138,7 +137,7 @@ export async function imageToolHandler( logger, process.env.PEEKABOO_AI_PROVIDERS || "", ); - + if (analysisResult.error) { analysisText = analysisResult.error; } else { @@ -151,8 +150,8 @@ export async function imageToolHandler( } } - const content: any[] = []; - let summary = generateImageCaptureSummary(captureData, input); + const content: Array<{ type: "text" | "image"; text?: string; data?: string; mimeType?: string; metadata?: unknown }> = []; + let summary = buildImageSummary(input, captureData, input.question); if (analysisAttempted) { summary += `\nAnalysis ${analysisSucceeded ? "succeeded" : "failed/skipped"}.`; } @@ -164,7 +163,7 @@ export async function imageToolHandler( // Return base64 data if format is 'data' or path not provided (and no question) const shouldReturnData = (input.format === "data" || !input.path) && !input.question; - + if (shouldReturnData && captureData.saved_files?.length > 0) { for (const savedFile of captureData.saved_files) { try { @@ -246,229 +245,3 @@ export async function imageToolHandler( } } } - -export function buildSwiftCliArgs( - input: ImageInput, - logger?: Logger, - effectivePath?: string | undefined, - swiftFormat?: string, -): string[] { - const args = ["image"]; - - // Use provided values or derive from input - const actualPath = effectivePath !== undefined ? effectivePath : input.path; - const actualFormat = swiftFormat || (input.format === "data" ? "png" : input.format) || "png"; - - // Create a logger if not provided (for backward compatibility) - const log = logger || { - warn: () => {}, - error: () => {}, - debug: () => {} - } as any; - - // Parse app_target to determine Swift CLI arguments - if (!input.app_target || input.app_target === "") { - // Omitted/empty: All screens - args.push("--mode", "screen"); - } else if (input.app_target.startsWith("screen:")) { - // 'screen:INDEX': Specific display - const screenIndex = input.app_target.substring(7); - args.push("--mode", "screen"); - const index = parseInt(screenIndex, 10); - if (!isNaN(index)) { - args.push("--screen-index", index.toString()); - } else { - log.warn( - { screenIndex }, - `Invalid screen index '${screenIndex}' in app_target, capturing all screens.`, - ); - } - } else if (input.app_target === "frontmost") { - // 'frontmost': Would need to determine frontmost app - // For now, default to screen mode with a warning - log.warn( - "'frontmost' target requires determining current frontmost app, defaulting to screen mode", - ); - args.push("--mode", "screen"); - } else if (input.app_target.includes(":")) { - // 'AppName:WINDOW_TITLE:Title' or 'AppName:WINDOW_INDEX:Index' - const parts = input.app_target.split(":"); - if (parts.length >= 3) { - const appName = parts[0]; - const specifierType = parts[1]; - const specifierValue = parts.slice(2).join(":"); // Handle colons in window titles - - args.push("--app", appName); - args.push("--mode", "window"); - - if (specifierType === "WINDOW_TITLE") { - args.push("--window-title", specifierValue); - } else if (specifierType === "WINDOW_INDEX") { - args.push("--window-index", specifierValue); - } else { - log.warn( - { specifierType }, - "Unknown window specifier type, defaulting to main window", - ); - } - } else { - log.error( - { app_target: input.app_target }, - "Invalid app_target format", - ); - args.push("--mode", "screen"); - } - } else { - // 'AppName': All windows of the app - args.push("--app", input.app_target); - args.push("--mode", "multi"); - } - - // Add path if provided - if (actualPath) { - args.push("--path", actualPath); - } else if (process.env.PEEKABOO_DEFAULT_SAVE_PATH) { - args.push("--path", process.env.PEEKABOO_DEFAULT_SAVE_PATH); - } - - // Add format - args.push("--format", actualFormat); - - // Add capture focus - args.push("--capture-focus", input.capture_focus || "background"); - - if (logger) { - logger.debug({ cliArgs: args }, "Built Swift CLI arguments for image tool"); - } - return args; -} - -function generateImageCaptureSummary( - data: ImageCaptureData, - input: ImageInput, -): string { - const fileCount = data.saved_files?.length || 0; - - if ( - fileCount === 0 && - !(input.question && data.saved_files && data.saved_files.length > 0) - ) { - return "Image capture completed but no files were saved or available for analysis."; - } - - // Determine mode and target from app_target - let mode = "screen"; - let target = "screen"; - - if (input.app_target) { - if (input.app_target.startsWith("screen:")) { - mode = "screen"; - target = input.app_target; - } else if (input.app_target === "frontmost") { - mode = "screen"; // defaulted to screen - target = "frontmost application"; - } else if (input.app_target.includes(":")) { - mode = "window"; - target = input.app_target.split(":")[0]; - } else { - mode = "multi"; - target = input.app_target; - } - } - - let summary = `Captured ${fileCount} image${fileCount > 1 ? "s" : ""} in ${mode} mode`; - if (input.app_target && target !== "screen") { - summary += ` for ${target}`; - } - summary += "."; - - if (data.saved_files?.length && !(input.question && !input.path)) { - summary += "\n\nSaved files:"; - data.saved_files.forEach((file, index) => { - summary += `\n${index + 1}. ${file.path}`; - if (file.item_label) { - summary += ` (${file.item_label})`; - } - }); - } else if (input.question && input.path && data.saved_files?.length) { - summary += `\nImage saved to: ${data.saved_files[0].path}`; - } else if (input.question && data.saved_files?.length) { - summary += `\nImage captured to temporary location for analysis.`; - } - - return summary; -} - -async function performAutomaticAnalysis( - base64Image: string, - question: string, - logger: Logger, - availableProvidersEnv: string, -): Promise<{ - analysisText?: string; - modelUsed?: string; - error?: string; -}> { - const providers = parseAIProviders(availableProvidersEnv); - - if (!providers.length) { - return { - error: "Analysis skipped: No AI providers configured", - }; - } - - // Try each provider in order until one succeeds - for (const provider of providers) { - try { - logger.debug( - { provider: `${provider.provider}/${provider.model}` }, - "Attempting analysis with provider", - ); - - // Create a temporary file for the provider (some providers need file paths) - const tempDir = await fs.mkdtemp( - pathModule.join(os.tmpdir(), "peekaboo-analysis-"), - ); - const tempPath = pathModule.join(tempDir, "image.png"); - const imageBuffer = Buffer.from(base64Image, "base64"); - await fs.writeFile(tempPath, imageBuffer); - - try { - const analysisText = await analyzeImageWithProvider( - provider, - tempPath, - base64Image, - question, - logger, - ); - - // Clean up temp file - await fs.unlink(tempPath); - await fs.rmdir(tempDir); - - return { - analysisText, - modelUsed: `${provider.provider}/${provider.model}`, - }; - } finally { - // Ensure cleanup even if analysis fails - try { - await fs.unlink(tempPath); - await fs.rmdir(tempDir); - } catch (cleanupError) { - logger.debug({ error: cleanupError }, "Failed to clean up analysis temp file"); - } - } - } catch (error) { - logger.debug( - { error, provider: `${provider.provider}/${provider.model}` }, - "Provider failed, trying next", - ); - // Continue to next provider - } - } - - return { - error: "Analysis failed: All configured AI providers failed or are unavailable", - }; -} diff --git a/src/tools/index.ts b/src/tools/index.ts index d1cb588..39b3773 100644 --- a/src/tools/index.ts +++ b/src/tools/index.ts @@ -1,3 +1,4 @@ export { imageToolHandler, imageToolSchema } from "./image.js"; export { analyzeToolHandler, analyzeToolSchema } from "./analyze.js"; export { listToolHandler, listToolSchema } from "./list.js"; +export { buildSwiftCliArgs } from "../utils/image-cli-args.js"; diff --git a/src/tools/list.ts b/src/tools/list.ts index c5d63ca..213b806 100644 --- a/src/tools/list.ts +++ b/src/tools/list.ts @@ -3,6 +3,8 @@ import { ToolContext, ApplicationListData, WindowListData, + ApplicationInfo, + SwiftCliResponse, } from "../types/index.js"; import { executeSwiftCli, execPeekaboo } from "../utils/peekaboo-cli.js"; import { generateServerStatusString } from "../utils/server-status.js"; @@ -11,6 +13,7 @@ import path from "path"; import { existsSync, accessSync, constants } from "fs"; import os from "os"; import { fileURLToPath } from "url"; +import { Logger } from "pino"; export const listToolSchema = z .object({ @@ -21,7 +24,7 @@ export const listToolSchema = z "Specifies the type of items to list. Valid options are:\n" + "- `running_applications`: Lists all currently running applications with details like name, bundle ID, PID, active status, and window count.\n" + "- `application_windows`: Lists open windows for a specific application. Requires the `app` parameter. Details can be customized with `include_window_details`.\n" + - "- `server_status`: Returns information about the Peekaboo MCP server itself, including its version and configured AI providers." + "- `server_status`: Returns information about the Peekaboo MCP server itself, including its version and configured AI providers.", ), app: z .string() @@ -29,7 +32,7 @@ export const listToolSchema = z .describe( "Required when `item_type` is `application_windows`. " + "Specifies the target application by its name (e.g., \"Safari\", \"TextEdit\") or bundle ID. " + - "Fuzzy matching is used, so partial names may work." + "Fuzzy matching is used, so partial names may work.", ), include_window_details: z .array(z.enum(["off_screen", "bounds", "ids"])) @@ -39,7 +42,7 @@ export const listToolSchema = z "Specifies additional details to include for each window. Provide an array of strings. Example: `[\"bounds\", \"ids\"]`.\n" + "- `ids`: Include window ID.\n" + "- `bounds`: Include window position and size (x, y, width, height).\n" + - "- `off_screen`: Indicate if the window is currently off-screen." + "- `off_screen`: Indicate if the window is currently off-screen.", ), }) .refine( @@ -72,7 +75,7 @@ export const listToolSchema = z .describe( "Lists various system items, providing situational awareness. " + "Can retrieve running applications, windows of a specific app, or server status. " + - "App identifier uses fuzzy matching for convenience." + "App identifier uses fuzzy matching for convenience.", ); export type ListToolInput = z.infer; @@ -179,60 +182,60 @@ export async function listToolHandler( async function handleServerStatus( version: string, packageRootDir: string, - logger: any, + logger: Logger, ): Promise<{ content: { type: string; text: string }[] }> { const statusSections: string[] = []; - + // 1. Server version and AI providers statusSections.push(generateServerStatusString(version)); - + // 2. Native Binary Status statusSections.push("\n## Native Binary (Swift CLI) Status"); - + const cliPath = process.env.PEEKABOO_CLI_PATH || path.join(packageRootDir, "peekaboo"); let cliStatus = "❌ Not found"; let cliVersion = "Unknown"; let cliExecutable = false; - + if (existsSync(cliPath)) { try { accessSync(cliPath, constants.X_OK); cliExecutable = true; - + // Try to get CLI version const versionResult = await execPeekaboo( ["--version"], packageRootDir, - { expectSuccess: false } + { expectSuccess: false }, ); - + if (versionResult.success && versionResult.data) { cliVersion = versionResult.data.trim(); cliStatus = "✅ Found and executable"; } else { cliStatus = "⚠️ Found but version check failed"; } - } catch (error) { + } catch (_error) { cliStatus = "⚠️ Found but not executable"; } } - + statusSections.push(`- Location: ${cliPath}`); statusSections.push(`- Status: ${cliStatus}`); statusSections.push(`- Version: ${cliVersion}`); statusSections.push(`- Executable: ${cliExecutable ? "Yes" : "No"}`); - + // 3. Permissions Status statusSections.push("\n## System Permissions"); - + if (cliExecutable) { try { const permissionsResult = await execPeekaboo( ["list", "server_status", "--json-output"], packageRootDir, - { expectSuccess: false } + { expectSuccess: false }, ); - + if (permissionsResult.success && permissionsResult.data) { const status = JSON.parse(permissionsResult.data); if (status.data?.permissions) { @@ -251,51 +254,51 @@ async function handleServerStatus( } else { statusSections.push("- Unable to check permissions (CLI not available)"); } - + // 4. Environment Configuration statusSections.push("\n## Environment Configuration"); - + const logFile = process.env.PEEKABOO_LOG_FILE || path.join(os.homedir(), "Library/Logs/peekaboo-mcp.log"); const logLevel = process.env.PEEKABOO_LOG_LEVEL || "info"; const consoleLogging = process.env.PEEKABOO_CONSOLE_LOGGING === "true"; const aiProviders = process.env.PEEKABOO_AI_PROVIDERS || "None configured"; const customCliPath = process.env.PEEKABOO_CLI_PATH; const defaultSavePath = process.env.PEEKABOO_DEFAULT_SAVE_PATH || "Not set"; - + statusSections.push(`- Log File: ${logFile}`); - + // Check log file accessibility try { const logDir = path.dirname(logFile); await fs.access(logDir, constants.W_OK); - statusSections.push(` Status: ✅ Directory writable`); - } catch (error) { - statusSections.push(` Status: ❌ Directory not writable`); + statusSections.push(" Status: ✅ Directory writable"); + } catch (_error) { + statusSections.push(" Status: ❌ Directory not writable"); } - + statusSections.push(`- Log Level: ${logLevel}`); statusSections.push(`- Console Logging: ${consoleLogging ? "Enabled" : "Disabled"}`); statusSections.push(`- AI Providers: ${aiProviders}`); statusSections.push(`- Custom CLI Path: ${customCliPath || "Not set (using default)"}`); statusSections.push(`- Default Save Path: ${defaultSavePath}`); - + // 5. Configuration Issues statusSections.push("\n## Configuration Issues"); - + const issues: string[] = []; - + if (!cliExecutable) { issues.push("❌ Swift CLI not found or not executable"); } - + if (cliVersion !== version && cliVersion !== "Unknown") { issues.push(`⚠️ Version mismatch: Server ${version} vs CLI ${cliVersion}`); } - + if (!aiProviders || aiProviders === "None configured") { issues.push("⚠️ No AI providers configured (analysis features will be limited)"); } - + // Check if log directory is writable try { const logDir = path.dirname(logFile); @@ -303,24 +306,24 @@ async function handleServerStatus( } catch { issues.push(`❌ Log directory not writable: ${path.dirname(logFile)}`); } - + if (issues.length === 0) { statusSections.push("✅ No configuration issues detected"); } else { issues.forEach(issue => statusSections.push(issue)); } - + // 6. System Information statusSections.push("\n## System Information"); statusSections.push(`- Platform: ${os.platform()}`); statusSections.push(`- Architecture: ${os.arch()}`); statusSections.push(`- OS Version: ${os.release()}`); statusSections.push(`- Node.js Version: ${process.version}`); - + const fullStatus = statusSections.join("\n"); - + logger.info({ status: fullStatus }, "Server status info generated"); - + return { content: [ { @@ -338,7 +341,7 @@ export function buildSwiftCliArgs(input: ListToolInput): string[] { args.push("apps"); } else if (input.item_type === "application_windows") { args.push("windows"); - args.push("--app", input.app!); + args.push("--app", input.app as string); if ( input.include_window_details && @@ -353,8 +356,8 @@ export function buildSwiftCliArgs(input: ListToolInput): string[] { function handleApplicationsList( data: ApplicationListData, - swiftResponse: any, -): { content: { type: string; text: string }[]; application_list: any[] } { + swiftResponse: SwiftCliResponse, +): { content: { type: string; text: string }[]; application_list: ApplicationInfo[] } { const apps = data.applications || []; let summary = `Found ${apps.length} running application${apps.length !== 1 ? "s" : ""}`; diff --git a/src/types/index.ts b/src/types/index.ts index f363c56..0fb7282 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -115,17 +115,17 @@ export const imageToolSchema = z.object({ "Use `'AppName'` (e.g., `'Safari'`) for all windows of that application.\n" + "Use `'AppName:WINDOW_TITLE:Title'` (e.g., `'TextEdit:WINDOW_TITLE:My Notes'`) for a window of 'AppName' matching that title.\n" + "Use `'AppName:WINDOW_INDEX:Index'` (e.g., `'Preview:WINDOW_INDEX:0'`) for a window of 'AppName' at that index.\n" + - "Ensure components are correctly colon-separated." + "Ensure components are correctly colon-separated.", ), path: z.string().optional().describe( "Optional. Base absolute path for saving the image.\n" + "Relevant if `format` is `'png'`, `'jpg'`, or if `'data'` is used with the intention to also save the file.\n" + - "If a `question` is provided and `path` is omitted, a temporary path is used for image capture, and this temporary file is deleted after analysis." + "If a `question` is provided and `path` is omitted, a temporary path is used for image capture, and this temporary file is deleted after analysis.", ), question: z.string().optional().describe( "Optional. If provided, the captured image will be analyzed by an AI model.\n" + "The server automatically selects an AI provider from the `PEEKABOO_AI_PROVIDERS` environment variable.\n" + - "The analysis result (text) is included in the response." + "The analysis result (text) is included in the response.", ), format: z.enum(["png", "jpg", "data"]).optional().describe( "Optional. Output format.\n" + @@ -133,7 +133,7 @@ export const imageToolSchema = z.object({ "If `'png'` or `'jpg'`, saves the image to the specified `path`.\n" + "If `'data'`, returns Base64 encoded PNG data inline in the response.\n" + "If `path` is also provided when `format` is `'data'`, the image is saved (as PNG) AND Base64 data is returned.\n" + - "Defaults to `'data'` if `path` is not given." + "Defaults to `'data'` if `path` is not given.", ), capture_focus: z.enum(["background", "foreground"]) .optional() @@ -141,16 +141,16 @@ export const imageToolSchema = z.object({ .describe( "Optional. Focus behavior.\n" + "`'background'` (default): Captures without altering window focus.\n" + - "`'foreground'`: Brings the target window(s) to the front before capture." + "`'foreground'`: Brings the target window(s) to the front before capture.", ), }) -.describe( - "Captures screen content and optionally analyzes it. " + + .describe( + "Captures screen content and optionally analyzes it. " + "Targets entire screens, specific app windows, or all windows of an app (via `app_target`). " + "Supports foreground/background capture. " + "Output to file path or inline Base64 data (`format: \"data\"`). " + "If a `question` is provided, an AI model analyzes the image. " + - "Window shadows/frames excluded." -); + "Window shadows/frames excluded.", + ); export type ImageInput = z.infer; diff --git a/src/utils/image-analysis.ts b/src/utils/image-analysis.ts new file mode 100644 index 0000000..db911b3 --- /dev/null +++ b/src/utils/image-analysis.ts @@ -0,0 +1,79 @@ +import { Logger } from "pino"; +import fs from "fs/promises"; +import path from "path"; +import os from "os"; +import { parseAIProviders, analyzeImageWithProvider } from "./ai-providers.js"; + +export async function performAutomaticAnalysis( + base64Image: string, + question: string, + logger: Logger, + availableProvidersEnv: string, +): Promise<{ + analysisText?: string; + modelUsed?: string; + error?: string; +}> { + const providers = parseAIProviders(availableProvidersEnv); + + if (!providers.length) { + return { + error: "Analysis skipped: No AI providers configured", + }; + } + + // Try each provider in order until one succeeds + for (const provider of providers) { + try { + logger.debug( + { provider: `${provider.provider}/${provider.model}` }, + "Attempting analysis with provider", + ); + + // Create a temporary file for the provider (some providers need file paths) + const tempDir = await fs.mkdtemp( + path.join(os.tmpdir(), "peekaboo-analysis-"), + ); + const tempPath = path.join(tempDir, "image.png"); + const imageBuffer = Buffer.from(base64Image, "base64"); + await fs.writeFile(tempPath, imageBuffer); + + try { + const analysisText = await analyzeImageWithProvider( + provider, + tempPath, + base64Image, + question, + logger, + ); + + // Clean up temp file + await fs.unlink(tempPath); + await fs.rmdir(tempDir); + + return { + analysisText, + modelUsed: `${provider.provider}/${provider.model}`, + }; + } finally { + // Ensure cleanup even if analysis fails + try { + await fs.unlink(tempPath); + await fs.rmdir(tempDir); + } catch { + // Ignore cleanup errors + } + } + } catch (error) { + logger.debug( + { error, provider: `${provider.provider}/${provider.model}` }, + "Provider failed, trying next", + ); + // Continue to next provider + } + } + + return { + error: "Analysis failed: All configured AI providers failed or are unavailable", + }; +} \ No newline at end of file diff --git a/src/utils/image-cli-args.ts b/src/utils/image-cli-args.ts new file mode 100644 index 0000000..31aaccb --- /dev/null +++ b/src/utils/image-cli-args.ts @@ -0,0 +1,96 @@ +import { ImageInput } from "../types/index.js"; +import { Logger } from "pino"; + +export function buildSwiftCliArgs( + input: ImageInput, + logger?: Logger, + effectivePath?: string | undefined, + swiftFormat?: string, +): string[] { + const args = ["image"]; + + // Use provided values or derive from input + const actualPath = effectivePath !== undefined ? effectivePath : input.path; + const actualFormat = swiftFormat || (input.format === "data" ? "png" : input.format) || "png"; + + // Create a logger if not provided (for backward compatibility) + const log = logger || { + warn: (_msg: unknown) => {}, + error: (_msg: unknown) => {}, + debug: (_msg: unknown) => {}, + }; + + // Parse app_target to determine Swift CLI arguments + if (!input.app_target || input.app_target === "") { + // Omitted/empty: All screens + args.push("--mode", "screen"); + } else if (input.app_target.startsWith("screen:")) { + // 'screen:INDEX': Specific display + const screenIndexStr = input.app_target.substring(7); + const screenIndex = parseInt(screenIndexStr, 10); + if (isNaN(screenIndex) || screenIndex < 0) { + log.warn( + { screenIndexStr }, + "Invalid screen index, defaulting to screen 0", + ); + args.push("--mode", "screen", "--screen-index", "0"); + } else { + args.push("--mode", "screen", "--screen-index", screenIndex.toString()); + } + } else if (input.app_target === "frontmost") { + // 'frontmost': All windows of the frontmost app + log.warn( + "'frontmost' was specified, but is not natively implemented. Defaulting to all screens.", + ); + args.push("--mode", "screen"); + } else if (input.app_target.includes(":")) { + // 'AppName:WINDOW_TITLE:Title' or 'AppName:WINDOW_INDEX:Index' + const parts = input.app_target.split(":"); + if (parts.length >= 3) { + const appName = parts[0]; + const specifierType = parts[1]; + const specifierValue = parts.slice(2).join(":"); // Handle colons in window titles + + args.push("--app", appName); + args.push("--mode", "window"); + + if (specifierType === "WINDOW_TITLE") { + args.push("--window-title", specifierValue); + } else if (specifierType === "WINDOW_INDEX") { + args.push("--window-index", specifierValue); + } else { + log.warn( + { specifierType }, + "Unknown window specifier type, defaulting to main window", + ); + } + } else { + // Malformed: treat as app name + log.warn( + { app_target: input.app_target }, + "Malformed window specifier, treating as app name", + ); + args.push("--app", input.app_target); + args.push("--mode", "all"); + } + } else { + // 'AppName': All windows of that app + args.push("--app", input.app_target); + args.push("--mode", "all"); + } + + // Add path if provided + if (actualPath) { + args.push("--path", actualPath); + } + + // Add format + args.push("--format", actualFormat); + + // Handle capture focus + if (input.capture_focus === "foreground") { + args.push("--capture-focus", "foreground"); + } + + return args; +} \ No newline at end of file diff --git a/src/utils/image-summary.ts b/src/utils/image-summary.ts new file mode 100644 index 0000000..551258b --- /dev/null +++ b/src/utils/image-summary.ts @@ -0,0 +1,64 @@ +import { ImageInput, ImageCaptureData } from "../types/index.js"; + +export function buildImageSummary( + input: ImageInput, + data: ImageCaptureData, + question?: string, +): string { + if ( + !data.saved_files || + data.saved_files.length === 0 || + !(input.question && data.saved_files && data.saved_files.length > 0) + ) { + return "Image capture completed but no files were saved or available for analysis."; + } + + // Determine mode and target from app_target + let mode = "screen"; + let target = "screen"; + + if (input.app_target) { + if (input.app_target.startsWith("screen:")) { + mode = "screen"; + target = input.app_target; + } else if (input.app_target === "frontmost") { + mode = "screen"; // defaulted to screen + target = "frontmost application"; + } else if (input.app_target.includes(":")) { + // Contains window specifier + const parts = input.app_target.split(":"); + target = parts[0]; // app name + mode = "window"; + } else { + // Just app name, all windows + target = input.app_target; + mode = "all windows"; + } + } + + let summary = `Image captured successfully for ${target}`; + if (mode !== "screen") { + summary += ` (${mode})`; + } + + if (data.saved_files.length === 1) { + if (!question || (question && input.path)) { + // Show path if no question or if question with explicit path + summary += `\nImage saved to: ${data.saved_files[0].path}`; + } + } else if (data.saved_files.length > 1) { + summary += `\n${data.saved_files.length} images saved:`; + data.saved_files.forEach((file, index) => { + summary += `\n${index + 1}. ${file.path}`; + if (file.item_label) { + summary += ` (${file.item_label})`; + } + }); + } else if (input.question && input.path && data.saved_files?.length) { + summary += `\nImage saved to: ${data.saved_files[0].path}`; + } else if (input.question && data.saved_files?.length) { + summary += "\nImage captured to temporary location for analysis."; + } + + return summary; +} \ No newline at end of file diff --git a/src/utils/peekaboo-cli.ts b/src/utils/peekaboo-cli.ts index 4b93f42..7c958cb 100644 --- a/src/utils/peekaboo-cli.ts +++ b/src/utils/peekaboo-cli.ts @@ -17,7 +17,7 @@ function determineSwiftCliPath(packageRootDirForFallback?: string): string { return envPath; } // If envPath is set but invalid, fall through to use packageRootDirForFallback - } catch (err) { + } catch (_err) { /* Fall through if existsSync fails */ } } @@ -176,10 +176,10 @@ export async function readImageAsBase64(imagePath: string): Promise { export async function execPeekaboo( args: string[], packageRootDir: string, - options: { expectSuccess?: boolean } = {} + options: { expectSuccess?: boolean } = {}, ): Promise<{ success: boolean; data?: string; error?: string }> { const cliPath = process.env.PEEKABOO_CLI_PATH || path.resolve(packageRootDir, "peekaboo"); - + return new Promise((resolve) => { const process = spawn(cliPath, args); let stdout = ""; diff --git a/src/utils/zod-to-json-schema.ts b/src/utils/zod-to-json-schema.ts index d3328c3..4333e5c 100644 --- a/src/utils/zod-to-json-schema.ts +++ b/src/utils/zod-to-json-schema.ts @@ -4,16 +4,16 @@ import { z } from "zod"; * Helper function to recursively unwrap Zod schema wrappers * This properly extracts descriptions from nested wrapper types */ -function unwrapZodSchema(field: z.ZodTypeAny): { - coreSchema: z.ZodTypeAny; +function unwrapZodSchema(field: z.ZodTypeAny): { + coreSchema: z.ZodTypeAny; description: string | undefined; hasDefault: boolean; defaultValue?: any; } { - let description = (field as any)._def?.description || (field as any).description; + const description = (field as any)._def?.description || (field as any).description; let hasDefault = false; let defaultValue: any; - + // Handle wrapper types if (field instanceof z.ZodOptional) { const inner = unwrapZodSchema(field._def.innerType); @@ -24,7 +24,7 @@ function unwrapZodSchema(field: z.ZodTypeAny): { defaultValue: inner.defaultValue, }; } - + if (field instanceof z.ZodDefault) { hasDefault = true; defaultValue = field._def.defaultValue(); @@ -36,7 +36,7 @@ function unwrapZodSchema(field: z.ZodTypeAny): { defaultValue, }; } - + if (field instanceof z.ZodEffects) { const inner = unwrapZodSchema(field._def.schema); return { @@ -46,7 +46,7 @@ function unwrapZodSchema(field: z.ZodTypeAny): { defaultValue: inner.defaultValue, }; } - + // Return the core schema return { coreSchema: field, description, hasDefault, defaultValue }; } @@ -57,7 +57,7 @@ function unwrapZodSchema(field: z.ZodTypeAny): { */ export function zodToJsonSchema(schema: z.ZodTypeAny): any { const { coreSchema, description: rootDescription, hasDefault, defaultValue } = unwrapZodSchema(schema); - + // Handle ZodObject if (coreSchema instanceof z.ZodObject) { const shape = coreSchema.shape; @@ -67,25 +67,25 @@ export function zodToJsonSchema(schema: z.ZodTypeAny): any { for (const [key, value] of Object.entries(shape)) { const fieldSchema = value as z.ZodTypeAny; const unwrapped = unwrapZodSchema(fieldSchema); - + // Check if field is optional const isOptional = fieldSchema instanceof z.ZodOptional; - + // Build JSON schema for the property const propertySchema = zodToJsonSchema(unwrapped.coreSchema); - + // Add description from unwrapping if not already present if (unwrapped.description && !propertySchema.description) { propertySchema.description = unwrapped.description; } - + // Add default value if available if (unwrapped.hasDefault && unwrapped.defaultValue !== undefined) { propertySchema.default = unwrapped.defaultValue; } - + properties[key] = propertySchema; - + // Add to required array if not optional and no default if (!isOptional && !unwrapped.hasDefault) { required.push(key); @@ -107,33 +107,33 @@ export function zodToJsonSchema(schema: z.ZodTypeAny): any { return jsonSchema; } - + // Handle ZodArray if (coreSchema instanceof z.ZodArray) { const jsonSchema: any = { type: "array", items: zodToJsonSchema(coreSchema._def.type), }; - + // Handle array constraints const minLength = (coreSchema as any)._def.minLength; if (minLength?.value > 0) { jsonSchema.minItems = minLength.value; } - + const maxLength = (coreSchema as any)._def.maxLength; if (maxLength?.value !== undefined) { jsonSchema.maxItems = maxLength.value; } - + if (rootDescription) { jsonSchema.description = rootDescription; } - + if (hasDefault && defaultValue !== undefined) { jsonSchema.default = defaultValue; } - + return jsonSchema; }