From 0ed1191a35a06161823fa7f0240d6f36d6dc3086 Mon Sep 17 00:00:00 2001 From: DjDeveloperr Date: Sat, 9 May 2026 10:56:00 -0400 Subject: [PATCH] Unify DevTools panels and add inspector port discovery --- AGENTS.md | 4 + client/src/api/simulators.ts | 22 + client/src/api/types.ts | 46 + client/src/app/AppShell.tsx | 43 +- client/src/app/uiState.ts | 5 + .../src/features/devtools/DevToolsPanel.tsx | 766 ++++++++ .../src/features/simulators/SimulatorMenu.tsx | 11 - client/src/features/toolbar/Toolbar.tsx | 58 +- .../features/viewport/SimulatorViewport.tsx | 3 + client/src/styles/components.css | 162 ++ client/src/styles/layout.css | 21 + docs/api/rest.md | 110 ++ docs/inspector/index.md | 52 +- package-lock.json | 15 + package.json | 3 + scripts/build-client.sh | 7 + server/Cargo.lock | 24 + server/Cargo.toml | 2 + server/src/api/routes.rs | 341 +++- server/src/devtools.rs | 1567 +++++++++++++++++ server/src/main.rs | 2 + server/src/static_files/mod.rs | 33 +- server/src/webkit.rs | 1271 +++++++++++++ 23 files changed, 4507 insertions(+), 61 deletions(-) create mode 100644 client/src/features/devtools/DevToolsPanel.tsx create mode 100644 server/src/devtools.rs create mode 100644 server/src/webkit.rs diff --git a/AGENTS.md b/AGENTS.md index 7e26c2da..b4a1bf05 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,6 +20,9 @@ The native side should own anything that depends on macOS frameworks, `xcrun sim Defines REST routes for simulator control, health, metrics, and chrome assets. - `server/src/transport/webrtc.rs` Exposes the H.264 WebRTC offer/answer endpoint for browser live video. +- `server/src/webkit.rs` + Discovers simulator WebKit Remote Inspector targets and bridges WebInspectorUI + WebSocket traffic to the simulator `webinspectord` binary-plist socket. - `server/src/simulators/registry.rs` Tracks Rust-side simulator session state and lazy native attachment by UDID. - `cli/XCWSimctl.*` @@ -71,6 +74,7 @@ Private simulator behavior is implemented locally in: The current repo uses the private boot path, private display bridge, and private accessibility translation bridge directly. The browser streams frames from that bridge, injects touch and keyboard events through the same native session layer, inspects accessibility through `AccessibilityPlatformTranslation`, and renders device chrome from `cli/XCWChromeRenderer.*`. Physical chrome button support uses DeviceKit `chrome.json` input geometry for browser hit targets. Volume, action, and mute buttons dispatch through `IndigoHIDMessageForHIDArbitrary` with consumer/telephony HID usage pairs from the device chrome metadata; home, lock, and app-switcher remain on the existing SimulatorKit button paths. +WebKit inspection uses the simulator `webinspectord` Unix socket named `com.apple.webinspectord_sim.socket` and WebKit's binary-plist Remote Inspector selectors. It lists only WebKit content that the runtime exposes as inspectable. For app-owned `WKWebView` on iOS 16.4 and newer, the app must set `isInspectable = true`. ## Build and Run diff --git a/client/src/api/simulators.ts b/client/src/api/simulators.ts index 58a80325..5e0c83b6 100644 --- a/client/src/api/simulators.ts +++ b/client/src/api/simulators.ts @@ -2,11 +2,13 @@ import { apiRequest } from "./client"; import type { AccessibilitySourcePreference, AccessibilityTreeResponse, + ChromeDevToolsTargetDiscovery, ChromeProfile, InspectorRequestResponse, SimulatorLogsResponse, SimulatorMetadata, SimulatorsResponse, + WebKitTargetDiscovery, } from "./types"; export async function listSimulators( @@ -74,6 +76,26 @@ export async function fetchSimulatorLogs( ); } +export async function fetchWebKitTargets( + udid: string, + options: RequestInit = {}, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/webkit/targets`, + options, + ); +} + +export async function fetchChromeDevToolsTargets( + udid: string, + options: RequestInit = {}, +): Promise { + return apiRequest( + `/api/simulators/${encodeURIComponent(udid)}/devtools/targets`, + options, + ); +} + export async function sendInspectorRequest( udid: string, method: string, diff --git a/client/src/api/types.ts b/client/src/api/types.ts index e45e625f..713a0155 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -40,6 +40,52 @@ export interface SimulatorsResponse { simulators: SimulatorMetadata[]; } +export interface WebKitTarget { + id: string; + appId: string; + appName?: string | null; + pageId: number; + title?: string | null; + url?: string | null; + kind: "safari-page" | "app-web-content" | "web-content-proxy" | string; + inspectorUrl: string; + webSocketUrl: string; +} + +export interface WebKitTargetDiscovery { + udid: string; + socketPath?: string | null; + targets: WebKitTarget[]; + warnings: string[]; +} + +export interface ChromeDevToolsTarget { + id: string; + appName?: string | null; + bundleIdentifier?: string | null; + description: string; + devtoolsFrontendUrl: string; + processIdentifier: number; + source: + | "react-native" + | "react-native-metro" + | "chrome-inspector" + | "nativescript" + | "swiftui" + | "in-app-inspector" + | string; + title: string; + type: string; + url: string; + webSocketDebuggerUrl: string; +} + +export interface ChromeDevToolsTargetDiscovery { + udid: string; + targets: ChromeDevToolsTarget[]; + warnings: string[]; +} + export interface HealthResponse { ok: boolean; videoCodec?: string; diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 16cd0b20..f475fccd 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -23,7 +23,6 @@ import { openSimulatorUrl, pressHome, pressSimulatorButton, - rotateLeft, rotateRight, simulatorControlSocketUrl, shutdownSimulator, @@ -41,6 +40,7 @@ import type { TouchPhase, } from "../api/types"; import { AccessibilityInspector } from "../features/accessibility/AccessibilityInspector"; +import { DevToolsPanel } from "../features/devtools/DevToolsPanel"; import { isEditableTarget } from "../features/input/keycodes"; import { useKeyboardInput } from "../features/input/useKeyboardInput"; import { usePointerInput } from "../features/input/usePointerInput"; @@ -86,9 +86,11 @@ import { import { useElementSize } from "../shared/hooks/useElementSize"; import { ACCESSIBILITY_SOURCE_STORAGE_KEY, + CHROME_DEVTOOLS_VISIBLE_STORAGE_KEY, clearLegacyVolatileUiState, DEFAULT_VIEWPORT_STATE, DEBUG_VISIBLE_STORAGE_KEY, + DEVTOOLS_VISIBLE_STORAGE_KEY, HIERARCHY_VISIBLE_STORAGE_KEY, nextAccessibilitySourcePreference, readPersistedUiState, @@ -97,6 +99,7 @@ import { sanitizeAccessibilitySources, TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, viewportStateForUDID, + WEBKIT_INSPECTOR_VISIBLE_STORAGE_KEY, writePersistedUiState, writeStoredFlag, } from "./uiState"; @@ -349,6 +352,12 @@ export function AppShell({ const [hierarchyVisible, setHierarchyVisible] = useState(() => readStoredFlag(HIERARCHY_VISIBLE_STORAGE_KEY), ); + const [devToolsVisible, setDevToolsVisible] = useState( + () => + readStoredFlag(DEVTOOLS_VISIBLE_STORAGE_KEY) || + readStoredFlag(CHROME_DEVTOOLS_VISIBLE_STORAGE_KEY) || + readStoredFlag(WEBKIT_INSPECTOR_VISIBLE_STORAGE_KEY), + ); const [selectedUDID, setSelectedUDID] = useState(initialSelectedUDID ?? ""); const [search, setSearch] = useState(""); const [openURLValue, setOpenURLValue] = useState( @@ -752,10 +761,18 @@ export function AppShell({ writeStoredFlag(HIERARCHY_VISIBLE_STORAGE_KEY, hierarchyVisible); }, [hierarchyVisible]); + useEffect(() => { + writeStoredFlag(DEVTOOLS_VISIBLE_STORAGE_KEY, devToolsVisible); + }, [devToolsVisible]); + useEffect(() => { writeStoredFlag(TOUCH_OVERLAY_VISIBLE_STORAGE_KEY, touchOverlayVisible); }, [touchOverlayVisible]); + const toggleDevTools = useCallback(() => { + setDevToolsVisible((current) => !current); + }, []); + useEffect(() => { window.localStorage.setItem( ACCESSIBILITY_SOURCE_STORAGE_KEY, @@ -1815,20 +1832,6 @@ export function AppShell({ ); } }} - onRotateLeft={() => { - if (!selectedSimulator) { - return; - } - beginZoomAnimation(); - if (sendControl(selectedSimulator.udid, { type: "rotateLeft" })) { - setRotationQuarterTurns((current) => (current + 3) % 4); - return; - } - void runAction(async () => { - await rotateLeft(selectedSimulator.udid); - setRotationQuarterTurns((current) => (current + 3) % 4); - }, false); - }} onOpenBundlePrompt={promptForBundleID} onOpenUrlPrompt={promptForURL} onRotateRight={() => { @@ -1873,6 +1876,7 @@ export function AppShell({ } }} onToggleDebug={() => setDebugVisible((current) => !current)} + onToggleDevTools={toggleDevTools} onToggleHierarchy={() => { setHierarchyVisible((current) => !current); if (hierarchyVisible) { @@ -1902,6 +1906,7 @@ export function AppShell({ selectedSimulator?.isBooted && !selectedSimulatorTransitionKind, )} touchOverlayVisible={touchOverlayVisible} + devToolsVisible={devToolsVisible} /> setDevToolsVisible(false)} + selectedSimulator={selectedSimulator} + /> + ) : null + } zoomDockRef={handleZoomDockRef} zoomAnimating={zoomAnimating} /> diff --git a/client/src/app/uiState.ts b/client/src/app/uiState.ts index 4722d13a..60956b70 100644 --- a/client/src/app/uiState.ts +++ b/client/src/app/uiState.ts @@ -23,6 +23,11 @@ export interface PersistedUiState { export const UI_STATE_STORAGE_KEY = "xcw-ui-state"; export const DEBUG_VISIBLE_STORAGE_KEY = "xcw-debug-visible"; export const HIERARCHY_VISIBLE_STORAGE_KEY = "xcw-hierarchy-visible"; +export const DEVTOOLS_VISIBLE_STORAGE_KEY = "xcw-devtools-visible"; +export const WEBKIT_INSPECTOR_VISIBLE_STORAGE_KEY = + "xcw-webkit-inspector-visible"; +export const CHROME_DEVTOOLS_VISIBLE_STORAGE_KEY = + "xcw-chrome-devtools-visible"; export const ACCESSIBILITY_SOURCE_STORAGE_KEY = "xcw-hierarchy-source"; export const TOUCH_OVERLAY_VISIBLE_STORAGE_KEY = "xcw-touch-overlay-visible"; diff --git a/client/src/features/devtools/DevToolsPanel.tsx b/client/src/features/devtools/DevToolsPanel.tsx new file mode 100644 index 00000000..12db9318 --- /dev/null +++ b/client/src/features/devtools/DevToolsPanel.tsx @@ -0,0 +1,766 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import type { + CSSProperties, + KeyboardEvent as ReactKeyboardEvent, + PointerEvent as ReactPointerEvent, +} from "react"; + +import { accessTokenFromLocation } from "../../api/client"; +import { apiUrl } from "../../api/config"; +import { + fetchChromeDevToolsTargets, + fetchWebKitTargets, +} from "../../api/simulators"; +import type { + ChromeDevToolsTarget, + SimulatorMetadata, + WebKitTarget, +} from "../../api/types"; + +const DEVTOOLS_TARGET_REFRESH_MS = 5000; +const CHROME_DEVTOOLS_REQUEST_TIMEOUT_MS = 6000; +const WEBKIT_DEVTOOLS_REQUEST_TIMEOUT_MS = 2500; +const DEVTOOLS_PANEL_WIDTH_STORAGE_KEY = "xcw-devtools-panel-width"; +const LEGACY_PANEL_WIDTH_STORAGE_KEYS = [ + "xcw-chrome-devtools-panel-width", + "xcw-webkit-panel-width", +]; +const DEVTOOLS_PANEL_DEFAULT_WIDTH = 720; +const DEVTOOLS_PANEL_MIN_WIDTH = 420; +const DEVTOOLS_PANEL_MIN_VIEWPORT_WIDTH = 340; +const DEVTOOLS_PANEL_WIDTH_STEP = 40; + +interface DevToolsPanelProps { + onClose: () => void; + selectedSimulator: SimulatorMetadata | null; +} + +interface ResizeState { + handle: HTMLDivElement; + pointerId: number; + startPointer: number; + startValue: number; +} + +interface DevToolsTarget { + frameUrl: string; + id: string; + meta: string; + source: string; + title: string; +} + +interface DevToolsDiscovery { + targets: DevToolsTarget[]; + warnings: string[]; +} + +export function DevToolsPanel({ + onClose, + selectedSimulator, +}: DevToolsPanelProps) { + const [panelWidth, setPanelWidth] = useState(readStoredPanelWidth); + const [isResizing, setIsResizing] = useState(false); + const [discovery, setDiscovery] = useState(null); + const [selectedTargetId, setSelectedTargetId] = useState(""); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(""); + const [frameLoaded, setFrameLoaded] = useState(false); + const discoveryRef = useRef(null); + const frameRef = useRef(null); + const panelWidthRef = useRef(panelWidth); + const requestIdRef = useRef(0); + const resizeStateRef = useRef(null); + const selectedTargetIdRef = useRef(""); + + const targets = discovery?.targets ?? []; + const selectedTarget = useMemo(() => { + if (targets.length === 0) { + return null; + } + return ( + targets.find((target) => target.id === selectedTargetId) ?? targets[0] + ); + }, [selectedTargetId, targets]); + const frameUrl = selectedTarget?.frameUrl ?? ""; + + useEffect(() => { + panelWidthRef.current = panelWidth; + }, [panelWidth]); + + const applyDiscovery = useCallback( + (nextDiscovery: DevToolsDiscovery | null) => { + discoveryRef.current = nextDiscovery; + setDiscovery(nextDiscovery); + }, + [], + ); + + const applySelectedTargetId = useCallback((nextTargetId: string) => { + selectedTargetIdRef.current = nextTargetId; + setSelectedTargetId(nextTargetId); + }, []); + + const loadTargets = useCallback(async () => { + if (!selectedSimulator) { + applyDiscovery(null); + applySelectedTargetId(""); + setError(""); + setIsLoading(false); + return; + } + + const requestId = ++requestIdRef.current; + setIsLoading(true); + setError(""); + try { + const chromeTargets = requestWithTimeout( + (signal) => + fetchChromeDevToolsTargets(selectedSimulator.udid, { signal }), + CHROME_DEVTOOLS_REQUEST_TIMEOUT_MS, + "Timed out loading Chrome DevTools targets.", + ); + const webKitTargets = selectedSimulator.isBooted + ? requestWithTimeout( + (signal) => fetchWebKitTargets(selectedSimulator.udid, { signal }), + WEBKIT_DEVTOOLS_REQUEST_TIMEOUT_MS, + "Timed out loading WebKit targets.", + ) + : Promise.resolve({ + socketPath: null, + targets: [], + udid: selectedSimulator.udid, + warnings: [], + }); + const [chromeResult, webKitResult] = await Promise.allSettled([ + chromeTargets, + webKitTargets, + ]); + if (requestId !== requestIdRef.current) { + return; + } + + const nextTargets: DevToolsTarget[] = []; + const warnings: string[] = []; + const errors: string[] = []; + if (chromeResult.status === "fulfilled") { + nextTargets.push(...chromeResult.value.targets.map(mapChromeTarget)); + warnings.push(...chromeResult.value.warnings); + } else { + errors.push(errorMessage(chromeResult.reason)); + } + + if (webKitResult.status === "fulfilled") { + nextTargets.push(...webKitResult.value.targets.map(mapWebKitTarget)); + warnings.push(...webKitResult.value.warnings); + } else { + errors.push(errorMessage(webKitResult.reason)); + } + + const previousDiscovery = discoveryRef.current; + if ( + nextTargets.length === 0 && + previousDiscovery && + previousDiscovery.targets.length > 0 + ) { + applyDiscovery({ + ...previousDiscovery, + warnings: mergeWarnings( + warnings, + errors, + previousDiscovery.warnings, + [ + "DevTools target discovery returned no targets; keeping the active target while debuggers reconnect.", + ], + ), + }); + return; + } + + const nextDiscovery = { + targets: nextTargets, + warnings: mergeWarnings(warnings, errors), + }; + applyDiscovery(nextDiscovery); + const current = selectedTargetIdRef.current; + const nextTargetId = + current && nextTargets.some((target) => target.id === current) + ? current + : (nextTargets[0]?.id ?? ""); + applySelectedTargetId(nextTargetId); + if (nextTargets.length === 0 && errors.length > 0) { + setError(errors.join(" ")); + } + } catch (targetError) { + if (requestId !== requestIdRef.current) { + return; + } + const message = errorMessage(targetError); + const previousDiscovery = discoveryRef.current; + if (previousDiscovery && previousDiscovery.targets.length > 0) { + applyDiscovery({ + ...previousDiscovery, + warnings: mergeWarnings(previousDiscovery.warnings, [message]), + }); + return; + } + applyDiscovery(null); + applySelectedTargetId(""); + setError(message); + } finally { + if (requestId === requestIdRef.current) { + setIsLoading(false); + } + } + }, [ + applyDiscovery, + applySelectedTargetId, + selectedSimulator?.isBooted, + selectedSimulator?.udid, + ]); + + useEffect(() => { + requestIdRef.current += 1; + applyDiscovery(null); + applySelectedTargetId(""); + setError(""); + setFrameLoaded(false); + }, [applyDiscovery, applySelectedTargetId, selectedSimulator?.udid]); + + useEffect(() => { + void loadTargets(); + const interval = window.setInterval(() => { + if (!selectedTargetIdRef.current) { + void loadTargets(); + } + }, DEVTOOLS_TARGET_REFRESH_MS); + return () => window.clearInterval(interval); + }, [loadTargets]); + + useEffect(() => { + setFrameLoaded(false); + }, [frameUrl]); + + useEffect(() => { + function handlePointerMove(event: PointerEvent) { + const resizeState = resizeStateRef.current; + if (!resizeState) { + return; + } + + event.preventDefault(); + const nextWidth = clampPanelWidth( + resizeState.startValue + resizeState.startPointer - event.clientX, + ); + panelWidthRef.current = nextWidth; + setPanelWidth(nextWidth); + } + + function finishResize() { + const resizeState = resizeStateRef.current; + resizeStateRef.current = null; + setIsResizing(false); + document.body.classList.remove("is-resizing-devtools"); + if (!resizeState) { + return; + } + if (resizeState.handle.hasPointerCapture(resizeState.pointerId)) { + resizeState.handle.releasePointerCapture(resizeState.pointerId); + } + storePanelWidth(panelWidthRef.current); + } + + function handleViewportResize() { + setPanelWidth((currentWidth) => { + const nextWidth = clampPanelWidth(currentWidth); + panelWidthRef.current = nextWidth; + return nextWidth; + }); + } + + window.addEventListener("pointermove", handlePointerMove); + window.addEventListener("pointerup", finishResize); + window.addEventListener("pointercancel", finishResize); + window.addEventListener("resize", handleViewportResize); + return () => { + window.removeEventListener("pointermove", handlePointerMove); + window.removeEventListener("pointerup", finishResize); + window.removeEventListener("pointercancel", finishResize); + window.removeEventListener("resize", handleViewportResize); + document.body.classList.remove("is-resizing-devtools"); + }; + }, []); + + useEffect(() => { + const frame = frameRef.current; + if (!frame || typeof ResizeObserver === "undefined") { + return; + } + + const observer = new ResizeObserver(() => { + frame.contentWindow?.dispatchEvent(new Event("resize")); + }); + observer.observe(frame); + return () => observer.disconnect(); + }, [frameUrl]); + + function beginResize(event: ReactPointerEvent) { + if (event.button !== 0) { + return; + } + + event.preventDefault(); + event.currentTarget.setPointerCapture(event.pointerId); + resizeStateRef.current = { + handle: event.currentTarget, + pointerId: event.pointerId, + startPointer: event.clientX, + startValue: panelWidthRef.current, + }; + setIsResizing(true); + document.body.classList.add("is-resizing-devtools"); + } + + function handleResizeKeyDown(event: ReactKeyboardEvent) { + let nextWidth: number | null = null; + if (event.key === "ArrowLeft") { + nextWidth = clampPanelWidth( + panelWidthRef.current + DEVTOOLS_PANEL_WIDTH_STEP, + ); + } else if (event.key === "ArrowRight") { + nextWidth = clampPanelWidth( + panelWidthRef.current - DEVTOOLS_PANEL_WIDTH_STEP, + ); + } else if (event.key === "Home") { + nextWidth = DEVTOOLS_PANEL_MIN_WIDTH; + } else if (event.key === "End") { + nextWidth = panelWidthMaximum(); + } + + if (nextWidth == null) { + return; + } + + event.preventDefault(); + panelWidthRef.current = nextWidth; + setPanelWidth(nextWidth); + storePanelWidth(nextWidth); + } + + function openDetachedInspector() { + if (!frameUrl) { + return; + } + window.open(frameUrl, "_blank", "noopener"); + } + + const statusMessage = + error || + (!selectedSimulator + ? "No simulator selected." + : isLoading && targets.length === 0 + ? "Loading DevTools targets..." + : targets.length === 0 + ? selectedSimulator.isBooted + ? "No DevTools targets. Open Safari, enable inspectable WKWebViews, start Metro, or launch a Chrome remote debugging target." + : "No DevTools targets. Boot the simulator for Safari/WebKit, or start Metro or Chrome remote debugging." + : ""); + const panelStyle = { + "--webkit-panel-width": `${panelWidth}px`, + } as CSSProperties; + + return ( +