diff --git a/README.md b/README.md index ddf7fde4..e638cfa7 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

SimDeck is a developer tool built for streamlining mobile app development for coding agents. - Drive Simulator from the CLI using agents, browser, and automated tests on macOS. + Drive iOS Simulators and Android emulators from the CLI using agents, browser, and automated tests on macOS.

@@ -35,8 +35,9 @@ view inside the editor. ## Features -- Local simulator video stream over browser-native WebRTC H.264 with H.264 WebSocket fallback -- Full simulator control & inspection using private accessibility APIs - available using `simdeck` CLI +- Local iOS Simulator and Android emulator video over browser-native WebRTC H.264 with H.264 WebSocket fallback +- Android emulator frames are sourced from emulator gRPC and encoded through macOS VideoToolbox +- Full simulator control & inspection using private iOS accessibility APIs and Android UIAutomator - available using `simdeck` CLI - Real-time screen `describe` command using accessibility view tree - available in token-efficient format for agents - CoreSimulator chrome asset rendering for device bezels - NativeScript, React Native, Flutter, UIKit and SwiftUI runtime inspector plugins to view app's view hierarchy live @@ -138,6 +139,7 @@ simdeck boot simdeck shutdown simdeck erase simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck open-url https://example.com simdeck launch com.apple.Preferences @@ -179,6 +181,14 @@ simdeck logs --seconds 30 --limit 200 without launching Simulator.app, then falls back to `xcrun simctl` when private booting is unavailable. +Android emulators appear in `simdeck list` with IDs like +`android:SimDeck_Pixel_8_API_36`. For Android IDs, lifecycle, install, launch, +URL, screenshot, logs, UIAutomator `describe`, tap, swipe, text, key, home, app +switcher, rotation, pasteboard, and browser live view route through the Android +SDK tools (`emulator` and `adb`) plus the emulator gRPC screenshot stream for +live video. `simdeck stream` remains iOS-only because it writes the iOS H.264 +transport stream. + `stream` writes an Annex B H.264 elementary stream to stdout for diagnostics or external tools such as `ffplay`. diff --git a/cli/native/XCWNativeBridge.h b/cli/native/XCWNativeBridge.h index a7d81d61..ff4ac7f1 100644 --- a/cli/native/XCWNativeBridge.h +++ b/cli/native/XCWNativeBridge.h @@ -89,6 +89,11 @@ bool xcw_native_session_rotate_right(void * _Nonnull handle, char * _Nullable * bool xcw_native_session_rotate_left(void * _Nonnull handle, char * _Nullable * _Nullable error_message); void xcw_native_session_set_frame_callback(void * _Nonnull handle, xcw_native_frame_callback _Nullable callback, void * _Nullable user_data); +void * _Nullable xcw_native_h264_encoder_create(xcw_native_frame_callback _Nullable callback, void * _Nullable user_data, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_destroy(void * _Nullable handle); +bool xcw_native_h264_encoder_encode_rgba(void * _Nonnull handle, const uint8_t * _Nonnull rgba, size_t length, uint32_t width, uint32_t height, uint64_t timestamp_us, char * _Nullable * _Nullable error_message); +void xcw_native_h264_encoder_request_keyframe(void * _Nonnull handle); + void xcw_native_free_string(char * _Nullable value); void xcw_native_free_bytes(xcw_native_owned_bytes bytes); void xcw_native_release_shared_bytes(xcw_native_shared_bytes bytes); diff --git a/cli/native/XCWNativeBridge.m b/cli/native/XCWNativeBridge.m index 3fc2376a..e93241e7 100644 --- a/cli/native/XCWNativeBridge.m +++ b/cli/native/XCWNativeBridge.m @@ -3,11 +3,13 @@ #import "DFPrivateSimulatorDisplayBridge.h" #import "XCWAccessibilityBridge.h" #import "XCWChromeRenderer.h" +#import "XCWH264Encoder.h" #import "XCWNativeSession.h" #import "XCWSimctl.h" #import #import +#import #include #include @@ -63,10 +65,190 @@ static xcw_native_owned_bytes XCWOwnedBytesFromData(NSData *data) { return bytes; } +static xcw_native_shared_bytes XCWSharedBytesFromData(NSData *data) { + if (data.length == 0) { + return (xcw_native_shared_bytes){0}; + } + + CFTypeRef owner = CFRetain((__bridge CFTypeRef)data); + return (xcw_native_shared_bytes){ + .data = data.bytes, + .length = data.length, + .owner = (const void *)owner, + }; +} + static XCWNativeSession *XCWNativeSessionFromHandle(void *handle) { return (__bridge XCWNativeSession *)handle; } +@interface XCWNativeH264Encoder : NSObject + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData; +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error; +- (void)requestKeyFrame; +- (void)invalidate; + +@end + +@implementation XCWNativeH264Encoder { + XCWH264Encoder *_encoder; + xcw_native_frame_callback _callback; + void *_callbackUserData; + uint64_t _frameSequence; +} + +- (instancetype)initWithFrameCallback:(xcw_native_frame_callback)callback + userData:(void *)userData { + self = [super init]; + if (self == nil) { + return nil; + } + + _callback = callback; + _callbackUserData = userData; + __weak typeof(self) weakSelf = self; + @synchronized (XCWNativeH264Encoder.class) { + const char *previousCodec = getenv("SIMDECK_VIDEO_CODEC"); + char *previousCodecCopy = previousCodec != NULL ? strdup(previousCodec) : NULL; + const char *androidCodec = getenv("SIMDECK_ANDROID_VIDEO_CODEC"); + if (androidCodec == NULL || strlen(androidCodec) == 0) { + androidCodec = "software"; + } + setenv("SIMDECK_VIDEO_CODEC", androidCodec, 1); + _encoder = [[XCWH264Encoder alloc] initWithOutputHandler:^(NSData *sampleData, + uint64_t timestampUs, + BOOL isKeyFrame, + NSString * _Nullable codec, + NSData * _Nullable decoderConfig, + CGSize dimensions) { + __strong typeof(weakSelf) strongSelf = weakSelf; + if (strongSelf == nil || strongSelf->_callback == NULL || sampleData.length == 0) { + return; + } + strongSelf->_frameSequence += 1; + xcw_native_frame frame = { + .frame_sequence = strongSelf->_frameSequence, + .timestamp_us = timestampUs, + .is_keyframe = isKeyFrame, + .width = (uint32_t)llround(dimensions.width), + .height = (uint32_t)llround(dimensions.height), + .codec = codec.UTF8String, + .description = XCWSharedBytesFromData(decoderConfig), + .data = XCWSharedBytesFromData(sampleData), + }; + strongSelf->_callback(&frame, strongSelf->_callbackUserData); + }]; + if (previousCodecCopy != NULL) { + setenv("SIMDECK_VIDEO_CODEC", previousCodecCopy, 1); + free(previousCodecCopy); + } else { + unsetenv("SIMDECK_VIDEO_CODEC"); + } + } + return self; +} + +- (void)dealloc { + [self invalidate]; +} + +- (BOOL)encodeRGBA:(const uint8_t *)rgba + length:(size_t)length + width:(uint32_t)width + height:(uint32_t)height + error:(NSError * _Nullable __autoreleasing *)error { + if (rgba == NULL || width == 0 || height == 0) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:1 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was empty." }]; + } + return NO; + } + size_t expectedLength = (size_t)width * (size_t)height * 4; + if (length < expectedLength) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:2 + userInfo:@{ NSLocalizedDescriptionKey: @"RGBA frame input was truncated." }]; + } + return NO; + } + + NSDictionary *attributes = @{ + (__bridge NSString *)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (__bridge NSString *)kCVPixelBufferWidthKey: @(width), + (__bridge NSString *)kCVPixelBufferHeightKey: @(height), + (__bridge NSString *)kCVPixelBufferIOSurfacePropertiesKey: @{}, + }; + CVPixelBufferRef pixelBuffer = NULL; + CVReturn createStatus = CVPixelBufferCreate(kCFAllocatorDefault, + (size_t)width, + (size_t)height, + kCVPixelFormatType_32BGRA, + (__bridge CFDictionaryRef)attributes, + &pixelBuffer); + if (createStatus != kCVReturnSuccess || pixelBuffer == NULL) { + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:createStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to allocate a VideoToolbox pixel buffer." }]; + } + return NO; + } + + CVReturn lockStatus = CVPixelBufferLockBaseAddress(pixelBuffer, 0); + if (lockStatus != kCVReturnSuccess) { + CVPixelBufferRelease(pixelBuffer); + if (error != NULL) { + *error = [NSError errorWithDomain:@"SimDeck.NativeH264Encoder" + code:lockStatus + userInfo:@{ NSLocalizedDescriptionKey: @"Unable to lock a VideoToolbox pixel buffer." }]; + } + return NO; + } + + uint8_t *dst = CVPixelBufferGetBaseAddress(pixelBuffer); + size_t dstRowBytes = CVPixelBufferGetBytesPerRow(pixelBuffer); + size_t srcRowBytes = (size_t)width * 4; + for (uint32_t y = 0; y < height; y += 1) { + const uint8_t *srcRow = rgba + ((size_t)y * srcRowBytes); + uint8_t *dstRow = dst + ((size_t)y * dstRowBytes); + for (uint32_t x = 0; x < width; x += 1) { + const uint8_t *src = srcRow + ((size_t)x * 4); + uint8_t *pixel = dstRow + ((size_t)x * 4); + pixel[0] = src[2]; + pixel[1] = src[1]; + pixel[2] = src[0]; + pixel[3] = src[3]; + } + } + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + [_encoder encodePixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + return YES; +} + +- (void)requestKeyFrame { + [_encoder requestKeyFrame]; +} + +- (void)invalidate { + [_encoder invalidate]; +} + +@end + +static XCWNativeH264Encoder *XCWNativeH264EncoderFromHandle(void *handle) { + return (__bridge XCWNativeH264Encoder *)handle; +} + static BOOL XCWPerformSimctlAction(char **errorMessage, BOOL (^action)(XCWSimctl *simctl, NSError **error)) { XCWSimctl *simctl = [[XCWSimctl alloc] init]; NSError *error = nil; @@ -889,6 +1071,58 @@ void xcw_native_session_set_frame_callback(void *handle, xcw_native_frame_callba } } +void *xcw_native_h264_encoder_create(xcw_native_frame_callback callback, void *user_data, char **error_message) { + @autoreleasepool { + XCWNativeH264Encoder *encoder = [[XCWNativeH264Encoder alloc] initWithFrameCallback:callback + userData:user_data]; + if (encoder == nil) { + if (error_message != NULL) { + *error_message = XCWCopyCString(@"Unable to create the native H.264 encoder."); + } + return NULL; + } + return (__bridge_retained void *)encoder; + } +} + +void xcw_native_h264_encoder_destroy(void *handle) { + if (handle == NULL) { + return; + } + @autoreleasepool { + XCWNativeH264Encoder *encoder = CFBridgingRelease(handle); + [encoder invalidate]; + } +} + +bool xcw_native_h264_encoder_encode_rgba(void *handle, + const uint8_t *rgba, + size_t length, + uint32_t width, + uint32_t height, + uint64_t timestamp_us, + char **error_message) { + (void)timestamp_us; + @autoreleasepool { + NSError *error = nil; + BOOL ok = [XCWNativeH264EncoderFromHandle(handle) encodeRGBA:rgba + length:length + width:width + height:height + error:&error]; + if (!ok) { + XCWSetErrorMessage(error_message, error); + } + return ok; + } +} + +void xcw_native_h264_encoder_request_keyframe(void *handle) { + @autoreleasepool { + [XCWNativeH264EncoderFromHandle(handle) requestKeyFrame]; + } +} + void xcw_native_free_string(char *value) { if (value != NULL) { free(value); diff --git a/client/src/api/types.ts b/client/src/api/types.ts index d201c83d..d4983644 100644 --- a/client/src/api/types.ts +++ b/client/src/api/types.ts @@ -28,11 +28,17 @@ export interface PrivateDisplayInfo { export interface SimulatorMetadata { udid: string; name: string; + platform?: "ios-simulator" | "android-emulator" | string; runtimeName?: string; runtimeIdentifier?: string; deviceTypeName?: string; deviceTypeIdentifier?: string; isBooted: boolean; + android?: { + avdName?: string; + grpcPort?: number; + serial?: string; + }; privateDisplay?: PrivateDisplayInfo; } @@ -107,6 +113,7 @@ export interface ChromeProfile { screenWidth: number; screenHeight: number; cornerRadius: number; + chromeStyle?: "asset" | string; hasScreenMask?: boolean; buttons?: ChromeButtonProfile[]; } @@ -153,15 +160,23 @@ export interface AccessibilityNode { AXUniqueId?: string | null; AXValue?: string | null; alpha?: number | null; + androidClass?: string | null; + androidPackage?: string | null; + androidResourceId?: string | null; backgroundColor?: Record | null; bounds?: AccessibilityFrame | null; + checkable?: boolean | null; + checked?: boolean | null; className?: string | null; + clickable?: boolean | null; children?: AccessibilityNode[]; control?: Record | null; content_required?: boolean | null; custom_actions?: string[] | null; debugDescription?: string | null; enabled?: boolean | null; + focusable?: boolean | null; + focused?: boolean | null; frame?: AccessibilityFrame | null; frameInScreen?: AccessibilityFrame | null; flutter?: Record | null; @@ -171,8 +186,10 @@ export interface AccessibilityNode { isHidden?: boolean | null; isOpaque?: boolean | null; isUserInteractionEnabled?: boolean | null; + longClickable?: boolean | null; moduleName?: string | null; nativeScript?: Record | null; + password?: boolean | null; pid?: number | null; placeholder?: string | null; reactNative?: Record | null; @@ -180,6 +197,8 @@ export interface AccessibilityNode { role_description?: string | null; scroll?: Record | null; semantics?: Record | null; + scrollable?: boolean | null; + selected?: boolean | null; source?: | "native-ax" | "in-app-inspector" @@ -206,6 +225,7 @@ export interface AccessibilityNode { } export type AccessibilitySource = + | "android-uiautomator" | "native-ax" | "in-app-inspector" | "nativescript" diff --git a/client/src/app/AppShell.tsx b/client/src/app/AppShell.tsx index 59f2b3dc..78252dc0 100644 --- a/client/src/app/AppShell.tsx +++ b/client/src/app/AppShell.tsx @@ -352,8 +352,8 @@ export function AppShell({ const [hierarchyVisible, setHierarchyVisible] = useState(() => readStoredFlag(HIERARCHY_VISIBLE_STORAGE_KEY), ); - const [devToolsVisible, setDevToolsVisible] = useState( - () => readStoredFlag(DEVTOOLS_VISIBLE_STORAGE_KEY, false), + const [devToolsVisible, setDevToolsVisible] = useState(() => + readStoredFlag(DEVTOOLS_VISIBLE_STORAGE_KEY, false), ); const [selectedUDID, setSelectedUDID] = useState(initialSelectedUDID ?? ""); const [search, setSearch] = useState(""); @@ -686,17 +686,28 @@ export function AppShell({ const shouldRenderChrome = selectedSimulator != null && shouldRenderNativeChrome(selectedSimulator); const viewportChromeProfile = shouldRenderChrome ? chromeProfile : null; - const effectiveDeviceNaturalSize = useMemo( - () => + const isAndroidViewport = isAndroidSimulator(selectedSimulator); + const effectiveDeviceNaturalSize = useMemo(() => { + const displaySize = simulatorDisplaySize(selectedSimulator); + if (isAndroidViewport && displaySize) { + return displaySize; + } + return ( deviceNaturalSize ?? (!shouldRenderChrome && chromeProfile ? { width: chromeProfile.screenWidth, height: chromeProfile.screenHeight, } - : simulatorDisplaySize(selectedSimulator)), - [chromeProfile, deviceNaturalSize, selectedSimulator, shouldRenderChrome], - ); + : displaySize) + ); + }, [ + chromeProfile, + deviceNaturalSize, + isAndroidViewport, + selectedSimulator, + shouldRenderChrome, + ]); const zoomDockReservedHeight = zoomDockElement && typeof window !== "undefined" @@ -743,9 +754,9 @@ export function AppShell({ : "", [selectedSimulator?.udid, streamStamp], ); + const chromeUsesAsset = Boolean(viewportChromeProfile && chromeUrl); const chromeRequired = Boolean( - (shouldRenderChrome && !chromeProfileReady) || - (viewportChromeProfile && chromeUrl), + (shouldRenderChrome && !chromeProfileReady) || chromeUsesAsset, ); const simulatorRotationQuarterTurns = normalizeSimulatorRotationQuarterTurns(selectedSimulator); @@ -1289,15 +1300,17 @@ export function AppShell({ } satisfies CSSProperties) : null; const screenOnlyStyle = - !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0 - ? ({ - borderRadius: `${Math.min( - chromeProfile.cornerRadius * - (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth), - DEVICE_SCREEN_WIDTH / 2, - )}px`, - } satisfies CSSProperties) - : null; + !viewportChromeProfile && isAndroidViewport + ? null + : !viewportChromeProfile && chromeProfile && chromeProfile.screenWidth > 0 + ? ({ + borderRadius: `${Math.min( + chromeProfile.cornerRadius * + (DEVICE_SCREEN_WIDTH / chromeProfile.screenWidth), + DEVICE_SCREEN_WIDTH / 2, + )}px`, + } satisfies CSSProperties) + : null; const viewportScreenStyle = chromeScreenStyle ?? screenOnlyStyle; const shellStyle = viewportChromeProfile ? { @@ -1840,14 +1853,25 @@ export function AppShell({ if (!selectedSimulator) { return; } + const androidViewport = isAndroidSimulator(selectedSimulator); beginZoomAnimation(); if (sendControl(selectedSimulator.udid, { type: "rotateRight" })) { - setRotationQuarterTurns((current) => (current + 1) % 4); + if (androidViewport) { + setRotationQuarterTurns(0); + window.setTimeout(() => void refresh(), 300); + } else { + setRotationQuarterTurns((current) => (current + 1) % 4); + } return; } void runAction(async () => { await rotateRight(selectedSimulator.udid); - setRotationQuarterTurns((current) => (current + 1) % 4); + if (androidViewport) { + setRotationQuarterTurns(0); + await refresh(); + } else { + setRotationQuarterTurns((current) => (current + 1) % 4); + } }, false); }} onStreamEncoderChange={updateStreamEncoder} @@ -2000,6 +2024,7 @@ export function AppShell({ outerCanvasRef={handleOuterCanvasRef} rotationQuarterTurns={rotationQuarterTurns} screenAspect={screenAspect} + screenClassName={isAndroidViewport ? "android-screen" : undefined} selectedSimulator={selectedSimulator} shellStyle={shellStyle} streamCanvasRef={handleStreamCanvasRef} @@ -2151,6 +2176,14 @@ function normalizeMaxEdge( : fallback; } +function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { + return Boolean( + simulator?.platform === "android-emulator" || + simulator?.deviceTypeIdentifier === "android-emulator" || + simulator?.udid.startsWith("android:"), + ); +} + function streamConfigsEqual(left: StreamConfig, right: StreamConfig): boolean { return ( left.encoder === right.encoder && diff --git a/client/src/app/uiState.ts b/client/src/app/uiState.ts index 98871d0c..0a00e027 100644 --- a/client/src/app/uiState.ts +++ b/client/src/app/uiState.ts @@ -37,6 +37,7 @@ const ACCESSIBILITY_SOURCE_ORDER: AccessibilitySource[] = [ "flutter", "swiftui", "in-app-inspector", + "android-uiautomator", "native-ax", ]; @@ -162,6 +163,7 @@ export function isAccessibilitySource( value === "flutter" || value === "swiftui" || value === "in-app-inspector" || + value === "android-uiautomator" || value === "native-ax" ); } diff --git a/client/src/features/accessibility/AccessibilityInspector.tsx b/client/src/features/accessibility/AccessibilityInspector.tsx index a36ecd2b..89f4af16 100644 --- a/client/src/features/accessibility/AccessibilityInspector.tsx +++ b/client/src/features/accessibility/AccessibilityInspector.tsx @@ -458,17 +458,24 @@ function NodeDetails({ node: AccessibilityNode; selectedSimulator: SimulatorMetadata | null; }) { + const isAndroid = isAndroidSimulator(selectedSimulator); const details = [ ["Type", accessibilityKind(node)], ["Label", primaryAccessibilityText(node)], ["Source", sourceLocationText(node)], - ["Identifier", accessibilityIdentifier(node)], + [ + isAndroid ? "Resource ID" : "Identifier", + isAndroid + ? (node.androidResourceId ?? "") + : accessibilityIdentifier(node), + ], ["Inspector ID", node.inspectorId ?? ""], ["Module", node.moduleName ?? ""], ["NativeScript", nativeScriptDescription(node.nativeScript)], ["React Native", reactNativeDescription(node.reactNative)], ["Flutter", flutterDescription(node.flutter)], - ["UIKit Class", node.className ?? ""], + [isAndroid ? "Android Class" : "UIKit Class", node.className ?? ""], + ["Package", isAndroid ? (node.androidPackage ?? "") : ""], ["Last JS", lastUIKitScriptText(node)], ["Value", node.AXValue ?? ""], ["Role", node.role ?? ""], @@ -477,6 +484,15 @@ function NodeDetails({ ["SwiftUI", swiftUIDescription(node.swiftUI)], ["Enabled", node.enabled == null ? "" : node.enabled ? "true" : "false"], ["Hidden", node.isHidden == null ? "" : node.isHidden ? "true" : "false"], + ["Clickable", boolDetail(isAndroid, node.clickable)], + ["Long Clickable", boolDetail(isAndroid, node.longClickable)], + ["Focusable", boolDetail(isAndroid, node.focusable)], + ["Focused", boolDetail(isAndroid, node.focused)], + ["Scrollable", boolDetail(isAndroid, node.scrollable)], + ["Checkable", boolDetail(isAndroid, node.checkable)], + ["Checked", boolDetail(isAndroid, node.checked)], + ["Selected", boolDetail(isAndroid, node.selected)], + ["Password", boolDetail(isAndroid, node.password)], ["Alpha", node.alpha == null ? "" : String(round(node.alpha))], ["Frame", validFrame(node.frame) ? frameText(node.frame) : ""], ["PID", node.pid == null ? "" : String(node.pid)], @@ -498,6 +514,18 @@ function NodeDetails({ ); } +function isAndroidSimulator(simulator: SimulatorMetadata | null): boolean { + return Boolean( + simulator?.platform === "android-emulator" || + simulator?.deviceTypeIdentifier === "android-emulator" || + simulator?.udid.startsWith("android:"), + ); +} + +function boolDetail(include: boolean, value: boolean | null | undefined) { + return include && value != null ? (value ? "true" : "false") : ""; +} + function UIKitScriptEditor({ node, selectedSimulator, @@ -731,6 +759,7 @@ const HIERARCHY_SOURCE_ORDER: AccessibilitySource[] = [ "flutter", "swiftui", "in-app-inspector", + "android-uiautomator", "native-ax", ]; @@ -764,6 +793,9 @@ function sourceLabel(source: AccessibilitySource): string { if (source === "swiftui") { return "SwiftUI"; } + if (source === "android-uiautomator") { + return "Android"; + } return source === "in-app-inspector" ? "UIKit" : "Native AX"; } diff --git a/client/src/features/simulators/simulatorDisplay.test.ts b/client/src/features/simulators/simulatorDisplay.test.ts index 58c0b2f9..f39a5507 100644 --- a/client/src/features/simulators/simulatorDisplay.test.ts +++ b/client/src/features/simulators/simulatorDisplay.test.ts @@ -51,4 +51,16 @@ describe("simulatorDisplay", () => { ), ).toBe(false); }); + + it("keeps native chrome off for Android emulators", () => { + expect( + shouldRenderNativeChrome( + simulator({ + deviceTypeIdentifier: "android-emulator", + name: "SimDeck Pixel", + platform: "android-emulator", + }), + ), + ).toBe(false); + }); }); diff --git a/client/src/features/stream/streamTypes.ts b/client/src/features/stream/streamTypes.ts index f55eb58c..4e9b6099 100644 --- a/client/src/features/stream/streamTypes.ts +++ b/client/src/features/stream/streamTypes.ts @@ -2,6 +2,7 @@ import type { Size } from "../viewport/types"; export interface StreamConnectTarget { clientId?: string; + platform?: string; remote?: boolean; streamConfig?: StreamConfig; transport?: StreamTransport; diff --git a/client/src/features/stream/streamWorkerClient.test.ts b/client/src/features/stream/streamWorkerClient.test.ts new file mode 100644 index 00000000..418e9ff6 --- /dev/null +++ b/client/src/features/stream/streamWorkerClient.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + buildStreamTarget, + initialStreamBackend, + preferredStreamBackend, +} from "./streamWorkerClient"; + +describe("streamWorkerClient", () => { + it("uses the common H264 WebSocket preference for Android emulator streams", () => { + const target = buildStreamTarget("android:emulator-5554", { + platform: "android-emulator", + transport: "h264", + }); + + expect(preferredStreamBackend(target)).toBe("h264-ws"); + }); + + it("uses the common WebRTC preference for Android emulator streams", () => { + const target = buildStreamTarget("android:Pixel_8", { + transport: "webrtc", + }); + + expect(preferredStreamBackend(target)).toBe("webrtc"); + }); + + it("treats explicit RGBA transport as a WebRTC backend", () => { + const previousWindow = globalThis.window; + Object.defineProperty(globalThis, "window", { + configurable: true, + value: { location: { search: "?stream=rgba" } }, + }); + + try { + expect(preferredStreamBackend(buildStreamTarget("android:Pixel_8"))).toBe( + "webrtc", + ); + } finally { + Object.defineProperty(globalThis, "window", { + configurable: true, + value: previousWindow, + }); + } + }); + + it("defaults Android auto streams to WebRTC when the browser supports it", () => { + const previousPeerConnection = globalThis.RTCPeerConnection; + ( + globalThis as unknown as { RTCPeerConnection: unknown } + ).RTCPeerConnection = function RTCPeerConnection() {}; + const target = buildStreamTarget("android:Pixel_8", { + transport: "auto", + }); + + try { + expect(preferredStreamBackend(target)).toBe("auto"); + expect(initialStreamBackend(target)).toBe("webrtc"); + } finally { + ( + globalThis as unknown as { RTCPeerConnection: unknown } + ).RTCPeerConnection = previousPeerConnection; + } + }); +}); diff --git a/client/src/features/stream/streamWorkerClient.ts b/client/src/features/stream/streamWorkerClient.ts index e5397753..ceb935e2 100644 --- a/client/src/features/stream/streamWorkerClient.ts +++ b/client/src/features/stream/streamWorkerClient.ts @@ -18,6 +18,7 @@ import type { const HAVE_CURRENT_DATA = 2; const WEBRTC_CONTROL_CHANNEL_LABEL = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL = "simdeck-telemetry"; +const WEBRTC_RGBA_CHANNEL_LABEL = "simdeck-rgba"; const WEBRTC_FIRST_FRAME_TIMEOUT_MS = 10000; const WEBRTC_STALLED_FRAME_TIMEOUT_MS = 3000; const WEBRTC_LOCAL_RECEIVER_BUFFER_SECONDS = 0.001; @@ -33,6 +34,10 @@ const H264_WS_HEADER_BYTES = 40; const H264_WS_MAGIC = 0x53444831; const H264_WS_FLAG_KEYFRAME = 1 << 0; const H264_WS_FLAG_CONFIG = 1 << 1; +const WEBRTC_RGBA_CHUNK_HEADER_BYTES = 48; +const WEBRTC_RGBA_CHUNK_MAGIC = 0x53445243; +const WEBRTC_RGBA_VERSION = 1; +const WEBRTC_RGBA_FORMAT_RGBA8888 = 1; const H264_WS_LOCAL_AUTO_PROFILES: StreamQualityPreset[] = [ "low", "economy", @@ -224,6 +229,7 @@ export function buildStreamTarget( udid: string, options: { clientId?: string; + platform?: string; remote?: boolean; streamConfig?: StreamConfig; transport?: StreamTransport; @@ -231,6 +237,7 @@ export function buildStreamTarget( ): StreamConnectTarget { return { clientId: options.clientId, + platform: options.platform, remote: options.remote, streamConfig: options.streamConfig, transport: options.transport, @@ -300,6 +307,34 @@ interface H264WebSocketFrame { width: number; } +interface WebRtcRgbaFrame { + height: number; + payload: Uint8Array; + sequence: number; + timestampUs: number; + width: number; +} + +interface WebRtcRgbaChunk { + chunkOffset: number; + height: number; + payload: Uint8Array; + payloadBytes: number; + sequence: number; + timestampUs: number; + width: number; +} + +interface WebRtcRgbaAssembly { + buffer: Uint8Array; + height: number; + receivedBytes: number; + receivedRanges: Array<[number, number]>; + sequence: number; + timestampUs: number; + width: number; +} + interface WebCodecsVideoFrame { close(): void; codedHeight?: number; @@ -351,6 +386,13 @@ interface WebCodecsVideoDecoderConstructor { }>; } +interface WebRtcAnswerPayload extends RTCSessionDescriptionInit { + video?: { + height?: number; + width?: number; + }; +} + interface PendingVideoFrame { frame: WebCodecsVideoFrame; sequence: number | null; @@ -1141,6 +1183,53 @@ function parseH264WebSocketFrame(data: unknown): H264WebSocketFrame | null { }; } +function parseWebRtcRgbaChunk(data: unknown): WebRtcRgbaChunk | null { + const bytes = bytesFromBinaryMessage(data); + if (!bytes || bytes.byteLength < WEBRTC_RGBA_CHUNK_HEADER_BYTES) { + return null; + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + if ( + view.getUint32(0, false) !== WEBRTC_RGBA_CHUNK_MAGIC || + view.getUint8(4) !== WEBRTC_RGBA_VERSION || + view.getUint8(5) !== WEBRTC_RGBA_FORMAT_RGBA8888 + ) { + return null; + } + const headerBytes = view.getUint16(6, false); + if ( + headerBytes < WEBRTC_RGBA_CHUNK_HEADER_BYTES || + headerBytes > bytes.byteLength + ) { + return null; + } + const width = view.getUint32(24, false); + const height = view.getUint32(28, false); + const payloadBytes = view.getUint32(32, false); + const chunkOffset = view.getUint32(36, false); + const chunkBytes = view.getUint32(40, false); + const payloadEnd = headerBytes + chunkBytes; + if ( + width <= 0 || + height <= 0 || + payloadBytes !== width * height * 4 || + chunkBytes <= 0 || + chunkOffset + chunkBytes > payloadBytes || + payloadEnd > bytes.byteLength + ) { + return null; + } + return { + chunkOffset, + height, + payload: bytes.subarray(headerBytes, payloadEnd), + payloadBytes, + sequence: Number(view.getBigUint64(8, false)), + timestampUs: Number(view.getBigUint64(16, false)), + width, + }; +} + function bytesFromBinaryMessage(data: unknown): Uint8Array | null { if (ArrayBuffer.isView(data)) { return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); @@ -1168,6 +1257,16 @@ function bytesFromBinaryMessage(data: unknown): Uint8Array | null { return null; } +function rangeAlreadyReceived( + ranges: Array<[number, number]>, + start: number, + end: number, +): boolean { + return ranges.some( + ([rangeStart, rangeEnd]) => start >= rangeStart && end <= rangeEnd, + ); +} + function h264CodecStringFromAvcC(config: Uint8Array): string | null { if (config.byteLength < 4 || config[0] !== 1) { return null; @@ -1196,6 +1295,7 @@ function hexByte(byte: number): string { class WebRtcStreamClient implements StreamClientBackend { private animationFrame = 0; private canvas: HTMLCanvasElement | null = null; + private canvasContext: CanvasRenderingContext2D | null = null; private connectGeneration = 0; private controlChannel: RTCDataChannel | null = null; private diagnostics = createWebRtcDiagnostics(); @@ -1215,6 +1315,10 @@ class WebRtcStreamClient implements StreamClientBackend { private receiverStatsInterval = 0; private receiverStatsSeen = false; private shouldReconnect = false; + private latestRgbaSequence = -1; + private rgbaAssemblies = new Map(); + private rgbaChannel: RTCDataChannel | null = null; + private rgbaMode = false; private streamConfigGeneration = 0; private streamTarget: StreamConnectTarget | null = null; private telemetryChannel: RTCDataChannel | null = null; @@ -1228,6 +1332,10 @@ class WebRtcStreamClient implements StreamClientBackend { attachCanvas(canvasElement: HTMLCanvasElement) { this.canvas = canvasElement; + this.canvasContext = canvasElement.getContext("2d", { + alpha: false, + desynchronized: true, + }); if ( this.video && this.video.parentElement !== canvasElement.parentElement @@ -1240,9 +1348,12 @@ class WebRtcStreamClient implements StreamClientBackend { } clear() { - this.canvas - ?.getContext("2d") - ?.clearRect(0, 0, this.canvas.width, this.canvas.height); + this.ensureCanvasContext()?.clearRect( + 0, + 0, + this.canvas?.width ?? 0, + this.canvas?.height ?? 0, + ); } async collectVisualArtifactSample( @@ -1317,16 +1428,20 @@ class WebRtcStreamClient implements StreamClientBackend { iceTransportPolicy: iceTransportPolicy(health), }); this.peerConnection = peerConnection; + const useRgbaTransport = shouldUseLocalAndroidRgbaWebRtc(target); + this.rgbaMode = useRgbaTransport; this.attachDiagnostics(peerConnection, target, generation); - this.startReceiverStatsPolling(peerConnection, target, generation); - const transceiver = peerConnection.addTransceiver("video", { - direction: "recvonly", - }); - configureReceiverCodecPreferences(transceiver); - configureLowLatencyReceiver( - transceiver.receiver, - receiverBufferSeconds(target), - ); + if (!useRgbaTransport) { + this.startReceiverStatsPolling(peerConnection, target, generation); + const transceiver = peerConnection.addTransceiver("video", { + direction: "recvonly", + }); + configureReceiverCodecPreferences(transceiver); + configureLowLatencyReceiver( + transceiver.receiver, + receiverBufferSeconds(target), + ); + } const controlChannel = peerConnection.createDataChannel( WEBRTC_CONTROL_CHANNEL_LABEL, { @@ -1355,7 +1470,20 @@ class WebRtcStreamClient implements StreamClientBackend { } }); + peerConnection.ondatachannel = (event) => { + if ( + generation !== this.connectGeneration || + event.channel.label !== WEBRTC_RGBA_CHANNEL_LABEL + ) { + return; + } + this.attachRgbaDataChannel(event.channel, generation); + }; + peerConnection.ontrack = (event) => { + if (useRgbaTransport) { + return; + } if (generation !== this.connectGeneration) { return; } @@ -1419,7 +1547,12 @@ class WebRtcStreamClient implements StreamClientBackend { if (this.reportedVideoWidth > 0 && this.reportedVideoHeight > 0) { this.onMessage({ type: "status", - status: { detail: "WebRTC media connected", state: "streaming" }, + status: { + detail: useRgbaTransport + ? "WebRTC RGBA stream connected" + : "WebRTC media connected", + state: "streaming", + }, }); } return; @@ -1454,6 +1587,7 @@ class WebRtcStreamClient implements StreamClientBackend { await this.negotiatePeerConnection(peerConnection, target, generation, { detailPrefix: "local", + transport: useRgbaTransport ? "rgba" : undefined, }); this.scheduleFrameWatchdog(target, generation); } catch (error) { @@ -1496,7 +1630,11 @@ class WebRtcStreamClient implements StreamClientBackend { peerConnection: RTCPeerConnection, target: StreamConnectTarget, generation: number, - options: { detailPrefix: string; iceRestart?: boolean }, + options: { + detailPrefix: string; + iceRestart?: boolean; + transport?: "rgba"; + }, ) { const offer = safariBaselineH264Offer( await peerConnection.createOffer({ iceRestart: options.iceRestart }), @@ -1529,8 +1667,9 @@ class WebRtcStreamClient implements StreamClientBackend { const response = await postWebRtcOfferWithAuthRetry( target, localDescription, + options.transport, ); - const answer = (await response.json()) as RTCSessionDescriptionInit; + const answer = (await response.json()) as WebRtcAnswerPayload; if (generation !== this.connectGeneration) { return; } @@ -1539,6 +1678,15 @@ class WebRtcStreamClient implements StreamClientBackend { ); this.postDiagnostics(target, `${options.detailPrefix}-answer`); await peerConnection.setRemoteDescription(answer); + if ( + typeof answer.video?.width === "number" && + typeof answer.video?.height === "number" && + answer.video.width > 0 && + answer.video.height > 0 + ) { + this.syncCanvasSize(answer.video.width, answer.video.height); + this.reportVideoConfig(answer.video.width, answer.video.height); + } } destroy() { @@ -1573,6 +1721,11 @@ class WebRtcStreamClient implements StreamClientBackend { activeWebRtcTelemetryChannel = null; } this.telemetryChannel = null; + this.rgbaChannel?.close(); + this.rgbaChannel = null; + this.rgbaAssemblies.clear(); + this.latestRgbaSequence = -1; + this.rgbaMode = false; this.peerConnection?.close(); this.peerConnection = null; } @@ -1685,10 +1838,14 @@ class WebRtcStreamClient implements StreamClientBackend { return; } const now = performance.now(); - const hasRenderedFrame = this.stats.renderedFrames > 0; + const hasMediaProgress = + this.hasRenderedFrame || + this.stats.renderedFrames > 0 || + this.stats.decodedFrames > 0 || + this.stats.receivedPackets > 0; const frameAgeMs = this.lastVideoFrameAt > 0 ? now - this.lastVideoFrameAt : Infinity; - if (!hasRenderedFrame) { + if (!hasMediaProgress) { this.handleConnectionError( target, generation, @@ -1697,6 +1854,10 @@ class WebRtcStreamClient implements StreamClientBackend { ); return; } + if (!this.hasRenderedFrame) { + this.scheduleFrameWatchdog(target, generation); + return; + } if (frameAgeMs > WEBRTC_STALLED_FRAME_TIMEOUT_MS) { this.sendControl({ snapshot: true, type: "streamControl" }); this.scheduleFrameWatchdog(target, generation); @@ -1966,6 +2127,171 @@ class WebRtcStreamClient implements StreamClientBackend { } } + private attachRgbaDataChannel(channel: RTCDataChannel, generation: number) { + this.rgbaChannel?.close(); + this.rgbaChannel = channel; + channel.binaryType = "arraybuffer"; + channel.addEventListener("message", (event) => { + if ( + generation !== this.connectGeneration || + channel !== this.rgbaChannel + ) { + return; + } + if (hasArrayBufferMethod(event.data)) { + void event.data.arrayBuffer().then((buffer) => { + if ( + generation === this.connectGeneration && + channel === this.rgbaChannel + ) { + this.handleRgbaMessage(buffer); + } + }); + return; + } + this.handleRgbaMessage(event.data); + }); + channel.addEventListener("close", () => { + if (this.rgbaChannel === channel) { + this.rgbaChannel = null; + } + }); + } + + private handleRgbaMessage(data: unknown) { + const chunk = parseWebRtcRgbaChunk(data); + if (!chunk) { + this.stats.h264ParseFailures += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + this.stats.receivedPackets += 1; + if (chunk.sequence < this.latestRgbaSequence) { + this.stats.droppedFrames += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + const frame = this.appendRgbaChunk(chunk); + if (!frame) { + return; + } + this.stats.decodedFrames += 1; + this.stats.width = frame.width; + this.stats.height = frame.height; + this.stats.frameSequence = frame.sequence; + this.stats.codec = "webrtc-rgba"; + this.latestRgbaSequence = frame.sequence; + this.paintRgbaFrame(frame); + } + + private appendRgbaChunk(chunk: WebRtcRgbaChunk): WebRtcRgbaFrame | null { + let assembly = this.rgbaAssemblies.get(chunk.sequence); + if ( + assembly && + (assembly.width !== chunk.width || + assembly.height !== chunk.height || + assembly.buffer.byteLength !== chunk.payloadBytes) + ) { + this.rgbaAssemblies.delete(chunk.sequence); + assembly = undefined; + } + if (!assembly) { + assembly = { + buffer: new Uint8Array(chunk.payloadBytes), + height: chunk.height, + receivedBytes: 0, + receivedRanges: [], + sequence: chunk.sequence, + timestampUs: chunk.timestampUs, + width: chunk.width, + }; + this.rgbaAssemblies.set(chunk.sequence, assembly); + this.trimRgbaAssemblies(chunk.sequence); + } + assembly.buffer.set(chunk.payload, chunk.chunkOffset); + const chunkEnd = chunk.chunkOffset + chunk.payload.byteLength; + if ( + !rangeAlreadyReceived( + assembly.receivedRanges, + chunk.chunkOffset, + chunkEnd, + ) + ) { + assembly.receivedRanges.push([chunk.chunkOffset, chunkEnd]); + assembly.receivedBytes += chunk.payload.byteLength; + } + if (assembly.receivedBytes < assembly.buffer.byteLength) { + return null; + } + this.rgbaAssemblies.delete(chunk.sequence); + return { + height: assembly.height, + payload: assembly.buffer, + sequence: assembly.sequence, + timestampUs: assembly.timestampUs, + width: assembly.width, + }; + } + + private trimRgbaAssemblies(latestSequence: number) { + while (this.rgbaAssemblies.size > 3) { + const firstKey = this.rgbaAssemblies.keys().next().value; + if (typeof firstKey !== "number") { + break; + } + this.rgbaAssemblies.delete(firstKey); + } + for (const sequence of this.rgbaAssemblies.keys()) { + if (latestSequence - sequence > 2) { + this.rgbaAssemblies.delete(sequence); + } + } + } + + private paintRgbaFrame(frame: WebRtcRgbaFrame) { + const canvas = this.canvas; + const context = this.ensureCanvasContext(); + if (!canvas || !context) { + return; + } + this.syncCanvasSize(frame.width, frame.height); + const startedAt = performance.now(); + const rgba = new Uint8ClampedArray( + frame.payload.buffer as ArrayBuffer, + frame.payload.byteOffset, + frame.payload.byteLength, + ); + try { + context.putImageData( + new ImageData(rgba, frame.width, frame.height), + 0, + 0, + ); + } catch { + this.stats.droppedFrames += 1; + this.onMessage({ type: "stats", stats: { ...this.stats } }); + return; + } + const finishedAt = performance.now(); + const previousFrameAt = this.lastVideoFrameAt; + this.lastVideoFrameAt = finishedAt; + this.hasRenderedFrame = true; + this.stats.renderedFrames += 1; + this.stats.latestRenderMs = finishedAt - startedAt; + this.stats.maxRenderMs = Math.max( + this.stats.maxRenderMs, + this.stats.latestRenderMs, + ); + this.stats.averageRenderMs = + this.stats.averageRenderMs <= 0 + ? this.stats.latestRenderMs + : this.stats.averageRenderMs * 0.9 + this.stats.latestRenderMs * 0.1; + this.stats.latestFrameGapMs = + previousFrameAt > 0 ? finishedAt - previousFrameAt : 0; + this.reportVideoConfig(frame.width, frame.height); + this.onMessage({ type: "stats", stats: { ...this.stats } }); + } + private drawVideoFrame = () => { this.videoFrameCallback = 0; if (!this.canvas || !this.video) { @@ -2031,7 +2357,12 @@ class WebRtcStreamClient implements StreamClientBackend { }); this.onMessage({ type: "status", - status: { detail: "WebRTC media connected", state: "streaming" }, + status: { + detail: this.rgbaMode + ? "WebRTC RGBA stream connected" + : "WebRTC media connected", + state: "streaming", + }, }); } @@ -2145,6 +2476,22 @@ class WebRtcStreamClient implements StreamClientBackend { this.videoFrameCallback = 0; } + private ensureCanvasContext(): CanvasRenderingContext2D | null { + const canvas = this.canvas; + if (!canvas) { + this.canvasContext = null; + return null; + } + if (this.canvasContext?.canvas === canvas) { + return this.canvasContext; + } + this.canvasContext = canvas.getContext("2d", { + alpha: false, + desynchronized: true, + }); + return this.canvasContext; + } + private syncCanvasSize(width: number, height: number) { if (!this.canvas) { return; @@ -2195,8 +2542,9 @@ function streamErrorIsServerUnreachable(message: string): boolean { async function postWebRtcOfferWithAuthRetry( target: StreamConnectTarget, localDescription: RTCSessionDescription, + transport?: "rgba", ): Promise { - const response = await postWebRtcOffer(target, localDescription); + const response = await postWebRtcOffer(target, localDescription, transport); if (response.status !== 401) { if (!response.ok) { throw new Error(await response.text()); @@ -2207,7 +2555,7 @@ async function postWebRtcOfferWithAuthRetry( throw new Error(await response.text()); } await fetchHealth(); - const retry = await postWebRtcOffer(target, localDescription); + const retry = await postWebRtcOffer(target, localDescription, transport); if (!retry.ok) { throw new Error(await retry.text()); } @@ -2273,6 +2621,7 @@ function streamQualityQuery(config: StreamConfig | undefined): string { function postWebRtcOffer( target: StreamConnectTarget, localDescription: RTCSessionDescription, + transport?: "rgba", ): Promise { return fetch( apiUrl(`/api/simulators/${encodeURIComponent(target.udid)}/webrtc/offer`), @@ -2283,6 +2632,7 @@ function postWebRtcOffer( streamConfig: target.streamConfig ? streamQualityPayload(target.streamConfig) : undefined, + transport, type: localDescription.type, }), headers: apiHeaders(), @@ -2334,6 +2684,17 @@ function shouldUseRemoteH264AutoProfile( return Boolean(target?.remote) || !isLoopbackHost(window.location.hostname); } +function shouldUseLocalAndroidRgbaWebRtc(target: StreamConnectTarget): boolean { + const stream = new URLSearchParams(window.location.search).get("stream"); + return ( + !target.remote && + isLoopbackHost(window.location.hostname) && + (stream === "rgba" || stream === "webrtc-rgba") && + (target.platform === "android-emulator" || + target.udid.startsWith("android:")) + ); +} + function configureLowLatencyReceiver( receiver: RTCRtpReceiver, bufferSeconds: number | null, @@ -2687,10 +3048,11 @@ export class StreamWorkerClient { return; } this.backend?.destroy(); - this.backend = - kind === "h264-ws" - ? new H264WebSocketStreamClient(this.handleBackendMessage) - : new WebRtcStreamClient(this.handleBackendMessage); + if (kind === "h264-ws") { + this.backend = new H264WebSocketStreamClient(this.handleBackendMessage); + } else { + this.backend = new WebRtcStreamClient(this.handleBackendMessage); + } this.backendKind = kind; if (this.canvasElement) { this.backend.attachCanvas(this.canvasElement); @@ -2725,7 +3087,7 @@ export class StreamWorkerClient { }; } -function preferredStreamBackend( +export function preferredStreamBackend( target?: StreamConnectTarget | null, ): "auto" | StreamBackend { const value = @@ -2734,10 +3096,14 @@ function preferredStreamBackend( if (value === "h264" || value === "h264-ws") { return "h264-ws"; } - return value === "webrtc" ? "webrtc" : "auto"; + return value === "webrtc" || value === "rgba" || value === "webrtc-rgba" + ? "webrtc" + : "auto"; } -function initialStreamBackend(target: StreamConnectTarget): StreamBackend { +export function initialStreamBackend( + target: StreamConnectTarget, +): StreamBackend { const preferredBackend = preferredStreamBackend(target); if (preferredBackend === "h264-ws") { return canUseH264WebSocket() ? "h264-ws" : "webrtc"; diff --git a/client/src/features/stream/useLiveStream.ts b/client/src/features/stream/useLiveStream.ts index 55f40118..5cf02fbe 100644 --- a/client/src/features/stream/useLiveStream.ts +++ b/client/src/features/stream/useLiveStream.ts @@ -299,10 +299,21 @@ export function useLiveStream({ return; } + const display = simulator.privateDisplay; + const displayKey = + simulator.platform === "android-emulator" && display + ? [ + Math.round(display.displayWidth), + Math.round(display.displayHeight), + display.rotationQuarterTurns ?? 0, + ].join("x") + : ""; const targetKey = [ simulator.udid, + simulator.platform ?? "", remote ? "remote" : "local", streamTransport, + displayKey, ].join("|"); if (connectedStreamTargetKeyRef.current === targetKey) { return; @@ -316,6 +327,7 @@ export function useLiveStream({ workerClient.connect( buildStreamTarget(simulator.udid, { clientId: clientTelemetryIdRef.current, + platform: simulator.platform, remote, streamConfig, transport: streamTransport, @@ -324,6 +336,10 @@ export function useLiveStream({ }, [ canvasElement, simulator?.isBooted, + simulator?.platform, + simulator?.privateDisplay?.displayHeight, + simulator?.privateDisplay?.displayWidth, + simulator?.privateDisplay?.rotationQuarterTurns, simulator?.udid, paused, remote, diff --git a/client/src/features/viewport/DeviceChrome.tsx b/client/src/features/viewport/DeviceChrome.tsx index 9ac199de..000bd8f2 100644 --- a/client/src/features/viewport/DeviceChrome.tsx +++ b/client/src/features/viewport/DeviceChrome.tsx @@ -42,6 +42,7 @@ interface DeviceChromeProps { onStartPanning: (event: React.PointerEvent) => void; rotationQuarterTurns: number; screenAspect: string; + screenClassName?: string; shellStyle: CSSProperties | null; simulatorName: string; streamBackend: string; @@ -80,6 +81,7 @@ export function DeviceChrome({ onStartPanning, rotationQuarterTurns, screenAspect, + screenClassName, shellStyle, simulatorName, streamBackend, @@ -138,6 +140,7 @@ export function DeviceChrome({ onPickerSelect={onPickerSelect} rotationQuarterTurns={rotationQuarterTurns} simulatorName={simulatorName} + screenClassName={screenClassName} streamBackend={streamBackend} streamCanvasRef={streamCanvasRef} streamCanvasKey={streamCanvasKey} @@ -179,6 +182,7 @@ export function DeviceChrome({ onPickerSelect={onPickerSelect} rotationQuarterTurns={rotationQuarterTurns} simulatorName={simulatorName} + screenClassName={screenClassName} streamBackend={streamBackend} streamCanvasRef={streamCanvasRef} streamCanvasKey={streamCanvasKey} @@ -418,6 +422,7 @@ interface ScreenLayerProps { onPickerHover: (id: string | null) => void; onPickerSelect: (id: string) => void; rotationQuarterTurns: number; + screenClassName?: string; simulatorName: string; streamBackend: string; streamCanvasRef: Ref; @@ -445,6 +450,7 @@ function ScreenLayer({ onPickerHover, onPickerSelect, rotationQuarterTurns, + screenClassName, simulatorName, streamBackend, streamCanvasRef, @@ -456,7 +462,13 @@ function ScreenLayer({ }: ScreenLayerProps) { return (
; rotationQuarterTurns: number; screenAspect: string; + screenClassName?: string; selectedSimulator: SimulatorMetadata | null; shellStyle: CSSProperties | null; streamBackend: string; @@ -112,6 +113,7 @@ export function SimulatorViewport({ outerCanvasRef, rotationQuarterTurns, screenAspect, + screenClassName, selectedSimulator, shellStyle, streamBackend, @@ -197,6 +199,7 @@ export function SimulatorViewport({ onStartPanning={onStartPanning} rotationQuarterTurns={rotationQuarterTurns} screenAspect={screenAspect} + screenClassName={screenClassName} shellStyle={shellStyle} simulatorName={selectedSimulator.name} streamBackend={streamBackend} diff --git a/client/src/styles/components.css b/client/src/styles/components.css index bb0275cb..e7c1de2e 100644 --- a/client/src/styles/components.css +++ b/client/src/styles/components.css @@ -642,6 +642,12 @@ color: color-mix(in srgb, #d7ba7d 82%, var(--text)); } +.hierarchy-source-pill.source-android-uiautomator { + border-color: color-mix(in srgb, #7fd97f 55%, var(--border)); + background: color-mix(in srgb, #7fd97f 13%, transparent); + color: color-mix(in srgb, #7fd97f 82%, var(--text)); +} + .hierarchy-source-pill.active { gap: 5px; padding-inline: 7px 8px; @@ -1497,6 +1503,26 @@ border-radius: 0; } +.device-screen.android-screen { + --android-screen-radius: 18px; + --android-stream-overscan: 2px; + + background: transparent; + border-radius: var(--android-screen-radius); + clip-path: inset(0 round var(--android-screen-radius)); +} + +.device-screen.android-screen .stream-canvas, +.device-screen.android-screen .stream-video { + inset: calc(-1 * var(--android-stream-overscan)); + width: calc( + 100% + var(--android-stream-overscan) + var(--android-stream-overscan) + ); + height: calc( + 100% + var(--android-stream-overscan) + var(--android-stream-overscan) + ); +} + .stream-canvas { position: absolute; inset: 0; diff --git a/docs/api/rest.md b/docs/api/rest.md index 2d9a01ec..f549bf9a 100644 --- a/docs/api/rest.md +++ b/docs/api/rest.md @@ -102,7 +102,8 @@ When ### `GET /api/simulators` -Returns every simulator known to the native bridge, enriched with any session state SimDeck has attached: +Returns every iOS Simulator known to the native bridge plus every Android AVD +found in the Android SDK, enriched with any session state SimDeck has attached: ```json { @@ -113,6 +114,7 @@ Returns every simulator known to the native bridge, enriched with any session st "runtimeName": "iOS 18.0", "deviceTypeIdentifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-15-Pro", "isBooted": true, + "platform": "ios-simulator", "privateDisplay": { "displayReady": true, "displayStatus": "running", @@ -126,13 +128,32 @@ Returns every simulator known to the native bridge, enriched with any session st } ``` -`privateDisplay` is `null` until a stream attaches. +Android emulators use IDs prefixed with `android:` and include Android metadata: + +```json +{ + "udid": "android:SimDeck_Pixel_8_API_36", + "name": "SimDeck_Pixel_8_API_36", + "platform": "android-emulator", + "runtimeName": "Android", + "deviceTypeName": "Android Emulator", + "isBooted": true, + "android": { + "avdName": "SimDeck_Pixel_8_API_36", + "serial": "emulator-5554", + "grpcPort": 8554 + } +} +``` + +For iOS, `privateDisplay` is `null` until a stream attaches. For Android, +SimDeck fills display size from `adb shell wm size` when the emulator is booted. ## Simulator lifecycle ### `POST /api/simulators/{udid}/boot` -Boots the simulator and returns the refreshed simulator metadata: +Boots the simulator or Android emulator and returns the refreshed device metadata: ```json { "simulator": { ... } } @@ -140,11 +161,12 @@ Boots the simulator and returns the refreshed simulator metadata: ### `POST /api/simulators/{udid}/shutdown` -Tears down the live session (if any) and shuts the simulator down. +Tears down the live session (if any) and shuts the simulator or emulator down. ### `POST /api/simulators/{udid}/toggle-appearance` -Toggles between light and dark appearance via `simctl ui appearance`. +Toggles between light and dark appearance via `simctl ui appearance` on iOS or +`cmd uimode night` on Android. ```json { "ok": true } @@ -152,7 +174,10 @@ Toggles between light and dark appearance via `simctl ui appearance`. ### `POST /api/simulators/{udid}/refresh` -Forces the encoder to emit a fresh keyframe. Useful after a discontinuity or when the client decoder drifts. +Forces the iOS encoder to emit a fresh frame. For Android IDs, this route is a +no-op that returns `{ "ok": true, "stream": "screenshot" }`; Android WebRTC +keyframe requests are handled through the WebRTC control channel and RTCP +feedback. ```json { "ok": true } @@ -182,13 +207,30 @@ and the server responds with an SDP answer for a receive-only H.264 video track: } ``` -The endpoint requires the active simulator stream to produce H.264-compatible -samples. The bundled browser client always uses this endpoint. +The endpoint requires the selected device stream to produce H.264-compatible +samples. For iOS, samples come from the native simulator display session. For +Android, the server reads raw frames from the emulator gRPC `streamScreenshot` +API, encodes them through VideoToolbox, and writes the resulting H.264 samples +to the same WebRTC video track. The browser also opens `simdeck-control` and `simdeck-telemetry` data channels. In addition to input messages, clients can request a keyframe or tune the stream attached to that peer: +### `GET /api/simulators/{udid}/android/frames` + +Android-only WebSocket stream backed by the emulator gRPC `streamScreenshot` +API. This is retained as a raw-frame diagnostic/fallback path; the browser live +view uses the WebRTC H.264 endpoint by default. The server sends binary raw RGBA +frames with a small SimDeck header, already flipped into top-down row order for +canvas rendering. +Query parameters: + +| Query parameter | Default | Notes | +| --------------- | ------- | ------------------------------------------ | +| `maxEdge` | `960` | Longest output edge requested from gRPC. | +| `maxFps` | `30` | Max frames per second forwarded to client. | + ```json { "type": "streamControl", "forceKeyframe": true } ``` @@ -274,6 +316,13 @@ Content-Type: application/json { "ok": true } ``` +### `GET /api/simulators/{udid}/screenshot.png` + +Returns a PNG screenshot for the selected device. iOS screenshots come from the +native simulator bridge; Android screenshots come from `adb exec-out screencap +-p`. The browser client uses this endpoint for still-image diagnostics and +fallbacks. + ## Input ### `POST /api/simulators/{udid}/touch` diff --git a/docs/cli/commands.md b/docs/cli/commands.md index c67386e9..2d29e3f0 100644 --- a/docs/cli/commands.md +++ b/docs/cli/commands.md @@ -177,12 +177,17 @@ simdeck shutdown simdeck erase ``` -`list` returns the same simulator inventory the browser UI renders. Lifecycle commands return JSON and use the native bridge, preferring private CoreSimulator paths when available and falling back to `xcrun simctl`. +`list` returns the same simulator inventory the browser UI renders, including +Android AVDs as IDs like `android:Pixel_8_API_36`. iOS lifecycle commands use +the native bridge, preferring private CoreSimulator paths when available and +falling back to `xcrun simctl`. Android lifecycle commands use the Android SDK +`emulator` and `adb` tools. ## Apps And URLs ```sh simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck uninstall com.example.App simdeck launch com.example.App simdeck open-url https://example.com @@ -269,9 +274,13 @@ simdeck chrome-profile `stream` writes Annex B H.264 samples to stdout and runs until interrupted, or until `--frames` samples have been written. It is intended for diagnostics and -external tools. +external tools, and is iOS-only. Android live viewing in the browser uses the +WebRTC H.264 endpoint; raw frames come from emulator gRPC and are encoded +through VideoToolbox. -`logs` fetches recent simulator logs. `chrome-profile` returns the CoreSimulator chrome layout used by the browser viewport. +`logs` fetches recent simulator logs or Android `logcat` output. `chrome-profile` +returns the CoreSimulator chrome layout for iOS and a screen-sized profile for +Android. ## HTTP Fast Path diff --git a/docs/extensions/browser-client.md b/docs/extensions/browser-client.md index b9124c30..336356cd 100644 --- a/docs/extensions/browser-client.md +++ b/docs/extensions/browser-client.md @@ -39,15 +39,15 @@ client/ └── styles/ ``` -| Folder | Responsibility | -| ------------------------- | ----------------------------------------------------------------------- | -| `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | -| `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | -| `features/viewport/` | Frame canvas, chrome compositing, hit testing. | -| `features/stream/` | WebRTC client, receiver stats, and video frame plumbing. | -| `features/input/` | Touch / keyboard / hardware-button affordances. | -| `features/accessibility/` | Accessibility tree pane and source switcher. | -| `features/toolbar/` | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh). | +| Folder | Responsibility | +| ------------------------- | ---------------------------------------------------------------------------- | +| `api/` | Typed wrappers around the SimDeck REST API and shared TypeScript types. | +| `features/simulators/` | Sidebar list of simulators plus boot/shutdown affordances. | +| `features/viewport/` | Frame canvas, chrome compositing, hit testing. | +| `features/stream/` | WebRTC H.264 client for iOS and Android, receiver stats, and frame plumbing. | +| `features/input/` | Touch / keyboard / hardware-button affordances. | +| `features/accessibility/` | Accessibility tree pane and source switcher. | +| `features/toolbar/` | Top toolbar (rotate, home, app switcher, dark mode toggle, refresh). | ## Bootstrap flow @@ -55,8 +55,8 @@ client/ 2. `main.tsx` mounts the React tree at `#root`. 3. `AppShell` calls `GET /api/health` to learn the active encoder mode. 4. The simulator sidebar fetches `GET /api/simulators` and renders the list. -5. Selecting a simulator posts an SDP offer to `/api/simulators//webrtc/offer`. -6. The browser renders the H.264 video track through native WebRTC playback. +5. Selecting a device posts an SDP offer to `/api/simulators//webrtc/offer`. +6. The browser renders the H.264 video track through native WebRTC playback. Android emulator frames are sourced from emulator gRPC on the server and encoded through VideoToolbox before WebRTC delivery. 7. Touch and key events round-trip through `POST /api/simulators//touch` and `/key`. ## Dev workflow diff --git a/docs/guide/architecture.md b/docs/guide/architecture.md index 4468b188..d7dd6fb7 100644 --- a/docs/guide/architecture.md +++ b/docs/guide/architecture.md @@ -4,13 +4,13 @@ SimDeck is intentionally split into a small number of clearly-scoped layers. Eve ## High-level layout -SimDeck has three layers stacked between the browser and the iOS Simulator: +SimDeck has three layers stacked between the browser and the target device: -1. **Browser / VS Code** runs the React client from `client/`. It speaks HTTP for control and WebRTC for live video, served by the Rust server. +1. **Browser / VS Code** runs the React client from `client/`. It speaks HTTP for control and WebRTC H.264 for live video, served by the Rust server. 2. **The Rust server** (`server/`, built on `axum` + `tokio`) owns the CLI entrypoint, project daemon lifecycle, REST routes (`api/`), the stream transports (`transport/`), the inspector WebSocket hub (`inspector.rs`), the per-UDID session registry (`simulators/`), metrics, and log streaming. -3. **The Objective-C bridge** (`cli/`) is reached through a narrow C ABI in `cli/native/XCWNativeBridge.*`. It wraps `xcrun simctl`, the private `CoreSimulator` direct-boot path, the per-session hardware/software H.264 encoder, the headless display bridge that produces frames and accepts HID input, and the device-chrome renderer. +3. **Native device bridges** own platform-specific work. The Objective-C bridge (`cli/`) is reached through a narrow C ABI in `cli/native/XCWNativeBridge.*` for iOS. The Rust Android bridge (`server/src/android.rs`) shells out to the Android SDK for AVD discovery, emulator lifecycle, ADB input, screenshots, UIAutomator, and logcat. -Underneath all of that is the iOS Simulator itself — `CoreSimulator` for lifecycle, `SimulatorKit` for chrome assets. +Underneath all of that are the iOS Simulator (`CoreSimulator` and `SimulatorKit`) and the Android emulator (`emulator` and `adb`). ## Layer responsibilities @@ -20,17 +20,18 @@ Owns the public CLI shape (`simdeck`, `simdeck ui`, `daemon`, `boot`, `shutdown` Key modules: -| Module | Responsibility | -| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | -| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | -| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | -| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | -| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | -| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | -| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | -| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | -| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | -| `server/src/logs.rs` | `os_log` log streaming and filtering. | +| Module | Responsibility | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------- | +| `server/src/main.rs` | CLI entrypoint, project daemon management, AppKit main-thread shim, tokio runtime bootstrap. | +| `server/src/api/routes.rs` | Every `/api/*` route, including simulator control, accessibility, and inspector proxy. | +| `server/src/android.rs` | Android AVD discovery, emulator lifecycle, emulator gRPC input/video, screenshots, UIAutomator, and logcat. | +| `server/src/transport/webrtc.rs` | WebRTC offer/answer endpoint for H.264 browser video. | +| `server/src/transport/packet.rs` | Shared encoded frame type used between simulator sessions and transports. | +| `server/src/inspector.rs` | WebSocket hub for the NativeScript runtime inspector. | +| `server/src/simulators/registry.rs` | Per-UDID session registry with lazy attachment to the native bridge. | +| `server/src/simulators/session.rs` | Frame broadcast channel, keyframe gating, refresh requests. | +| `server/src/metrics/counters.rs` | Atomic counters and per-client stream stats accepted from stream transports or `/api/client-stream-stats`. | +| `server/src/logs.rs` | `os_log` log streaming and filtering. | The Rust server runs the tokio runtime on a worker thread while the AppKit main loop spins on the main thread. The native bridge needs the main loop to deliver display callbacks and HID events. @@ -53,13 +54,16 @@ Inside the bridge: ### `client/` — React browser UI -The React app served at `/` is a thin shell that calls the REST API and consumes live video over WebRTC H.264. +The React app served at `/` is a thin shell that calls the REST API. It consumes +live device video over WebRTC H.264. iOS frames come from the native simulator +display bridge; Android frames come from emulator gRPC `streamScreenshot` and +are encoded through VideoToolbox on the server. Layout under `client/src/`: - `app/AppShell.tsx` — top-level shell. - `api/` — typed wrappers around `/api/*` (`client.ts`, `controls.ts`, `simulators.ts`, `types.ts`). -- `features/stream/` — WebRTC client, receiver stats, and video frame plumbing. +- `features/stream/` — WebRTC client, receiver stats, and frame plumbing. - `features/viewport/` — frame canvas, hit testing, chrome compositing. - `features/input/` — touch/keyboard/hardware button affordances. - `features/accessibility/` — accessibility tree pane and source switcher. @@ -85,7 +89,7 @@ Most control endpoints follow the same path: a typed Rust handler in `server/src ### Live video -The browser posts an SDP offer to `/api/simulators/{udid}/webrtc/offer`. The handler in `transport::webrtc` ensures the per-UDID `SimulatorSession` is started, waits up to ~3 s for the first H.264 keyframe, returns an SDP answer, and writes the simulator frame source to a WebRTC video track. +The browser posts an SDP offer to `/api/simulators/{udid}/webrtc/offer`. The handler in `transport::webrtc` starts the selected frame source, waits for the first H.264 keyframe, returns an SDP answer, and writes H.264 samples to a WebRTC video track. For Android, that source is emulator gRPC raw pixels passed through the shared VideoToolbox encoder path. ### Input diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 43e33926..0fabaa30 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -10,6 +10,7 @@ SimDeck only runs on macOS. The native bridge links private `CoreSimulator` and | ---------------------------------- | ------------------------------------------------------------------------------------ | | **macOS 13+** | Required for current `CoreSimulator` and Apple's VideoToolbox H.264 encoder. | | **Xcode + iOS Simulator runtimes** | The native bridge invokes `xcrun simctl` and the Simulator app. | +| **Android SDK tools** | Optional. Required for Android emulator support (`emulator`, `adb`, and AVD images). | | **Node.js ≥ 18** | The launcher (`bin/simdeck.mjs`) and the bundled client tooling. | | **Rust (stable)** | Required only when building from source. Installed via [rustup](https://rustup.rs/). | diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 6a5f3a75..8aa3b283 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -28,21 +28,29 @@ try { The current session object exposes: -| Method | Purpose | -| -------------- | ----------------------------------------------------------------- | -| `list()` | Fetch simulator inventory from `GET /api/simulators`. | -| `launch()` | Launch an installed bundle ID. | -| `openUrl()` | Open a URL or deep link. | -| `tap()` | Tap normalized screen coordinates. | -| `key()` | Send one HID key code. | -| `button()` | Press a hardware button. | -| `tree()` | Fetch an accessibility hierarchy. | -| `query()` | Return compact matches for a selector. | -| `waitFor()` | Poll until a selector appears. | -| `assert()` | Assert a selector is present. | -| `batch()` | Run multiple REST actions through `/api/simulators/{udid}/batch`. | -| `screenshot()` | Return a PNG buffer. | -| `close()` | Stop the daemon if this session started it. | +| Method | Purpose | +| -------------------------------------- | ----------------------------------------------------------------- | +| `list()` | Fetch simulator inventory from `GET /api/simulators`. | +| `install()`, `uninstall()` | Install or remove an app. | +| `launch()` | Launch an installed bundle ID or Android package. | +| `openUrl()` | Open a URL or deep link. | +| `tap()`, `tapElement()` | Tap normalized coordinates or a matching accessibility element. | +| `touch()`, `swipe()`, `gesture()` | Send normalized pointer gestures. | +| `typeText()`, `key()`, `keySequence()` | Send text or HID keyboard input. | +| `button()` | Press a hardware button. | +| `home()`, `dismissKeyboard()` | Trigger common system controls. | +| `appSwitcher()` | Open the app switcher. | +| `rotateLeft()`, `rotateRight()` | Rotate the simulator display. | +| `toggleAppearance()` | Toggle light/dark appearance. | +| `pasteboardSet()`, `pasteboardGet()` | Read or write pasteboard text. | +| `chromeProfile()` | Fetch screen/chrome geometry. | +| `tree()` | Fetch an accessibility hierarchy. | +| `query()` | Return compact matches for a selector. | +| `waitFor()` | Poll until a selector appears. | +| `assert()` | Assert a selector is present. | +| `batch()` | Run multiple REST actions through `/api/simulators/{udid}/batch`. | +| `screenshot()` | Return a PNG buffer. | +| `close()` | Stop the daemon if this session started it. | Selectors can match `id`, `label`, `value`, or `type`. Query options accept `source`, `maxDepth`, and `includeHidden`. diff --git a/packages/simdeck-test/dist/index.d.ts b/packages/simdeck-test/dist/index.d.ts index 78875ccf..a9e2a23e 100644 --- a/packages/simdeck-test/dist/index.d.ts +++ b/packages/simdeck-test/dist/index.d.ts @@ -13,7 +13,8 @@ export type QueryOptions = { | "react-native" | "flutter" | "uikit" - | "native-ax"; + | "native-ax" + | "android-uiautomator"; maxDepth?: number; includeHidden?: boolean; }; @@ -28,6 +29,19 @@ export type TapOptions = QueryOptions & { waitTimeoutMs?: number; pollMs?: number; }; +export type SwipeOptions = { + durationMs?: number; + steps?: number; +}; +export type GestureOptions = SwipeOptions & { + delta?: number; +}; +export type TypeTextOptions = { + delayMs?: number; +}; +export type KeySequenceOptions = { + delayMs?: number; +}; export type SimDeckSession = { endpoint: string; pid: number; @@ -44,8 +58,37 @@ export type SimDeckSession = { options?: TapOptions, ): Promise; touch(udid: string, x: number, y: number, phase: string): Promise; + swipe( + udid: string, + startX: number, + startY: number, + endX: number, + endY: number, + options?: SwipeOptions, + ): Promise; + gesture( + udid: string, + preset: string, + options?: GestureOptions, + ): Promise; + typeText( + udid: string, + text: string, + options?: TypeTextOptions, + ): Promise; key(udid: string, keyCode: number, modifiers?: number): Promise; + keySequence( + udid: string, + keyCodes: number[], + options?: KeySequenceOptions, + ): Promise; button(udid: string, button: string, durationMs?: number): Promise; + home(udid: string): Promise; + dismissKeyboard(udid: string): Promise; + appSwitcher(udid: string): Promise; + rotateLeft(udid: string): Promise; + rotateRight(udid: string): Promise; + toggleAppearance(udid: string): Promise; pasteboardSet(udid: string, text: string): Promise; pasteboardGet(udid: string): Promise; chromeProfile(udid: string): Promise; diff --git a/packages/simdeck-test/dist/index.js b/packages/simdeck-test/dist/index.js index 0259c11d..20f1593b 100644 --- a/packages/simdeck-test/dist/index.js +++ b/packages/simdeck-test/dist/index.js @@ -67,11 +67,68 @@ export async function connect(options = {}) { y, phase, }), + swipe: (udid, startX, startY, endX, endY, swipeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "swipe", + startX, + startY, + endX, + endY, + ...swipeOptions, + }, + ], + }, + ), + gesture: (udid, preset, gestureOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "gesture", + preset, + ...gestureOptions, + }, + ], + }, + ), + typeText: (udid, text, typeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "type", + text, + ...typeOptions, + }, + ], + }, + ), key: (udid, keyCode, modifiers = 0) => requestOk(endpoint, `/api/simulators/${encodeURIComponent(udid)}/key`, { keyCode, modifiers, }), + keySequence: (udid, keyCodes, keySequenceOptions = {}) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/key-sequence`, + { + keyCodes, + ...keySequenceOptions, + }, + ), button: (udid, button, durationMs = 0) => requestOk( endpoint, @@ -81,6 +138,42 @@ export async function connect(options = {}) { durationMs, }, ), + home: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/home`, + null, + ), + dismissKeyboard: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/dismiss-keyboard`, + null, + ), + appSwitcher: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/app-switcher`, + null, + ), + rotateLeft: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-left`, + null, + ), + rotateRight: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-right`, + null, + ), + toggleAppearance: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/toggle-appearance`, + null, + ), pasteboardSet: (udid, text) => requestOk( endpoint, diff --git a/packages/simdeck-test/src/index.ts b/packages/simdeck-test/src/index.ts index db2ffd9f..379475ac 100644 --- a/packages/simdeck-test/src/index.ts +++ b/packages/simdeck-test/src/index.ts @@ -22,7 +22,8 @@ export type QueryOptions = { | "react-native" | "flutter" | "uikit" - | "native-ax"; + | "native-ax" + | "android-uiautomator"; maxDepth?: number; includeHidden?: boolean; }; @@ -40,6 +41,23 @@ export type TapOptions = QueryOptions & { pollMs?: number; }; +export type SwipeOptions = { + durationMs?: number; + steps?: number; +}; + +export type GestureOptions = SwipeOptions & { + delta?: number; +}; + +export type TypeTextOptions = { + delayMs?: number; +}; + +export type KeySequenceOptions = { + delayMs?: number; +}; + export type SimDeckSession = { endpoint: string; pid: number; @@ -56,8 +74,37 @@ export type SimDeckSession = { options?: TapOptions, ): Promise; touch(udid: string, x: number, y: number, phase: string): Promise; + swipe( + udid: string, + startX: number, + startY: number, + endX: number, + endY: number, + options?: SwipeOptions, + ): Promise; + gesture( + udid: string, + preset: string, + options?: GestureOptions, + ): Promise; + typeText( + udid: string, + text: string, + options?: TypeTextOptions, + ): Promise; key(udid: string, keyCode: number, modifiers?: number): Promise; + keySequence( + udid: string, + keyCodes: number[], + options?: KeySequenceOptions, + ): Promise; button(udid: string, button: string, durationMs?: number): Promise; + home(udid: string): Promise; + dismissKeyboard(udid: string): Promise; + appSwitcher(udid: string): Promise; + rotateLeft(udid: string): Promise; + rotateRight(udid: string): Promise; + toggleAppearance(udid: string): Promise; pasteboardSet(udid: string, text: string): Promise; pasteboardGet(udid: string): Promise; chromeProfile(udid: string): Promise; @@ -163,11 +210,68 @@ export async function connect( y, phase, }), + swipe: (udid, startX, startY, endX, endY, swipeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "swipe", + startX, + startY, + endX, + endY, + ...swipeOptions, + }, + ], + }, + ), + gesture: (udid, preset, gestureOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "gesture", + preset, + ...gestureOptions, + }, + ], + }, + ), + typeText: (udid, text, typeOptions = {}) => + requestJson( + endpoint, + "POST", + `/api/simulators/${encodeURIComponent(udid)}/batch`, + { + steps: [ + { + action: "type", + text, + ...typeOptions, + }, + ], + }, + ), key: (udid, keyCode, modifiers = 0) => requestOk(endpoint, `/api/simulators/${encodeURIComponent(udid)}/key`, { keyCode, modifiers, }), + keySequence: (udid, keyCodes, keySequenceOptions = {}) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/key-sequence`, + { + keyCodes, + ...keySequenceOptions, + }, + ), button: (udid, button, durationMs = 0) => requestOk( endpoint, @@ -177,6 +281,42 @@ export async function connect( durationMs, }, ), + home: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/home`, + null, + ), + dismissKeyboard: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/dismiss-keyboard`, + null, + ), + appSwitcher: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/app-switcher`, + null, + ), + rotateLeft: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-left`, + null, + ), + rotateRight: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/rotate-right`, + null, + ), + toggleAppearance: (udid) => + requestOk( + endpoint, + `/api/simulators/${encodeURIComponent(udid)}/toggle-appearance`, + null, + ), pasteboardSet: (udid, text) => requestOk( endpoint, diff --git a/scripts/integration/cli.mjs b/scripts/integration/cli.mjs index 461cf2b7..c9551b7f 100644 --- a/scripts/integration/cli.mjs +++ b/scripts/integration/cli.mjs @@ -837,6 +837,28 @@ async function ensureFixtureForeground(label, options = {}) { if (launchError === null) { throw verifyError; } + logStep(`${label}: opening fixture URL after launch timeout`); + } + + try { + await retrySimdeckJson( + cliArgs(["open-url", simulatorUDID, fixtureUrl]), + `${label} fixture URL fallback`, + { + attempts: 2, + delayMs: 2_000, + timeoutMs: 180_000, + }, + ); + return await verifyUi(label, { + expectFixture: true, + attempts: options.fallbackVerifyAttempts ?? 12, + delayMs: options.fallbackVerifyDelayMs ?? 1_500, + }); + } catch (urlError) { + logStep( + `${label}: fixture URL fallback failed: ${summarizeError(urlError)}`, + ); logStep(`${label}: tapping fixture icon after launch timeout`); } diff --git a/server/Cargo.lock b/server/Cargo.lock index a0120b67..9c7c7db3 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -150,6 +150,28 @@ dependencies = [ "syn", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -173,13 +195,40 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper", + "tower 0.5.3", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core", + "axum-core 0.5.6", "base64", "bytes", "form_urlencoded", @@ -190,7 +239,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -203,12 +252,32 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.6" @@ -314,9 +383,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "shlex", @@ -486,9 +555,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "der" @@ -561,6 +630,12 @@ dependencies = [ "spki", ] +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + [[package]] name = "elliptic-curve" version = "0.13.8" @@ -620,6 +695,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -791,6 +872,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -802,9 +908,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -897,6 +1003,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -905,6 +1012,20 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -914,12 +1035,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "libc", "pin-project-lite", + "socket2 0.6.3", "tokio", "tower-service", + "tracing", ] [[package]] @@ -1023,14 +1149,24 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1038,7 +1174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1085,6 +1221,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1093,10 +1238,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1115,9 +1262,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "litemap" @@ -1149,6 +1296,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -1378,6 +1531,26 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf0d9e68100b3a7989b4901972f265cd542e560a3a8a724e1e20322f4d06ce9" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a990e22f43e84855daf260dded30524ef4a9021cc7541c26540500a50b624389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1407,7 +1580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "092791278e026273c1b65bbdcfbba3a300f2994c896bd01ab01da613c29c46f1" dependencies = [ "base64", - "indexmap", + "indexmap 2.14.0", "quick-xml", "serde", "time", @@ -1483,6 +1656,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quick-xml" version = "0.39.4" @@ -1648,6 +1844,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + [[package]] name = "rtcp" version = "0.12.0" @@ -1694,9 +1896,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "once_cell", "ring", @@ -1708,9 +1910,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -1904,7 +2106,7 @@ name = "simdeck-server" version = "0.1.0" dependencies = [ "anyhow", - "axum", + "axum 0.8.9", "base64", "bytes", "cc", @@ -1914,6 +2116,8 @@ dependencies = [ "http", "libc", "plist", + "prost", + "roxmltree", "serde", "serde_json", "sha2", @@ -1921,6 +2125,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite", + "tonic", "tower-http", "tracing", "tracing-subscriber", @@ -2144,9 +2349,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -2206,6 +2411,56 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.7.9", + "base64", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.6", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -2224,9 +2479,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" dependencies = [ "bitflags 2.11.1", "bytes", @@ -2322,6 +2577,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "tungstenite" version = "0.29.0" @@ -2455,6 +2716,15 @@ dependencies = [ "atomic-waker", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -2481,9 +2751,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -2494,9 +2764,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2504,9 +2774,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -2517,9 +2787,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] @@ -2541,7 +2811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -2554,7 +2824,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.1", "hashbrown 0.15.5", - "indexmap", + "indexmap 2.14.0", "semver", ] @@ -2911,7 +3181,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap", + "indexmap 2.14.0", "prettyplease", "syn", "wasm-metadata", @@ -2942,7 +3212,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.1", - "indexmap", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -2961,7 +3231,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap", + "indexmap 2.14.0", "log", "semver", "serde", diff --git a/server/Cargo.toml b/server/Cargo.toml index 90270a9b..0a376bc6 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -15,6 +15,8 @@ hex = "0.4" http = "1.1" libc = "0.2" plist = "1.7" +prost = "0.13" +roxmltree = "0.20" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "0.10" @@ -22,6 +24,7 @@ thiserror = "2.0" tokio = { version = "1.42", features = ["fs", "io-util", "macros", "process", "rt-multi-thread", "signal", "sync", "time"] } tokio-stream = "0.1" tokio-tungstenite = "0.29" +tonic = { version = "0.12", features = ["transport"] } tower-http = { version = "0.6", features = ["cors", "fs", "trace"] } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } diff --git a/server/src/android.rs b/server/src/android.rs new file mode 100644 index 00000000..0060e8cb --- /dev/null +++ b/server/src/android.rs @@ -0,0 +1,1959 @@ +use crate::error::AppError; +use bytes::BytesMut; +use http::uri::PathAndQuery; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::env; +use std::ffi::OsString; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::{Mutex, OnceLock}; +use std::thread; +use std::time::{Duration, Instant}; +use tonic::metadata::MetadataValue; +use tonic::transport::{Channel, Endpoint}; + +const ANDROID_ID_PREFIX: &str = "android:"; +const DEFAULT_GRPC_PORT_BASE: u16 = 8554; +const ANDROID_GRPC_FRAME_MESSAGE_LIMIT: usize = 64 * 1024 * 1024; +const ANDROID_TOUCH_IDENTIFIER: i32 = 1; +const RUNNING_EMULATOR_CACHE_TTL: Duration = Duration::from_secs(2); +const AVD_GRPC_PORT_CACHE_TTL: Duration = Duration::from_secs(60); +const SCREEN_SIZE_CACHE_TTL: Duration = Duration::from_secs(1); +const MODIFIER_SHIFT: u32 = 1 << 0; +const MODIFIER_CONTROL: u32 = 1 << 1; +const MODIFIER_OPTION: u32 = 1 << 2; +const MODIFIER_COMMAND: u32 = 1 << 3; +const MODIFIER_CAPS_LOCK: u32 = 1 << 4; + +type TimedMap = Option<(Instant, HashMap)>; +type DisplayMetricsCache = HashMap; + +#[derive(Clone, Copy, Debug, PartialEq)] +struct AndroidDisplayMetrics { + width: f64, + height: f64, + rotation_quarter_turns: u16, +} + +#[derive(Clone, Default)] +pub struct AndroidBridge; + +#[derive(Clone, Debug)] +pub struct AndroidDevice { + pub avd_name: String, + pub serial: Option, + pub is_booted: bool, + pub grpc_port: u16, +} + +#[derive(Debug)] +pub struct AndroidFrame { + pub width: u32, + pub height: u32, + pub timestamp_us: u64, + pub rgba: Vec, +} + +pub struct AndroidGrpcFrameStream { + inner: tonic::Streaming, + target: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct AndroidFrameTarget { + width: u32, + height: u32, + rotation_quarter_turns: u16, +} + +pub fn is_android_id(id: &str) -> bool { + id.starts_with(ANDROID_ID_PREFIX) +} + +pub fn avd_from_id(id: &str) -> Result { + id.strip_prefix(ANDROID_ID_PREFIX) + .filter(|value| !value.trim().is_empty()) + .map(ToOwned::to_owned) + .ok_or_else(|| AppError::bad_request(format!("Invalid Android emulator id `{id}`."))) +} + +pub fn id_for_avd(avd_name: &str) -> String { + format!("{ANDROID_ID_PREFIX}{avd_name}") +} + +impl AndroidBridge { + pub fn list_devices(&self) -> Result, AppError> { + if !self.emulator_path().exists() { + return Ok(Vec::new()); + } + + let avds = self + .run_emulator(["-list-avds"])? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(ToOwned::to_owned) + .collect::>(); + if avds.is_empty() { + return Ok(Vec::new()); + } + + let running = self.running_emulators().unwrap_or_default(); + Ok(avds + .into_iter() + .enumerate() + .map(|(index, avd_name)| AndroidDevice { + serial: running.get(&avd_name).cloned(), + is_booted: running.contains_key(&avd_name), + grpc_port: DEFAULT_GRPC_PORT_BASE + index as u16, + avd_name, + }) + .collect()) + } + + pub fn enrich_devices(&self, devices: Vec) -> Vec { + devices + .into_iter() + .map(|device| self.device_value(device)) + .collect() + } + + pub fn boot(&self, id: &str) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + if self.resolve_serial(&avd_name).is_ok() { + return Ok(()); + } + let grpc_port = self.grpc_port_for_avd(&avd_name)?; + Command::new(self.emulator_path()) + .args([ + "-avd", + &avd_name, + "-no-window", + "-no-audio", + "-gpu", + "swiftshader_indirect", + "-grpc", + &grpc_port.to_string(), + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .map_err(|error| { + AppError::native(format!( + "Unable to start Android emulator `{avd_name}`: {error}" + )) + })?; + Ok(()) + } + + pub fn shutdown(&self, id: &str) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + let serial = self.resolve_serial(&avd_name)?; + let _ = self.run_adb(["-s", &serial, "emu", "kill"])?; + Ok(()) + } + + pub fn erase(&self, id: &str) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + if self.resolve_serial(&avd_name).is_ok() { + return Err(AppError::bad_request( + "Shutdown the Android emulator before erasing it.", + )); + } + let avd_dir = self.avd_dir(&avd_name); + for file_name in [ + "userdata-qemu.img", + "cache.img", + "data.img", + "sdcard.img", + "snapshots.img", + ] { + let path = avd_dir.join(file_name); + if path.exists() { + std::fs::remove_file(&path).map_err(|error| { + AppError::native(format!("Unable to remove {}: {error}", path.display())) + })?; + } + } + Ok(()) + } + + pub fn wait_until_booted(&self, id: &str, timeout_duration: Duration) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + let deadline = Instant::now() + timeout_duration; + loop { + if let Ok(serial) = self.resolve_serial(&avd_name) { + if self + .run_adb(["-s", &serial, "shell", "getprop", "sys.boot_completed"]) + .unwrap_or_default() + .trim() + == "1" + { + return Ok(()); + } + } + if Instant::now() >= deadline { + return Err(AppError::native(format!( + "Android emulator `{avd_name}` did not finish booting in time." + ))); + } + thread::sleep(Duration::from_millis(500)); + } + } + + pub fn screenshot_png(&self, id: &str) -> Result, AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb_bytes(["-s", &serial, "exec-out", "screencap", "-p"]) + } + + pub fn install_app(&self, id: &str, app_path: &str) -> Result<(), AppError> { + if !app_path.ends_with(".apk") { + return Err(AppError::bad_request( + "Android install expects an `.apk` path.", + )); + } + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "install", "-r", app_path])?; + Ok(()) + } + + pub fn uninstall_app(&self, id: &str, package_name: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "uninstall", package_name])?; + Ok(()) + } + + pub fn open_url(&self, id: &str, url: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb([ + "-s", + &serial, + "shell", + "am", + "start", + "-a", + "android.intent.action.VIEW", + "-d", + url, + ])?; + Ok(()) + } + + pub fn launch_package(&self, id: &str, package: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb([ + "-s", + &serial, + "shell", + "monkey", + "-p", + package, + "-c", + "android.intent.category.LAUNCHER", + "1", + ])?; + Ok(()) + } + + pub fn set_pasteboard_text(&self, id: &str, text: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb_shell(&serial, &format!("cmd clipboard set {}", shell_quote(text)))?; + Ok(()) + } + + pub fn pasteboard_text(&self, id: &str) -> Result { + let serial = self.serial_for_id(id)?; + self.run_adb_shell(&serial, "cmd clipboard get") + } + + pub fn send_touch(&self, id: &str, x: f64, y: f64, phase: &str) -> Result<(), AppError> { + if self.send_touch_grpc(id, x, y, phase).is_ok() { + return Ok(()); + } + if phase != "ended" && phase != "cancelled" { + return Ok(()); + } + let serial = self.serial_for_id(id)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + let px = (x.clamp(0.0, 1.0) * (width - 1.0)).round().max(0.0); + let py = (y.clamp(0.0, 1.0) * (height - 1.0)).round().max(0.0); + self.run_adb([ + "-s", + &serial, + "shell", + "input", + "tap", + &px.to_string(), + &py.to_string(), + ])?; + Ok(()) + } + + pub fn send_swipe( + &self, + id: &str, + start_x: f64, + start_y: f64, + end_x: f64, + end_y: f64, + duration_ms: u64, + ) -> Result<(), AppError> { + if self + .send_swipe_grpc(id, start_x, start_y, end_x, end_y, duration_ms) + .is_ok() + { + return Ok(()); + } + let serial = self.serial_for_id(id)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + let coords = [start_x, start_y, end_x, end_y] + .into_iter() + .enumerate() + .map(|(index, value)| { + let max = if index % 2 == 0 { + width - 1.0 + } else { + height - 1.0 + }; + (value.clamp(0.0, 1.0) * max).round().max(0.0).to_string() + }) + .collect::>(); + self.run_adb([ + "-s", + &serial, + "shell", + "input", + "swipe", + &coords[0], + &coords[1], + &coords[2], + &coords[3], + &duration_ms.to_string(), + ])?; + Ok(()) + } + + pub fn send_key(&self, id: &str, key_code: u16, modifiers: u32) -> Result<(), AppError> { + if let Some(text) = hid_text_for_key(key_code, modifiers) { + return self.type_text_adb(id, &text); + } + + let serial = self.serial_for_id(id)?; + let android_key = android_key_code(key_code); + if has_android_key_modifiers(modifiers) { + return self.press_android_key_combination(&serial, android_key, modifiers); + } + self.press_android_key(&serial, android_key) + } + + pub fn type_text(&self, id: &str, text: &str) -> Result<(), AppError> { + self.type_text_adb(id, text) + } + + pub fn dismiss_keyboard(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.press_android_key(&serial, 4) + } + + fn type_text_adb(&self, id: &str, text: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let escaped = android_input_text_arg(text); + self.run_adb(["-s", &serial, "shell", "input", "text", &escaped])?; + Ok(()) + } + + fn press_android_key(&self, serial: &str, key_code: u16) -> Result<(), AppError> { + let key_code = key_code.to_string(); + self.run_adb(["-s", serial, "shell", "input", "keyevent", &key_code])?; + Ok(()) + } + + fn press_android_key_combination( + &self, + serial: &str, + key_code: u16, + modifiers: u32, + ) -> Result<(), AppError> { + let mut parts = vec!["input".to_owned(), "keycombination".to_owned()]; + parts.extend( + android_modifier_key_codes(modifiers) + .into_iter() + .map(|key| key.to_string()), + ); + parts.push(key_code.to_string()); + match self.run_adb_shell(serial, &parts.join(" ")) { + Ok(_) => Ok(()), + Err(_) => self.press_android_key(serial, key_code), + } + } + + pub fn press_home(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "3"])?; + Ok(()) + } + + pub fn open_app_switcher(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "187"])?; + Ok(()) + } + + pub fn press_button(&self, id: &str, button: &str, duration_ms: u32) -> Result<(), AppError> { + match button { + "home" => self.press_home(id), + "lock" | "side-button" => { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "26"])?; + if duration_ms > 500 { + thread::sleep(Duration::from_millis(u64::from(duration_ms))); + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "26"])?; + } + Ok(()) + } + "back" => { + let serial = self.serial_for_id(id)?; + self.run_adb(["-s", &serial, "shell", "input", "keyevent", "4"])?; + Ok(()) + } + _ => Err(AppError::bad_request(format!( + "Unsupported Android hardware button `{button}`." + ))), + } + } + + pub fn rotate_right(&self, id: &str) -> Result<(), AppError> { + self.rotate_by_quarter_turns(id, 1) + } + + pub fn rotate_left(&self, id: &str) -> Result<(), AppError> { + self.rotate_by_quarter_turns(id, -1) + } + + fn rotate_by_quarter_turns(&self, id: &str, delta: i16) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let current = self + .display_metrics_for_serial(&serial) + .map(|metrics| metrics.rotation_quarter_turns) + .unwrap_or(0); + let next = (i16::try_from(current).unwrap_or(0) + delta).rem_euclid(4) as u16; + let rotation = next.to_string(); + self.run_adb([ + "-s", + &serial, + "shell", + "cmd", + "window", + "user-rotation", + "lock", + &rotation, + ])?; + self.invalidate_display_metrics_for_serial(&serial); + Ok(()) + } + + pub fn toggle_appearance(&self, id: &str) -> Result<(), AppError> { + let serial = self.serial_for_id(id)?; + let current = self.run_adb_shell(&serial, "cmd uimode night")?; + let mode = if current.to_lowercase().contains("yes") { + "no" + } else { + "yes" + }; + self.run_adb(["-s", &serial, "shell", "cmd", "uimode", "night", mode])?; + Ok(()) + } + + pub fn logs(&self, id: &str, limit: usize) -> Result, AppError> { + let serial = self.serial_for_id(id)?; + let raw = self.run_adb([ + "-s", + &serial, + "logcat", + "-d", + "-v", + "threadtime", + "-t", + &limit.max(1).to_string(), + ])?; + Ok(raw + .lines() + .map(|line| { + json!({ + "timestamp": "", + "level": android_log_level(line), + "process": "", + "pid": Value::Null, + "subsystem": "android", + "category": "logcat", + "message": line, + }) + }) + .collect()) + } + + pub fn chrome_profile(&self, id: &str) -> Result { + let serial = self.serial_for_id(id)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + Ok(json!({ + "totalWidth": width, + "totalHeight": height, + "screenX": 0, + "screenY": 0, + "screenWidth": width, + "screenHeight": height, + "cornerRadius": 0, + "hasScreenMask": false, + })) + } + + pub async fn grpc_frame_stream( + &self, + id: &str, + max_edge: Option, + ) -> Result { + let avd_name = avd_from_id(id)?; + let port = self.grpc_port_for_avd(&avd_name)?; + let mut format = grpc::ImageFormat { + format: grpc::image_format::ImgFormat::Rgba8888 as i32, + width: 0, + height: 0, + display: 0, + transport: None, + }; + let target = self + .resolve_serial(&avd_name) + .ok() + .and_then(|serial| self.display_metrics_for_serial(&serial).ok()) + .map(|metrics| AndroidFrameTarget { + width: metrics.width.round().max(1.0) as u32, + height: metrics.height.round().max(1.0) as u32, + rotation_quarter_turns: metrics.rotation_quarter_turns, + }); + if let (Some(max_edge), Some(target)) = (max_edge, target) { + let max_edge = max_edge.clamp(240, 2400); + let largest = target.width.max(target.height); + if largest > max_edge { + if target.width >= target.height { + format.width = max_edge; + } else { + format.height = max_edge; + } + } + } + + let endpoint = Endpoint::from_shared(format!("http://127.0.0.1:{port}")) + .map_err(|error| AppError::native(format!("Invalid Android gRPC endpoint: {error}")))? + .connect() + .await + .map_err(|error| { + AppError::native(format!( + "Unable to connect to Android emulator gRPC: {error}" + )) + })?; + let mut grpc = tonic::client::Grpc::new(endpoint) + .max_decoding_message_size(ANDROID_GRPC_FRAME_MESSAGE_LIMIT); + grpc.ready().await.map_err(|error| { + AppError::native(format!("Android emulator gRPC is not ready: {error}")) + })?; + let path = PathAndQuery::from_static( + "/android.emulation.control.EmulatorController/streamScreenshot", + ); + let mut request = tonic::Request::new(format); + if let Some(token) = emulator_grpc_token(port) { + let value = MetadataValue::try_from(format!("Bearer {token}")).map_err(|error| { + AppError::native(format!("Invalid Android emulator gRPC token: {error}")) + })?; + request.metadata_mut().insert("authorization", value); + } + let response = grpc + .server_streaming(request, path, tonic::codec::ProstCodec::default()) + .await + .map_err(|error| { + AppError::native(format!( + "Android emulator screenshot stream failed: {error}" + )) + })?; + Ok(AndroidGrpcFrameStream { + inner: response.into_inner(), + target, + }) + } + + pub fn accessibility_tree( + &self, + id: &str, + max_depth: Option, + ) -> Result { + let serial = self.serial_for_id(id)?; + let raw = self.run_adb_shell( + &serial, + "uiautomator dump /sdcard/simdeck_ui.xml >/dev/null && cat /sdcard/simdeck_ui.xml", + )?; + let xml = extract_xml(&raw); + let document = roxmltree::Document::parse(xml).map_err(|error| { + AppError::native(format!("Unable to parse UIAutomator XML: {error}")) + })?; + let mut roots = Vec::new(); + let root = document.root_element(); + let max_depth = max_depth.unwrap_or(80).min(80); + for child in root.children().filter(|node| node.has_tag_name("node")) { + roots.push(android_node_value(child, 0, max_depth)); + } + let (width, height) = self.screen_size_for_serial(&serial)?; + if roots.is_empty() { + roots.push(json!({ + "type": "screen", + "role": "screen", + "frame": frame_value(0.0, 0.0, width, height), + "children": [], + })); + } + Ok(json!({ + "source": "android-uiautomator", + "availableSources": ["android-uiautomator"], + "roots": roots, + })) + } + + fn send_touch_grpc(&self, id: &str, x: f64, y: f64, phase: &str) -> Result<(), AppError> { + self.block_on_grpc(self.send_touch_grpc_async(id, x, y, phase)) + } + + async fn send_touch_grpc_async( + &self, + id: &str, + x: f64, + y: f64, + phase: &str, + ) -> Result<(), AppError> { + let avd_name = avd_from_id(id)?; + let serial = self.resolve_serial(&avd_name)?; + let (width, height) = self.screen_size_for_serial(&serial)?; + let pressure = match phase { + "began" | "moved" => 1, + "ended" | "cancelled" => 0, + _ => return Ok(()), + }; + let event = grpc::TouchEvent { + touches: vec![grpc::Touch { + x: normalized_to_pixel(x, width), + y: normalized_to_pixel(y, height), + identifier: ANDROID_TOUCH_IDENTIFIER, + pressure, + touch_major: 8, + touch_minor: 8, + expiration: grpc::touch::EventExpiration::NeverExpire as i32, + orientation: 0, + }], + display: 0, + }; + self.grpc_unary_for_avd::( + &avd_name, + "/android.emulation.control.EmulatorController/sendTouch", + event, + ) + .await?; + Ok(()) + } + + fn send_swipe_grpc( + &self, + id: &str, + start_x: f64, + start_y: f64, + end_x: f64, + end_y: f64, + duration_ms: u64, + ) -> Result<(), AppError> { + let duration_ms = duration_ms.clamp(50, 1500); + let steps = (duration_ms / 8).clamp(4, 120); + self.send_touch_grpc(id, start_x, start_y, "began")?; + for step in 1..steps { + let t = step as f64 / steps as f64; + self.send_touch_grpc( + id, + start_x + (end_x - start_x) * t, + start_y + (end_y - start_y) * t, + "moved", + )?; + thread::sleep(Duration::from_millis((duration_ms / steps).max(1))); + } + self.send_touch_grpc(id, end_x, end_y, "ended") + } + + fn block_on_grpc(&self, future: F) -> Result + where + F: Future>, + { + if let Ok(handle) = tokio::runtime::Handle::try_current() { + return handle.block_on(future); + } + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| AppError::internal(format!("Unable to create gRPC runtime: {error}")))? + .block_on(future) + } + + async fn grpc_unary_for_avd( + &self, + avd_name: &str, + path: &'static str, + request: Req, + ) -> Result + where + Req: prost::Message + Default + Send + 'static, + Resp: prost::Message + Default + Send + 'static, + { + let port = self.grpc_port_for_avd(avd_name)?; + let channel = grpc_channel_for_port(port)?; + let mut grpc = tonic::client::Grpc::new(channel); + grpc.ready().await.map_err(|error| { + AppError::native(format!("Android emulator gRPC is not ready: {error}")) + })?; + let mut request = tonic::Request::new(request); + if let Some(token) = emulator_grpc_token(port) { + let value = MetadataValue::try_from(format!("Bearer {token}")).map_err(|error| { + AppError::native(format!("Invalid Android emulator gRPC token: {error}")) + })?; + request.metadata_mut().insert("authorization", value); + } + let response = grpc + .unary( + request, + PathAndQuery::from_static(path), + tonic::codec::ProstCodec::default(), + ) + .await + .map_err(|error| { + AppError::native(format!("Android emulator gRPC input failed: {error}")) + })?; + Ok(response.into_inner()) + } + + fn device_value(&self, device: AndroidDevice) -> Value { + let id = id_for_avd(&device.avd_name); + let private_display = if let Some(serial) = device.serial.as_deref() { + let metrics = + self.display_metrics_for_serial(serial) + .unwrap_or(AndroidDisplayMetrics { + width: 0.0, + height: 0.0, + rotation_quarter_turns: 0, + }); + json!({ + "displayReady": metrics.width > 0.0 && metrics.height > 0.0, + "displayStatus": "Ready", + "displayWidth": metrics.width, + "displayHeight": metrics.height, + "frameSequence": 0, + "rotationQuarterTurns": 0, + }) + } else { + json!({ + "displayReady": false, + "displayStatus": "Boot required", + "displayWidth": 0, + "displayHeight": 0, + "frameSequence": 0, + "rotationQuarterTurns": 0, + }) + }; + json!({ + "udid": id, + "id": id, + "platform": "android-emulator", + "name": device.avd_name, + "state": if device.is_booted { "Booted" } else { "Shutdown" }, + "isBooted": device.is_booted, + "isAvailable": true, + "lastBootedAt": Value::Null, + "dataPath": self.avd_dir(&device.avd_name), + "logPath": Value::Null, + "deviceTypeIdentifier": "android-emulator", + "deviceTypeName": "Android Emulator", + "runtimeIdentifier": "android", + "runtimeName": "Android", + "android": { + "avdName": device.avd_name, + "serial": device.serial, + "grpcPort": device.grpc_port, + }, + "privateDisplay": private_display, + }) + } + + fn serial_for_id(&self, id: &str) -> Result { + self.resolve_serial(&avd_from_id(id)?) + } + + fn resolve_serial(&self, avd_name: &str) -> Result { + self.running_emulators()?.remove(avd_name).ok_or_else(|| { + AppError::native(format!("Android emulator `{avd_name}` is not running.")) + }) + } + + fn running_emulators(&self) -> Result, AppError> { + static CACHE: OnceLock>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((updated_at, running)) = cache.lock().unwrap().as_ref() { + if updated_at.elapsed() < RUNNING_EMULATOR_CACHE_TTL { + return Ok(running.clone()); + } + } + if !self.adb_path().exists() { + return Ok(HashMap::new()); + } + let output = self.run_adb(["devices"])?; + let mut result = HashMap::new(); + for line in output.lines().skip(1) { + let mut parts = line.split_whitespace(); + let Some(serial) = parts.next() else { continue }; + let Some(state) = parts.next() else { continue }; + if state != "device" || !serial.starts_with("emulator-") { + continue; + } + if let Ok(name_output) = self.run_adb(["-s", serial, "emu", "avd", "name"]) { + if let Some(name) = name_output + .lines() + .map(str::trim) + .find(|line| !line.is_empty() && *line != "OK") + { + result.insert(name.to_owned(), serial.to_owned()); + } + } + } + *cache.lock().unwrap() = Some((Instant::now(), result.clone())); + Ok(result) + } + + fn grpc_port_for_avd(&self, avd_name: &str) -> Result { + static CACHE: OnceLock>> = OnceLock::new(); + let cache = CACHE.get_or_init(|| Mutex::new(None)); + if let Some((updated_at, ports)) = cache.lock().unwrap().as_ref() { + if updated_at.elapsed() < AVD_GRPC_PORT_CACHE_TTL { + if let Some(port) = ports.get(avd_name) { + return Ok(*port); + } + } + } + + let ports = self + .run_emulator(["-list-avds"])? + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .enumerate() + .map(|(index, name)| (name.to_owned(), DEFAULT_GRPC_PORT_BASE + index as u16)) + .collect::>(); + let port = ports + .get(avd_name) + .copied() + .ok_or_else(|| AppError::not_found(format!("Unknown Android AVD `{avd_name}`.")))?; + *cache.lock().unwrap() = Some((Instant::now(), ports)); + Ok(port) + } + + fn screen_size_for_serial(&self, serial: &str) -> Result<(f64, f64), AppError> { + let metrics = self.display_metrics_for_serial(serial)?; + Ok((metrics.width, metrics.height)) + } + + fn display_metrics_for_serial(&self, serial: &str) -> Result { + let cache = android_display_metrics_cache(); + if let Some((updated_at, metrics)) = cache.lock().unwrap().get(serial) { + if updated_at.elapsed() < SCREEN_SIZE_CACHE_TTL { + return Ok(*metrics); + } + } + let output = self.run_adb(["-s", serial, "shell", "dumpsys", "display"])?; + let metrics = parse_android_display_metrics(&output) + .or_else(|| self.wm_display_metrics_for_serial(serial).ok()) + .ok_or_else(|| AppError::native("Android emulator did not report display metrics."))?; + cache + .lock() + .unwrap() + .insert(serial.to_owned(), (Instant::now(), metrics)); + Ok(metrics) + } + + fn wm_display_metrics_for_serial( + &self, + serial: &str, + ) -> Result { + let output = self.run_adb(["-s", serial, "shell", "wm", "size"])?; + let size = output + .split_whitespace() + .find(|part| part.contains('x')) + .ok_or_else(|| AppError::native("Android emulator did not report a screen size."))?; + let (width, height) = size + .split_once('x') + .ok_or_else(|| AppError::native("Android emulator reported an invalid screen size."))?; + let width = width + .parse::() + .map_err(|_| AppError::native("Android emulator reported an invalid width."))?; + let height = height + .parse::() + .map_err(|_| AppError::native("Android emulator reported an invalid height."))?; + Ok(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: 0, + }) + } + + fn invalidate_display_metrics_for_serial(&self, serial: &str) { + android_display_metrics_cache() + .lock() + .unwrap() + .remove(serial); + } + + fn run_adb_shell(&self, serial: &str, script: &str) -> Result { + self.run_adb(["-s", serial, "shell", script]) + } + + fn run_adb(&self, args: [&str; N]) -> Result { + run_command_text(self.adb_path(), args) + } + + fn run_adb_bytes(&self, args: [&str; N]) -> Result, AppError> { + run_command_bytes(self.adb_path(), args) + } + + fn run_emulator(&self, args: [&str; N]) -> Result { + run_command_text(self.emulator_path(), args) + } + + fn adb_path(&self) -> PathBuf { + sdk_root().join("platform-tools/adb") + } + + fn emulator_path(&self) -> PathBuf { + sdk_root().join("emulator/emulator") + } + + fn avd_dir(&self, avd_name: &str) -> PathBuf { + home_dir().join(format!(".android/avd/{avd_name}.avd")) + } +} + +impl AndroidGrpcFrameStream { + pub async fn next_frame(&mut self) -> Result, AppError> { + let Some(image) = self.inner.message().await.map_err(|error| { + AppError::native(format!( + "Android emulator screenshot stream failed: {error}" + )) + })? + else { + return Ok(None); + }; + let format = image.format.ok_or_else(|| { + AppError::native("Android emulator screenshot did not include an image format.") + })?; + let width = if format.width > 0 { + format.width + } else { + image.width + }; + let height = if format.height > 0 { + format.height + } else { + image.height + }; + if width == 0 || height == 0 { + return Err(AppError::native( + "Android emulator screenshot did not include dimensions.", + )); + } + let rgba = rgba_display_order( + &image.image, + width, + height, + grpc::image_format::ImgFormat::try_from(format.format) + .unwrap_or(grpc::image_format::ImgFormat::Rgba8888), + )?; + let (width, height, mut rgba) = + normalize_android_frame_orientation(width, height, rgba, self.target); + flatten_android_frame_alpha(&mut rgba, width, height); + Ok(Some(AndroidFrame { + width, + height, + timestamp_us: image.timestamp_us, + rgba, + })) + } +} + +fn normalize_android_frame_orientation( + width: u32, + height: u32, + mut rgba: Vec, + target: Option, +) -> (u32, u32, Vec) { + let Some(target) = target else { + return (width, height, rgba); + }; + if width == 0 || height == 0 || target.width == 0 || target.height == 0 { + return (width, height, rgba); + } + + let (width, height) = if (width > height) == (target.width > target.height) { + (width, height) + } else { + rgba = rotate_rgba_clockwise(&rgba, width, height); + (height, width) + }; + + if target.rotation_quarter_turns >= 2 { + rotate_rgba_180_in_place(&mut rgba, width, height); + } + + (width, height, rgba) +} + +fn rotate_rgba_clockwise(rgba: &[u8], width: u32, height: u32) -> Vec { + let width = width as usize; + let height = height as usize; + let mut out = vec![0; rgba.len()]; + for y in 0..height { + for x in 0..width { + let src = (y * width + x) * 4; + let dst_x = height - 1 - y; + let dst_y = x; + let dst = (dst_y * height + dst_x) * 4; + out[dst..dst + 4].copy_from_slice(&rgba[src..src + 4]); + } + } + out +} + +fn rotate_rgba_180_in_place(rgba: &mut [u8], width: u32, height: u32) { + let pixel_count = width as usize * height as usize; + for pixel in 0..(pixel_count / 2) { + let opposite = pixel_count - 1 - pixel; + for channel in 0..4 { + rgba.swap(pixel * 4 + channel, opposite * 4 + channel); + } + } +} + +fn flatten_android_frame_alpha(rgba: &mut [u8], width: u32, height: u32) { + if !rgba.chunks_exact(4).any(|pixel| pixel[3] != 255) { + return; + } + + let width = width as usize; + let height = height as usize; + let Some(default_fill) = first_opaque_rgb(rgba) else { + for pixel in rgba.chunks_exact_mut(4) { + pixel[3] = 255; + } + return; + }; + + for y in 0..height { + let row_start = y * width * 4; + let row = &mut rgba[row_start..row_start + width * 4]; + let mut fill = first_opaque_rgb(row).unwrap_or(default_fill); + for pixel in row.chunks_exact_mut(4) { + if pixel[3] == 255 { + fill = [pixel[0], pixel[1], pixel[2]]; + continue; + } + composite_pixel_over_rgb(pixel, fill); + } + } +} + +fn first_opaque_rgb(rgba: &[u8]) -> Option<[u8; 3]> { + rgba.chunks_exact(4) + .find(|pixel| pixel[3] == 255) + .map(|pixel| [pixel[0], pixel[1], pixel[2]]) +} + +fn composite_pixel_over_rgb(pixel: &mut [u8], background: [u8; 3]) { + let alpha = u32::from(pixel[3]); + if alpha == 0 { + pixel[0] = background[0]; + pixel[1] = background[1]; + pixel[2] = background[2]; + pixel[3] = 255; + return; + } + + for channel in 0..3 { + pixel[channel] = ((u32::from(pixel[channel]) * alpha + + u32::from(background[channel]) * (255 - alpha) + + 127) + / 255) as u8; + } + pixel[3] = 255; +} + +fn run_command_text(program: PathBuf, args: [&str; N]) -> Result { + let output = run_command(program, args)?; + String::from_utf8(output) + .map_err(|error| AppError::native(format!("Command returned non-UTF8 output: {error}"))) +} + +fn run_command_bytes( + program: PathBuf, + args: [&str; N], +) -> Result, AppError> { + run_command(program, args) +} + +fn run_command(program: PathBuf, args: [&str; N]) -> Result, AppError> { + if !program.exists() { + return Err(AppError::native(format!( + "Android SDK binary not found at {}.", + program.display() + ))); + } + let output = Command::new(&program) + .args(args) + .env("ANDROID_HOME", sdk_root()) + .env("ANDROID_SDK_ROOT", sdk_root()) + .env("JAVA_HOME", java_home()) + .output() + .map_err(|error| { + AppError::native(format!("Unable to run {}: {error}", program.display())) + })?; + if output.status.success() { + return Ok(output.stdout); + } + let stderr = String::from_utf8_lossy(&output.stderr); + let stdout = String::from_utf8_lossy(&output.stdout); + Err(AppError::native(format!( + "{} failed: {}{}", + program + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("Android command"), + stderr.trim(), + if stdout.trim().is_empty() { + String::new() + } else { + format!(" {}", stdout.trim()) + } + ))) +} + +fn grpc_channel_for_port(port: u16) -> Result { + static CHANNELS: OnceLock>> = OnceLock::new(); + let channels = CHANNELS.get_or_init(|| Mutex::new(HashMap::new())); + let mut channels = channels.lock().unwrap(); + if let Some(channel) = channels.get(&port) { + return Ok(channel.clone()); + } + let endpoint = Endpoint::from_shared(format!("http://127.0.0.1:{port}")) + .map_err(|error| AppError::native(format!("Invalid Android gRPC endpoint: {error}")))?; + let channel = endpoint.connect_lazy(); + channels.insert(port, channel.clone()); + Ok(channel) +} + +fn normalized_to_pixel(value: f64, extent: f64) -> i32 { + (value.clamp(0.0, 1.0) * (extent - 1.0).max(0.0)) + .round() + .max(0.0) as i32 +} + +fn sdk_root() -> PathBuf { + env::var_os("ANDROID_HOME") + .or_else(|| env::var_os("ANDROID_SDK_ROOT")) + .map(PathBuf::from) + .filter(|path| path.exists()) + .unwrap_or_else(|| home_dir().join("Library/Android/sdk")) +} + +fn java_home() -> OsString { + env::var_os("JAVA_HOME").unwrap_or_else(|| OsString::from("/opt/homebrew/opt/openjdk")) +} + +fn home_dir() -> PathBuf { + env::var_os("HOME") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new("/").to_path_buf()) +} + +fn emulator_grpc_token(port: u16) -> Option { + per_instance_grpc_token(port).or_else(global_grpc_token) +} + +fn per_instance_grpc_token(port: u16) -> Option { + let running_dir = home_dir().join("Library/Caches/TemporaryItems/avd/running"); + let entries = std::fs::read_dir(running_dir).ok()?; + let port_value = port.to_string(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|ext| ext.to_str()) != Some("ini") { + continue; + } + let contents = std::fs::read_to_string(path).ok()?; + let fields = parse_ini(&contents); + if fields.get("grpc.port") == Some(&port_value) { + if let Some(token) = fields.get("grpc.token").filter(|token| !token.is_empty()) { + return Some(token.to_owned()); + } + } + } + None +} + +fn global_grpc_token() -> Option { + std::fs::read_to_string(home_dir().join(".emulator_console_auth_token")) + .ok() + .map(|token| token.trim().to_owned()) + .filter(|token| !token.is_empty()) +} + +fn parse_ini(contents: &str) -> HashMap { + contents + .lines() + .filter_map(|line| { + let line = line.trim(); + let (key, value) = line.split_once('=')?; + Some((key.trim().to_owned(), value.trim().to_owned())) + }) + .collect() +} + +fn rgba_display_order( + image: &[u8], + width: u32, + height: u32, + format: grpc::image_format::ImgFormat, +) -> Result, AppError> { + let width = width as usize; + let height = height as usize; + match format { + grpc::image_format::ImgFormat::Rgba8888 => { + let row_len = width * 4; + if image.len() < row_len * height { + return Err(AppError::native( + "Android emulator returned a truncated RGBA frame.", + )); + } + Ok(image[..row_len * height].to_vec()) + } + grpc::image_format::ImgFormat::Rgb888 => { + let src_row_len = width * 3; + if image.len() < src_row_len * height { + return Err(AppError::native( + "Android emulator returned a truncated RGB frame.", + )); + } + let mut out = BytesMut::with_capacity(width * height * 4); + out.resize(width * height * 4, 255); + for y in 0..height { + let src_row = y * src_row_len; + let dst_row = y * width * 4; + for x in 0..width { + let src = src_row + x * 3; + let dst = dst_row + x * 4; + out[dst] = image[src]; + out[dst + 1] = image[src + 1]; + out[dst + 2] = image[src + 2]; + out[dst + 3] = 255; + } + } + Ok(out.to_vec()) + } + grpc::image_format::ImgFormat::Png => Err(AppError::native( + "Android emulator gRPC returned PNG instead of raw pixels.", + )), + } +} + +fn extract_xml(output: &str) -> &str { + output + .find(", depth: usize, max_depth: usize) -> Value { + let bounds = parse_bounds(node.attribute("bounds").unwrap_or("")); + let class_name = node.attribute("class").unwrap_or(""); + let short_class = class_name.rsplit('.').next().unwrap_or(class_name); + let text = node.attribute("text").unwrap_or(""); + let content_desc = node.attribute("content-desc").unwrap_or(""); + let label = if !text.is_empty() { text } else { content_desc }; + let resource_id = node.attribute("resource-id").unwrap_or(""); + let role = android_role(node, short_class); + let mut children = Vec::new(); + if depth < max_depth { + for child in node.children().filter(|child| child.has_tag_name("node")) { + children.push(android_node_value(child, depth + 1, max_depth)); + } + } + json!({ + "source": "android-uiautomator", + "type": android_type(short_class, class_name), + "role": role, + "className": class_name, + "AXIdentifier": resource_id, + "AXLabel": label, + "AXValue": text, + "androidClass": class_name, + "androidPackage": node.attribute("package").unwrap_or(""), + "androidResourceId": resource_id, + "checkable": bool_attr(node, "checkable"), + "checked": bool_attr(node, "checked"), + "clickable": bool_attr(node, "clickable"), + "focusable": bool_attr(node, "focusable"), + "focused": bool_attr(node, "focused"), + "longClickable": bool_attr(node, "long-clickable"), + "password": bool_attr(node, "password"), + "scrollable": bool_attr(node, "scrollable"), + "selected": bool_attr(node, "selected"), + "text": text, + "title": label, + "enabled": bool_attr(node, "enabled"), + "isHidden": node.attribute("visible-to-user") == Some("false"), + "frame": frame_value(bounds.0, bounds.1, bounds.2, bounds.3), + "frameInScreen": frame_value(bounds.0, bounds.1, bounds.2, bounds.3), + "children": children, + }) +} + +fn parse_bounds(value: &str) -> (f64, f64, f64, f64) { + let numbers = value + .replace("][", ",") + .replace(['[', ']'], "") + .split(',') + .filter_map(|part| part.parse::().ok()) + .collect::>(); + if numbers.len() != 4 { + return (0.0, 0.0, 0.0, 0.0); + } + ( + numbers[0], + numbers[1], + (numbers[2] - numbers[0]).max(0.0), + (numbers[3] - numbers[1]).max(0.0), + ) +} + +fn frame_value(x: f64, y: f64, width: f64, height: f64) -> Value { + json!({ "x": x, "y": y, "width": width, "height": height }) +} + +fn bool_attr(node: roxmltree::Node<'_, '_>, name: &str) -> bool { + node.attribute(name) == Some("true") +} + +fn android_type(short_class: &str, class_name: &str) -> String { + let fallback = if short_class.is_empty() { + class_name + } else { + short_class + }; + if fallback.is_empty() { + "View".to_owned() + } else { + fallback.to_owned() + } +} + +fn android_display_metrics_cache() -> &'static Mutex { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn parse_android_display_metrics(output: &str) -> Option { + let rotation = parse_android_display_rotation(output).unwrap_or(0); + if let Some(line) = output + .lines() + .find(|line| line.contains("mOverrideDisplayInfo=DisplayInfo")) + { + if let Some((width, height)) = parse_display_info_app_size(line) { + return Some(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: rotation, + }); + } + } + if let Some((width, height)) = output.lines().find_map(parse_logical_frame_size) { + return Some(AndroidDisplayMetrics { + width, + height, + rotation_quarter_turns: rotation, + }); + } + None +} + +fn parse_android_display_rotation(output: &str) -> Option { + output + .lines() + .find(|line| line.contains("mOverrideDisplayInfo=DisplayInfo")) + .and_then(parse_display_info_rotation) + .or_else(|| { + output + .lines() + .find_map(|line| { + line.split_once("mCurrentOrientation=") + .map(|(_, value)| value) + }) + .and_then(parse_rotation_token) + }) +} + +fn parse_display_info_app_size(line: &str) -> Option<(f64, f64)> { + let (_, value) = line.rsplit_once(", app ")?; + parse_size_prefix(value) +} + +fn parse_display_info_rotation(line: &str) -> Option { + let (_, value) = line.rsplit_once(", rotation ")?; + parse_rotation_token(value) +} + +fn parse_rotation_token(value: &str) -> Option { + let digits = value + .trim() + .chars() + .take_while(|character| character.is_ascii_digit()) + .collect::(); + let rotation = digits.parse::().ok()?; + Some(rotation % 4) +} + +fn parse_size_prefix(value: &str) -> Option<(f64, f64)> { + let mut parts = value.split_whitespace(); + let width = parts.next()?.trim_end_matches(',').parse::().ok()?; + if parts.next()? != "x" { + return None; + } + let height = parts.next()?.trim_end_matches(',').parse::().ok()?; + Some((width, height)) +} + +fn parse_logical_frame_size(line: &str) -> Option<(f64, f64)> { + let (_, value) = line.split_once("logicalFrame=Rect(")?; + let (frame, _) = value.split_once(')')?; + let (_, max_values) = frame.split_once(" - ")?; + let (width, height) = max_values.split_once(',')?; + Some((width.trim().parse().ok()?, height.trim().parse().ok()?)) +} + +fn android_role(node: roxmltree::Node<'_, '_>, class_name: &str) -> &'static str { + let clickable = bool_attr(node, "clickable"); + let scrollable = bool_attr(node, "scrollable"); + match class_name { + "Button" | "ImageButton" | "FloatingActionButton" => "button", + "EditText" => "textField", + "TextView" => "staticText", + "ImageView" => "image", + "CheckBox" => "checkBox", + "RadioButton" => "radioButton", + "Switch" | "ToggleButton" => "switch", + "SeekBar" => "slider", + "RecyclerView" | "ListView" | "GridView" => "collection", + "ScrollView" | "HorizontalScrollView" | "NestedScrollView" | "ViewPager" => "scrollView", + "WebView" => "webView", + "ProgressBar" => "progressIndicator", + "Spinner" => "popUpButton", + "TabWidget" | "TabLayout" => "tabGroup", + "Toolbar" | "ActionBar" => "toolbar", + "ViewGroup" | "FrameLayout" | "LinearLayout" | "RelativeLayout" | "ConstraintLayout" + | "CoordinatorLayout" | "DrawerLayout" => "container", + _ if scrollable => "scrollView", + _ if clickable => "button", + _ => "view", + } +} + +fn android_key_code(hid: u16) -> u16 { + match hid { + 4..=29 => 29 + (hid - 4), + 30 => 8, + 31 => 9, + 32 => 10, + 33 => 11, + 34 => 12, + 35 => 13, + 36 => 14, + 37 => 15, + 38 => 16, + 39 => 7, + 40 => 66, + 41 => 111, + 42 => 67, + 43 => 61, + 44 => 62, + 45 => 69, + 46 => 70, + 47 => 71, + 48 => 72, + 49 => 73, + 51 => 74, + 52 => 75, + 53 => 68, + 54 => 55, + 55 => 56, + 56 => 76, + 57 => 115, + 58..=69 => 131 + (hid - 58), + 73 => 124, + 74 => 122, + 75 => 92, + 76 => 112, + 77 => 123, + 78 => 93, + 79 => 22, + 80 => 21, + 81 => 20, + 82 => 19, + _ => hid, + } +} + +fn hid_text_for_key(hid: u16, modifiers: u32) -> Option { + if modifiers & (MODIFIER_CONTROL | MODIFIER_OPTION | MODIFIER_COMMAND) != 0 { + return None; + } + + let shifted = modifiers & MODIFIER_SHIFT != 0; + let caps_locked = modifiers & MODIFIER_CAPS_LOCK != 0; + + if (4..=29).contains(&hid) { + let offset = (hid - 4) as u8; + let base = if shifted ^ caps_locked { b'A' } else { b'a' }; + return Some(char::from(base + offset).to_string()); + } + + let text = match (hid, shifted) { + (30, false) => "1", + (30, true) => "!", + (31, false) => "2", + (31, true) => "@", + (32, false) => "3", + (32, true) => "#", + (33, false) => "4", + (33, true) => "$", + (34, false) => "5", + (34, true) => "%", + (35, false) => "6", + (35, true) => "^", + (36, false) => "7", + (36, true) => "&", + (37, false) => "8", + (37, true) => "*", + (38, false) => "9", + (38, true) => "(", + (39, false) => "0", + (39, true) => ")", + (44, _) => " ", + (45, false) => "-", + (45, true) => "_", + (46, false) => "=", + (46, true) => "+", + (47, false) => "[", + (47, true) => "{", + (48, false) => "]", + (48, true) => "}", + (49, false) => "\\", + (49, true) => "|", + (51, false) => ";", + (51, true) => ":", + (52, false) => "'", + (52, true) => "\"", + (53, false) => "`", + (53, true) => "~", + (54, false) => ",", + (54, true) => "<", + (55, false) => ".", + (55, true) => ">", + (56, false) => "/", + (56, true) => "?", + _ => return None, + }; + Some(text.to_owned()) +} + +fn android_input_text_arg(text: &str) -> String { + let mut escaped = String::new(); + for character in text.chars() { + match character { + ' ' => escaped.push_str("%s"), + '%' => escaped.push_str("%25"), + '&' | '(' | ')' | '<' | '>' | ';' | '|' | '*' | '\\' | '"' | '\'' | '`' | '$' => { + escaped.push('\\'); + escaped.push(character); + } + _ => escaped.push(character), + } + } + escaped +} + +fn has_android_key_modifiers(modifiers: u32) -> bool { + modifiers & (MODIFIER_SHIFT | MODIFIER_CONTROL | MODIFIER_OPTION | MODIFIER_COMMAND) != 0 +} + +fn android_modifier_key_codes(modifiers: u32) -> Vec { + let mut keys = Vec::new(); + if modifiers & MODIFIER_CONTROL != 0 { + keys.push(113); + } + if modifiers & MODIFIER_OPTION != 0 { + keys.push(57); + } + if modifiers & MODIFIER_SHIFT != 0 { + keys.push(59); + } + if modifiers & MODIFIER_COMMAND != 0 { + keys.push(117); + } + keys +} + +fn android_log_level(line: &str) -> &'static str { + if line.contains(" E ") { + "error" + } else if line.contains(" W ") { + "warning" + } else if line.contains(" D ") { + "debug" + } else { + "info" + } +} + +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', "'\\''")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn android_nodes_keep_class_type_and_semantic_role() { + let document = roxmltree::Document::parse( + r#""#, + ) + .unwrap(); + + let value = android_node_value(document.root_element(), 0, 10); + + assert_eq!(value["type"], "ViewGroup"); + assert_eq!(value["role"], "container"); + assert_eq!(value["AXIdentifier"], "com.example:id/hotseat"); + assert_eq!(value["androidClass"], "android.view.ViewGroup"); + assert_eq!(value["androidResourceId"], "com.example:id/hotseat"); + assert_eq!(value["enabled"], true); + } + + #[test] + fn clickable_unknown_android_nodes_are_buttons() { + let document = roxmltree::Document::parse( + r#""#, + ) + .unwrap(); + + let value = android_node_value(document.root_element(), 0, 10); + + assert_eq!(value["type"], "CustomTile"); + assert_eq!(value["role"], "button"); + } + + #[test] + fn android_key_code_maps_usb_hid_keyboard_usages() { + assert_eq!(android_key_code(4), 29); + assert_eq!(android_key_code(29), 54); + assert_eq!(android_key_code(30), 8); + assert_eq!(android_key_code(39), 7); + assert_eq!(android_key_code(40), 66); + assert_eq!(android_key_code(42), 67); + assert_eq!(android_key_code(58), 131); + assert_eq!(android_key_code(69), 142); + assert_eq!(android_key_code(73), 124); + assert_eq!(android_key_code(79), 22); + } + + #[test] + fn hid_text_for_key_uses_shift_and_caps_for_printable_input() { + assert_eq!(hid_text_for_key(4, 0).as_deref(), Some("a")); + assert_eq!(hid_text_for_key(4, MODIFIER_SHIFT).as_deref(), Some("A")); + assert_eq!( + hid_text_for_key(4, MODIFIER_CAPS_LOCK).as_deref(), + Some("A") + ); + assert_eq!( + hid_text_for_key(4, MODIFIER_SHIFT | MODIFIER_CAPS_LOCK).as_deref(), + Some("a") + ); + assert_eq!(hid_text_for_key(30, 0).as_deref(), Some("1")); + assert_eq!(hid_text_for_key(30, MODIFIER_SHIFT).as_deref(), Some("!")); + assert_eq!(hid_text_for_key(56, MODIFIER_SHIFT).as_deref(), Some("?")); + assert_eq!(hid_text_for_key(80, 0), None); + assert_eq!(hid_text_for_key(4, MODIFIER_COMMAND), None); + } + + #[test] + fn android_input_text_arg_escapes_adb_shell_text() { + assert_eq!(android_input_text_arg("hello world"), "hello%sworld"); + assert_eq!(android_input_text_arg("100%"), "100%25"); + assert_eq!(android_input_text_arg("a&b"), "a\\&b"); + } + + #[test] + fn android_modifier_key_codes_match_android_meta_keys() { + assert_eq!(android_modifier_key_codes(MODIFIER_CONTROL), vec![113]); + assert_eq!(android_modifier_key_codes(MODIFIER_OPTION), vec![57]); + assert_eq!(android_modifier_key_codes(MODIFIER_SHIFT), vec![59]); + assert_eq!(android_modifier_key_codes(MODIFIER_COMMAND), vec![117]); + assert_eq!( + android_modifier_key_codes(MODIFIER_CONTROL | MODIFIER_SHIFT), + vec![113, 59] + ); + } + + #[test] + fn parse_android_display_metrics_prefers_current_app_size() { + let output = r#" + mViewports=[DisplayViewport{type=INTERNAL, logicalFrame=Rect(0, 0 - 2400, 1080), physicalFrame=Rect(0, 0 - 2400, 1080)}] + mCurrentOrientation=3 + mOverrideDisplayInfo=DisplayInfo{"Built-in Screen", real 2400 x 1080, largest app 2400 x 2400, smallest app 1080 x 1080, rotation 3, state ON, app 2400 x 1080, density 420} +"#; + + assert_eq!( + parse_android_display_metrics(output), + Some(AndroidDisplayMetrics { + width: 2400.0, + height: 1080.0, + rotation_quarter_turns: 3, + }) + ); + } + + #[test] + fn parse_android_display_metrics_falls_back_to_logical_frame() { + let output = r#" + mViewports=[DisplayViewport{type=INTERNAL, logicalFrame=Rect(0, 0 - 1080, 2400), physicalFrame=Rect(0, 0 - 1080, 2400)}] + mCurrentOrientation=0 +"#; + + assert_eq!( + parse_android_display_metrics(output), + Some(AndroidDisplayMetrics { + width: 1080.0, + height: 2400.0, + rotation_quarter_turns: 0, + }) + ); + } + + #[test] + fn android_frame_orientation_rotates_when_stream_aspect_is_swapped() { + let rgba = vec![ + 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, // + 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255, + ]; + + let (width, height, rotated) = normalize_android_frame_orientation( + 3, + 2, + rgba, + Some(AndroidFrameTarget { + width: 2, + height: 3, + rotation_quarter_turns: 0, + }), + ); + + assert_eq!((width, height), (2, 3)); + assert_eq!( + rotated, + vec![ + 4, 0, 0, 255, 1, 0, 0, 255, // + 5, 0, 0, 255, 2, 0, 0, 255, // + 6, 0, 0, 255, 3, 0, 0, 255, + ] + ); + } + + #[test] + fn android_frame_orientation_keeps_matching_aspect() { + let rgba = vec![1, 2, 3, 255, 5, 6, 7, 255]; + + let (width, height, out) = normalize_android_frame_orientation( + 1, + 2, + rgba.clone(), + Some(AndroidFrameTarget { + width: 1080, + height: 2400, + rotation_quarter_turns: 0, + }), + ); + + assert_eq!((width, height), (1, 2)); + assert_eq!(out, rgba); + } + + #[test] + fn android_frame_alpha_flattens_transparent_edges_to_nearest_row_pixel() { + let mut rgba = vec![0, 0, 0, 0, 10, 20, 30, 255, 40, 50, 60, 255, 0, 0, 0, 0]; + + flatten_android_frame_alpha(&mut rgba, 4, 1); + + assert_eq!( + rgba, + vec![10, 20, 30, 255, 10, 20, 30, 255, 40, 50, 60, 255, 40, 50, 60, 255,] + ); + } + + #[test] + fn android_frame_alpha_composites_partial_alpha() { + let mut rgba = vec![100, 50, 0, 255, 200, 200, 200, 128]; + + flatten_android_frame_alpha(&mut rgba, 2, 1); + + assert_eq!(rgba, vec![100, 50, 0, 255, 150, 125, 100, 255]); + } + + #[test] + fn android_frame_orientation_rotates_reverse_orientations_180() { + let rgba = vec![ + 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, // + 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255, + ]; + + let (width, height, rotated) = normalize_android_frame_orientation( + 3, + 2, + rgba, + Some(AndroidFrameTarget { + width: 3, + height: 2, + rotation_quarter_turns: 3, + }), + ); + + assert_eq!((width, height), (3, 2)); + assert_eq!( + rotated, + vec![ + 6, 0, 0, 255, 5, 0, 0, 255, 4, 0, 0, 255, // + 3, 0, 0, 255, 2, 0, 0, 255, 1, 0, 0, 255, + ] + ); + } +} + +mod grpc { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Empty {} + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Touch { + #[prost(int32, tag = "1")] + pub x: i32, + #[prost(int32, tag = "2")] + pub y: i32, + #[prost(int32, tag = "3")] + pub identifier: i32, + #[prost(int32, tag = "4")] + pub pressure: i32, + #[prost(int32, tag = "5")] + pub touch_major: i32, + #[prost(int32, tag = "6")] + pub touch_minor: i32, + #[prost(enumeration = "touch::EventExpiration", tag = "7")] + pub expiration: i32, + #[prost(int32, tag = "8")] + pub orientation: i32, + } + + pub mod touch { + #[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, + )] + #[repr(i32)] + pub enum EventExpiration { + Unspecified = 0, + NeverExpire = 1, + } + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct TouchEvent { + #[prost(message, repeated, tag = "1")] + pub touches: Vec, + #[prost(int32, tag = "2")] + pub display: i32, + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ImageTransport { + #[prost(enumeration = "image_transport::TransportChannel", tag = "1")] + pub channel: i32, + #[prost(string, tag = "2")] + pub handle: String, + } + + pub mod image_transport { + #[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, + )] + #[repr(i32)] + pub enum TransportChannel { + Unspecified = 0, + Mmap = 1, + } + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct ImageFormat { + #[prost(enumeration = "image_format::ImgFormat", tag = "1")] + pub format: i32, + #[prost(uint32, tag = "3")] + pub width: u32, + #[prost(uint32, tag = "4")] + pub height: u32, + #[prost(uint32, tag = "5")] + pub display: u32, + #[prost(message, optional, tag = "6")] + pub transport: Option, + } + + pub mod image_format { + #[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration, + )] + #[repr(i32)] + pub enum ImgFormat { + Png = 0, + Rgba8888 = 1, + Rgb888 = 2, + } + } + + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct Image { + #[prost(message, optional, tag = "1")] + pub format: Option, + #[prost(uint32, tag = "2")] + pub width: u32, + #[prost(uint32, tag = "3")] + pub height: u32, + #[prost(bytes = "vec", tag = "4")] + pub image: Vec, + #[prost(uint32, tag = "5")] + pub seq: u32, + #[prost(uint64, tag = "6")] + pub timestamp_us: u64, + } +} diff --git a/server/src/api/routes.rs b/server/src/api/routes.rs index a74ad820..5eed4f85 100644 --- a/server/src/api/routes.rs +++ b/server/src/api/routes.rs @@ -1,3 +1,4 @@ +use crate::android::{self, AndroidBridge}; use crate::api::json::json; use crate::auth; use crate::config::Config; @@ -11,6 +12,7 @@ use crate::simulators::registry::SessionRegistry; use crate::simulators::session::SimulatorSession; use crate::static_files; use crate::transport::packet::FramePacket; +use crate::transport::webrtc::AndroidWebRtcSource; use crate::webkit; use axum::body::Body; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; @@ -33,7 +35,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpStream; use tokio::process::Command; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{broadcast, mpsc, Mutex}; use tokio::task; use tokio::time::timeout; use tower_http::trace::TraceLayer; @@ -54,6 +56,7 @@ pub struct AppState { pub inspectors: InspectorHub, pub metrics: Arc, pub simulator_inventory: SimulatorInventoryCache, + pub android: AndroidBridge, } #[derive(Clone, Default)] @@ -1314,9 +1317,9 @@ async fn inspector_response( } async fn list_simulators(State(state): State) -> Result, AppError> { - let simulators = list_simulators_cached(state.clone(), false).await?; + let simulators = all_device_values(state.clone(), false).await?; Ok(json(json_value!({ - "simulators": state.registry.enrich_simulators(simulators), + "simulators": simulators, }))) } @@ -1324,6 +1327,15 @@ async fn boot_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state.clone(), move |android| { + android.boot(&action_udid)?; + android.wait_until_booted(&action_udid, Duration::from_secs(120)) + }) + .await?; + return simulator_payload(state, udid).await; + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state.clone(), move |bridge| { @@ -1337,6 +1349,11 @@ async fn shutdown_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state.clone(), move |android| android.shutdown(&action_udid)).await?; + return simulator_payload(state, udid).await; + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state.clone(), move |bridge| { @@ -1350,6 +1367,11 @@ async fn erase_simulator( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| android.erase(&action_udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } forget_lifecycle_session(&state, &udid); let action_udid = udid.clone(); run_bridge_action(state, move |bridge| bridge.erase_simulator(&action_udid)).await?; @@ -1374,6 +1396,14 @@ async fn install_app( "Request body must include `appPath`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.install_app(&action_udid, &payload.app_path) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.install_app(&action_udid, &payload.app_path) @@ -1392,6 +1422,14 @@ async fn uninstall_app( "Request body must include `bundleId`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.uninstall_app(&action_udid, &payload.bundle_id) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.uninstall_app(&action_udid, &payload.bundle_id) @@ -1404,6 +1442,10 @@ async fn get_pasteboard( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let text = run_android_action(state, move |android| android.pasteboard_text(&udid)).await?; + return Ok(json(json_value!({ "text": text }))); + } let text = run_bridge_action(state, move |bridge| bridge.pasteboard_text(&udid)).await?; Ok(json(json_value!({ "text": text }))) } @@ -1413,6 +1455,13 @@ async fn set_pasteboard( Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.set_pasteboard_text(&udid, &payload.text) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { bridge.set_pasteboard_text(&udid, &payload.text) }) @@ -1424,7 +1473,11 @@ async fn screenshot_png( State(state): State, Path(udid): Path, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { - let png = run_bridge_action(state, move |bridge| bridge.screenshot_png(&udid)).await?; + let png = if android::is_android_id(&udid) { + run_android_action(state, move |android| android.screenshot_png(&udid)).await? + } else { + run_bridge_action(state, move |bridge| bridge.screenshot_png(&udid)).await? + }; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); headers.insert( @@ -1438,6 +1491,10 @@ async fn toggle_appearance( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.toggle_appearance(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| bridge.toggle_appearance(&action_udid)).await?; Ok(json(json_value!({ "ok": true }))) @@ -1447,6 +1504,9 @@ async fn refresh_stream( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + return Ok(json(json_value!({ "ok": true, "stream": "screenshot" }))); + } let session = state.registry.get_or_create_async(&udid).await?; if let Err(error) = session.ensure_started_async().await { state.registry.remove(&udid); @@ -1464,6 +1524,14 @@ async fn open_url( if payload.url.trim().is_empty() { return Err(AppError::bad_request("Request body must include `url`.")); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.open_url(&action_udid, &payload.url) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.open_url(&action_udid, &payload.url) @@ -1482,6 +1550,14 @@ async fn launch_bundle( "Request body must include `bundleId`.", )); } + if android::is_android_id(&udid) { + let action_udid = udid.clone(); + run_android_action(state, move |android| { + android.launch_package(&action_udid, &payload.bundle_id) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } let action_udid = udid.clone(); run_bridge_action(state, move |bridge| { bridge.launch_bundle(&action_udid, &payload.bundle_id) @@ -1612,6 +1688,13 @@ async fn send_touch( let x = payload.x.clamp(0.0, 1.0); let y = payload.y.clamp(0.0, 1.0); let phase = payload.phase; + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_touch(&udid, x, y, &phase) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; input.send_touch(x, y, &phase) @@ -1642,6 +1725,24 @@ async fn send_touch_sequence( )); } } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + for event in payload.events { + android.send_touch( + &udid, + event.x.clamp(0.0, 1.0), + event.y.clamp(0.0, 1.0), + &event.phase, + )?; + if let Some(delay_ms) = event.delay_ms_after.filter(|delay_ms| *delay_ms > 0) { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for event in payload.events { @@ -1665,6 +1766,10 @@ async fn control_socket( Path(udid): Path, websocket: WebSocketUpgrade, ) -> impl IntoResponse { + if android::is_android_id(&udid) { + return websocket + .on_upgrade(move |socket| handle_android_control_socket(state, udid, socket)); + } websocket.on_upgrade(move |socket| handle_control_socket(state, udid, socket)) } @@ -1677,12 +1782,182 @@ async fn h264_socket( websocket.on_upgrade(move |socket| handle_h264_socket(state, udid, query, socket)) } +async fn handle_android_control_socket(state: AppState, udid: String, socket: WebSocket) { + let (mut sender, mut receiver) = socket.split(); + let mut active_touch: Option = None; + let _ = sender + .send(Message::Text( + json_value!({ "type": "ready", "udid": udid, "platform": "android-emulator" }) + .to_string() + .into(), + )) + .await; + while let Some(message) = receiver.next().await { + let text = match message { + Ok(Message::Text(text)) => text, + Ok(Message::Binary(bytes)) => match String::from_utf8(bytes.to_vec()) { + Ok(text) => text.into(), + Err(_) => continue, + }, + Ok(Message::Close(_)) => break, + Ok(Message::Ping(_)) | Ok(Message::Pong(_)) => continue, + Err(_) => break, + }; + let Ok(control_message) = serde_json::from_str::(&text) else { + continue; + }; + let state = state.clone(); + let udid = udid.clone(); + let _ = run_android_control_message(state, udid, control_message, &mut active_touch).await; + } +} + +struct AndroidControlTouch { + started_at: Instant, + start_x: f64, + start_y: f64, + latest_x: f64, + latest_y: f64, +} + +async fn run_android_control_message( + state: AppState, + udid: String, + message: ControlMessage, + active_touch: &mut Option, +) -> Result<(), AppError> { + match message { + ControlMessage::Touch { x, y, phase } => { + handle_android_control_touch(state, udid, x, y, phase, active_touch).await + } + ControlMessage::EdgeTouch { x, y, phase, .. } => { + handle_android_control_touch(state, udid, x, y, phase, active_touch).await + } + ControlMessage::MultiTouch { x1, y1, phase, .. } => { + handle_android_control_touch(state, udid, x1, y1, phase, active_touch).await + } + other => { + run_android_action(state, move |android| match other { + ControlMessage::Key { + key_code, + modifiers, + } => android.send_key(&udid, key_code, modifiers.unwrap_or(0)), + ControlMessage::Button { + button, + duration_ms, + phase, + .. + } => match phase.as_deref() { + Some("down" | "began") => Ok(()), + Some("up" | "ended" | "cancelled") | None => { + android.press_button(&udid, &button, duration_ms.unwrap_or(0)) + } + Some(_) => Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )), + }, + ControlMessage::DismissKeyboard => android.dismiss_keyboard(&udid), + ControlMessage::Home => android.press_home(&udid), + ControlMessage::AppSwitcher => android.open_app_switcher(&udid), + ControlMessage::RotateLeft => android.rotate_left(&udid), + ControlMessage::RotateRight => android.rotate_right(&udid), + ControlMessage::ToggleAppearance => android.toggle_appearance(&udid), + ControlMessage::Touch { .. } + | ControlMessage::EdgeTouch { .. } + | ControlMessage::MultiTouch { .. } => Ok(()), + }) + .await + } + } +} + +async fn handle_android_control_touch( + state: AppState, + udid: String, + x: f64, + y: f64, + phase: String, + active_touch: &mut Option, +) -> Result<(), AppError> { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + let x = x.clamp(0.0, 1.0); + let y = y.clamp(0.0, 1.0); + match phase.as_str() { + "began" => { + *active_touch = Some(AndroidControlTouch { + started_at: Instant::now(), + start_x: x, + start_y: y, + latest_x: x, + latest_y: y, + }); + run_android_action(state, move |android| { + android.send_touch(&udid, x, y, "began") + }) + .await + } + "moved" => { + if let Some(touch) = active_touch.as_mut() { + touch.latest_x = x; + touch.latest_y = y; + } + run_android_action(state, move |android| { + android.send_touch(&udid, x, y, "moved") + }) + .await + } + "ended" => { + let touch = active_touch.take().unwrap_or(AndroidControlTouch { + started_at: Instant::now(), + start_x: x, + start_y: y, + latest_x: x, + latest_y: y, + }); + let end_x = x; + let end_y = y; + let distance = + ((end_x - touch.start_x).powi(2) + (end_y - touch.start_y).powi(2)).sqrt(); + let duration_ms = touch.started_at.elapsed().as_millis().clamp(80, 1500) as u64; + run_android_action(state, move |android| { + if distance >= 0.025 { + android + .send_touch(&udid, end_x, end_y, "ended") + .or_else(|_| { + android.send_swipe( + &udid, + touch.start_x, + touch.start_y, + end_x, + end_y, + duration_ms, + ) + }) + } else { + android.send_touch(&udid, end_x, end_y, "ended") + } + }) + .await + } + "cancelled" => { + *active_touch = None; + Ok(()) + } + _ => Ok(()), + } +} + async fn webrtc_offer( State(state): State, + ConnectInfo(address): ConnectInfo, Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { - crate::transport::webrtc::create_answer(state, udid, payload) + crate::transport::webrtc::create_answer(state, udid, payload, address.ip().is_loopback()) .await .map(Json) } @@ -1693,6 +1968,11 @@ async fn handle_h264_socket( initial_quality: StreamQualityPayload, socket: WebSocket, ) { + if android::is_android_id(&udid) { + handle_android_h264_socket(state, udid, initial_quality, socket).await; + return; + } + if initial_quality.has_any_value() { if let Err(error) = apply_stream_quality_payload(&state, &initial_quality) { tracing::debug!("Failed to apply H264 WebSocket stream quality for {udid}: {error}"); @@ -1804,6 +2084,115 @@ async fn handle_h264_socket( } } +async fn handle_android_h264_socket( + state: AppState, + udid: String, + initial_quality: StreamQualityPayload, + socket: WebSocket, +) { + let source = match AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + initial_quality.max_edge, + true, + ) + .await + { + Ok(source) => source, + Err(error) => { + tracing::debug!("Failed to create Android H264 WebSocket source for {udid}: {error}"); + return; + } + }; + + let mut subscription = source.subscribe(); + let (mut sender, mut receiver) = socket.split(); + let mut decoder_synced = false; + let mut last_sent_sequence: Option = None; + + let initial_keyframe = source + .wait_for_keyframe(H264_WS_KEYFRAME_WAIT_TIMEOUT) + .await + .filter(|frame| h264_ws_frame_is_decoder_sync(frame)); + + if let Some(keyframe) = initial_keyframe { + if h264_ws_frame_is_supported(&keyframe) { + let message_bytes = h264_ws_frame_message(&keyframe); + let message = Message::Binary(message_bytes); + if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) + .await + .ok() + .and_then(Result::ok) + .is_none() + { + return; + } + last_sent_sequence = Some(keyframe.frame_sequence); + decoder_synced = true; + } + } else { + source.request_keyframe(); + } + + loop { + tokio::select! { + received = receiver.next() => { + let Some(received) = received else { break }; + let message = match received { + Ok(message) => message, + Err(error) => { + tracing::debug!("Android H264 WebSocket closed for {udid}: {error}"); + break; + } + }; + if !handle_android_h264_socket_message(&state, &source, &message) { + break; + } + } + frame = subscription.recv() => { + let frame = match frame { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(_)) => { + decoder_synced = false; + source.request_keyframe(); + continue; + } + Err(broadcast::error::RecvError::Closed) => break, + }; + if !h264_ws_frame_is_supported(&frame) { + continue; + } + if last_sent_sequence + .map(|sequence| frame.frame_sequence <= sequence) + .unwrap_or(false) + { + continue; + } + if !decoder_synced && !frame.is_keyframe { + source.request_keyframe(); + continue; + } + let is_keyframe = frame.is_keyframe; + let message_bytes = h264_ws_frame_message(&frame); + let message = Message::Binary(message_bytes); + if timeout(H264_WS_SEND_TIMEOUT, sender.send(message)) + .await + .ok() + .and_then(Result::ok) + .is_none() + { + break; + } + last_sent_sequence = Some(frame.frame_sequence); + if is_keyframe { + decoder_synced = true; + } + } + } + } +} + fn handle_h264_socket_message( state: &AppState, session: &SimulatorSession, @@ -1849,6 +2238,47 @@ fn handle_h264_socket_message( true } +fn handle_android_h264_socket_message( + state: &AppState, + source: &AndroidWebRtcSource, + message: &Message, +) -> bool { + let text = match message { + Message::Text(text) => text.as_str(), + Message::Binary(bytes) => match std::str::from_utf8(bytes) { + Ok(text) => text, + Err(_) => return true, + }, + Message::Close(_) => return false, + Message::Ping(_) | Message::Pong(_) => return true, + }; + let Ok(message) = serde_json::from_str::(text) else { + return true; + }; + match message { + H264SocketMessage::ClientStats { stats } => { + if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { + state.metrics.record_client_stream_stats(*stats); + } + } + H264SocketMessage::StreamControl { + force_keyframe, + snapshot, + } => { + if force_keyframe.unwrap_or(false) { + source.request_keyframe(); + } + if snapshot.unwrap_or(false) { + source.request_refresh(); + } + } + H264SocketMessage::StreamQuality { config: _ } => { + source.request_keyframe(); + } + } + true +} + #[derive(Debug, Deserialize)] #[serde(tag = "type", rename_all = "camelCase")] enum H264SocketMessage { @@ -2106,6 +2536,13 @@ async fn send_key( Path(udid): Path, Json(payload): Json, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_key(&udid, payload.key_code, payload.modifiers.unwrap_or(0)) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { bridge.send_key(&udid, payload.key_code, payload.modifiers.unwrap_or(0)) }) @@ -2128,6 +2565,21 @@ async fn send_key_sequence( "Key sequence cannot contain more than 512 key codes.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let delay_ms = payload.delay_ms.unwrap_or(0); + let key_count = payload.key_codes.len(); + for (index, key_code) in payload.key_codes.into_iter().enumerate() { + android.send_key(&udid, key_code, 0)?; + if delay_ms > 0 && index + 1 < key_count { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let delay_ms = payload.delay_ms.unwrap_or(0); @@ -2148,6 +2600,10 @@ async fn dismiss_keyboard( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.dismiss_keyboard(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.send_key(&udid, 41, 0)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2160,6 +2616,13 @@ async fn press_button( if payload.button.trim().is_empty() { return Err(AppError::bad_request("Request body must include `button`.")); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.press_button(&udid, &payload.button, payload.duration_ms.unwrap_or(0)) + }) + .await?; + return Ok(json(json_value!({ "ok": true }))); + } if let Some(phase) = payload.phase.as_deref() { let pressed = match phase { "down" | "began" => true, @@ -2193,6 +2656,10 @@ async fn press_home( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.press_home(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.press_home(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2201,6 +2668,10 @@ async fn open_app_switcher( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_app_switcher(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.open_app_switcher(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2209,6 +2680,10 @@ async fn rotate_right( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_right(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.rotate_right(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2217,6 +2692,10 @@ async fn rotate_left( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_left(&udid)).await?; + return Ok(json(json_value!({ "ok": true }))); + } run_bridge_action(state, move |bridge| bridge.rotate_left(&udid)).await?; Ok(json(json_value!({ "ok": true }))) } @@ -2225,6 +2704,11 @@ async fn chrome_profile( State(state): State, Path(udid): Path, ) -> Result, AppError> { + if android::is_android_id(&udid) { + let profile = + run_android_action(state, move |android| android.chrome_profile(&udid)).await?; + return Ok(json(profile)); + } let profile = run_bridge_action(state, move |bridge| bridge.chrome_profile(&udid)).await?; Ok(json(json_value!(profile))) } @@ -2234,6 +2718,11 @@ async fn chrome_png( Path(udid): Path, Query(query): Query, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + if android::is_android_id(&udid) { + return Err(AppError::not_found( + "Android emulators do not expose device chrome assets.", + )); + } let include_buttons = query .buttons .as_deref() @@ -2305,6 +2794,11 @@ async fn screen_mask_png( State(state): State, Path(udid): Path, ) -> Result<(StatusCode, HeaderMap, Vec), AppError> { + if android::is_android_id(&udid) { + return Err(AppError::not_found( + "Android emulators do not expose screen mask assets.", + )); + } let png = run_bridge_action(state, move |bridge| bridge.screen_mask_png(&udid)).await?; let mut headers = HeaderMap::new(); headers.insert(header::CONTENT_TYPE, "image/png".parse().unwrap()); @@ -2339,6 +2833,22 @@ async fn accessibility_tree_value( max_depth: Option, include_hidden: bool, ) -> Result { + if android::is_android_id(&udid) { + let requested_source = source + .filter(|source| *source != "auto") + .map(|source| source.to_owned()); + return run_android_action(state, move |android| { + let mut tree = android.accessibility_tree(&udid, max_depth)?; + if include_hidden { + tree["includeHidden"] = Value::Bool(true); + } + if let Some(source) = requested_source { + tree["requestedSource"] = Value::String(source); + } + Ok(tree) + }) + .await; + } let requested_source = AccessibilityHierarchySource::parse(source)?; let max_depth = max_depth.map(|depth| depth.min(80)); @@ -2477,6 +2987,13 @@ async fn accessibility_point( )); } + if android::is_android_id(&udid) { + let snapshot = run_android_action(state, move |android| { + android.accessibility_tree(&udid, None) + }) + .await?; + return Ok(json(snapshot)); + } let snapshot = accessibility_snapshot(state, udid, Some((query.x, query.y)), None).await?; Ok(json(snapshot)) } @@ -2525,6 +3042,17 @@ async fn perform_tap_payload( tap_point_from_snapshot(&snapshot, &payload.selector)? }; + if android::is_android_id(&udid) { + return run_android_action(state, move |android| { + android.send_touch(&udid, x, y, "began")?; + if duration_ms > 0 { + std::thread::sleep(Duration::from_millis(duration_ms)); + } + android.send_touch(&udid, x, y, "ended") + }) + .await; + } + run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; input.send_touch(x, y, "began")?; @@ -2607,6 +3135,13 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul key_code, modifiers, } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_key(&udid, key_code, modifiers.unwrap_or(0)) + }) + .await?; + return Ok(json_value!({ "action": "key" })); + } run_bridge_action(state, move |bridge| { bridge.send_key(&udid, key_code, modifiers.unwrap_or(0)) }) @@ -2625,6 +3160,21 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "keySequence cannot contain more than 512 key codes.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let delay_ms = delay_ms.unwrap_or(0); + let key_count = key_codes.len(); + for (index, key_code) in key_codes.into_iter().enumerate() { + android.send_key(&udid, key_code, 0)?; + if delay_ms > 0 && index + 1 < key_count { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "keySequence" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let delay_ms = delay_ms.unwrap_or(0); @@ -2653,6 +3203,28 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "touch requires finite normalized x and y.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + let x = x.clamp(0.0, 1.0); + let y = y.clamp(0.0, 1.0); + if down.unwrap_or(false) || up.unwrap_or(false) { + if down.unwrap_or(false) { + android.send_touch(&udid, x, y, "began")?; + } + if down.unwrap_or(false) && up.unwrap_or(false) { + std::thread::sleep(Duration::from_millis(delay_ms.unwrap_or(100))); + } + if up.unwrap_or(false) { + android.send_touch(&udid, x, y, "ended")?; + } + } else { + android.send_touch(&udid, x, y, phase.as_deref().unwrap_or("began"))?; + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "touch" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; let x = x.clamp(0.0, 1.0); @@ -2684,6 +3256,31 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "touchSequence cannot contain more than 64 events.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + for event in events { + if !event.x.is_finite() || !event.y.is_finite() { + return Err(AppError::bad_request( + "touchSequence requires finite normalized x and y.", + )); + } + android.send_touch( + &udid, + event.x.clamp(0.0, 1.0), + event.y.clamp(0.0, 1.0), + &event.phase, + )?; + if let Some(delay_ms) = + event.delay_ms_after.filter(|delay_ms| *delay_ms > 0) + { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + }) + .await?; + return Ok(json_value!({ "action": "touchSequence" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for event in events { @@ -2723,6 +3320,20 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul "swipe requires finite normalized coordinates.", )); } + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_swipe( + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms.unwrap_or(350), + ) + }) + .await?; + return Ok(json_value!({ "action": "swipe" })); + } run_bridge_action(state, move |bridge| { let step_count = steps.unwrap_or(12).max(1); let delay = @@ -2755,6 +3366,20 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul } => { let (start_x, start_y, end_x, end_y, default_duration_ms) = normalized_gesture_coordinates(&preset, delta)?; + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.send_swipe( + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms.unwrap_or(default_duration_ms), + ) + }) + .await?; + return Ok(json_value!({ "action": "gesture", "preset": preset })); + } run_bridge_action(state, move |bridge| { let step_count = steps.unwrap_or(12).max(1); let delay = Duration::from_millis( @@ -2777,6 +3402,23 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul Ok(json_value!({ "action": "gesture", "preset": preset })) } BatchStep::Type { text, delay_ms } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + if delay_ms.is_some() { + for character in text.chars() { + android.type_text(&udid, &character.to_string())?; + if let Some(delay_ms) = delay_ms.filter(|delay_ms| *delay_ms > 0) { + std::thread::sleep(Duration::from_millis(delay_ms)); + } + } + Ok(()) + } else { + android.type_text(&udid, &text) + } + }) + .await?; + return Ok(json_value!({ "action": "type" })); + } run_bridge_action(state, move |bridge| { let input = bridge.create_input_session(&udid)?; for character in text.chars() { @@ -2799,6 +3441,13 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul button, duration_ms, } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.press_button(&udid, &button, duration_ms.unwrap_or(0)) + }) + .await?; + return Ok(json_value!({ "action": "button" })); + } run_bridge_action(state, move |bridge| { bridge.press_button(&udid, &button, duration_ms.unwrap_or(0)) }) @@ -2806,34 +3455,69 @@ async fn run_batch_step(state: AppState, udid: String, step: BatchStep) -> Resul Ok(json_value!({ "action": "button" })) } BatchStep::Launch { bundle_id } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| { + android.launch_package(&udid, &bundle_id) + }) + .await?; + return Ok(json_value!({ "action": "launch" })); + } run_bridge_action(state, move |bridge| bridge.launch_bundle(&udid, &bundle_id)).await?; Ok(json_value!({ "action": "launch" })) } BatchStep::OpenUrl { url } => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_url(&udid, &url)).await?; + return Ok(json_value!({ "action": "openUrl" })); + } run_bridge_action(state, move |bridge| bridge.open_url(&udid, &url)).await?; Ok(json_value!({ "action": "openUrl" })) } BatchStep::Home => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.press_home(&udid)).await?; + return Ok(json_value!({ "action": "home" })); + } run_bridge_action(state, move |bridge| bridge.press_home(&udid)).await?; Ok(json_value!({ "action": "home" })) } BatchStep::DismissKeyboard => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.dismiss_keyboard(&udid)).await?; + return Ok(json_value!({ "action": "dismissKeyboard" })); + } run_bridge_action(state, move |bridge| bridge.send_key(&udid, 41, 0)).await?; Ok(json_value!({ "action": "dismissKeyboard" })) } BatchStep::AppSwitcher => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.open_app_switcher(&udid)).await?; + return Ok(json_value!({ "action": "appSwitcher" })); + } run_bridge_action(state, move |bridge| bridge.open_app_switcher(&udid)).await?; Ok(json_value!({ "action": "appSwitcher" })) } BatchStep::RotateLeft => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_left(&udid)).await?; + return Ok(json_value!({ "action": "rotateLeft" })); + } run_bridge_action(state, move |bridge| bridge.rotate_left(&udid)).await?; Ok(json_value!({ "action": "rotateLeft" })) } BatchStep::RotateRight => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.rotate_right(&udid)).await?; + return Ok(json_value!({ "action": "rotateRight" })); + } run_bridge_action(state, move |bridge| bridge.rotate_right(&udid)).await?; Ok(json_value!({ "action": "rotateRight" })) } BatchStep::ToggleAppearance => { + if android::is_android_id(&udid) { + run_android_action(state, move |android| android.toggle_appearance(&udid)).await?; + return Ok(json_value!({ "action": "toggleAppearance" })); + } run_bridge_action(state, move |bridge| bridge.toggle_appearance(&udid)).await?; Ok(json_value!({ "action": "toggleAppearance" })) } @@ -3270,6 +3954,10 @@ async fn simulator_logs( Query(query): Query, ) -> Result, AppError> { let limit = query.limit.unwrap_or(250).clamp(1, 1000); + if android::is_android_id(&udid) { + let entries = run_android_action(state, move |android| android.logs(&udid, limit)).await?; + return Ok(json(json_value!({ "entries": entries }))); + } let filters = LogFilters::new( split_filter_values(query.levels.as_deref()), split_filter_values(query.processes.as_deref()), @@ -4543,6 +5231,28 @@ where })? } +async fn run_android_action(state: AppState, action: F) -> Result +where + F: FnOnce(AndroidBridge) -> Result + Send + 'static, + T: Send + 'static, +{ + let android = state.android.clone(); + task::spawn_blocking(move || action(android)) + .await + .map_err(|error| { + AppError::internal(format!("Failed to join Android bridge task: {error}")) + })? +} + +async fn all_device_values(state: AppState, force_refresh: bool) -> Result, AppError> { + let ios = list_simulators_cached(state.clone(), force_refresh).await?; + let mut values = state.registry.enrich_simulators(ios); + let android_devices = + run_android_action(state.clone(), |android| android.list_devices()).await?; + values.extend(state.android.enrich_devices(android_devices)); + Ok(values) +} + async fn list_simulators_cached( state: AppState, force_refresh: bool, @@ -4575,8 +5285,7 @@ async fn accessibility_snapshot( } async fn simulator_payload(state: AppState, udid: String) -> Result, AppError> { - let simulators = list_simulators_cached(state.clone(), true).await?; - let enriched = state.registry.enrich_simulators(simulators); + let enriched = all_device_values(state.clone(), true).await?; let simulator = enriched .into_iter() .find(|entry| entry.get("udid").and_then(Value::as_str) == Some(udid.as_str())) diff --git a/server/src/main.rs b/server/src/main.rs index 6b0f6b99..b85947e0 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,3 +1,4 @@ +mod android; mod api; mod auth; mod config; @@ -815,8 +816,9 @@ fn studio_stream_quality_profile( }) } -fn command_service_url(explicit: Option) -> anyhow::Result { +fn command_service_url(explicit: Option<&str>) -> anyhow::Result { if let Some(url) = explicit + .map(ToOwned::to_owned) .or_else(|| env::var("SIMDECK_SERVER_URL").ok()) .filter(|value| !value.trim().is_empty()) { @@ -825,6 +827,18 @@ fn command_service_url(explicit: Option) -> anyhow::Result { Ok(ensure_project_daemon(DaemonLaunchOptions::default())?.http_url) } +fn command_service_url_for_udid( + udid: &str, + explicit: &Option, + service_url: &Option, +) -> anyhow::Result> { + if android::is_android_id(udid) { + Ok(Some(command_service_url(explicit.as_deref())?)) + } else { + Ok(service_url.clone()) + } +} + impl Default for DaemonLaunchOptions { fn default() -> Self { Self { @@ -2031,7 +2045,7 @@ fn main() -> anyhow::Result<()> { CoreSimulatorCommand::Restart => core_simulator::restart(), }, Command::List => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let simulators = service_get_json(&service_url, "/api/simulators")? .get("simulators") .cloned() @@ -2043,7 +2057,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Boot { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "boot", &Value::Null)?; println!( "{}", @@ -2054,7 +2068,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Shutdown { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "shutdown", &Value::Null)?; println!( "{}", @@ -2065,7 +2079,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::OpenUrl { udid, url } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_open_url(&service_url, &udid, &url)?; println!( "{}", @@ -2076,7 +2090,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Launch { udid, bundle_id } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_launch(&service_url, &udid, &bundle_id)?; println!( "{}", @@ -2087,7 +2101,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::ToggleAppearance { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "toggle-appearance", &Value::Null)?; println_json( &serde_json::json!({ "ok": true, "udid": udid, "action": "toggle-appearance" }), @@ -2095,13 +2109,13 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Erase { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok(&service_url, &udid, "erase", &Value::Null)?; println_json(&serde_json::json!({ "ok": true, "udid": udid, "action": "erase" }))?; Ok(()) } Command::Install { udid, app_path } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok( &service_url, &udid, @@ -2114,7 +2128,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Uninstall { udid, bundle_id } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; service_post_ok( &service_url, &udid, @@ -2128,7 +2142,7 @@ fn main() -> anyhow::Result<()> { } Command::Pasteboard { command } => match command { PasteboardCommand::Get { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let text = service_get_json( &service_url, &format!("/api/simulators/{}/pasteboard", url_path_component(&udid)), @@ -2146,7 +2160,7 @@ fn main() -> anyhow::Result<()> { stdin, file, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let text = read_text_input(text, stdin, file)?; service_post_ok( &service_url, @@ -2165,7 +2179,7 @@ fn main() -> anyhow::Result<()> { seconds, limit, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let filters = native::bridge::LogFilters::new(Vec::new(), Vec::new(), String::new()); let _ = filters; let entries = service_get_json( @@ -2186,7 +2200,7 @@ fn main() -> anyhow::Result<()> { output, stdout, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let png = service_get_bytes( &service_url, &format!( @@ -2221,7 +2235,7 @@ fn main() -> anyhow::Result<()> { include_hidden, direct, } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let snapshot = describe_ui_snapshot( &bridge, &udid, @@ -2245,7 +2259,13 @@ fn main() -> anyhow::Result<()> { up, delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + if android_device && !normalized { + anyhow::bail!("Android touch coordinates require --normalized."); + } + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { if down || up { let mut events = Vec::new(); if down { @@ -2300,8 +2320,10 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; if let (Some(server_url), Some(x), Some(y), true, None, None, None, None) = ( - service_url.as_deref(), + command_server_url.as_deref(), x, y, normalized, @@ -2313,7 +2335,7 @@ fn main() -> anyhow::Result<()> { sleep_ms(pre_delay_ms); service_tap(server_url, &udid, x, y, duration_ms)?; sleep_ms(post_delay_ms); - } else if let Some(server_url) = service_url.as_deref() { + } else if let Some(server_url) = command_server_url.as_deref() { sleep_ms(pre_delay_ms); service_tap_element( server_url, @@ -2375,18 +2397,41 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + if android_device && !normalized { + anyhow::bail!("Android swipe coordinates require --normalized."); + } + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { sleep_ms(pre_delay_ms); - service_swipe( - server_url, - &udid, - start_x, - start_y, - end_x, - end_y, - duration_ms, - steps, - )?; + if android_device { + service_batch( + server_url, + &udid, + vec![serde_json::json!({ + "action": "swipe", + "startX": start_x, + "startY": start_y, + "endX": end_x, + "endY": end_y, + "durationMs": duration_ms, + "steps": steps, + })], + false, + )?; + } else { + service_swipe( + server_url, + &udid, + start_x, + start_y, + end_x, + end_y, + duration_ms, + steps, + )?; + } sleep_ms(post_delay_ms); } else { let (start_x, start_y) = @@ -2421,7 +2466,33 @@ fn main() -> anyhow::Result<()> { pre_delay_ms, post_delay_ms, } => { - if let Some(server_url) = service_url.as_deref().filter(|_| normalized) { + let android_device = android::is_android_id(&udid); + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if android_device { + let server_url = command_server_url + .as_deref() + .ok_or_else(|| anyhow::anyhow!("Android command requires SimDeck daemon."))?; + sleep_ms(pre_delay_ms); + service_batch( + server_url, + &udid, + vec![serde_json::json!({ + "action": "gesture", + "preset": preset, + "durationMs": duration_ms, + "delta": delta, + "steps": 4, + })], + false, + )?; + sleep_ms(post_delay_ms); + println_json( + &serde_json::json!({ "ok": true, "udid": udid, "action": "gesture", "preset": preset }), + )?; + return Ok(()); + } + if let Some(server_url) = command_server_url.as_deref().filter(|_| normalized) { let gesture = gesture_coordinates( &bridge, &udid, @@ -2484,6 +2555,9 @@ fn main() -> anyhow::Result<()> { duration_ms, steps, } => { + if android::is_android_id(&udid) { + anyhow::bail!("Android pinch gestures are not supported by the ADB input bridge."); + } let frames = pinch_frames( &bridge, &udid, @@ -2509,6 +2583,9 @@ fn main() -> anyhow::Result<()> { duration_ms, steps, } => { + if android::is_android_id(&udid) { + anyhow::bail!("Android rotate gestures are not supported by the ADB input bridge."); + } let frames = rotate_gesture_frames( &bridge, &udid, @@ -2537,7 +2614,9 @@ fn main() -> anyhow::Result<()> { } => { let key_code = parse_hid_key(&key)?; sleep_ms(pre_delay_ms); - if let Some(server_url) = service_url.as_deref().filter(|_| duration_ms == 0) { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref().filter(|_| duration_ms == 0) { service_key(server_url, &udid, key_code, modifiers)?; } else if duration_ms > 0 && modifiers == 0 { let input = bridge.create_input_session(&udid)?; @@ -2557,7 +2636,9 @@ fn main() -> anyhow::Result<()> { delay_ms, } => { let keys = parse_key_list(&keycodes)?; - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_key_sequence(server_url, &udid, &keys, delay_ms)?; } else { let input = bridge.create_input_session(&udid)?; @@ -2580,7 +2661,9 @@ fn main() -> anyhow::Result<()> { } => { let modifier_mask = parse_modifier_mask(&modifiers)?; let key_code = parse_hid_key(&key)?; - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_key(server_url, &udid, key_code, modifier_mask)?; } else { bridge.send_key(&udid, key_code, modifier_mask)?; @@ -2596,7 +2679,21 @@ fn main() -> anyhow::Result<()> { delay_ms, } => { let text = read_text_input(text, stdin, file)?; - type_text(&bridge, &udid, &text, delay_ms)?; + if android::is_android_id(&udid) { + let server_url = command_service_url(explicit_server_url.as_deref())?; + service_batch( + &server_url, + &udid, + vec![serde_json::json!({ + "action": "type", + "text": text, + "delayMs": delay_ms, + })], + false, + )?; + } else { + type_text(&bridge, &udid, &text, delay_ms)?; + } println_json(&serde_json::json!({ "ok": true, "udid": udid, "action": "type" }))?; Ok(()) } @@ -2605,7 +2702,9 @@ fn main() -> anyhow::Result<()> { button, duration_ms, } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_button(server_url, &udid, &button, duration_ms)?; } else { bridge.press_button(&udid, &button, duration_ms)?; @@ -2622,7 +2721,9 @@ fn main() -> anyhow::Result<()> { stdin, continue_on_error, } => { - let report = if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + let report = if let Some(server_url) = command_server_url.as_deref() { let step_lines = read_batch_steps(steps, file, stdin)?; service_batch( server_url, @@ -2637,7 +2738,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::DismissKeyboard { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "dismiss-keyboard", &Value::Null)?; } else { bridge.send_key(&udid, 41, 0)?; @@ -2651,7 +2754,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::Home { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "home", &Value::Null)?; } else { bridge.press_home(&udid)?; @@ -2660,7 +2765,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::AppSwitcher { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "app-switcher", &Value::Null)?; } else { bridge.open_app_switcher(&udid)?; @@ -2671,7 +2778,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::RotateLeft { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "rotate-left", &Value::Null)?; } else { bridge.rotate_left(&udid)?; @@ -2682,7 +2791,9 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::RotateRight { udid } => { - if let Some(server_url) = service_url.as_deref() { + let command_server_url = + command_service_url_for_udid(&udid, &explicit_server_url, &service_url)?; + if let Some(server_url) = command_server_url.as_deref() { service_post_ok(server_url, &udid, "rotate-right", &Value::Null)?; } else { bridge.rotate_right(&udid)?; @@ -2693,7 +2804,7 @@ fn main() -> anyhow::Result<()> { Ok(()) } Command::ChromeProfile { udid } => { - let service_url = command_service_url(explicit_server_url.clone())?; + let service_url = command_service_url(explicit_server_url.as_deref())?; let profile = service_get_json( &service_url, &format!( @@ -5179,6 +5290,7 @@ async fn serve( inspectors, metrics, simulator_inventory: Default::default(), + android: Default::default(), }; let http_router = app_router( diff --git a/server/src/native/ffi.rs b/server/src/native/ffi.rs index 2d0659fb..9d6b2df1 100644 --- a/server/src/native/ffi.rs +++ b/server/src/native/ffi.rs @@ -268,6 +268,23 @@ unsafe extern "C" { error_message: *mut *mut c_char, ) -> bool; + pub fn xcw_native_h264_encoder_create( + callback: Option, + user_data: *mut c_void, + error_message: *mut *mut c_char, + ) -> *mut c_void; + pub fn xcw_native_h264_encoder_destroy(handle: *mut c_void); + pub fn xcw_native_h264_encoder_encode_rgba( + handle: *mut c_void, + rgba: *const u8, + length: usize, + width: u32, + height: u32, + timestamp_us: u64, + error_message: *mut *mut c_char, + ) -> bool; + pub fn xcw_native_h264_encoder_request_keyframe(handle: *mut c_void); + pub fn xcw_native_free_string(value: *mut c_char); pub fn xcw_native_free_bytes(bytes: xcw_native_owned_bytes); pub fn xcw_native_release_shared_bytes(bytes: xcw_native_shared_bytes); diff --git a/server/src/transport/webrtc.rs b/server/src/transport/webrtc.rs index d0d13553..e6864107 100644 --- a/server/src/transport/webrtc.rs +++ b/server/src/transport/webrtc.rs @@ -1,23 +1,29 @@ +use crate::android; use crate::api::routes::{ apply_stream_quality_payload, run_control_message, AppState, ControlMessage, StreamQualityPayload, }; use crate::error::AppError; use crate::metrics::counters::ClientStreamStats; -use bytes::Bytes; +use crate::native::ffi; +use crate::transport::packet::{FramePacket, SharedFrame}; +use bytes::{BufMut, Bytes, BytesMut}; use serde::{Deserialize, Serialize}; use std::collections::{HashMap, VecDeque}; -use std::sync::atomic::Ordering; -use std::sync::{Arc, Mutex, OnceLock}; +use std::ffi::{c_void, CStr}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, RwLock, Weak}; use std::time::Duration; use tokio::sync::{broadcast, mpsc}; use tokio::task; -use tokio::time; +use tokio::time::{self, Instant}; use tracing::{info, warn}; use webrtc::api::interceptor_registry::register_default_interceptors; use webrtc::api::media_engine::{MediaEngine, MIME_TYPE_H264}; use webrtc::api::APIBuilder; +use webrtc::data_channel::data_channel_init::RTCDataChannelInit; use webrtc::data_channel::data_channel_message::DataChannelMessage; +use webrtc::data_channel::data_channel_state::RTCDataChannelState; use webrtc::data_channel::RTCDataChannel; use webrtc::ice_transport::ice_server::RTCIceServer; use webrtc::interceptor::registry::Registry; @@ -43,6 +49,7 @@ const ANNEX_B_START_CODE: &[u8] = &[0, 0, 0, 1]; const DEFAULT_STUN_URL: &str = "stun:stun.l.google.com:19302"; const WEBRTC_CONTROL_CHANNEL_LABEL: &str = "simdeck-control"; const WEBRTC_TELEMETRY_CHANNEL_LABEL: &str = "simdeck-telemetry"; +const WEBRTC_RGBA_CHANNEL_LABEL: &str = "simdeck-rgba"; const WEBRTC_DEFAULT_LOCAL_STREAM_FPS: u32 = 60; const WEBRTC_MAX_LOCAL_STREAM_FPS: u32 = 240; const WEBRTC_WRITE_TIMEOUT: Duration = Duration::from_millis(120); @@ -53,6 +60,15 @@ const WEBRTC_FAST_ICE_GATHER_TIMEOUT: Duration = Duration::from_millis(250); const WEBRTC_FULL_ICE_GATHER_TIMEOUT: Duration = Duration::from_secs(3); const WEBRTC_RTP_OUTBOUND_MTU: usize = 1200; const WEBRTC_PEER_DISCONNECTED_TIMEOUT: Duration = Duration::from_secs(12); +const ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY: usize = 128; +const ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY: usize = 8; +const ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES: usize = 48; +const ANDROID_WEBRTC_RGBA_CHUNK_BYTES: usize = 256 * 1024; +const ANDROID_WEBRTC_RGBA_CHUNK_MAGIC: u32 = 0x5344_5243; // "SDRC" +const ANDROID_WEBRTC_RGBA_VERSION: u8 = 1; +const ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888: u8 = 1; +const ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT: usize = 2; +const ANDROID_WEBRTC_FPS: u64 = 30; static WEBRTC_MEDIA_STREAMS: OnceLock>>> = OnceLock::new(); const MAX_WEBRTC_MEDIA_STREAMS_PER_UDID: usize = 16; @@ -81,6 +97,14 @@ pub struct WebRtcAnswerPayload { pub sdp: String, #[serde(rename = "type")] pub kind: String, + pub video: WebRtcVideoMetadata, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct WebRtcVideoMetadata { + pub width: u32, + pub height: u32, } #[derive(Debug, Clone, Serialize)] @@ -97,26 +121,63 @@ pub async fn create_answer( state: AppState, udid: String, payload: WebRtcOfferPayload, + peer_is_loopback: bool, ) -> Result { if payload.kind != "offer" { return Err(AppError::bad_request( "WebRTC payload must include type `offer`.", )); } - - let session = state.registry.get_or_create_async(&udid).await?; - if let Err(error) = session.ensure_started_async().await { - state.registry.remove(&udid); - return Err(error); + let is_android = android::is_android_id(&udid); + let transport = payload + .transport + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()); + if transport.is_some_and(|value| value.eq_ignore_ascii_case("rgba")) { + if !is_android { + return Err(AppError::bad_request( + "RGBA WebRTC transport is only supported for Android emulators.", + )); + } + if !peer_is_loopback { + return Err(AppError::bad_request( + "RGBA WebRTC transport is only available to loopback clients.", + )); + } + return create_android_rgba_answer(state, udid, payload).await; } if payload.transport.is_some() { return Err(AppError::bad_request( - "WebRTC preview supports media tracks only.", + "Unsupported WebRTC transport. Supported transports are media tracks and Android loopback RGBA.", )); } - if let Some(stream_config) = payload.stream_config.as_ref() { - apply_stream_quality_payload(&state, stream_config)?; + if !is_android { + if let Some(stream_config) = payload.stream_config.as_ref() { + apply_stream_quality_payload(&state, stream_config)?; + } } + + let source = if is_android { + WebRtcVideoSource::Android( + AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + android_webrtc_max_edge(), + true, + ) + .await?, + ) + } else { + let session = state.registry.get_or_create_async(&udid).await?; + if let Err(error) = session.ensure_started_async().await { + state.registry.remove(&udid); + return Err(error); + } + WebRtcVideoSource::Simulator(session) + }; + info!( "WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", count_sdp_candidates(&payload.sdp), @@ -128,9 +189,9 @@ pub async fn create_answer( ice_transport_policy_label() ); - let first_frame = wait_for_h264_sync_keyframe(&session, WEBRTC_INITIAL_KEYFRAME_TIMEOUT) + let first_frame = wait_for_h264_sync_keyframe(&source, WEBRTC_INITIAL_KEYFRAME_TIMEOUT) .await - .ok_or_else(|| AppError::native("Timed out waiting for a simulator H.264 keyframe."))?; + .ok_or_else(|| AppError::native("Timed out waiting for a device H.264 keyframe."))?; let codec = first_frame .codec .as_deref() @@ -179,13 +240,22 @@ pub async fn create_answer( ); register_diagnostics(&peer_connection, &udid); let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); - register_control_data_channel( - &peer_connection, - session.clone(), - state.clone(), - udid.clone(), - stream_control_tx, - ); + match &source { + WebRtcVideoSource::Simulator(session) => register_control_data_channel( + &peer_connection, + session.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ), + WebRtcVideoSource::Android(source) => register_android_data_channel( + &peer_connection, + source.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ), + } let video_track = Arc::new(TrackLocalStaticRTP::new( RTCRtpCodecCapability { @@ -203,7 +273,7 @@ pub async fn create_answer( .add_track(video_track.clone() as Arc) .await .map_err(|error| AppError::internal(format!("add WebRTC video track: {error}")))?; - let rtcp_session = session.clone(); + let rtcp_source = source.clone(); let rtcp_udid = udid.clone(); tokio::spawn(async move { while let Ok((packets, _attributes)) = rtp_sender.read_rtcp().await { @@ -212,7 +282,7 @@ pub async fn create_answer( .any(|packet| rtcp_packet_requests_keyframe(packet.as_ref())) { info!("WebRTC RTCP requested keyframe for {rtcp_udid}"); - rtcp_session.request_keyframe(); + rtcp_source.request_keyframe(); } } }); @@ -258,13 +328,15 @@ pub async fn create_answer( summarize_sdp_candidate_types(&local_description.sdp) ); + let first_frame_width = first_frame.width; + let first_frame_height = first_frame.height; let (cancellation_token, cancellation) = register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true); tokio::spawn( WebRtcMediaStream { state, udid, - session, + source, first_frame, peer_connection, video_track, @@ -278,6 +350,131 @@ pub async fn create_answer( Ok(WebRtcAnswerPayload { sdp: local_description.sdp, kind: "answer".to_owned(), + video: WebRtcVideoMetadata { + width: first_frame_width, + height: first_frame_height, + }, + }) +} + +async fn create_android_rgba_answer( + state: AppState, + udid: String, + payload: WebRtcOfferPayload, +) -> Result { + let source = AndroidWebRtcSource::start( + state.android.clone(), + state.metrics.clone(), + udid.clone(), + None, + false, + ) + .await?; + info!( + "Android RGBA WebRTC offer for {udid}: remote_candidates={} remote_candidate_types={} ice_servers={} ice_transport_policy={}", + count_sdp_candidates(&payload.sdp), + summarize_sdp_candidate_types(&payload.sdp), + std::env::var("SIMDECK_WEBRTC_ICE_SERVERS") + .ok() + .filter(|value| !value.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_STUN_URL.to_owned()), + ice_transport_policy_label() + ); + + let api = APIBuilder::new().build(); + let peer_connection = Arc::new( + api.new_peer_connection(RTCConfiguration { + ice_servers: ice_servers(), + ice_transport_policy: ice_transport_policy(), + ..Default::default() + }) + .await + .map_err(|error| AppError::internal(format!("create WebRTC peer connection: {error}")))?, + ); + register_diagnostics(&peer_connection, &udid); + let (stream_control_tx, stream_control_rx) = mpsc::unbounded_channel(); + register_android_data_channel( + &peer_connection, + source.clone(), + state.clone(), + udid.clone(), + stream_control_tx, + ); + let rgba_channel = peer_connection + .create_data_channel( + WEBRTC_RGBA_CHANNEL_LABEL, + Some(RTCDataChannelInit { + ordered: Some(false), + max_retransmits: Some(0), + ..Default::default() + }), + ) + .await + .map_err(|error| AppError::internal(format!("create RGBA WebRTC data channel: {error}")))?; + + let fast_gather = + has_sdp_candidate_type(&payload.sdp, "host") && ice_transport_policy_label() == "all"; + let offer = RTCSessionDescription::offer(payload.sdp) + .map_err(|error| AppError::bad_request(format!("invalid WebRTC offer: {error}")))?; + peer_connection + .set_remote_description(offer) + .await + .map_err(|error| AppError::bad_request(format!("set remote WebRTC offer: {error}")))?; + let answer = peer_connection + .create_answer(None) + .await + .map_err(|error| AppError::internal(format!("create WebRTC answer: {error}")))?; + let mut gather_complete = peer_connection.gathering_complete_promise().await; + peer_connection + .set_local_description(answer) + .await + .map_err(|error| AppError::internal(format!("set WebRTC answer: {error}")))?; + let gather_timeout = if fast_gather { + WEBRTC_FAST_ICE_GATHER_TIMEOUT + } else { + WEBRTC_FULL_ICE_GATHER_TIMEOUT + }; + let gather_result = time::timeout(gather_timeout, gather_complete.recv()).await; + let mut local_description = peer_connection + .local_description() + .await + .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; + if gather_result.is_err() && count_sdp_candidates(&local_description.sdp) == 0 { + let _ = time::timeout(WEBRTC_FULL_ICE_GATHER_TIMEOUT, gather_complete.recv()).await; + local_description = peer_connection + .local_description() + .await + .ok_or_else(|| AppError::internal("WebRTC local description was not set."))?; + } + info!( + "Android RGBA WebRTC answer for {udid}: local_candidates={} local_candidate_types={}", + count_sdp_candidates(&local_description.sdp), + summarize_sdp_candidate_types(&local_description.sdp) + ); + + let (cancellation_token, cancellation) = + register_webrtc_media_stream(&udid, payload.client_id.as_deref(), true); + tokio::spawn( + WebRtcRgbaStream { + state, + udid, + source, + peer_connection, + rgba_channel, + cancellation_token, + cancellation, + stream_control_rx, + } + .run(), + ); + + Ok(WebRtcAnswerPayload { + sdp: local_description.sdp, + kind: "answer".to_owned(), + video: WebRtcVideoMetadata { + width: 0, + height: 0, + }, }) } @@ -481,6 +678,196 @@ fn attach_control_data_channel( })); } +fn register_android_data_channel( + peer_connection: &Arc, + source: AndroidWebRtcSource, + state: AppState, + udid: String, + stream_control_tx: mpsc::UnboundedSender, +) { + peer_connection.on_data_channel(Box::new(move |channel: Arc| { + let source = source.clone(); + let state = state.clone(); + let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); + Box::pin(async move { + let label = channel.label(); + if label != WEBRTC_CONTROL_CHANNEL_LABEL && label != WEBRTC_TELEMETRY_CHANNEL_LABEL { + return; + } + attach_android_data_channel(channel, source, state, udid, stream_control_tx); + }) + })); +} + +fn attach_android_data_channel( + channel: Arc, + source: AndroidWebRtcSource, + state: AppState, + udid: String, + stream_control_tx: mpsc::UnboundedSender, +) { + let (control_tx, control_rx) = mpsc::unbounded_channel::(); + task::spawn(run_android_webrtc_control_queue( + state.clone(), + udid.clone(), + control_rx, + )); + channel.on_message(Box::new(move |message: DataChannelMessage| { + let source = source.clone(); + let state = state.clone(); + let udid = udid.clone(); + let stream_control_tx = stream_control_tx.clone(); + let control_tx = control_tx.clone(); + Box::pin(async move { + let Ok(text) = std::str::from_utf8(&message.data) else { + warn!("Invalid Android WebRTC control message bytes for {udid}"); + return; + }; + if let Ok(message) = serde_json::from_str::(text) { + match message { + WebRtcDataChannelMessage::ClientStats { stats } => { + if !stats.client_id.trim().is_empty() && !stats.kind.trim().is_empty() { + state.metrics.record_client_stream_stats(*stats); + } + } + WebRtcDataChannelMessage::StreamControl { + force_keyframe, + snapshot, + } => { + let command = WebRtcStreamCommand { + force_keyframe: force_keyframe.unwrap_or(false), + snapshot: snapshot.unwrap_or(false), + }; + if command.force_keyframe || command.snapshot { + source.request_keyframe(); + } + let _ = stream_control_tx.send(command); + } + WebRtcDataChannelMessage::StreamQuality { config } => { + let _ = config; + source.request_keyframe(); + } + } + return; + } + + let control_message = match serde_json::from_str::(text) { + Ok(message) => message, + Err(error) => { + warn!("Invalid Android WebRTC control message for {udid}: {error}"); + return; + } + }; + if control_tx.send(control_message).is_err() { + warn!("Android WebRTC control queue closed for {udid}"); + } + }) + })); +} + +async fn run_android_webrtc_control_queue( + state: AppState, + udid: String, + mut receiver: mpsc::UnboundedReceiver, +) { + let mut pending = VecDeque::new(); + loop { + let mut message = match pending.pop_front() { + Some(message) => message, + None => match receiver.recv().await { + Some(message) => message, + None => break, + }, + }; + if webrtc_control_message_is_move(&message) { + while let Ok(next_message) = receiver.try_recv() { + if webrtc_control_message_is_move(&next_message) { + message = next_message; + } else { + pending.push_back(next_message); + break; + } + } + } + + if let Err(error) = + run_android_webrtc_control_message(state.clone(), udid.clone(), message).await + { + warn!("Android WebRTC control message failed for {udid}: {error}"); + } + } +} + +async fn run_android_webrtc_control_message( + state: AppState, + udid: String, + message: ControlMessage, +) -> Result<(), AppError> { + task::spawn_blocking(move || match message { + ControlMessage::Touch { x, y, phase } => { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + state + .android + .send_touch(&udid, x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), &phase) + } + ControlMessage::EdgeTouch { x, y, phase, .. } => { + if !x.is_finite() || !y.is_finite() { + return Err(AppError::bad_request( + "`x` and `y` must be finite normalized numbers.", + )); + } + state + .android + .send_touch(&udid, x.clamp(0.0, 1.0), y.clamp(0.0, 1.0), &phase) + } + ControlMessage::MultiTouch { x1, y1, phase, .. } => { + if !x1.is_finite() || !y1.is_finite() { + return Err(AppError::bad_request( + "`x1` and `y1` must be finite normalized numbers.", + )); + } + state + .android + .send_touch(&udid, x1.clamp(0.0, 1.0), y1.clamp(0.0, 1.0), &phase) + } + ControlMessage::Key { + key_code, + modifiers, + } => state + .android + .send_key(&udid, key_code, modifiers.unwrap_or(0)), + ControlMessage::Button { + button, + duration_ms, + phase, + .. + } => match phase.as_deref() { + Some("down" | "began") => Ok(()), + Some("up" | "ended" | "cancelled") | None => { + state + .android + .press_button(&udid, &button, duration_ms.unwrap_or(0)) + } + Some(_) => Err(AppError::bad_request( + "`phase` must be `down`, `up`, `began`, `ended`, or `cancelled`.", + )), + }, + ControlMessage::DismissKeyboard => state.android.dismiss_keyboard(&udid), + ControlMessage::Home => state.android.press_home(&udid), + ControlMessage::AppSwitcher => state.android.open_app_switcher(&udid), + ControlMessage::RotateLeft => state.android.rotate_left(&udid), + ControlMessage::RotateRight => state.android.rotate_right(&udid), + ControlMessage::ToggleAppearance => state.android.toggle_appearance(&udid), + }) + .await + .map_err(|error| AppError::internal(format!("Failed to join Android control task: {error}")))? +} + async fn run_webrtc_control_queue( session: crate::simulators::session::SimulatorSession, state: AppState, @@ -847,32 +1234,487 @@ fn ice_transport_policy() -> RTCIceTransportPolicy { } } -async fn wait_for_h264_sync_keyframe( - session: &crate::simulators::session::SimulatorSession, - timeout_duration: Duration, -) -> Option { - if let Some(frame) = session.latest_keyframe() { - if h264_frame_is_decoder_sync(&frame) { - return Some(frame); +#[derive(Clone)] +pub(crate) struct AndroidWebRtcSource { + inner: Arc, +} + +struct AndroidWebRtcSourceInner { + udid: String, + encoder_handle: AtomicUsize, + callback_user_data: AtomicUsize, + shutdown_tx: broadcast::Sender<()>, + sender: broadcast::Sender, + raw_sender: broadcast::Sender>, + latest_keyframe: RwLock>, + metrics: Arc, +} + +unsafe impl Send for AndroidWebRtcSourceInner {} +unsafe impl Sync for AndroidWebRtcSourceInner {} + +impl AndroidWebRtcSource { + pub(crate) async fn start( + bridge: android::AndroidBridge, + metrics: Arc, + udid: String, + max_edge: Option, + encode_h264: bool, + ) -> Result { + let mut frame_stream = bridge.grpc_frame_stream(&udid, max_edge).await?; + let (sender, _) = broadcast::channel(ANDROID_WEBRTC_FRAME_BROADCAST_CAPACITY); + let (raw_sender, _) = broadcast::channel(ANDROID_WEBRTC_RAW_FRAME_BROADCAST_CAPACITY); + let (shutdown_tx, _) = broadcast::channel(1); + let inner = Arc::new(AndroidWebRtcSourceInner { + udid: udid.clone(), + encoder_handle: AtomicUsize::new(0), + callback_user_data: AtomicUsize::new(0), + shutdown_tx, + sender, + raw_sender, + latest_keyframe: RwLock::new(None), + metrics, + }); + if encode_h264 { + let user_data = Weak::into_raw(Arc::downgrade(&inner)) as *mut c_void; + let mut error = std::ptr::null_mut(); + let handle = unsafe { + ffi::xcw_native_h264_encoder_create( + Some(android_h264_encoder_frame_callback), + user_data, + &mut error, + ) + }; + if handle.is_null() { + unsafe { + let _ = Weak::from_raw(user_data as *const AndroidWebRtcSourceInner); + } + return Err(unsafe { take_native_error(error) }.unwrap_or_else(|| { + AppError::native("Unable to create Android H.264 encoder.") + })); + } + inner + .encoder_handle + .store(handle as usize, Ordering::Release); + inner + .callback_user_data + .store(user_data as usize, Ordering::Release); + } + + let source = Self { inner }; + let latest_frame = Arc::new(Mutex::new(None::>)); + let reader_inner = Arc::downgrade(&source.inner); + let reader_latest_frame = latest_frame.clone(); + let mut reader_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = reader_shutdown_rx.recv() => break, + frame = frame_stream.next_frame() => { + match frame { + Ok(Some(frame)) => { + let frame = Arc::new(frame); + if let Some(inner) = reader_inner.upgrade() { + let _ = inner.raw_sender.send(frame.clone()); + } + *reader_latest_frame.lock().unwrap() = Some(frame); + } + Ok(None) => break, + Err(error) => { + let udid = reader_inner + .upgrade() + .map(|inner| inner.udid.clone()) + .unwrap_or_else(|| "android".to_owned()); + warn!("Android WebRTC raw frame stream failed for {udid}: {error}"); + break; + } + } + } + } + } + }); + + if encode_h264 { + let encoder_inner = Arc::downgrade(&source.inner); + let encoder_latest_frame = latest_frame; + let mut encoder_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + let min_frame_gap = android_webrtc_frame_interval(); + let mut ticker = time::interval(min_frame_gap); + ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = encoder_shutdown_rx.recv() => break, + _ = ticker.tick() => { + let Some(inner) = encoder_inner.upgrade() else { + break; + }; + let frame = encoder_latest_frame.lock().unwrap().clone(); + let Some(frame) = frame else { + continue; + }; + if inner.latest_keyframe.read().unwrap().is_none() { + inner.request_keyframe(); + } + let handle = inner.encoder_handle.load(Ordering::Acquire); + let udid = inner.udid.clone(); + let encode_result = task::spawn_blocking(move || { + encode_android_rgba_frame(handle, &frame) + }) + .await + .map_err(|error| AppError::internal(format!("Failed to join Android encoder task: {error}"))) + .and_then(|result| result); + if let Err(error) = encode_result { + warn!("Android VideoToolbox encode failed for {udid}: {error}"); + } + } + } + } + }); + source.request_keyframe(); + } else { + let raw_inner = Arc::downgrade(&source.inner); + let raw_latest_frame = latest_frame; + let mut raw_shutdown_rx = source.inner.shutdown_tx.subscribe(); + tokio::spawn(async move { + let min_frame_gap = android_webrtc_frame_interval(); + let mut ticker = time::interval(min_frame_gap); + ticker.set_missed_tick_behavior(time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _ = raw_shutdown_rx.recv() => break, + _ = ticker.tick() => { + let Some(inner) = raw_inner.upgrade() else { + break; + }; + let frame = raw_latest_frame.lock().unwrap().clone(); + let Some(frame) = frame else { + continue; + }; + let _ = inner.raw_sender.send(frame); + } + } + } + }); + } + Ok(source) + } + + pub(crate) fn subscribe(&self) -> broadcast::Receiver { + self.inner.sender.subscribe() + } + + fn subscribe_raw(&self) -> broadcast::Receiver> { + self.inner.raw_sender.subscribe() + } + + pub(crate) async fn wait_for_keyframe( + &self, + timeout_duration: Duration, + ) -> Option { + let deadline = Instant::now() + timeout_duration; + let baseline_sequence = self + .inner + .latest_keyframe + .read() + .unwrap() + .as_ref() + .map_or(0, |frame| frame.frame_sequence); + let mut receiver = self.inner.sender.subscribe(); + self.request_keyframe(); + + loop { + if let Some(frame) = self.inner.latest_keyframe.read().unwrap().clone() { + if frame.frame_sequence > baseline_sequence { + return Some(frame); + } + } + let remaining = deadline.checked_duration_since(Instant::now())?; + match time::timeout(remaining, receiver.recv()).await { + Ok(Ok(frame)) if frame.is_keyframe && frame.frame_sequence > baseline_sequence => { + return Some(frame) + } + Ok(Ok(_)) | Ok(Err(broadcast::error::RecvError::Lagged(_))) => { + self.request_keyframe(); + } + Ok(Err(_)) | Err(_) => return None, + } + } + } + + pub(crate) fn request_refresh(&self) {} + + pub(crate) fn request_keyframe(&self) { + self.inner.request_keyframe(); + } +} + +impl Drop for AndroidWebRtcSourceInner { + fn drop(&mut self) { + let _ = self.shutdown_tx.send(()); + let encoder_handle = self.encoder_handle.load(Ordering::Acquire); + let callback_user_data = self.callback_user_data.load(Ordering::Acquire); + unsafe { + if encoder_handle != 0 { + ffi::xcw_native_h264_encoder_destroy(encoder_handle as *mut c_void); + } + if callback_user_data != 0 { + let _ = Weak::from_raw(callback_user_data as *const AndroidWebRtcSourceInner); + } + } + } +} + +unsafe extern "C" fn android_h264_encoder_frame_callback( + frame: *const ffi::xcw_native_frame, + user_data: *mut c_void, +) { + if frame.is_null() || user_data.is_null() { + return; + } + + let weak = unsafe { Weak::from_raw(user_data as *const AndroidWebRtcSourceInner) }; + if let Some(inner) = weak.upgrade() { + unsafe { + inner.handle_encoded_frame(&*frame); + } + } + let _ = Weak::into_raw(weak); +} + +impl AndroidWebRtcSourceInner { + fn request_keyframe(&self) { + self.metrics + .keyframe_requests + .fetch_add(1, Ordering::Relaxed); + let encoder_handle = self.encoder_handle.load(Ordering::Acquire); + if encoder_handle == 0 { + return; + } + unsafe { + ffi::xcw_native_h264_encoder_request_keyframe(encoder_handle as *mut c_void); + } + } + + fn handle_encoded_frame(&self, frame: &ffi::xcw_native_frame) { + let description = unsafe { copy_native_shared_bytes(frame.description) }; + let Some(data) = (unsafe { copy_native_shared_bytes(frame.data) }) else { + return; + }; + let packet = Arc::new(FramePacket { + frame_sequence: frame.frame_sequence, + timestamp_us: frame.timestamp_us, + is_keyframe: frame.is_keyframe, + width: frame.width, + height: frame.height, + codec: native_c_string(frame.codec), + description, + data, + }); + self.metrics.frames_encoded.fetch_add(1, Ordering::Relaxed); + if packet.is_keyframe { + self.metrics + .keyframes_encoded + .fetch_add(1, Ordering::Relaxed); + *self.latest_keyframe.write().unwrap() = Some(packet.clone()); + } + let _ = self.sender.send(packet); + } +} + +fn encode_android_rgba_frame( + encoder_handle: usize, + frame: &android::AndroidFrame, +) -> Result<(), AppError> { + unsafe { + let mut error = std::ptr::null_mut(); + let ok = ffi::xcw_native_h264_encoder_encode_rgba( + encoder_handle as *mut c_void, + frame.rgba.as_ptr(), + frame.rgba.len(), + frame.width, + frame.height, + frame.timestamp_us, + &mut error, + ); + if ok { + Ok(()) + } else { + Err(take_native_error(error) + .unwrap_or_else(|| AppError::native("Android VideoToolbox encode failed."))) + } + } +} + +fn android_rgba_webrtc_frame_chunks( + sequence: u64, + frame: &android::AndroidFrame, +) -> Option> { + let expected_bytes = frame.width as usize * frame.height as usize * 4; + if expected_bytes == 0 || frame.rgba.len() != expected_bytes { + return None; + } + let mut chunks = Vec::with_capacity(frame.rgba.len().div_ceil(ANDROID_WEBRTC_RGBA_CHUNK_BYTES)); + for (chunk_index, chunk) in frame + .rgba + .chunks(ANDROID_WEBRTC_RGBA_CHUNK_BYTES) + .enumerate() + { + let chunk_offset = chunk_index * ANDROID_WEBRTC_RGBA_CHUNK_BYTES; + let mut bytes = + BytesMut::with_capacity(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES + chunk.len()); + bytes.put_u32(ANDROID_WEBRTC_RGBA_CHUNK_MAGIC); + bytes.put_u8(ANDROID_WEBRTC_RGBA_VERSION); + bytes.put_u8(ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); + bytes.put_u16(ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES as u16); + bytes.put_u64(sequence); + bytes.put_u64(frame.timestamp_us); + bytes.put_u32(frame.width); + bytes.put_u32(frame.height); + bytes.put_u32(frame.rgba.len() as u32); + bytes.put_u32(chunk_offset as u32); + bytes.put_u32(chunk.len() as u32); + bytes.put_u8(if chunk_offset + chunk.len() >= frame.rgba.len() { + 1 + } else { + 0 + }); + bytes.put_u8(0); + bytes.put_u16(0); + bytes.extend_from_slice(chunk); + chunks.push(bytes.freeze()); + } + Some(chunks) +} + +unsafe fn copy_native_shared_bytes(bytes: ffi::xcw_native_shared_bytes) -> Option { + if bytes.data.is_null() || bytes.length == 0 { + if !bytes.owner.is_null() { + unsafe { + ffi::xcw_native_release_shared_bytes(bytes); + } + } + return None; + } + + let copied = + unsafe { Bytes::copy_from_slice(std::slice::from_raw_parts(bytes.data, bytes.length)) }; + unsafe { + ffi::xcw_native_release_shared_bytes(bytes); + } + Some(copied) +} + +fn native_c_string(ptr: *const i8) -> Option { + if ptr.is_null() { + return None; + } + let value = unsafe { CStr::from_ptr(ptr) } + .to_string_lossy() + .trim() + .to_owned(); + if value.is_empty() { + None + } else { + Some(value) + } +} + +unsafe fn take_native_error(raw: *mut i8) -> Option { + if raw.is_null() { + return None; + } + let message = unsafe { CStr::from_ptr(raw) } + .to_string_lossy() + .into_owned(); + unsafe { + ffi::xcw_native_free_string(raw); + } + Some(AppError::native(message)) +} + +fn android_webrtc_max_edge() -> Option { + std::env::var("SIMDECK_ANDROID_WEBRTC_MAX_EDGE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .map(|value| value.clamp(360, 4096)) +} + +fn android_webrtc_frame_interval() -> Duration { + Duration::from_micros(1_000_000 / ANDROID_WEBRTC_FPS) +} + +#[derive(Clone)] +enum WebRtcVideoSource { + Simulator(crate::simulators::session::SimulatorSession), + Android(AndroidWebRtcSource), +} + +impl WebRtcVideoSource { + fn subscribe(&self) -> WebRtcFrameReceiver { + match self { + Self::Simulator(session) => WebRtcFrameReceiver::Simulator(session.subscribe()), + Self::Android(source) => WebRtcFrameReceiver::Android(source.subscribe()), + } + } + + async fn wait_for_keyframe(&self, timeout_duration: Duration) -> Option { + match self { + Self::Simulator(session) => session.wait_for_keyframe(timeout_duration).await, + Self::Android(source) => source.wait_for_keyframe(timeout_duration).await, + } + } + + fn request_refresh(&self) { + match self { + Self::Simulator(session) => session.request_refresh(), + Self::Android(source) => source.request_refresh(), + } + } + + fn request_keyframe(&self) { + match self { + Self::Simulator(session) => session.request_keyframe(), + Self::Android(source) => source.request_keyframe(), + } + } +} + +enum WebRtcFrameReceiver { + Simulator(crate::simulators::session::FrameSubscription), + Android(broadcast::Receiver), +} + +impl WebRtcFrameReceiver { + async fn recv(&mut self) -> Result { + match self { + Self::Simulator(receiver) => receiver.recv().await, + Self::Android(receiver) => receiver.recv().await, } } +} +async fn wait_for_h264_sync_keyframe( + source: &WebRtcVideoSource, + timeout_duration: Duration, +) -> Option { let deadline = time::Instant::now() + timeout_duration; loop { let remaining = deadline.checked_duration_since(time::Instant::now())?; - let frame = session.wait_for_keyframe(remaining).await?; + let frame = source.wait_for_keyframe(remaining).await?; if h264_frame_is_decoder_sync(&frame) { return Some(frame); } - session.request_keyframe(); + source.request_keyframe(); } } struct WebRtcMediaStream { state: AppState, - session: crate::simulators::session::SimulatorSession, + source: WebRtcVideoSource, udid: String, - first_frame: crate::transport::packet::SharedFrame, + first_frame: SharedFrame, peer_connection: Arc, video_track: Arc, cancellation_token: broadcast::Sender<()>, @@ -880,11 +1722,140 @@ struct WebRtcMediaStream { stream_control_rx: mpsc::UnboundedReceiver, } +struct WebRtcRgbaStream { + state: AppState, + source: AndroidWebRtcSource, + udid: String, + peer_connection: Arc, + rgba_channel: Arc, + cancellation_token: broadcast::Sender<()>, + cancellation: broadcast::Receiver<()>, + stream_control_rx: mpsc::UnboundedReceiver, +} + +impl WebRtcRgbaStream { + async fn run(self) { + let Self { + state, + source, + udid, + peer_connection, + rgba_channel, + cancellation_token, + mut cancellation, + mut stream_control_rx, + } = self; + let mut rx = source.subscribe_raw(); + let mut peer_state_interval = time::interval(Duration::from_millis(250)); + let mut peer_disconnected_since: Option = None; + let mut sequence = 0u64; + let _guard = WebRtcMetricsGuard::new(state.metrics.clone()); + rgba_channel.on_open(Box::new({ + let udid = udid.clone(); + move || { + let udid = udid.clone(); + Box::pin(async move { + info!("Android RGBA WebRTC data channel open for {udid}"); + }) + } + })); + + loop { + tokio::select! { + _ = cancellation.recv() => { + warn!("Android RGBA WebRTC stream replaced for {udid}"); + break; + } + _ = peer_state_interval.tick() => { + let peer_state = peer_connection.connection_state(); + if matches!(peer_state, RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) { + warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); + break; + } + if peer_state == RTCPeerConnectionState::Disconnected { + let disconnected_since = + peer_disconnected_since.get_or_insert_with(time::Instant::now); + if disconnected_since.elapsed() >= WEBRTC_PEER_DISCONNECTED_TIMEOUT { + warn!("Android RGBA WebRTC stream closing for {udid}: peer state {peer_state}"); + break; + } + } else { + peer_disconnected_since = None; + } + } + command = stream_control_rx.recv() => { + let Some(command) = command else { + continue; + }; + if command.force_keyframe || command.snapshot { + source.request_refresh(); + } + } + frame = rx.recv() => { + let frame = match frame { + Ok(frame) => frame, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + state + .metrics + .frames_dropped_server + .fetch_add(skipped, Ordering::Relaxed); + continue; + } + Err(broadcast::error::RecvError::Closed) => { + warn!("Android RGBA WebRTC stream closing for {udid}: raw frame channel closed"); + break; + } + }; + if rgba_channel.ready_state() != RTCDataChannelState::Open { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + let Some(chunks) = android_rgba_webrtc_frame_chunks(sequence, &frame) else { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + }; + let buffered_amount = rgba_channel.buffered_amount().await; + if buffered_amount > frame.rgba.len() * ANDROID_WEBRTC_RGBA_BUFFERED_FRAME_LIMIT { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + continue; + } + sequence = sequence.wrapping_add(1); + let mut sent_frame = true; + for chunk in chunks { + let send_result = time::timeout(WEBRTC_REALTIME_WRITE_TIMEOUT, rgba_channel.send(&chunk)).await; + match send_result { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + warn!("Android RGBA WebRTC data channel send failed for {udid}: {error}"); + sent_frame = false; + break; + } + Err(_) => { + sent_frame = false; + break; + } + } + } + if sent_frame { + state.metrics.frames_sent.fetch_add(1, Ordering::Relaxed); + } else { + state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); + } + } + } + } + + warn!("Android RGBA WebRTC stream ended for {udid}"); + clear_webrtc_media_stream(&udid, &cancellation_token); + let _ = peer_connection.close().await; + } +} + impl WebRtcMediaStream { async fn run(self) { let Self { state, - session, + source, udid, first_frame, peer_connection, @@ -893,7 +1864,7 @@ impl WebRtcMediaStream { mut cancellation, mut stream_control_rx, } = self; - let mut rx = session.subscribe(); + let mut rx = source.subscribe(); let mut send_timing = WebRtcSendTiming::new(); let mut peer_state_interval = time::interval(Duration::from_millis(250)); let realtime_stream = realtime_stream_enabled(); @@ -930,10 +1901,10 @@ impl WebRtcMediaStream { if recovery_action_for_write_timeout(realtime_stream) == FrameRecoveryAction::Refresh { - session.request_refresh(); + source.request_refresh(); } else { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); } } Err(error) => { @@ -972,9 +1943,9 @@ impl WebRtcMediaStream { }; if command.force_keyframe || command.snapshot { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); } else { - session.request_refresh(); + source.request_refresh(); } } frame = rx.recv() => { @@ -986,7 +1957,7 @@ impl WebRtcMediaStream { .frames_dropped_server .fetch_add(skipped, Ordering::Relaxed); waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); continue; } Err(broadcast::error::RecvError::Closed) => { @@ -1002,7 +1973,7 @@ impl WebRtcMediaStream { waiting_for_keyframe = false; } else if frame.is_keyframe { waiting_for_keyframe = true; - session.request_keyframe(); + source.request_keyframe(); state.metrics.frames_dropped_server.fetch_add(1, Ordering::Relaxed); continue; } @@ -1027,9 +1998,9 @@ impl WebRtcMediaStream { let recovery_action = recovery_action_for_write_timeout(realtime_stream); waiting_for_keyframe = recovery_action == FrameRecoveryAction::Keyframe; if recovery_action == FrameRecoveryAction::Refresh { - session.request_refresh(); + source.request_refresh(); } else { - session.request_keyframe(); + source.request_keyframe(); } } Err(error) => { @@ -1469,11 +2440,14 @@ impl Drop for WebRtcMetricsGuard { #[cfg(test)] mod tests { use super::{ - append_avcc_parameter_sets, append_length_prefixed_nalus, h264_annex_b_sample, - h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, is_annex_b, - is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, WebRtcMetricsGuard, - WebRtcSendTiming, ANNEX_B_START_CODE, + android_rgba_webrtc_frame_chunks, append_avcc_parameter_sets, append_length_prefixed_nalus, + h264_annex_b_sample, h264_frame_has_idr, h264_frame_is_decoder_sync, h264_sdp_fmtp_line, + is_annex_b, is_h264_codec, rtcp_packet_requests_keyframe, rtp_packet_pacing, + WebRtcMetricsGuard, WebRtcSendTiming, ANDROID_WEBRTC_RGBA_CHUNK_BYTES, + ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES, ANDROID_WEBRTC_RGBA_CHUNK_MAGIC, + ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888, ANDROID_WEBRTC_RGBA_VERSION, ANNEX_B_START_CODE, }; + use crate::android; use crate::metrics::counters::Metrics; use crate::transport::packet::FramePacket; use bytes::Bytes; @@ -1934,6 +2908,71 @@ mod tests { ); } + #[test] + fn android_rgba_webrtc_frame_chunks_use_fixed_binary_header() { + let rgba = (0..(320 * 240 * 4)) + .map(|value| (value % 251) as u8) + .collect::>(); + let frame = android::AndroidFrame { + width: 320, + height: 240, + timestamp_us: 123_456, + rgba: rgba.clone(), + }; + + let chunks = android_rgba_webrtc_frame_chunks(7, &frame).unwrap(); + + assert_eq!(chunks.len(), 2); + let first = &chunks[0]; + assert_eq!( + u32::from_be_bytes(first[0..4].try_into().unwrap()), + ANDROID_WEBRTC_RGBA_CHUNK_MAGIC + ); + assert_eq!(first[4], ANDROID_WEBRTC_RGBA_VERSION); + assert_eq!(first[5], ANDROID_WEBRTC_RGBA_FORMAT_RGBA8888); + assert_eq!( + u16::from_be_bytes(first[6..8].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES + ); + assert_eq!(u64::from_be_bytes(first[8..16].try_into().unwrap()), 7); + assert_eq!( + u64::from_be_bytes(first[16..24].try_into().unwrap()), + 123_456 + ); + assert_eq!(u32::from_be_bytes(first[24..28].try_into().unwrap()), 320); + assert_eq!(u32::from_be_bytes(first[28..32].try_into().unwrap()), 240); + assert_eq!( + u32::from_be_bytes(first[32..36].try_into().unwrap()) as usize, + rgba.len() + ); + assert_eq!(u32::from_be_bytes(first[36..40].try_into().unwrap()), 0); + assert_eq!( + u32::from_be_bytes(first[40..44].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!(first[44], 0); + + let second = &chunks[1]; + assert_eq!( + u32::from_be_bytes(second[36..40].try_into().unwrap()) as usize, + ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!( + u32::from_be_bytes(second[40..44].try_into().unwrap()) as usize, + rgba.len() - ANDROID_WEBRTC_RGBA_CHUNK_BYTES + ); + assert_eq!(second[44], 1); + let reassembled = chunks + .iter() + .flat_map(|chunk| { + chunk[ANDROID_WEBRTC_RGBA_CHUNK_HEADER_BYTES..] + .iter() + .copied() + }) + .collect::>(); + assert_eq!(reassembled, rgba); + } + trait CloneFrameForTest { fn clone_for_test(&self, frame_sequence: u64) -> Self; } diff --git a/skills/simdeck/SKILL.md b/skills/simdeck/SKILL.md index c37ea448..aced60a0 100644 --- a/skills/simdeck/SKILL.md +++ b/skills/simdeck/SKILL.md @@ -5,7 +5,7 @@ description: Use for simulator lifecycle, app install/launch, live viewing, UI i # SimDeck Agent Guide -SimDeck automates iOS Simulators. Use the CLI for automation and the browser UI for live human visibility. Works with UIKit, SwiftUI, React Native, Expo, and NativeScript apps. +SimDeck automates iOS Simulators and Android emulators. Use the CLI for automation and the browser UI for live human visibility. iOS works with UIKit, SwiftUI, React Native, Expo, and NativeScript apps; Android works through ADB, emulator lifecycle, screenshots, logs, and UIAutomator hierarchy dumps. SimDeck uses one warm daemon per project. Check it with `simdeck daemon status`; start it or open the browser UI when needed: @@ -47,6 +47,7 @@ simdeck shutdown simdeck erase simdeck core-simulator restart simdeck install /path/to/App.app +simdeck install android: /path/to/app.apk simdeck launch com.example.App simdeck uninstall com.example.App simdeck open-url myapp://route @@ -56,6 +57,12 @@ simdeck toggle-appearance Build apps with project tooling. +Android devices use IDs like `android:Pixel_8_API_36`. `simdeck list` discovers +AVDs from the Android SDK, `boot` starts `emulator -avd ... -no-window`, and +live browser viewing uses the same WebRTC H.264 endpoint as iOS. Android frames +come from emulator gRPC and are encoded through VideoToolbox. `simdeck stream` +is still iOS-only. + ## Fast Agent Inspection Use targeted checks for test loops. `describe` is a diagnostic snapshot of the whole hierarchy; it is useful for planning, but it is expensive. For verification, prefer the daemon APIs exposed by `simdeck/test`: `query`, `waitFor`, `assert`, selector `tap`, and `batch`. @@ -75,6 +82,8 @@ simdeck describe --direct ``` Use `--source auto` with the project daemon. Use `--direct` or `--source native-ax` for the private CoreSimulator accessibility bridge. NativeScript, React Native, and Flutter inspector runtimes can add richer hierarchy data. +For Android IDs, `describe` uses `uiautomator dump`; use `--format agent` or +`--format compact-json` the same way as iOS. Prefer selectors, coordinates only when needed. Selector taps go through the daemon and wait for the element server-side.