Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/agents/web-backend.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ Default first-run flow:
agent-device web setup
agent-device open "https://example.com" --platform web
agent-device snapshot -i --platform web
agent-device viewport 1280 900 --platform web
agent-device screenshot ./artifacts/web-full.png --platform web --fullscreen
agent-device network dump 25 --platform web
agent-device close --platform web
```
Expand Down
11 changes: 11 additions & 0 deletions src/client-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,16 @@ export type PrepareCommandOptions = DeviceCommandBaseOptions & {
timeoutMs?: number;
};

export type ViewportCommandOptions = DeviceCommandBaseOptions & {
width: number;
height: number;
};

export type ViewportCommandResult = CommandRequestResult & {
width: number;
height: number;
};

export type AgentDeviceCommandClient = {
wait: (options: WaitCommandOptions) => Promise<WaitCommandResult>;
alert: (options?: AlertCommandOptions) => Promise<AlertCommandResult>;
Expand All @@ -521,6 +531,7 @@ export type AgentDeviceCommandClient = {
clipboard: (options: ClipboardCommandOptions) => Promise<ClipboardCommandResult>;
reactNative: (options: ReactNativeCommandOptions) => Promise<CommandRequestResult>;
prepare: (options: PrepareCommandOptions) => Promise<CommandRequestResult>;
viewport: (options: ViewportCommandOptions) => Promise<ViewportCommandResult>;
};

type SelectorSnapshotCommandOptions = Pick<CaptureSnapshotOptions, 'depth' | 'scope' | 'raw'>;
Expand Down
2 changes: 2 additions & 0 deletions src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import type {
Lease,
MaterializationReleaseOptions,
MetroPrepareOptions,
ViewportCommandResult,
} from './client-types.ts';
import { readSerializedSnapshotCaptureAnnotations } from './snapshot-capture-annotations.ts';
import { readSnapshotDiagnosticsSummary } from './snapshot-diagnostics.ts';
Expand Down Expand Up @@ -101,6 +102,7 @@ export function createAgentDeviceClient(
clipboard: async (options) => await executeCommand('clipboard', options),
reactNative: async (options) => await executeCommand('react-native', options),
prepare: async (options) => await executeCommand('prepare', options),
viewport: async (options) => await executeCommand<ViewportCommandResult>('viewport', options),
},
devices: {
list: async (options = {}) => {
Expand Down
1 change: 1 addition & 0 deletions src/command-catalog.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ export const PUBLIC_COMMANDS = {
trace: 'trace',
triggerAppEvent: 'trigger-app-event',
type: 'type',
viewport: 'viewport',
wait: 'wait',
} as const;

Expand Down
11 changes: 9 additions & 2 deletions src/commands/capture/index.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,11 +82,18 @@ describe('capture command interface', () => {
['page.png'],
flags({ screenshotFullscreen: true, screenshotMaxSize: 1024 }),
);
expect(input).toMatchObject({ path: 'page.png', fullscreen: true, maxSize: 1024 });
expect(input).toMatchObject({
path: 'page.png',
fullscreen: true,
maxSize: 1024,
});
expect(screenshotDaemonWriter(input)).toMatchObject({
command: 'screenshot',
positionals: ['page.png'],
options: { screenshotFullscreen: true, screenshotMaxSize: 1024 },
options: {
screenshotFullscreen: true,
screenshotMaxSize: 1024,
},
});
});

Expand Down
10 changes: 7 additions & 3 deletions src/commands/capture/screenshot-options.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,13 @@ test('screenshot flag projection maps public options to request flags', () => {
assert.deepEqual(
screenshotFlagsFromOptions({
overlayRefs: true,
fullscreen: false,
fullscreen: true,
maxSize: 512,
stabilize: false,
}),
{
overlayRefs: true,
screenshotFullscreen: false,
screenshotFullscreen: true,
screenshotMaxSize: 512,
screenshotNoStabilize: true,
},
Expand All @@ -48,7 +48,11 @@ test('screenshot script flags use the shared recorded flag contract', () => {
const parts: string[] = [];
const flags = {};

let result = readScreenshotScriptFlag({ args: ['--fullscreen'], index: 0, flags });
let result = readScreenshotScriptFlag({ args: ['--full'], index: 0, flags });
assert.deepEqual(result, { handled: true, nextIndex: 0 });
result = readScreenshotScriptFlag({ args: ['-f'], index: 0, flags });
assert.deepEqual(result, { handled: true, nextIndex: 0 });
result = readScreenshotScriptFlag({ args: ['--fullscreen'], index: 0, flags });
assert.deepEqual(result, { handled: true, nextIndex: 0 });
result = readScreenshotScriptFlag({ args: ['--max-size', '640'], index: 0, flags });
assert.deepEqual(result, { handled: true, nextIndex: 1 });
Expand Down
5 changes: 3 additions & 2 deletions src/commands/capture/screenshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ const screenshotCommandDefinition = defineExecutableCommand(

const screenshotCliSchema = {
helpDescription:
'Capture screenshot (macOS app sessions default to the app window; use --fullscreen for full desktop, --max-size to downscale, --overlay-refs to annotate current refs, or --no-stabilize for low-latency Android capture loops)',
summary: 'Capture screenshot with optional desktop, downscale, or ref overlay modes',
'Capture screenshot (web defaults to the viewport; use --fullscreen, --full, or -f for the entire page. macOS app sessions default to the app window; use --fullscreen for full desktop, --max-size to downscale, --overlay-refs to annotate current refs, or --no-stabilize for low-latency Android capture loops)',
summary:
'Capture screenshot with optional web full-page, desktop, downscale, or ref overlay modes',
positionalArgs: ['path?'],
allowedFlags: SCREENSHOT_COMMAND_FLAG_KEYS,
} as const;
Expand Down
2 changes: 2 additions & 0 deletions src/commands/management/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { installManagementCommandFacets } from './install.ts';
import { prepareCommandFacet } from './prepare.ts';
import { pushManagementCommandFacets } from './push.ts';
import { sessionCommandFacet } from './session.ts';
import { viewportCommandFacet } from './viewport.ts';

export const managementCommandFamily = defineCommandFamilyFromFacets({
name: 'management',
Expand All @@ -15,6 +16,7 @@ export const managementCommandFamily = defineCommandFamilyFromFacets({
sessionCommandFacet,
openCommandFacet,
closeCommandFacet,
viewportCommandFacet,
...installManagementCommandFacets,
...pushManagementCommandFacets,
],
Expand Down
1 change: 1 addition & 0 deletions src/commands/management/output.ts
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,7 @@ export const managementCliOutputFormatters = {
reinstall: resultOutput(deployCliOutput),
'install-from-source': resultOutput(installFromSourceCliOutput),
prepare: messageOutput,
viewport: messageOutput,
} as const satisfies Record<string, CliOutputFormatter>;

function formatDeviceLine(device: AgentDeviceDevice): string {
Expand Down
61 changes: 61 additions & 0 deletions src/commands/management/viewport.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import { PUBLIC_COMMANDS } from '../../command-catalog.ts';
import type { ViewportCommandOptions } from '../../client-types.ts';
import type { CommandSchemaOverride } from '../../utils/cli-command-schema-types.ts';
import { AppError } from '../../utils/errors.ts';
import { integerField, requiredField } from '../command-input.ts';
import { defineExecutableCommand } from '../command-contract.ts';
import { commonInputFromFlags, direct } from '../cli-grammar/common.ts';
import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts';
import { defineCommandFacet } from '../family/types.ts';
import { defineFieldCommandMetadata } from '../field-command-contract.ts';
import { managementCliOutputFormatters } from './output.ts';

const viewportCommandMetadata = defineFieldCommandMetadata(
'viewport',
'Resize the active web viewport.',
{
width: requiredField(integerField('Viewport width in CSS pixels.', { min: 1 })),
height: requiredField(integerField('Viewport height in CSS pixels.', { min: 1 })),
},
);

const viewportCommandDefinition = defineExecutableCommand(
viewportCommandMetadata,
(client, input) => client.command.viewport(input),
);

const viewportCliSchema = {
helpDescription:
'Resize the active web viewport before taking snapshots or screenshots. Useful for fixed-layout or 100vh apps where changing the viewport reveals different content.',
summary: 'Resize the active web viewport for the current session',
positionalArgs: ['width', 'height'],
} as const satisfies CommandSchemaOverride;

const viewportCliReader: CliReader = (positionals, flags) => ({
...commonInputFromFlags(flags),
width: readViewportDimension(positionals[0], 'width'),
height: readViewportDimension(positionals[1], 'height'),
});

const viewportDaemonWriter: DaemonWriter = direct(PUBLIC_COMMANDS.viewport, (input) => {
const { width, height } = input as ViewportCommandOptions;
return [String(width), String(height)];
});

export const viewportCommandFacet = defineCommandFacet({
name: 'viewport',
metadata: viewportCommandMetadata,
definition: viewportCommandDefinition,
cliSchema: viewportCliSchema,
cliReader: viewportCliReader,
daemonWriter: viewportDaemonWriter,
cliOutputFormatter: managementCliOutputFormatters.viewport,
});

function readViewportDimension(value: string | undefined, label: 'width' | 'height'): number {
const parsed = value === undefined ? NaN : Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new AppError('INVALID_ARGS', `viewport ${label} must be a positive integer`);
}
return parsed;
}
9 changes: 5 additions & 4 deletions src/contracts/screenshot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,11 @@ type ScreenshotSpecificFlagDefinition = {
export const SCREENSHOT_SPECIFIC_FLAG_DEFINITIONS: readonly ScreenshotSpecificFlagDefinition[] = [
{
key: 'screenshotFullscreen',
names: ['--fullscreen'],
names: ['--fullscreen', '--full', '-f'],
type: 'boolean',
usageLabel: '--fullscreen',
usageDescription: 'Screenshot: capture the full screen instead of the app window',
usageLabel: '--fullscreen, --full, -f',
usageDescription:
'Screenshot: on web capture the full page; on macOS app sessions capture the full desktop instead of the app window',
},
{
key: 'screenshotMaxSize',
Expand Down Expand Up @@ -124,7 +125,7 @@ export function readScreenshotScriptFlag(params: {
}): { handled: true; nextIndex: number } | { handled: false } {
const { args, flags, index } = params;
const token = args[index];
if (token === '--fullscreen') {
if (token === '--fullscreen' || token === '--full' || token === '-f') {
flags.screenshotFullscreen = true;
return { handled: true, nextIndex: index };
}
Expand Down
6 changes: 6 additions & 0 deletions src/core/__tests__/web-interactor.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ test('web interactor delegates first-slice operations to the scoped provider', a
async screenshot(outPath, options) {
calls.push(`screenshot:${outPath}:${options?.fullscreen === true}`);
},
async setViewport(width, height) {
calls.push(`viewport:${width}:${height}`);
},
async click(x, y) {
calls.push(`click:${x}:${y}`);
},
Expand All @@ -48,6 +51,7 @@ test('web interactor delegates first-slice operations to the scoped provider', a
await interactor.type('world', 6);
await interactor.scroll('down', { pixels: 400 });
await interactor.screenshot('/tmp/web.png', { fullscreen: true });
await interactor.setViewport?.(1280, 900);
return await interactor.snapshot({ scope: 'main' });
});

Expand All @@ -61,6 +65,7 @@ test('web interactor delegates first-slice operations to the scoped provider', a
'type:world:6',
'scroll:down:400',
'screenshot:/tmp/web.png:true',
'viewport:1280:900',
'snapshot:main',
]);
assert.equal(snapshot.backend, 'web');
Expand All @@ -86,6 +91,7 @@ function makeWebProvider(overrides: Partial<WebProvider> = {}): WebProvider {
close: async () => {},
snapshot: async () => ({ nodes: [] }),
screenshot: async () => {},
setViewport: async () => {},
click: async () => {},
fill: async () => {},
typeText: async () => {},
Expand Down
7 changes: 7 additions & 0 deletions src/core/capabilities.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,12 @@ const WEB_QUERY_COMMANDS = [
'wait',
] as const;
const WEB_INTERACTION_COMMANDS = ['click', 'fill', 'focus', 'press', 'scroll', 'type'] as const;
const WEB_SETTING_COMMANDS = ['viewport'] as const;
const WEB_SUPPORTED_COMMANDS = new Set<string>([
...WEB_RUNTIME_COMMANDS,
...WEB_QUERY_COMMANDS,
...WEB_INTERACTION_COMMANDS,
...WEB_SETTING_COMMANDS,
]);
const ALL_DEVICE_COMMAND_CAPABILITY = {
apple: { simulator: true, device: true },
Expand Down Expand Up @@ -259,6 +261,11 @@ const BASE_COMMAND_CAPABILITY_MATRIX: Record<string, CommandCapability> = {
supports: (device) =>
device.platform === 'android' || device.platform === 'macos' || device.kind === 'simulator',
},
viewport: {
apple: { simulator: true, device: true },
android: { emulator: true, device: true, unknown: true },
linux: LINUX_NONE,
},
'trigger-app-event': {
apple: { simulator: true, device: true },
android: { emulator: true, device: true, unknown: true },
Expand Down
26 changes: 26 additions & 0 deletions src/core/dispatch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,8 @@ async function dispatchKnownCommand(
return await handleTriggerAppEventCommand(device, interactor, positionals, context);
case 'screenshot':
return await handleScreenshotCommand(interactor, positionals, outPath, context);
case 'viewport':
return await handleViewportCommand(interactor, positionals);
case 'back':
await interactor.back(context?.backMode);
return { action: 'back', mode: context?.backMode ?? 'in-app', ...successText('Back') };
Expand Down Expand Up @@ -282,6 +284,22 @@ async function handleScreenshotCommand(
return { path: screenshotPath, ...successText(`Saved screenshot: ${screenshotPath}`) };
}

async function handleViewportCommand(
interactor: Interactor,
positionals: string[],
): Promise<Record<string, unknown>> {
if (positionals.length !== 2) {
throw new AppError('INVALID_ARGS', 'viewport requires exactly two arguments: <width> <height>');
}
const width = readViewportDimension(positionals[0], 'width');
const height = readViewportDimension(positionals[1], 'height');
if (!interactor.setViewport) {
throw new AppError('UNSUPPORTED_OPERATION', 'viewport is not supported by this backend');
}
await interactor.setViewport(width, height);
return { width, height, ...successText(`Viewport set: ${width}x${height}`) };
}

async function handleClipboardCommand(
interactor: Interactor,
positionals: string[],
Expand Down Expand Up @@ -309,6 +327,14 @@ async function handleClipboardCommand(
};
}

function readViewportDimension(value: string | undefined, label: 'width' | 'height'): number {
const parsed = value === undefined ? NaN : Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new AppError('INVALID_ARGS', `viewport ${label} must be a positive integer`);
}
return parsed;
}

async function handleKeyboardCommand(
device: DeviceInfo,
positionals: string[],
Expand Down
1 change: 1 addition & 0 deletions src/core/interactor-types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ export type Interactor = {
): Promise<Record<string, unknown> | void>;
pinch(scale: number, x?: number, y?: number): Promise<Record<string, unknown> | void>;
screenshot(outPath: string, options?: ScreenshotOptions): Promise<void>;
setViewport?(width: number, height: number): Promise<Record<string, unknown> | void>;
snapshot(options?: SnapshotOptions): Promise<SnapshotResult>;
back(mode?: BackMode): Promise<void>;
home(): Promise<void>;
Expand Down
1 change: 1 addition & 0 deletions src/core/interactors/web.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ export function createWebInteractor(): Interactor {
scroll: (direction, options) => provider().scroll(direction, options),
pinch: () => unsupportedWebOperation('pinch'),
screenshot: (outPath, options) => provider().screenshot(outPath, options),
setViewport: (width, height) => provider().setViewport(width, height),
snapshot: async (options) => {
const result = await withDiagnosticTimer(
'snapshot_capture',
Expand Down
1 change: 1 addition & 0 deletions src/daemon/__tests__/request-handler-catalog.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ test('catalog commands use generic routing only when intentionally passthrough o
PUBLIC_COMMANDS.screenshot,
PUBLIC_COMMANDS.scroll,
PUBLIC_COMMANDS.swipe,
PUBLIC_COMMANDS.viewport,
].sort();
const genericCatalogCommands = [
...Object.values(PUBLIC_COMMANDS),
Expand Down
1 change: 1 addition & 0 deletions src/daemon/__tests__/request-platform-providers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ function makeWebProvider(overrides: Partial<WebProvider> = {}): WebProvider {
close: async () => {},
snapshot: async () => ({ nodes: [] }),
screenshot: async () => {},
setViewport: async () => {},
click: async () => {},
fill: async () => {},
typeText: async () => {},
Expand Down
1 change: 1 addition & 0 deletions src/daemon/daemon-command-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ const DAEMON_COMMAND_DESCRIPTORS = [
),
descriptor(PUBLIC_COMMANDS.focus, 'generic', { androidBlockingDialogGuard: true }),
descriptor(PUBLIC_COMMANDS.screenshot, 'generic', { replayScopedAction: true }),
descriptor(PUBLIC_COMMANDS.viewport, 'generic', { replayScopedAction: true }),
...descriptors(
'generic',
{ androidBlockingDialogGuard: true },
Expand Down
2 changes: 2 additions & 0 deletions src/platforms/web/agent-browser-provider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ test('agent-browser provider maps supported operations to session-scoped JSON co
await withCommandExecutorOverride(recordingExecutor(calls), async () => {
await provider.open('https://example.test');
await provider.screenshot('/tmp/page.png', { fullscreen: true });
await provider.setViewport(1280, 900);
await provider.click(10.4, 20.6);
await provider.clickRef?.('@e3');
await provider.fill(11, 22, 'Ada');
Expand All @@ -41,6 +42,7 @@ test('agent-browser provider maps supported operations to session-scoped JSON co
[
['open', 'https://example.test', '--json', '--session', 'web-session'],
['screenshot', '--full', '/tmp/page.png', '--json', '--session', 'web-session'],
['set', 'viewport', '1280', '900', '--json', '--session', 'web-session'],
['mouse', 'move', '10', '21', '--json', '--session', 'web-session'],
['mouse', 'down', '--json', '--session', 'web-session'],
['mouse', 'up', '--json', '--session', 'web-session'],
Expand Down
Loading
Loading