diff --git a/.changeset/computer-workspace-file-tools.md b/.changeset/computer-workspace-file-tools.md new file mode 100644 index 0000000..d574c10 --- /dev/null +++ b/.changeset/computer-workspace-file-tools.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +Add bounded `find` and `grep` tools, a read-only-aware `delete` tool, and shared locking for file mutations. diff --git a/packages/computer/src/stub.test.ts b/packages/computer/src/stub.test.ts index 2ec8eb6..0836000 100644 --- a/packages/computer/src/stub.test.ts +++ b/packages/computer/src/stub.test.ts @@ -226,6 +226,18 @@ describe("WorkspaceStub", () => { }); }); + it("fs.find forwards bounded search options", async () => { + await withStub(async (ws) => { + const stub = ws.stub(); + await ws.fs.writeFile("/a.ts", ""); + await ws.fs.writeFile("/b.ts", ""); + await ws.fs.writeFile("/c.ts", ""); + expect(await stub.fs.find("/", "*.ts", { limit: 1, offset: 1 })).toEqual([ + { path: "/b.ts", type: "file" }, + ]); + }); + }); + it("fs.stat propagates ENOENT for missing paths", async () => { await withStub(async (ws) => { const stub = ws.stub(); diff --git a/packages/computer/src/stub.ts b/packages/computer/src/stub.ts index 0099424..794b534 100644 --- a/packages/computer/src/stub.ts +++ b/packages/computer/src/stub.ts @@ -41,6 +41,7 @@ import { trackStub, untrackStub } from "@cloudflare/computer-rpc/debug"; import type { + FindOptions, GrepOptions, MkdirOptions, ReaddirOptions, @@ -194,12 +195,16 @@ export class WorkspaceFilesystemStub extends RpcTarget { ); } - find(directory: string, pattern?: string): Promise { + find( + directory: string, + pattern?: string, + options: FindOptions = {}, + ): Promise { return withSpan( this.#ws.observer, "workspace.fs.find", { "workspace.fs.path": directory, "workspace.fs.pattern": pattern }, - () => this.#ws.fs.find(directory, pattern), + () => this.#ws.fs.find(directory, pattern, options), (span, outcome) => { if (outcome.ok) span.setAttribute("workspace.fs.matches", outcome.value.length); }, diff --git a/packages/computer/src/tools/ai.test.ts b/packages/computer/src/tools/ai.test.ts index 1b86141..33c305b 100644 --- a/packages/computer/src/tools/ai.test.ts +++ b/packages/computer/src/tools/ai.test.ts @@ -4,7 +4,10 @@ import type { WorkspaceRuntimeExecHandle, WorkspaceRuntimeResult } from "../runt import { Workspace } from "../workspace.js"; import { createAITools, + createDeleteTool, createEditTool, + createFindTool, + createGrepTool, createReadTool, createWriteTool, type FileStore, @@ -259,10 +262,18 @@ describe("WorkspaceFileStore", () => { }); describe("createAITools filesystem tools", () => { - it("creates fixed read, write, edit, and ls tools by default", () => { + it("creates the complete filesystem tool set by default", () => { const tools = createAITools({ workspace: makeWorkspace() }); - expect(Object.keys(tools).sort()).toEqual(["edit", "ls", "read", "write"]); + expect(Object.keys(tools).sort()).toEqual([ + "delete", + "edit", + "find", + "grep", + "ls", + "read", + "write", + ]); }); it("returns only read-only tools when readonly is true", () => { @@ -275,7 +286,7 @@ describe("createAITools filesystem tools", () => { }, }); - expect(Object.keys(tools).sort()).toEqual(["ls", "read"]); + expect(Object.keys(tools).sort()).toEqual(["find", "grep", "ls", "read"]); }); it("reads, lists, writes, and edits workspace files", async () => { @@ -349,6 +360,367 @@ describe("createAITools filesystem tools", () => { }); }); + it("serializes write behind an edit on the same store and path", async () => { + let releaseRead: (() => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + const writes: string[] = []; + const store: FileStore = { + async stat() { + return { size: 3, mtime: 1 }; + }, + async readAll() { + markReadStarted?.(); + await readGate; + return bytes("old"); + }, + async *readChunks() { + yield bytes("old"); + }, + async write(_path, content) { + writes.push(decode(content)); + }, + }; + const edit = executeTool(createEditTool({ store }), { + path: "/workspace/file.txt", + edits: [{ oldText: "old", newText: "edited" }], + }); + await readStarted; + + const write = executeTool(createWriteTool({ store }), { + path: "/workspace/file.txt", + content: "written", + }); + await Promise.resolve(); + const writesBeforeEditFinished = [...writes]; + + releaseRead?.(); + await Promise.all([edit, write]); + expect(writesBeforeEditFinished).toEqual([]); + expect(writes).toEqual(["edited", "written"]); + }); + + it("shares mutation locks across tool sets for the same workspace", async () => { + const workspace = makeWorkspace(); + await workspace.fs.mkdir("/workspace", { recursive: true }); + await workspace.fs.writeFile("/workspace/file.txt", "old"); + let releaseRead: (() => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + const originalReadFile = workspace.fs.readFile.bind(workspace.fs); + const originalWriteFile = workspace.fs.writeFile.bind(workspace.fs); + const writes: string[] = []; + workspace.fs.readFile = async (...args: Parameters) => { + markReadStarted?.(); + await readGate; + return originalReadFile(...args); + }; + workspace.fs.writeFile = async (...args: Parameters) => { + const content = args[1]; + if (content instanceof Uint8Array) writes.push(decode(content)); + return originalWriteFile(...args); + }; + + const firstTools = createAITools({ workspace }); + const secondTools = createAITools({ workspace }); + const edit = executeTool(firstTools.edit, { + path: "/workspace/file.txt", + edits: [{ oldText: "old", newText: "edited" }], + }); + await readStarted; + const write = executeTool(secondTools.write, { + path: "/workspace/file.txt", + content: "written", + }); + await Promise.resolve(); + const writesBeforeEditFinished = [...writes]; + + releaseRead?.(); + await Promise.all([edit, write]); + expect(writesBeforeEditFinished).toEqual([]); + expect(writes).toEqual(["edited", "written"]); + }); + + it("does not share edit locks between stores", async () => { + let releaseRead: (() => void) | undefined; + let markFirstStarted: (() => void) | undefined; + let markSecondStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const firstStarted = new Promise((resolve) => { + markFirstStarted = resolve; + }); + const secondStarted = new Promise((resolve) => { + markSecondStarted = resolve; + }); + const first = memoryStore({ content: "old" }); + first.readAll = async () => { + markFirstStarted?.(); + await readGate; + return bytes("old"); + }; + const second = memoryStore({ content: "old" }); + second.readAll = async () => { + markSecondStarted?.(); + return bytes("old"); + }; + + const firstEdit = executeTool(createEditTool({ store: first }), { + path: "/workspace/file.txt", + edits: [{ oldText: "old", newText: "first" }], + }); + await firstStarted; + const secondEdit = executeTool(createEditTool({ store: second }), { + path: "/workspace/file.txt", + edits: [{ oldText: "old", newText: "second" }], + }); + + const secondAcquired = await Promise.race([ + secondStarted.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 0)), + ]); + + releaseRead?.(); + await Promise.all([firstEdit, secondEdit]); + expect(secondAcquired).toBe(true); + }); + + it("passes find pagination to the workspace filesystem", async () => { + let received: { limit?: number; offset?: number } | undefined; + const tool = createFindTool({ + workspace: { + fs: { + async find(_path, _pattern, options) { + received = options; + return [{ path: "/workspace/a.ts", type: "file" }]; + }, + }, + }, + }); + + await executeTool(tool, { + path: "/workspace", + pattern: "**/*.ts", + limit: 2, + offset: 7, + }); + expect(received).toEqual({ limit: 3, offset: 7 }); + }); + + it("passes grep include and pagination to one filesystem search", async () => { + let received: Record | undefined; + const tool = createGrepTool({ + workspace: { + fs: { + async find() { + throw new Error("find must not be called by the grep tool"); + }, + async grep(_query, _path, options) { + received = options; + return []; + }, + }, + }, + }); + + await executeTool(tool, { + path: "/workspace", + query: "TODO", + include: "**/*.ts", + limit: 2, + offset: 7, + }); + expect(received).toMatchObject({ include: "**/*.ts", limit: 3, offset: 7 }); + }); + + it("accepts grep continuation offsets produced after large result sets", () => { + const tool = createGrepTool({ + workspace: { + fs: { + async find() { + return []; + }, + async grep() { + return []; + }, + }, + }, + }); + const schema = tool.inputSchema as { + safeParse(input: unknown): { success: boolean }; + }; + + expect(schema.safeParse({ path: "/workspace", query: "needle", offset: 10_200 }).success).toBe( + true, + ); + }); + + it("finds, greps, and deletes files through a real Workspace", async () => { + const workspace = makeWorkspace(); + const tools = createAITools({ workspace }); + await workspace.fs.mkdir("/workspace/src", { recursive: true }); + await workspace.fs.writeFile("/workspace/src/a.ts", "const value = 'TODO';\n"); + await workspace.fs.writeFile("/workspace/src/b.md", "todo in docs\n"); + + await expect( + executeTool(tools.find, { path: "/workspace", pattern: "**/*.ts", limit: 20 }), + ).resolves.toEqual({ + path: "/workspace", + pattern: "**/*.ts", + count: 1, + entries: [{ path: "/workspace/src/a.ts", type: "file" }], + }); + await expect( + executeTool(tools.grep, { + path: "/workspace", + query: "todo", + include: "**/*.ts", + limit: 20, + }), + ).resolves.toMatchObject({ + count: 1, + matches: [{ path: "/workspace/src/a.ts", line: 1, text: "const value = 'TODO';" }], + }); + await expect(executeTool(tools.delete, { path: "/workspace/src/a.ts" })).resolves.toEqual({ + deleted: "/workspace/src/a.ts", + }); + await expect(workspace.fs.stat("/workspace/src/a.ts")).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it("serializes delete behind an edit on the same store and path", async () => { + let releaseRead: (() => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + const events: string[] = []; + const store = memoryStore({ + content: "old", + onWrite() { + events.push("edit"); + }, + }); + store.readAll = async () => { + markReadStarted?.(); + await readGate; + return bytes("old"); + }; + const deleteStore = Object.assign(store, { + async remove() { + events.push("delete"); + }, + }); + const edit = executeTool(createEditTool({ store }), { + path: "/workspace/file.txt", + edits: [{ oldText: "old", newText: "edited" }], + }); + await readStarted; + const deletion = executeTool(createDeleteTool({ store: deleteStore }), { + path: "/workspace/file.txt", + }); + await Promise.resolve(); + const eventsBeforeEditFinished = [...events]; + + releaseRead?.(); + await Promise.all([edit, deletion]); + expect(eventsBeforeEditFinished).toEqual([]); + expect(events).toEqual(["edit", "delete"]); + }); + + it("serializes recursive delete behind a mutation in its subtree", async () => { + let releaseRead: (() => void) | undefined; + let markReadStarted: (() => void) | undefined; + const readGate = new Promise((resolve) => { + releaseRead = resolve; + }); + const readStarted = new Promise((resolve) => { + markReadStarted = resolve; + }); + const events: string[] = []; + const store = memoryStore({ + content: "old", + onWrite() { + events.push("edit"); + }, + }); + store.readAll = async () => { + markReadStarted?.(); + await readGate; + return bytes("old"); + }; + const deleteStore = Object.assign(store, { + async remove() { + events.push("delete"); + }, + }); + const edit = executeTool(createEditTool({ store }), { + path: "/workspace/tree/file.txt", + edits: [{ oldText: "old", newText: "edited" }], + }); + await readStarted; + const deletion = executeTool(createDeleteTool({ store: deleteStore }), { + path: "/workspace/tree", + recursive: true, + }); + await Promise.resolve(); + const eventsBeforeEditFinished = [...events]; + + releaseRead?.(); + await Promise.all([edit, deletion]); + expect(eventsBeforeEditFinished).toEqual([]); + expect(events).toEqual(["edit", "delete"]); + }); + + it("allows unrelated mutations while a recursive delete is pending", async () => { + let releaseRemove: (() => void) | undefined; + let markRemoveStarted: (() => void) | undefined; + const removeGate = new Promise((resolve) => { + releaseRemove = resolve; + }); + const removeStarted = new Promise((resolve) => { + markRemoveStarted = resolve; + }); + const events: string[] = []; + const store = Object.assign(memoryStore({ content: "old" }), { + async remove() { + markRemoveStarted?.(); + await removeGate; + events.push("delete"); + }, + }); + const deletion = executeTool(createDeleteTool({ store }), { + path: "/workspace/tree", + recursive: true, + }); + await removeStarted; + const write = executeTool(createWriteTool({ store }), { + path: "/workspace/other.txt", + content: "new", + }).then(() => events.push("write")); + await write; + + expect(events).toEqual(["write"]); + releaseRemove?.(); + await deletion; + expect(events).toEqual(["write", "delete"]); + }); + it("preserves file mode when write overwrites an existing file", async () => { const writes: Array<{ path: string; content: string; mode?: number }> = []; const tool = createWriteTool({ diff --git a/packages/computer/src/tools/ai.ts b/packages/computer/src/tools/ai.ts index fb3dd96..a4bc5e0 100644 --- a/packages/computer/src/tools/ai.ts +++ b/packages/computer/src/tools/ai.ts @@ -1,6 +1,9 @@ import type { ToolSet } from "ai"; import { createExecTool, type ExecToolOptions, type ExecWorkspaceLike } from "./exec.js"; +import { createDeleteTool, type DeleteToolOptions } from "./fs/delete.js"; import { createEditTool, type EditToolOptions } from "./fs/edit.js"; +import { createFindTool, type FindToolOptions } from "./fs/find.js"; +import { createGrepTool, type GrepToolOptions } from "./fs/grep.js"; import { createListTool } from "./fs/list.js"; import { createReadTool, type ReadToolOptions } from "./fs/read.js"; import { type WorkspaceLike as FileWorkspaceLike, WorkspaceFileStore } from "./fs/store.js"; @@ -14,6 +17,9 @@ export interface CreateAIToolsOptions { read?: Omit; write?: Omit; edit?: Omit; + find?: Omit; + grep?: Omit; + delete?: Omit; shell?: Omit; } @@ -22,12 +28,15 @@ export function createAITools(options: CreateAIToolsOptions): ToolSet { const tools: ToolSet = { read: createReadTool({ store, ...options.read }), ls: createListTool({ workspace: options.workspace }), + find: createFindTool({ workspace: options.workspace, ...options.find }), + grep: createGrepTool({ workspace: options.workspace, ...options.grep }), }; if (options.readonly === true) return tools; tools.write = createWriteTool({ store, ...options.write }); tools.edit = createEditTool({ store, ...options.edit }); + tools.delete = createDeleteTool({ store, ...options.delete }); if (options.shell !== undefined) { tools.exec = createExecTool({ diff --git a/packages/computer/src/tools/fs/delete.ts b/packages/computer/src/tools/fs/delete.ts new file mode 100644 index 0000000..44c5288 --- /dev/null +++ b/packages/computer/src/tools/fs/delete.ts @@ -0,0 +1,39 @@ +import { type Tool, tool } from "ai"; +import { z } from "zod"; +import { withFileLock } from "./locks.js"; +import type { MutableFileStore } from "./types.js"; + +export interface DeleteToolOptions { + store: MutableFileStore; +} + +const inputSchema = z.object({ + path: z.string().describe("Absolute path to the file or directory to delete."), + recursive: z + .boolean() + .optional() + .describe("Remove a directory and all of its contents. Defaults to false."), +}); + +export function createDeleteTool(options: DeleteToolOptions): Tool> { + const { store } = options; + return tool({ + description: + "Delete a file or directory. Set recursive to true to remove a non-empty directory.", + inputSchema, + execute: async ({ path, recursive }) => + withFileLock( + store, + path, + async () => { + try { + await store.remove(path, { recursive, force: true }); + return { deleted: path }; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } + }, + { subtree: recursive === true }, + ), + }); +} diff --git a/packages/computer/src/tools/fs/edit.ts b/packages/computer/src/tools/fs/edit.ts index 3076c8d..ec338b9 100644 --- a/packages/computer/src/tools/fs/edit.ts +++ b/packages/computer/src/tools/fs/edit.ts @@ -10,6 +10,7 @@ import { restoreLineEndings, stripBom, } from "./edit-diff.js"; +import { withFileLock } from "./locks.js"; import type { FileStore } from "./types.js"; export interface EditToolOptions { @@ -71,25 +72,6 @@ function prepareArguments(input: unknown): { path: string; edits: Edit[] } { return args as { path: string; edits: Edit[] }; } -// Per-path mutation queue. Edit and write should never race on the same file: -// fuzzy matching reads the entire buffer, applies a textual change, then -// writes — a concurrent edit landing between read and write would silently -// clobber the first edit. Module-scoped so all tools sharing a store also -// share the queue. -const fileLocks = new Map>(); -async function withFileLock(path: string, fn: () => Promise): Promise { - const prev = fileLocks.get(path) ?? Promise.resolve(); - const next = prev.then(fn, fn); - fileLocks.set( - path, - next.finally(() => { - // Clear only if we're still the tail of the chain. - if (fileLocks.get(path) === next) fileLocks.delete(path); - }), - ); - return next; -} - export function createEditTool(options: EditToolOptions): Tool> { const { store } = options; const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; @@ -105,7 +87,7 @@ export function createEditTool(options: EditToolOptions): Tool { + return withFileLock(store, path, async () => { try { const stat = await store.stat(path); if (!stat) return { error: `File not found: ${path}` }; diff --git a/packages/computer/src/tools/fs/find.ts b/packages/computer/src/tools/fs/find.ts new file mode 100644 index 0000000..f64822f --- /dev/null +++ b/packages/computer/src/tools/fs/find.ts @@ -0,0 +1,64 @@ +import { type Tool, tool } from "ai"; +import { z } from "zod"; + +interface FoundEntry { + path: string; + type: "file" | "dir"; +} + +export interface FindWorkspaceLike { + fs: { + find( + directory: string, + pattern?: string, + options?: { limit?: number; offset?: number }, + ): Promise; + }; +} + +export interface FindToolOptions { + workspace: FindWorkspaceLike; +} + +const DEFAULT_LIMIT = 200; +const MAX_LIMIT = 1000; + +const inputSchema = z.object({ + path: z.string().default("/workspace").describe("Absolute directory to search."), + pattern: z + .string() + .describe('Glob pattern relative to path, for example "**/*.ts" or "src/?.js".'), + limit: z.number().int().min(1).max(MAX_LIMIT).optional(), + offset: z.number().int().min(0).optional(), +}); + +export function createFindTool(options: FindToolOptions): Tool> { + return tool({ + description: + "Find files and directories matching a glob. * stays within one path segment, ** crosses directories, and ? matches one character.", + inputSchema, + execute: async ({ path, pattern, limit, offset }) => { + try { + const pageSize = limit ?? DEFAULT_LIMIT; + const pageOffset = offset ?? 0; + const matches = await options.workspace.fs.find(path, pattern, { + limit: pageSize + 1, + offset: pageOffset, + }); + const truncated = matches.length > pageSize; + const entries = truncated ? matches.slice(0, pageSize) : matches; + const result: { + path: string; + pattern: string; + count: number; + entries: FoundEntry[]; + nextOffset?: number; + } = { path, pattern, count: entries.length, entries }; + if (truncated) result.nextOffset = pageOffset + pageSize; + return result; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } + }, + }); +} diff --git a/packages/computer/src/tools/fs/grep.ts b/packages/computer/src/tools/fs/grep.ts new file mode 100644 index 0000000..8746fec --- /dev/null +++ b/packages/computer/src/tools/fs/grep.ts @@ -0,0 +1,98 @@ +import { type Tool, tool } from "ai"; +import { z } from "zod"; + +interface GrepContextLine { + line: number; + text: string; + isMatch: boolean; +} + +interface GrepMatch { + path: string; + line: number; + text: string; + context?: GrepContextLine[]; +} + +interface GrepOptions { + fixedString?: boolean; + caseSensitive?: boolean; + contextLines?: number; + limit?: number; + offset?: number; + include?: string; +} + +export interface GrepWorkspaceLike { + fs: { + grep(pattern: string, path: string, options?: GrepOptions): Promise; + }; +} + +export interface GrepToolOptions { + workspace: GrepWorkspaceLike; +} + +const DEFAULT_LIMIT = 200; +const MAX_LIMIT = 1000; + +const inputSchema = z.object({ + path: z.string().default("/workspace").describe("Absolute file or directory to search."), + query: z.string().describe("Regular expression or fixed string to search for."), + include: z + .string() + .optional() + .describe('Glob relative to path that limits searched files, for example "**/*.ts".'), + fixedString: z.boolean().optional().describe("Treat query as plain text instead of a regex."), + caseSensitive: z.boolean().optional().describe("Match letter case. Defaults to false."), + contextLines: z.number().int().min(0).max(10).optional(), + limit: z.number().int().min(1).max(MAX_LIMIT).optional(), + offset: z.number().int().min(0).optional(), +}); + +export function createGrepTool(options: GrepToolOptions): Tool> { + return tool({ + description: + "Search workspace text with a regular expression or fixed string. Results include paths and line numbers and can include surrounding lines.", + inputSchema, + execute: async ({ + path, + query, + include, + fixedString, + caseSensitive, + contextLines, + limit, + offset, + }) => { + try { + const pageSize = limit ?? DEFAULT_LIMIT; + const pageOffset = offset ?? 0; + const searchOptions = { + fixedString: fixedString ?? false, + caseSensitive: caseSensitive ?? false, + contextLines: contextLines ?? 0, + }; + const matches = await options.workspace.fs.grep(query, path, { + ...searchOptions, + include, + limit: pageSize + 1, + offset: pageOffset, + }); + const truncated = matches.length > pageSize; + const page = truncated ? matches.slice(0, pageSize) : matches; + const result: { + path: string; + query: string; + count: number; + matches: GrepMatch[]; + nextOffset?: number; + } = { path, query, count: page.length, matches: page }; + if (truncated) result.nextOffset = pageOffset + pageSize; + return result; + } catch (error) { + return { error: error instanceof Error ? error.message : String(error) }; + } + }, + }); +} diff --git a/packages/computer/src/tools/fs/locks.ts b/packages/computer/src/tools/fs/locks.ts new file mode 100644 index 0000000..56842a4 --- /dev/null +++ b/packages/computer/src/tools/fs/locks.ts @@ -0,0 +1,85 @@ +import type { FileStore } from "./types.js"; + +interface LockEntry { + path: string; + subtree: boolean; + done: Promise; +} + +export interface FileLockOptions { + /** Exclude mutations at this path and every ancestor or descendant. */ + subtree?: boolean; +} + +const storeLocks = new WeakMap>(); + +export async function withFileLock( + store: FileStore, + path: string, + operation: () => Promise, + options: FileLockOptions = {}, +): Promise { + const identity = store.lockIdentity ?? store; + let locks = storeLocks.get(identity); + if (locks === undefined) { + locks = new Set(); + storeLocks.set(identity, locks); + } + + const normalizedPath = normalizePath(path); + const subtree = options.subtree === true; + const previous = [...locks] + .filter((entry) => conflicts(normalizedPath, subtree, entry.path, entry.subtree)) + .map((entry) => entry.done); + let release: (() => void) | undefined; + const current: LockEntry = { + path: normalizedPath, + subtree, + done: new Promise((resolve) => { + release = resolve; + }), + }; + locks.add(current); + + await Promise.all(previous); + try { + return await operation(); + } finally { + release?.(); + locks.delete(current); + if (locks.size === 0) storeLocks.delete(identity); + } +} + +function conflicts( + leftPath: string, + leftSubtree: boolean, + rightPath: string, + rightSubtree: boolean, +) { + if (leftPath === rightPath) return true; + if (!leftSubtree && !rightSubtree) return false; + return isDescendant(leftPath, rightPath) || isDescendant(rightPath, leftPath); +} + +function isDescendant(path: string, ancestor: string): boolean { + if (path === ancestor) return true; + if (ancestor === "/") return path.startsWith("/"); + if (ancestor === "") return !path.startsWith("/"); + return path.startsWith(`${ancestor}/`); +} + +function normalizePath(path: string): string { + const absolute = path.startsWith("/"); + const parts: string[] = []; + for (const part of path.split("/")) { + if (part === "" || part === ".") continue; + if (part === "..") { + parts.pop(); + } else { + parts.push(part); + } + } + const normalized = parts.join("/"); + return absolute ? `/${normalized}` : normalized; +} diff --git a/packages/computer/src/tools/fs/store.ts b/packages/computer/src/tools/fs/store.ts index 76172e8..7e3c078 100644 --- a/packages/computer/src/tools/fs/store.ts +++ b/packages/computer/src/tools/fs/store.ts @@ -10,7 +10,7 @@ * and multimodal output still drain `fs.readFile(path)`. */ -import type { FileStat, FileStore } from "./types.js"; +import type { FileStat, MutableFileStore } from "./types.js"; const RANGE_CHUNK_BYTES = 64 * 1024; @@ -32,6 +32,30 @@ export interface WorkspaceLike { writeFile(path: string, content: Uint8Array, options?: { mode?: number }): Promise; mkdir(path: string, options?: { recursive?: boolean }): Promise; rm(path: string, options?: { recursive?: boolean; force?: boolean }): Promise; + find( + directory: string, + pattern?: string, + options?: { limit?: number; offset?: number }, + ): Promise>; + grep( + pattern: string, + path: string, + options?: { + fixedString?: boolean; + caseSensitive?: boolean; + contextLines?: number; + limit?: number; + offset?: number; + include?: string; + }, + ): Promise< + Array<{ + path: string; + line: number; + text: string; + context?: Array<{ line: number; text: string; isMatch: boolean }>; + }> + >; readdir( path: string, options?: { limit?: number; offset?: number }, @@ -48,8 +72,16 @@ export interface WorkspaceLike { }; } -export class WorkspaceFileStore implements FileStore { - constructor(private readonly ws: WorkspaceLike) {} +type WorkspaceFileStoreLike = { + fs: Pick; +}; + +export class WorkspaceFileStore implements MutableFileStore { + readonly lockIdentity: object; + + constructor(private readonly ws: WorkspaceFileStoreLike) { + this.lockIdentity = ws.fs; + } async stat(path: string): Promise { try { @@ -77,6 +109,10 @@ export class WorkspaceFileStore implements FileStore { await this.ws.fs.writeFile(path, content, opts); } + async remove(path: string, opts?: { recursive?: boolean; force?: boolean }): Promise { + await this.ws.fs.rm(path, opts); + } + async *readChunks(path: string, byteOffset = 0, byteLength?: number): AsyncIterable { if (!Number.isSafeInteger(byteOffset) || byteOffset < 0) { throw new Error("readChunks: byteOffset must be a non-negative safe integer"); @@ -128,7 +164,7 @@ async function drain(stream: ReadableStream): Promise { return out; } -async function ensureParentDir(ws: WorkspaceLike, path: string): Promise { +async function ensureParentDir(ws: WorkspaceFileStoreLike, path: string): Promise { const i = path.lastIndexOf("/"); if (i <= 0) return; const parent = path.slice(0, i); diff --git a/packages/computer/src/tools/fs/types.ts b/packages/computer/src/tools/fs/types.ts index eac26d6..2eda78a 100644 --- a/packages/computer/src/tools/fs/types.ts +++ b/packages/computer/src/tools/fs/types.ts @@ -18,6 +18,9 @@ export interface FileStat { } export interface FileStore { + /** Shared identity used to coordinate mutations across adapters. */ + readonly lockIdentity?: object; + /** Return file metadata, or null if the path does not exist or is not a file. */ stat(path: string): Promise; @@ -43,3 +46,8 @@ export interface FileStore { */ write(path: string, content: Uint8Array, opts?: { mode?: number }): Promise; } + +export interface MutableFileStore extends FileStore { + /** Remove a file or directory. */ + remove(path: string, opts?: { recursive?: boolean; force?: boolean }): Promise; +} diff --git a/packages/computer/src/tools/fs/write.ts b/packages/computer/src/tools/fs/write.ts index d7c1132..139b01f 100644 --- a/packages/computer/src/tools/fs/write.ts +++ b/packages/computer/src/tools/fs/write.ts @@ -1,5 +1,6 @@ import { type Tool, tool } from "ai"; import { z } from "zod"; +import { withFileLock } from "./locks.js"; import type { FileStore } from "./types.js"; export interface WriteToolOptions { @@ -32,16 +33,18 @@ export function createWriteTool(options: WriteToolOptions): Tool { + try { + // Preserve the existing file's mode when overwriting so executable + // scripts don't silently lose its executable bits. For new files we + // let the store apply its own default. + const existing = await store.stat(path); + await store.write(path, bytes, existing ? { mode: existing.mode } : undefined); + return { path, bytesWritten: bytes.length }; + } catch (err) { + return { error: err instanceof Error ? err.message : String(err) }; + } + }); }, }); } diff --git a/packages/computer/src/tools/index.ts b/packages/computer/src/tools/index.ts index b2686a5..7e06468 100644 --- a/packages/computer/src/tools/index.ts +++ b/packages/computer/src/tools/index.ts @@ -7,10 +7,13 @@ export { type ExecToolOptions, type ExecToolOutput, } from "./exec.js"; +export { createDeleteTool, type DeleteToolOptions } from "./fs/delete.js"; export { createEditTool, type EditToolOptions } from "./fs/edit.js"; +export { createFindTool, type FindToolOptions } from "./fs/find.js"; +export { createGrepTool, type GrepToolOptions } from "./fs/grep.js"; export { createListTool, type ListToolOptions } from "./fs/list.js"; export { createReadTool, type ReadToolOptions } from "./fs/read.js"; export { WorkspaceFileStore, type WorkspaceLike } from "./fs/store.js"; -export type { FileStat, FileStore } from "./fs/types.js"; +export type { FileStat, FileStore, MutableFileStore } from "./fs/types.js"; export { createWriteTool, type WriteToolOptions } from "./fs/write.js"; export { createPublishTool, type PublishToolOptions } from "./publish.js"; diff --git a/packages/dofs/src/fs/filesystem.ts b/packages/dofs/src/fs/filesystem.ts index ff23489..ffe8431 100644 --- a/packages/dofs/src/fs/filesystem.ts +++ b/packages/dofs/src/fs/filesystem.ts @@ -15,7 +15,7 @@ import type { Database } from "../storage.js"; import { chmod } from "./chmod.js"; -import { find, type WorkspaceFoundEntry } from "./find.js"; +import { type FindOptions, find, type WorkspaceFoundEntry } from "./find.js"; import { type GrepOptions, grep, type WorkspaceGrepMatch } from "./grep.js"; import { ls } from "./ls.js"; import { type MkdirOptions, mkdir } from "./mkdir.js"; @@ -86,8 +86,12 @@ export class WorkspaceFilesystem { return readdir(this.db, path, options); } - async find(directory: string, pattern?: string): Promise { - return find(this.db, directory, pattern); + async find( + directory: string, + pattern?: string, + options: FindOptions = {}, + ): Promise { + return find(this.db, directory, pattern, options); } async ls(prefix: string): Promise { diff --git a/packages/dofs/src/fs/find.test.ts b/packages/dofs/src/fs/find.test.ts index bd47264..b2b470b 100644 --- a/packages/dofs/src/fs/find.test.ts +++ b/packages/dofs/src/fs/find.test.ts @@ -90,6 +90,21 @@ describe("find", () => { }); }); + it("applies limit and offset while walking in deterministic order", async () => { + await withDB(async (db) => { + mkdir(db, "/a/b", { recursive: true }, () => 0); + await writeFile(db, "/a/1.ts", "", {}, () => 0); + await writeFile(db, "/a/b/2.ts", "", {}, () => 0); + await writeFile(db, "/a/b/3.ts", "", {}, () => 0); + await writeFile(db, "/a/z.ts", "", {}, () => 0); + + expect(find(db, "/a", "**/*.ts", { offset: 1, limit: 2 })).toEqual([ + { path: "/a/b/2.ts", type: "file" }, + { path: "/a/b/3.ts", type: "file" }, + ]); + }); + }); + it("does not match files outside the start directory even with **", async () => { await withDB(async (db) => { mkdir(db, "/a", {}, () => 0); diff --git a/packages/dofs/src/fs/find.ts b/packages/dofs/src/fs/find.ts index 9caaf2d..f4ac294 100644 --- a/packages/dofs/src/fs/find.ts +++ b/packages/dofs/src/fs/find.ts @@ -8,13 +8,34 @@ export interface WorkspaceFoundEntry { type: "file" | "dir"; } +export interface FindOptions { + /** Maximum matching entries to return. */ + limit?: number; + /** Matching entries to skip in traversal order. */ + offset?: number; +} + interface ChildRow { name: string; child_inode: number; type: "file" | "dir"; } -export function find(db: Database, directory: string, pattern?: string): WorkspaceFoundEntry[] { +interface WalkState { + seen: number; + offset: number; + limit: number; + out: WorkspaceFoundEntry[]; +} + +const CHILD_PAGE_SIZE = 128; + +export function find( + db: Database, + directory: string, + pattern?: string, + options: FindOptions = {}, +): WorkspaceFoundEntry[] { const { path: canonical } = canonicalizePath(directory); const node = resolveInode(db, canonical); if (node === null) { @@ -24,42 +45,71 @@ export function find(db: Database, directory: string, pattern?: string): Workspa throw createWorkspaceError("ENOTDIR", `not a directory: ${canonical}`, canonical); } - const out: WorkspaceFoundEntry[] = []; + const limit = options.limit ?? Number.MAX_SAFE_INTEGER; + if (!Number.isSafeInteger(limit) || limit < 0) { + throw new TypeError("find limit must be a non-negative safe integer"); + } + const offset = options.offset ?? 0; + if (!Number.isSafeInteger(offset) || offset < 0) { + throw new TypeError("find offset must be a non-negative safe integer"); + } + if (limit === 0) return []; + // An empty pattern is equivalent to no pattern: walk and return // everything rather than compiling it into `^$`, which would match // only empty relative paths and yield no results. const regex = pattern ? compileGlob(pattern) : undefined; + const prefix = canonical === "/" ? "/" : `${canonical}/`; + const state: WalkState = { seen: 0, offset, limit, out: [] }; + walk(db, node.inode, canonical, prefix, regex, state); + return state.out; +} - walk(db, node.inode, canonical, out); +function walk( + db: Database, + parentInode: number, + parentPath: string, + prefix: string, + regex: RegExp | undefined, + state: WalkState, +): boolean { + let afterName = ""; + while (true) { + const children = readChildren(db, parentInode, afterName); + if (children.length === 0) return false; - if (regex === undefined) { - return out; + for (const child of children) { + const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; + const relativePath = childPath.slice(prefix.length); + if (regex === undefined || regex.test(relativePath)) { + if (state.seen >= state.offset) { + state.out.push({ path: childPath, type: child.type }); + if (state.out.length >= state.limit) return true; + } + state.seen += 1; + } + if (child.type === "dir" && walk(db, child.child_inode, childPath, prefix, regex, state)) { + return true; + } + } + + if (children.length < CHILD_PAGE_SIZE) return false; + afterName = children[children.length - 1].name; } - // Glob matches against the path relative to the start directory. - const prefix = canonical === "/" ? "/" : `${canonical}/`; - return out.filter((entry) => { - if (!entry.path.startsWith(prefix)) return false; - const rel = entry.path.slice(prefix.length); - return regex.test(rel); - }); } -function walk(db: Database, parentInode: number, parentPath: string, out: WorkspaceFoundEntry[]) { - const children = db.all( +function readChildren(db: Database, parentInode: number, afterName: string): ChildRow[] { + return db.all( `SELECT d.name AS name, d.child_inode AS child_inode, n.type AS type FROM vfs_dirents d JOIN vfs_nodes n ON n.inode = d.child_inode - WHERE d.parent_inode = ? - ORDER BY d.name`, + WHERE d.parent_inode = ? AND d.name > ? + ORDER BY d.name + LIMIT ?`, parentInode, + afterName, + CHILD_PAGE_SIZE, ); - for (const child of children) { - const childPath = parentPath === "/" ? `/${child.name}` : `${parentPath}/${child.name}`; - out.push({ path: childPath, type: child.type }); - if (child.type === "dir") { - walk(db, child.child_inode, childPath, out); - } - } } // Compile a simple glob into a regex. Supported: diff --git a/packages/dofs/src/fs/grep.test.ts b/packages/dofs/src/fs/grep.test.ts index 1823851..c407e3d 100644 --- a/packages/dofs/src/fs/grep.test.ts +++ b/packages/dofs/src/fs/grep.test.ts @@ -102,6 +102,20 @@ describe("grep", () => { }); }); + it("filters directory searches by an include glob before applying pagination", async () => { + await withDB(async (db) => { + await writeFile(db, "/a.md", "TODO markdown\n", {}, () => 0); + await writeFile(db, "/b.ts", "TODO one\nTODO two\n", {}, () => 0); + await writeFile(db, "/c.ts", "TODO three\n", {}, () => 0); + + expect( + (await grep(db, "TODO", "/", { include: "**/*.ts", offset: 1, limit: 2 })).map( + (match) => `${match.path}:${match.line}`, + ), + ).toEqual(["/b.ts:2", "/c.ts:1"]); + }); + }); + it("matches across a chunk boundary", async () => { await withDB(async (db) => { // Lay out a file whose line straddles the 512KiB chunk boundary. diff --git a/packages/dofs/src/fs/grep.ts b/packages/dofs/src/fs/grep.ts index 3d81e20..68c42cd 100644 --- a/packages/dofs/src/fs/grep.ts +++ b/packages/dofs/src/fs/grep.ts @@ -31,6 +31,8 @@ export interface GrepOptions { limit?: number; /** Matching lines to skip before collecting results. */ offset?: number; + /** Glob relative to a searched directory that limits files. */ + include?: string; } interface ScanState { @@ -63,16 +65,9 @@ export async function grep( const settings = normalizeOptions(options); if (settings.limit === 0) return []; const matcher = compileMatcher(pattern, settings.fixedString, settings.caseSensitive); - const filePaths = - node.type === "file" - ? [canonical] - : find(db, canonical) - .filter((entry) => entry.type === "file") - .map((entry) => entry.path) - .sort(); - const matches: WorkspaceGrepMatch[] = []; const state: ScanState = { seen: 0, accepted: 0 }; + const filePaths = node.type === "file" ? [canonical] : filesUnder(db, canonical, options.include); for (const filePath of filePaths) { const complete = await scanFile( db, @@ -124,6 +119,23 @@ function normalizeOptions(options: GrepOptions): { }; } +function* filesUnder( + db: Database, + directory: string, + include: string | undefined, +): Iterable { + const pageSize = 128; + let offset = 0; + while (true) { + const page = find(db, directory, include, { limit: pageSize, offset }); + for (const entry of page) { + if (entry.type === "file") yield entry.path; + } + if (page.length < pageSize) return; + offset += page.length; + } +} + function compileMatcher(pattern: string, fixedString: boolean, caseSensitive: boolean): RegExp { const source = fixedString ? pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") : pattern; try { diff --git a/packages/dofs/src/index.ts b/packages/dofs/src/index.ts index 3924fc9..e5638a4 100644 --- a/packages/dofs/src/index.ts +++ b/packages/dofs/src/index.ts @@ -5,7 +5,7 @@ export { WorkspaceFilesystem, type WorkspaceFilesystemOptions, } from "./fs/filesystem.js"; -export type { WorkspaceFoundEntry } from "./fs/find.js"; +export type { FindOptions, WorkspaceFoundEntry } from "./fs/find.js"; export type { GrepOptions, WorkspaceGrepContextLine,