From dcae3c6c148582d235f5650aace6f9744eeb153c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:53:28 +0800 Subject: [PATCH 001/791] fix(ghidra): reject address truncation before reads and edits --- bridge/ghidra/ReaGhidraBridge.java | 13 ++- docs/ghidra-dos.md | 8 +- docs/testing.md | 8 +- .../lib/real-ghidra-address-boundaries.mjs | 85 +++++++++++++++++++ scripts/verify-real-ghidra-com.mjs | 3 + scripts/verify-real-ghidra-dos.mjs | 7 ++ 6 files changed, 118 insertions(+), 6 deletions(-) create mode 100644 scripts/lib/real-ghidra-address-boundaries.mjs diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 784a33da8e6..d5f6cd9e4f4 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -3004,8 +3004,17 @@ else if (value.startsWith("0x") || value.startsWith("0X")) { AddressSpace space = spaceName == null ? currentProgram.getAddressFactory().getDefaultAddressSpace() : currentProgram.getAddressFactory().getAddressSpace(spaceName); - return space == null ? null : space.getAddress(offset); - } + if (space == null) return null; + Address address = space.getAddress(offset); + BigInteger requestedOffset = new BigInteger(offset, 16); + BigInteger parsedOffset = new BigInteger(Long.toUnsignedString(address.getOffset())); + if (!requestedOffset.equals(parsedOffset)) + throw new RequestFailure("invalid_request", + "Ghidra address offset cannot be represented without truncation in space " + + space.getName() + ": requested " + value + ", parsed " + canonicalAddress(address)); + return address; + } + catch (RequestFailure failure) { throw failure; } catch (Exception exception) { return null; } diff --git a/docs/ghidra-dos.md b/docs/ghidra-dos.md index 893c3506d71..8083036818d 100644 --- a/docs/ghidra-dos.md +++ b/docs/ghidra-dos.md @@ -1,7 +1,7 @@ # DOS MZ and COM analysis with Ghidra REA can analyze DOS MZ executables and explicitly selected COM images through the bring-your-own Ghidra adapter. -The Linux x64 verification lane exercises actual 16-bit disassembly and +The Linux x64 and macOS arm64 verification lanes exercise actual 16-bit disassembly and decompilation through the CLI and stdio MCP. Provider admission accepts Ghidra 12.1.x and that installation's declared JDK range. The verification lane uses Ghidra 12.1.4 and JDK 21; no DOS emulator or cross-compiler is required. @@ -87,6 +87,10 @@ address for subsequent lookups and leaves instruction decoding context with Ghidra. These coordinates are analysis load addresses, not file offsets or observations of a running DOS system. +Offsets that Ghidra would truncate to a different address are rejected before +reading or annotating a function. This also applies to explicit or encoded +address-space names; leading zeros preserve the numeric offset. + `procedure_info` and function dossiers include `body` evidence: ```json @@ -150,7 +154,7 @@ are separate from the initialized MZ module. Raw COM files and DOS extenders without an admitted MZ real-mode entry are outside this import boundary. DOS MZ is available through Ghidra, with no Hopper support claim. It is outside -the Windows native PE P0 boundary. macOS DOS verification remains unverified. +the Windows native PE P0 boundary. macOS x64 DOS verification remains unverified. ## Real verification diff --git a/docs/testing.md b/docs/testing.md index 051122f00ff..4c14a0c1789 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -420,14 +420,18 @@ a DOS emulator or compiler, then checks real 16-bit decoding, segment relocation, near/far calls, decompilation, disjoint function body ranges, stable CLI/MCP observations, unchanged source bytes, and owned process/project cleanup. Raw p-code address-space selector tokens are reported separately from -the stable observation comparison. Linux x64 is verified; macOS DOS remains -unverified. This lane is separate from host-native and optional cross-format +the stable observation comparison. Linux x64 and macOS arm64 are verified; +macOS x64 remains unverified. This lane is separate from host-native and optional cross-format verification. See [DOS analysis](ghidra-dos.md). `npm run verify:ghidra:com` uses a generated headerless fixture with no compiler, DOS emulator or game data. It exercises explicit admission, BinaryLoader entry preparation, measured register context, whole-file byte readback, source offsets, unmapped PSP/partial reads, actual decompilation, CLI/MCP parity and owned cleanup. +Both segmented-address lanes reject oversized default, explicit-space and encoded-space +coordinates through real reads, function queries and annotation attempts. Rejected +annotations must preserve the live function dossier; CLI and MCP must report the +truncation constraint, while leading-zero coordinates still resolve correctly. It has the same Ghidra/JDK prerequisites as the MZ lane. Neither lane claims DOS runtime or PC-98 device execution. diff --git a/scripts/lib/real-ghidra-address-boundaries.mjs b/scripts/lib/real-ghidra-address-boundaries.mjs new file mode 100644 index 00000000000..032e65be844 --- /dev/null +++ b/scripts/lib/real-ghidra-address-boundaries.mjs @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; + +/** Exercise segmented-address truncation through real CLI/MCP reads and edits. */ +export async function verifyGhidraAddressBoundaries(client, entry, cli) { + const query = async (name, args) => { + const reply = await client.callTool( + { name, arguments: args }, + { timeout: 240000 }, + ); + assert.notEqual(reply.isError, true, JSON.stringify(reply)); + return reply.structuredContent.result; + }; + const before = await query("analyze_function", { procedure: entry }); + // The segmented parser used to discard the high 32 bits and alias this entry. + const overflow = (0x100000000n + BigInt(entry)).toString(16); + const representations = [ + `0x${overflow}`, + `0X${overflow}`, + overflow, + `ram:0x${overflow}`, + `%72am:0x${overflow}`, + ]; + for (const address of representations) { + for (const [name, args] of [ + ["read_bytes", { address, length: 1 }], + ["inspect_native_instruction", { address }], + ["resolve_containing_procedure", { address }], + ["procedure_info", { procedure: address }], + [ + "annotate_native_function", + { + procedure: address, + name: "MUST_NOT_ALIAS", + comment: "MUST NOT WRITE", + }, + ], + ]) { + const reply = await client.callTool( + { name, arguments: args }, + { timeout: 240000 }, + ); + assert.equal(reply.isError, true, `${name} accepted ${address}`); + const error = reply.structuredContent?.error; + assert.equal(error?.code, "invalid_request", JSON.stringify(reply)); + assert.equal(error.details.operation, name); + const issues = JSON.stringify(error.details.issues); + assert.match(issues, /cannot be represented without truncation/u); + assert.ok(issues.includes(address), issues); + assert.equal(reply.structuredContent.evidence, undefined); + } + } + assert.deepEqual( + await query("analyze_function", { procedure: entry }), + before, + "Rejected oversized annotations changed the aliased function", + ); + // Leading zeros and an encoded explicit space preserve the requested offset. + const padded = `%72am:0x0000${BigInt(entry).toString(16)}`; + assert.deepEqual( + await query("read_bytes", { address: padded, length: 1 }), + await query("read_bytes", { address: entry, length: 1 }), + ); + for (const [command, flags, operation] of [ + ["read-bytes", ["--length", "1"], "read_bytes"], + [ + "annotate-native-function", + ["--name", "MUST_NOT_ALIAS"], + "annotate_native_function", + ], + ]) { + let failure; + try { + await cli(command, representations[0], flags); + } catch (error) { + assert.equal(error.code, 1); + failure = JSON.parse(error.stdout); + } + assert.equal(failure?.code, "invalid_request"); + assert.equal(failure.details.operation, operation); + assert.match( + JSON.stringify(failure.details.issues), + /cannot be represented without truncation/u, + ); + } +} diff --git a/scripts/verify-real-ghidra-com.mjs b/scripts/verify-real-ghidra-com.mjs index 455c0e73548..0db71b48614 100644 --- a/scripts/verify-real-ghidra-com.mjs +++ b/scripts/verify-real-ghidra-com.mjs @@ -14,6 +14,7 @@ import { functionDossierSchema } from "../dist/domain/hopperValues.js"; import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; import { buildDosComFixture } from "../tests/conformance/ghidra/dos-com-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { verifyGhidraAddressBoundaries } from "./lib/real-ghidra-address-boundaries.mjs"; // This lane intentionally has no game paths, binary seeds or compiler dependency. if (process.argv.length !== 2) @@ -228,6 +229,7 @@ try { ); assert.equal(retained.annotations.inline_comment, changes.inline_comment); assert.equal(retained.dossier.procedure.name, changes.name); + await verifyGhidraAddressBoundaries(client, fixture.entry, cli); const cleared = await call("annotate_native_function", { procedure: fixture.entry, comment: "", @@ -294,6 +296,7 @@ try { cli_mcp_parity: true, }, original_unchanged: true, + address_truncation_rejected: true, limitations: image.limitations, }; } catch (error) { diff --git a/scripts/verify-real-ghidra-dos.mjs b/scripts/verify-real-ghidra-dos.mjs index 1a786992a2b..114f920e26b 100644 --- a/scripts/verify-real-ghidra-dos.mjs +++ b/scripts/verify-real-ghidra-dos.mjs @@ -24,6 +24,7 @@ import { import { inspectGhidraInstallation } from "../dist/ghidra/GhidraInstallation.js"; import { buildDosMzFixture } from "../tests/conformance/ghidra/dos-mz-fixture.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; +import { verifyGhidraAddressBoundaries } from "./lib/real-ghidra-address-boundaries.mjs"; // This lane intentionally has no game paths, binary seeds or compiler dependency. if (process.argv.length !== 2) @@ -367,6 +368,11 @@ try { arguments: { procedure: "0xffffffffffffffff" }, }); assert.equal(invalid.isError, true, "Unmapped address did not fail"); + await verifyGhidraAddressBoundaries( + client, + identities.entry.address, + cliEvidence, + ); const cliWindow = await cliEvidence("instructions", identities.entry.address); assert.deepEqual( cliWindow.normalized_result, @@ -428,6 +434,7 @@ try { far_target: expected.far, }, invalid_address_rejected: true, + address_truncation_rejected: true, original_unchanged: true, limitations: [ "Read-only import/decompilation evidence, not DOS runtime execution or whole-program semantic equivalence.", From 549effa056729e5f9239d9ed7864537a9c49e421 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:02 +0800 Subject: [PATCH 002/791] fix(ghidra): resolve exact external function entries --- docs/native-investigation.md | 3 ++ docs/testing.md | 2 + scripts/lib/real-ghidra-boundaries.mjs | 47 +++++++++++++++++++ src/ghidra/GhidraInventoryValues.test.ts | 23 --------- src/ghidra/GhidraInventoryValues.ts | 4 ++ src/ghidra/GhidraProviderCapabilities.ts | 5 ++ tests/boundary/mcp/ghidraInventoryMcp.test.ts | 23 +++++++++ 7 files changed, 84 insertions(+), 23 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index f394573f0d0..efa6e625375 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -82,6 +82,9 @@ by the inventory, including their namespaces and any platform symbol prefix. Overloads can share a fully qualified name; ambiguity errors return every matching entry address so the caller can select the intended function directly. +`resolve_containing_procedure` also resolves an exact external entry. Its empty +body remains explicit; nearby external addresses do not inherit that identity. + Ghidra function references cover the complete function-body AddressSet and its exact entry, including references into instruction interiors and references from embedded data. Addresses inside an enclosing span but outside the owned diff --git a/docs/testing.md b/docs/testing.md index 4c14a0c1789..ebebdb4c166 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,6 +280,8 @@ xrefs, raw procedure references, and CLI/MCP dossiers must retain that data edge A valid legacy snapshot reconstructs the former omitted edge under its older profile; CLI and MCP must reject that binding with a mismatch reason and recovery advice. The rejected open must preserve the active live session. +Exact external entries must resolve while retaining an empty body; unknown +external addresses remain unresolved and external annotations are rejected. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index c89e4533f6b..98d2f10859f 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -95,6 +95,52 @@ export async function verifyGhidraBoundaries( assert.equal((await stat(socketRoot)).mode & 0o777, 0o700); const address = original.procedure.address; const name = original.procedure.name; + const procedures = await call("list_procedures"); + const external = procedures.find((item) => item.procedure.external); + assert.ok(external, "Source fixture lacks an external function"); + const externalDossier = await call("analyze_function", { + procedure: external.address, + }); + const externalEntry = await call("resolve_containing_procedure", { + address: external.address, + }); + assert.equal(externalEntry.found, true); + assert.equal(externalEntry.procedure.address, external.address); + assert.equal(externalEntry.procedure.classification.external, true); + assert.deepEqual( + externalEntry.procedure.body, + externalDossier.procedure.body, + ); + assert.equal(externalEntry.procedure.body.contains_entry, false); + assert.deepEqual(externalEntry.procedure.body.ranges, []); + const externalSpace = external.address.slice( + 0, + external.address.indexOf(":"), + ); + const lastExternalOffset = procedures + .filter((item) => item.procedure.external) + .map((item) => BigInt(item.address.slice(item.address.indexOf(":") + 1))) + .reduce((maximum, offset) => (offset > maximum ? offset : maximum), 0n); + const unknownExternal = `${externalSpace}:0x${(lastExternalOffset + 1n).toString(16)}`; + assert.deepEqual( + await call("resolve_containing_procedure", { address: unknownExternal }), + { + query_address: unknownExternal, + found: false, + procedure: null, + reason: "outside_segments", + }, + ); + assert.deepEqual( + (await cli("function", external.address)).procedure, + externalDossier.procedure, + "CLI and MCP must preserve external identity without inventing body bytes", + ); + await invalid( + "annotate_native_function", + { procedure: external.address, name: "rea_external_edit" }, + /Annotations require a local function entry/u, + ); const names = await call("list_names"); const leaf = names.find((item) => item.value.endsWith("rea_ghidra_inventory_leaf"), @@ -550,6 +596,7 @@ export async function verifyGhidraBoundaries( mutation_rollback: true, annotation_native_text_validation: true, lossless_unicode_transport: true, + exact_external_entry_resolution: true, complete_function_body_references: true, ambiguity_candidates_inline: true, legacy_reference_snapshot_rejected: true, diff --git a/src/ghidra/GhidraInventoryValues.test.ts b/src/ghidra/GhidraInventoryValues.test.ts index 8a882e27b3c..4d441ec394c 100644 --- a/src/ghidra/GhidraInventoryValues.test.ts +++ b/src/ghidra/GhidraInventoryValues.test.ts @@ -31,18 +31,6 @@ describe("containing-procedure complete body evidence", () => { found: true, procedure: identity(), }); - it("accepts the Java producer identity including complete inclusive body ranges", () => { - const value = found(); - expect( - parseGhidraInventoryResult("resolve_containing_procedure", value), - ).toEqual({ ok: true, value }); - expect( - parseGhidraInventoryResult("resolve_containing_procedure", { - ...value, - query_address: "0x1021", - }).ok, - ).toBe(true); - }); it("rejects a missing body rather than treating a legacy identity as complete", () => { const { body: omitted, ...legacy } = identity(); expect(omitted.total_bytes).toBe(5); @@ -108,15 +96,4 @@ describe("containing-procedure complete body evidence", () => { }).ok, ).toBe(false); }); - it("retains explicit not-found outcomes without fabricating a body", () => { - const value = { - query_address: "0x1010", - found: false, - procedure: null, - reason: "not_in_procedure", - }; - expect( - parseGhidraInventoryResult("resolve_containing_procedure", value), - ).toEqual({ ok: true, value }); - }); }); diff --git a/src/ghidra/GhidraInventoryValues.ts b/src/ghidra/GhidraInventoryValues.ts index 1ed9d480959..ebb4c30cc02 100644 --- a/src/ghidra/GhidraInventoryValues.ts +++ b/src/ghidra/GhidraInventoryValues.ts @@ -219,7 +219,11 @@ const containingProcedure = z.discriminatedUnion("found", [ .strict() .superRefine((value, context) => { const body = value.procedure.body; + const exactExternalEntry = + value.procedure.classification.external && + value.query_address === value.procedure.address; if ( + !exactExternalEntry && body.available && !functionBodyEntryAgrees( { ...body, contains_entry: true }, diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index a0f246d3a90..28ddff2476f 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -85,6 +85,11 @@ export const limitationsFor = (operation: string): readonly string[] => { ...common, "Only Ghidra-defined string Data is observed; charset is reported, while a non-missing terminator cannot distinguish a present terminator from a fixed or Pascal layout.", ]; + case "resolve_containing_procedure": + return [ + ...common, + "An exact external entry resolves to its observed function identity even when its body is empty. This does not establish mapped executable bytes or containment of nearby external addresses.", + ]; case "list_segments": return [ ...common, diff --git a/tests/boundary/mcp/ghidraInventoryMcp.test.ts b/tests/boundary/mcp/ghidraInventoryMcp.test.ts index 0c6a660ec1b..3a484cae190 100644 --- a/tests/boundary/mcp/ghidraInventoryMcp.test.ts +++ b/tests/boundary/mcp/ghidraInventoryMcp.test.ts @@ -67,6 +67,29 @@ it("rejects contradictory provider output before emitting MCP Evidence", async ( limitations: ["Ghidra-specific instruction text."], }, }, + { + name: "resolve_containing_procedure", + arguments: { address: "EXTERNAL:0x2" }, + output: { + query_address: "EXTERNAL:0x2", + found: true, + procedure: { + ...ghidraFunctionIdentity(), + address: "EXTERNAL:0x1", + classification: { + ...ghidraFunctionIdentity().classification, + external: true, + }, + body: { + ...ghidraFunctionIdentity().body, + ranges: [], + total_bytes: 0, + span_bytes: 0, + contains_entry: false, + }, + }, + }, + }, ]; for (const probe of cases) { output = jsonValueSchema.parse(probe.output); From bc4aa46d7c3960a1f325523e48aed2089b176e84 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:54:11 +0800 Subject: [PATCH 003/791] fix(ghidra): recover regex stack exhaustion without losing annotations --- bridge/ghidra/ReaGhidraBridge.java | 23 +++++++- docs/native-investigation.md | 8 +++ docs/testing.md | 3 + scripts/lib/real-ghidra-boundaries.mjs | 74 ++++++++++++++++++++++++ src/domain/analysisErrorCore.ts | 4 +- src/domain/analysisErrorPresentation.ts | 17 +++--- src/ghidra/GhidraProviderCapabilities.ts | 6 +- src/ghidra/GhidraProviderClient.ts | 16 +++++ tests/conformance/ghidra/inventory.c | 10 ++++ 9 files changed, 151 insertions(+), 10 deletions(-) diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index d5f6cd9e4f4..326d36665f3 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1490,6 +1490,7 @@ private JsonArray search(JsonObject params, boolean procedureSearch) throws Exce : regexMatcher(expression, caseSensitive); JsonArray items = new JsonArray(); for (InventoryItem item : inventory) { + monitor.checkCancelled(); if (!matcher.matches(item.value)) { continue; } @@ -3098,9 +3099,29 @@ private static ValueMatcher regexMatcher(String expression, boolean caseSensitiv ); } catch (PatternSyntaxException exception) { + // Java wraps compiler stack exhaustion in PatternSyntaxException + // without retaining its cause. This producer reason is not bad syntax. + if (exception.getDescription().equals("Stack overflow during pattern compilation")) + throw regexStackExhausted(expression, "compiling"); throw new RequestFailure("invalid_request", "Invalid regex pattern " + expression + ": " + exception.getDescription() + " at index " + exception.getIndex()); } - return value -> pattern.matcher(value).find(); + catch (StackOverflowError exhausted) { + throw regexStackExhausted(expression, "compiling"); + } + return value -> { + try { + return pattern.matcher(value).find(); + } + catch (StackOverflowError exhausted) { + throw regexStackExhausted(expression, "matching"); + } + }; + } + + private static RequestFailure regexStackExhausted(String expression, String stage) { + return new RequestFailure("regex_stack_exhausted", + "Ghidra's Java regex engine exhausted its stack while " + stage + + " pattern " + expression + ". Use literal mode or simplify the regex; the session remains usable."); } private static SessionDescriptor readDescriptor(Path path) throws IOException { diff --git a/docs/native-investigation.md b/docs/native-investigation.md index efa6e625375..ca840e8f898 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -93,6 +93,14 @@ body are excluded. `procedure_references` returns internal and external edges; The reference collector's semantics participate in the analysis profile, so older snapshots cannot replay the former incomplete results under this profile. +Ghidra regex searches use Java Pattern semantics. If compiling or matching a +pattern exhausts the engine's stack, REA returns a resource constraint and +preserves the active database and annotations. Retry in literal mode or simplify +the regex; successful literal searches retain complete matching strings. +Cancelling an active request terminates the ephemeral database and discards its +annotations. REA retains the selected target and imports the original artifact +again on the next Ghidra query. + ```bash rea inspect-native-instruction
--provider ghidra rea resolve-native-call-targets --provider ghidra diff --git a/docs/testing.md b/docs/testing.md index ebebdb4c166..21923f00b7c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -282,6 +282,9 @@ profile; CLI and MCP must reject that binding with a mismatch reason and recovery advice. The rejected open must preserve the active live session. Exact external entries must resolve while retaining an empty body; unknown external addresses remain unresolved and external annotations are rejected. +An adversarial regex over a full 12 KB literal must report stack exhaustion as +a resource constraint, preserve live annotations, and allow complete literal +searches afterward; CLI and MCP must agree on both results and recovery advice. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 98d2f10859f..b31fa61cd74 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -414,6 +414,79 @@ export async function verifyGhidraBoundaries( comment: changes.comment, inline_comment: changes.inline_comment, }); + const regexPattern = "^(a|aa)*b$"; + const regexFailure = await client.callTool( + { + name: "search_strings", + arguments: { pattern: regexPattern, mode: "regex" }, + }, + options, + ); + assert.equal(regexFailure.isError, true); + const regexError = regexFailure.structuredContent.error; + assert.equal(regexError.code, "resource_constraint"); + assert.equal(regexError.details.operation, "search_strings"); + assert.equal(regexError.details.resource, "memory"); + assert.match( + regexError.remediation.action, + /literal mode or simplify the regex/u, + ); + assert.match( + regexError.details.reason, + /exhausted its stack.*Use literal mode/u, + ); + rejectedCalls++; + const nestedPattern = "(".repeat(12000) + "a" + ")".repeat(12000); + const compileFailure = await client.callTool( + { + name: "search_procedures", + arguments: { pattern: nestedPattern, mode: "regex" }, + }, + options, + ); + assert.equal(compileFailure.isError, true); + const compileError = compileFailure.structuredContent.error; + assert.equal(compileError.code, "resource_constraint"); + assert.equal(compileError.details.operation, "search_procedures"); + assert.match(compileError.details.reason, /stack while compiling pattern/u); + assert.equal(compileError.remediation.action, regexError.remediation.action); + rejectedCalls++; + assert.deepEqual( + await call("analyze_function", { procedure: address }), + updated.dossier, + "Regex stack exhaustion must preserve the live annotation database", + ); + assert.equal( + (await call("binary_session")).capabilities.find( + (item) => item.operation === "search_strings", + ).available, + true, + ); + assert.ok( + (await call("search_procedures", { pattern: changes.name })).some( + (item) => item.address === address, + ), + "Regex compiler exhaustion must preserve edited names and search availability", + ); + const longLiteral = "a".repeat(3 * 16 ** 3) + "!"; + const longString = strings.find((item) => item.value === longLiteral); + assert.ok(longString, "The provider must retain the full regression literal"); + const literalMatches = await call("search_strings", { pattern: longLiteral }); + assert.deepEqual(literalMatches, [ + { address: longString.address, value: longLiteral }, + ]); + assert.deepEqual(await cli("search", longLiteral), literalMatches); + await assert.rejects( + cli("search", regexPattern, ["--mode", "regex"]), + (error) => { + assert.equal(error.code, 1); + const rejected = JSON.parse(error.stdout); + assert.equal(rejected.code, regexError.code); + assert.equal(rejected.details.reason, regexError.details.reason); + assert.equal(rejected.remediation.action, regexError.remediation.action); + return true; + }, + ); assert.deepEqual(updated.effects, { scope: "session-analysis-database", source_bytes_modified: false, @@ -597,6 +670,7 @@ export async function verifyGhidraBoundaries( annotation_native_text_validation: true, lossless_unicode_transport: true, exact_external_entry_resolution: true, + recoverable_regex_stack_exhaustion: true, complete_function_body_references: true, ambiguity_candidates_inline: true, legacy_reference_snapshot_rejected: true, diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index 47f0e0867fd..22ddc4921c5 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -20,15 +20,17 @@ export class AnalysisUnsupportedTargetError extends AnalysisError { /** An analysis or transport resource failure, distinct from malformed input or unsupported coverage. */ export class AnalysisResourceConstraintError extends AnalysisError { readonly _tag = "AnalysisResourceConstraintError"; + readonly remediationAction: string | undefined; constructor( readonly operation: string, readonly resource: "memory" | "cpu" | "file-size" | "transport", readonly reason: string, readonly reportedLimits: Readonly> | null, - options?: AnalysisErrorOptions, + options?: AnalysisErrorOptions & { readonly remediationAction?: string }, ) { super(`Resource constraint during ${operation}: ${reason}`, options); + this.remediationAction = options?.remediationAction; } } diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index f1c8e942dc5..ff18238974d 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -41,13 +41,16 @@ export const analysisErrorRemediationAction = ( if (error instanceof AnalysisUnsupportedTargetError) return "Select a target supported by this operation or choose an operation supporting the reported target format."; if (error instanceof AnalysisResourceConstraintError) - return error.resource === "transport" - ? "Use the reported Evidence reference with a focused analysis tool when retained in this session, or export the session through export_evidence_bundle to a caller-selected path. Complete CLI JSON output is also available. The connection remains usable." - : error.resource === "cpu" - ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." - : error.resource === "file-size" - ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." - : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits."; + return ( + error.remediationAction ?? + (error.resource === "transport" + ? "Use the reported Evidence reference with a focused analysis tool when retained in this session, or export the session through export_evidence_bundle to a caller-selected path. Complete CLI JSON output is also available. The connection remains usable." + : error.resource === "cpu" + ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." + : error.resource === "file-size" + ? "Review the reported worker file-size limits and write failure. Retry with a sufficient file-size allowance for the evidence reply; REA retains tighter inherited limits." + : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits.") + ); if (error instanceof HopperTimeoutError) return error.providerState === "busy" ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry." diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index 28ddff2476f..640444e0189 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -26,6 +26,7 @@ export const GHIDRA_OPERATIONS = Object.freeze([ export const healthLimitations = Object.freeze([ "The session serves operations only after Ghidra reports default auto-analysis complete; incomplete analysis does not expose partial results.", "The imported Program and temporary project are ephemeral and deleted on close. Annotation operations edit session database metadata; original executable bytes are never written.", + "Cancelling an active Ghidra request terminates its ephemeral database and discards session annotations. The selected target remains open; the next provider query imports the original artifact again.", ]); /** Additional limitations applied to the experimental Windows x64 P0 boundary. */ @@ -97,7 +98,10 @@ export const limitationsFor = (operation: string): readonly string[] => { ]; case "search_procedures": case "search_strings": - return common; + return [ + ...common, + "Regex mode uses Java Pattern semantics. Matcher or compiler stack exhaustion returns a resource constraint without closing the session; literal mode avoids regex recursion and returns complete matching values.", + ]; case "procedure_pseudo_code": return [ ...common, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 1e8fa55e626..00a1cfa8762 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -13,6 +13,7 @@ import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, AnalysisInputError, + AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; @@ -335,6 +336,21 @@ const projectSessionError = ( ); if (failure.kind === "remote" && failure.remoteCode === "decompile_cancelled") return new AnalysisCancelledError(operation); + if ( + failure.kind === "remote" && + failure.remoteCode === "regex_stack_exhausted" + ) + return new AnalysisResourceConstraintError( + operation, + "memory", + failure.message, + null, + { + cause: failure, + remediationAction: + "Retry this search in literal mode or simplify the regex. The active analysis session and annotations remain available.", + }, + ); if ( failure.kind === "remote" && ["invalid_request", "not_found", "ambiguous"].includes( diff --git a/tests/conformance/ghidra/inventory.c b/tests/conformance/ghidra/inventory.c index 188e513c7d0..3655de0bb32 100644 --- a/tests/conformance/ghidra/inventory.c +++ b/tests/conformance/ghidra/inventory.c @@ -2,6 +2,16 @@ volatile int rea_ghidra_inventory_global = 7; +// This literal reproduces stack exhaustion in Java's recursive regex matcher. +#define REA_REPEAT_4(value) value value value value +#define REA_REPEAT_16(value) REA_REPEAT_4(REA_REPEAT_4(value)) +#define REA_REGEX_RUN REA_REPEAT_16(REA_REPEAT_16(REA_REPEAT_16("a"))) +__attribute__((used)) const char *volatile rea_ghidra_regex_subject = + REA_REGEX_RUN REA_REGEX_RUN REA_REGEX_RUN "!"; +#undef REA_REGEX_RUN +#undef REA_REPEAT_16 +#undef REA_REPEAT_4 + enum ReaFixtureState { REA_FIXTURE_IDLE = 0, REA_FIXTURE_READY = 3 }; union ReaFixturePayload { unsigned int word; unsigned char bytes[4]; }; struct ReaFixtureLayout { From 6b92002597d34555c32ebf24b9db6e4c17c0a08d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:25:38 +0800 Subject: [PATCH 004/791] fix(browser): preserve command failures and reject malformed replies --- docs/browser-observation.md | 2 +- src/browser/CdpConnection.ts | 51 ++++++++++++------ src/browser/CdpOptionalCommand.ts | 7 +-- ...BrowserProvider.lifecycle_failures.test.ts | 26 +++++++++ tests/boundary/browser/cdpConnection.test.ts | 53 +++++++++++++++++++ 5 files changed, 117 insertions(+), 22 deletions(-) diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 0d9bd8a35a8..e1ce74fd1f3 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -157,7 +157,7 @@ coverage. It reports excluded items and any values omitted or truncated by validation; it does not drop records to meet an internal collection quota. Disallowed-origin entries are filtered before retention. -CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited protocol events poison the connection so later commands fail closed. +CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited events and malformed replies to pending commands poison the connection so later commands fail closed. An optional method is reported unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. diff --git a/src/browser/CdpConnection.ts b/src/browser/CdpConnection.ts index cbf6bae445e..c547e20baf4 100644 --- a/src/browser/CdpConnection.ts +++ b/src/browser/CdpConnection.ts @@ -211,24 +211,43 @@ export class CdpConnection { #receiveResponse(message: Record, id: number): void { const pending = this.#pending.get(id); if (pending === undefined) return; - this.#complete(id, pending); - if ("error" in message) { - const reported = message.error; - pending.reject( - new CdpCommandRejection( - this.operation, - pending.method, - isRecord(reported) && typeof reported.code === "number" - ? reported.code - : null, - isRecord(reported) && typeof reported.message === "string" - ? reported.message - : null, - ), - ); + const hasResult = "result" in message; + const hasError = "error" in message; + if ( + ("sessionId" in message && typeof message.sessionId !== "string") || + hasResult === hasError + ) { + this.#failPending("protocol_error"); + return; + } + if (hasResult) { + if (!isRecord(message.result)) { + this.#failPending("protocol_error"); + return; + } + this.#complete(id, pending); + pending.resolve(message.result); + return; + } + const reported = message.error; + if ( + !isRecord(reported) || + typeof reported.code !== "number" || + !Number.isSafeInteger(reported.code) || + typeof reported.message !== "string" + ) { + this.#failPending("protocol_error"); return; } - pending.resolve("result" in message ? message.result : {}); + this.#complete(id, pending); + pending.reject( + new CdpCommandRejection( + this.operation, + pending.method, + reported.code, + reported.message, + ), + ); } #complete(id: number, pending: PendingCommand): void { diff --git a/src/browser/CdpOptionalCommand.ts b/src/browser/CdpOptionalCommand.ts index b02b2f78ff9..8939ce90400 100644 --- a/src/browser/CdpOptionalCommand.ts +++ b/src/browser/CdpOptionalCommand.ts @@ -1,4 +1,4 @@ -import { BrowserObservationError } from "../domain/browserObservationError.js"; +import { CdpCommandRejection } from "./CdpCommandRejection.js"; import { CdpConnection } from "./CdpConnection.js"; /** Execute an optional CDP method while preserving transport and cancellation failures. */ @@ -20,10 +20,7 @@ export const optionalCdpCommand = async ( context.signal, ); } catch (cause: unknown) { - if ( - !(cause instanceof BrowserObservationError) || - cause.reason !== "protocol_error" - ) + if (!(cause instanceof CdpCommandRejection) || cause.code !== -32_601) throw cause; limitations.push(`${method} was unavailable from this browser target.`); return undefined; diff --git a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts index 3d5fc7a6f80..3109e9d003e 100644 --- a/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.lifecycle_failures.test.ts @@ -269,6 +269,32 @@ describe("CdpBrowserProvider: protocol failures and cancellation cleanup", () => "Accessibility.getFullAXTree was unavailable from this browser target.", ); + const invalidAccessibilityParams = await startFakeCdpBrowser({ + commandError: (command) => + command.method === "Accessibility.getFullAXTree" + ? { code: -32_602, message: "Invalid params" } + : undefined, + }); + trackBrowser(invalidAccessibilityParams); + const invalidParamsResult = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: invalidAccessibilityParams.endpoint, + allowed_origins: [invalidAccessibilityParams.allowedOrigin], + target_id: "allowed-page", + observation_ms: 0, + }), + ); + expect(invalidParamsResult).toMatchObject({ + ok: false, + error: { + _tag: "BrowserObservationError", + reason: "protocol_error", + command: "Accessibility.getFullAXTree", + code: -32_602, + reportedMessage: "Invalid params", + }, + }); + for (const [options, reason] of [ [{ oversizedDiscovery: true }, "invalid_endpoint_response"], [{ invalidBrowserWebSocket: true }, "invalid_endpoint_response"], diff --git a/tests/boundary/browser/cdpConnection.test.ts b/tests/boundary/browser/cdpConnection.test.ts index 5df64c46b17..2b2bb0c08da 100644 --- a/tests/boundary/browser/cdpConnection.test.ts +++ b/tests/boundary/browser/cdpConnection.test.ts @@ -32,6 +32,8 @@ describe("CDP connection", () => { expect(frames).toMatchObject({ frameTree: { frame: { id: "frame-main" } }, }); + browser.emitRawMessage(JSON.stringify({ id: 999 })); + await expect(connection.send("Runtime.enable")).resolves.toEqual({}); } finally { await connection.close(); } @@ -84,6 +86,57 @@ describe("CDP connection", () => { } }); + it.each([ + ["missing result and error", (id: number) => ({ id })], + [ + "both result and error", + (id: number) => ({ + id, + result: {}, + error: { code: -32_602, message: "Invalid params" }, + }), + ], + [ + "malformed error", + (id: number) => ({ + id, + error: { code: "bad", message: "Invalid params" }, + }), + ], + ["invalid result", (id: number) => ({ id, result: [] })], + ["invalid session id", (id: number) => ({ id, result: {}, sessionId: 7 })], + ])( + "fails the connection for a correlated reply with %s", + async (_case, reply) => { + const browser = await startFakeCdpBrowser({ + hangOnMethod: "Page.enable", + }); + browsers.push(browser); + const connection = await CdpConnection.connect( + browser.browserWebSocketUrl, + "observe_web_session", + ); + try { + const pending = connection.send("Page.enable"); + const rejection = expect(pending).rejects.toMatchObject({ + _tag: "BrowserObservationError", + reason: "protocol_error", + }); + await vi.waitFor(() => expect(browser.commands).toHaveLength(1)); + const command = browser.commands[0]; + if (command === undefined) + throw new Error("Fake browser received no command"); + browser.emitRawMessage(JSON.stringify(reply(command.id))); + await rejection; + await expect(connection.send("Runtime.enable")).rejects.toMatchObject({ + reason: "protocol_error", + }); + } finally { + await connection.close(); + } + }, + ); + it("preserves the selected payload limit reason for pending and subsequent commands", async () => { const browser = await startFakeCdpBrowser({ commandResult: () => ({ oversized: "x".repeat(2_048) }), From 90b06db6c0b2c5863f292f63a226a5c5678a86ac Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:25:48 +0800 Subject: [PATCH 005/791] test(process): expose unavailable capture and unexpected teardown failures --- .../process/processCaptureTerminal.test.ts | 93 ++++++++++--------- tests/fixtures/android/jadx.ts | 14 ++- 2 files changed, 61 insertions(+), 46 deletions(-) diff --git a/tests/boundary/process/processCaptureTerminal.test.ts b/tests/boundary/process/processCaptureTerminal.test.ts index 3eb0ff39fa8..6fe449b6e7a 100644 --- a/tests/boundary/process/processCaptureTerminal.test.ts +++ b/tests/boundary/process/processCaptureTerminal.test.ts @@ -1,52 +1,55 @@ import { rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { expect, it } from "vitest"; +import { expect } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { itWithCaptureCapability } from "./processCaptureCapability.js"; -import { - captureProcessScenario, - probeProcessCaptureCapability, -} from "../../../src/process/capture/ProcessHarness.js"; +import { captureProcessScenario } from "../../../src/process/capture/ProcessHarness.js"; import { parseProcessScenario } from "../../../src/domain/process/processCapture.js"; -it("captures one command's terminal output, selected files, and owned cleanup", async () => { - const root = await createTestTempDirectory("rea-process-capture-"); - const script = join(root, "task.mjs"); - await writeFile( - script, - [ - 'import { writeFile } from "node:fs/promises";', - "process.stdout.write(`value:${process.env.CAPTURE_VALUE}\\n`);", - 'await writeFile("state.txt", "created");', - ].join("\n"), - ); - try { - const capability = await probeProcessCaptureCapability(); - if (!capability.available) return; - const result = await captureProcessScenario( - parseProcessScenario({ - executable: process.execPath, - arguments: [script], - working_directory: root, - environment: { CAPTURE_VALUE: "caller-selected" }, - filesystem_observation_paths: ["."], - }), +itWithCaptureCapability( + "captures one command's terminal output, selected files, and owned cleanup", + async () => { + const root = await createTestTempDirectory("rea-process-capture-"); + const script = join(root, "task.mjs"); + await writeFile( + script, + [ + 'import { writeFile } from "node:fs/promises";', + "process.stdout.write(`value:${process.env.CAPTURE_VALUE}\\n`);", + 'await writeFile("state.txt", "created");', + ].join("\n"), ); - if (!result.ok) throw result.error; - expect(result.value.frames.map(({ data }) => data).join("")).toContain( - "value:caller-selected", - ); - expect(result.value.files_after).toContainEqual( - expect.objectContaining({ path: "root_0:state.txt" }), - ); - expect(result.value.filesystem_effects).toContainEqual( - expect.objectContaining({ status: "created", path: "root_0:state.txt" }), - ); - expect(result.value.cleanup).toEqual({ - owned_process_group: "verified", - temporary_root: "removed", - }); - } finally { - await rm(root, { recursive: true, force: true }); - } -}, 20_000); + try { + const result = await captureProcessScenario( + parseProcessScenario({ + executable: process.execPath, + arguments: [script], + working_directory: root, + environment: { CAPTURE_VALUE: "caller-selected" }, + filesystem_observation_paths: ["."], + }), + ); + if (!result.ok) throw result.error; + expect(result.value.frames.map(({ data }) => data).join("")).toContain( + "value:caller-selected", + ); + expect(result.value.files_after).toContainEqual( + expect.objectContaining({ path: "root_0:state.txt" }), + ); + expect(result.value.filesystem_effects).toContainEqual( + expect.objectContaining({ + status: "created", + path: "root_0:state.txt", + }), + ); + expect(result.value.cleanup).toEqual({ + owned_process_group: "verified", + temporary_root: "removed", + }); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, + 20_000, +); diff --git a/tests/fixtures/android/jadx.ts b/tests/fixtures/android/jadx.ts index bc6a1850ddf..a9633e1de22 100644 --- a/tests/fixtures/android/jadx.ts +++ b/tests/fixtures/android/jadx.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import { expect, onTestFinished } from "vitest"; import { AndroidAnalysisService } from "../../../src/application/android/AndroidAnalysisService.js"; import { createAndroidAnalysisProvider } from "../../../src/composition/android.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; import { ProviderProcessSupervisor, spawnOwnedProviderProcess, @@ -74,7 +75,18 @@ export const createJadxProtocolFixture = async (mode = "normal") => { return spawned; }, ); - onTestFinished(() => provider.close().catch(() => undefined)); + onTestFinished(async () => { + try { + await provider.close(); + } catch (cause) { + if ( + mode !== "cleanup-failure" || + !(cause instanceof ProviderCleanupError) || + cause.providerId !== "jadx" + ) + throw cause; + } + }); return { service: new AndroidAnalysisService(provider), provider, From 3eb7dbdc1689737b44cc4ff044bcb1a311511969 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:26:25 +0800 Subject: [PATCH 006/791] fix(verify): isolate package archives and published canary caches --- scripts/verify-package-install.mjs | 12 +-- scripts/verify-package-pack.mjs | 27 ++++-- scripts/verify-package.mjs | 4 +- scripts/verify-packed-bridge.mjs | 2 +- scripts/verify-published-package.mjs | 17 ++-- .../process/packagePackIsolation.test.ts | 84 +++++++++++++++++++ 6 files changed, 118 insertions(+), 28 deletions(-) create mode 100644 tests/boundary/process/packagePackIsolation.test.ts diff --git a/scripts/verify-package-install.mjs b/scripts/verify-package-install.mjs index 81ee5afc66b..544ff93008e 100644 --- a/scripts/verify-package-install.mjs +++ b/scripts/verify-package-install.mjs @@ -6,7 +6,6 @@ import { exec, json, run } from "./lib/verify-package-core.mjs"; /** Install the packed CLI globally with and without optional dependencies and verify PTY degradation. */ export async function verifyPackageInstall({ - root, tarball, prefix, workspace, @@ -14,14 +13,7 @@ export async function verifyPackageInstall({ }) { await exec( "npm", - [ - "install", - "--global", - "--ignore-scripts", - "--prefix", - prefix, - join(root, tarball), - ], + ["install", "--global", "--ignore-scripts", "--prefix", prefix, tarball], { env: environment }, ); const cli = join(prefix, "bin", "rea"); @@ -61,7 +53,7 @@ export async function verifyPackageInstall({ "--omit=optional", "--prefix", noOptionalPrefix, - join(root, tarball), + tarball, ], { env: environment }, ); diff --git a/scripts/verify-package-pack.mjs b/scripts/verify-package-pack.mjs index 4a674075abd..2d5ebed08e6 100644 --- a/scripts/verify-package-pack.mjs +++ b/scripts/verify-package-pack.mjs @@ -1,3 +1,4 @@ +import { mkdir } from "node:fs/promises"; import { join } from "node:path"; import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; @@ -6,15 +7,23 @@ import { verifyPackedBridge } from "./verify-packed-bridge.mjs"; /** Create a tarball and assert production packaging constraints. */ export async function verifyPackagePack({ root, workspace }) { - const tarball = ( - await exec("npm", ["pack", "--silent"], { cwd: root }) - ).stdout.trim(); - const packedFiles = ( - await exec("tar", ["-tf", join(root, tarball)]) - ).stdout.split("\n"); + const packDirectory = join(workspace, "pack"); + await mkdir(packDirectory); + const tarball = join( + packDirectory, + ( + await exec( + "npm", + ["pack", "--silent", "--pack-destination", packDirectory], + { + cwd: root, + }, + ) + ).stdout.trim(), + ); + const packedFiles = (await exec("tar", ["-tf", tarball])).stdout.split("\n"); const packedManifest = JSON.parse( - (await exec("tar", ["-xOf", join(root, tarball), "package/package.json"])) - .stdout, + (await exec("tar", ["-xOf", tarball, "package/package.json"])).stdout, ); if ( packedManifest.scripts?.postinstall !== undefined || @@ -42,7 +51,7 @@ export async function verifyPackagePack({ root, workspace }) { const skill = ( await exec("tar", [ "-xOf", - join(root, tarball), + tarball, "package/skills/reverse-engineer-anything/SKILL.md", ]) ).stdout; diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index edd01f31155..9e2544e1412 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -44,7 +44,6 @@ try { referenceRoot, }); const { cli, packageRunnerCli } = await verifyPackageInstall({ - root, tarball, prefix: environmentData.prefix, workspace, @@ -58,7 +57,7 @@ try { "node_modules", PRODUCT_IDENTITY.packageName, ), - tarball: join(root, tarball), + tarball, workspace, environment: environmentData.environment, }); @@ -137,6 +136,5 @@ try { `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 4 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true, update, mcpStartup, mcpModuleLoading })}\n`, ); } finally { - if (tarball) await rm(join(root, tarball), { force: true }); await rm(workspace, { recursive: true, force: true }); } diff --git a/scripts/verify-packed-bridge.mjs b/scripts/verify-packed-bridge.mjs index 85477268359..a2ce508428f 100644 --- a/scripts/verify-packed-bridge.mjs +++ b/scripts/verify-packed-bridge.mjs @@ -18,7 +18,7 @@ export async function verifyPackedBridge({ throw new Error("package omitted the Hopper bridge"); if (!packedFiles.includes(packedGhidraBridge)) throw new Error("package omitted the Ghidra bridge"); - await exec("tar", ["-xf", join(root, tarball), "-C", workspace]); + await exec("tar", ["-xf", tarball, "-C", workspace]); const ghidraSource = await readFile( join(workspace, packedGhidraBridge), "utf8", diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index 35c5224efb3..aa1c05b386d 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -18,9 +18,12 @@ const version = process.argv[2]; if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u.test(version ?? "")) throw new Error("Usage: node scripts/verify-published-package.mjs "); -const serverEnvironment = { ...process.env }; -delete serverEnvironment.HOPPER_TARGET_PATH; const canaryRoot = await mkdtemp(join(tmpdir(), "rea-published-canary-")); +const serverEnvironment = { + ...process.env, + npm_config_cache: join(canaryRoot, "npm-cache"), +}; +delete serverEnvironment.HOPPER_TARGET_PATH; const transport = new StdioClientTransport({ command: "npm", args: [ @@ -91,7 +94,6 @@ process.stdout.write( ); async function verifyPublishedUpdate({ version: targetVersion, canaryRoot }) { - const previousVersion = await findPreviousPublishedVersion(targetVersion); const prefix = join(canaryRoot, "update-prefix"); const home = join(canaryRoot, "update-home"); const environment = { @@ -101,6 +103,11 @@ async function verifyPublishedUpdate({ version: targetVersion, canaryRoot }) { npm_config_prefix: prefix, PATH: `${join(prefix, "bin")}${delimiter}${process.env.PATH ?? ""}`, }; + const previousVersion = await findPreviousPublishedVersion( + targetVersion, + canaryRoot, + environment, + ); await execFileAsync( "npm", @@ -193,11 +200,11 @@ if (result.status === "failed") process.exitCode = 1; }; } -async function findPreviousPublishedVersion(targetVersion) { +async function findPreviousPublishedVersion(targetVersion, cwd, environment) { const { stdout } = await execFileAsync( "npm", ["view", "rea-agents", "versions", "--json"], - { maxBuffer: 16 * 1024 * 1024 }, + { cwd, env: environment, maxBuffer: 16 * 1024 * 1024 }, ); const versions = JSON.parse(stdout); const previous = versions diff --git a/tests/boundary/process/packagePackIsolation.test.ts b/tests/boundary/process/packagePackIsolation.test.ts new file mode 100644 index 00000000000..48c88eaf26e --- /dev/null +++ b/tests/boundary/process/packagePackIsolation.test.ts @@ -0,0 +1,84 @@ +import { execFile } from "node:child_process"; +import { + access, + mkdir, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execFileAsync = promisify(execFile); +const expectedFailure = + "package did not retain its MCP Registry ownership marker"; + +it("keeps npm pack output in the owned workspace when package validation fails", async () => { + const testRoot = await createTestTempDirectory("rea-pack-isolation-"); + const packageRoot = join(testRoot, "source"); + const workspace = join(testRoot, "workspace"); + const npmCache = join(testRoot, "npm-cache"); + const archiveName = "rea-agents-6.0.0.tgz"; + const archivePath = join(packageRoot, archiveName); + const sentinel = Buffer.from("caller-owned archive\n"); + await Promise.all([mkdir(packageRoot), mkdir(workspace), mkdir(npmCache)]); + await Promise.all([ + writeFile( + join(packageRoot, "package.json"), + JSON.stringify({ + name: "rea-agents", + version: "6.0.0", + mcpName: "invalid-owner", + bin: { rea: "scripts/rea.mjs", "rea-agents": "scripts/rea.mjs" }, + dependencies: { "@lydell/node-pty": "1.1.0" }, + }), + ), + writeFile(archivePath, sentinel), + ]); + const packageContentsBefore = (await readdir(packageRoot)).sort(); + const packerUrl = pathToFileURL( + resolve("scripts/verify-package-pack.mjs"), + ).href; + const harness = ` +import { rm } from "node:fs/promises"; +const workspace = ${JSON.stringify(workspace)}; +let failure; +try { + const { verifyPackagePack } = await import(${JSON.stringify(packerUrl)}); + await verifyPackagePack({ root: ${JSON.stringify(packageRoot)}, workspace }); +} catch (cause) { + if (!(cause instanceof Error) || cause.message !== ${JSON.stringify(expectedFailure)}) throw cause; + failure = cause.message; +} finally { + await rm(workspace, { recursive: true, force: true }); +} +if (failure !== ${JSON.stringify(expectedFailure)}) throw new Error("Expected package metadata validation to fail after packing"); +process.stdout.write(JSON.stringify({ failure })); +`; + const { stdout } = await execFileAsync( + process.execPath, + ["--input-type=module", "--eval", harness], + { + cwd: process.cwd(), + env: { + ...process.env, + npm_config_cache: npmCache, + TEMP: npmCache, + TMP: npmCache, + TMPDIR: npmCache, + }, + maxBuffer: 1_024 * 1_024, + }, + ); + + expect(JSON.parse(stdout)).toEqual({ failure: expectedFailure }); + expect((await readdir(packageRoot)).sort()).toEqual(packageContentsBefore); + expect(await readFile(archivePath)).toEqual(sentinel); + await expect(access(workspace)).rejects.toMatchObject({ code: "ENOENT" }); +}); From ee043e2cbf30152c60b403fff050a0fc848a3286 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:30:07 +0800 Subject: [PATCH 007/791] fix(verify): report final success only after resource cleanup --- docs/testing.md | 3 + scripts/verify-browser-dom-destinations.mjs | 11 +- scripts/verify-native-ui.mjs | 17 ++- scripts/verify-package.mjs | 42 +++++++- scripts/verify-real-browser.mjs | 62 ++++++----- scripts/verify-real-ghidra.mjs | 49 ++++----- scripts/verify-test-discovery.mjs | 6 +- scripts/verify-test-temp-hygiene.mjs | 6 +- scripts/verify-windows-package.mjs | 29 ++--- scripts/verify/apple/asset-catalog.mjs | 14 ++- scripts/verify/apple/dispatch.mjs | 6 +- scripts/verify/apple/interface-builder.mjs | 30 +++--- scripts/verify/apple/keyed-archive.mjs | 19 +++- scripts/verify/apple/macos-bundle.mjs | 26 ++--- scripts/verify/javascript/output.mjs | 22 ++-- scripts/verify/javascript/recovery.mjs | 16 +-- scripts/verify/managed/conformance.mjs | 114 ++++++++++---------- 17 files changed, 277 insertions(+), 195 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 21923f00b7c..fbd1765a44e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -43,6 +43,9 @@ through the compiled public CLI or MCP runtime. Calling a compiled private workflow or injecting a provider still establishes its narrower integration claim, even if a directory name calls it acceptance. +Emit a verifier's final success report only after awaited cleanup. Observe final +process lineage after closing its owned resources. + Measure slow files before removing capacity regressions. Optimize repeated process startup, fixture construction, and production algorithms when they dominate runtime. Keep at least one input beyond each formerly failing size or diff --git a/scripts/verify-browser-dom-destinations.mjs b/scripts/verify-browser-dom-destinations.mjs index 7390caf11ae..31b5d1e1746 100644 --- a/scripts/verify-browser-dom-destinations.mjs +++ b/scripts/verify-browser-dom-destinations.mjs @@ -13,6 +13,7 @@ if (!executable) const profile = await mkdtemp(join(tmpdir(), "rea-browser-dom-")); const site = await startBrowserVerifierSite(); let context; +let report; try { context = await chromium.launchPersistentContext(profile, { executablePath: executable, @@ -43,11 +44,15 @@ try { process.argv[2], page.url(), ); - process.stdout.write( - `${JSON.stringify({ ...result, browser: context.browser().version(), native_oracle: true, verified: true })}\n`, - ); + report = { + ...result, + browser: context.browser().version(), + native_oracle: true, + verified: true, + }; } finally { await context?.close(); await site.close(); await rm(profile, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify-native-ui.mjs b/scripts/verify-native-ui.mjs index 34616a72a91..8e2f3e05e88 100644 --- a/scripts/verify-native-ui.mjs +++ b/scripts/verify-native-ui.mjs @@ -15,6 +15,7 @@ if (process.platform !== "darwin") const root = await mkdtemp(join(tmpdir(), "rea-ui-fixture-")); let child; let fixturePid; +let report; try { const childRetrievalTest = join(root, "child-retrieval-test"); await promisify(execFile)("/usr/bin/xcrun", [ @@ -187,9 +188,18 @@ try { ); if (mismatch.ok || !mismatch.error.message.includes("target-mismatch")) throw new Error("Real helper did not reject changed target bytes"); - process.stdout.write( - `${JSON.stringify({ ok: true, verification_status: observation.ok ? "passed" : "permission-boundary-only", positive_e2e: observation.ok, observation: observation.ok ? "captured-selected-fixture-window" : "os-permission-denial-verified", scenario: scenarioStatus, permission_failure: observation.ok ? null : observation.error.message, mismatch_rejected: true, target_owned: true })}\n`, - ); + report = { + ok: true, + verification_status: observation.ok ? "passed" : "permission-boundary-only", + positive_e2e: observation.ok, + observation: observation.ok + ? "captured-selected-fixture-window" + : "os-permission-denial-verified", + scenario: scenarioStatus, + permission_failure: observation.ok ? null : observation.error.message, + mismatch_rejected: true, + target_owned: true, + }; } finally { if (fixturePid !== undefined) { try { @@ -205,3 +215,4 @@ try { } await rm(root, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 9e2544e1412..24923f7f6e5 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -34,6 +34,7 @@ let tarball; let mcpStartup; let mcpModuleLoading; let update; +let report; try { ({ tarball } = await verifyPackagePack({ root, workspace })); @@ -132,9 +133,44 @@ try { evidenceRoot, artifactArchive, }); - process.stdout.write( - `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 4 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true, update, mcpStartup, mcpModuleLoading })}\n`, - ); + report = { + cli: true, + analysisCli: true, + artifactCli: true, + managedCli: true, + managedReconstructionCli: true, + managedNativeVerificationCli: true, + managedApplicationGraphCli: true, + evidenceCli: true, + incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, + lifecycleScriptsRequired: false, + doctor: "platform-appropriate", + setup: supportedSetupHost + ? "planned-then-idempotent" + : "unsupported-host-rejected", + setupPlanReadOnly: supportedSetupHost, + existingHopperPreserved: supportedSetupHost, + clients: supportedSetupHost ? 4 : 0, + backupReadback: supportedSetupHost, + failureRecovery: supportedSetupHost, + configSymlinkLifecycle: supportedSetupHost, + skill: supportedSetupHost, + skillReferences: supportedSetupHost, + mcpTools: TOOL_CONTRACTS.length, + mcpPrompts: prompts.names.length, + promptCompletion: true, + promptCompletionLifecycle: true, + evidenceMcp: true, + targetFree: true, + targetLifecycle: true, + boundedRegexBridge: true, + update, + mcpStartup, + mcpModuleLoading, + }; } finally { await rm(workspace, { recursive: true, force: true }); } +process.stdout.write( + `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`, +); diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index e29cd7d78b8..b1cf21006ac 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -52,6 +52,7 @@ const profile = await mkdtemp(join(tmpdir(), "rea-real-browser-")); const site = await startBrowserVerifierSite(); let browser; let pageProxy; +let report; try { browser = spawn( executable, @@ -338,35 +339,32 @@ try { target_id: target, }); const moduleTrace = await verifyBrowserModules(executable); - process.stdout.write( - `${JSON.stringify({ - verifier_run: await completeVerifierRun(verifierRun), - browser: observed.value.browser.product, - endpoint, - target, - domNodes: observed.value.dom.nodes.length, - accessibilityNodes: observed.value.accessibility.nodes.length, - scripts: observed.value.scripts.items.length, - networkRequests: observed.value.network.requests.length, - consoleEvents: observed.value.console.events.length, - websocketEvents: observed.value.network.websocket_events.length, - bundleScripts: bundle.value.capture.scripts_analyzed, - sourceMaps: bundle.value.observations.source_maps.processed, - sessionEvents: session.value.timeline.length, - pageScopedTransport: true, - screenshotBytes: screenshot.value.artifact.bytes, - largeScreenshot, - popupEvents, - networkEvidence, - scriptExport, - moduleTrace, - domDestinations, - browserScenarioCli: true, - browserScenarioAttachCleanup: "disconnected-external", - browserScenarioLaunchCleanup: "terminated-owned-process", - verified: true, - })}\n`, - ); + report = { + browser: observed.value.browser.product, + endpoint, + target, + domNodes: observed.value.dom.nodes.length, + accessibilityNodes: observed.value.accessibility.nodes.length, + scripts: observed.value.scripts.items.length, + networkRequests: observed.value.network.requests.length, + consoleEvents: observed.value.console.events.length, + websocketEvents: observed.value.network.websocket_events.length, + bundleScripts: bundle.value.capture.scripts_analyzed, + sourceMaps: bundle.value.observations.source_maps.processed, + sessionEvents: session.value.timeline.length, + pageScopedTransport: true, + screenshotBytes: screenshot.value.artifact.bytes, + largeScreenshot, + popupEvents, + networkEvidence, + scriptExport, + moduleTrace, + domDestinations, + browserScenarioCli: true, + browserScenarioAttachCleanup: "disconnected-external", + browserScenarioLaunchCleanup: "terminated-owned-process", + verified: true, + }; } finally { if (pageProxy !== undefined) await pageProxy.close(); if (browser !== undefined) await stopProcess(browser); @@ -378,6 +376,12 @@ try { retryDelay: 100, }); } +process.stdout.write( + `${JSON.stringify({ + verifier_run: await completeVerifierRun(verifierRun), + ...report, + })}\n`, +); async function verifyPageScopedTransport(provider, proxy, origin) { const target = await pageTarget(provider, proxy.endpoint); diff --git a/scripts/verify-real-ghidra.mjs b/scripts/verify-real-ghidra.mjs index 3dafec49610..d960b3d1b5d 100644 --- a/scripts/verify-real-ghidra.mjs +++ b/scripts/verify-real-ghidra.mjs @@ -43,6 +43,7 @@ import { } from "./verify-real-ghidra-function.mjs"; const exec = promisify(execFile); const verifierRun = createVerifierRun(); +let report; const installDir = process.env.GHIDRA_INSTALL_DIR; if (installDir === undefined || !isAbsolute(installDir)) throw new Error( @@ -265,33 +266,33 @@ try { const customPath = process.env.GHIDRA_TARGET_PATH; const custom = customPath === undefined ? null : await verifyTarget(customPath, "custom"); - process.stdout.write( - `${JSON.stringify({ - verifier_run: await completeVerifierRun(verifierRun), - ok: true, - provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION }, - verification_lane: aarch64JumpTableOnly - ? "aarch64-jump-table" - : crossFormat - ? "cross-format" - : "host-native", - fixture_sources: - crossFormat || aarch64JumpTableOnly - ? [sourcePath, crossFormatSourcePath] - : [sourcePath], - fixtures: [debug, stripped, layout, ...crossResults].map(summary), - malformed_target: "rejected-before-provider-start", - native_api_cli: aarch64JumpTableOnly - ? (crossResults[0]?.native_api_cli ?? null) - : debug.native_api_cli, - native_value_e2e: debug.native_values, - custom_target: custom === null ? null : summary(custom), - cleanup: "complete", - })}\n`, - ); + report = { + ok: true, + provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION }, + verification_lane: aarch64JumpTableOnly + ? "aarch64-jump-table" + : crossFormat + ? "cross-format" + : "host-native", + fixture_sources: + crossFormat || aarch64JumpTableOnly + ? [sourcePath, crossFormatSourcePath] + : [sourcePath], + fixtures: [debug, stripped, layout, ...crossResults].map(summary), + malformed_target: "rejected-before-provider-start", + native_api_cli: aarch64JumpTableOnly + ? (crossResults[0]?.native_api_cli ?? null) + : debug.native_api_cli, + native_value_e2e: debug.native_values, + custom_target: custom === null ? null : summary(custom), + cleanup: "complete", + }; } finally { await rm(fixtureRoot, { recursive: true, force: true }); } +process.stdout.write( + `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`, +); function nativeFixtureTarget(platform, architecture) { const supportedTargets = { diff --git a/scripts/verify-test-discovery.mjs b/scripts/verify-test-discovery.mjs index b5101d2553e..46a6aced877 100644 --- a/scripts/verify-test-discovery.mjs +++ b/scripts/verify-test-discovery.mjs @@ -10,6 +10,7 @@ const execute = promisify(execFile); const root = resolve(import.meta.dirname, ".."); const temporary = await mkdtemp(join(tmpdir(), "rea-test-discovery-")); +let discovered; try { const output = join(temporary, "discovery.json"); await execute( @@ -58,8 +59,9 @@ try { ) { console.error(JSON.stringify(report, null, 2)); process.exitCode = 1; - } else - console.log(`Discovered all ${report.discovered} test files exactly once.`); + } else discovered = report.discovered; } finally { await rm(temporary, { recursive: true, force: true }); } +if (discovered !== undefined) + console.log(`Discovered all ${discovered} test files exactly once.`); diff --git a/scripts/verify-test-temp-hygiene.mjs b/scripts/verify-test-temp-hygiene.mjs index e4aecab131f..cdfdb9fa9e0 100644 --- a/scripts/verify-test-temp-hygiene.mjs +++ b/scripts/verify-test-temp-hygiene.mjs @@ -59,9 +59,6 @@ try { } if (failures.length > 0) throw new Error(failures.join("\n\n")); - process.stdout.write( - "Test temporary-directory hygiene verified: no owned residues remain.\n", - ); } finally { await rm(isolatedTemporaryRoot, { recursive: true, @@ -70,3 +67,6 @@ try { retryDelay: 25, }); } +process.stdout.write( + "Test temporary-directory hygiene verified: no owned residues remain.\n", +); diff --git a/scripts/verify-windows-package.mjs b/scripts/verify-windows-package.mjs index 52361cad34c..3d3fabf3531 100644 --- a/scripts/verify-windows-package.mjs +++ b/scripts/verify-windows-package.mjs @@ -27,6 +27,7 @@ const verifierRun = createVerifierRun(); const root = process.cwd(); const workspace = await mkdtemp(join(tmpdir(), "rea-windows-package-")); const prefix = join(workspace, "prefix"); +let report; try { const packed = JSON.parse( @@ -212,23 +213,23 @@ try { environment, }); - process.stdout.write( - `${JSON.stringify({ - verifier_run: await completeVerifierRun(verifierRun), - ok: true, - platform: process.platform, - package: packageResult.filename, - tools: toolCount, - ghidra_bridge: "present", - agent_setup: ["opencode", "vscode", "copilot_cli"], - setup_idempotent: true, - uninstall: true, - update, - })}\n`, - ); + report = { + ok: true, + platform: process.platform, + package: packageResult.filename, + tools: toolCount, + ghidra_bridge: "present", + agent_setup: ["opencode", "vscode", "copilot_cli"], + setup_idempotent: true, + uninstall: true, + update, + }; } finally { await rm(workspace, { recursive: true, force: true }); } +process.stdout.write( + `${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`, +); function npm(arguments_, cwd) { const npmExecPath = process.env.npm_execpath; diff --git a/scripts/verify/apple/asset-catalog.mjs b/scripts/verify/apple/asset-catalog.mjs index 8a62396f148..f29d79386a6 100644 --- a/scripts/verify/apple/asset-catalog.mjs +++ b/scripts/verify/apple/asset-catalog.mjs @@ -21,6 +21,7 @@ await exec("/usr/bin/xcrun", ["--find", "actool"]); await exec("/usr/bin/xcrun", ["--find", "clang"]); await exec("/usr/bin/assetutil", ["--version"]); const root = await mkdtemp(join(tmpdir(), "rea-asset-e2e-")); +let report; try { const app = join(root, "Fixture.app"); const resources = join(app, "Contents", "Resources"); @@ -98,9 +99,16 @@ try { }); assert.equal(invalid.isError, true); }); - process.stdout.write( - `${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, records: raw.length, pagination: true, malformed_input_rejected: true })}\n`, - ); + report = { + ok: true, + mocked: false, + cli: true, + stdio_mcp: true, + records: raw.length, + pagination: true, + malformed_input_rejected: true, + }; } finally { await rm(root, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify/apple/dispatch.mjs b/scripts/verify/apple/dispatch.mjs index 3e108c10ede..bf86d85d657 100644 --- a/scripts/verify/apple/dispatch.mjs +++ b/scripts/verify/apple/dispatch.mjs @@ -142,6 +142,7 @@ const checkSwift = (metadata, mode) => { }; const root = await mkdtemp(join(tmpdir(), "rea-dispatch-")); +let report; try { const results = []; for (const mode of LINK_MODES) { @@ -197,9 +198,8 @@ try { }); } } - process.stdout.write( - `${JSON.stringify({ ok: true, target_executed: false, fixtures: results })}\n`, - ); + report = { ok: true, target_executed: false, fixtures: results }; } finally { await rm(root, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify/apple/interface-builder.mjs b/scripts/verify/apple/interface-builder.mjs index 1f8ec323e05..fb3346f738c 100644 --- a/scripts/verify/apple/interface-builder.mjs +++ b/scripts/verify/apple/interface-builder.mjs @@ -21,6 +21,7 @@ const sourceRoot = fileURLToPath( new URL("../../../tests/conformance/interface-builder/", import.meta.url), ); +let report; try { await mkdir(join(appPath, "Contents", "MacOS"), { recursive: true }); await mkdir(resourcesPath, { recursive: true }); @@ -83,21 +84,20 @@ try { ); assert.equal(decoded.graph.truncated, false); - process.stdout.write( - `${JSON.stringify({ - ok: true, - fixture: "compiled-appkit-nib", - provider: execution.value.provider, - target_sha256: decoded.target_sha256, - archive_sha256: decoded.documents[0].archive_sha256, - objects: decoded.documents[0].object_count, - connections: decoded.documents[0].connection_count, - recovered_control: button.name, - recovered_action: action.name, - hierarchy_complete: decoded.documents[0].hierarchy_complete, - truncated: decoded.graph.truncated, - })}\n`, - ); + report = { + ok: true, + fixture: "compiled-appkit-nib", + provider: execution.value.provider, + target_sha256: decoded.target_sha256, + archive_sha256: decoded.documents[0].archive_sha256, + objects: decoded.documents[0].object_count, + connections: decoded.documents[0].connection_count, + recovered_control: button.name, + recovered_action: action.name, + hierarchy_complete: decoded.documents[0].hierarchy_complete, + truncated: decoded.graph.truncated, + }; } finally { await rm(fixtureRoot, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index 924c3bb0e1f..4a162e72e87 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -17,6 +17,7 @@ if (process.platform !== "darwin") "Real Foundation keyed archive verification requires macOS and Xcode Swift tools", ); const root = await mkdtemp(join(tmpdir(), "rea-keyed-fixture-")); +let report; try { const archive = join(root, "model.plist"); await promisify(execFile)("/usr/bin/xcrun", [ @@ -150,9 +151,21 @@ try { [{ source: null, path: ["UID"], target: 1, status: "resolved" }], ); } - process.stdout.write( - `${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, xml_golden: true, uid_named_root: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, shared_identity: true, cyclic_identity: true, target_classes_instantiated_by_reader: false })}\n`, - ); + report = { + ok: true, + mocked: false, + cli: true, + stdio_mcp: true, + xml_golden: true, + uid_named_root: true, + format: graph.archive_format, + objects: graph.total_objects, + references: graph.total_references, + shared_identity: true, + cyclic_identity: true, + target_classes_instantiated_by_reader: false, + }; } finally { await rm(root, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify/apple/macos-bundle.mjs b/scripts/verify/apple/macos-bundle.mjs index 4108f19888f..815c50a846c 100644 --- a/scripts/verify/apple/macos-bundle.mjs +++ b/scripts/verify/apple/macos-bundle.mjs @@ -65,6 +65,7 @@ const EXPECTED_BUNDLES = [ ["Contents/XPCServices/Svc.xpc", "xpc-service", "macos-deep", "."], ]; +let report; try { const { app } = await buildMacosBundleFixture(root); const zip = join(root, "MacFixture.zip"); @@ -155,19 +156,18 @@ try { const dylibs = await verifyDylibResolution(app); const dyldEnvironment = await verifyDyldEnvironment(root); - process.stdout.write( - `${JSON.stringify({ - ok: true, - mocked: false, - cli: true, - stdio_mcp: true, - containers: containers.map(({ format }) => format), - bundles: EXPECTED_BUNDLES.length, - dmg_detached: true, - dylib_resolution: dylibs, - dyld_environment: dyldEnvironment, - })}\n`, - ); + report = { + ok: true, + mocked: false, + cli: true, + stdio_mcp: true, + containers: containers.map(({ format }) => format), + bundles: EXPECTED_BUNDLES.length, + dmg_detached: true, + dylib_resolution: dylibs, + dyld_environment: dyldEnvironment, + }; } finally { await rm(root, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjs index 0e6d5635ea8..13dfa916c79 100644 --- a/scripts/verify/javascript/output.mjs +++ b/scripts/verify/javascript/output.mjs @@ -34,6 +34,7 @@ const sampleRss = () => { }; const sampler = setInterval(sampleRss, 25); sampler.unref(); +let finalReport; try { const evidence = createEvidence( @@ -112,17 +113,15 @@ try { assert.equal(digest, expectedHash.digest("hex")); assert.equal((await stat(path)).size, expectedBytes); sampleRss(); - process.stdout.write( - `${JSON.stringify({ - json_output_verified: true, - format, - output_bytes: expectedBytes, - engine_string_limit: constants.MAX_STRING_LENGTH, - output_sha256: digest, - evidence_id: evidence.evidence_id, - rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss }, - })}\n`, - ); + finalReport = { + json_output_verified: true, + format, + output_bytes: expectedBytes, + engine_string_limit: constants.MAX_STRING_LENGTH, + output_sha256: digest, + evidence_id: evidence.evidence_id, + rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss }, + }; } finally { clearInterval(sampler); if (destination !== undefined && !destination.closed) { @@ -132,3 +131,4 @@ try { } await rm(workspace, { recursive: true, force: true }); } +process.stdout.write(`${JSON.stringify(finalReport)}\n`); diff --git a/scripts/verify/javascript/recovery.mjs b/scripts/verify/javascript/recovery.mjs index f200b355acf..fcee30ba8c5 100644 --- a/scripts/verify/javascript/recovery.mjs +++ b/scripts/verify/javascript/recovery.mjs @@ -49,6 +49,7 @@ const call = async (name, args) => { }; const fingerprint = (bytes) => createHash("sha256").update(bytes).digest("hex"); const counts = []; +let finalReport; try { const fixtures = await buildRecoveryFixtures( root, @@ -128,19 +129,18 @@ try { }); } } - console.log( - JSON.stringify({ - engine: "wakaru 1.13.0", - toolchains: fixtures.toolchains, - results: counts, - verified: true, - }), - ); + finalReport = { + engine: "wakaru 1.13.0", + toolchains: fixtures.toolchains, + results: counts, + verified: true, + }; } finally { await client.close(); await transport.close(); await rm(root, { recursive: true, force: true }); } +console.log(JSON.stringify(finalReport)); async function assertRecoveredArtifacts(evidence, result, fixture) { assert.deepEqual(result.options, { diff --git a/scripts/verify/managed/conformance.mjs b/scripts/verify/managed/conformance.mjs index c274fff409c..4a0ed5c3b12 100644 --- a/scripts/verify/managed/conformance.mjs +++ b/scripts/verify/managed/conformance.mjs @@ -34,6 +34,8 @@ import { const verifierRun = createVerifierRun(); const workspace = await mkdtemp(join(tmpdir(), "rea-managed-conformance-")); +let completionReportInput; +let finalReport; const { fixture, fixtureBytes, @@ -378,65 +380,61 @@ try { const manifestSelfTest = await runManagedAppManifestSelfTest(); const operatorManifest = await runOptionalManagedAppManifest(); const ilspyOracle = await runOptionalIlspyOracle(); - const completionReport = createManagedCompletionReport( - { - modern, - framework, - nativeFunctionTarget, - readyToRun, - obfuscated, - left, - right, - nativeOnly, - malformed, - manifestSelfTest, - operatorManifest, - ilspyOracle, + completionReportInput = { + modern, + framework, + nativeFunctionTarget, + readyToRun, + obfuscated, + left, + right, + nativeOnly, + malformed, + manifestSelfTest, + operatorManifest, + ilspyOracle, + }; + finalReport = { + verified: + 11 + (operatorManifest === null ? 0 : 1) + (ilspyOracle === null ? 0 : 1), + managedSurfaces: [ + "inspect_managed_artifact", + "inspect_managed_members", + "inspect_managed_native_boundaries", + "compare_managed_members", + "verify_managed_native_boundaries", + "import_managed_reconstruction", + "project_managed_application_graph", + ], + coverage: [ + "modern-dotnet-anycpu", + "dotnet-framework-x86-pinvoke", + "x64-ready-to-run-native-body", + "managed-native-verification", + "managed-application-graph-projection", + "unicode-obfuscated-identifiers", + "decompiler-reconstruction-import", + "mvid-and-token-drift", + "not-managed", + "malformed-metadata", + "operator-local-managed-manifest", + ...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]), + ], + managedAppManifest: { + env: "REA_MANAGED_APP_MANIFEST_PATH", + selfTest: manifestSelfTest, + operator: operatorManifest ?? { configured: false }, }, - await completeVerifierRun(verifierRun), - ); - - process.stdout.write( - `${JSON.stringify({ - verified: - 11 + - (operatorManifest === null ? 0 : 1) + - (ilspyOracle === null ? 0 : 1), - managedSurfaces: [ - "inspect_managed_artifact", - "inspect_managed_members", - "inspect_managed_native_boundaries", - "compare_managed_members", - "verify_managed_native_boundaries", - "import_managed_reconstruction", - "project_managed_application_graph", - ], - coverage: [ - "modern-dotnet-anycpu", - "dotnet-framework-x86-pinvoke", - "x64-ready-to-run-native-body", - "managed-native-verification", - "managed-application-graph-projection", - "unicode-obfuscated-identifiers", - "decompiler-reconstruction-import", - "mvid-and-token-drift", - "not-managed", - "malformed-metadata", - "operator-local-managed-manifest", - ...(ilspyOracle === null ? [] : ["ilspy-reconstruction-oracle"]), - ], - managedAppManifest: { - env: "REA_MANAGED_APP_MANIFEST_PATH", - selfTest: manifestSelfTest, - operator: operatorManifest ?? { configured: false }, - }, - ilspyOracle: ilspyOracle ?? { - env: "REA_ILSPY_CMD_PATH", - configured: false, - }, - completionReport, - })}\n`, - ); + ilspyOracle: ilspyOracle ?? { + env: "REA_ILSPY_CMD_PATH", + configured: false, + }, + }; } finally { await rm(workspace, { recursive: true, force: true }); } +finalReport.completionReport = createManagedCompletionReport( + completionReportInput, + await completeVerifierRun(verifierRun), +); +process.stdout.write(`${JSON.stringify(finalReport)}\n`); From 96d3cafb0725881b6cdab0e2185104957dc69f71 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:48:45 +0800 Subject: [PATCH 008/791] fix(browser): bind correlated replies to their selected session --- docs/browser-observation.md | 2 +- src/browser/CdpConnection.ts | 6 ++- tests/boundary/browser/cdpConnection.test.ts | 48 +++++++++++--------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/docs/browser-observation.md b/docs/browser-observation.md index e1ce74fd1f3..c0840111de6 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -157,7 +157,7 @@ coverage. It reports excluded items and any values omitted or truncated by validation; it does not drop records to meet an internal collection quota. Disallowed-origin entries are filtered before retention. -CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited events and malformed replies to pending commands poison the connection so later commands fail closed. An optional method is reported unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. +CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited events and malformed replies to pending commands poison the connection so later commands fail closed. When a correlated reply reports a session ID, it must match the command's selected session. An optional method is reported unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. diff --git a/src/browser/CdpConnection.ts b/src/browser/CdpConnection.ts index c547e20baf4..fae6046ed27 100644 --- a/src/browser/CdpConnection.ts +++ b/src/browser/CdpConnection.ts @@ -15,6 +15,7 @@ export interface CdpEvent { interface PendingCommand { readonly method: string; + readonly sessionId?: string; readonly resolve: (value: unknown) => void; readonly reject: (error: AnalysisError) => void; readonly removeAbort: () => void; @@ -120,6 +121,7 @@ export class CdpConnection { signal?.addEventListener("abort", onAbort, { once: true }); this.#pending.set(id, { method, + ...(sessionId === undefined ? {} : { sessionId }), resolve, reject, removeAbort: () => signal?.removeEventListener("abort", onAbort), @@ -214,7 +216,9 @@ export class CdpConnection { const hasResult = "result" in message; const hasError = "error" in message; if ( - ("sessionId" in message && typeof message.sessionId !== "string") || + ("sessionId" in message && + (typeof message.sessionId !== "string" || + message.sessionId !== pending.sessionId)) || hasResult === hasError ) { this.#failPending("protocol_error"); diff --git a/tests/boundary/browser/cdpConnection.test.ts b/tests/boundary/browser/cdpConnection.test.ts index 2b2bb0c08da..5ce4195a743 100644 --- a/tests/boundary/browser/cdpConnection.test.ts +++ b/tests/boundary/browser/cdpConnection.test.ts @@ -6,6 +6,31 @@ import { type FakeCdpBrowser, } from "../../fixtures/fakeCdpBrowser.js"; +const INVALID_REPLIES = [ + ["missing result and error", (id: number) => ({ id })], + [ + "both result and error", + (id: number) => ({ + id, + result: {}, + error: { code: -32_602, message: "Invalid params" }, + }), + ], + [ + "malformed error", + (id: number) => ({ + id, + error: { code: "bad", message: "Invalid params" }, + }), + ], + ["invalid result", (id: number) => ({ id, result: [] })], + ["invalid session id", (id: number) => ({ id, result: {}, sessionId: 7 })], + [ + "foreign session id", + (id: number) => ({ id, result: {}, sessionId: "foreign-session" }), + ], +] as const; + describe("CDP connection", () => { const browsers: FakeCdpBrowser[] = []; @@ -86,26 +111,7 @@ describe("CDP connection", () => { } }); - it.each([ - ["missing result and error", (id: number) => ({ id })], - [ - "both result and error", - (id: number) => ({ - id, - result: {}, - error: { code: -32_602, message: "Invalid params" }, - }), - ], - [ - "malformed error", - (id: number) => ({ - id, - error: { code: "bad", message: "Invalid params" }, - }), - ], - ["invalid result", (id: number) => ({ id, result: [] })], - ["invalid session id", (id: number) => ({ id, result: {}, sessionId: 7 })], - ])( + it.each(INVALID_REPLIES)( "fails the connection for a correlated reply with %s", async (_case, reply) => { const browser = await startFakeCdpBrowser({ @@ -117,7 +123,7 @@ describe("CDP connection", () => { "observe_web_session", ); try { - const pending = connection.send("Page.enable"); + const pending = connection.send("Page.enable", {}, "selected-session"); const rejection = expect(pending).rejects.toMatchObject({ _tag: "BrowserObservationError", reason: "protocol_error", From a0784c0d2cb70c48ecfeb0be23c6d68de5ee5be1 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:49:31 +0800 Subject: [PATCH 009/791] test(process): retire native caches at worker teardown --- docs/testing.md | 7 ++++++- knip.json | 7 ++++++- .../androidAnalysisMcp.test.ts | 0 .../jadxIntegration.test.ts | 0 tests/processOwnershipRunner.ts | 18 ++++++++++++++++++ vitest.config.ts | 1 + 6 files changed, 31 insertions(+), 2 deletions(-) rename tests/boundary/{mcp => process}/androidAnalysisMcp.test.ts (100%) rename tests/boundary/{android => process}/jadxIntegration.test.ts (100%) create mode 100644 tests/processOwnershipRunner.ts diff --git a/docs/testing.md b/docs/testing.md index fbd1765a44e..afca620548b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -46,6 +46,11 @@ claim, even if a directory name calls it acceptance. Emit a verifier's final success report only after awaited cleanup. Observe final process lineage after closing its owned resources. +The Vitest runner releases cached native process inspectors during awaited +worker teardown. Fork termination does not run Node's normal exit hooks; a +per-file teardown would instead retire shared inspectors before later files +can use them. + Measure slow files before removing capacity regressions. Optimize repeated process startup, fixture construction, and production algorithms when they dominate runtime. Keep at least one input beyond each formerly failing size or @@ -356,7 +361,7 @@ are separate proof levels. Synthetic producer regressions run independently: ```sh -npm run test:focused -- tests/boundary/android/jadxIntegration.test.ts tests/boundary/mcp/androidAnalysisMcp.test.ts +npm run test:focused -- tests/boundary/process/jadxIntegration.test.ts tests/boundary/process/androidAnalysisMcp.test.ts ``` ## Optional NativeAOT Ghidra analysis diff --git a/knip.json b/knip.json index 5ac657f65f3..ae8742b2ce7 100644 --- a/knip.json +++ b/knip.json @@ -1,6 +1,11 @@ { "$schema": "./node_modules/knip/schema.json", - "entry": ["src/main.ts", "src/cli.ts", "src/contracts/errorSchemas.ts"], + "entry": [ + "src/main.ts", + "src/cli.ts", + "src/contracts/errorSchemas.ts", + "tests/processOwnershipRunner.ts" + ], "project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"], "vitepress": { "entry": ["docs/.vitepress/config.ts"] diff --git a/tests/boundary/mcp/androidAnalysisMcp.test.ts b/tests/boundary/process/androidAnalysisMcp.test.ts similarity index 100% rename from tests/boundary/mcp/androidAnalysisMcp.test.ts rename to tests/boundary/process/androidAnalysisMcp.test.ts diff --git a/tests/boundary/android/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts similarity index 100% rename from tests/boundary/android/jadxIntegration.test.ts rename to tests/boundary/process/jadxIntegration.test.ts diff --git a/tests/processOwnershipRunner.ts b/tests/processOwnershipRunner.ts new file mode 100644 index 00000000000..108c76295f6 --- /dev/null +++ b/tests/processOwnershipRunner.ts @@ -0,0 +1,18 @@ +import { TestRunner } from "vitest"; + +import type { ProcessOwnershipHost } from "../src/process/ProcessOwnership.js"; + +/** Retire each isolated graph's native cache when its test worker stops. */ +export default class ProcessOwnershipRunner extends TestRunner { + readonly #registeredHosts = new WeakSet(); + + async onBeforeRunFiles(): Promise { + const { systemProcessOwnershipHost } = + await import("../src/process/ProcessOwnershipObservation.js"); + if (this.#registeredHosts.has(systemProcessOwnershipHost)) return; + if (this.onCleanupWorkerContext === undefined) + throw new Error("The test pool does not support worker resource cleanup"); + this.#registeredHosts.add(systemProcessOwnershipHost); + this.onCleanupWorkerContext(() => systemProcessOwnershipHost.close?.()); + } +} diff --git a/vitest.config.ts b/vitest.config.ts index 0ab5bbc44b9..8c629580d46 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -134,6 +134,7 @@ export default defineConfig({ projects, retry: 0, reporters: ["default"], + runner: "./tests/processOwnershipRunner.ts", // Boundary projects may compete with TypeScript, docs, and package checks // under Turbo. Keep the deadline bounded while avoiding false failures from // host-level CPU and filesystem contention. From f591e37a1e45d9b7179ed788c4e7ee77a708c6b0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:03:54 +0800 Subject: [PATCH 010/791] test(artifacts): retire simulated mounts after detach failures --- .../native/nativeDmgArtifactReader.test.ts | 92 +++++++++++-------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts index 6de63381317..8b0df273056 100644 --- a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts +++ b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts @@ -77,6 +77,11 @@ describe("native DMG artifact reader", () => { ), }); + let mountRoot: string | undefined; + onTestFinished(async () => { + if (mountRoot !== undefined) + await rm(mountRoot, { recursive: true, force: true }); + }); const host: NativeDmgHost = { run(arguments_) { if (arguments_[0] === "detach") @@ -92,6 +97,8 @@ describe("native DMG artifact reader", () => { }); if (arguments_[0] !== "attach") return Promise.resolve({ stdout: "", exitCode: 0 }); + mountRoot = arguments_[arguments_.indexOf("-mountroot") + 1]; + if (mountRoot === undefined) throw new Error("missing mount root"); return Promise.resolve({ stdout: build({ "system-entities": [ @@ -387,53 +394,58 @@ it("owns the canonical mount root and detaches each observed whole image once", ]); }); -describe("APFS disk image detach", () => { - // hdiutil reports the image disk and its synthesized APFS container as two - // whole disks; detaching the container ejects the image disk as well. - const apfsHost = (stillListed: readonly string[]) => { - const calls: string[][] = []; - const host: NativeDmgHost = { - async run(arguments_) { - calls.push([...arguments_]); - if (arguments_[0] === "detach" && arguments_[1] === "/dev/disk4") - throw new Error("hdiutil: detach failed - No such file or directory"); - if (arguments_[0] === "info") - return { - stdout: build({ - images: - stillListed.length === 0 - ? [] - : [ - { - "system-entities": stillListed.map((device) => ({ - "dev-entry": device, - })), - }, - ], - }), - exitCode: 0, - }; - if (arguments_[0] !== "attach") return { stdout: "", exitCode: 0 }; - const mountRoot = arguments_[arguments_.indexOf("-mountroot") + 1]; - if (mountRoot === undefined) throw new Error("missing mount root"); - const mountPoint = join(mountRoot, "Fixture"); - await mkdir(mountPoint); +// hdiutil reports the image disk and synthesized APFS container separately; +// detaching the container can already eject the image disk. +const apfsHost = (stillListed: readonly string[]) => { + const calls: string[][] = []; + let mountRoot: string | undefined; + onTestFinished(async () => { + if (mountRoot !== undefined) + await rm(mountRoot, { recursive: true, force: true }); + }); + const host: NativeDmgHost = { + async run(arguments_) { + calls.push([...arguments_]); + if (arguments_[0] === "detach" && arguments_[1] === "/dev/disk4") + throw new Error("hdiutil: detach failed - No such file or directory"); + if (arguments_[0] === "info") return { stdout: build({ - "system-entities": [ - { "dev-entry": "/dev/disk4" }, - { "dev-entry": "/dev/disk4s1" }, - { "dev-entry": "/dev/disk5s1", "mount-point": mountPoint }, - { "dev-entry": "/dev/disk5" }, - ], + images: + stillListed.length === 0 + ? [] + : [ + { + "system-entities": stillListed.map((device) => ({ + "dev-entry": device, + })), + }, + ], }), exitCode: 0, }; - }, - }; - return { calls, host }; + if (arguments_[0] !== "attach") return { stdout: "", exitCode: 0 }; + mountRoot = arguments_[arguments_.indexOf("-mountroot") + 1]; + if (mountRoot === undefined) throw new Error("missing mount root"); + const mountPoint = join(mountRoot, "Fixture"); + await mkdir(mountPoint); + return { + stdout: build({ + "system-entities": [ + { "dev-entry": "/dev/disk4" }, + { "dev-entry": "/dev/disk4s1" }, + { "dev-entry": "/dev/disk5s1", "mount-point": mountPoint }, + { "dev-entry": "/dev/disk5" }, + ], + }), + exitCode: 0, + }; + }, }; + return { calls, host }; +}; +describe("APFS disk image detach", () => { it("accepts an image disk that its container detach already ejected", async () => { const { calls, host } = apfsHost([]); const reader = await NativeDmgArtifactReader.create( From 9a70fb482c974658088167c9d279bd70159e5dc8 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:06:02 +0800 Subject: [PATCH 011/791] test(process): distinguish native startup from producer verification --- docs/testing.md | 6 ++++++ tests/boundary/process/ownedCommand.test.ts | 7 ++++++- tests/boundary/process/processCaptureTerminal.test.ts | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index afca620548b..96fb27a7be2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -174,6 +174,12 @@ case. Real capture tests preserve actual descendant and cleanup checks. The existing Apple job also exercises the relocated filesystem snapshot identity, cancellation and descriptor cleanup regressions on macOS. +Prepare the native inspector before cases that measure producer output or exit +behavior; keep startup deadlines and cancellation in distinct cases. Run real +process-capture verification separately from package or build checks. On macOS, +new npm processes can become token-unreadable after changing their display +title and prevent verified cleanup during a capture's ownership sweep. + MCP SDK transport tests with recording providers remain integration tests. They are useful for schema drift and failure projection but do not prove that Hopper, Ghidra or another substituted engine works. `verify:package` proves diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index c40e5fd57c8..e647263315c 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -1,12 +1,13 @@ import { randomUUID } from "node:crypto"; import { setTimeout as delay } from "node:timers/promises"; -import { expect, it } from "vitest"; +import { beforeAll, expect, it } from "vitest"; import { OwnedCommandFailure, runOwnedCommand, } from "../../../src/process/OwnedCommand.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js"; +import { prepareProcessOwnershipInspection } from "../../../src/process/ProcessOwnershipObservation.js"; import { waitForProviderProcessReady } from "../../fixtures/providerProcess.js"; const command = (script: string) => ({ @@ -15,6 +16,10 @@ const command = (script: string) => ({ runId: `rea-owned-command-test-${randomUUID()}`, }); +// Output/exit cases start with a prepared inspector; startup deadline cases +// below inject their own preparation so they remain independent of compiler speed. +beforeAll(() => prepareProcessOwnershipInspection()); + it("includes ownership preparation in the command deadline before provider creation", async () => { let prepared = false; await expect( diff --git a/tests/boundary/process/processCaptureTerminal.test.ts b/tests/boundary/process/processCaptureTerminal.test.ts index 6fe449b6e7a..355f4168391 100644 --- a/tests/boundary/process/processCaptureTerminal.test.ts +++ b/tests/boundary/process/processCaptureTerminal.test.ts @@ -43,7 +43,7 @@ itWithCaptureCapability( path: "root_0:state.txt", }), ); - expect(result.value.cleanup).toEqual({ + expect(result.value.cleanup).toMatchObject({ owned_process_group: "verified", temporary_root: "removed", }); From 09d7b9bab53ae968502fd6e19df694cca8cab817 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:06:02 +0800 Subject: [PATCH 012/791] test(process): cover argv rewrites in native ownership inspection --- docs/process-capture.md | 8 ++++ .../darwinProcessRunTokenReader.test.ts | 47 ++++++++++++++----- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index 20195bdc4ad..2b443813909 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -108,6 +108,14 @@ and process residual unknowns. Successful cleanup verifies the owned group; it does not attribute those unrelated processes. Related or otherwise unexplained unreadable processes still prevent successful cleanup. +On macOS, changing a Node process's `process.title` can make its run token +unreadable. Node documents that [setting the title overwrites argv memory](https://nodejs.org/download/release/v24.18.0/docs/api/process.html#processtitle). +With the pinned toolchain, a live child retained a readable start identity +while its token became unavailable after this change; npm changes its title +as well. A newly started unreadable process can prevent verified cleanup even +after the selected command exits. REA preserves this uncertainty and leaves +that process untouched. + ## Compare two captures Compare saved capture Evidence with: diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 0c41c0ffd05..81fbf999723 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -614,18 +614,36 @@ it.skipIf(!onDarwin)( ); it.skipIf(!onDarwin)( - "reads only the exact ownership key from live process environments", + "reads exact ownership keys and preserves unknowns after process title changes", async () => { const host = createSystemProcessOwnershipHost("darwin"); - const decoy = await startNodeChild({ - DECOY: "words REA_PROCESS_RUN_ID=synthetic-decoy", - }); - const owned = await startNodeChild({ - REA_PROCESS_RUN_ID: "synthetic-owned", - }); + const children: Awaited>[] = []; try { + const decoy = await startNodeChild({ + DECOY: "words REA_PROCESS_RUN_ID=synthetic-decoy", + }); + children.push(decoy); + const owned = await startNodeChild({ + REA_PROCESS_RUN_ID: "synthetic-owned", + }); + children.push(owned); + const titleMutated = await startNodeChild( + { REA_PROCESS_RUN_ID: "synthetic-title-mutated" }, + process.execPath, + ["-e", 'process.title = "npm run check"; setInterval(() => {}, 1_000)'], + ); + children.push(titleMutated); + await expect + .poll(async () => { + const process = (await host.listProcesses()).find( + ({ pid }) => pid === titleMutated.pid, + ); + return process?.command ?? ""; + }) + .toMatch(/^npm run check/u); + const processes = await host.listProcesses(); - const entries = [decoy, owned].map(({ pid }) => { + const entries = [decoy, owned, titleMutated].map(({ pid }) => { const entry = processes.find((process) => process.pid === pid); if (entry === undefined) throw new Error("test child is absent from process table"); @@ -641,12 +659,19 @@ it.skipIf(!onDarwin)( state: "readable", runId: "synthetic-owned", }); + expect(observations?.get(titleMutated.pid)).toMatchObject({ + state: "unavailable", + reason: expect.stringMatching(/\S/u), + }); expect(identities?.get(decoy.pid)?.state).toBe("readable"); expect(identities?.get(owned.pid)?.state).toBe("readable"); + expect(identities?.get(titleMutated.pid)?.state).toBe("readable"); } finally { - await stopNodeChild(decoy.child); - await stopNodeChild(owned.child); - await host.close?.(); + try { + await Promise.all(children.map(({ child }) => stopNodeChild(child))); + } finally { + await host.close?.(); + } } }, ); From 8d39eae6b950057a1713a4ee8247ff17ef2e1e9e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:29:57 +0800 Subject: [PATCH 013/791] chore(website): configure Cloudflare hosting for rea.tools --- .gitignore | 7 +++ README.md | 20 +++--- website/README.md | 78 ++++++++++++++++++++++++ website/public/guides/browser/index.html | 2 +- website/wrangler.toml | 22 +++++++ 5 files changed, 118 insertions(+), 11 deletions(-) create mode 100644 website/wrangler.toml diff --git a/.gitignore b/.gitignore index a362e63e81a..670f8a1be99 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,13 @@ Thumbs.db # Generated website download /website/public/examples/notes-example.zip +# Local Cloudflare development state and credentials +/website/.wrangler/ +/website/.dev.vars* +/website/.env* +!/website/.dev.vars.example +!/website/.env*.example + # Build-generated MCP tool catalog (regenerated by `npm run build`; never commit) src/generatedMcpToolCatalog.ts diff --git a/README.md b/README.md index 3ac6a90da2b..4ffd6f4607d 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ morluto%2Frea | Trendshift -**[Website](https://morluto.github.io/rea/) · [Guides](https://morluto.github.io/rea/guides/) · [Showcases](https://morluto.github.io/rea/showcase/)** +**[Website](https://rea.tools/) · [Guides](https://rea.tools/guides/) · [Showcases](https://rea.tools/showcase/)** [Quick start](#quick-start) · [How REA works](#how-rea-works) · [What you can analyze](#what-you-can-analyze) · [Showcases](#showcases) · [FAQ](#faq) · [Documentation](#documentation) @@ -54,7 +54,7 @@ REA connects your agent to tools for inspecting native binaries, JavaScript and Setup registers REA with your agent and installs matching workflow instructions. Native analysis can use an existing Hopper or Ghidra installation; setup can optionally install Hopper with approval. Static JavaScript analysis needs neither engine. -> **[Visit the REA website](https://morluto.github.io/rea/)** for setup instructions, illustrated guides, and real case studies. +> **[Visit the REA website](https://rea.tools/)** for setup instructions, illustrated guides, and real case studies. ## Quick start @@ -149,12 +149,12 @@ Additional tools and host support depend on the target: | Target | What REA returns | Requirements and guide | | ---------------------- | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------- | -| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](https://morluto.github.io/rea/guides/native/) | +| Native binaries | Pseudocode, assembly, strings, symbols, calls and references | Hopper, Ghidra or IDA; [native analysis](https://rea.tools/guides/native/) | | Offline ELF layout | Sections, segments, original symbols/relocations and static mitigation candidates | Caller-supplied pwntools on Linux x64; [binary diagnostics](docs/binary-diagnostics.md) | | EVM bytecode | Dispatch selectors, byte offsets, inferred arguments and mutability | Local raw/hex carrier; [offline EVM guide](docs/evm-bytecode.md) | | Recorded Linux crashes | Raw notes, every recorded thread's registers/signals and optional mapping candidates | Caller-supplied pwntools; optional GDB/pwndbg; [recorded crashes](docs/recorded-crashes.md) | -| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; [application analysis](https://morluto.github.io/rea/guides/javascript/) | -| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](https://morluto.github.io/rea/guides/browser/) | +| JavaScript / Electron | Modules, imports, source maps, routes, IPC and native add-on relationships | Node.js and npm; [application analysis](https://rea.tools/guides/javascript/) | +| Websites | Page structure, scripts, network observations and requested screenshots | A Chrome-family browser; [browser analysis](https://rea.tools/guides/browser/) | | Saved network captures | Requests, responses, exposed payloads and source locations | HAR; mitmdump on Linux for native mitmproxy captures; [capture guide](docs/web-network-captures.md) | | .NET assemblies | Metadata, CIL instructions, declared native dependencies and build comparisons | Static inspection; [managed-code guide](docs/managed-code-analysis.md) | | Android APKs | Manifest declarations, classes, decompiled methods and references | Headless JADX and a full JDK on Linux/macOS; [Android guide](docs/android-analysis.md) | @@ -185,7 +185,7 @@ Follow a sound call into its position-to-pan helper, inspect the instructions, and turn incomplete pseudocode into C. The reconstruction passes 3,205 original-x86 cases and reproduces all 63 compiled function bytes. -[Read the case study](https://morluto.github.io/rea/showcase/dx-ball/) · +[Read the case study](https://rea.tools/showcase/dx-ball/) · [Reconstruction repository](https://github.com/N0zoM1z0/dx-ball) ### Notion: trace the Electron clipboard bridge @@ -193,7 +193,7 @@ original-x86 cases and reproduces all 63 compiled function bytes. Find the renderer's clipboard API, follow it through preload and IPC into the main process, and inspect the rich clipboard format. -[Read the case study](https://morluto.github.io/rea/showcase/notion/) +[Read the case study](https://rea.tools/showcase/notion/) ### TH04: recover a DOS bullet-ring calculation @@ -201,7 +201,7 @@ Inspect the original PC-98 game's 16-bit instructions, recover the fixed and aimed angle calculations, and compare the reconstructed C++ with the historical compiler output. -[Read the case study](https://morluto.github.io/rea/showcase/th04/) · +[Read the case study](https://rea.tools/showcase/th04/) · [Reconstruction repository](https://github.com/N0zoM1z0/th04) If you've used REA on something interesting, we'd love to see it. Share your @@ -289,7 +289,7 @@ with your REA version, target type, steps to reproduce and error output. ## Documentation -Start with the website's [worked guides](https://morluto.github.io/rea/guides/). +Start with the website's [worked guides](https://rea.tools/guides/). For exact options, prerequisites and result contracts: - [Installation and setup](docs/installation.md): agent registration, provider configuration, updates and uninstall. @@ -313,7 +313,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and checks, ## Project links -[Website](https://morluto.github.io/rea/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Security](SECURITY.md) +[Website](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Security](SECURITY.md) ## Star history diff --git a/website/README.md b/website/README.md index 23fbca05265..09c1f8b6732 100644 --- a/website/README.md +++ b/website/README.md @@ -130,6 +130,84 @@ The mask SVG is drawn from the seventh decoded mask. It selects zero-based position 21 and compares its character code with 55 (`7`). Keep the figure, solver constants and evidence notes aligned when changing this case. +## Cloudflare Workers + +`wrangler.toml` serves `website/public/` through Workers Static Assets. There is +no Worker script or frontend build. Before a preview or deployment, Wrangler +runs the existing Python scripts to prepare the example ZIP and verify the +website. Node.js and Python 3 must be available; the commands below use a pinned +Wrangler version without adding it to REA's package dependencies. + +Cloudflare documents static-asset requests as free and unlimited, with no +additional storage cost. See [billing and limitations](https://developers.cloudflare.com/workers/static-assets/billing-and-limitations/). + +### Preview and deploy + +Run these commands from the repository root. `--cwd website` makes the build +paths consistent with the website configuration. Preview locally first: + +```sh +npx wrangler@4.149.0 dev --cwd website --env "" +``` + +If `CLOUDFLARE_API_TOKEN` is already set in your shell, Wrangler uses it for +authentication; skip `wrangler login`. If you are not using an API token, sign +in once with `npx wrangler@4.149.0 login`. See Cloudflare's +[authentication commands](https://developers.cloudflare.com/workers/wrangler/commands/general/). + +To publish a preview: + +```sh +npx wrangler@4.149.0 deploy --cwd website --env "" +``` + +This creates `rea-website-preview` at the `workers.dev` URL printed by Wrangler. +It does not attach `rea.tools`. Check the homepage, a nested guide, case studies, +the Notes browser example and the Electron ZIP at that URL. + +### Connect rea.tools + +Serving the custom hostname requires an +[active Cloudflare zone](https://developers.cloudflare.com/workers/configuration/routing/custom-domains/). +When DNS is managed elsewhere: + +1. Add `rea.tools` to your Cloudflare account and choose the Free plan. +2. Export or save the current DNS records. Review Cloudflare's imported records + against them, including subdomains, MX and TXT records for email and domain + verification; the automatic scan can miss records. +3. If DNSSEC is enabled, follow Cloudflare's instructions to disable it at the + registrar before changing nameservers. +4. At the domain registrar, replace the current nameservers with the two assigned + by Cloudflare. The domain can remain registered with the current registrar. +5. Wait for Cloudflare to report the zone as Active. Re-enable DNSSEC through + Cloudflare afterward if required. + +See Cloudflare's [nameserver setup procedure](https://developers.cloudflare.com/dns/zone-setups/full-setup/setup/) +for the complete migration steps. A CNAME to the preview's `workers.dev` hostname +at the current DNS provider does not configure a Workers Custom Domain. + +Publish to the production Worker: + +```sh +npx wrangler@4.149.0 deploy --cwd website --env production +``` + +This creates or updates `rea-website`, attaches `rea.tools`, and lets Cloudflare +manage its DNS record and HTTPS certificate. Deployment can succeed while the +zone is pending; verify public HTTPS after activation and certificate issuance. +An existing CNAME at `rea.tools` must be resolved before adding the Custom Domain. The production Worker also +retains its `workers.dev` address for direct checks. Only `rea.tools` is attached; +`www.rea.tools` is not configured by this file. + +Verify , , + and the Electron ZIP. Unknown paths return +404, and directory pages retain trailing slashes so relative assets and links +resolve correctly. The root README and browser guide use `https://rea.tools/` +as the public website URL. + +Deployments are manual. The existing GitHub Pages publisher remains available +as a separate host; it does not publish to Cloudflare. + ## GitHub Pages `.github/workflows/website-pages.yml` prepares and deploys only `website/public`. diff --git a/website/public/guides/browser/index.html b/website/public/guides/browser/index.html index 095c82b3fd6..b330215c0b3 100644 --- a/website/public/guides/browser/index.html +++ b/website/public/guides/browser/index.html @@ -120,7 +120,7 @@

Open Chrome with remote debugging

google-chrome \
   --remote-debugging-port=9222 \
   --user-data-dir=/tmp/rea-notes-browser \
-  https://morluto.github.io/rea/examples/notes-web/
+ https://rea.tools/examples/notes-web/

Use your installed browser's executable path on macOS or Windows. diff --git a/website/wrangler.toml b/website/wrangler.toml new file mode 100644 index 00000000000..06be8ad532c --- /dev/null +++ b/website/wrangler.toml @@ -0,0 +1,22 @@ +# Serve only the public website. The default deployment is a workers.dev preview. +name = "rea-website-preview" +compatibility_date = "2026-10-08" +workers_dev = true +preview_urls = false + +[assets] +directory = "./public" +html_handling = "auto-trailing-slash" +not_found_handling = "none" + +[build] +cwd = ".." +command = "python3 scripts/prepare-website.py && python3 scripts/verify-website.py" +watch_dir = "./public/examples/notes-electron" + +# Public HTTPS depends on zone activation and certificate readiness. +[env.production] +name = "rea-website" +workers_dev = true +preview_urls = false +routes = [{ pattern = "rea.tools", custom_domain = true }] From e44cb45908af8d3902e0db044e0cf7b86cdd7ed0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:34:42 +0800 Subject: [PATCH 014/791] fix(session): serialize snapshot lifecycle with admitted requests --- docs/native-investigation.md | 9 + docs/testing.md | 9 + scripts/lib/real-ghidra-boundaries.mjs | 13 +- .../lib/real-ghidra-snapshot-lifecycle.mjs | 180 ++++++++++++++ src/application/binary/BinarySession.ts | 222 +++++++++++------- src/application/binary/BinarySessionPort.ts | 13 + .../binary/BinarySessionRecords.ts | 19 ++ src/contracts/sessionToolContracts.ts | 4 +- src/server/registerCloseLifecycleTool.ts | 34 +-- tests/boundary/mcp/sessionMcp.test.ts | 111 ++++++++- .../analysis-sessions/binarySession.test.ts | 121 +++++++--- 11 files changed, 587 insertions(+), 148 deletions(-) create mode 100644 scripts/lib/real-ghidra-snapshot-lifecycle.mjs diff --git a/docs/native-investigation.md b/docs/native-investigation.md index ca840e8f898..eea5a8d7da5 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -101,6 +101,15 @@ Cancelling an active request terminates the ephemeral database and discards its annotations. REA retains the selected target and imports the original artifact again on the next Ghidra query. +Opening the same target and profile again retains the live database, including +annotations. It does not make those edits eligible for an immutable snapshot. +Snapshot closes drain earlier provider requests before saving and keep later +requests waiting until the save and close finish. A concurrent successful edit +therefore rejects the snapshot save and leaves the edited session open. +After metadata edits, snapshot saves and imports remain unavailable in that +session. Use `export_evidence_bundle` to retain the observations, close without +`snapshot_path`, and reopen the target before importing or saving a snapshot. + ```bash rea inspect-native-instruction

--provider ghidra rea resolve-native-call-targets --provider ghidra diff --git a/docs/testing.md b/docs/testing.md index 96fb27a7be2..e0ffb7370fa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -299,6 +299,15 @@ external addresses remain unresolved and external annotations are rejected. An adversarial regex over a full 12 KB literal must report stack exhaustion as a resource constraint, preserve live annotations, and allow complete literal searches afterward; CLI and MCP must agree on both results and recovery advice. +Real snapshot lifecycle checks retain edited API results as Evidence while +rejecting immutable snapshot saves and imports before and after a repeated +open of the same target. They verify unchanged live annotations and run identity, +absent rejected output files, an unchanged source snapshot, and successful +snapshot import/save after closing and recreating the database. The pristine +snapshot is written by an independent real CLI session. +They also start a real annotation and snapshot close concurrently: the edit +must succeed, the snapshot must be rejected without creating a file, and the +edited session must remain usable until explicitly closed. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index b31fa61cd74..8ba6974a313 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -9,6 +9,7 @@ import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot-e2e.mjs"; +import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; /** Probe real Ghidra location, annotation and error contracts through public adapters. */ export async function verifyGhidraBoundaries( @@ -639,10 +640,16 @@ export async function verifyGhidraBoundaries( selected, ); } - await call("close_binary"); + const snapshotLifecycle = await verifyGhidraSnapshotLifecycle( + client, + target, + await call("analyze_function", { procedure: address }), + cli, + ); + successfulCalls += snapshotLifecycle.successfulCalls; + rejectedCalls += snapshotLifecycle.rejectedCalls; await assert.rejects(access(socketRoot), { code: "ENOENT" }); await assert.rejects(access(runtimeRoot), { code: "ENOENT" }); - await call("open_binary", { path: target.path, provider_id: "ghidra" }); assert.equal(await call("procedure_address", { procedure: name }), address); assert.equal(await call("address_name", { address }), name); assert.equal( @@ -674,6 +681,8 @@ export async function verifyGhidraBoundaries( complete_function_body_references: true, ambiguity_candidates_inline: true, legacy_reference_snapshot_rejected: true, + mutation_snapshot_lifecycle: true, + concurrent_annotation_snapshot_close: true, source_immutable: true, reopen_discards_edits: true, long_tmpdir_private_socket_cleanup: true, diff --git a/scripts/lib/real-ghidra-snapshot-lifecycle.mjs b/scripts/lib/real-ghidra-snapshot-lifecycle.mjs new file mode 100644 index 00000000000..7333b72f50e --- /dev/null +++ b/scripts/lib/real-ghidra-snapshot-lifecycle.mjs @@ -0,0 +1,180 @@ +import assert from "node:assert/strict"; +import { access, mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parseAnalysisSnapshot } from "../../dist/domain/analysisSnapshot.js"; +import { parseEvidenceBundle } from "../../dist/domain/evidenceBundle.js"; +import { requireMcpResult } from "./mcp-verifier-results.mjs"; + +/** Keep mutable Ghidra observations out of immutable snapshots across target opens. */ +export async function verifyGhidraSnapshotLifecycle( + client, + target, + edited, + cli, +) { + const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-snapshot-proof-")); + let successfulCalls = 0; + let rejectedCalls = 0; + const call = async (name, args = {}) => { + const result = requireMcpResult( + await client.callTool({ name, arguments: args }, { timeout: 180000 }), + name, + ); + successfulCalls++; + return result; + }; + const rejectSnapshot = async (name, args) => { + const reply = await client.callTool({ name, arguments: args }); + assert.equal( + reply.isError, + true, + `${name} accepted a mutated snapshot binding`, + ); + const error = reply.structuredContent?.error; + assert.equal( + error?.code, + "evidence_integrity_mismatch", + JSON.stringify(reply), + ); + assert.match(error.message, /after analysis metadata mutations/u); + assert.match( + error.remediation.action, + /close.*without saving a snapshot/iu, + ); + assert.match(error.remediation.action, /export_evidence_bundle/u); + rejectedCalls++; + }; + try { + const entry = edited.procedure.address; + const snapshotPath = join(workspace, "original.json"); + // The CLI has its own unmodified database and writes the production snapshot format. + const original = await cli("function", entry, ["--snapshot", snapshotPath]); + const snapshotBytes = await readFile(snapshotPath); + const snapshot = parseAnalysisSnapshot( + JSON.parse(snapshotBytes.toString("utf8")), + ); + assert.equal(snapshot.target.sha256, target.sha256); + assert.notEqual(original.procedure.name, edited.procedure.name); + const runId = (await call("binary_session")).analysis_run.run_id; + const api = await call("inspect_native_api", { procedure: entry }); + assert.equal(api.procedure.name, edited.procedure.name); + for (const repeatedOpen of [false, true]) { + if (repeatedOpen) + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + const destination = join( + workspace, + repeatedOpen ? "repeated.json" : "edited.json", + ); + await rejectSnapshot("close_binary", { snapshot_path: destination }); + await assert.rejects(access(destination), { code: "ENOENT" }); + assert.equal((await call("binary_session")).analysis_run.run_id, runId); + assert.deepEqual( + await call("analyze_function", { procedure: entry }), + edited, + ); + } + await rejectSnapshot("open_binary", { + path: target.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }); + assert.deepEqual(await readFile(snapshotPath), snapshotBytes); + assert.equal((await call("binary_session")).analysis_run.run_id, runId); + assert.deepEqual( + await call("analyze_function", { procedure: entry }), + edited, + ); + + // Mutable results remain exportable as observations, with their mutation Evidence. + const bundlePath = join(workspace, "observations.json"); + await call("export_evidence_bundle", { path: bundlePath }); + const bundle = parseEvidenceBundle( + JSON.parse(await readFile(bundlePath, "utf8")), + ); + assert.ok( + bundle.records.some( + (record) => record.operation === "annotate_native_function", + ), + ); + assert.ok( + bundle.records.some( + (record) => + record.operation === "inspect_native_api" && + record.normalized_result.procedure.name === edited.procedure.name, + ), + ); + + await call("close_binary"); + await call("open_binary", { + path: target.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }); + const restored = await call("analyze_function", { procedure: entry }); + assert.deepEqual(restored.procedure, original.procedure); + assert.deepEqual(restored.comments, original.comments); + assert.notEqual((await call("binary_session")).analysis_run.run_id, runId); + const freshPath = join(workspace, "fresh.json"); + await call("close_binary", { snapshot_path: freshPath }); + const fresh = parseAnalysisSnapshot( + JSON.parse(await readFile(freshPath, "utf8")), + ); + assert.ok( + !fresh.evidence_bundle.records.some( + (record) => + record.operation === "annotate_native_function" || + (record.operation === "inspect_native_api" && + record.normalized_result.procedure.name === edited.procedure.name), + ), + ); + await call("open_binary", { + path: target.path, + provider_id: "ghidra", + snapshot_path: freshPath, + }); + await verifyConcurrentSnapshotClose({ + call, + rejectSnapshot, + workspace, + target, + entry, + original, + }); + return { successfulCalls, rejectedCalls }; + } finally { + await rm(workspace, { recursive: true, force: true }); + } +} + +async function verifyConcurrentSnapshotClose({ + call, + rejectSnapshot, + workspace, + target, + entry, + original, +}) { + const destination = join(workspace, "concurrent.json"); + const annotation = call("annotate_native_function", { + procedure: entry, + name: "rea_concurrent_snapshot_edit", + comment: + "A successful edit must invalidate a concurrently requested snapshot.", + }); + const close = rejectSnapshot("close_binary", { snapshot_path: destination }); + const [updated] = await Promise.all([annotation, close]); + await assert.rejects(access(destination), { code: "ENOENT" }); + assert.equal((await call("binary_session")).open, true); + assert.deepEqual( + await call("analyze_function", { procedure: entry }), + updated.dossier, + ); + // Discard the ephemeral edit and leave the caller with a pristine database. + await call("close_binary"); + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + assert.deepEqual( + (await call("analyze_function", { procedure: entry })).procedure, + original.procedure, + ); +} diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 6526d58cd97..184fea127f9 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -20,7 +20,11 @@ import type { import type { AnalysisOperation } from "../AnalysisProvider.js"; import { OFFICIAL_TOOL_CONTRACTS } from "../../contracts/officialToolContracts.js"; import { ENHANCED_TOOL_CONTRACTS } from "../../contracts/enhancedToolContracts.js"; -import type { BinarySessionPort } from "./BinarySessionPort.js"; +import type { + BinarySessionPort, + SavedAnalysisSnapshot, +} from "./BinarySessionPort.js"; +import { writeAnalysisSnapshot } from "./AnalysisSnapshotFiles.js"; import { SessionProviderRouter, type SessionProviderRoute, @@ -71,7 +75,8 @@ export class BinarySession } | undefined; #transition: Promise = Promise.resolve(); - readonly #calls = new Set>(); + #transitionGeneration = 0; + readonly #calls = new Map, number>(); readonly #providerRouter: SessionProviderRouter; readonly #runtimeUnavailability = new Map< string, @@ -220,7 +225,7 @@ export class BinarySession resolve: () => Promise>, options: Pick, ): Promise> { - return this.#serialize(async () => { + return this.#serialize(async (admittedThrough) => { if (isAborted(options.signal)) return err(new AnalysisCancelledError("open_binary")); const resolved = await resolve(); @@ -234,15 +239,16 @@ export class BinarySession activeProfile === null || activeProfile === undefined ? profile === null : profile !== null && analysisProfilesEqual(activeProfile, profile); + await this.#drainCalls(admittedThrough); + if (isAborted(options.signal)) + return err(new AnalysisCancelledError("open_binary")); if (sameTarget && sameProfile) { if (options.snapshot !== undefined) { const imported = this.importAnalysisSnapshot(options.snapshot); if (!imported.ok) return imported; } - this.resetSnapshotInvalidation(); return ok(target); } - await this.#drainCalls(); const previous = this.#active; this.#active = undefined; if (previous !== undefined) { @@ -278,6 +284,7 @@ export class BinarySession runId, }; this.#clearRuntimeAvailability(); + this.resetSnapshotInvalidation(); if (options.snapshot === undefined) { if (profile === null) this.clearSnapshot(); else this.selectSnapshot(target, profile); @@ -291,7 +298,6 @@ export class BinarySession return imported; } } - this.resetSnapshotInvalidation(); return ok(target); }); } @@ -302,31 +308,62 @@ export class BinarySession readonly retainProviderDocuments?: boolean; } = {}, ): Promise> { - return this.#serialize(async () => { - const previous = this.#active; - this.#active = undefined; - await this.#drainCalls(); - const closed = - previous === undefined - ? ok(null) - : await closeAnalysisClient( - previous.client, - previous.route.identity.id, - { - ...(options.progress === undefined - ? {} - : { progress: options.progress }), - ...(options.retainProviderDocuments === true - ? { retainDocument: true } - : {}), - }, - ); - this.clearSessionRecords(); - this.#clearRuntimeAvailability(); - return closed; + return this.#serialize(async (admittedThrough) => { + await this.#drainCalls(admittedThrough); + return this.#closeActive(options); }); } + /** Drain earlier requests, save an immutable snapshot, and close under one lifecycle lock. */ + closeWithSnapshot( + path: string, + overwrite: boolean, + options: Pick = {}, + ): Promise> { + return this.#serialize(async (admittedThrough) => { + await this.#drainCalls(admittedThrough); + const snapshot = this.exportAnalysisSnapshot(); + if (!snapshot.ok) return snapshot; + const written = await writeAnalysisSnapshot( + snapshot.value, + path, + overwrite, + ); + if (!written.ok) return written; + const closed = await this.#closeActive(options); + return closed.ok + ? ok({ ...written.value, entries: snapshot.value.entries.length }) + : closed; + }); + } + + async #closeActive( + options: Pick & { + readonly retainProviderDocuments?: boolean; + }, + ): Promise> { + const previous = this.#active; + this.#active = undefined; + const closed = + previous === undefined + ? ok(null) + : await closeAnalysisClient( + previous.client, + previous.route.identity.id, + { + ...(options.progress === undefined + ? {} + : { progress: options.progress }), + ...(options.retainProviderDocuments === true + ? { retainDocument: true } + : {}), + }, + ); + this.clearSessionRecords(); + this.#clearRuntimeAvailability(); + return closed; + } + /** Describe the active binary session. */ status(): JsonValue { return binarySessionStatus({ @@ -360,12 +397,28 @@ export class BinarySession * Calls may overlap, but a pending target transition prevents new calls from * entering until the transition has settled. */ - async execute( + execute( name: Parameters[0], arguments_: Readonly>, options?: { readonly signal?: AbortSignal }, ): Promise> { - const transitioned = await this.#waitForTransition(name, options?.signal); + const generation = this.#transitionGeneration; + const call = this.#execute(name, arguments_, options, this.#transition); + this.#calls.set(call, generation); + return call.finally(() => this.#calls.delete(call)); + } + + async #execute( + name: Parameters[0], + arguments_: Readonly>, + options: { readonly signal?: AbortSignal } | undefined, + transition: Promise, + ): Promise> { + const transitioned = await this.#waitForTransition( + name, + options?.signal, + transition, + ); if (!transitioned.ok) return transitioned; const prepared = prepareSessionExecution({ active: this.#active, @@ -385,53 +438,48 @@ export class BinarySession const { active, capability, profile, cacheable, cached } = prepared.value; if (cached !== undefined) return ok(cached); const call = active.client.execute(name, arguments_, options); - this.#calls.add(call); - try { - const result = await call; - const profiled = bindExecutionTarget( - commitExecutionProfile(name, result, profile), - name, - active.target, - ); - this.#observeRuntimeAvailability(name, profiled); - if ( - profiled.ok && - cacheable && - profile !== undefined && - name !== "decode_interface_builder" && - name !== "inspect_asset_catalog" && - name !== "inspect_keyed_archive" && - name !== "trace_dylib_resolution" - ) { - const evidence = createEvidence( - profiled.value.subject ?? active.target, - profiled.value.provider, - { - operation: name, - parameters: arguments_, - result: profiled.value.result, - analysisProfile: profile, - rawResult: profiled.value.rawResult, - limitations: profiled.value.limitations, - locations: profiled.value.locations, - }, - ); - const recorded = this.recordEvidence(evidence); - if (!recorded.ok) return recorded; - this.recordSnapshot({ - target: active.target, - profile, + const result = await call; + const profiled = bindExecutionTarget( + commitExecutionProfile(name, result, profile), + name, + active.target, + ); + this.#observeRuntimeAvailability(name, profiled); + if ( + profiled.ok && + cacheable && + profile !== undefined && + name !== "decode_interface_builder" && + name !== "inspect_asset_catalog" && + name !== "inspect_keyed_archive" && + name !== "trace_dylib_resolution" + ) { + const evidence = createEvidence( + profiled.value.subject ?? active.target, + profiled.value.provider, + { operation: name, parameters: arguments_, - execution: profiled.value, - }); - } else if (profiled.ok && capability?.effects.mutatesArtifact === true) { - this.invalidateSnapshot(); - } - return profiled; - } finally { - this.#calls.delete(call); + result: profiled.value.result, + analysisProfile: profile, + rawResult: profiled.value.rawResult, + limitations: profiled.value.limitations, + locations: profiled.value.locations, + }, + ); + const recorded = this.recordEvidence(evidence); + if (!recorded.ok) return recorded; + this.recordSnapshot({ + target: active.target, + profile, + operation: name, + parameters: arguments_, + execution: profiled.value, + }); + } else if (profiled.ok && capability?.effects.mutatesArtifact === true) { + this.invalidateSnapshot(); } + return profiled; } #observeRuntimeAvailability( @@ -507,8 +555,15 @@ export class BinarySession return this.#active?.route ?? this.#providerRouter.initialRoute(); } - #serialize(operation: () => Promise): Promise { - const result = this.#transition.then(operation, operation); + #serialize( + operation: (admittedThrough: number) => Promise, + ): Promise { + // Later calls wait for this transition and must not be included in its drain. + const admittedThrough = this.#transitionGeneration++; + const result = this.#transition.then( + () => operation(admittedThrough), + () => operation(admittedThrough), + ); this.#transition = result.then( () => undefined, () => undefined, @@ -516,8 +571,12 @@ export class BinarySession return result; } - async #drainCalls(): Promise { - await Promise.allSettled(this.#calls); + async #drainCalls(admittedThrough: number): Promise { + await Promise.allSettled( + [...this.#calls] + .filter(([, generation]) => generation <= admittedThrough) + .map(([call]) => call), + ); } async #restore( @@ -550,12 +609,13 @@ export class BinarySession async #waitForTransition( operation: string, - signal?: AbortSignal, + signal: AbortSignal | undefined, + transition: Promise, ): Promise> { if (signal?.aborted === true) return err(new AnalysisCancelledError(operation)); if (signal === undefined) { - await this.#transition; + await transition; return ok(undefined); } return new Promise((resolve) => { @@ -563,7 +623,7 @@ export class BinarySession resolve(err(new AnalysisCancelledError(operation))); }; signal.addEventListener("abort", onAbort, { once: true }); - this.#transition.then( + transition.then( () => { signal.removeEventListener("abort", onAbort); resolve( diff --git a/src/application/binary/BinarySessionPort.ts b/src/application/binary/BinarySessionPort.ts index fb7317f0006..cfee0250148 100644 --- a/src/application/binary/BinarySessionPort.ts +++ b/src/application/binary/BinarySessionPort.ts @@ -17,6 +17,13 @@ import type { ProviderIdentity, } from "../AnalysisProvider.js"; +/** Receipt for an atomically saved snapshot followed by provider cleanup. */ +export interface SavedAnalysisSnapshot { + readonly path: string; + readonly bytes: number; + readonly entries: number; +} + /** Target lifecycle used by CLI and MCP without exposing a concrete provider. */ export interface BinarySessionPort extends AnalysisOperationPort, InvestigationRecordPort { @@ -35,6 +42,12 @@ export interface BinarySessionPort readonly retainProviderDocuments?: boolean; }, ): Promise>; + /** Save a snapshot and close while excluding later provider requests. */ + closeWithSnapshot( + path: string, + overwrite: boolean, + options?: Pick, + ): Promise>; status(): JsonValue; activeTarget(): BinaryTarget | undefined; exportAnalysisSnapshot(): Result; diff --git a/src/application/binary/BinarySessionRecords.ts b/src/application/binary/BinarySessionRecords.ts index bafce6b0f95..4c2307ef210 100644 --- a/src/application/binary/BinarySessionRecords.ts +++ b/src/application/binary/BinarySessionRecords.ts @@ -25,6 +25,9 @@ import type { import { AnalysisSnapshotCache } from "./AnalysisSnapshotCache.js"; import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; +const SNAPSHOT_MUTATION_RECOVERY = + "Export session observations through export_evidence_bundle if needed. Close the active target without saving a snapshot, then reopen it before importing or saving a snapshot."; + export interface ActiveAnalysisBinding { readonly target: BinaryTarget; readonly profile: AnalysisProfileCommitment | null; @@ -104,6 +107,9 @@ export abstract class BinarySessionRecords { return err( new EvidenceIntegrityError( "Analysis snapshots are unavailable after analysis metadata mutations", + { + userMessage: `Analysis snapshots are unavailable after analysis metadata mutations. ${SNAPSHOT_MUTATION_RECOVERY}`, + }, ), ); const target = active?.target; @@ -130,6 +136,15 @@ export abstract class BinarySessionRecords { snapshot: AnalysisSnapshot, ): Result { const active = this.activeAnalysisBinding(); + if (active !== undefined && this.#snapshotInvalidated) + return err( + new EvidenceIntegrityError( + "Analysis snapshots cannot be imported after analysis metadata mutations", + { + userMessage: `Analysis snapshots cannot be imported after analysis metadata mutations. ${SNAPSHOT_MUTATION_RECOVERY}`, + }, + ), + ); if (active?.profile === null) return err( new AnalysisSnapshotMismatchError( @@ -173,12 +188,14 @@ export abstract class BinarySessionRecords { Record >, ): AnalysisExecution | undefined { + if (this.#snapshotInvalidated) return undefined; return this.#snapshot.lookup(target, profile, operation, parameters); } protected recordSnapshot( input: Parameters[0], ): void { + if (this.#snapshotInvalidated) return; this.#snapshot.record(input); this.#emitSnapshotChanged(); } @@ -194,6 +211,7 @@ export abstract class BinarySessionRecords { "Workflow snapshot entries require an active concrete provider profile", ), ); + if (this.#snapshotInvalidated) return ok(null); try { this.#snapshot.recordWorkflow({ target: active.target, @@ -222,6 +240,7 @@ export abstract class BinarySessionRecords { protected resetSnapshotInvalidation(): void { if (!this.#snapshotInvalidated) return; + this.#snapshot.clear(); this.#snapshotInvalidated = false; this.#emitSnapshotChanged(); } diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index ebd7983bd1c..5e9de09cfdd 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -65,12 +65,12 @@ export const exportEvidenceBundleInputSchema = z.strictObject({ export const SESSION_TOOL_CONTRACTS = [ session( "open_binary", - "Open a local executable, application bundle, archive, JavaScript, source map, plist, or analysis database after validation. format=dos-com explicitly interprets 1..65280 headerless bytes as a DOS COM analysis image; omission preserves header-based detection. provider_id selects one deep provider or deterministic auto selection; the binding remains stable until close or an explicit switch, with no failure fallback. An optional analysis snapshot is imported atomically and must match the binary identity, concrete provider, and canonical analysis profile exactly.", + "Open a local executable, application bundle, archive, JavaScript, source map, plist, or analysis database after validation. format=dos-com explicitly interprets 1..65280 headerless bytes as a DOS COM analysis image; omission preserves header-based detection. provider_id selects one deep provider or deterministic auto selection; the binding remains stable until close or an explicit switch, with no failure fallback. Reopening the same target and profile retains its active database and metadata edits. An optional analysis snapshot is imported atomically and must match the binary identity, concrete provider, and canonical analysis profile exactly; importing into a session with metadata mutations requires closing and reopening it first.", openBinaryInputSchema, ), session( "close_binary", - "Optionally write a provider-neutral analysis snapshot atomically to the caller-supplied path, then close the active target and every provider resource started for it. Existing files require explicit overwrite; a failed save leaves the session open so cached analysis is not lost.", + "Drain earlier provider requests, optionally write a provider-neutral analysis snapshot atomically to the caller-supplied path, then close the active target and every provider resource started for it. Later provider requests wait until this lifecycle operation finishes. Existing files require explicit overwrite; a failed save leaves the session open so cached analysis is not lost. Metadata edits prevent immutable snapshot saves until the session is recreated; export_evidence_bundle preserves mutable observations before closing without a snapshot.", closeBinaryInputSchema, ), session( diff --git a/src/server/registerCloseLifecycleTool.ts b/src/server/registerCloseLifecycleTool.ts index 19f66e786cb..48d6d933cd2 100644 --- a/src/server/registerCloseLifecycleTool.ts +++ b/src/server/registerCloseLifecycleTool.ts @@ -1,9 +1,8 @@ -import { writeAnalysisSnapshot } from "../application/binary/AnalysisSnapshotFiles.js"; -import { ok } from "../domain/result.js"; import { reportLifecycleEnd, reportLifecycleStart, } from "./lifecycleProgress.js"; +import { ok } from "../domain/result.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import type { LifecycleToolRegistration } from "./registerSessionTools.js"; import { logToolExecution } from "./toolLogging.js"; @@ -22,7 +21,8 @@ export const registerCloseLifecycleTool = ({ toolRegistrationOptions(closeContract), async (input, context) => { const progress = mcpProgressReporter(context); - if (input.snapshot_path === undefined) { + const snapshotPath = input.snapshot_path; + if (snapshotPath === undefined) { await reportLifecycleStart(progress, closeContract.name); const closed = await logToolExecution(logger, closeContract.name, () => session.close({ progress }), @@ -30,33 +30,21 @@ export const registerCloseLifecycleTool = ({ await reportLifecycleEnd(progress, closeContract.name, closed.ok); return toCallToolResult(closed, closeContract); } - const snapshot = session.exportAnalysisSnapshot(); - if (!snapshot.ok) return toCallToolResult(snapshot, closeContract); - const written = await writeAnalysisSnapshot( - snapshot.value, - input.snapshot_path, - input.overwrite, - ); - if (!written.ok) return toCallToolResult(written, closeContract); await reportLifecycleStart( progress, closeContract.name, - "snapshot written; closing provider", + "saving snapshot; closing provider", ); const closed = await logToolExecution(logger, closeContract.name, () => - session.close({ progress }), + session.closeWithSnapshot(snapshotPath, input.overwrite, { + progress, + }), ); await reportLifecycleEnd(progress, closeContract.name, closed.ok); - return closed.ok - ? toCallToolResult( - ok({ - path: written.value.path, - bytes: written.value.bytes, - entries: snapshot.value.entries.length, - }), - closeContract, - ) - : toCallToolResult(closed, closeContract); + return toCallToolResult( + closed.ok ? ok({ ...closed.value }) : closed, + closeContract, + ); }, ); }; diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index bd065fa69b7..1d40bec8439 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -7,7 +7,10 @@ import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + createDeferred, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; import type { AnalysisClient, AnalysisProvider, @@ -18,6 +21,7 @@ import { createServer } from "../../../src/server/createServer.js"; import { silentLogger } from "../../../src/logger.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { MAX_JSON_DEPTH } from "../../../src/domain/jsonValue.js"; +import { parseAnalysisSnapshot } from "../../../src/domain/analysisSnapshot.js"; import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { @@ -39,6 +43,111 @@ afterEach(async () => { directory = undefined; }); +describe("MCP snapshot lifecycle ordering", () => { + it("saves earlier in-flight observations, blocks later calls, and preserves the session on write failure", async () => { + directory = await createTestTempDirectory("rea-mcp-snapshot-order-"); + const started = createDeferred(); + const release = createDeferred>(); + const { mcp, first, closed } = await createSessionMcpHarness( + directory, + (closed) => { + const base = provider(closed); + return { + ...base, + capabilities: () => + base.capabilities().map((capability) => ({ + ...capability, + operation: "address_name", + })), + createClient: (target) => ({ + ...client(target.path, closed), + execute: (name) => { + if (name === "health") return Promise.resolve(ok(null)); + started.resolve(); + return release.promise; + }, + }), + }; + }, + resources, + ); + expect( + (await mcp.callTool({ name: "open_binary", arguments: { path: first } })) + .isError, + ).not.toBe(true); + const observation = mcp.callTool({ + name: "address_name", + arguments: { address: "0x1000", document: "fixture" }, + }); + await started.promise; + const snapshotPath = join(directory, "analysis.json"); + const closing = mcp.callTool({ + name: "close_binary", + arguments: { snapshot_path: snapshotPath }, + }); + const later = mcp.callTool({ + name: "address_name", + arguments: { address: "0x1001", document: "fixture" }, + }); + expect( + structured(await mcp.callTool({ name: "binary_session", arguments: {} })) + .result, + ).toMatchObject({ open: true }); + expect(closed).toEqual([]); + release.resolve(ok(first)); + expect((await observation).isError).not.toBe(true); + expect((await closing).isError).not.toBe(true); + expect((await later).isError).toBe(true); + const snapshot = parseAnalysisSnapshot( + JSON.parse(await readFile(snapshotPath, "utf8")), + ); + expect(snapshot.entries).toHaveLength(1); + expect(snapshot.entries[0]?.execution.result).toBe(first); + expect( + snapshot.evidence_bundle.records.some( + (record) => + record.operation === "address_name" && + record.normalized_result === first, + ), + ).toBe(true); + + expect( + (await mcp.callTool({ name: "open_binary", arguments: { path: first } })) + .isError, + ).not.toBe(true); + const bytes = await readFile(snapshotPath); + expect( + ( + await mcp.callTool({ + name: "close_binary", + arguments: { snapshot_path: snapshotPath }, + }) + ).isError, + ).toBe(true); + expect(await readFile(snapshotPath)).toEqual(bytes); + expect( + ( + await mcp.callTool({ + name: "address_name", + arguments: { address: "0x1000", document: "fixture" }, + }) + ).isError, + ).not.toBe(true); + expect( + structured(await mcp.callTool({ name: "binary_session", arguments: {} })) + .result, + ).toMatchObject({ open: true }); + expect( + ( + await mcp.callTool({ + name: "close_binary", + arguments: { snapshot_path: snapshotPath, overwrite: true }, + }) + ).isError, + ).not.toBe(true); + }, 10_000); +}); + describe("target-free MCP lifecycle", () => { it("reopens replaced content at one canonical path through MCP", async () => { directory = await createTestTempDirectory("rea-mcp-replaced-target-"); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index ec066fa923c..167db30f6cb 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -414,17 +414,10 @@ describe("evidence metadata imports and snapshot cache", () => { ).ok, ).toBe(true); const before = session.exportAnalysisSnapshot(); - expect(before.ok).toBe(true); - if (!before.ok) { - await session.close(); - return; - } + if (!before.ok) throw before.error; const existing = session.exportEvidenceBundle().records[0]; - expect(existing?.subject).not.toBeNull(); - if (existing?.subject === null || existing === undefined) { - await session.close(); - return; - } + if (existing === undefined || existing.subject === null) + throw new Error("Expected artifact-bound Evidence"); const addition = createEvidence( { path: existing.subject.local_path, @@ -445,6 +438,9 @@ describe("evidence metadata imports and snapshot cache", () => { expect( session.importEvidenceBundle(createEvidenceBundle([addition])), ).toEqual({ ok: true, value: 1 }); + expect( + session.importEvidenceBundle(createEvidenceBundle([addition])), + ).toEqual({ ok: true, value: 0 }); expect(session.exportAnalysisSnapshot()).toMatchObject({ ok: true, value: { @@ -466,7 +462,7 @@ describe("evidence metadata imports and snapshot cache", () => { await session.close(); }); - it("invalidates cached analysis when an import changes evidence path metadata", async () => { + it("keeps changed evidence metadata out of snapshots until the session is recreated", async () => { const [target] = await createBinarySessionTargets(); const calls: string[] = []; const session = createTestBinarySession(createCacheProvider(calls)); @@ -479,30 +475,15 @@ describe("evidence metadata imports and snapshot cache", () => { }) ).ok, ).toBe(true); - expect(session.exportAnalysisSnapshot().ok).toBe(true); + const baseline = session.exportAnalysisSnapshot(); + if (!baseline.ok) throw baseline.error; const evidence = session .exportEvidenceBundle() .records.find(({ operation }) => operation === "address_name"); - expect(evidence?.subject).not.toBeNull(); - if (evidence?.subject === null || evidence === undefined) { - await session.close(); - return; - } - expect( - session.importEvidenceBundle(createEvidenceBundle([evidence])), - ).toEqual({ ok: true, value: 0 }); - expect(session.exportAnalysisSnapshot().ok).toBe(true); + if (evidence === undefined || evidence.subject === null) + throw new Error("Expected artifact-bound Evidence"); const callsAfterInitialRead = [...calls]; - expect( - ( - await session.execute("address_name", { - address: "0x1000", - document: "first", - }) - ).ok, - ).toBe(true); - expect(calls).toEqual(callsAfterInitialRead); const relocated = { ...evidence, @@ -533,6 +514,32 @@ describe("evidence metadata imports and snapshot cache", () => { ), }, }); + expect((await session.open(target)).ok).toBe(true); + expect(session.exportAnalysisSnapshot().ok).toBe(false); + expect(session.importAnalysisSnapshot(baseline.value)).toMatchObject({ + ok: false, + error: { + userMessage: expect.stringContaining( + "Close the active target without saving a snapshot", + ), + }, + }); + expect(session.exportEvidenceBundle().records).toContainEqual(relocated); + const callsAfterInvalidation = calls.length; + expect( + ( + await session.execute("address_name", { + address: "0x1000", + document: "first", + }) + ).ok, + ).toBe(true); + expect(calls).toHaveLength(callsAfterInvalidation + 1); + await session.close(); + expect((await session.open(target, { snapshot: baseline.value })).ok).toBe( + true, + ); + expect(session.exportAnalysisSnapshot().ok).toBe(true); await session.close(); }); }); @@ -591,11 +598,6 @@ describe("replay of exact immutable calls", () => { ).toBe(true); expect(replayCalls).toEqual(["health", "set_address_name", "address_name"]); expect(replay.exportAnalysisSnapshot().ok).toBe(false); - expect((await replay.open(first)).ok).toBe(true); - expect(replay.exportAnalysisSnapshot()).toMatchObject({ - ok: true, - value: { entries: [{ operation: "address_name" }] }, - }); await replay.close(); const mismatch = createTestBinarySession(createCacheProvider([])); @@ -863,9 +865,16 @@ describe("typed unavailability without dispatching", () => { await session.close(); }); - it("requires an open binary and closes idempotently", async () => { + it("drains an admitted call before closing and closes idempotently", async () => { + const [first] = await createBinarySessionTargets(); const session = createTestBinarySession(() => client()); expect((await session.execute("binary_overview", {})).ok).toBe(false); + expect((await session.open(first)).ok).toBe(true); + // The call has been admitted but has not resumed after the transition wait. + const admitted = session.execute("binary_overview", {}); + const closing = session.close(); + expect((await admitted).ok).toBe(true); + expect(await closing).toEqual({ ok: true, value: null }); expect(await session.close()).toEqual({ ok: true, value: null }); expect(await session.close()).toEqual({ ok: true, value: null }); }); @@ -886,7 +895,7 @@ describe("typed unavailability without dispatching", () => { expect(created).toBe(3); }); - it("serializes concurrent opens and leaves the last target active", async () => { + it("orders calls between queued transitions without deadlocking", async () => { const [first, second] = await createBinarySessionTargets(); const clients: ControllableAnalysisClient[] = []; const session = createTestBinarySession(() => { @@ -895,12 +904,31 @@ describe("typed unavailability without dispatching", () => { return value; }); const one = session.open(first); + const firstCall = session.execute("binary_overview", {}); + const controller = new AbortController(); + const cancelled = session.execute( + "binary_overview", + {}, + { signal: controller.signal }, + ); + controller.abort(); const two = session.open(second); + const secondCall = session.execute("binary_overview", {}); + const closing = session.close(); + const afterClose = session.execute("binary_overview", {}); expect((await one).ok).toBe(true); + expect((await firstCall).ok).toBe(true); + expect(await cancelled).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); expect((await two).ok).toBe(true); - expect(session.status()).toMatchObject({ open: true }); - expect(JSON.stringify(session.status())).toContain("second.hop"); + expect((await secondCall).ok).toBe(true); + expect((await closing).ok).toBe(true); + expect((await afterClose).ok).toBe(false); + expect(session.status()).toMatchObject({ open: false }); expect(clients[0]?.closed).toBe(1); + expect(clients[1]?.closed).toBe(1); }); }); @@ -977,12 +1005,27 @@ describe("active client replacement", () => { return value; }); expect((await session.open(first)).ok).toBe(true); + const current = session.activeTarget(); + if (current === undefined) throw new Error("Missing active target"); + const preview = await session.previewTarget(current); + if (!preview.ok) throw preview.error; const call = session.execute("procedure_pseudo_code", {}); + const controller = new AbortController(); + const reopening = session.openResolvedTarget(preview.value, { + signal: controller.signal, + }); + // The resolved route needs no I/O; one event-loop turn lets it reach the drain. + await new Promise((resolve) => setImmediate(resolve)); + controller.abort(); const switching = session.open(second); await Promise.resolve(); expect(clients[0]?.closed).toBe(0); active.resolve(ok(null)); expect((await call).ok).toBe(true); + expect(await reopening).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", operation: "open_binary" }, + }); expect((await switching).ok).toBe(true); expect(clients[0]?.closed).toBe(1); }); From 1136ea881029eab59ffb0834149e86a6aff69e3b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:43:19 +0800 Subject: [PATCH 015/791] test(session): preserve live binding after cancelled switches --- .../analysis-sessions/providerBinding.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/composition/analysis-sessions/providerBinding.test.ts b/tests/composition/analysis-sessions/providerBinding.test.ts index 7434a839923..4ca489da554 100644 --- a/tests/composition/analysis-sessions/providerBinding.test.ts +++ b/tests/composition/analysis-sessions/providerBinding.test.ts @@ -149,7 +149,7 @@ describe("target-bound provider routing", () => { }); }); - it("cancels a provider switch, closes its old client, and restores the binding", async () => { + it("preserves the active client when a provider switch is cancelled while draining", async () => { const target = await databaseTarget(); const alpha = deepProvider("alpha"); const beta = deepProvider("beta"); @@ -176,18 +176,18 @@ describe("target-bound provider routing", () => { ok: false, error: new AnalysisCancelledError("open_binary"), }); - expect(alpha.clients[0]?.closed).toBe(true); + expect(alpha.clients[0]?.closed).toBe(false); expect(beta.clients).toHaveLength(0); expect(bindingStatus(session)).toMatchObject({ analysis_provider_binding: { provider: { id: "alpha" } }, }); alpha.pending = undefined; - const restored = await session.execute("address_name", { address: "0x2" }); - expect(restored.ok && restored.value.provider.id).toBe("alpha"); - expect(alpha.clients).toHaveLength(2); + const retained = await session.execute("address_name", { address: "0x2" }); + expect(retained.ok && retained.value.provider.id).toBe("alpha"); + expect(alpha.clients).toHaveLength(1); await session.close(); - expect(alpha.clients[1]?.closed).toBe(true); + expect(alpha.clients[0]?.closed).toBe(true); }); }); From d83f0983d5aa18d29d95d24c4ccb3928be6f29f5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:55:43 +0800 Subject: [PATCH 016/791] docs: align translated READMEs and website links with English --- CONTRIBUTING.md | 12 +- README_ar.md | 352 ++++++++---------- README_ja.md | 376 ++++++++------------ README_ko.md | 343 ++++++++---------- README_zh.md | 346 ++++++++---------- docs/roadmap.md | 2 +- tests/conformance/readmeConsistency.test.ts | 84 +++-- 7 files changed, 644 insertions(+), 871 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a36554490b3..ee321d58a4e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,10 +39,14 @@ use `npm run docs:generate`; `docs:build` generates them before building the web Pull requests run `npm run docs:check`. `.github/workflows/pages.yml` is a manual VitePress build and does not publish. The public site at - is published only by the manual website -workflow on `main`. The repository's **Settings → Pages → Build and -deployment → Source** must be set to **GitHub Actions** before the first -deployment. + serves `website/public/` through Cloudflare Workers. See +the [website README](website/README.md#cloudflare-workers) for previews and +manual deployments. + +The separate GitHub Pages host uses only the manual +`.github/workflows/website-pages.yml` workflow on `main`. The repository's +**Settings → Pages → Build and deployment → Source** must be set to +**GitHub Actions** before deploying there. ## Development feedback and PR verification diff --git a/README_ar.md b/README_ar.md index 79f5e8e2c24..e8cee241ce0 100644 --- a/README_ar.md +++ b/README_ar.md @@ -2,354 +2,286 @@ [English](README.md) · [简体中文](README_zh.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **العربية** -# REA: اعكس هندسة أي شيء +# REA: هندسة عكسية لأي شيء -### وسيط MCP واحد للهندسة العكسية عبر الملفات التنفيذية والتطبيقات وسلوك وقت التشغيل. +### خادم MCP واحد للهندسة العكسية للملفات الثنائية والتطبيقات والسلوك أثناء التشغيل. -**اعثر على ميزة تعجبك. افهم آلية عملها. ابنها بالطريقة التي تريدها.** +**وجدت ميزة تعجبك؟ افهم طريقة عملها حتى مستوى الشيفرة الثنائية.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](#منصة-أدوات-التحقيق) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) morluto%2Frea | Trendshift -**[الموقع (بالإنجليزية)](https://morluto.github.io/rea/) · [أدلة الاستخدام](https://morluto.github.io/rea/guides/) · [دراسة حالة DX-Ball](https://morluto.github.io/rea/showcase/dx-ball/)** +**[الموقع](https://rea.tools/) · [الأدلة](https://rea.tools/guides/) · [دراسات الحالة](https://rea.tools/showcase/)** -[البدء السريع](#البدء-السريع) · [حالة الدعم](#حالة-الدعم) · [من الملف التنفيذي إلى السلوك](#من-الملف-التنفيذي-إلى-السلوك) · [منصة أدوات التحقيق](#منصة-أدوات-التحقيق) · [خطة العمل](#خطة-العمل) · [كيف يعمل؟](#كيف-يعمل) +[البدء السريع](#البدء-السريع) · [كيف يعمل REA](#كيف-يعمل-rea) · [ما الذي يمكنك تحليله](#ما-الذي-يمكنك-تحليله) · [دراسات الحالة](#دراسات-الحالة) · [الأسئلة الشائعة](#الأسئلة-الشائعة) · [التوثيق](#التوثيق) - +npx rea-agents setup + +
+ +REA يشغّل جسر التحليل داخل Hopper لفحص ملف ثنائي للشيفرة الأصلية + +
+ +
Discord
انضم إلى مجتمع الهندسة العكسية

- Discord · أسئلة وأجوبة · شارك أعمالك + Discord · أسئلة وأجوبة · مشاركة النتائج

-npx rea-agents setup - -
- -REA يشغّل جسر التحليل داخل Hopper لفحص ملف تنفيذي أصلي - --- -هل وجدت في تطبيق ميزة تريدها في منتجك؟ اطلب من وكيلك استقصاءها باستخدام REA. يستطيع فحص التطبيق من دون شيفرته المصدرية، وشرح طريقة عمل الميزة وعرض الأدلة، ثم بناء نسخة تناسب مشروعك. - -يوفر REA أدوات لتحليل الملفات التنفيذية الأصلية وتطبيقات JavaScript وElectron وتجميعات .NET والمواقع. يمكنك استخدامها من وكيلك أو من الطرفية. يعمل التحليل محليًا، وتتضمن النتائج الأدلة والقيود. - -يضبط Setup إعدادات الوكيل ويربطه بتثبيت موجود من Hopper أو Ghidra، ويثبّت سير عمل REA الموجّه. ويمكنه تثبيت Hopper بعد موافقتك. ولا يحتاج التحليل الثابت لتطبيقات JavaScript إلى أي محرك. - -## اطلب من وكيلك مباشرة +هل وجدت في تطبيق ميزة تريد إضافتها إلى منتجك؟ اطلب من وكيلك استقصاءها باستخدام REA. يستطيع فحص التطبيق من دون شيفرته المصدرية، وشرح طريقة عمل الميزة وعرض الأدلة، ثم بناء نسخة تناسب مشروعك. -بعد [الإعداد](#البدء-السريع)، أعد تشغيل الوكيل ثم اطلب: - -```text -استقصِ طريقة عمل البحث في تطبيق Notes، واعرض الأدلة، ثم ابنِ ميزة مشابهة لمشروعي. -``` +يربط REA وكيلك بأدوات لفحص الملفات الثنائية للشيفرة الأصلية، وتطبيقات JavaScript وElectron، وتجميعات .NET، والمواقع. يمكنك أيضًا استخدام الأدوات نفسها من الطرفية. يجري التحليل محليًا، وتتضمن النتائج الأدلة والقيود التي تستند إليها كل خلاصة. -استبدل Notes بالتطبيق الذي تريد فهمه، أو اطلب نظرة عامة أولًا. +يسجّل الإعداد REA لدى وكيلك ويثبّت تعليمات سير العمل المطابقة. يمكن لتحليل الشيفرة الأصلية استخدام تثبيت موجود من Hopper أو Ghidra؛ ويمكن للإعداد أيضًا تثبيت Hopper اختياريًا بعد موافقتك. لا يحتاج التحليل الثابت لـ JavaScript إلى أي من المحركين. -## من الملف التنفيذي إلى السلوك - -| فك الترجمة | الفهم | إعادة الإنشاء | -| ---------------------------------------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------ | -| استخرج الشيفرة شبه المصدرية، والتعليمات، والسلاسل النصية، والرموز | تتبع تدفق التحكم والمراجع المتبادلة والاستدعاءات والأنواع | حوّل ما تعلمه الوكيل إلى ميزة تناسب تقنياتك وواجهتك ومتطلباتك | - -يوضح REA كيف وصل إلى نتائجه. وهو لا يدّعي استعادة الشيفرة المصدرية الأصلية أو استنساخ التطبيق تلقائيًا. - -## لماذا REA؟ - -| | | -| ------------------------ | ---------------------------------------------------------------------------- | -| **مصمم للوكلاء** | اسأل عما يفعله التطبيق ودع الوكيل يفحصه بدلًا من التخمين. | -| **CLI وMCP** | استخدم قدرات الهندسة العكسية نفسها من الطرفية أو الوكيل. | -| **إعداد موجّه** | اضبط وكيلك، واربط أداة تحليل موجودة، أو ثبّت Hopper بموافقتك. | -| **من الفهم إلى الشيفرة** | افهم الميزة ثم ابنِ نسختك الخاصة في جلسة البرمجة نفسها. | -| **محلي حسب التصميم** | يجري التحليل على مضيفك المحلي المدعوم، ولا يرفع REA التطبيق إلى خدمة مستضافة. | -| **يحافظ على السياق** | استقصِ عدة تطبيقات دون بدء التحليل من جديد عند كل سؤال. | +> **[زر موقع REA](https://rea.tools/)** للاطلاع على تعليمات الإعداد والأدلة المصوّرة ودراسات الحالة الفعلية. ## البدء السريع -### تشغيل الإعداد (موصى به) +### إعداد وكيلك -اربط REA بوكيلك: +بعد تثبيت Node.js وnpm، نفّذ: ```bash npx rea-agents setup ``` -يعرض Setup أولاً قائمة متعددة الاختيار للوكلاء الذين تريد ربطهم. تكون تسجيلات REA الموجودة محددة افتراضياً، ولا يؤدي اكتشاف عميل جديد وحده إلى تحديده؛ ويمكنك اختيار عميل غير مهيأ. راجع المسارات والتغييرات المحددة قبل الموافقة. يثبت Setup سير عمل REA للوكلاء المحددين افتراضياً. Hopper خيار منفصل يتطلب موافقة مستقلة، ويمكن أيضاً حفظ مسارات تثبيت Ghidra الموجودة. +اختر الوكلاء، وراجع التغييرات المقترحة، ثم وافق عليها. يضيف الإعداد خادم MCP الخاص بـ REA وتعليمات سير العمل المطابقة، مع الاحتفاظ بنسخ احتياطية من الإعدادات الموجودة. أعد تشغيل وكيلك بعد ذلك. -يعرض التغييرات قبل تطبيقها وينشئ نسخة احتياطية من الإعدادات الموجودة. راجع [دليل التثبيت والإعداد](docs/installation.md) للمتطلبات والخيارات الإضافية. +يدعم الإعداد Claude Code وCodex وCursor وGemini CLI و[وكلاء آخرين](docs/installation.md#supported-agents). راجع [التثبيت والإعداد](docs/installation.md) لتهيئة موفّري التحليل وتسجيل MCP يدويًا. -### الاستخدام مع وكيل +### اطلب من وكيلك -بعد الإعداد، أعد تشغيل الوكيل واشرح التطبيق أو الميزة التي تريد استقصاءها. يدعم REA Claude Code وClaude Desktop وCodex وCursor وGemini CLI وWindsurf وDevin وOpenCode وAntigravity وGitHub Copilot CLI وCommand Code وVS Code. تكون تسجيلات REA الموجودة محددة افتراضياً، ويجب اختيار العملاء المكتشفين الآخرين يدوياً. يمكن للوكلاء الآخرين استخدام إعداد MCP أدناه. +```text +استقصِ طريقة عمل البحث في تطبيق Notes، واعرض الأدلة، ثم ابنِ ميزة مشابهة لمشروعي. +``` -يمكن تشغيل Hopper في الوضع التجريبي. إذا ظهرت رسالة عند التشغيل الأول، فاختر التجربة أو أدخل ترخيصًا موجودًا. +استبدل Notes بالتطبيق المستهدف وحدّد الميزة التي تريد فهمها. -### مهارة مساعد البرمجة (اختياري) +### استخدام الطرفية -أضف المهارة إلى مساعد البرمجة للحصول على سياق أغنى: +افحص مجلد تطبيق JavaScript/Electron بعد فك حزمه، أو ملف ASAR: ```bash -npx skills add morluto/rea --skill reverse-engineer-anything +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json ``` -توفر المهارة سير عمل التحقيق في REA. نفّذ الإعداد أعلاه لربط REA بالوكيل وضبط أدوات التحليل. يثبّت الإعداد افتراضيًا مهارة مطابقة للإصدار؛ وهذا الأمر يثبّت نسخة المستودع. +تتضمن النتيجة الوحدات وعمليات الاستيراد وحدود Electron والأدلة المتعلقة بها. استبدل المسار بمسار هدفك، مثل `"D:/apps/example"` على Windows. -لشجرة JavaScript/Electron المستخرجة أو ملف ASAR، لا يلزم إعداد MCP ولا محرك أصلي: +لتثبيت أمر `rea` للاستخدام المعتاد: ```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +npm install --global rea-agents +rea --help ``` -استبدل المسار بهدفك (مثل `"D:/apps/example"` على Windows). يدعم سير العمل الثابت هذه المسارات، بينما يتطلب التحليل الأصلي محركه الخاص. +لتحليل الشيفرة الأصلية، هيّئ موفّرًا أولًا. راجع [دليل CLI وEvidence](docs/cli.md) لأوامر تحليل الشيفرة الأصلية، واختيار الموفّر، واللقطات، والاستخدام في السكربتات. + +### تحديث REA -### تثبيت أمر rea +يتطور REA بسرعة، وتتضمن الإصدارات الجديدة إصلاحات متكررة للأخطاء. حافظ على تحديث نسختك. -ثبت واجهة سطر الأوامر: +لواجهة CLI المثبّتة عبر npm: ```bash -curl -fsSL https://raw.githubusercontent.com/morluto/rea/main/install.sh | bash +rea update ``` -يجب تثبيت Node.js وnpm مسبقًا. عند تشغيله من طرفية، يثبت البرنامج `rea` ويبدأ الإعداد. +لتحديث تسجيلات الوكلاء والمهارة، نفّذ أمر الإعداد الذي يعرضه التحديث. -يمكنك بدلًا من ذلك التثبيت باستخدام npm ثم تشغيل الإعداد: +إذا كنت تستخدم `npx`، فحدّث إعداد وكيلك بالأمر التالي: ```bash -npm install --global rea-agents -rea setup +npx rea-agents@latest setup ``` -### المتطلبات +راجع تغييرات الإعداد وأعد تشغيل وكيلك. لأوامر CLI التي تنفّذها مرة واحدة، استخدم `npx rea-agents@latest` متبوعًا بالأمر المطلوب. -- macOS 12 أو أحدث -- Ubuntu 24.04+ أو Fedora 41+ أو Arch Linux بنواة 64 بت أو CachyOS -- Node.js 22.x (>=22.19) أو 24.x (>=24.11) أو 26+ وnpm +## كيف يعمل REA -يتطلب تحليل الملفات التنفيذية الأصلية Hopper أو Ghidra. Hopper برنامج منفصل، وللوضع التجريبي قيود يحددها المورّد، لكن الترخيص المدفوع ليس إلزاميًا. +يستدعي وكيلك REA عبر MCP لفحص الهدف وتتبع الشيفرة ذات الصلة. يعيد REA النتائج مع أدلتها. يستخدمها الوكيل لطرح أسئلة متابعة، أو شرح السلوك، أو كتابة تنفيذ واختباره. تستخدم أوامر CLI سير العمل نفسه. -يدعم Ghidra نظامي Linux x64 وmacOS x64/arm64. ثبت Ghidra 12.1.x وJDK الكامل بنواة 64 بت الذي يعلنه ذلك التثبيت (`application.java.min` حتى `application.java.max`) بصورة منفصلة، ثم اضبط REA لاستخدامهما. تتطلب إصدارات 12.1 الحالية JDK 21 أو أحدث ولا تحدد حدًا أعلى. جرى التحقق من الجسر مع Ghidra 12.1.4 وJDK 21. يحتاج macOS أيضًا إلى أداة فك الترجمة الأصلية المطابقة لمعمارية الجهاز. +![مسار الاستقصاء في REA: يسأل وكيلك عن هدف محلي، ويفحصه REA ويتتبعه بأدوات التحليل، ثم يستخدم الوكيل الشيفرة والمراجع والجوانب غير المحسومة التي أعادتها الأدوات لشرح السلوك وتنفيذه واختباره.](website/public/assets/figures/rea-investigation-flow.svg) -يستطيع Setup التحقق من التثبيت وحفظ المسارات؛ ولا يثبت أو يحدّث Ghidra أو Java أو Node.js أو npm أو Homebrew. +[افتح الشكل بحجمه الكامل](website/public/assets/figures/rea-investigation-flow.svg). -يتضمن فرع main والحزمة npm 4.1.0 دعمًا تجريبيًا لـ Windows x64 Ghidra P0 لتطبيقات PE الأصلية x86-64 (غير المُدارة وغير DLL) على NTFS محلي، مع ضوابط Job Object وقوائم DACL الخاصة وقبول المسارات المجمّعة. تحقق من [حد الإصدار](docs/installation.md#released-package-and-main) قبل توقع ذلك من حزمة npm أقدم. راجع [دليل Windows Ghidra P0](docs/windows-ghidra-p0.md) للمتطلبات والنطاق الموثق. + -### تشخيص المشكلات +## ما الذي يمكنك تحليله -يشخّص `npx -y rea-agents@latest doctor` الجهاز والاعتماديات وأدوات التحليل وإعدادات الوكيل من دون تغييرها. أضف `--json` للحصول على تشخيص منظم. +يتطلب REA إصدار Node.js 22.x (>=22.19) أو 24.x (>=24.11) أو 26+، إضافة إلى npm. تعتمد الأدوات الإضافية والأنظمة المضيفة المدعومة على الهدف: -على Linux، يفضّل REA الملف التنفيذي `/opt/hopper/bin/Hopper`، وإذا لم يكن متاحًا يتحقق تلقائيًا من `~/.local/share/rea/hopper/bin/Hopper`. استخدم `HOPPER_LAUNCHER_PATH` للمسارات الأخرى. إذا كان الملف موجودًا لكن محرك التحليل لا يظهر، فشغّل `ldd /absolute/path/to/Hopper | grep 'not found'` على مسار Hopper الفعلي للتحقق من المكتبات الناقصة. راجع [دليل Hopper](docs/installation.md#hopper). +| الهدف | ما يعيده REA | المتطلبات والدليل | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| الملفات الثنائية للشيفرة الأصلية | شيفرة شبه برمجية، وتعليمات تجميع، وسلاسل نصية، ورموز، واستدعاءات، ومراجع | Hopper أو Ghidra أو IDA؛ [تحليل الشيفرة الأصلية](https://rea.tools/guides/native/) | +| بنية ELF دون تشغيل | الأقسام والمقاطع والرموز ومعلومات إعادة التموضع الأصلية، وآليات الحماية المحتملة المستدلّ عليها بالفحص الثابت | pwntools يوفّره المستدعي على Linux x64؛ [تشخيص الملفات الثنائية](docs/binary-diagnostics.md) | +| شيفرة EVM البايتية | محددات التوجيه، وإزاحات البايتات، والمعاملات المستنتجة، وقابلية تغيير الحالة | مدخل محلي يحمل بايتات خامًا أو تمثيلًا سداسيًا عشريًا؛ [دليل EVM دون تشغيل](docs/evm-bytecode.md) | +| أعطال Linux المسجّلة | سجلات note الخام، ومسجّلات المعالج/الإشارات لكل خيط مسجّل، وترشيحات اختيارية لخرائط الذاكرة | pwntools يوفّره المستدعي؛ GDB/pwndbg اختياريان؛ [الأعطال المسجّلة](docs/recorded-crashes.md) | +| JavaScript / Electron | الوحدات، وعمليات الاستيراد، وخرائط المصدر، والمسارات، وIPC، وعلاقات إضافات الشيفرة الأصلية | Node.js وnpm؛ [تحليل التطبيقات](https://rea.tools/guides/javascript/) | +| المواقع | بنية الصفحة، والسكربتات، وملاحظات الشبكة، ولقطات الشاشة المطلوبة | متصفح من عائلة Chrome؛ [تحليل المتصفح](https://rea.tools/guides/browser/) | +| تسجيلات الشبكة المحفوظة | الطلبات، والاستجابات، ومحتويات الحمولة المتاحة، ومواضعها في المصدر | HAR؛ وmitmdump على Linux لتسجيلات mitmproxy بصيغتها الأصلية؛ [دليل تسجيلات الشبكة](docs/web-network-captures.md) | +| تجميعات .NET | البيانات الوصفية، وتعليمات CIL، واعتماديات الشيفرة الأصلية المعلنة، ومقارنات البناء | فحص ثابت؛ [دليل الشيفرة المُدارة](docs/managed-code-analysis.md) | +| حزم Android APK | تصريحات ملف manifest، والأصناف، والدوال الناتجة عن فك الترجمة، والمراجع | JADX دون واجهة رسومية وJDK كامل على Linux/macOS؛ [دليل Android](docs/android-analysis.md) | +| البرامج الثابتة | المناطق، ونتائج الاستخراج، وما يُمرّر إلى تحليل الشيفرة الأصلية | Binwalk / Unblob على Linux؛ [دليل البرامج الثابتة](docs/firmware-analysis.md) | +| الحزم والموارد | قوائم الملفات، والبصمات، وملفات plist، وبنية حزم Apple، والموارد المستخرجة | [دليل تحليل الملفات وJavaScript](docs/javascript-artifact-reconstruction.md)، [تطبيقات Apple](docs/apple-application-analysis.md) | +| سلوك العمليات | مخرجات الطرفية، والتفاعلات، وملاحظات الخروج ونظام الملفات، ومقارنات التشغيل | Linux/macOS مع PTY أصلي؛ [تسجيل سلوك العمليات](docs/process-capture.md) | -### التحديث والإزالة +يقرأ الفحص الثابت لـ JavaScript و.NET الملفات المقدّمة دون تشغيل التطبيق. يشغّل التسجيل أثناء التنفيذ الهدف المحدد أو يتفاعل معه بصلاحيات مستخدمك؛ ويشرح كل دليل للتسجيل أثناء التنفيذ آثاره. -- يحدّث `rea update` تثبيت REA الحالي. -- يزيل `rea uninstall` تسجيلات الوكلاء وملفات سير العمل التي يديرها REA، ويترك Hopper مثبتًا. -- يزيل `rea uninstall --purge-data` أيضًا ذاكرة REA المؤقتة وحالته. استخدمه فقط إذا أردت حذف هذه البيانات. + -## حالة الدعم +تختلف صيغ ملفات الشيفرة الأصلية والأنظمة المضيفة المدعومة باختلاف الموفّر. راجع [إعداد Hopper وGhidra](docs/installation.md#hopper)، و[دليل IDA](docs/ida-provider.md)، و[دعم Ghidra التجريبي على Windows](docs/windows-ghidra-p0.md). يدعم Ghidra أيضًا [تحليل DOS ذي 16 بت](docs/ghidra-dos.md). لاختيار الموفّر، راجع [دليل CLI](docs/cli.md#choose-a-provider). تحقّق من [توفر الميزات في الإصدارات](docs/installation.md#released-package-and-main) للميزات المضافة بعد أحدث إصدار على npm. -تُوصف قدرات المستودع الحالية ومتطلبات الأنظمة في [دليل الدعم الإنجليزي](README.md#current-status). قد يسبق فرع main [إصدار npm](docs/installation.md#released-package-and-main). +## دراسات الحالة -- يوفر Ghidra 25 عملية للقراءة فقط على Linux x64 وmacOS x64/arm64 وحدود Windows x64 P0 التجريبية. يدعم Linux/macOS أيضًا تعليقات الدوال الذرية على مستوى الجلسة. نظام Windows P0 للقراءة فقط، ولا يوفر Ghidra أي تحكم في الواجهة. -- يتطلب الفحص الثابت لنظام Android تجهيز JADX/Java بشكل منفصل؛ ويغطي التحقق بالموفر الحقيقي نظام Linux. راجع [تحليل Android](docs/android-analysis.md). -- تحدد طلبات المتصفح وElectron والعمليات هدفها وإجراءاتها ودورة حياتها مباشرة، وتعتمد على وصول المضيف الفعلي دون منح أذونات منفصلة من REA. ما زالت كتابة إعدادات Setup وتثبيت Hopper تتطلب الموافقة على خطتها الدقيقة. -- يصف `rea capabilities` عمليات جلسة الملف التنفيذي لا كل أدوات التطبيقات ووقت التشغيل. راجع قائمة أدوات MCP الكاملة والدليل ذا الصلة. +### DX-Ball: إعادة بناء حساب توزيع الصوت بين القناتين -## استقصاء كامل بطلب واحد +تتبّع استدعاء صوت إلى الدالة المساعدة التي تحوّل الموضع إلى توزيع بين القناتين، وافحص التعليمات، وحوّل الشيفرة شبه البرمجية غير المكتملة إلى C. تجتاز إعادة البناء 3,205 حالات اختبار على x86 الأصلي، وتعيد إنتاج جميع بايتات الدالة المترجمة البالغ عددها 63 بايتًا. -بعد الإعداد، يمكنك أن تطلب من وكيلك: +[اقرأ دراسة الحالة](https://rea.tools/showcase/dx-ball/) · +[مستودع إعادة البناء](https://github.com/N0zoM1z0/dx-ball) -> افتح التطبيق، واكتشف آلية عمل البحث من دون اتصال، واشرح تدفق التحكم، ثم ابنِ نسخة لمشروعي باستخدام TypeScript وSQLite. +### Notion: تتبّع جسر الحافظة في Electron -يمكن للوكيل تحويل ذلك إلى سير عمل قابل للتحقق: +اعثر على API الحافظة في عملية العرض، وتتبّعه عبر preload وIPC إلى العملية الرئيسية، وافحص صيغة الحافظة ذات التنسيق الغني. -```text -فتح الهدف - → انتظار اكتمال تحليل Hopper - → جرد المعماريات والرموز والسلاسل النصية - → تحديد نقاط الدخول والدوال المهمة - → تتبع الاستدعاءات والمراجع المتبادلة وتدفق التحكم - → فك ترجمة التنفيذ ذي الصلة - → تلخيص السلوك والقيود والحالات الطرفية - → بناء الميزة بما يلائم تقنيات المشروع ومتطلباته -``` +[اقرأ دراسة الحالة](https://rea.tools/showcase/notion/) -## ما الذي يمكنك فعله؟ +### TH04: استعادة حساب حلقة المقذوفات في DOS -### فهم تطبيق لا تملك مصدره +افحص تعليمات اللعبة الأصلية على PC-98 ذات 16 بت، واستعد حساب الزوايا الثابتة والموجّهة نحو الهدف، وقارن شيفرة C++ المعاد بناؤها بمخرجات المترجم المستخدم آنذاك. -اكشف نقاط الدخول، وأسماء الفئات، والسلاسل النصية، والأطر المستخدمة، ومسارات الميزات من الملف التنفيذي وحده. +[اقرأ دراسة الحالة](https://rea.tools/showcase/th04/) · +[مستودع إعادة البناء](https://github.com/N0zoM1z0/th04) -### بناء ميزة تعجبك بطريقتك +إذا استخدمت REA لفحص هدف مثير للاهتمام، فنود الاطلاع عليه. شارك حالتك في [issue](https://github.com/morluto/rea/issues) أو [pull request](https://github.com/morluto/rea/pulls)، مع ذكر الهدف، وسؤالك، وكيف ساعدك REA، وما توصلت إليه. -استقصِ الميزة، وافهم سلوكها، ثم ابنِ نسخة ملائمة لمنتجك وتقنياتك ومتطلباتك. +## الأسئلة الشائعة -### توثيق بروتوكول أو تنسيق غير موثق +
+أي وكلاء يمكنهم استخدام REA؟ -تتبع الموزعات، والثوابت، والمقارنات، والقراءة والكتابة لفهم الرسائل وتنسيقات الملفات وآلات الحالات. +أي وكيل يدعم خوادم MCP المحلية. يهيّئ الإعداد [الوكلاء المدعومين](docs/installation.md#supported-agents)؛ ويمكن للعملاء الآخرين استخدام [تسجيل MCP يدويًا](docs/installation.md#mcp-registry). -### إعادة إنشاء سلوك متوافق +
-حوّل منطقًا مفكوك الترجمة إلى مواصفات واضحة، ثم نفذ بديلًا نظيفًا تحكمه اختبارات السلوك الملحوظ. +
+هل أحتاج إلى Hopper أو Ghidra أو IDA؟ -### تدقيق منطق أمني حساس +يستخدم تحليل الشيفرة الأصلية المتعمق أحدها. يعمل الفحص الثابت لـ JavaScript و.NET دون محرك لتحليل الشيفرة الأصلية. يستطيع الإعداد تثبيت Hopper بعد الموافقة؛ أما Ghidra وIDA فيستخدمان تثبيتاتك الموجودة. راجع [إعداد الموفّرين](docs/installation.md#hopper). -افحص التحقق من الإدخال، والتشفير، والصلاحيات، والتخزين، ومسارات الأخطاء من دون رفع الملف التنفيذي إلى خدمة بعيدة. +
-## منصة أدوات التحقيق +
+هل أحتاج إلى تشغيل Hopper أولًا؟ -| عائلة الأدوات | العدد | الاستخدام | -| ---------------------- | ----: | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| الفحص الأصلي | 40 | الدوال والشيفرة شبه المصدرية والتعليمات والسلاسل والرموز والاستدعاءات والمراجع والتعليقات وقراءة البايتات ومواضع الملفات | -| سير عمل التحقيق | 14 | نظرات عامة على التطبيقات وملفات الدوال وواجهات API الأصلية والتوزيع وفك الترجمة الدفعي وتتبع الميزات ومسارات الاستدعاء ومخططاته واكتشاف Swift وObjective-C | -| أدوات الملفات الثنائية | 10 | بيانات Mach-O والتوقيعات وplist والمعماريات وأسماء Swift ومراقبة الاستدعاءات عبر LLDB على macOS؛ تخطيط ELF وأدلة الحماية الثابتة وسجلات الأعطال على Linux | -| مخطط المخرجات | 7 | جرد الدلائل والحزم وملفات Interface Builder المترجمة وكتالوجات موارد Apple وتحليل تحميل Mach-O dylib والاستخراج واستدلال واجهات EVM دون اتصال | -| PE/CLI المُدار | 7 | هوية تجميعات .NET وبياناتها الوصفية وتعليمات CIL والاعتماديات الأصلية واستيراد إعادة البناء ومقارنة الإصدارات | -| البرامج الثابتة | 2 | فحص مناطق البرامج الثابتة على Linux واستخراجها الصريح | -| Android APK | 5 | بيانات الحزمة وmanifest والبحث عن الأصناف وقوائم الأعضاء وفك ترجمة الأساليب والمراجع الثابتة | -| مراقبة المتصفح | 12 | بنية الصفحات وبيانات الشبكة والبرامج النصية وخرائط المصدر واكتشاف WebMCP والصور ومقارنة الالتقاط | -| تحليل Electron | 5 | مراقبة العارضين وخريطة التطبيق الثابتة والمواءمة بين نتائج الثبات والتشغيل | -| وقت تشغيل JavaScript | 2 | اكتشاف أهداف Node/Electron Inspector ومواضع البرامج وأحداث سياقات التنفيذ | -| سير عمل التطبيقات | 13 | تصدير النصوص البرمجية الملتقطة، تتبع الميزات عبر الطبقات ومقارنة الإصدارات ومواءمة المصدر التاريخي ومقارنة أشكال القيم المعادة والتحقق من إعادة الإنشاء | -| مساحة العمل والملاحظة | 21 | الجلسات وحزم الأدلة وسياق التنقل ومقارنة العمليات والمخرجات والدوال وتسجيل الأسئلة غير المحسومة | +يشغّل REA برنامج Hopper عندما تحتاج العملية إليه. على macOS، قد تظهر عند التشغيل الأول نافذة تطلب اختيار وضع العرض التجريبي أو تفعيل الترخيص. راجع [تشغيل Hopper واستكشاف الأخطاء](docs/installation.md#launcher-paths-and-troubleshooting). -## خطة العمل +
-يشمل العمل التالي توسيع التحقق من الأهداف الأصلية، ومقارنة إصدارات JavaScript و.NET، ومراقبة وقت التشغيل والتحقق من إعادة الإنشاء. يتيح Setup بالفعل اختيار تكامل الوكيل وتثبيت Hopper. تثبيت أدوات تحليل أخرى ما زال عملًا لاحقًا. راجع [خطة التثبيت](docs/roadmap.md) و[تقييم أدوات التحليل](docs/provider-evaluation.md). +
+ماذا يفعل تثبيت المهارة من skills.sh؟ -## استخدام REA مع وكلاء برمجة آخرين +توفّر المهارة تعليمات الاستقصاء لوكيلك. استخدم `rea setup` لتسجيل خادم MCP الخاص بـ REA وتثبيت التعليمات المطابقة، ثم أعد تشغيل وكيلك. راجع [تثبيت المهارة وحدها](docs/installation.md#skill-only-installation). -يدعم Setup Claude Code وClaude Desktop وCodex وCursor وGemini CLI وWindsurf وDevin وOpenCode وAntigravity وGitHub Copilot CLI وCommand Code وVS Code. تكون تسجيلات REA الموجودة محددة افتراضياً، ويجب اختيار العملاء المكتشفين الآخرين يدوياً. يمكن لأي وكيل يدعم خوادم MCP المحلية الاتصال باستخدام الإعداد التالي: +
- +
+ما الشيفرة التي يعيدها REA؟ -```json -{ - "mcpServers": { - "rea": { - "command": "npx", - "args": ["-y", "rea-agents@6.0.0", "mcp"] - } - } -} -``` +يعيد تحليل الشيفرة الأصلية شيفرة شبه برمجية وتعليمات تجميع. يستعيد تحليل JavaScript/Electron الوحدات والعلاقات بينها. يستخدم وكيلك هذه النتائج لكتابة تنفيذ واختباره؛ وتقدّم [دراسات الحالة](#دراسات-الحالة) أمثلة عملية. + +
- +
+هل يرفع REA تطبيقي؟ -## كيف يعمل؟ +يحلّل REA الأهداف محليًا. يتلقى وكيلك نتائج الأدوات، ولموفّر النموذج الذي يستخدمه سياسة بيانات خاصة به. -```mermaid -flowchart LR - Human[المستخدم] --> Agent[وكيل البرمجة] - Agent --> REA[REA] - Human -->|CLI| REA - REA --> Hopper[عمليات تحليل Hopper] - Hopper --> App[تطبيقك] - REA --> Ghidra[تحليل Ghidra للقراءة فقط
جرد + تحليل الدوال] - Ghidra --> App -``` +
-يستخدم CLI وخادم MCP سير عمل التحليل نفسه. تحرر أوامر الطرفية جلسة الجسر الخاصة بها عند انتهائها، بينما يمكن لجلسة الوكيل الحفاظ على الاتصال أثناء الاستقصاء. إغلاق جلسة REA لا يغلق تطبيق Hopper الذي يستخدمه المستخدم. +
+ماذا أفعل إذا واجهت خطأ؟ -## أوامر CLI +حدّث أولًا؛ فقد يكون إصدار حديث قد أصلح المشكلة بالفعل. -سير عمل الوكيل أعلاه هو أسهل طريقة لاستخدام REA. للحصول على نظرة سريعة على تطبيق من الطرفية: +لواجهة CLI المثبّتة عبر npm: ```bash -npx -y rea-agents@latest analyze /Applications/Notes.app -npx -y rea-agents@latest compare /absolute/path/to/left-evidence.json /absolute/path/to/right-evidence.json +rea update ``` -شغّل `npx -y rea-agents@latest --help` لمعرفة خيارات فك الترجمة المباشر والخيارات الأخرى. - -يمكن لـ REA فتح مجلد `.app` على Mac مباشرة. إذا لم يعثر الوكيل على التطبيق، فأخبره بمكان تثبيته. - -## سلوك Hopper - -يشغّل REA برنامج Hopper عند الحاجة. على macOS قد تظهر نافذته في المقدمة أو يحتاج أحد الحوارات إلى استجابة منك. لا يمكن ضمان بقاء النافذة في الخلفية. - -يحدد REA تنسيق الهدف ومعماريته لتجنب حوارات الاختيار المعتادة. عند إغلاق الجلسة، يوقف الجسر وينظف موارده الخاصة، لكنه لا يغلق تطبيق Hopper الذي تستخدمه. - -## الأمان والخصوصية +لإعداد الوكيل عبر `npx`: -- يبقى التحليل محليًا بين REA وموفر التحليل المختار عبر مقابس محلية خاصة، ومع IDA عبر تسجيل MCP المحلي المضبوط. -- لا يرفع REA الأهداف إلى خدمة مستضافة. -- تعمل أدوات التحليل والأهداف المطلقة بصلاحيات المستخدم نفسه؛ وما زال التقاط واجهة النظام على macOS يعتمد على الوصول إلى تسهيلات الاستخدام وتسجيل الشاشة. ولا ينفّذ التحليل الثابت للـ JavaScript الوحدات المستخرجة. -- تستخدم جلسات Ghidra مشروعًا مؤقتًا معزولًا ولا تفتح مشاريع Ghidra التي يملكها المستخدم أو تعدّلها. -- راجع الملفات التنفيذية غير الموثوقة واعزلها كما تفعل مع أي مدخل أصلي قد يكون ضارًا. -- أبلغ عن الثغرات وفق [سياسة الأمان](SECURITY.md)، وليس عبر قضية عامة. - -## الأسئلة الشائعة - -
-هل يتضمن REA محرك فك ترجمة خاصًا به؟ - -لا. يستطيع Setup تثبيت Hopper، لكنه يظل برنامجًا منفصلًا بترخيص خاص به. يوفر REA واجهة CLI وخادم MCP وسير العمل المخصص للوكلاء. - -
- -
-هل يجب أن يكون Hopper مفتوحًا مسبقًا؟ +```bash +npx rea-agents@latest setup +``` -لا. يستطيع REA تشغيله. لكن Hopper قد يصبح مرئيًا أو يطلب إدخالًا عند الحاجة إلى قرار تفاعلي. +إذا كنت تستخدم وكيلًا، فأكمل [تحديث الإعداد](#تحديث-rea) وأعد تشغيله. أعد محاولة المهمة نفسها. إذا استمرت المشكلة، [افتح issue](https://github.com/morluto/rea/issues) يتضمن إصدار REA، ونوع الهدف، وخطوات إعادة الإنتاج، ومخرجات الخطأ.
-
-هل يعمل REA على Linux أو Windows؟ +## التوثيق -يدعم REA نظام macOS 12+ وUbuntu 24.04+ وFedora 41+ وArch Linux بنواة 64 بت وCachyOS. يعمل تحليل Ghidra للقراءة فقط على Linux x64 وmacOS x64/arm64 مع Ghidra 12.1.x وJDK الكامل الذي يعلنه التثبيت (JDK 21 أو أحدث دون حد أعلى في إصدارات 12.1 الحالية)، إضافة إلى حد Windows x64 P0 التجريبي لتطبيقات PE الأصلية x86-64 على NTFS محلي؛ راجع [دليل Windows Ghidra P0](docs/windows-ghidra-p0.md) للنطاق الموثق. +ابدأ بـ [الأدلة العملية](https://rea.tools/guides/) على الموقع. للخيارات الدقيقة والمتطلبات وعقود النتائج: -
+- [التثبيت والإعداد](docs/installation.md): تسجيل الوكلاء، وتهيئة الموفّرين، والتحديثات، وإلغاء التثبيت. +- [الجاهزية واستكشاف الأخطاء](docs/installation.md#check-readiness-for-your-task): تشخيص وكيل أو محرك تحليل محدد. +- [CLI وEvidence](docs/cli.md): الأوامر، واختيار الموفّر، واللقطات، والاستيراد/التصدير، وحالات الخروج. +- [عقود MCP](docs/mcp-contracts.md) و[تعليمات الوكلاء](docs/mcp-prompts.md): نتائج الأدوات، والجلسات، والاستقصاءات الموجّهة. +- [كتالوج الأدوات](docs/mcp-contracts.md#generated-catalog): قائمة الأدوات والموفّرين وأوامر CLI المولّدة أثناء البناء. +- [خطة العمل](docs/roadmap.md): الأعمال المخطط لها ومتابعة القدرات. -
-هل يمكنني تحليل قواعد Hopper المحفوظة؟ +أبلغ عن الثغرات وفق [SECURITY.md](SECURITY.md). -نعم. يدعم وقت التشغيل الملفات التنفيذية وقواعد `.hop`. +## المساهمة -
- -## التطوير +نرحّب بمساعدتك في تطوير REA! [افتح issue](https://github.com/morluto/rea/issues) للإبلاغ عن خطأ أو اقتراح ميزة، أو [أرسل pull request](https://github.com/morluto/rea/pulls) لتحسين الشيفرة أو التوثيق. -راجع [CONTRIBUTING.md](CONTRIBUTING.md) لإعداد بيئة التطوير وبنية المشروع والاختبارات وإرشادات الإصدار. +راجع [CONTRIBUTING.md](CONTRIBUTING.md) لإعداد بيئة التطوير والفحوص، و[دليل الاختبار](docs/testing.md) لمسارات التحقق، و[خريطة البنية](docs/architecture.mermaid) لهيكل المشروع. -## المشروع +## روابط المشروع -- [حزمة npm](https://www.npmjs.com/package/rea-agents) -- [GitHub](https://github.com/morluto/rea) -- [القضايا والطلبات](https://github.com/morluto/rea/issues) -- [Hopper Disassembler](https://www.hopperapp.com/) -- [سياسة الأمان](SECURITY.md) +[الموقع](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [الأمان](SECURITY.md) -## تاريخ النجوم +## سجل النجوم -🎉 **وصلنا إلى 20,000 نجمة على GitHub، شكرًا لكم!** +🎉 **20,000 نجمة على GitHub — شكرًا لكم!** -شكرًا لكل من يستخدم REA ويبلّغ عن الأخطاء ويختبر الإصدارات ويساهم في إصلاحها. +شكرًا لكل من يستخدم REA، ويبلّغ عن الأخطاء، ويختبر إصدارات البناء، ويساهم بالإصلاحات. - تاريخ نجوم REA على GitHub + سجل نجوم REA على GitHub +## إخلاء المسؤولية + +يوفّر REA أدوات لأبحاث الهندسة العكسية والتحليل وإعادة البناء المشروعة. أنت مسؤول عن الحصول على أي تفويض مطلوب والالتزام بالقوانين المعمول بها. لا يؤيد المشروع الاستخدام غير القانوني أو غير المصرّح به. + ## الترخيص [MIT](LICENSE) diff --git a/README_ja.md b/README_ja.md index 5db65e77e73..c25a0c13ccb 100644 --- a/README_ja.md +++ b/README_ja.md @@ -4,24 +4,33 @@ # REA:あらゆるものをリバースエンジニアリング -### バイナリ、アプリ、実行時の動作をひとつの MCP でリバースエンジニアリング。 +### バイナリ、アプリケーション、実行時の動作を、ひとつの MCP でリバースエンジニアリング。 **気に入った機能を見つけたら、その仕組みをバイナリのレベルまで理解する。** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](#調査ツールカタログ) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) morluto%2Frea | Trendshift -**[ウェブサイト(英語)](https://morluto.github.io/rea/) · [ガイド](https://morluto.github.io/rea/guides/) · [DX-Ball の事例](https://morluto.github.io/rea/showcase/dx-ball/)** +**[ウェブサイト](https://rea.tools/) · [ガイド](https://rea.tools/guides/) · [事例](https://rea.tools/showcase/)** -[クイックスタート](#クイックスタート) · [現在の対応状況](#現在の対応状況) · [バイナリから動作へ](#バイナリから動作へ) · [調査ツールカタログ](#調査ツールカタログ) · [ロードマップ](#ロードマップ) · [仕組み](#仕組み) +[クイックスタート](#クイックスタート) · [REA の仕組み](#rea-の仕組み) · [分析できる対象](#分析できる対象) · [事例](#事例) · [よくある質問](#よくある質問) · [ドキュメント](#ドキュメント) - +npx rea-agents setup + +
+ +REA が Hopper 内で分析ブリッジを起動し、ネイティブバイナリを調べる様子 + +
+ +
@@ -35,342 +44,231 @@
-npx rea-agents setup - -
- -Hopper 内で解析ブリッジを起動し、ネイティブバイナリを調べる REA - --- -自分のプロダクトにも欲しい機能を他のアプリで見つけたら、REA を使ってエージェントに調査を依頼できます。エージェントはソースコードがなくてもアプリを調べ、機能の仕組みを説明して根拠を示し、あなたのプロジェクト向けの実装まで進められます。 - -REA は、ネイティブバイナリ、JavaScript/Electron アプリ、.NET アセンブリ、Web サイトを調べるためのツールをエージェントに提供します。同じツールはターミナルからも使えます。解析はローカルで実行され、結果には各結論の根拠と制限が含まれます。 +アプリに、自分の製品にも取り入れたい機能を見つけましたか?エージェントに REA で調査を依頼してください。ソースコードがなくてもアプリを調べ、機能の仕組みを説明し、根拠を示したうえで、あなたのプロジェクト向けに実装できます。 -セットアップは REA をエージェントに登録し、対応するワークフロー手順もインストールします。ネイティブ解析には既存の Hopper または Ghidra を使えます。承認すれば、セットアップで Hopper をインストールすることもできます。静的な JavaScript 解析にはどちらのエンジンも不要です。 +REA はエージェントを分析ツールにつなぎ、ネイティブバイナリ、JavaScript や Electron のアプリ、.NET アセンブリ、ウェブサイトを調べられるようにします。同じツールをターミナルからも使えます。分析はローカルで実行され、結果には各結論を裏付ける根拠と制限事項が含まれます。 -## エージェントに頼むだけ - -[セットアップ](#クイックスタート)後にエージェントを再起動し、次のように依頼します: - -```text -メモアプリの検索機能を調べて根拠を示し、私のプロジェクト向けに同様の機能を実装してください。 -``` +セットアップはエージェントに REA を登録し、対応するワークフローの手順をインストールします。ネイティブ分析には既存の Hopper または Ghidra を使えます。承認すれば、セットアップで Hopper をインストールすることもできます。静的な JavaScript 分析には、どちらのエンジンも必要ありません。 -「メモアプリ」を調べたいアプリに置き換えてください。まずは概要の調査を依頼することもできます。 - -## バイナリから動作へ - -| 逆コンパイル | 理解 | 再現 | -| -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | -| ネイティブアプリや実行ファイルから、プロシージャ、疑似コード、アセンブリ、文字列、シンボル、セグメント、メタデータを復元します。 | 呼び出し元、呼び出し先、クロスリファレンス、コールグラフをたどり、機能やアルゴリズムの実際の動作を説明できる状態にします。 | エージェントが得た知見を、あなたの技術スタック、画面、要件に合うプロダクト機能へ変えます。 | - -REA は結論に至った根拠を示します。元のソースコードを復元できる、あるいはアプリ全体を自動的に複製できるとは主張しません。 - -## REA を選ぶ理由 - -| | | -| -------------------------- | -------------------------------------------------------------------------------------------------------- | -| **エージェント向け** | アプリの動作を質問すると、エージェントが推測ではなく実際に調べて答えます。 | -| **CLI と MCP** | 同じリバースエンジニアリング機能を、ターミナルからもエージェントからも使えます。 | -| **ガイド付きセットアップ** | エージェントの設定、既存の解析ツールへの接続、承認したうえでの Hopper のインストールに対応します。 | -| **洞察からコードへ** | 機能を理解したら、同じコーディングセッションで自分のプロダクト向けに実装できます。 | -| **ローカルで解析** | 解析は対応するローカルホストで実行されます。REA がアプリをホスト型の解析サービスへ送ることはありません。 | -| **コンテキストを維持** | 質問のたびに解析をやり直すことなく、複数のアプリを続けて調査できます。 | +> **[REA のウェブサイト](https://rea.tools/)**で、セットアップ手順、図解ガイド、実際の事例を確認できます。 ## クイックスタート -### セットアップを実行(推奨) +### エージェントを設定する -REA をエージェントで使えるように設定します: +Node.js と npm がインストールされている環境で実行します。 ```bash npx rea-agents setup ``` -セットアップでは、まず REA を使うエージェントを選び(複数選択可)、具体的なパスと変更内容を確認してから承認します。既存の REA 登録は最初から選択されています。新しく検出されたエージェントは選択肢に表示されますが、検出されただけでは選択されません。選択したエージェントには、MCP 接続と REA のガイド付きワークフローが追加されます。Hopper のインストールは別の任意項目で、個別に承認が必要です。既存の Ghidra のインストール先を登録することもできます。 - -セットアップは変更を適用する前に内容を表示し、既存の設定をバックアップします。要件とセットアップのオプションは[インストールガイド](docs/installation.md)を参照してください。 - -### エージェントで使う(推奨) - -セットアップ後にエージェントを再起動し、理解したいアプリや機能を[説明してください](#エージェントに頼むだけ)。Hopper はデモモードでも動作します。初回起動の画面が表示されたら、デモを選ぶか既存のライセンスを入力してください。 +使うエージェントを選び、予定されている変更を確認して承認してください。セットアップは REA の MCP サーバーと対応するワークフローの手順を追加し、既存の設定をバックアップします。完了後にエージェントを再起動してください。 -REA は Claude Code、Claude Desktop、Codex、Cursor、Gemini CLI、Windsurf、Devin、OpenCode、Antigravity、GitHub Copilot CLI、Command Code、VS Code に対応しています。セットアップでは既存の REA 登録が最初から選択され、それ以外の検出済みエージェントは選択するまで対象になりません。その他のエージェントは[手動の MCP 設定](#他のコーディングエージェントで使う)を使えます。 +セットアップは Claude Code、Codex、Cursor、Gemini CLI、および[その他のエージェント](docs/installation.md#supported-agents)に対応しています。プロバイダーの設定と MCP の手動登録については、[インストールとセットアップ](docs/installation.md)を参照してください。 -### AI コーディングアシスタント向けスキル(任意) +### エージェントに依頼する -AI コーディングアシスタントにスキルを追加すると、より豊富なコンテキストを利用できます: - -```bash -npx skills add morluto/rea --skill reverse-engineer-anything +```text +Notes アプリの検索機能の仕組みを調べ、根拠を示して、私のプロジェクトにも同様の機能を実装してください。 ``` -このスキルは REA の調査ワークフローを提供します。REA をエージェントに接続して解析ツールを設定するには、上記のセットアップを実行してください。セットアップは既定でバージョンの一致したスキルをインストールします。このコマンドはリポジトリ版のスキルをインストールします。 +Notes を対象のアプリに置き換え、理解したい機能を指定してください。 -### ターミナルで最初の結果を得る +### ターミナルを使う -展開済みの JavaScript/Electron アプリのディレクトリまたは ASAR に対して、次を実行します: +展開済みの JavaScript/Electron アプリのディレクトリ、または ASAR を調べます。 ```bash npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json ``` -パスは調査対象に置き換えてください(Windows では `"D:/apps/example"` など)。MCP の設定、Hopper、Ghidra を必要とせず、アプリを実行することもなく、Evidence、復元したグラフ、制限、未解決事項をその場で返します。ネイティブアプリの場合は、先にエンジンを設定してから、そのアプリのパスを指定して `analyze` を使います。`doctor` は問題を診断したいときに使うもので、解析のたびに実行する必要はありません。 - -### 作業に合わせて準備状況を確認する +結果にはモジュール、インポート、Electron の境界と、それらの根拠が含まれます。パスは対象に合わせて変更してください。Windows では、たとえば `"D:/apps/example"` を指定します。 -オプションなしの `rea doctor` は連携全体の監査です。検出したすべてのエージェント登録、インストール済みのスキル、すべての任意の解析エンジンを確認するため、今の作業には支障がなくても `healthy: false` を報告することがあります。作業に合わせて確認範囲を指定してください: +日常的に使う場合は、`rea` コマンドをインストールできます。 -| 作業 | 準備状況の確認 | -| ------------------------------------------ | ---------------------------------------------------------------------------------------------- | -| 静的な JavaScript/Electron 解析 | 不要です。`analyze-javascript-application` を直接実行します。 | -| 特定の解析エンジンのトラブルシューティング | `rea doctor --provider ghidra --json`(または `hopper`、`ida`) | -| 特定のエージェントの MCP 登録を確認 | `rea doctor --client codex --json`([クライアント ID](docs/installation.md#supported-agents)) | -| インストール済みのスキルを確認 | `rea doctor --skill --json` | +```bash +npm install --global rea-agents +rea --help +``` -範囲を指定したレポートでは、`healthy` と終了ステータスを決めるのは `scope_checks` だけです。それ以外の項目は `informational_checks` に表示され、今の作業のために修正する必要はありません。たとえば、使っていないエンジンが見つからなくても対処は不要です。 +ネイティブ分析には、先にプロバイダーの設定が必要です。ネイティブ分析のコマンド、プロバイダーの選択、スナップショット、スクリプトでの利用は、[CLI と Evidence のガイド](docs/cli.md)を参照してください。 -ネイティブ対象に対応するエンジンが複数インストールされている場合、REA は自動ではどれも選びません。対象を開くと `code: "capability_unavailable"` と `details.selection_reason: "ambiguous"` で失敗し、`details.candidate_ids` に候補が示されます。CLI の `--provider` か `open_binary` の `provider_id` で一度選ぶか、常用する設定として `REA_ANALYSIS_PROVIDER` を指定してください。詳しくは[英語版の解説](README.md#choosing-a-deep-analysis-provider)を参照してください。 +### REA を更新する -### rea コマンドをインストール +REA は更新が速く、新しいリリースには頻繁にバグ修正が含まれます。インストールしたバージョンを最新に保ってください。 -コマンドラインツールをインストールします: +npm でインストールした CLI の場合: ```bash -curl -fsSL https://raw.githubusercontent.com/morluto/rea/main/install.sh | bash +rea update ``` -Node.js と npm を先にインストールしておいてください。ターミナルで実行すると、インストーラーは `rea` を追加してセットアップを開始します。 +エージェントの登録とスキルを更新するには、更新処理が表示するセットアップコマンドを実行してください。 -npm でインストールしてから、セットアップを実行することもできます: +`npx` を使っている場合は、次のコマンドでエージェントの設定を更新します。 ```bash -npm install --global rea-agents -rea setup +npx rea-agents@latest setup ``` -### 要件 +設定の変更を確認し、エージェントを再起動してください。単発の CLI 操作では、`npx rea-agents@latest` に続けてコマンドを指定します。 -静的な JavaScript 解析に必要なのは Node.js と npm だけです。ホストと外部ツールの前提条件は使うワークフローによって異なり、ネイティブ解析プロバイダーの対応プラットフォームは各ガイドで説明しています。 +## REA の仕組み -- macOS 12 以降 -- Ubuntu 24.04+、Fedora 41+、64 ビット Arch Linux、または CachyOS -- Node.js 22.x (>=22.19)、24.x (>=24.11)、または 26+ -- npm(特定のバージョンは不要で、REA が npm をインストールすることもありません) +エージェントは MCP 経由で REA を呼び出し、対象を調べて関連するコードを追跡します。REA は調査結果と根拠を返します。エージェントはそれを使って追加の質問をしたり、動作を説明したり、実装を書いてテストしたりします。CLI コマンドも同じワークフローを使います。 -ネイティブバイナリの詳細な解析には Hopper、Ghidra、IDA Pro のいずれかが必要です。Hopper は独自のライセンスを持つ別製品で、デモ版でもベンダーが定める制限の範囲で解析できます。Ghidra と IDA は、利用者が用意したものを使います。 +![REA の調査フロー:エージェントがローカルの対象について質問し、REA が分析ツールで調査・追跡します。エージェントは返されたコード、参照、未解明の点を使い、説明、実装、テストを行います。](website/public/assets/figures/rea-investigation-flow.svg) -Ghidra は Linux x64 と macOS x64/arm64 に対応しています。Ghidra 12.1.x と、そのインストールが宣言する 64 ビットの完全な JDK(`application.java.min` から `application.java.max` まで)を別途インストールし、REA で使うように設定してください。現行の 12.1 リリースは JDK 21 以降を必要とし、上限はありません。ブリッジは Ghidra 12.1.4 と JDK 21 で検証しています。macOS では、ホストのアーキテクチャに合うネイティブデコンパイラーも必要です。 +[原寸大の図を開く](website/public/assets/figures/rea-investigation-flow.svg)。 -```bash -export GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra_12.1.4_PUBLIC -export JAVA_HOME=/absolute/path/to/jdk-21 # java と javac が PATH から解決できる場合は省略可 -rea doctor --provider ghidra --json -rea setup -``` +
-セットアップはこれらのインストールを確認し、承認後にパスを保存します。Ghidra、Java、Node.js、npm、Homebrew のインストールや更新は行いません。 +## 分析できる対象 -IDA Pro は、すでに動作している [mrexodia/ida-pro-mcp](https://github.com/mrexodia/ida-pro-mcp) の MCP 登録を再利用します。開いている GUI の対象を読み取り専用で解析するか(`attached`、既定)、データベースのスーパーバイザーで指定したバイナリをヘッドレスで開いて解析します(`headless`)。REA が attached の GUI データベースを保存したり閉じたりすることはありません。最初の実環境での検証は、Windows の GUI と Windows x64 のヘッドレス IDA 9.3 が対象です。設定方法と対応範囲は [IDA プロバイダーガイド](docs/ida-provider.md)を参照してください。 +REA には Node.js 22.x(>=22.19)、24.x(>=24.11)、または 26+ と npm が必要です。追加のツールや対応ホストは、対象によって異なります。 -```bash -export REA_IDA_MCP_CONFIG=/absolute/path/to/ida-mcp.json -rea function /absolute/path/to/program main --provider ida --json -``` +| 対象 | REA が返すもの | 要件とガイド | +| -------------------------------- | -------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| ネイティブバイナリ | 疑似コード、アセンブリ、文字列、シンボル、呼び出し、参照 | Hopper、Ghidra、または IDA;[ネイティブ分析](https://rea.tools/guides/native/) | +| オフライン ELF レイアウト | セクション、セグメント、元のシンボル/再配置情報、静的解析によるセキュリティ対策の候補 | Linux x64 で利用者が用意する pwntools;[バイナリ診断](docs/binary-diagnostics.md) | +| EVM バイトコード | ディスパッチセレクター、バイトオフセット、推定された引数と状態変更の可否 | ローカルの生バイト/16 進数の入力ファイル;[オフライン EVM ガイド](docs/evm-bytecode.md) | +| 記録済みの Linux クラッシュ | 生の note レコード、記録された全スレッドのレジスター/シグナル、任意のマッピング候補 | 利用者が用意する pwntools;GDB/pwndbg は任意;[記録済みクラッシュ](docs/recorded-crashes.md) | +| JavaScript / Electron | モジュール、インポート、ソースマップ、ルート、IPC、ネイティブアドオンの関係 | Node.js と npm;[アプリケーション分析](https://rea.tools/guides/javascript/) | +| ウェブサイト | ページ構造、スクリプト、ネットワークの観測結果、依頼されたスクリーンショット | Chrome 系ブラウザー;[ブラウザー分析](https://rea.tools/guides/browser/) | +| 保存済みのネットワークキャプチャ | リクエスト、レスポンス、取得可能なペイロード、ソース内の位置 | HAR;mitmproxy ネイティブ形式のキャプチャには Linux 上の mitmdump;[キャプチャガイド](docs/web-network-captures.md) | +| .NET アセンブリ | メタデータ、CIL 命令、宣言されたネイティブ依存関係、ビルドの比較 | 静的な検査;[マネージドコードのガイド](docs/managed-code-analysis.md) | +| Android APK | マニフェストの宣言、クラス、逆コンパイルされたメソッド、参照 | Linux/macOS 上のヘッドレス JADX と完全な JDK;[Android ガイド](docs/android-analysis.md) | +| ファームウェア | 領域、抽出結果、ネイティブ分析への引き継ぎ | Linux 上の Binwalk / Unblob;[ファームウェアガイド](docs/firmware-analysis.md) | +| パッケージとリソース | ファイル一覧、ダイジェスト、plist、Apple バンドルの構造、抽出されたリソース | [アーティファクトと JavaScript のガイド](docs/javascript-artifact-reconstruction.md)、[Apple アプリケーション](docs/apple-application-analysis.md) | +| プロセスの動作 | ターミナル出力、対話、終了とファイルシステムの観測結果、実行の比較 | ネイティブ PTY を備えた Linux/macOS;[プロセスキャプチャ](docs/process-capture.md) | -リポジトリの main と npm 4.1.0 には、ローカル NTFS 上のネイティブ x86-64 PE アプリケーション(マネージドコードと DLL を除く)を対象とする、実験的な Windows x64 Ghidra P0 対応が含まれています。Job Object、プライベート DACL、パスの受け入れ検査の各制御も同梱されています。古い npm パッケージでこの機能を使えるか確認するには[リリース境界](docs/installation.md#released-package-and-main)を、前提条件と検証済みの範囲は [Windows Ghidra P0](docs/windows-ghidra-p0.md) を参照してください。 +静的な JavaScript と .NET の検査は、指定されたファイルを読み取り、アプリケーションを実行しません。実行時キャプチャは、あなたのユーザー権限で対象を実行したり、対象とやり取りしたりします。それぞれの実行時ガイドに、どのような操作が行われるかを記載しています。 -### トラブルシューティング + -`npx -y rea-agents@latest doctor` は、ホスト、依存関係、解析ツール、エージェント設定を変更せずに確認します。構造化された診断結果が必要な場合は `--json` を追加してください。 +対応するネイティブ形式とホストは、プロバイダーによって異なります。[Hopper と Ghidra のセットアップ](docs/installation.md#hopper)、[IDA ガイド](docs/ida-provider.md)、[Windows 向け Ghidra の実験的対応](docs/windows-ghidra-p0.md)を参照してください。Ghidra は [16 ビット DOS 分析](docs/ghidra-dos.md)にも対応しています。プロバイダーの選択は [CLI ガイド](docs/cli.md#choose-a-provider)を参照してください。最新の npm リリース以降に追加された機能については、[リリースでの利用可否](docs/installation.md#released-package-and-main)を確認してください。 -Linux では実行可能な `/opt/hopper/bin/Hopper` を優先し、利用できない場合は `~/.local/share/rea/hopper/bin/Hopper` を自動的に確認します。それ以外の場所にインストールした場合は `HOPPER_LAUNCHER_PATH` を設定してください。ファイルがあるのに解析エンジンが見つからないと報告される場合は、実際の Hopper のパスに対して `ldd /absolute/path/to/Hopper | grep 'not found'` を実行し、不足しているライブラリを確認してください。詳しくは [Hopper ガイド](docs/installation.md#hopper)を参照してください。 +## 事例 -### 更新とアンインストール +### DX-Ball:音声のパン計算を再構築する -- `rea update` は現在の REA インストールを更新します。 -- `rea uninstall` は REA が管理するエージェント登録とスキルだけを削除します。Hopper、Node.js、Evidence ファイル、キャプチャ、無関係なスキル、他の MCP サーバーは残ります。 -- `rea uninstall --purge-data` は `~/.rea/cache` と `~/.rea/state` も削除します。それらを消したい場合にだけ使ってください。 +音声の呼び出しから位置をパンに変換する補助関数をたどり、命令を調べ、不完全な疑似コードを C に書き直します。再構築した実装は、元の x86 に対する 3,205 ケースの検証を通過し、コンパイル後の関数の全 63 バイトを再現しています。 -## 現在の対応状況 +[事例を読む](https://rea.tools/showcase/dx-ball/) · +[再構築リポジトリ](https://github.com/N0zoM1z0/dx-ball) -リポジトリの現在の機能とプラットフォーム要件は[英語の対応状況ガイド](README.md#current-status)で説明しています。main は [npm リリース](docs/installation.md#released-package-and-main)より先行する場合があります。 +### Notion:Electron のクリップボードブリッジを追跡する -- Ghidra は Linux x64、macOS x64/arm64、実験的な Windows x64 P0 境界で 25 種類の読み取り専用操作を提供します。Linux と macOS では、セッション内での関数名とエントリーコメントのアトミックな編集にも対応します。Windows P0 には変更の権限がなく、Ghidra には REA から操作できる GUI 機能はありません。 -- IDA アダプターは、ドキュメントに記載された読み取り専用の関数・文字列操作を提供します。 -- 静的な Android APK の検査は Linux と macOS に対応し、別途用意したヘッドレス JADX と完全な JDK が必要です。現在のメタデータブリッジは macOS arm64 で検証しています。詳しくは [Android 解析](docs/android-analysis.md)を参照してください。 -- ブラウザー、Electron、プロセスの各リクエストは対象、操作、ライフサイクルを直接指定し、ホストで実際に許可されたアクセスの範囲で動作します。REA 側で別途権限を付与する必要はありません。セットアップによる設定の書き込みと Hopper のインストールには、引き続き具体的な計画の承認が必要です。 -- `rea capabilities` と `rea providers` はバイナリセッションのプロバイダーと補助操作を説明するもので、ブラウザー、Android、アプリのすべてのワークフローの一覧ではありません。MCP の機能全体は、接続中の MCP ツール一覧と `binary_session` が示すツールの利用可否で確認してください。 +レンダラーのクリップボード API を見つけ、preload と IPC を経由してメインプロセスまでたどり、リッチなクリップボード形式を調べます。 -## ひとつのプロンプトで調査を完結 +[事例を読む](https://rea.tools/showcase/notion/) -```text -メモアプリをリバースエンジニアリングし、オフライン検索機能の仕組みを説明して、 -TypeScript と SQLite を使って私のプロジェクト向けに実装してください。 -``` +### TH04:DOS の弾幕リングの計算を復元する -| 手順 | エージェントの処理 | REA ツール | -| ---: | ------------------------------ | ---------------------------------------------------------------- | -| 1 | バイナリを開いて識別 | `open_binary`, `binary_overview` | -| 2 | オフライン検索の手掛かりを探す | `search_strings`, `search_procedures`, `list_names` | -| 3 | 手掛かりを実行コードに結び付け | `find_xrefs_to_name`, `xrefs`, `procedure_callers` | -| 4 | 関連する制御フローを復元 | `get_call_graph`, `procedure_callees`, `procedure_info` | -| 5 | 関連する処理を逆コンパイル | `procedure_pseudo_code`, `procedure_assembly`, `batch_decompile` | -| 6 | プロジェクトに機能を実装 | 技術スタック、プロダクト、要件に合わせたコード | - -手順 1〜5 のアプリ解析は REA が担当します。手順 6 は、アプリについて得た知見をもとに、エージェントが通常の編集ツールとテストツールで行います。 - -## エージェントにできること - -- 気に入った機能を調査し、自分のプロダクトに合わせた形で実装する。 -- ソースコードが手に入らない機能の仕組みを説明する。 -- アプリの認証、保存、更新、ネットワーク処理の流れを復元する。 -- 文書化されていない形式やインターフェースについて、文書化に必要な構造を復元する。 -- 文字列やシンボルから、疑わしい動作を実装しているコードまでたどる。 -- 復元した動作を、プロダクト機能、テスト、移行メモ、移植版、相互運用できる代替実装に変える。 -- マングルされたシンボルをひとつずつ手作業で読み解かなくても、Swift と Objective-C のメタデータを解析する。 -- Hopper に名前、コメント、ブックマークを残し、人とエージェントの解析を互いに活かす。 - -## 調査ツールカタログ - -| ツール分類 | 数 | 用途 | -| -------------------- | --: | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ネイティブ検査 | 40 | 関数、疑似コード、アセンブリ、文字列、シンボル、呼び出し、参照、注釈、バイト読み取り、ファイルオフセット | -| 調査ワークフロー | 14 | アプリ概要、関数ドシエ、ネイティブ API とディスパッチ、一括逆コンパイル、機能トレース、コールパス、コールグラフ、Swift と Objective-C の検出 | -| ネイティブバイナリ | 10 | macOS の Mach-O メタデータ、署名、plist、アーキテクチャ、Swift シンボル、LLDB による呼び出し観測;Linux の ELF レイアウト、静的な保護機構の証拠と記録済みクラッシュ | -| 成果物グラフ | 7 | ディレクトリとパッケージの目録、コンパイル済み Interface Builder ファイル、Apple アセットカタログ、Mach-O dylib のロード解決、抽出、オフライン EVM インターフェース推定 | -| マネージド PE/CLI | 7 | .NET の識別情報、メタデータ、CIL 命令、ネイティブ依存関係、再構築結果のインポート、ビルド比較 | -| ファームウェア | 2 | Linux ファームウェア領域の検査と明示的な展開 | -| Android APK | 5 | パッケージと manifest の宣言、クラス検索、メンバー一覧、メソッドの逆コンパイル、静的な被参照 | -| ブラウザー観察 | 12 | ページ構成、ネットワークメタデータ、スクリプト、ソースマップ、WebMCP の検出、スクリーンショット、キャプチャ比較 | -| Electron 解析 | 5 | レンダラーの観察、静的なアプリのマッピング、静的結果と実行時結果の照合 | -| JavaScript 実行時 | 2 | Node/Electron Inspector の対象検出、スクリプトの位置、実行コンテキストのイベント | -| アプリワークフロー | 13 | 取得した Web スクリプトのエクスポート、Android/Apple の目録の投影、層をまたぐ機能トレース、ビルド比較、過去のソースとの対応付け、静的な戻り値形状の比較、再構築の検証 | -| ワークスペースと観察 | 21 | セッション、Evidence バンドル、ナビゲーションの文脈、プロセス・成果物・関数の比較、未解決事項の記録 | - -## ロードマップ - -[現在の対応状況](#現在の対応状況)は提供済みの機能です。今後の作業は次のとおりです。 - -- **現在:** ツール、プロバイダー、セットアップの選択肢、バージョンの変更に合わせてドキュメントを正確に保ち、Hopper と Ghidra で検証するネイティブバイナリのアーキテクチャと間接呼び出しの範囲を広げます。 -- **次:** 機能トレースに静的抽出と実行時の観察を追加し、難読化された .NET アセンブリの比較を改善し、プロセス、プロトコル、ファイルシステム、再接続、バージョン比較の範囲を広げます。 -- **その後:** ブラウザーと Electron のシナリオ操作を増やし、LLDB や Frida などによるネイティブアプリの実行時観察と、他の解析ツールや対象の評価を進めます。 - -セットアップでは、すでにエージェント連携と Hopper のインストールを選べます。その他の解析ツールのインストールへの対応は今後の作業です。[インストールのロードマップ](docs/roadmap.md)と[解析ツールの評価](docs/provider-evaluation.md)を参照してください。 - -## 他のコーディングエージェントで使う - -セットアップは Claude Code、Claude Desktop、Codex、Cursor、Gemini CLI、Windsurf、Devin、OpenCode、Antigravity、GitHub Copilot CLI、Command Code、VS Code に対応しています。既存の REA 登録は最初から選択され、それ以外の検出済みエージェントは選択するまで対象になりません。ローカル MCP サーバーに対応するエージェントであれば、次の設定でも接続できます。 - - - -```json -{ - "mcpServers": { - "rea": { - "command": "npx", - "args": ["-y", "rea-agents@6.0.0", "mcp"] - } - } -} -``` +元の PC-98 ゲームの 16 ビット命令を調べ、固定角度と自機を狙う角度の計算を復元し、再構築した C++ を当時のコンパイラーの出力と比較します。 - +[事例を読む](https://rea.tools/showcase/th04/) · +[再構築リポジトリ](https://github.com/N0zoM1z0/th04) -常用する登録では、パッケージのバージョンを 1 つに固定してください。`rea setup` はその固定バージョンを維持し、同梱のスキルも同時に更新します。 +REA で興味深い対象を調べたら、ぜひ共有してください。[issue](https://github.com/morluto/rea/issues) または [pull request](https://github.com/morluto/rea/pulls) に、対象、調べたかったこと、REA がどう役立ったか、何が分かったかを記載してください。 -## 仕組み +## よくある質問 -```mermaid -flowchart LR - Agent["コーディングエージェント"] --> REA["REA
CLI + MCP"] - Terminal["ターミナル"] --> REA - REA --> Hopper["Hopper プロバイダー"] - REA --> Ghidra["Ghidra プロバイダー
インベントリ + 関数解析 + 注釈"] - REA --> Ida["IDA MCP プロバイダー
attached GUI または headless データベース"] - Hopper --> App["あなたのアプリ"] - Ghidra --> App - Ida --> App -``` +
+どのエージェントで REA を使えますか? -CLI と MCP サーバーは、同じアプリケーションワークフローと Evidence の契約を使います。ターミナルのコマンドは完了すると自身のブリッジセッションを解放し、エージェントのセッションは調査中の対象と Evidence を保持できます。REA のセッションを閉じても、利用中の Hopper アプリは終了しません。 +ローカル MCP サーバーに対応するエージェントなら使えます。セットアップは[対応エージェント](docs/installation.md#supported-agents)を設定します。その他のクライアントでは、[MCP の手動登録](docs/installation.md#mcp-registry)を利用できます。 -## CLI +
-上のエージェントワークフローが、REA を使う最も簡単な方法です。ターミナルからアプリの概要を一度だけ調べる場合は、次を実行します。 +
+Hopper、Ghidra、IDA は必要ですか? -```bash -npx -y rea-agents@latest analyze /Applications/Notes.app -``` +深いネイティブ分析には、そのいずれかを使います。静的な JavaScript と .NET の検査には、ネイティブ分析エンジンは必要ありません。セットアップは承認後に Hopper をインストールできます。Ghidra と IDA は既存のインストールを使います。[プロバイダーのセットアップ](docs/installation.md#hopper)を参照してください。 -直接の逆コンパイルやその他のオプションは、`npx -y rea-agents@latest --help` で確認できます。 +
-グローバルな `rea` コマンドとしてもインストールできます。 +
+先に Hopper を起動する必要はありますか? -```bash -npm install --global rea-agents -rea --help -rea mcp -``` +REA は必要な操作の際に Hopper を起動します。macOS では、初回起動時にデモモードを選ぶか、ライセンスを有効化するよう求められることがあります。[Hopper の起動とトラブルシューティング](docs/installation.md#launcher-paths-and-troubleshooting)を参照してください。 -REA は Mac の `.app` フォルダーを直接開けます。エージェントがアプリを名前で見つけられない場合は、インストール場所を伝えてください。 - -## Hopper アプリの動作 +
-REA は操作に必要になった時点で Hopper を起動します。Hopper のランチャーは内部でアプリをアクティブにするため、REA がバックグラウンドでの起動を要求しても、macOS では Hopper のウィンドウやダイアログが前面に出る場合があります。デモやライセンスの画面では操作が必要になることがあります。 +
+skills.sh からスキルをインストールすると何ができますか? -Hopper は解析リクエストを一度に 1 つずつ処理します。待機をキャンセルしても、Hopper 内ですでに実行中の処理は止まりません。処理中かどうかはセッションが報告します。セッションを閉じると REA のブリッジと一時ソケットディレクトリを削除しますが、利用中の Hopper アプリは終了しません。後片付けを確認できなかった場合、`close_binary` は `cleanup_incomplete` と対象のリソースを報告します。 +スキルはエージェントに調査の手順を提供します。`rea setup` で REA の MCP サーバーを登録し、対応する手順をインストールしてから、エージェントを再起動してください。[スキルのみのインストール](docs/installation.md#skill-only-installation)を参照してください。 -## セキュリティモデル +
-解析はローカルで実行されます。REA は Hopper や Ghidra とは認証付きのプライベートなローカルソケットで、IDA とは設定済みのローカル MCP 登録を通じて通信します。Ghidra セッションは隔離された一時プロジェクトを使い、ユーザーの Ghidra プロジェクトを開いたり変更したりしません。 +
+REA はどのようなコードを返しますか? -これはサンドボックスではありません。解析ツールと起動した対象は現在のユーザー権限で動作し、同じ OS ユーザー権限で動く悪意のあるプロセスは防げません。Windows Ghidra P0 では、ネイティブの Job Object、保護されたプライベート DACL、ハンドルベースの受け入れ検査を自動的に使います。脆弱性は [SECURITY.md](SECURITY.md) に記載された非公開の手順で報告してください。 +ネイティブ分析は疑似コードとアセンブリを返します。JavaScript/Electron 分析はモジュールとその関係を復元します。エージェントはその結果を使って実装を書き、テストします。[事例](#事例)に具体的な手順があります。 -## FAQ +
-
Hopper を先に起動しておく必要はありますか? +
+REA はアプリをアップロードしますか? -いいえ。REA は必要になった時点で Hopper を起動します。すでに起動している Hopper でも使えますが、既存の GUI ドキュメントには接続せず、新しい解析ドキュメントを開きます。 +REA は対象をローカルで分析します。エージェントはツールの結果を受け取り、そのモデルプロバイダーには独自のデータポリシーがあります。
-
REA に Hopper は含まれていますか? +
+バグに遭遇したらどうすればよいですか? -含まれていません。セットアップで Hopper をインストールできますが、Hopper は独自のライセンスを持つ別製品です。REA は、エージェントが Hopper を使えるようにする CLI、MCP サーバー、ワークフローを提供します。 +まず更新してください。最近のリリースですでに修正されている可能性があります。 -
+npm でインストールした CLI の場合: + +```bash +rea update +``` -
REA は Ghidra や Java をインストールしたり同梱したりしますか? +`npx` でエージェントを設定している場合: -いいえ。REA は既存の Ghidra のインストールに接続します。[要件](#要件)にある Ghidra と Java のパスを設定してから、セットアップを実行してください。 +```bash +npx rea-agents@latest setup +``` + +エージェントを使っている場合は、[設定の更新](#rea-を更新する)を完了し、再起動してください。同じタスクをもう一度試し、問題が続く場合は [issue を作成](https://github.com/morluto/rea/issues)して、REA のバージョン、対象の種類、再現手順、エラー出力を記載してください。
-
アプリはアップロードされますか? +## ドキュメント -REA にホスト型の解析サービスはありません。現在のプロバイダーは、成果物の解析と動作のキャプチャをローカルで行います。エージェントやモデルの提供元には独自のデータポリシーがあるため、別途確認してください。 +まずはウェブサイトの[実践ガイド](https://rea.tools/guides/)を参照してください。正確なオプション、前提条件、結果の契約については、以下を参照してください。 -
+- [インストールとセットアップ](docs/installation.md):エージェントの登録、プロバイダーの設定、更新、アンインストール。 +- [準備状況の確認とトラブルシューティング](docs/installation.md#check-readiness-for-your-task):特定のエージェントや分析エンジンを診断。 +- [CLI と Evidence](docs/cli.md):コマンド、プロバイダーの選択、スナップショット、インポート/エクスポート、終了ステータス。 +- [MCP 契約](docs/mcp-contracts.md)と[エージェント向けプロンプト](docs/mcp-prompts.md):ツールの結果、セッション、ガイド付き調査。 +- [ツールカタログ](docs/mcp-contracts.md#generated-catalog):ビルド時に生成されるツール、プロバイダー、CLI コマンドの一覧。 +- [ロードマップ](docs/roadmap.md):計画中の作業と機能の進捗。 -
元のソースコードを復元できますか? +脆弱性の報告は [SECURITY.md](SECURITY.md) に従ってください。 -元のソースコードを確実に復元できるデコンパイラーはありません。REA はエージェントに疑似コード、アセンブリ、シンボル、文字列、メタデータ、それらの関係を提供し、観察された動作の説明や互換性のある再現に役立てられるようにします。 +## 貢献する -
+REA への貢献を歓迎します![issue を作成](https://github.com/morluto/rea/issues)してバグや機能の提案を報告したり、[pull request を送信](https://github.com/morluto/rea/pulls)してコードやドキュメントを改善したりできます。 + +開発環境とチェックは [CONTRIBUTING.md](CONTRIBUTING.md)、検証レーンは[テストガイド](docs/testing.md)、プロジェクト構造は[アーキテクチャ図](docs/architecture.mermaid)を参照してください。 -## 開発 +## プロジェクトリンク -開発環境、アーキテクチャ、テスト、リリース手順は [CONTRIBUTING.md](CONTRIBUTING.md) を参照してください。 +[ウェブサイト](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [セキュリティ](SECURITY.md) ## スター履歴 -🎉 **GitHub スターが 20,000 を突破しました。ありがとうございます!** +🎉 **GitHub スター 20,000 件、ありがとうございます!** -REA を使い、問題を報告し、ビルドをテストし、修正を届けてくださる皆さんに感謝します。 +REA の利用、バグ報告、ビルドのテスト、修正への貢献に感謝します。 @@ -380,6 +278,10 @@ REA を使い、問題を報告し、ビルドをテストし、修正を届け +## 免責事項 + +REA は、合法的なリバースエンジニアリングの研究、分析、再構築のためのツールを提供します。必要な許可を得て、適用される法律に従う責任は利用者にあります。このプロジェクトは違法または無許可の利用を支持しません。 + ## ライセンス [MIT](LICENSE) diff --git a/README_ko.md b/README_ko.md index 5df5089cacd..9f6787042c3 100644 --- a/README_ko.md +++ b/README_ko.md @@ -4,322 +4,271 @@ # REA: 무엇이든 리버스 엔지니어링 -### 에이전트로 앱의 동작부터 네이티브 바이너리까지 조사하세요. +### 하나의 MCP로 바이너리, 애플리케이션, 런타임 동작을 리버스 엔지니어링합니다. -**마음에 드는 기능을 찾고, 작동 방식을 이해하고, 원하는 방식으로 구현하세요.** +**마음에 드는 기능을 발견했다면, 바이너리 수준까지 작동 원리를 이해하세요.** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](#조사-도구-카탈로그) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) morluto%2Frea | Trendshift -**[웹사이트(영문)](https://morluto.github.io/rea/) · [가이드](https://morluto.github.io/rea/guides/) · [DX-Ball 사례](https://morluto.github.io/rea/showcase/dx-ball/)** +**[웹사이트](https://rea.tools/) · [가이드](https://rea.tools/guides/) · [사례](https://rea.tools/showcase/)** -[빠른 시작](#빠른-시작) · [현재 지원 범위](#현재-지원-범위) · [바이너리에서 동작까지](#바이너리에서-동작까지) · [조사 도구 카탈로그](#조사-도구-카탈로그) · [로드맵](#로드맵) · [작동 방식](#작동-방식) +[빠른 시작](#빠른-시작) · [REA의 작동 방식](#rea의-작동-방식) · [분석할 수 있는 대상](#분석할-수-있는-대상) · [사례](#사례) · [자주 묻는 질문](#자주-묻는-질문) · [문서](#문서) - +npx rea-agents setup + +
+ +REA가 Hopper 안에서 분석 브리지를 실행해 네이티브 바이너리를 검사하는 모습 + +
+ +
Discord
리버스 엔지니어링 커뮤니티에 참여하세요

- Discord · 질문과 답변 · 작업 공유 + Discord · 질문과 답변 · 결과 공유

-npx rea-agents setup - -
- -Hopper에서 분석 브리지를 시작하고 네이티브 바이너리를 조사하는 REA - --- -앱에서 마음에 드는 기능을 찾았다면 에이전트에게 REA로 조사해 달라고 요청하세요. 소스 코드가 없어도 앱을 분석하고, 기능의 동작과 근거를 설명한 뒤, 프로젝트에 맞는 기능을 구현할 수 있습니다. +앱에서 내 제품에도 넣고 싶은 기능을 발견했나요? 에이전트에게 REA로 조사해 달라고 요청하세요. 소스 코드가 없어도 앱을 검사하고, 기능의 작동 원리를 설명하고, 근거를 보여 주고, 프로젝트에 맞는 기능을 구현할 수 있습니다. -REA는 네이티브 바이너리, JavaScript/Electron 앱, .NET 어셈블리, 웹사이트 분석 도구를 제공합니다. 에이전트와 터미널에서 같은 도구를 사용할 수 있습니다. 분석은 로컬에서 실행되며 결과에는 근거와 제한 사항이 포함됩니다. +REA는 에이전트를 분석 도구에 연결해 네이티브 바이너리, JavaScript와 Electron 앱, .NET 어셈블리, 웹사이트를 검사할 수 있게 합니다. 터미널에서도 같은 도구를 사용할 수 있습니다. 분석은 로컬에서 실행되며, 결과에는 각 결론의 근거와 한계가 포함됩니다. -Setup은 에이전트를 설정하고 기존 Hopper 또는 Ghidra 설치에 연결합니다. 승인 후 Hopper를 설치할 수도 있습니다. +설정 과정은 에이전트에 REA를 등록하고 해당 버전에 맞는 워크플로 지침을 설치합니다. 네이티브 분석에는 기존 Hopper 또는 Ghidra 설치를 사용할 수 있습니다. 승인하면 설정 과정에서 Hopper를 설치할 수도 있습니다. 정적 JavaScript 분석에는 두 엔진 모두 필요하지 않습니다. -## 에이전트에게 바로 요청하기 - -[설정](#빠른-시작)을 마친 뒤 에이전트를 다시 시작하고 요청하세요. - -```text -메모 앱의 검색 기능이 어떻게 동작하는지 조사하고 근거를 보여 주세요. -그다음 제 프로젝트에 비슷한 기능을 구현해 주세요. -``` - -메모 앱 대신 조사할 앱을 지정하거나 먼저 개요를 요청할 수 있습니다. - -## 바이너리에서 동작까지 - -| 디컴파일 | 이해 | 재현 | -| ------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | -| 네이티브 앱이나 실행 파일에서 프로시저, 의사 코드, 어셈블리, 문자열, 심볼, 세그먼트, 메타데이터를 복구합니다. | 호출자, 피호출자, 상호 참조, 호출 그래프를 따라 기능이나 알고리즘의 실제 동작을 설명합니다. | 에이전트가 배운 내용을 사용자의 기술 스택, 화면, 요구 사항에 맞는 제품 기능으로 만듭니다. | - -REA는 분석을 바이너리 증거에 근거하게 합니다. 원본 소스 코드를 복원하거나 앱 전체를 자동으로 복제한다고 주장하지 않습니다. - -## REA를 사용하는 이유 - -| | | -| ------------------- | --------------------------------------------------------------------------------------------------- | -| **에이전트용** | 앱의 동작에 대해 질문하고 추측 대신 에이전트가 조사하게 합니다. | -| **CLI와 MCP** | 터미널과 에이전트에서 동일한 리버스 엔지니어링 기능을 사용합니다. | -| **안내식 설정** | 에이전트 설정, 기존 분석 도구 연결, 승인 후 Hopper 설치를 지원합니다. | -| **통찰에서 코드로** | 기능을 이해한 뒤 같은 코딩 세션에서 자신의 제품에 맞는 버전을 구현합니다. | -| **로컬 분석** | 분석은 지원되는 로컬 호스트에서 실행되며 REA는 바이너리를 호스팅 분석 서비스에 업로드하지 않습니다. | -| **컨텍스트 유지** | 질문마다 분석을 처음부터 시작하지 않고 여러 앱을 연속으로 조사합니다. | +> **[REA 웹사이트](https://rea.tools/)**에서 설정 방법, 그림으로 설명한 가이드, 실제 사례를 확인하세요. ## 빠른 시작 -### 설정 실행하기(권장) +### 에이전트 설정 -REA를 에이전트에 연결하세요. +Node.js와 npm이 설치된 환경에서 다음 명령을 실행하세요. ```bash npx rea-agents setup ``` -Setup은 먼저 연결할 에이전트를 여러 개 선택하도록 안내합니다. 기존 REA 등록은 기본 선택되며, 감지만 된 클라이언트는 자동 선택되지 않습니다. 아직 설정되지 않은 클라이언트도 직접 선택할 수 있습니다. 정확한 경로와 변경 사항을 검토한 뒤 승인하세요. 선택한 에이전트에는 기본적으로 REA 워크플로가 설치됩니다. Hopper는 별도의 선택 사항이며 따로 승인해야 합니다. 기존 Ghidra 경로도 등록할 수 있습니다. +에이전트를 선택하고, 예정된 변경 사항을 검토한 뒤 승인하세요. 설정 과정은 기존 설정을 백업하고 REA의 MCP 서버와 버전에 맞는 워크플로 지침을 추가합니다. 완료 후 에이전트를 다시 시작하세요. -변경 전에 계획을 표시하고 기존 설정을 백업합니다. 요구 사항과 추가 옵션은 [설치 안내](docs/installation.md)를 참고하세요. +설정은 Claude Code, Codex, Cursor, Gemini CLI 및 [다른 에이전트](docs/installation.md#supported-agents)를 지원합니다. 제공자 설정과 수동 MCP 등록은 [설치 및 설정](docs/installation.md)을 참고하세요. -### 에이전트에서 사용하기 +### 에이전트에게 요청 -설정 후 에이전트를 다시 시작하고 조사할 앱이나 기능을 설명하세요. REA는 Claude Code, Claude Desktop, Codex, Cursor, Gemini CLI, Windsurf, Devin, OpenCode, Antigravity, GitHub Copilot CLI, Command Code, VS Code를 지원합니다. 기존 REA 등록은 기본 선택되고, 그 밖의 감지된 클라이언트는 직접 선택해야 합니다. 다른 에이전트는 아래 MCP 설정을 사용할 수 있습니다. +```text +Notes 앱의 검색 기능이 어떻게 작동하는지 조사하고, 근거를 보여 준 다음, 내 프로젝트에 비슷한 기능을 구현해 줘. +``` -Hopper는 데모 모드로 사용할 수 있습니다. 첫 실행 안내가 나오면 데모를 선택하거나 기존 라이선스를 입력하세요. +Notes를 대상 앱으로 바꾸고 이해하고 싶은 기능을 지정하세요. -### AI 코딩 어시스턴트용 스킬(선택) +### 터미널 사용 -AI 코딩 어시스턴트에 스킬을 추가하면 더 풍부한 컨텍스트를 얻을 수 있습니다. +압축을 푼 JavaScript/Electron 앱 디렉터리 또는 ASAR를 검사합니다. ```bash -npx skills add morluto/rea --skill reverse-engineer-anything +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json ``` -이 스킬은 REA의 조사 워크플로를 제공합니다. 위 Setup을 실행해 REA를 에이전트에 연결하고 분석 도구를 설정하세요. Setup은 기본적으로 버전에 맞는 스킬을 설치합니다. 이 명령은 저장소 버전의 스킬을 설치합니다. +결과에는 모듈, 임포트, Electron 경계와 관련 근거가 포함됩니다. 경로를 분석할 대상으로 바꾸세요. Windows에서는 `"D:/apps/example"` 같은 경로를 사용할 수 있습니다. -압축을 해제한 JavaScript/Electron 앱 트리나 ASAR라면 MCP 설정이나 네이티브 엔진 없이 바로 분석할 수 있습니다. +자주 사용할 경우 `rea` 명령을 설치하세요. ```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +npm install --global rea-agents +rea --help ``` -경로를 조사 대상의 경로로 바꾸세요(Windows에서는 `"D:/apps/example"` 같은 경로). 이 정적 워크플로에서는 이런 경로도 지원됩니다. 네이티브 분석에는 전용 엔진 설정이 필요합니다. +네이티브 분석에는 먼저 제공자를 설정해야 합니다. 네이티브 명령, 제공자 선택, 스냅샷, 스크립트 사용은 [CLI 및 Evidence 가이드](docs/cli.md)를 참고하세요. -### rea 명령 설치하기 +### REA 업데이트 -명령줄 도구를 설치하세요. +REA는 빠르게 바뀌며 새 릴리스에는 버그 수정이 자주 포함됩니다. 설치된 버전을 최신으로 유지하세요. + +npm으로 설치한 CLI에서는 다음 명령을 사용합니다. ```bash -curl -fsSL https://raw.githubusercontent.com/morluto/rea/main/install.sh | bash +rea update ``` -Node.js와 npm이 먼저 설치되어 있어야 합니다. 터미널에서 실행하면 설치 프로그램이 `rea`를 설치하고 설정을 시작합니다. +에이전트 등록과 스킬을 갱신하려면 업데이트 과정에서 출력된 설정 명령을 실행하세요. -npm으로 설치한 뒤 설정을 실행할 수도 있습니다. +`npx`를 사용한다면 다음 명령으로 에이전트 설정을 갱신하세요. ```bash -npm install --global rea-agents -rea setup +npx rea-agents@latest setup ``` -### 요구 사항 - -- macOS 12 이상 -- Ubuntu 24.04+, Fedora 41+, 64비트 Arch Linux 또는 CachyOS -- Node.js 22.x (>=22.19), 24.x (>=24.11) 또는 26+와 npm +설정 변경 사항을 검토하고 에이전트를 다시 시작하세요. 일회성 CLI 작업에는 `npx rea-agents@latest` 뒤에 필요한 명령을 붙여 실행하세요. -네이티브 바이너리 분석에는 Hopper 또는 Ghidra가 필요합니다. Hopper는 별도 소프트웨어입니다. 데모에는 공급업체의 제한이 있지만 유료 라이선스가 필수는 아닙니다. +## REA의 작동 방식 -Ghidra는 Linux x64와 macOS x64/arm64를 지원합니다. Ghidra 12.1.x와 그 설치본이 선언한 완전한 64비트 JDK(`application.java.min`부터 `application.java.max`까지)를 별도로 설치한 뒤 REA에서 사용하도록 설정하세요. 현재 12.1 릴리스는 JDK 21 이상을 요구하고 상한은 없습니다. 브리지는 Ghidra 12.1.4와 JDK 21에서 검증됩니다. macOS에서는 호스트 아키텍처에 맞는 네이티브 디컴파일러도 필요합니다. +에이전트는 MCP를 통해 REA를 호출하여 대상을 검사하고 관련 코드를 추적합니다. REA는 조사 결과와 근거를 반환합니다. 에이전트는 이를 바탕으로 추가 질문을 하고, 동작을 설명하거나, 구현을 작성하고 테스트합니다. CLI 명령도 같은 워크플로를 사용합니다. -Setup은 설치를 확인하고 경로를 저장할 수 있습니다. Ghidra, Java, Node.js, npm, Homebrew를 설치하거나 업데이트하지 않습니다. +![REA 조사 흐름: 에이전트가 로컬 대상에 대해 질문하면 REA가 분석 도구로 대상을 검사하고 추적합니다. 에이전트는 반환된 코드, 참조, 확인되지 않은 사항을 이용해 설명하고 구현하고 테스트합니다.](website/public/assets/figures/rea-investigation-flow.svg) -저장소 main과 npm 4.1.0에는 로컬 NTFS상의 네이티브 x86-64 PE 애플리케이션(비관리, 비 DLL 대상)을 위한 실험적인 Windows x64 Ghidra P0 지원이 포함되어 있습니다. Job Object, 비공개 DACL, 경로 승인 제어가 함께 제공됩니다. 이전 npm 패키지에 이 기능이 있을 것으로 기대하기 전에 [릴리스 경계](docs/installation.md#released-package-and-main)를 확인하세요. 전제 조건과 검증 범위는 [Windows Ghidra P0](docs/windows-ghidra-p0.md)를 참고하세요. +[원본 크기의 그림 열기](website/public/assets/figures/rea-investigation-flow.svg). -### 문제 해결 + -`npx -y rea-agents@latest doctor`는 호스트, 의존성, 분석 도구, 에이전트 설정을 변경 없이 확인합니다. 구조화된 진단에는 `--json`을 추가하세요. +## 분석할 수 있는 대상 -Linux에서는 실행 가능한 `/opt/hopper/bin/Hopper`를 우선 사용하고, 사용할 수 없으면 `~/.local/share/rea/hopper/bin/Hopper`를 자동으로 확인합니다. 다른 위치에는 `HOPPER_LAUNCHER_PATH`를 설정하세요. 파일이 있는데도 분석 엔진이 없다고 하면 실제 Hopper 경로에 대해 `ldd /absolute/path/to/Hopper | grep 'not found'`를 실행해 누락된 라이브러리를 확인하세요. 자세한 내용은 [Hopper 안내](docs/installation.md#hopper)를 참고하세요. +REA에는 Node.js 22.x(>=22.19), 24.x(>=24.11) 또는 26+와 npm이 필요합니다. 추가 도구와 호스트 지원은 대상에 따라 다릅니다. -### 업데이트와 제거 +| 대상 | REA가 반환하는 내용 | 요구 사항 및 가이드 | +| --------------------- | ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| 네이티브 바이너리 | 의사 코드, 어셈블리, 문자열, 심볼, 호출, 참조 | Hopper, Ghidra 또는 IDA; [네이티브 분석](https://rea.tools/guides/native/) | +| 오프라인 ELF 레이아웃 | 섹션, 세그먼트, 원본 심볼/재배치 정보, 정적 분석으로 얻은 보안 완화 기법 후보 | Linux x64에서 호출자가 제공하는 pwntools; [바이너리 진단](docs/binary-diagnostics.md) | +| EVM 바이트코드 | 디스패치 선택자, 바이트 오프셋, 추론한 인자와 상태 변경 가능 여부 | 로컬 원시 바이트/16진수 입력 파일; [오프라인 EVM 가이드](docs/evm-bytecode.md) | +| 기록된 Linux 크래시 | 원시 note 레코드, 기록된 모든 스레드의 레지스터/시그널, 선택적 매핑 후보 | 호출자가 제공하는 pwntools; GDB/pwndbg는 선택 사항; [기록된 크래시](docs/recorded-crashes.md) | +| JavaScript / Electron | 모듈, 임포트, 소스 맵, 라우트, IPC, 네이티브 애드온 관계 | Node.js와 npm; [애플리케이션 분석](https://rea.tools/guides/javascript/) | +| 웹사이트 | 페이지 구조, 스크립트, 네트워크 관찰 결과, 요청한 스크린샷 | Chrome 계열 브라우저; [브라우저 분석](https://rea.tools/guides/browser/) | +| 저장된 네트워크 캡처 | 요청, 응답, 접근 가능한 페이로드, 소스 위치 | HAR; mitmproxy 네이티브 캡처에는 Linux의 mitmdump; [캡처 가이드](docs/web-network-captures.md) | +| .NET 어셈블리 | 메타데이터, CIL 명령, 선언된 네이티브 의존성, 빌드 비교 | 정적 검사; [관리 코드 가이드](docs/managed-code-analysis.md) | +| Android APK | 매니페스트 선언, 클래스, 디컴파일된 메서드, 참조 | Linux/macOS의 헤드리스 JADX와 전체 JDK; [Android 가이드](docs/android-analysis.md) | +| 펌웨어 | 영역, 추출 결과, 네이티브 분석으로 전달할 정보 | Linux의 Binwalk / Unblob; [펌웨어 가이드](docs/firmware-analysis.md) | +| 패키지 및 리소스 | 파일 목록, 다이제스트, plist, Apple 번들 구조, 추출한 리소스 | [아티팩트 및 JavaScript 가이드](docs/javascript-artifact-reconstruction.md), [Apple 애플리케이션](docs/apple-application-analysis.md) | +| 프로세스 동작 | 터미널 출력, 상호작용, 종료 및 파일 시스템 관찰 결과, 실행 비교 | 네이티브 PTY를 지원하는 Linux/macOS; [프로세스 캡처](docs/process-capture.md) | -- `rea update`는 현재 REA 설치를 업데이트합니다. -- `rea uninstall`은 REA가 관리하는 에이전트 등록과 워크플로 파일을 제거합니다. Hopper는 유지합니다. -- `rea uninstall --purge-data`는 REA 캐시와 상태도 삭제합니다. 해당 데이터를 지우려는 경우에만 사용하세요. +정적 JavaScript 및 .NET 검사는 제공된 파일을 읽으며 애플리케이션을 실행하지 않습니다. 런타임 캡처는 사용자의 권한으로 선택한 대상을 실행하거나 대상과 상호작용합니다. 각 런타임 가이드에서 그 영향을 설명합니다. -## 현재 지원 범위 + -저장소의 현재 기능과 플랫폼 요구 사항은 [영문 지원 가이드](README.md#current-status)에 설명되어 있습니다. main은 [npm 릴리스](docs/installation.md#released-package-and-main)보다 앞선 내용이 포함될 수 있습니다. +지원하는 네이티브 형식과 호스트는 제공자마다 다릅니다. [Hopper 및 Ghidra 설정](docs/installation.md#hopper), [IDA 가이드](docs/ida-provider.md), [실험적 Windows Ghidra 지원](docs/windows-ghidra-p0.md)을 참고하세요. Ghidra는 [16비트 DOS 분석](docs/ghidra-dos.md)도 지원합니다. 제공자 선택은 [CLI 가이드](docs/cli.md#choose-a-provider)를 참고하세요. 최신 npm 릴리스 이후 추가된 기능은 [릴리스에서의 사용 가능 여부](docs/installation.md#released-package-and-main)를 확인하세요. -- Ghidra는 Linux x64, macOS x64/arm64, 실험적인 Windows x64 P0 경계에서 25개의 읽기 전용 작업을 제공합니다. Linux/macOS에서는 원자적 세션 함수 주석도 추가로 지원합니다. Windows P0는 읽기 전용이며 Ghidra에는 GUI 제어가 없습니다. -- 정적 Android 검사는 별도로 준비한 JADX/Java가 필요합니다. 실제 제공자 검증은 Linux를 대상으로 합니다. [Android 분석](docs/android-analysis.md)을 참고하세요. -- 브라우저, Electron, 프로세스 요청은 대상·동작·수명 주기를 직접 지정하며 호스트의 실제 접근에 의존합니다. 별도의 REA 권한 승인은 필요 없습니다. Setup의 설정 쓰기와 Hopper 설치는 여전히 정확한 계획에 대한 승인이 필요합니다. -- `rea capabilities`는 모든 앱/런타임 도구의 목록이 아니라 바이너리 세션 작업을 설명합니다. 연결된 MCP 도구 목록과 관련 가이드를 참고하세요. +## 사례 -## 하나의 프롬프트로 전체 조사 +### DX-Ball: 사운드 패닝 계산 재구현 -```text -메모 앱을 리버스 엔지니어링해 오프라인 검색 기능의 작동 방식을 설명한 다음, -TypeScript와 SQLite를 사용해 제 프로젝트에 맞는 버전을 구현해 주세요. -``` +사운드 호출에서 위치를 패닝 값으로 변환하는 보조 함수까지 추적하고, 명령을 검사하여 불완전한 의사 코드를 C로 옮깁니다. 재구현한 코드는 원본 x86에 대한 3,205개 테스트 케이스를 통과했으며, 컴파일된 함수의 63바이트 전체를 재현합니다. -| 단계 | 에이전트 작업 | REA 도구 | -| ---: | ----------------------- | ---------------------------------------------------------------- | -| 1 | 바이너리 열기 및 식별 | `open_binary`, `binary_overview` | -| 2 | 오프라인 검색 단서 찾기 | `search_strings`, `search_procedures`, `list_names` | -| 3 | 단서를 실행 코드에 연결 | `find_xrefs_to_name`, `xrefs`, `procedure_callers` | -| 4 | 제어 흐름 재구성 | `get_call_graph`, `procedure_callees`, `procedure_info` | -| 5 | 관련 루틴 디컴파일 | `procedure_pseudo_code`, `procedure_assembly`, `batch_decompile` | -| 6 | 프로젝트에 기능 구현 | 기술 스택, 제품, 요구 사항에 맞는 코드 | - -REA는 1~5단계의 바이너리 분석을 처리합니다. 6단계는 에이전트가 일반 파일 편집 및 테스트 도구로 수행합니다. - -## 에이전트가 할 수 있는 일 - -- 소스 코드가 없는 기능의 동작을 설명합니다. -- 앱의 인증, 저장소, 업데이트 또는 네트워크 흐름을 재구성합니다. -- 문서화되지 않은 형식이나 인터페이스의 구조를 복구합니다. -- 문자열이나 심볼에서 의심스러운 동작을 구현한 코드까지 추적합니다. -- 한 세션에서 두 앱 버전을 전환하며 구현 경로를 비교합니다. -- 마음에 드는 기능을 조사하고 자신의 제품에 맞는 버전으로 구현합니다. -- 복구한 동작을 제품 기능, 테스트, 마이그레이션 문서, 포팅 또는 상호 운용 가능한 대체품으로 변환합니다. -- Swift 및 Objective-C 메타데이터를 분석합니다. -- Hopper에 이름, 주석, 북마크를 남겨 사람과 에이전트의 분석을 연결합니다. - -## 조사 도구 카탈로그 - -| 도구 분류 | 수 | 용도 | -| ----------------- | --: | ---------------------------------------------------------------------------------------------------------------------------------------- | -| 네이티브 검사 | 40 | 함수, 의사 코드, 어셈블리, 문자열, 심볼, 호출, 참조, 주석, 바이트 읽기, 파일 오프셋 | -| 조사 워크플로 | 14 | 앱 개요, 함수 기록서, 네이티브 API와 디스패치, 일괄 디컴파일, 기능 추적, 호출 경로, 호출 그래프, Swift와 Objective-C 탐색 | -| 네이티브 바이너리 | 10 | macOS의 Mach-O 메타데이터, 서명, plist, 아키텍처, Swift 심볼과 LLDB 호출 관찰; Linux의 ELF 레이아웃, 정적 보호 증거와 기록된 크래시 | -| 아티팩트 그래프 | 7 | 디렉터리와 패키지 목록, 컴파일된 Interface Builder 파일, Apple 애셋 카탈로그, Mach-O dylib 로드 해석, 추출, 오프라인 EVM 인터페이스 추론 | -| 관리 PE/CLI | 7 | .NET 식별, 메타데이터, CIL 명령어, 네이티브 종속성, 재구성 가져오기, 빌드 비교 | -| 펌웨어 | 2 | Linux 펌웨어 영역 검사 및 명시적 추출 | -| Android APK | 5 | 패키지와 manifest 선언, 클래스 검색, 멤버 목록, 메서드 디컴파일, 정적 참조 | -| 브라우저 관찰 | 12 | 페이지 구조, 네트워크 메타데이터, 스크립트, 소스 맵, WebMCP 탐색, 스크린샷, 캡처 비교 | -| Electron 분석 | 5 | 렌더러 관찰, 정적 앱 매핑, 정적·런타임 결과 연결 | -| JavaScript 런타임 | 2 | Node/Electron Inspector 대상 탐색, 스크립트 위치, 실행 컨텍스트 이벤트 | -| 앱 워크플로 | 13 | 캡처한 웹 스크립트 내보내기, 계층 간 기능 추적, 빌드 비교, 히스토리 소스 매핑, 정적 반환 구조 비교, 재구현 검증 | -| 작업 공간과 관찰 | 21 | 세션, 근거 번들, 탐색 컨텍스트, 프로세스·아티팩트·함수 비교, 미해결 항목 기록 | - -## 로드맵 - -다음 작업은 네이티브 대상 검증 확대, JavaScript와 .NET의 버전 비교, 런타임 관찰과 재구현 검증입니다. Setup은 에이전트 연결과 Hopper 설치 선택을 이미 지원합니다. 다른 분석 도구 설치는 향후 작업입니다. [설치 로드맵](docs/roadmap.md)과 [분석 도구 평가](docs/provider-evaluation.md)를 참고하세요. - -## 다른 코딩 에이전트에서 사용하기 - -Setup은 Claude Code, Claude Desktop, Codex, Cursor, Gemini CLI, Windsurf, Devin, OpenCode, Antigravity, GitHub Copilot CLI, Command Code, VS Code를 지원합니다. 기존 REA 등록은 기본 선택되고, 그 밖의 감지된 클라이언트는 직접 선택해야 합니다. 로컬 MCP 서버를 지원하는 에이전트는 다음 설정으로 연결할 수 있습니다. - - - -```json -{ - "mcpServers": { - "rea": { - "command": "npx", - "args": ["-y", "rea-agents@6.0.0", "mcp"] - } - } -} -``` +[사례 읽기](https://rea.tools/showcase/dx-ball/) · +[재구현 저장소](https://github.com/N0zoM1z0/dx-ball) - +### Notion: Electron 클립보드 브리지 추적 -## 작동 방식 +렌더러의 클립보드 API를 찾고, preload와 IPC를 거쳐 메인 프로세스까지 추적한 뒤, 서식 있는 클립보드 형식을 검사합니다. -```mermaid -flowchart LR - Agent["코딩 에이전트"] --> REA["REA
CLI + MCP"] - Terminal["터미널"] --> REA - REA --> Hopper["Hopper 분석 작업"] - Hopper --> App["사용자의 앱"] - REA --> Ghidra["Ghidra 읽기 전용 분석
인벤토리 + 함수 분석"] - Ghidra --> App -``` +[사례 읽기](https://rea.tools/showcase/notion/) -CLI와 MCP 서버는 같은 분석 워크플로를 사용합니다. 터미널 명령은 완료 후 자신의 브리지 세션을 해제하고, 에이전트 세션은 조사 중 연결을 유지할 수 있습니다. REA 세션을 닫아도 사용자가 이용 중인 Hopper 앱은 종료하지 않습니다. +### TH04: DOS 원형 탄막 계산 복원 -## CLI +원본 PC-98 게임의 16비트 명령을 검사하고 고정 각도와 조준 각도의 계산을 복원합니다. 재구현한 C++를 당시 컴파일러의 출력과 비교합니다. -위의 에이전트 워크플로가 REA를 사용하는 가장 쉬운 방법입니다. 터미널에서 앱을 한 번 빠르게 살펴보려면 다음을 실행하세요. +[사례 읽기](https://rea.tools/showcase/th04/) · +[재구현 저장소](https://github.com/N0zoM1z0/th04) -```bash -npx -y rea-agents@latest analyze /Applications/Notes.app -``` +REA로 흥미로운 대상을 분석했다면 알려 주세요. [issue](https://github.com/morluto/rea/issues) 또는 [pull request](https://github.com/morluto/rea/pulls)에 대상, 조사하려던 질문, REA가 어떻게 도움이 되었는지, 무엇을 알아냈는지를 담아 공유하세요. -직접 디컴파일하는 방법과 기타 옵션은 `npx -y rea-agents@latest --help`에서 확인할 수 있습니다. +## 자주 묻는 질문 -전역 `rea` 명령으로 설치할 수도 있습니다. +
+어떤 에이전트에서 REA를 사용할 수 있나요? -```bash -npm install --global rea-agents -rea --help -rea mcp -``` +로컬 MCP 서버를 지원하는 에이전트라면 사용할 수 있습니다. 설정 과정은 [지원하는 에이전트](docs/installation.md#supported-agents)를 구성합니다. 다른 클라이언트는 [수동 MCP 등록](docs/installation.md#mcp-registry)을 이용할 수 있습니다. -REA는 macOS의 `.app` 폴더를 직접 열 수 있습니다. 에이전트가 앱을 찾지 못하면 설치 위치를 알려 주세요. +
-## Hopper 앱 동작 +
+Hopper, Ghidra 또는 IDA가 필요한가요? -REA는 필요할 때 Hopper를 시작합니다. Hopper 런처는 내부적으로 앱을 활성화하므로 대상을 열 때 Hopper가 다른 창 앞으로 나타날 수 있습니다. REA는 백그라운드 시작을 요청하지만 항상 뒤에 머무르는 것을 보장할 수 없습니다. +심층 네이티브 분석에는 이 중 하나를 사용합니다. 정적 JavaScript 및 .NET 검사에는 네이티브 분석 엔진이 필요하지 않습니다. 설정 과정에서 승인 후 Hopper를 설치할 수 있으며, Ghidra와 IDA는 기존 설치를 사용합니다. [제공자 설정](docs/installation.md#hopper)을 참고하세요. -명시적인 형식과 아키텍처 인자로 일반적인 FAT/ARM 선택 대화상자를 피하지만, 다른 Hopper 또는 macOS 대화상자에는 사람이 응답해야 할 수 있습니다. 세션 종료 시 브리지와 소켓 디렉터리를 제거하지만 사용자가 작업 중인 Hopper 앱은 종료하지 않습니다. +
-## 보안 모델 +
+Hopper를 먼저 실행해야 하나요? -각 세션은 무작위 capability token과 현재 사용자 전용 Unix 소켓을 사용합니다. Ghidra 세션은 격리된 임시 프로젝트도 사용하며 사용자 소유 Ghidra 프로젝트를 열거나 수정하지 않습니다. 이는 샌드박스가 아니며 같은 운영 체제 사용자로 실행되는 악성 프로세스를 방어하지 않습니다. 신뢰할 수 없는 바이너리를 열면 선택한 로컬 공급자가 현재 사용자 권한으로 분석합니다. 취약점은 [SECURITY.md](SECURITY.md)의 비공개 절차로 신고하세요. +REA는 작업에 필요할 때 Hopper를 실행합니다. macOS에서는 처음 실행할 때 데모 모드를 선택하거나 라이선스를 활성화하라는 대화 상자가 나타날 수 있습니다. [Hopper 실행 및 문제 해결](docs/installation.md#launcher-paths-and-troubleshooting)을 참고하세요. -## FAQ +
-
Hopper를 미리 실행해야 하나요? +
+skills.sh에서 설치하는 스킬은 어떤 역할을 하나요? -아니요. REA가 필요할 때 시작합니다. Hopper가 실행 중이어도 사용할 수 있지만 기존 GUI 문서에 연결하는 대신 새 분석 문서를 엽니다. +스킬은 에이전트에게 조사 지침을 제공합니다. `rea setup`으로 REA의 MCP 서버를 등록하고 버전에 맞는 지침을 설치한 뒤 에이전트를 다시 시작하세요. [스킬만 설치하기](docs/installation.md#skill-only-installation)를 참고하세요.
-
REA에 Hopper가 포함되나요? +
+REA는 어떤 코드를 반환하나요? -아니요. Setup이 Hopper를 설치할 수 있지만 Hopper는 자체 라이선스가 필요한 별도 소프트웨어입니다. REA는 CLI, MCP 서버, 에이전트용 워크플로를 제공합니다. +네이티브 분석은 의사 코드와 어셈블리를 반환합니다. JavaScript/Electron 분석은 모듈과 그 관계를 복원합니다. 에이전트는 이 결과를 사용해 구현을 작성하고 테스트합니다. [사례](#사례)에서 구체적인 예를 볼 수 있습니다.
-
바이너리가 업로드되나요? +
+REA가 내 앱을 업로드하나요? -REA에는 호스팅 분석 서비스가 없습니다. 로컬 Unix 소켓을 통해 Hopper를 조작합니다. 에이전트나 모델 공급자의 데이터 정책은 별도로 확인하세요. +REA는 대상을 로컬에서 분석합니다. 에이전트는 도구 결과를 받으며, 에이전트의 모델 제공자에는 별도의 데이터 정책이 있습니다.
-
원본 소스 코드를 복구할 수 있나요? +
+버그가 발생하면 어떻게 해야 하나요? + +먼저 업데이트하세요. 최근 릴리스에서 이미 수정되었을 수 있습니다. -보장할 수 없습니다. REA는 의사 코드, 어셈블리, 심볼, 문자열, 메타데이터 및 관계를 제공하여 에이전트가 관찰된 동작을 설명하거나 호환되게 재현하도록 돕습니다. +npm으로 설치한 CLI에서는 다음 명령을 사용합니다. + +```bash +rea update +``` + +`npx`로 에이전트를 설정했다면 다음 명령을 사용합니다. + +```bash +npx rea-agents@latest setup +``` + +에이전트를 사용 중이라면 [설정 갱신](#rea-업데이트)을 완료하고 다시 시작하세요. 같은 작업을 다시 시도하세요. 문제가 계속되면 REA 버전, 대상 유형, 재현 단계, 오류 출력을 포함해 [issue를 등록](https://github.com/morluto/rea/issues)하세요.
-## 개발 +## 문서 + +웹사이트의 [실습 가이드](https://rea.tools/guides/)부터 읽어 보세요. 정확한 옵션, 사전 요구 사항, 결과 계약은 다음 문서를 참고하세요. + +- [설치 및 설정](docs/installation.md): 에이전트 등록, 제공자 설정, 업데이트, 제거. +- [준비 상태 확인 및 문제 해결](docs/installation.md#check-readiness-for-your-task): 특정 에이전트 또는 분석 엔진 진단. +- [CLI 및 Evidence](docs/cli.md): 명령, 제공자 선택, 스냅샷, 가져오기/내보내기, 종료 상태. +- [MCP 계약](docs/mcp-contracts.md) 및 [에이전트 프롬프트](docs/mcp-prompts.md): 도구 결과, 세션, 안내에 따른 조사. +- [도구 카탈로그](docs/mcp-contracts.md#generated-catalog): 빌드 시 생성되는 도구, 제공자, CLI 명령 목록. +- [로드맵](docs/roadmap.md): 계획된 작업과 기능 진행 상황. -개발 환경, 아키텍처, 테스트, 릴리스 지침은 [CONTRIBUTING.md](CONTRIBUTING.md)를 참고하세요. +취약점은 [SECURITY.md](SECURITY.md)에 따라 보고하세요. + +## 기여 + +REA에 도움을 보태 주세요! [issue를 등록](https://github.com/morluto/rea/issues)해 버그나 기능을 제안하거나, [pull request를 보내](https://github.com/morluto/rea/pulls) 코드와 문서를 개선할 수 있습니다. + +개발 환경과 검사 항목은 [CONTRIBUTING.md](CONTRIBUTING.md), 검증 절차는 [테스트 가이드](docs/testing.md), 프로젝트 구조는 [아키텍처 지도](docs/architecture.mermaid)를 참고하세요. + +## 프로젝트 링크 + +[웹사이트](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [보안](SECURITY.md) ## 스타 기록 🎉 **GitHub 스타 20,000개, 감사합니다!** -REA를 사용하고, 버그를 제보하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. +REA를 사용하고, 버그를 보고하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. @@ -329,6 +278,10 @@ REA를 사용하고, 버그를 제보하고, 빌드를 테스트하고, 수정 +## 면책 조항 + +REA는 합법적인 리버스 엔지니어링 연구, 분석, 재구성을 위한 도구를 제공합니다. 필요한 권한을 얻고 관련 법률을 준수할 책임은 사용자에게 있습니다. 이 프로젝트는 불법적이거나 허가받지 않은 사용을 지지하지 않습니다. + ## 라이선스 [MIT](LICENSE) diff --git a/README_zh.md b/README_zh.md index b1b1e4cac10..c9dd4a0a04b 100644 --- a/README_zh.md +++ b/README_zh.md @@ -2,26 +2,35 @@ [English](README.md) · **简体中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [العربية](README_ar.md) -# REA:逆向工程一切 +# REA:逆向分析一切 -### 使用智能体逆向工程任何程序,从应用行为到原生二进制文件。 +### 通过一个 MCP 服务,逆向分析二进制文件、应用程序和运行时行为。 -**看到喜欢的功能。理解它的原理。按照你的方式实现。** +**看到喜欢的功能,弄清它的工作原理,深入到二进制层面。** [![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) [![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) -[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](#调查工具目录) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) [![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) morluto%2Frea | Trendshift -**[网站(英文)](https://morluto.github.io/rea/) · [使用指南](https://morluto.github.io/rea/guides/) · [DX-Ball 案例](https://morluto.github.io/rea/showcase/dx-ball/)** +**[网站](https://rea.tools/) · [指南](https://rea.tools/guides/) · [案例](https://rea.tools/showcase/)** -[快速开始](#快速开始) · [当前状态](#当前状态) · [从二进制到行为](#从二进制到行为) · [调查工具目录](#调查工具目录) · [路线图](#路线图) · [工作原理](#工作原理) +[快速开始](#快速开始) · [REA 的工作原理](#rea-的工作原理) · [可以分析什么](#可以分析什么) · [案例](#案例) · [常见问题](#常见问题) · [文档](#文档) - +npx rea-agents setup + +
+ +REA 在 Hopper 中启动分析桥,检查原生二进制文件 + +
+ +
@@ -35,301 +44,244 @@
-npx rea-agents setup - -
- -REA 在 Hopper 中启动分析桥并检查原生二进制文件 - --- -看到某个应用中想加入自己产品的功能?让智能体用 REA 调查它。即使没有源代码,智能体也可以检查应用、解释功能的工作方式、展示证据,并为你的项目实现类似功能。 - -REA 提供原生二进制、JavaScript 与 Electron 应用、.NET 程序集和网站的分析工具。你可以从智能体或终端使用它们。分析在本机运行,结果会说明证据和限制。 - -Setup 可以配置智能体、连接已安装的 Hopper 或 Ghidra,也可以在你批准后安装 Hopper。 - -## 直接询问智能体 - -完成[设置](#快速开始)后,重启智能体并提出问题: +在应用中看到一个想加入自己产品的功能?让智能体用 REA 调查它。即使没有源代码,智能体也可以检查应用、解释功能的工作方式、展示证据,并为你的项目实现类似功能。 -```text -解释“备忘录”应用的搜索功能,展示证据,然后为我的项目实现类似功能。 -``` - -把“备忘录”替换成你想了解的应用,也可以先请求概览。 - -## 从二进制到行为 +REA 将智能体连接到分析工具,用于检查原生二进制文件、JavaScript 和 Electron 应用、.NET 程序集以及网站。你也可以在终端中使用同样的工具。分析在本机运行,结果包含支撑各项结论的证据和相关限制。 -| 反编译 | 理解 | 重建 | -| ---------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | ------------------------------------------------------------ | -| 打开原生应用或可执行文件,恢复过程、伪代码、汇编、字符串、符号、段和元数据。 | 沿调用者、被调用者、交叉引用和调用图追踪,直到智能体能够解释功能或算法的实际工作方式。 | 将智能体学到的内容变成适合你的技术栈、界面和需求的产品功能。 | +设置流程会向智能体注册 REA,并安装匹配的工作流指引。原生分析可以使用已有的 Hopper 或 Ghidra;设置流程也可以在你批准后安装 Hopper。静态 JavaScript 分析不需要这两个引擎。 -REA 让调查始终以二进制证据为依据。它不会声称能恢复原始源代码,也不会自动克隆整个应用。 - -## 为什么选择 REA - -| | | -| ---------------- | -------------------------------------------------------------------- | -| **为智能体设计** | 直接询问应用的行为,让智能体去检查,而不是猜测。 | -| **CLI 与 MCP** | 在终端或智能体中使用同一套逆向工程能力。 | -| **引导式设置** | 配置智能体、连接已有分析工具,或在你批准后安装 Hopper。 | -| **从洞察到代码** | 理解功能后,在同一次编程会话中构建属于你自己的版本。 | -| **本地运行** | 分析在受支持的本机系统上运行;REA 不会把二进制上传到托管式分析服务。 | -| **保留上下文** | 连续调查多个应用,无需为每个问题重新开始整个分析过程。 | +> **[访问 REA 网站](https://rea.tools/)**,查看设置说明、图解指南和真实案例。 ## 快速开始 -### 运行设置(推荐) +### 设置智能体 -配置 REA 与智能体的连接: +安装 Node.js 和 npm 后,运行: ```bash npx rea-agents setup ``` -Setup 首先让你多选要连接的智能体。已有 REA 注册默认选中;仅被检测到的客户端不会自动选中,也可以手动选择尚未配置的客户端。查看具体路径和变更后再批准。所选智能体默认安装 REA 工作流;Hopper 是单独的可选操作,需要单独批准。Setup 也可以记录现有 Ghidra 的路径。 +选择智能体,检查计划中的变更并批准。设置流程会添加 REA 的 MCP 服务和匹配的工作流指引,并备份已有配置。完成后重启智能体。 -Setup 会先展示变更并备份已有配置。要求与更多选项见[安装与设置](docs/installation.md)。 +设置流程支持 Claude Code、Codex、Cursor、Gemini CLI 和[其他智能体](docs/installation.md#supported-agents)。提供方配置和手动注册 MCP 的方法见[安装与设置](docs/installation.md)。 -### 使用智能体 +### 询问智能体 -设置完成后,重启智能体并描述你想了解的应用或功能。REA 支持 Claude Code、Claude Desktop、Codex、Cursor、Gemini CLI、Windsurf、Devin、OpenCode、Antigravity、GitHub Copilot CLI、Command Code 和 VS Code。已有 REA 注册默认选中;其他检测到的客户端需手动选择。其他智能体可以使用下方的 MCP 配置。 +```text +弄清 Notes 应用的搜索功能如何工作,展示证据,并为我的项目实现类似功能。 +``` -Hopper 支持演示模式。如果首次启动时出现提示,选择演示模式或输入已有许可证。 +把 Notes 替换为目标应用,并描述你想了解的功能。 -### AI 编程助手技能(可选) +### 使用终端 -为你的 AI 编程助手添加 skill,以获得更丰富的上下文: +检查已解包的 JavaScript/Electron 应用目录或 ASAR 文件: ```bash -npx skills add morluto/rea --skill reverse-engineer-anything +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json ``` -该 skill 提供 REA 的调查工作流。先运行上文的 setup,把 REA 连接到你的智能体并配置分析工具。Setup 默认会安装版本匹配的 skill;这条命令安装的是仓库版本。 +结果包含模块、导入、Electron 边界及相关证据。将路径替换为你的目标路径,例如 Windows 上的 `"D:/apps/example"`。 -对于已解包的 JavaScript/Electron 应用目录或 ASAR,无需 MCP 配置,也无需原生引擎: +如果需要经常使用,可以安装 `rea` 命令: ```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +npm install --global rea-agents +rea --help ``` -把路径换成你的目标(例如 Windows 上的 `"D:/apps/example"`)。这条静态流程支持此类路径。原生分析仍需要先配置对应的分析引擎。 +原生分析需要先配置分析提供方。原生命令、提供方选择、快照和脚本用法见 [CLI 与 Evidence 指南](docs/cli.md)。 -### 安装 rea 命令 +### 更新 REA -安装命令行工具: +REA 更新较快,新版本经常包含错误修复。请保持安装的版本为最新。 + +通过 npm 安装的 CLI 使用: ```bash -curl -fsSL https://raw.githubusercontent.com/morluto/rea/main/install.sh | bash +rea update ``` -需要预先安装 Node.js 和 npm。在终端中运行时,安装程序会安装 `rea` 并启动设置。 +要更新智能体注册和 skill,请运行更新命令输出的设置命令。 -也可以用 npm 安装,再运行设置: +如果使用 `npx`,通过以下命令更新智能体设置: ```bash -npm install --global rea-agents -rea setup +npx rea-agents@latest setup ``` -### 系统要求 +检查设置变更并重启智能体。单次 CLI 操作可使用 `npx rea-agents@latest`,后接所需命令。 -- macOS 12 或更高版本 -- Ubuntu 24.04+、Fedora 41+、64 位 Arch Linux 或 CachyOS -- Node.js 22.x (>=22.19)、24.x (>=24.11) 或 26+,以及 npm +## REA 的工作原理 -原生二进制分析需要 Hopper 或 Ghidra。Hopper 是独立软件;演示模式有厂商规定的限制,不要求购买许可证。 +智能体通过 MCP 调用 REA,检查目标并追踪相关代码。REA 返回发现及对应证据。智能体据此提出后续问题、解释行为,或编写并测试实现。CLI 命令使用同样的工作流。 -Ghidra 支持 Linux x64 和 macOS x64/arm64。单独安装 Ghidra 12.1.x,以及该安装在 `application.java.min` 到 `application.java.max` 中声明的完整 64 位 JDK,然后配置 REA 使用它们。当前 12.1 发行版要求 JDK 21 或更新版本,并且不设置上限。桥接在 Ghidra 12.1.4 和 JDK 21 上完成验证。macOS 还需要对应架构的原生反编译器。 +![REA 调查流程:智能体询问本地目标的行为,REA 使用分析工具检查和追踪目标,智能体再利用返回的代码、引用和未知项进行解释、实现与测试。](website/public/assets/figures/rea-investigation-flow.svg) -Setup 可以验证安装并保存路径,不会安装或升级 Ghidra、Java、Node.js、npm 或 Homebrew。 +[打开完整尺寸的图示](website/public/assets/figures/rea-investigation-flow.svg)。 -仓库 main 分支与 npm 4.1.0 已包含实验性的 Windows x64 Ghidra P0 支持,适用于本地 NTFS 上的原生 x86-64 PE 应用(非托管、非 DLL),并附带 Job Object、私有 DACL 与路径准入控制。请先确认[发布边界](docs/installation.md#released-package-and-main),再假定旧版 npm 包具备该能力。详见 [Windows Ghidra P0](docs/windows-ghidra-p0.md) 的前置条件与已验证范围。 +
-### 故障排除 +## 可以分析什么 -运行 `npx -y rea-agents@latest doctor`,检查主机、依赖、分析工具和智能体配置。该命令不会修改文件。添加 `--json` 可获取结构化诊断。 +REA 需要 Node.js 22.x(>=22.19)、24.x(>=24.11)或 26+,以及 npm。额外工具和主机支持取决于分析目标: -Linux 上 REA 会优先使用可执行的 `/opt/hopper/bin/Hopper`;如果不可用,则自动检查 `~/.local/share/rea/hopper/bin/Hopper`。其他路径可通过 `HOPPER_LAUNCHER_PATH` 指定。如果文件存在但仍报告缺少分析引擎,请对实际 Hopper 路径运行 `ldd /absolute/path/to/Hopper | grep 'not found'` 检查缺少的系统库。安装详情见 [Hopper 指南](docs/installation.md#hopper)。 +| 目标 | REA 返回的内容 | 要求与指南 | +| --------------------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| 原生二进制文件 | 伪代码、汇编、字符串、符号、调用和引用 | Hopper、Ghidra 或 IDA;[原生分析](https://rea.tools/guides/native/) | +| 离线 ELF 布局 | 节、段、原始符号/重定位信息及静态防护机制候选项 | Linux x64 上由调用方提供的 pwntools;[二进制诊断](docs/binary-diagnostics.md) | +| EVM 字节码 | 分派选择器、字节偏移、推断的参数和状态可变性 | 本地原始字节/十六进制输入载体;[离线 EVM 指南](docs/evm-bytecode.md) | +| 已记录的 Linux 崩溃 | 原始 note 记录、每个已记录线程的寄存器/信号及可选的映射候选项 | 调用方提供的 pwntools;可选的 GDB/pwndbg;[已记录崩溃](docs/recorded-crashes.md) | +| JavaScript / Electron | 模块、导入、source map、路由、IPC 和原生扩展关系 | Node.js 和 npm;[应用分析](https://rea.tools/guides/javascript/) | +| 网站 | 页面结构、脚本、网络观察结果和按请求获取的截图 | Chrome 系浏览器;[浏览器分析](https://rea.tools/guides/browser/) | +| 已保存的网络捕获 | 请求、响应、可访问的载荷和来源位置 | HAR;原生 mitmproxy 捕获需要 Linux 上的 mitmdump;[捕获指南](docs/web-network-captures.md) | +| .NET 程序集 | 元数据、CIL 指令、声明的原生依赖和构建对比 | 静态检查;[托管代码指南](docs/managed-code-analysis.md) | +| Android APK | 清单声明、类、反编译的方法和引用 | Linux/macOS 上的无界面 JADX 和完整 JDK;[Android 指南](docs/android-analysis.md) | +| 固件 | 区域、提取结果和转交原生分析的内容 | Linux 上的 Binwalk / Unblob;[固件指南](docs/firmware-analysis.md) | +| 软件包与资源 | 文件清单、摘要、plist、Apple bundle 结构和提取的资源 | [制品与 JavaScript 指南](docs/javascript-artifact-reconstruction.md)、[Apple 应用](docs/apple-application-analysis.md) | +| 进程行为 | 终端输出、交互、退出和文件系统观察结果,以及运行对比 | 支持原生 PTY 的 Linux/macOS;[进程捕获](docs/process-capture.md) | -### 更新与卸载 +静态 JavaScript 和 .NET 检查读取提供的文件,不运行应用。运行时捕获会以你的用户权限运行目标或与之交互;各运行时指南说明了具体影响。 -- `rea update` 更新当前 REA 安装。 -- `rea uninstall` 移除 REA 管理的智能体配置项和工作流文件,保留 Hopper。 -- `rea uninstall --purge-data` 还会删除 REA 的缓存和状态。仅在需要移除这些数据时使用。 + -## 当前状态 +不同提供方支持的原生格式和主机系统有所不同。请参阅 [Hopper 与 Ghidra 设置](docs/installation.md#hopper)、[IDA 指南](docs/ida-provider.md)和[实验性 Windows Ghidra 支持](docs/windows-ghidra-p0.md)。Ghidra 也支持 [16 位 DOS 分析](docs/ghidra-dos.md)。提供方选择见 [CLI 指南](docs/cli.md#choose-a-provider)。对于最新 npm 版本发布后新增的功能,请查看[发布可用性](docs/installation.md#released-package-and-main)。 -仓库当前能力与平台要求见[英文支持指南](README.md#current-status)。main 分支可能领先于 [npm 发布版](docs/installation.md#released-package-and-main)。 +## 案例 -- Ghidra 在 Linux x64、macOS x64/arm64 以及实验性的 Windows x64 P0 边界上提供 25 个只读操作。Linux/macOS 额外支持原子化的会话级函数注释。Windows P0 为只读;Ghidra 不提供 GUI 控制。 -- 静态 Android 检查需要自备 JADX/Java,前置条件独立;真机验证覆盖 Linux。详见 [Android 分析](docs/android-analysis.md)。 -- 浏览器、Electron 与进程请求需直接声明目标、动作与生命周期,并依赖宿主的实际访问权限,无需 REA 另行授权。Setup 的配置写入与 Hopper 安装仍需逐项批准确切计划。 -- `rea capabilities` 描述的是二进制会话操作,而非全部应用/运行时工具。请以已连接 MCP 工具列表与相关指南为准。 +### DX-Ball:重建声像计算 -## 一个提示词,完成一次完整调查 +沿声音调用追踪到根据位置计算声像的辅助函数,检查指令,将不完整的伪代码转为 C。重建结果通过了 3,205 个原始 x86 测试用例,并复现了编译后函数的全部 63 个字节。 -```text -逆向工程“备忘录”应用,找到离线搜索功能的工作方式,解释其控制流, -并使用 TypeScript 和 SQLite 为我的项目构建一个版本。 -``` +[阅读案例](https://rea.tools/showcase/dx-ball/) · +[重建项目仓库](https://github.com/N0zoM1z0/dx-ball) -| 步骤 | 智能体的操作 | REA 工具 | -| ---: | ---------------------- | ---------------------------------------------------------------- | -| 1 | 打开并识别二进制文件 | `open_binary`, `binary_overview` | -| 2 | 搜索可能的离线搜索线索 | `search_strings`, `search_procedures`, `list_names` | -| 3 | 将线索连接到可执行代码 | `find_xrefs_to_name`, `xrefs`, `procedure_callers` | -| 4 | 重建相关控制流 | `get_call_graph`, `procedure_callees`, `procedure_info` | -| 5 | 反编译相关程序 | `procedure_pseudo_code`, `procedure_assembly`, `batch_decompile` | -| 6 | 在你的项目中构建该功能 | 适合你的技术栈、产品和需求的代码 | - -REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常规文件编辑与测试工具完成。 - -## 智能体可以完成什么 - -- 在没有源代码时解释某项功能的实现方式。 -- 重建应用的身份验证、存储、更新或网络流程。 -- 恢复足够的结构,以记录未公开的格式或接口。 -- 从字符串或符号追踪到实现可疑行为的代码。 -- 在一个会话中切换两个应用版本并比较实现路径。 -- 调查你喜欢的功能,并为自己的产品构建量身定制的版本。 -- 将恢复的行为转换为产品功能、测试、迁移说明、移植代码或互操作替代品。 -- 分析 Swift 和 Objective-C 元数据。 -- 在 Hopper 中留下名称、注释与书签,使人与智能体的分析互相增强。 - -## 调查工具目录 - -| 工具类别 | 数量 | 用途 | -| ----------------- | ---: | -------------------------------------------------------------------------------------------------------------------- | -| 原生检查 | 40 | 函数、伪代码、汇编、字符串、符号、调用、引用、注释、字节读取与文件偏移 | -| 调查工作流 | 14 | 应用概览、函数档案、原生 API 与分发、批量反编译、功能追踪、调用路径与调用图、Swift 与 Objective-C 发现 | -| 原生二进制工具 | 10 | macOS 的 Mach-O 元数据、签名、plist、架构、Swift 符号与 LLDB 调用观察;Linux 的 ELF 布局、静态保护证据与历史崩溃记录 | -| 产物图 | 7 | 目录与软件包清单、编译后的 Interface Builder 文件、Apple 资源目录、Mach-O dylib 加载解析、提取与离线 EVM 接口推断 | -| 托管 PE/CLI | 7 | .NET 程序集身份、元数据、CIL 指令、原生依赖声明、重建导入与构建比较 | -| 固件 | 2 | Linux 固件区域检查与显式提取 | -| Android APK | 5 | 包与 manifest 声明、类搜索、成员清单、方法反编译与静态引用 | -| 浏览器观察 | 12 | 页面结构、网络元数据、脚本、来源映射、WebMCP 发现、截图与捕获比较 | -| Electron 分析 | 5 | 渲染进程观察、静态应用映射、静态/运行时结果关联 | -| JavaScript 运行时 | 2 | Node/Electron Inspector 目标发现、脚本位置与执行上下文事件 | -| 应用工作流 | 13 | 捕获的网站脚本导出、跨层功能追踪、构建比较、历史源码映射、静态返回结构比较与重建验证 | -| 工作区与观察 | 21 | 会话、证据包、导航上下文、进程/产物/函数比较与待解决问题跟踪 | - -## 路线图 - -接下来的工作包括更多原生目标验证、JavaScript 与 .NET 跨版本比较,以及运行时观察和重建验证。当前设置已支持选择智能体集成和 Hopper 安装;安装其他分析工具仍是后续工作。详见[安装路线图](docs/roadmap.md)和[分析工具评估](docs/provider-evaluation.md)。 - -## 与其他编程智能体一起使用 - -Setup 支持 Claude Code、Claude Desktop、Codex、Cursor、Gemini CLI、Windsurf、Devin、OpenCode、Antigravity、GitHub Copilot CLI、Command Code 和 VS Code。已有 REA 注册默认选中;其他检测到的客户端需手动选择。任何支持本地 MCP 服务器的智能体都可以使用以下配置连接 REA。 - - - -```json -{ - "mcpServers": { - "rea": { - "command": "npx", - "args": ["-y", "rea-agents@6.0.0", "mcp"] - } - } -} -``` +### Notion:追踪 Electron 剪贴板桥 - +找到渲染进程的剪贴板 API,沿 preload 和 IPC 追踪到主进程,并检查富格式剪贴板数据。 -## 工作原理 +[阅读案例](https://rea.tools/showcase/notion/) -```mermaid -flowchart LR - Agent["编程智能体"] --> REA["REA
CLI + MCP"] - Terminal["终端"] --> REA - REA --> Hopper["Hopper 分析操作"] - Hopper --> App["你的应用"] - REA --> Ghidra["Ghidra 只读分析
清单 + 函数分析"] - Ghidra --> App -``` +### TH04:恢复 DOS 环形子弹的计算 -CLI 与 MCP 服务器使用相同的分析流程。终端命令完成后释放自己创建的桥接会话;智能体会话可以在调查期间保持连接。关闭 REA 会话不会退出用户正在使用的 Hopper 应用。 +检查原始 PC-98 游戏的 16 位指令,恢复固定角度和瞄准角度的计算,并将重建的 C++ 与当年的编译器输出进行对比。 -## CLI +[阅读案例](https://rea.tools/showcase/th04/) · +[重建项目仓库](https://github.com/N0zoM1z0/th04) -上面的智能体工作流是使用 REA 最简单的方式。如果只想在终端中快速了解一个应用: +如果你用 REA 分析了有趣的目标,我们很想了解。欢迎通过 [issue](https://github.com/morluto/rea/issues) 或 [pull request](https://github.com/morluto/rea/pulls) 分享案例,说明目标、你的问题、REA 如何提供帮助,以及你的发现。 -```bash -npx -y rea-agents@latest analyze /Applications/Notes.app -``` +## 常见问题 -运行 `npx -y rea-agents@latest --help` 查看直接反编译和其他选项。 +
+哪些智能体可以使用 REA? -也可以全局安装 `rea` 命令: +任何支持本地 MCP 服务的智能体都可以使用。设置流程会配置[受支持的智能体](docs/installation.md#supported-agents);其他客户端可通过[手动注册 MCP](docs/installation.md#mcp-registry) 使用。 -```bash -npm install --global rea-agents -rea --help -rea mcp -``` +
-REA 可以直接打开 Mac 的 `.app` 文件夹。如果智能体找不到应用,请告诉它应用安装在哪里。 +
+需要 Hopper、Ghidra 或 IDA 吗? -## Hopper 应用行为 +深入的原生分析需要其中一个引擎。静态 JavaScript 和 .NET 检查不需要原生分析引擎。设置流程可以在你批准后安装 Hopper;Ghidra 和 IDA 则使用已有安装。见[提供方设置](docs/installation.md#hopper)。 + +
-REA 会在需要时启动 Hopper,无需预先运行。Hopper 启动器内部会激活应用,因此打开目标时 Hopper 可能出现在其他窗口前。REA 会请求 macOS 在后台启动 Hopper,但无法保证窗口始终位于后台。 +
+需要先启动 Hopper 吗? -REA 会推导明确的格式和架构参数,以避免常见的 FAT 与 ARM 选择对话框。其他 Hopper 或 macOS 对话框仍可能需要人工响应。关闭 REA 会话会终止桥并删除私有套接字目录,但不会退出用户正在使用的 Hopper 应用。 +REA 会在操作需要时启动 Hopper。在 macOS 上,首次运行时可能会出现对话框,让你选择演示模式或激活许可证。见 [Hopper 启动与故障排查](docs/installation.md#launcher-paths-and-troubleshooting)。 -## 安全模型 +
-每个桥会话都使用随机能力令牌和仅限当前用户的 Unix 套接字。诊断保留本机路径、摘要和错误位置等排查信息,同时移除凭据和认证令牌。Ghidra 会话还使用隔离的临时项目,不会打开或修改用户自己的 Ghidra 项目。 +
+从 skills.sh 安装 skill 有什么作用? -这不是沙箱,也无法防御以同一操作系统用户身份运行的恶意进程。打开不可信二进制文件会让所选本地提供商以当前用户权限进行解析和分析。请按照 [SECURITY.md](SECURITY.md) 中的私密流程报告漏洞。 +skill 为智能体提供调查指引。使用 `rea setup` 注册 REA 的 MCP 服务并安装匹配的指引,然后重启智能体。见[仅安装 skill](docs/installation.md#skill-only-installation)。 -## 常见问题 +
-
Hopper 是否需要提前运行? +
+REA 返回什么代码? -不需要。REA 会在操作需要时启动 Hopper。Hopper 已经运行时也可以使用,但 REA 会打开新的分析文档,不会连接并复用现有 GUI 文档。 +原生分析返回伪代码和汇编。JavaScript/Electron 分析恢复模块及其关系。智能体利用这些发现编写并测试实现;[案例](#案例)提供了完整示例。
-
REA 是否包含 Hopper? +
+REA 会上传我的应用吗? -不包含。Setup 可以为你安装 Hopper,但 Hopper 仍是需要单独授权的软件。REA 提供 CLI、MCP 服务器和面向智能体的工作流。 +REA 在本机分析目标。智能体会接收工具结果,其模型提供商有自己的数据政策。
-
REA 会上传我的二进制文件吗? +
+遇到错误时该怎么办? -REA 不提供托管分析服务,而是通过本地 Unix 套接字把操作交给 Hopper。你的智能体或模型服务商可能有自己的数据政策,请单独核查。 +先更新;最近的版本可能已经修复了问题。 -
+通过 npm 安装的 CLI 使用: -
REA 能恢复原始源代码吗? +```bash +rea update +``` + +通过 `npx` 设置智能体时使用: + +```bash +npx rea-agents@latest setup +``` -不能保证。REA 提供伪代码、汇编、符号、字符串、元数据和关系,智能体可据此解释或兼容地重建观察到的行为。 +如果正在使用智能体,请完成[设置更新](#更新-rea)并重启智能体。重试同一任务。如果问题仍然存在,请[提交 issue](https://github.com/morluto/rea/issues),附上 REA 版本、目标类型、复现步骤和错误输出。
-## 开发 +## 文档 + +可以先阅读网站上的[实操指南](https://rea.tools/guides/)。具体选项、前提条件和结果契约见: + +- [安装与设置](docs/installation.md):智能体注册、提供方配置、更新和卸载。 +- [就绪检查与故障排查](docs/installation.md#check-readiness-for-your-task):诊断特定智能体或分析引擎。 +- [CLI 与 Evidence](docs/cli.md):命令、提供方选择、快照、导入/导出和退出状态。 +- [MCP 契约](docs/mcp-contracts.md)和[智能体提示词](docs/mcp-prompts.md):工具结果、会话和引导式调查。 +- [工具目录](docs/mcp-contracts.md#generated-catalog):构建时生成的工具、提供方和 CLI 命令清单。 +- [路线图](docs/roadmap.md):计划中的工作和能力跟踪。 -开发环境、架构、测试和发布说明请参阅 [CONTRIBUTING.md](CONTRIBUTING.md)。 +请按照 [SECURITY.md](SECURITY.md) 报告漏洞。 + +## 参与贡献 + +欢迎帮助改进 REA!你可以[提交 issue](https://github.com/morluto/rea/issues) 报告错误或建议功能,也可以[提交 pull request](https://github.com/morluto/rea/pulls) 改进代码或文档。 + +开发设置和检查见 [CONTRIBUTING.md](CONTRIBUTING.md),验证流程见[测试指南](docs/testing.md),项目结构见[架构图](docs/architecture.mermaid)。 + +## 项目链接 + +[网站](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [安全](SECURITY.md) ## Star 历史 -🎉 **GitHub Star 突破 20,000,感谢大家!** +🎉 **GitHub Star 达到 20,000 个,感谢大家!** -感谢每一位使用 REA、反馈问题、测试构建和贡献修复的朋友。 +感谢每一位使用 REA、报告错误、测试构建和贡献修复的朋友。 - REA GitHub Star 历史 + REA 的 GitHub Star 历史 +## 免责声明 + +REA 为合法的逆向工程研究、分析和重建提供工具。你有责任取得所需授权并遵守适用法律。项目不支持非法或未经授权的使用。 + ## 许可证 [MIT](LICENSE) diff --git a/docs/roadmap.md b/docs/roadmap.md index cd78ee103be..3832bf23882 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -22,7 +22,7 @@ installation changes and requires approval before writing or installing. ## Planned work -The [website guides](https://morluto.github.io/rea/guides/) and +The [website guides](https://rea.tools/guides/) and [generated catalog](product-catalog.json) describe available workflows. Current development priorities are: diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index 42dfe1db8f1..e1f7029c5d2 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -5,8 +5,7 @@ import { describe, expect, it } from "vitest"; import { SUPPORTED_CLIENT_DEFINITIONS } from "../../src/application/SupportedClients.js"; import { PRODUCT_IDENTITY } from "../../src/identity.js"; -const setupDocumentation = [ - "docs/installation.md", +const translatedReadmes = [ "README_zh.md", "README_ja.md", "README_ko.md", @@ -21,35 +20,66 @@ const jsonExamples = (content: string): unknown[] => JSON.parse(match[1] ?? ""), ); +const shellExamples = (content: string): string[] => + [...content.matchAll(/```bash\s*([\s\S]*?)```/gu)].map((match) => + (match[1] ?? "").trim(), + ); + +const linkedTargets = (content: string): string[] => + [...content.matchAll(/\]\(([^)\s]+)\)/gu)] + .map((match) => match[1] ?? "") + .filter( + (target) => + !target.startsWith("#") && !/^README(?:_[a-z]+)?\.md$/u.test(target), + ) + .sort(); + describe("onboarding documentation product facts", () => { - it.each(setupDocumentation)( - "keeps requirements and versioned MCP configuration aligned in %s", - async (path) => { - const content = await readFile(resolve(path), "utf8"); - expect(jsonExamples(content)).toContainEqual( - expect.objectContaining({ - mcpServers: expect.objectContaining({ - rea: expect.objectContaining({ - command: "npx", - args: [ - "-y", - PRODUCT_IDENTITY.registrationPackageSpecifier, - "mcp", - ], - }), + it("keeps detailed requirements and versioned MCP configuration aligned in the installation guide", async () => { + const content = await readFile(resolve("docs/installation.md"), "utf8"); + expect(jsonExamples(content)).toContainEqual( + expect.objectContaining({ + mcpServers: expect.objectContaining({ + rea: expect.objectContaining({ + command: "npx", + args: ["-y", PRODUCT_IDENTITY.registrationPackageSpecifier, "mcp"], }), }), + }), + ); + expect(content).toContain("Node.js 22"); + expect(content).toContain("macOS 12"); + expect(content).toContain("Ubuntu 24.04"); + expect(content).toContain("Fedora 41"); + expect(content).toContain("Arch Linux"); + expect(content).toContain("CachyOS"); + for (const client of SUPPORTED_CLIENT_DEFINITIONS) + expect(content).toContain(client.displayName); + }); + + it.each(translatedReadmes)( + "keeps onboarding commands, linked guidance and structure aligned with English in %s", + async (path) => { + const [english, translated] = await Promise.all([ + readFile(resolve("README.md"), "utf8"), + readFile(resolve(path), "utf8"), + ]); + expect(shellExamples(translated)).toEqual(shellExamples(english)); + expect(linkedTargets(translated)).toEqual(linkedTargets(english)); + const headingLevels = (content: string): number[] => + [...content.matchAll(/^(#{1,6})\s/gmu)].map( + (match) => match[1]?.length ?? 0, + ); + expect(headingLevels(translated)).toEqual(headingLevels(english)); + expect(translated.match(//gu)?.length).toBe( + english.match(//gu)?.length, ); - expect(content).toContain("Node.js 22"); - expect(content).toContain("macOS 12"); - expect(content).toContain("Ubuntu 24.04"); - expect(content).toContain("Fedora 41"); - expect(content).toContain("Arch Linux"); - expect(content).toContain("CachyOS"); - for (const client of SUPPORTED_CLIENT_DEFINITIONS) - expect(content).toContain(client.displayName); - if (path.startsWith("README_")) - expect(content).toContain("MCP-tool_catalog"); + expect(translated).toContain("Node.js 22.x"); + expect(translated).toContain(">=22.19"); + expect(translated).toContain("24.x"); + expect(translated).toContain(">=24.11"); + expect(translated).toContain("26+"); + expect(translated).toContain("MCP-tool_catalog"); }, ); From 2f59287f1a1794a68102a45c61cc657659fa5de6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:55:51 +0800 Subject: [PATCH 017/791] feat(website): add GitHub link with live star count --- website/public/assets/main.js | 29 +++++++++++++++ website/public/assets/styles.css | 40 ++++++++++++++++++--- website/public/faq/index.html | 21 ++++++++++- website/public/get-started/index.html | 21 ++++++++++- website/public/guides/browser/index.html | 21 ++++++++++- website/public/guides/index.html | 21 ++++++++++- website/public/guides/javascript/index.html | 21 ++++++++++- website/public/guides/native/index.html | 21 ++++++++++- website/public/index.html | 21 ++++++++++- website/public/showcase/ctf/index.html | 21 ++++++++++- website/public/showcase/dx-ball/index.html | 21 ++++++++++- website/public/showcase/index.html | 21 ++++++++++- website/public/showcase/notion/index.html | 21 ++++++++++- website/public/showcase/th04/index.html | 21 ++++++++++- 14 files changed, 305 insertions(+), 16 deletions(-) diff --git a/website/public/assets/main.js b/website/public/assets/main.js index e11e0c1614e..98751f28081 100644 --- a/website/public/assets/main.js +++ b/website/public/assets/main.js @@ -106,3 +106,32 @@ function initializeBackToTop() { } initializeBackToTop(); + +async function initializeGitHubStars() { + const link = document.querySelector(".nav-github"); + const count = link?.querySelector(".nav-github-count"); + if (count === null || count === undefined) return; + + try { + const response = await fetch("https://api.github.com/repos/morluto/rea", { + headers: { Accept: "application/vnd.github+json" }, + credentials: "omit", + signal: AbortSignal.timeout(5000), + }); + if (!response.ok) return; + const repository = await response.json(); + const stars = repository.stargazers_count; + if (!Number.isSafeInteger(stars) || stars < 0) return; + + const formatted = new Intl.NumberFormat("en-US").format(stars); + const label = `REA on GitHub (${formatted} ${stars === 1 ? "star" : "stars"})`; + count.textContent = formatted; + count.hidden = false; + link.setAttribute("aria-label", label); + link.setAttribute("title", label); + } catch { + // Keep the repository link usable when GitHub is unavailable. + } +} + +initializeGitHubStars(); diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index a7694af9b78..8a21650a89e 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -129,9 +129,9 @@ section[id] { } .header-inner { min-height: 88px; - display: flex; + display: grid; + grid-template-columns: auto minmax(0, 1fr) auto; align-items: center; - justify-content: space-between; gap: 2rem; } .brand { @@ -153,6 +153,7 @@ section[id] { .navigation { display: flex; align-items: center; + justify-content: flex-end; gap: 1.8rem; font-size: 0.9rem; } @@ -171,6 +172,32 @@ section[id] { .navigation .nav-start { color: var(--ink); } +.nav-github { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 44px; + height: 44px; + padding-inline: 10px; + gap: 8px; + border: 1px solid var(--line); + border-radius: 10px; + color: var(--muted); + text-decoration: none; +} +.nav-github:hover { + background: var(--soft); + color: var(--ink); +} +.nav-github svg { + display: block; + flex-shrink: 0; +} +.nav-github-count { + font-size: 0.875rem; + font-weight: 600; + font-variant-numeric: tabular-nums; +} .faq-list { border-top: 1px solid var(--line); } @@ -1078,18 +1105,23 @@ pre code { } .header-inner { min-height: 78px; - flex-wrap: wrap; - justify-content: flex-start; + grid-template-columns: minmax(0, 1fr) auto; gap: 0.55rem; padding-block: 0.8rem; } .navigation { + grid-row: 2; + grid-column: 1 / -1; width: 100%; justify-content: space-between; flex-wrap: wrap; gap: 0.5rem 0.25rem; font-size: 0.875rem; } + .nav-github { + grid-row: 1; + grid-column: 2; + } .hero { padding-block: 3.5rem 3rem; } diff --git a/website/public/faq/index.html b/website/public/faq/index.html index f6e24ca095c..9ef107adc3a 100644 --- a/website/public/faq/index.html +++ b/website/public/faq/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/get-started/index.html b/website/public/get-started/index.html index 042c74c4570..25b3e57c609 100644 --- a/website/public/get-started/index.html +++ b/website/public/get-started/index.html @@ -25,8 +25,27 @@ ShowcasesGet startedGuidesFAQ - GitHub + + + +
diff --git a/website/public/guides/browser/index.html b/website/public/guides/browser/index.html index b330215c0b3..1858727d140 100644 --- a/website/public/guides/browser/index.html +++ b/website/public/guides/browser/index.html @@ -25,8 +25,27 @@ ShowcasesGet startedGuidesFAQ - GitHub + + + +
diff --git a/website/public/guides/index.html b/website/public/guides/index.html index d6c953d4d2f..b84b11176cf 100644 --- a/website/public/guides/index.html +++ b/website/public/guides/index.html @@ -25,8 +25,27 @@ ShowcasesGet startedGuidesFAQ - GitHub + + + +
diff --git a/website/public/guides/javascript/index.html b/website/public/guides/javascript/index.html index e7186f0ce54..b7d559c50ec 100644 --- a/website/public/guides/javascript/index.html +++ b/website/public/guides/javascript/index.html @@ -25,8 +25,27 @@ ShowcasesGet startedGuidesFAQ - GitHub + + + +
diff --git a/website/public/guides/native/index.html b/website/public/guides/native/index.html index ba28535c3c1..d5fd3005e82 100644 --- a/website/public/guides/native/index.html +++ b/website/public/guides/native/index.html @@ -25,8 +25,27 @@ ShowcasesGet startedGuidesFAQ - GitHub + + + +
diff --git a/website/public/index.html b/website/public/index.html index c08ccca61c7..e315e63ff28 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/showcase/ctf/index.html b/website/public/showcase/ctf/index.html index e7055038e95..d3ed8e85cb2 100644 --- a/website/public/showcase/ctf/index.html +++ b/website/public/showcase/ctf/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/showcase/dx-ball/index.html b/website/public/showcase/dx-ball/index.html index 12cc6854faf..8394bb56c5c 100644 --- a/website/public/showcase/dx-ball/index.html +++ b/website/public/showcase/dx-ball/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/showcase/index.html b/website/public/showcase/index.html index 23a112d9b78..4f30835639a 100644 --- a/website/public/showcase/index.html +++ b/website/public/showcase/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/showcase/notion/index.html b/website/public/showcase/notion/index.html index d5dacc17828..134d173c503 100644 --- a/website/public/showcase/notion/index.html +++ b/website/public/showcase/notion/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
diff --git a/website/public/showcase/th04/index.html b/website/public/showcase/th04/index.html index 1fd59e7079f..84937ee32d0 100644 --- a/website/public/showcase/th04/index.html +++ b/website/public/showcase/th04/index.html @@ -26,8 +26,27 @@ Get started Guides FAQ - GitHub + + + +
From 704fa7699ca183d5d5e0518a6fcdbb953ec35a10 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:10:31 +0800 Subject: [PATCH 018/791] fix(ghidra): preserve source failures and snapshot ownership --- docs/native-investigation.md | 10 + docs/testing.md | 12 ++ scripts/lib/real-ghidra-boundaries.mjs | 13 ++ scripts/lib/real-ghidra-target-admission.mjs | 191 ++++++++++++++++++ src/domain/analysisErrorPresentation.ts | 2 +- src/ghidra/GhidraProviderClient.ts | 16 ++ src/ghidra/GhidraTargetSnapshot.ts | 80 ++++++-- .../ghidra/ghidraTargetSnapshot.test.ts | 54 +---- 8 files changed, 316 insertions(+), 62 deletions(-) create mode 100644 scripts/lib/real-ghidra-target-admission.mjs diff --git a/docs/native-investigation.md b/docs/native-investigation.md index eea5a8d7da5..6255d5e4e17 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -110,6 +110,16 @@ After metadata edits, snapshot saves and imports remain unavailable in that session. Use `export_evidence_bundle` to retain the observations, close without `snapshot_path`, and reopen the target before importing or saving a snapshot. +Ghidra checks its private import copy against the artifact digest selected by +`open_binary`. If the source changes before that copy is acquired, the query +reports `artifact_changed` with the selected path and both digests. Reopen the +stable target to acquire its current identity. An already imported database +continues to describe its captured bytes even if the original path is changed +or removed; this identity failure does not mark the Ghidra installation unavailable. +On Linux and macOS, an unreadable source reports `access_denied`; a removed or +nonregular replacement reports `artifact_changed`. Source copying is cancellable +and never replaces an existing private snapshot. + ```bash rea inspect-native-instruction
--provider ghidra rea resolve-native-call-targets --provider ghidra diff --git a/docs/testing.md b/docs/testing.md index e0ffb7370fa..3dad3fd7ad8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -308,6 +308,18 @@ snapshot is written by an independent real CLI session. They also start a real annotation and snapshot close concurrently: the edit must succeed, the snapshot must be rejected without creating a file, and the edited session must remain usable until explicitly closed. +Source-admission probes change a caller-owned fixture after `open_binary` but +before the first Ghidra query. They require an actionable `artifact_changed` +error preserving both digests and the selected path, unchanged provider +availability, failed-copy cleanup, and successful recovery after reopening. +After import, deleting that source must preserve the captured database identity. +Instruction inspection and containing-function lookup also agree across +hexadecimal case, leading zeros, and encoded default address-space spellings. +The same source-acquisition workflow exercises missing and directory-replaced +inputs, plus real read-permission denial on a non-root host. Root runs report +that permission-denial check as unverified. A focused filesystem integration +check retains selected-platform routing and exclusive creation; two copy/digest +checks duplicated by the real workflow have been removed. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 8ba6974a313..0de39100230 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -10,6 +10,7 @@ import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot-e2e.mjs"; import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; +import { verifyGhidraTargetAdmission } from "./real-ghidra-target-admission.mjs"; /** Probe real Ghidra location, annotation and error contracts through public adapters. */ export async function verifyGhidraBoundaries( @@ -648,6 +649,13 @@ export async function verifyGhidraBoundaries( ); successfulCalls += snapshotLifecycle.successfulCalls; rejectedCalls += snapshotLifecycle.rejectedCalls; + const targetAdmission = await verifyGhidraTargetAdmission({ + call, + reject: invalid, + target, + entry: address, + runtimeRoot: env.TMPDIR, + }); await assert.rejects(access(socketRoot), { code: "ENOENT" }); await assert.rejects(access(runtimeRoot), { code: "ENOENT" }); assert.equal(await call("procedure_address", { procedure: name }), address); @@ -683,6 +691,11 @@ export async function verifyGhidraBoundaries( legacy_reference_snapshot_rejected: true, mutation_snapshot_lifecycle: true, concurrent_annotation_snapshot_close: true, + changed_source_admission_and_recovery: true, + source_permission_denial_verified: targetAdmission.permissionDenied, + missing_and_nonregular_source_rejected: true, + imported_source_identity_retained: true, + equivalent_instruction_address_spellings: true, source_immutable: true, reopen_discards_edits: true, long_tmpdir_private_socket_cleanup: true, diff --git a/scripts/lib/real-ghidra-target-admission.mjs b/scripts/lib/real-ghidra-target-admission.mjs new file mode 100644 index 00000000000..2c256c15428 --- /dev/null +++ b/scripts/lib/real-ghidra-target-admission.mjs @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { + chmod, + mkdir, + mkdtemp, + readFile, + realpath, + readdir, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** Verify changed-source admission and retained private database identity through MCP. */ +export async function verifyGhidraTargetAdmission({ + call, + reject, + target, + entry, + runtimeRoot, +}) { + const workspace = await mkdtemp( + join(tmpdir(), "rea-ghidra-source-admission-"), + ); + const source = join(workspace, "caller-selected.%PATH%"); + try { + const original = await readFile(target.path); + await writeFile(source, original); + await call("close_binary"); + const opened = await call("open_binary", { + path: source, + provider_id: "ghidra", + }); + assert.equal(opened.sha256, target.sha256); + const selectedPath = await realpath(source); + assert.equal(opened.path, selectedPath); + const before = await call("binary_session"); + const changed = Buffer.concat([ + original, + Buffer.from("REA_SOURCE_CHANGED_AFTER_OPEN"), + ]); + const changedSha256 = createHash("sha256").update(changed).digest("hex"); + await writeFile(source, changed); + const failure = await reject("analyze_function", { procedure: entry }); + assert.equal(failure.code, "artifact_changed"); + assert.equal(failure.category, "integrity_mismatch"); + assert.equal(failure.details.operation, "analyze_function"); + assert.equal(failure.details.path, selectedPath); + assert.ok(failure.details.reason.includes(target.sha256)); + assert.ok(failure.details.reason.includes(changedSha256)); + assert.match(failure.remediation.action, /open_binary/u); + assert.doesNotMatch(failure.remediation.action, /doctor/u); + const after = await call("binary_session"); + assert.equal(after.analysis_run.run_id, before.analysis_run.run_id); + assert.equal(after.sha256, target.sha256); + assert.equal( + after.capabilities.find( + ({ operation }) => operation === "analyze_function", + )?.available, + true, + "An artifact identity mismatch must not mark the installed provider unavailable", + ); + assert.deepEqual( + (await readdir(runtimeRoot)).filter((name) => + name.startsWith("rea-ghidra-"), + ), + [], + "Failed admission must remove the private snapshot without launching Ghidra", + ); + assert.deepEqual(await readFile(source), changed); + const permissionDenied = await verifySourceReadFailures({ + call, + reject, + source, + bytes: changed, + entry, + runtimeRoot, + }); + + const reopened = await call("open_binary", { + path: source, + provider_id: "ghidra", + }); + assert.equal(reopened.sha256, changedSha256); + assert.notEqual( + (await call("binary_session")).analysis_run.run_id, + before.analysis_run.run_id, + ); + const dossier = await call("analyze_function", { procedure: entry }); + const instruction = await call("inspect_native_instruction", { + address: entry, + }); + assert.equal(dossier.procedure.address, entry); + assert.equal(instruction.status, "decoded"); + for (const address of [ + entry.toUpperCase(), + entry.slice(2), + `0x0000${entry.slice(2)}`, + `ram:${entry}`, + `%72%61%6D:${entry}`, + ]) { + assert.deepEqual( + await call("inspect_native_instruction", { address }), + instruction, + ); + assert.equal( + (await call("resolve_containing_procedure", { address })).procedure + .address, + entry, + ); + } + // Imported facts belong to the captured bytes even if the caller's path disappears. + await rm(source); + assert.deepEqual( + await call("analyze_function", { procedure: entry }), + dossier, + ); + assert.deepEqual( + await call("inspect_native_instruction", { address: entry }), + instruction, + ); + assert.equal((await call("binary_session")).sha256, changedSha256); + await call("close_binary"); + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + return { permissionDenied }; + } finally { + await rm(workspace, { recursive: true, force: true }); + } +} + +async function verifySourceReadFailures({ + call, + reject, + source, + bytes, + entry, + runtimeRoot, +}) { + const permissionDenied = + typeof process.getuid === "function" && process.getuid() !== 0; + const scenarios = [ + "missing", + "directory", + ...(permissionDenied ? ["permission"] : []), + ]; + for (const scenario of scenarios) { + await call("open_binary", { path: source, provider_id: "ghidra" }); + const before = await call("binary_session"); + if (scenario === "permission") await chmod(source, 0); + else { + await rm(source); + if (scenario === "directory") await mkdir(source); + } + try { + const failure = await reject("analyze_function", { procedure: entry }); + assert.equal( + failure.code, + scenario === "permission" ? "access_denied" : "artifact_changed", + ); + assert.equal(failure.details.path, before.path); + assert.equal(failure.details.operation, "analyze_function"); + if (scenario === "permission") { + assert.ok(["EACCES", "EPERM"].includes(failure.details.system_code)); + assert.match(failure.remediation.action, /read access/u); + } else assert.match(failure.remediation.action, /open_binary/u); + assert.doesNotMatch(failure.remediation.action, /doctor/u); + const after = await call("binary_session"); + assert.equal(after.analysis_run.run_id, before.analysis_run.run_id); + assert.equal(after.sha256, before.sha256); + assert.equal( + after.capabilities.find( + ({ operation }) => operation === "analyze_function", + )?.available, + true, + ); + assert.deepEqual( + (await readdir(runtimeRoot)).filter((name) => + name.startsWith("rea-ghidra-"), + ), + [], + ); + } finally { + if (scenario === "permission") await chmod(source, 0o600); + await rm(source, { recursive: true, force: true }); + await writeFile(source, bytes); + } + } + return permissionDenied; +} diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index ff18238974d..56b0ad2736a 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -75,7 +75,7 @@ export const analysisErrorRemediationAction = ( if (error instanceof AnalysisAccessDeniedError) return "Check the current process's read access to the selected path. Retry with a readable local file."; if (error instanceof AnalysisArtifactChangedError) - return "Wait until the selected file is stable, then retry this operation."; + return "Wait until the selected file is stable. For an active binary session, reopen the target with open_binary before retrying so REA acquires its current identity; for a CLI command or target-free tool, rerun the operation."; if (error instanceof AnalysisInputError) return "Correct the listed arguments and retry."; if (error instanceof UnknownRegistryError && error.reason === "not-found") diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 00a1cfa8762..aed08e04b44 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -11,6 +11,8 @@ import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js"; import type { BinaryTarget } from "../domain/binaryTarget.js"; import { AnalysisCancelledError, + AnalysisArtifactChangedError, + AnalysisAccessDeniedError, AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisResourceConstraintError, @@ -319,6 +321,20 @@ const projectSessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, ): AnalysisError => { + if (failure.cause instanceof AnalysisAccessDeniedError) + return new AnalysisAccessDeniedError( + operation, + failure.cause.path, + failure.cause.systemCode, + { cause: failure }, + ); + if (failure.cause instanceof AnalysisArtifactChangedError) + return new AnalysisArtifactChangedError( + operation, + failure.cause.path, + failure.cause.reason, + { cause: failure }, + ); if ( operation === "annotate_native_function" && failure.kind === "remote" && diff --git a/src/ghidra/GhidraTargetSnapshot.ts b/src/ghidra/GhidraTargetSnapshot.ts index d30618d0f3a..b6447bc7577 100644 --- a/src/ghidra/GhidraTargetSnapshot.ts +++ b/src/ghidra/GhidraTargetSnapshot.ts @@ -1,9 +1,15 @@ import { createHash } from "node:crypto"; -import { constants, createReadStream } from "node:fs"; -import { chmod, copyFile, rm } from "node:fs/promises"; +import { constants } from "node:fs"; +import { open, rm, type FileHandle } from "node:fs/promises"; import { extname, join } from "node:path"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; import type { JsonValue } from "../domain/jsonValue.js"; +import { + AnalysisArtifactChangedError, + AnalysisAccessDeniedError, +} from "../domain/analysisErrorCore.js"; import { windowsPrivateRuntime } from "../windows/WindowsPrivateRuntime.js"; /** Immutable target copy admitted to one ephemeral Ghidra import. */ @@ -37,7 +43,9 @@ export const createGhidraTargetSnapshot = async ( options.signal, ); if (observed.sha256 !== expectedSha256) - throw new Error( + throw new AnalysisArtifactChangedError( + "open_binary", + sourcePath, `Ghidra target snapshot digest mismatch: expected ${expectedSha256}, observed ${observed.sha256}`, ); return { @@ -46,23 +54,71 @@ export const createGhidraTargetSnapshot = async ( admission: { source: observed.source, snapshot: observed.snapshot }, }; } + let source: FileHandle | undefined; + let destination: FileHandle | undefined; try { - await copyFile(sourcePath, snapshotPath, constants.COPYFILE_EXCL); - await chmod(snapshotPath, 0o600); + source = await openSnapshotSource(sourcePath); + if (!(await source.stat()).isFile()) + throw new AnalysisArtifactChangedError( + "open_binary", + sourcePath, + "The selected Ghidra source is no longer a regular file", + ); + destination = await open(snapshotPath, "wx", 0o600); + await destination.chmod(0o600); const hash = createHash("sha256"); - for await (const chunk of createReadStream(snapshotPath)) - hash.update(chunk); + const digest = new Transform({ + transform(chunk: unknown, _encoding, done) { + if (!Buffer.isBuffer(chunk)) { + done(new TypeError("Ghidra snapshot stream must contain bytes")); + return; + } + hash.update(chunk); + done(null, chunk); + }, + }); + await pipeline( + source.createReadStream(), + digest, + destination.createWriteStream(), + options.signal === undefined ? {} : { signal: options.signal }, + ); const observedSha256 = hash.digest("hex"); if (observedSha256 !== expectedSha256) - throw new Error( + throw new AnalysisArtifactChangedError( + "open_binary", + sourcePath, `Ghidra target snapshot digest mismatch: expected ${expectedSha256}, observed ${observedSha256}`, ); return { path: snapshotPath, sha256: observedSha256 }; } catch (cause: unknown) { - await rm(snapshotPath, { force: true }).catch((cause: unknown) => { - // best-effort cleanup: snapshot removal must not mask the copy failure. - void cause; - }); + if (destination !== undefined) + await rm(snapshotPath, { force: true }).catch((cause: unknown) => { + // best-effort cleanup: snapshot removal must not mask the copy failure. + void cause; + }); + throw cause; + } finally { + await Promise.all([source?.close(), destination?.close()]); + } +}; + +const openSnapshotSource = async (path: string): Promise => { + try { + // A path replaced by a FIFO must not block startup before its type is checked. + return await open(path, constants.O_RDONLY | constants.O_NONBLOCK); + } catch (cause: unknown) { + const code = + cause instanceof Error && "code" in cause ? cause.code : undefined; + if (code === "EACCES" || code === "EPERM") + throw new AnalysisAccessDeniedError("open_binary", path, code, { cause }); + if (code === "ENOENT" || code === "ENOTDIR" || code === "ELOOP") + throw new AnalysisArtifactChangedError( + "open_binary", + path, + `The selected Ghidra source could not be opened (${code})`, + { cause }, + ); throw cause; } }; diff --git a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts index 30e796d1504..13660b944a7 100644 --- a/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts +++ b/tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts @@ -1,20 +1,16 @@ import { createHash } from "node:crypto"; -import { access, readFile, rm, writeFile } from "node:fs/promises"; +import { readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; -import { createGhidraTestRuntime } from "../../../fixtures/ghidraRuntime.js"; -import type { PrivateRuntimeRoot } from "../../../../src/process/PrivateRuntimeRoot.js"; import { createGhidraTargetSnapshot } from "../../../../src/ghidra/GhidraTargetSnapshot.js"; const roots: string[] = []; -const runtimes: PrivateRuntimeRoot[] = []; afterEach(async () => { - await Promise.all(runtimes.splice(0).map((runtime) => runtime.close())); await Promise.all( roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), ); @@ -34,51 +30,11 @@ describe("Ghidra target snapshot", () => { expect(snapshot.admission).toBeUndefined(); await expect(readFile(snapshot.path)).resolves.toEqual(bytes); await expect( - createGhidraTargetSnapshot(source, root, sha256, { platform: "win32" }), - ).rejects.toThrow(/No native private runtime owns/u); - }); - it("copies an exact digest-bound target into the private runtime", async () => { - const root = await createTestTempDirectory("rea-ghidra-snapshot-"); - roots.push(root); - const source = join(root, "source.exe"); - const bytes = Buffer.from("native PE fixture"); - const sha256 = createHash("sha256").update(bytes).digest("hex"); - await writeFile(source, bytes); - const runtime = await createGhidraTestRuntime(root); - runtimes.push(runtime); - - const snapshot = await createGhidraTargetSnapshot( - source, - runtime.path, - sha256, - ); - - expect(snapshot).toMatchObject({ - path: join(runtime.path, `target-${sha256.slice(0, 12)}.exe`), - sha256, - }); + createGhidraTargetSnapshot(source, root, sha256, { platform: "linux" }), + ).rejects.toMatchObject({ code: "EEXIST" }); await expect(readFile(snapshot.path)).resolves.toEqual(bytes); - }); - - it("rejects a mismatched digest and removes the failed runtime snapshot", async () => { - const root = await createTestTempDirectory("rea-ghidra-snapshot-"); - roots.push(root); - const source = join(root, "source with unsafe extension.%PATH%"); - await writeFile(source, "changed target"); - const runtime = await createGhidraTestRuntime(root); - runtimes.push(runtime); - const expected = "a".repeat(64); - const snapshotPath = join( - runtime.path, - `target-${expected.slice(0, 12)}.bin`, - ); - await expect( - createGhidraTargetSnapshot(source, runtime.path, expected), - ).rejects.toThrow(/digest mismatch/u); - await runtime.close(); - await expect(access(snapshotPath)).rejects.toMatchObject({ - code: "ENOENT", - }); + createGhidraTargetSnapshot(source, root, sha256, { platform: "win32" }), + ).rejects.toThrow(/No native private runtime owns/u); }); }); From b2d5bc110914f2ad42d17bc739d5f37c1b0f0a4d Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:38:55 +0800 Subject: [PATCH 019/791] fix(hopper): validate annotation destinations and symbol extents --- bridge/hopper_bridge.py | 29 ++- docs/installation.md | 7 +- scripts/lib/real-hopper-navigation.mjs | 315 ++++++++++++++----------- src/contracts/officialToolContracts.ts | 8 +- 4 files changed, 218 insertions(+), 141 deletions(-) diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 8f3397e3e01..49240834256 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -13,6 +13,8 @@ from typing import Any, Optional, Protocol, Sequence BAD_ADDRESSES = (-1, 0xFFFFFFFFFFFFFFFF, None) +# The native label setter truncates NSString names at this UTF-16 extent. +HOPPER_SYMBOL_NAME_UTF16_UNITS = 1024 _rea_session_document = None _search_inventory_cache = {} _pseudocode_cache = {} @@ -211,6 +213,7 @@ def _validate_name_targets(document, names): raise InvalidRequestError("Batch rename selects the same address more than once; use one canonical address per target") labels = set() for _, address, name in names: + _segment(document, address) if not name: continue if name in labels: @@ -221,6 +224,19 @@ def _validate_name_targets(document, names): raise InvalidRequestError("Name %s is already assigned to %s; include that address in the batch with a replacement name" % (name, _hex(owner))) +def _symbol_name(value, field): + """Reject native label truncation without altering the caller's name.""" + value = _api_text(value, field) + if len(value.encode("utf-16-le")) // 2 > HOPPER_SYMBOL_NAME_UTF16_UNITS: + raise InvalidRequestError("%s exceeds Hopper's %s UTF-16 code-unit symbol-name limit" % (field, HOPPER_SYMBOL_NAME_UTF16_UNITS)) + return value + + +def _name_matches(document, address, name): + observed = _segment(document, address).getNameAtAddress(address) + return observed == name or (not name and observed is None) + + def _address(document, value=None): """Keep explicit hexadecimal coordinates distinct from symbol names.""" if value is None: @@ -849,25 +865,25 @@ def _dispatch(method, params): } if method == "set_address_name": address = _address(document, params.get("address")) - name = _api_text(params["name"], "Name") + name = _symbol_name(params["name"], "Name") _validate_name_targets(document, [(params.get("address"), address, name)]) try: result = document.setNameAtAddress(address, name) finally: _invalidate_search_inventory(document) _invalidate_pseudocode(document) - return result + return bool(result) and _name_matches(document, address, name) if method == "set_addresses_names": # Validate all destinations before applying any annotation. A malformed # later address must not discard the result of an earlier mutation. - names = [(key, _address(document, key), _api_text(value, "Name for %s" % key)) for key, value in params["names"].items()] + names = [(key, _address(document, key), _symbol_name(value, "Name for %s" % key)) for key, value in params["names"].items()] _validate_name_targets(document, names) try: result = {key: document.setNameAtAddress(address, value) for key, address, value in names} finally: _invalidate_search_inventory(document) _invalidate_pseudocode(document) - return result + return {key: bool(result[key]) and _name_matches(document, address, value) for key, address, value in names} if method in ("set_comment", "set_inline_comment"): address = _address(document, params.get("address")) comment = _api_text(params["comment"], "Comment") @@ -882,6 +898,7 @@ def _dispatch(method, params): return [{"address": _hex(item), "name": document.getBookmarkName(item)} for item in document.getBookmarks()] if method == "set_bookmark": address = _address(document, params.get("address")) + _segment(document, address) name = params.get("name") if name is not None: _api_text(name, "Bookmark name") @@ -889,6 +906,10 @@ def _dispatch(method, params): return document.hasBookmarkAtAddress(address) if method == "unset_bookmark": address = _address(document, params.get("address")) + # Permit removing a legacy orphan bookmark, but do not report success + # for an unmapped address with no bookmark to remove. + if not document.hasBookmarkAtAddress(address): + _segment(document, address) document.removeBookmarkAtAddress(address) return not document.hasBookmarkAtAddress(address) raise InvalidRequestError("Unknown bridge method") diff --git a/docs/installation.md b/docs/installation.md index dec6f60d098..7ac8053f356 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -323,7 +323,12 @@ Cursor navigation returns the observed object start when Hopper snaps an interio address; adjacent-object navigation rejects unmapped inputs and document ends. Native API text rejects NUL characters and unpaired Unicode surrogates before annotation changes. Renames preserve unselected label owners; use a batch with -all affected addresses to move or swap existing labels explicitly. +all affected addresses to move or swap existing labels explicitly. Every rename +destination must be mapped, and native symbol names must fit Hopper's 1024 UTF-16 +code-unit limit. Oversized names fail before any batch edits; bookmarks and +literal string results are not subject to that symbol-name limit. Rename success +requires exact final readback. New bookmarks must point into mapped memory; +existing legacy bookmarks outside it can still be removed. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/scripts/lib/real-hopper-navigation.mjs b/scripts/lib/real-hopper-navigation.mjs index 9d1eb1577b9..3ee47a7a5ad 100644 --- a/scripts/lib/real-hopper-navigation.mjs +++ b/scripts/lib/real-hopper-navigation.mjs @@ -13,144 +13,18 @@ export async function verifyHopperNavigationAndText(call, invalid, address) { (item) => item.address === address, ); try { - const instructions = await call("read_function_instructions", { - procedure: address, + await verifyNavigation(call, invalid, address, { + originalName, + originalComment, + originalInline, }); - const info = await call("procedure_info", { procedure: address }); - assert.ok(info.locals.length > 0, "fixture omitted real stack locals"); - for (const local of info.locals) { - assert.equal(typeof local.name, "string"); - assert.match(local.stack_displacement, /^-?\d+$/u); - assert.ok(local.description.includes(local.name)); - assert.ok(local.description.includes(local.stack_displacement)); - assert.doesNotMatch(local.description, /object at 0x/u); - } - const dossier = await call("analyze_function", { procedure: address }); - assert.deepEqual(dossier.procedure.locals, info.locals); - const coordinates = instructions.instructions.map( - (line) => /^0x[0-9a-f]+/u.exec(line)?.[0], - ); - assert.equal(coordinates[0], address); - assert.ok(coordinates[1], "fixture omitted a second instruction"); - const interior = `0x${(BigInt(address) + 1n).toString(16)}`; - assert.equal(await call("goto_address", { address: interior }), address); - assert.equal(await call("current_address"), address); - assert.equal( - await call("next_address", { address: interior }), - coordinates[1], - ); - assert.equal(await call("prev_address", { address: interior }), address); - for (const unmapped of ["0x0", "0xfffffffffffffffe"]) { - for (const operation of ["goto_address", "next_address", "prev_address"]) - await invalid( - operation, - { address: unmapped }, - /Address is outside every segment/u, - ); - assert.equal(await call("current_address"), address); - } - const segments = await call("list_segments"); - const first = segments.reduce((left, right) => - BigInt(left.start) < BigInt(right.start) ? left : right, - ); - const last = segments.reduce((left, right) => - BigInt(left.end) > BigInt(right.end) ? left : right, - ); - await invalid( - "prev_address", - { address: first.start }, - /No previous address exists in mapped memory/u, - ); - await invalid( - "next_address", - { address: `0x${(BigInt(last.end) - 1n).toString(16)}` }, - /No adjacent analyzed object exists in mapped memory/u, - ); - - const comment = "REA representation boundary 测试"; - await call("set_comment", { address, comment }); - await call("set_inline_comment", { address, comment }); - await call("set_bookmark", { address, name: comment }); - for (const [text, reason] of [ - ["before\0after", /contains a NUL character/u], - ["before\ud800after", /contains an unpaired Unicode surrogate/u], - ]) { - for (const operation of ["procedure_info", "read_bytes"]) - await invalid( - operation, - operation === "procedure_info" - ? { procedure: text } - : { address: text, length: 1 }, - reason, - ); - for (const operation of ["set_address_name", "set_bookmark"]) - await invalid(operation, { address, name: text }, reason); - for (const operation of ["set_comment", "set_inline_comment"]) - await invalid(operation, { address, comment: text }, reason); - await invalid( - "set_addresses_names", - { names: { [address]: "must_not_apply", [other.address]: text } }, - reason, - ); - assert.equal(await call("address_name", { address }), originalName); - assert.equal( - await call("address_name", { address: other.address }), - other.value, - ); - assert.equal(await call("comment", { address }), comment); - assert.equal(await call("inline_comment", { address }), comment); - assert.equal( - (await call("list_bookmarks")).find((item) => item.address === address) - ?.name, - comment, - ); - } - await invalid( - "set_address_name", - { address, name: other.value }, - /already assigned to/u, - ); - await invalid( - "set_addresses_names", - { names: { [address]: other.value } }, - /already assigned to/u, - ); - await invalid( - "set_addresses_names", - { names: { [address]: "duplicate", [other.address]: "duplicate" } }, - /assigns the same name more than once/u, - ); - await invalid( - "set_addresses_names", - { names: { [address]: "first", [address.slice(2)]: "second" } }, - /selects the same address more than once/u, - ); - assert.equal(await call("address_name", { address }), originalName); - assert.equal( - await call("address_name", { address: other.address }), - other.value, - ); - try { - const swapped = await call("set_addresses_names", { - names: { [address]: other.value, [other.address]: originalName }, - }); - assert.equal(swapped[address], true); - assert.equal(swapped[other.address], true); - assert.equal(await call("address_name", { address }), other.value); - assert.equal( - await call("address_name", { address: other.address }), - originalName, - ); - } finally { - await call("set_addresses_names", { - names: { [address]: originalName, [other.address]: other.value }, - }); - } + await verifyAnnotationText(call, invalid, { address, other, originalName }); return { cursorReadback: true, interiorObjectNavigation: true, unmappedNavigationRejected: true, nativeTextValidationBeforeMutation: true, + mappedAnnotationValidationBeforeMutation: true, unselectedLabelOwnersPreserved: true, explicitLabelSwap: true, observedLocalMetadata: true, @@ -173,3 +47,180 @@ export async function verifyHopperNavigationAndText(call, invalid, address) { }); } } + +async function verifyNavigation( + call, + invalid, + address, + { originalName, originalComment, originalInline }, +) { + const instructions = await call("read_function_instructions", { + procedure: address, + }); + const info = await call("procedure_info", { procedure: address }); + assert.ok(info.locals.length > 0, "fixture omitted real stack locals"); + for (const local of info.locals) { + assert.equal(typeof local.name, "string"); + assert.match(local.stack_displacement, /^-?\d+$/u); + assert.ok(local.description.includes(local.name)); + assert.ok(local.description.includes(local.stack_displacement)); + assert.doesNotMatch(local.description, /object at 0x/u); + } + const dossier = await call("analyze_function", { procedure: address }); + assert.deepEqual(dossier.procedure.locals, info.locals); + const coordinates = instructions.instructions.map( + (line) => /^0x[0-9a-f]+/u.exec(line)?.[0], + ); + assert.equal(coordinates[0], address); + assert.ok(coordinates[1], "fixture omitted a second instruction"); + const interior = `0x${(BigInt(address) + 1n).toString(16)}`; + assert.equal(await call("goto_address", { address: interior }), address); + assert.equal(await call("current_address"), address); + assert.equal( + await call("next_address", { address: interior }), + coordinates[1], + ); + assert.equal(await call("prev_address", { address: interior }), address); + for (const unmapped of ["0x0", "0xfffffffffffffffe"]) { + for (const operation of ["goto_address", "next_address", "prev_address"]) + await invalid( + operation, + { address: unmapped }, + /Address is outside every segment/u, + ); + assert.equal(await call("current_address"), address); + for (const operation of [ + "set_address_name", + "set_bookmark", + "unset_bookmark", + "set_comment", + "set_inline_comment", + ]) + await invalid( + operation, + { + address: unmapped, + ...(operation === "unset_bookmark" + ? {} + : operation.endsWith("comment") + ? { comment: "must_not_apply" } + : { name: "must_not_apply" }), + }, + /Address is outside every segment/u, + ); + await invalid( + "set_addresses_names", + { names: { [address]: "must_not_apply", [unmapped]: "unmapped" } }, + /Address is outside every segment/u, + ); + assert.equal(await call("address_name", { address }), originalName); + assert.equal(await call("comment", { address }), originalComment); + assert.equal(await call("inline_comment", { address }), originalInline); + assert.ok( + !(await call("list_bookmarks")).some((item) => item.address === unmapped), + ); + } + const segments = await call("list_segments"); + const first = segments.reduce((left, right) => + BigInt(left.start) < BigInt(right.start) ? left : right, + ); + const last = segments.reduce((left, right) => + BigInt(left.end) > BigInt(right.end) ? left : right, + ); + await invalid( + "prev_address", + { address: first.start }, + /No previous address exists in mapped memory/u, + ); + await invalid( + "next_address", + { address: `0x${(BigInt(last.end) - 1n).toString(16)}` }, + /No adjacent analyzed object exists in mapped memory/u, + ); +} + +async function verifyAnnotationText( + call, + invalid, + { address, other, originalName }, +) { + const comment = "REA representation boundary 测试"; + await call("set_comment", { address, comment }); + await call("set_inline_comment", { address, comment }); + await call("set_bookmark", { address, name: comment }); + for (const [text, reason] of [ + ["before\0after", /contains a NUL character/u], + ["before\ud800after", /contains an unpaired Unicode surrogate/u], + ]) { + for (const operation of ["procedure_info", "read_bytes"]) + await invalid( + operation, + operation === "procedure_info" + ? { procedure: text } + : { address: text, length: 1 }, + reason, + ); + for (const operation of ["set_address_name", "set_bookmark"]) + await invalid(operation, { address, name: text }, reason); + for (const operation of ["set_comment", "set_inline_comment"]) + await invalid(operation, { address, comment: text }, reason); + await invalid( + "set_addresses_names", + { names: { [address]: "must_not_apply", [other.address]: text } }, + reason, + ); + assert.equal(await call("address_name", { address }), originalName); + assert.equal( + await call("address_name", { address: other.address }), + other.value, + ); + assert.equal(await call("comment", { address }), comment); + assert.equal(await call("inline_comment", { address }), comment); + assert.equal( + (await call("list_bookmarks")).find((item) => item.address === address) + ?.name, + comment, + ); + } + await invalid( + "set_address_name", + { address, name: other.value }, + /already assigned to/u, + ); + await invalid( + "set_addresses_names", + { names: { [address]: other.value } }, + /already assigned to/u, + ); + await invalid( + "set_addresses_names", + { names: { [address]: "duplicate", [other.address]: "duplicate" } }, + /assigns the same name more than once/u, + ); + await invalid( + "set_addresses_names", + { names: { [address]: "first", [address.slice(2)]: "second" } }, + /selects the same address more than once/u, + ); + assert.equal(await call("address_name", { address }), originalName); + assert.equal( + await call("address_name", { address: other.address }), + other.value, + ); + try { + const swapped = await call("set_addresses_names", { + names: { [address]: other.value, [other.address]: originalName }, + }); + assert.equal(swapped[address], true); + assert.equal(swapped[other.address], true); + assert.equal(await call("address_name", { address }), other.value); + assert.equal( + await call("address_name", { address: other.address }), + originalName, + ); + } finally { + await call("set_addresses_names", { + names: { [address]: originalName, [other.address]: other.value }, + }); + } +} diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 82502e2115d..0377f4b61cd 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -203,17 +203,17 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "set_address_name", - "Assign an analyst name to one hexadecimal address and report Hopper's boolean result. Reject names owned by another address; use set_addresses_names to explicitly move or swap labels among selected addresses. This mutates analysis metadata.", + "Assign an analyst name to one mapped hexadecimal address and report whether the native edit succeeded and exact readback matched. Hopper rejects symbol names exceeding 1024 UTF-16 code units before mutation, and rejects names owned by another address; use set_addresses_names to explicitly move or swap labels among selected addresses. This mutates analysis metadata.", z.object({ address, name: z.string(), document }), ), official( "set_addresses_names", - "Assign analyst names to multiple addresses in one call and return per-address success booleans. Hopper validates addresses, native string representations, and unique destinations/names before applying edits. Existing labels may move or swap only among explicitly selected addresses; unselected label owners remain unchanged. This mutates analysis metadata; verify failures individually.", + "Assign analyst names to multiple mapped addresses in one call and return per-address edit/readback success booleans. Hopper validates mapped destinations, native string representations, its 1024 UTF-16 code-unit symbol-name limit, and unique destinations/names before applying edits. Existing labels may move or swap only among explicitly selected addresses; unselected label owners remain unchanged. This mutates analysis metadata; verify failures individually.", z.object({ names: z.record(z.string(), z.string()), document }), ), official( "set_bookmark", - "Create or replace a bookmark at a hexadecimal address and report success. This mutates navigation metadata; bookmarks are analyst-authored navigation aids, not binary evidence.", + "Create or replace a bookmark at a mapped hexadecimal address and report success. This mutates navigation metadata; bookmarks are analyst-authored navigation aids, not binary evidence.", z.object({ address, name: z.string().optional(), document }), ), official( @@ -228,7 +228,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "unset_bookmark", - "Remove the bookmark at a hexadecimal address and return whether it is absent. This mutates navigation metadata and does not alter binary bytes.", + "Remove the bookmark at a mapped hexadecimal address and return whether it is absent. Hopper also permits removing an existing legacy bookmark outside mapped memory. This mutates navigation metadata and does not alter binary bytes.", z.object({ address, document }), ), official( From 35f1ddd6bc77588b9971961cd70968a815aae266 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:39:44 +0800 Subject: [PATCH 020/791] fix(hopper): resolve FAT loaders and original file coordinates --- bridge/hopper_bridge.py | 74 +++++- docs/installation.md | 14 +- docs/testing.md | 12 + package.json | 2 + scripts/verify-real-hopper-fat.mjs | 244 ++++++++++++++++++ .../binary/AnalysisProviderEvaluation.ts | 34 ++- .../binary/AnalysisProviderRegistry.test.ts | 13 +- .../binary/AnalysisProviderRegistry.ts | 24 +- src/contracts/hopperMemoryContracts.ts | 2 +- src/contracts/toolOutputSchemaGroups.ts | 24 ++ src/hopper/HopperAnalysisProfile.ts | 77 +++++- src/hopper/HopperProvider.ts | 12 +- .../hopper/hopperAnalysisProfile.test.ts | 69 ----- 13 files changed, 489 insertions(+), 112 deletions(-) create mode 100644 scripts/verify-real-hopper-fat.mjs diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 49240834256..cb0e9c6dc67 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -10,6 +10,7 @@ import os import re import socket +import struct from typing import Any, Optional, Protocol, Sequence BAD_ADDRESSES = (-1, 0xFFFFFFFFFFFFFFFF, None) @@ -277,6 +278,76 @@ def _procedure(document, value=None): return result +def _original_file_offset(document, offset): + """Translate Hopper's image-relative mapping using the observed loaded header.""" + if offset > 9007199254740991: + raise CapabilityUnavailableError("Provider file offset exceeds the exact JSON integer range") + path = document.getExecutableFilePath() + if not path: + raise CapabilityUnavailableError("Hopper did not identify the original executable for file-offset verification") + try: + with open(path, "rb") as source: + size = os.fstat(source.fileno()).st_size + header = source.read(8) + formats = { + b"\xca\xfe\xba\xbe": (">", False), + b"\xbe\xba\xfe\xca": ("<", False), + b"\xca\xfe\xba\xbf": (">", True), + b"\xbf\xba\xfe\xca": ("<", True), + } + fat = formats.get(header[:4]) + if fat is None: + if offset >= size: + raise InvalidRequestError("Mapped offset is outside the original executable file at %s" % path) + return offset, path, 0 + if len(header) != 8: + raise CapabilityUnavailableError("Original FAT executable has a truncated header at %s" % path) + byte_order, wide = fat + count = struct.unpack(byte_order + "I", header[4:])[0] + stride = 32 if wide else 20 + table_end = 8 + count * stride + if count == 0 or table_end > size: + raise CapabilityUnavailableError("Original FAT executable has an invalid architecture table at %s" % path) + header_address = document.getAddressFromFileOffset(0) + reader = getattr(document, "readBytes", None) + if header_address in BAD_ADDRESSES or document.getSegmentAtAddress(header_address) is None or not callable(reader): + raise CapabilityUnavailableError("Hopper did not expose a readable loaded Mach-O header to identify its FAT slice") + loaded = reader(header_address, 12) + if not isinstance(loaded, (bytes, bytearray)) or len(loaded) != 12: + raise CapabilityUnavailableError("Hopper did not expose the loaded Mach-O header to identify its FAT slice") + loaded_order = {b"\xce\xfa\xed\xfe": "<", b"\xcf\xfa\xed\xfe": "<", b"\xfe\xed\xfa\xce": ">", b"\xfe\xed\xfa\xcf": ">"}.get(bytes(loaded[:4])) + if loaded_order is None: + raise CapabilityUnavailableError("Hopper's loaded header is not a supported Mach-O image") + identity = struct.unpack(loaded_order + "II", loaded[4:12]) + selected = None + for index in range(count): + source.seek(8 + index * stride) + entry = source.read(stride) + if len(entry) != stride: + raise CapabilityUnavailableError("Original FAT executable architecture table changed during verification") + cpu, subtype, start, length, alignment = struct.unpack(byte_order + ("IIQQI" if wide else "IIIII"), entry[:28] if wide else entry) + if length < 12 or start < table_end or start > size or length > size - start or alignment > 52 or start % (1 << alignment): + raise CapabilityUnavailableError("Original FAT executable has an invalid slice range at index %s" % index) + if (cpu, subtype) != identity: + continue + source.seek(start) + if source.read(12) != bytes(loaded): + raise CapabilityUnavailableError("FAT slice declaration disagrees with Hopper's loaded header") + if selected is not None: + raise CapabilityUnavailableError("Multiple FAT slices match Hopper's loaded header; original-file mapping is ambiguous") + selected = (start, length) + if selected is None: + raise CapabilityUnavailableError("No original FAT slice matches Hopper's loaded header") + start, length = selected + if offset >= length: + raise InvalidRequestError("Mapped offset is outside Hopper's loaded FAT slice") + if start + offset > 9007199254740991: + raise CapabilityUnavailableError("Original file offset exceeds the exact JSON integer range") + return start + offset, path, start + except OSError as error: + raise CapabilityUnavailableError("Cannot read the original executable for file-offset verification at %s: %s" % (path, error)) from error + + def _procedure_name(procedure): entry = procedure.getEntryPoint() return procedure.getSegment().getNameAtAddress(entry) or _hex(entry) @@ -738,7 +809,8 @@ def _dispatch(method, params): # file header. An integer alone does not prove source-file provenance. if inverse(offset) != address: raise InvalidRequestError("Address has no authoritative file-offset mapping: reverse lookup disagrees at %s" % _hex(address)) - return {"address": _hex(address), "file_offset": offset} + original_offset, source_path, image_base = _original_file_offset(document, offset) + return {"address": _hex(address), "file_offset": original_offset, "provider_file_offset": offset, "image_base_file_offset": image_base, "source_path": source_path} if method == "resolve_containing_procedure": address = _address(document, params.get("address")) procedure, reason = _containing_procedure(document, address) diff --git a/docs/installation.md b/docs/installation.md index 7ac8053f356..62e07ee768a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -318,7 +318,19 @@ Analysis and annotation calls stay bound to the active target's native Hopper document, even when GUI focus changes or other documents have the same display name. Use `open_binary` to change targets. Byte reads stop at a segment boundary and return the readable prefix with `complete: false`. File-offset mapping checks -the reverse lookup; synthetic external-symbol memory has no original file offset. +the reverse lookup and original executable bounds; synthetic external-symbol +memory has no original file offset. For FAT Mach-O, offsets refer to the original +container file. Results retain Hopper's image-relative offset, the observed slice +base, and the source executable path. +Loader selection reads the Mach-O container header, including FAT files with a +single architecture. FAT64 uses the same derived FAT/Mach-O loader chain; +loading depends on the installed Hopper build. Hopper 6.1.0-demo timed out on +a FAT64 fixture while FAT32 controls and its extracted thin image loaded. +This does not establish that other builds reject FAT64. If loading fails, use +`lipo -archs ` to read the exact architecture names, then +`lipo -thin -output ` and open the +resulting image. FAT64 is distinct from the CPU architecture: FAT32 containers +can contain 64-bit executable images. Cursor navigation returns the observed object start when Hopper snaps an interior address; adjacent-object navigation rejects unmapped inputs and document ends. Native API text rejects NUL characters and unpaired Unicode surrogates before diff --git a/docs/testing.md b/docs/testing.md index 3dad3fd7ad8..a58381590aa 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -205,6 +205,18 @@ literal-trace results and invalid-address diagnostics agree with MCP. No provider is mocked in this lane. The Linux demo lane remains a separate `verify:hopper:linux` command. +`verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the +existing Xcode clang/lipo toolchain. It compiles arm64/x86-64 thin executables and +one- and two-slice FAT32 containers, verifies exact/interior address mappings +against bytes in the original files, checks CLI/MCP parity, and checks owned +runtime cleanup. `verify:hopper:fat64` additionally requires the installed Hopper +to load a compiled FAT64 container and verifies its original-file mappings. It +fails on a native load or analysis failure rather than treating a timeout as +proof of unsupported format. This additional lane remains unverified on Hopper +6.1.0-demo: FAT64 timed out while FAT32 controls and the extracted thin image +loaded successfully. +Cross-architecture fixture compilation is not required by `verify:hopper`. + Golden tests use immutable captured text inputs with producer/source provenance under `tests/fixtures/golden/`. Expected results are reviewed for the semantic claim; capture commands do not automatically approve new expected outputs. diff --git a/package.json b/package.json index aaa41580c2f..9276c2e1417 100644 --- a/package.json +++ b/package.json @@ -138,6 +138,8 @@ "typecheck": "tsc --noEmit", "verify:test-temp-hygiene": "npm run build:cached && node scripts/verify-test-temp-hygiene.mjs", "verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs", + "verify:hopper:fat": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs", + "verify:hopper:fat64": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs --fat64", "verify:apple-dispatch": "npm run build:cached && node scripts/verify/apple/dispatch.mjs", "verify:native-ui": "npm run build:cached && node scripts/verify-native-ui.mjs", "verify:keyed-archive": "npm run build:cached && node scripts/verify/apple/keyed-archive.mjs", diff --git a/scripts/verify-real-hopper-fat.mjs b/scripts/verify-real-hopper-fat.mjs new file mode 100644 index 00000000000..516229cc487 --- /dev/null +++ b/scripts/verify-real-hopper-fat.mjs @@ -0,0 +1,244 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { access, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; +import { constants } from "node:fs"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import Ajv from "ajv/dist/2020.js"; +import addFormats from "ajv-formats"; +import { requireMcpResult } from "./lib/mcp-verifier-results.mjs"; +import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; +import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js"; +import { parseConfig } from "../dist/config.js"; + +const run = promisify(execFile); +const verifyFat64 = process.argv[2] === "--fat64"; +assert.deepEqual(process.argv.slice(2), verifyFat64 ? ["--fat64"] : []); +if (process.platform !== "darwin" || !["arm64", "x64"].includes(process.arch)) + throw new Error( + "Hopper FAT verification requires macOS arm64 or x64 and its installed clang/lipo toolchain", + ); +const config = parseConfig(process.env); +if (!config.ok) throw config.error; +await access(config.value.hopperLauncherPath, constants.X_OK).catch((cause) => { + throw new Error( + `Hopper FAT verification requires the configured executable: ${config.value.hopperLauncherPath}`, + { cause }, + ); +}); +const toolPaths = {}; +for (const command of ["clang", "lipo"]) + toolPaths[command] = ( + await run("xcrun", ["--find", command]).catch((cause) => { + throw new Error(`Hopper FAT verification requires ${command}`, { cause }); + }) + ).stdout.trim(); +const sdk = ( + await run("xcrun", ["--sdk", "macosx", "--show-sdk-path"]).catch((cause) => { + throw new Error( + "Hopper FAT verification requires the installed macOS SDK", + { cause }, + ); + }) +).stdout.trim(); + +const directory = await realpath(await mkdtemp(join(tmpdir(), "hopper-fat-"))); +const source = fileURLToPath( + new URL("../tests/conformance/c/fixture.c", import.meta.url), +); +const arm = join(directory, "c-arm64"); +const intel = join(directory, "c-x86_64"); +const fat = join(directory, "c-fat"); +const fat64 = join(directory, "c-fat64"); +const singleFat = join(directory, "c-single-fat"); +const owned = new Set(); +const before = await snapshotHopperRuntime( + "/tmp", + HOPPER_TARGET_LEASE_DIRECTORY, +); +const client = new Client({ name: "hopper-fat-verifier", version: "1" }); +const transport = new StdioClientTransport({ + command: process.execPath, + args: ["dist/main.js"], + env: { + ...process.env, + REA_ANALYSIS_PROVIDER: "hopper", + HOPPER_LOADER_ARGS_JSON: "[]", + }, + stderr: "pipe", +}); +transport.stderr?.on("data", (chunk) => process.stderr.write(chunk)); +const ajv = new Ajv({ strict: false }); +addFormats(ajv); +const observations = []; +try { + await run(toolPaths.clang, [ + "-isysroot", + sdk, + "-arch", + "arm64", + "-O0", + "-fno-inline", + source, + "-o", + arm, + ]); + await run(toolPaths.clang, [ + "-isysroot", + sdk, + "-arch", + "x86_64", + "-O0", + "-fno-inline", + source, + "-o", + intel, + ]); + await run(toolPaths.lipo, ["-create", arm, intel, "-output", fat]); + if (verifyFat64) + await run(toolPaths.lipo, [ + "-create", + arm, + intel, + "-fat64", + "-output", + fat64, + ]); + await run(toolPaths.lipo, [ + "-create", + process.arch === "arm64" ? arm : intel, + "-output", + singleFat, + ]); + await client.connect(transport); + assert.ok(transport.pid !== null); + owned.add(transport.pid); + const catalog = await client.listTools(); + const call = async (name, args = {}) => { + const reply = await client.callTool( + { name, arguments: args }, + { timeout: 180_000 }, + ); + const result = requireMcpResult(reply, name); + const schema = catalog.tools.find( + (tool) => tool.name === name, + )?.outputSchema; + assert.ok(schema, `${name} omitted its output schema`); + assert.equal( + ajv.validate(schema, reply.structuredContent), + true, + JSON.stringify(ajv.errors), + ); + return result; + }; + let fatExpected; + let fatAddress; + for (const path of [ + arm, + intel, + fat, + singleFat, + ...(verifyFat64 ? [fat64] : []), + ]) { + console.error(`Verifying Hopper source-file mappings for ${path}`); + await call("open_binary", { path }); + const file = await readFile(path); + const procedures = await call("list_procedures"); + const entry = procedures.find((item) => item.value.endsWith("rea_entry")); + assert.ok(entry, "compiler fixture omitted rea_entry"); + for (const address of [ + entry.address, + `0x${(BigInt(entry.address) + 1n).toString(16)}`, + ]) { + const mapping = await call("address_to_file_offset", { address }); + const bytes = await call("read_bytes", { address, length: 16 }); + assert.equal( + mapping.file_offset, + mapping.image_base_file_offset + mapping.provider_file_offset, + ); + assert.equal(mapping.source_path, path); + assert.equal(bytes.complete, true); + assert.equal( + file + .subarray(mapping.file_offset, mapping.file_offset + 16) + .toString("hex"), + bytes.bytes_hex, + ); + if (path === fat || path === singleFat || path === fat64) { + assert.ok( + mapping.image_base_file_offset > 0, + "FAT mapping omitted its physical slice offset", + ); + assert.notEqual( + file + .subarray( + mapping.provider_file_offset, + mapping.provider_file_offset + 16, + ) + .toString("hex"), + bytes.bytes_hex, + "fixture did not distinguish image-relative and original-file coordinates", + ); + } else assert.equal(mapping.image_base_file_offset, 0); + if (path === fat) { + fatExpected = mapping; + fatAddress = address; + } + observations.push({ path, address, ...mapping, matched_bytes: 16 }); + } + await call("close_binary"); + } + const pending = run( + process.execPath, + [ + "scripts/rea.mjs", + "address-to-file-offset", + fat, + fatAddress, + "--provider", + "hopper", + "--format", + "json", + ], + { + timeout: 180_000, + env: { ...process.env, HOPPER_LOADER_ARGS_JSON: "[]" }, + }, + ); + assert.ok(pending.child.pid !== undefined); + owned.add(pending.child.pid); + const cli = JSON.parse((await pending).stdout); + assert.deepEqual(cli.normalized_result, fatExpected); +} finally { + await client.close(); + await transport.close(); + await rm(directory, { recursive: true, force: true }); +} +const retained = [ + ...(await snapshotHopperRuntime( + "/tmp", + HOPPER_TARGET_LEASE_DIRECTORY, + owned, + )), +].filter((path) => !before.has(path)); +assert.deepEqual( + retained, + [], + "FAT verification retained owned Hopper sessions or leases", +); +console.log( + JSON.stringify( + { + observations, + cliMcpParity: true, + fat64: verifyFat64 ? "verified" : "not_run", + cleanShutdown: true, + }, + null, + 2, + ), +); diff --git a/src/application/binary/AnalysisProviderEvaluation.ts b/src/application/binary/AnalysisProviderEvaluation.ts index c8cd651b6b5..eb907b153db 100644 --- a/src/application/binary/AnalysisProviderEvaluation.ts +++ b/src/application/binary/AnalysisProviderEvaluation.ts @@ -3,7 +3,11 @@ import { type AnalysisProfileCommitment, } from "../../domain/analysisProfile.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; -import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisUnsupportedTargetError, +} from "../../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { jsonObjectSchema, type JsonValue } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; import type { AnalysisProviderCandidate } from "../AnalysisProvider.js"; @@ -16,6 +20,7 @@ export interface AnalysisProviderCandidateEvaluation { readonly status: AnalysisProviderCandidateStatus; readonly profile?: AnalysisProfileCommitment; readonly compatibility?: Readonly>; + readonly profileError?: AnalysisError; } /** Resolve and validate one provider profile without starting its client. */ @@ -51,12 +56,27 @@ export const evaluateAnalysisProviderCandidate = async ( return err(new AnalysisCancelledError("open_binary")); if (!resolved.ok && resolved.error instanceof AnalysisCancelledError) return err(resolved.error); - if (!resolved.ok) - return ok( - rejectedProfile(candidate, status, resolved.error.message, { - error_tag: resolved.error._tag, - }), - ); + if (!resolved.ok) { + const error = resolved.error; + const rejection = + error instanceof AnalysisUnsupportedTargetError + ? { + candidate, + status: { + ...status, + targetSupport: { + status: "unsupported" as const, + code: "target_format_unsupported" as const, + reason: error.reason, + diagnostics: { path: error.path, error_tag: error._tag }, + }, + }, + } + : rejectedProfile(candidate, status, error.message, { + error_tag: error._tag, + }); + return ok({ ...rejection, profileError: error }); + } try { if (resolved.value.profile === null) return ok( diff --git a/src/application/binary/AnalysisProviderRegistry.test.ts b/src/application/binary/AnalysisProviderRegistry.test.ts index 5423044d81f..935a72f8ee5 100644 --- a/src/application/binary/AnalysisProviderRegistry.test.ts +++ b/src/application/binary/AnalysisProviderRegistry.test.ts @@ -212,15 +212,10 @@ describe("analysis provider registry: binding failures and cancellation", () => failed.provider, ]).select(DATABASE_TARGET, "failed"); if (explicit.ok) throw new Error("expected provider selection rejection"); - expect(explicit.error).toBeInstanceOf(ProviderSelectionError); - if (explicit.error instanceof ProviderSelectionError) - expect(explicit.error.rejections).toMatchObject([ - { - providerId: "failed", - code: "version_unresolved", - diagnostics: { error_tag: "ProviderAdapterError" }, - }, - ]); + expect(explicit.error).toMatchObject({ + _tag: "ProviderAdapterError", + providerId: "failed", + }); }); it("cancels a pending profile probe even when the adapter ignores its signal", async () => { diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index cfc2180c367..5fe08d030b1 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -6,6 +6,7 @@ import { import type { AnalysisProfileCommitment } from "../../domain/analysisProfile.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { ProviderSelectionError, type ProviderSelectionFailureReason, @@ -147,12 +148,7 @@ export class AnalysisProviderRegistry { target: BinaryTarget, requestedSelector?: AnalysisProviderSelector, options: { readonly signal?: AbortSignal } = {}, - ): Promise< - Result< - AnalysisProviderSelection, - ProviderSelectionError | AnalysisCancelledError - > - > { + ): Promise> { if (signalIsAborted(options.signal)) return err(new AnalysisCancelledError("open_binary")); const parsedRequest = @@ -224,12 +220,7 @@ export class AnalysisProviderRegistry { context: TargetSelectionContext, providerId: string, source: AnalysisProviderBinding["selectionSource"], - ): Promise< - Result< - AnalysisProviderSelection, - ProviderSelectionError | AnalysisCancelledError - > - > { + ): Promise> { const { baseline, signal, target } = context; const provider = this.#byId.get(providerId); if (provider === undefined) @@ -253,6 +244,8 @@ export class AnalysisProviderRegistry { ); if (!evaluation.ok) return evaluation; const evaluated = evaluation.value; + if (evaluated.profileError !== undefined) + return err(evaluated.profileError); const statuses = replaceCandidate(baseline, evaluated.status); if ( evaluated.profile === undefined || @@ -275,12 +268,7 @@ export class AnalysisProviderRegistry { async #selectAutomatically( context: TargetSelectionContext, - ): Promise< - Result< - AnalysisProviderSelection, - ProviderSelectionError | AnalysisCancelledError - > - > { + ): Promise> { const { baseline, signal, target } = context; const results = await Promise.all( this.#providers.map( diff --git a/src/contracts/hopperMemoryContracts.ts b/src/contracts/hopperMemoryContracts.ts index ddebc3e8191..5e5a84e1126 100644 --- a/src/contracts/hopperMemoryContracts.ts +++ b/src/contracts/hopperMemoryContracts.ts @@ -16,7 +16,7 @@ export const HOPPER_MEMORY_TOOL_DEFINITIONS = [ { name: "address_to_file_offset", description: - "Map one provider-normalized virtual address to its original nonnegative file offset. Unmapped addresses fail explicitly, and providers without an authoritative mapping API return typed capability unavailability.", + "Map one provider-normalized virtual address to its original nonnegative file offset. Hopper translates FAT Mach-O image-relative offsets using the observed loaded header, preserving its provider coordinate, image base, and source path. Unmapped addresses fail explicitly, and unavailable or ambiguous original-file mappings return typed capability unavailability.", inputSchema: z.object({ address, document }), }, ] as const; diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 13a94b210eb..b60608ae16e 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -147,6 +147,30 @@ export const officialOutputSchemas: Readonly> = { z.object({ address: z.string(), file_offset: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + provider_file_offset: z + .number() + .int() + .min(0) + .max(Number.MAX_SAFE_INTEGER) + .exactOptional() + .describe( + "Original coordinate returned by the provider's mapping API.", + ), + image_base_file_offset: z + .number() + .int() + .min(0) + .max(Number.MAX_SAFE_INTEGER) + .exactOptional() + .describe( + "File offset of the loaded image within its source container; zero for a thin executable.", + ), + source_path: z + .string() + .exactOptional() + .describe( + "Observed original executable path used to verify the file mapping.", + ), }), ), procedure_references: resultOf( diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index 1b17bedb505..d168e0e3d47 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -10,7 +10,11 @@ import type { BinaryArchitecture, BinaryTarget, } from "../domain/binaryTarget.js"; -import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, +} from "../domain/analysisErrorCore.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; @@ -29,7 +33,17 @@ export const resolveHopperAnalysisProfile = async ( ): Promise> => { if (target.kind !== "executable" && target.kind !== "database") return ok({ profile: null, compatibility: {} }); - const derived = hopperLoaderArgsForTarget(target); + let machoContainer: "thin" | "fat32" | "fat64" | undefined; + if (target.kind === "executable" && target.format === "mach-o") { + const header = await readMachoMagic(target.path, options.signal); + if (!header.ok) return header; + machoContainer = [0xcafebabf, 0xbfbafeca].includes(header.value) + ? "fat64" + : [0xcafebabe, 0xbebafeca].includes(header.value) + ? "fat32" + : "thin"; + } + const derived = hopperLoaderArgsForTarget(target, machoContainer); if (!derived.ok) return derived; const loaderArgs = options.loaderArgsOverride.length === 0 @@ -53,6 +67,9 @@ export const resolveHopperAnalysisProfile = async ( available_architectures: [ ...(target.availableArchitectures ?? []), ].sort(), + ...(machoContainer === undefined + ? {} + : { macho_container: machoContainer }), loader: { source: options.loaderArgsOverride.length === 0 @@ -71,7 +88,11 @@ export const resolveHopperAnalysisProfile = async ( /** Derive Hopper's complete non-interactive CLI loader selection. */ export const hopperLoaderArgsForTarget = ( target: BinaryTarget, -): Result => { + machoContainer?: "thin" | "fat32" | "fat64", +): Result< + readonly string[], + ProviderAdapterError | AnalysisCapabilityUnavailableError +> => { if (target.kind === "database") return ok([]); if (target.kind !== "executable") return err(new ProviderAdapterError("hopper", "resolve_analysis_profile")); @@ -79,8 +100,16 @@ export const hopperLoaderArgsForTarget = ( const flag = hopperArchitectureFlag(architecture); switch (target.format) { case "mach-o": + if (machoContainer === undefined) + return err( + new AnalysisCapabilityUnavailableError( + "hopper", + "resolve_analysis_profile", + "Mach-O container kind is unknown; resolve the Hopper analysis profile before starting its client.", + ), + ); return ok( - (target.availableArchitectures?.length ?? 0) > 1 + machoContainer === "fat32" || machoContainer === "fat64" ? ["-l", "FAT", flag, "-l", "Mach-O"] : ["-l", "Mach-O", flag], ); @@ -109,6 +138,46 @@ const hopperArchitectureFlag = (architecture: BinaryArchitecture): string => { } }; +const readMachoMagic = async ( + path: string, + signal?: AbortSignal, +): Promise> => { + if (signalIsAborted(signal)) + return err(new AnalysisCancelledError("open_binary")); + const stream = createReadStream(path, { start: 0, end: 3 }); + const onAbort = (): void => { + stream.destroy(); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + try { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + if (signalIsAborted(signal)) + return err(new AnalysisCancelledError("open_binary")); + const bytes = Buffer.concat(chunks); + return bytes.length === 4 + ? ok(bytes.readUInt32BE(0)) + : err( + new BinaryTargetError( + path, + "Mach-O header became truncated before Hopper loader selection", + ), + ); + } catch (cause: unknown) { + return signalIsAborted(signal) + ? err(new AnalysisCancelledError("open_binary")) + : err( + new BinaryTargetError( + path, + `Cannot read the Mach-O container header: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ), + ); + } finally { + signal?.removeEventListener("abort", onAbort); + } +}; + const sha256File = async ( path: string, signal: AbortSignal | undefined, diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index a07f9994b36..b03d73fd340 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -158,7 +158,13 @@ export class HopperProvider implements AnalysisProviderCandidate { ), close: () => Promise.resolve(), }; - const derivedLoaderArgs = hopperLoaderArgsForTarget(target); + const container = profile?.parameters.macho_container; + const derivedLoaderArgs = hopperLoaderArgsForTarget( + target, + container === "thin" || container === "fat32" || container === "fat64" + ? container + : undefined, + ); if (!derivedLoaderArgs.ok) return { execute: () => Promise.resolve(err(derivedLoaderArgs.error)), @@ -200,7 +206,9 @@ export class HopperProvider implements AnalysisProviderCandidate { value: createAnalysisExecution(result.value, executionProvider, { ...(profile === undefined ? {} : { analysisProfile: profile }), limitations: [ - "Results depend on Hopper's completed static analysis.", + ...(CAPABILITIES.find( + (descriptor) => descriptor.operation === operation, + )?.limitations ?? []), ...(operation === "read_bytes" ? [ "Hopper byte reads stop at the containing segment's exclusive end; a readable prefix is returned with complete=false when the requested range crosses it.", diff --git a/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts b/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts index 391b38cf2aa..13a3df281cd 100644 --- a/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts +++ b/tests/boundary/providers/hopper/hopperAnalysisProfile.test.ts @@ -32,29 +32,8 @@ describe("Hopper analysis profiles", () => { }); it.each([ - [target("mach-o", "arm64"), ["-l", "Mach-O", "--aarch64"]], [target("elf", "x86_64"), ["-l", "ELF", "--intel-64"]], [target("pe", "x86"), ["-l", "WinPE", "--intel-32"]], - [ - { - path: "/tmp/fixture", - sha256: "a".repeat(64), - kind: "executable", - format: "mach-o", - architecture: "arm64", - availableArchitectures: ["x86_64", "arm64"] as const, - } satisfies BinaryTarget, - ["-l", "FAT", "--aarch64", "-l", "Mach-O"], - ], - [ - { - path: "/tmp/fixture.hop", - sha256: "b".repeat(64), - kind: "database", - format: "analysis-database", - } satisfies BinaryTarget, - [], - ], ] as const)("derives complete non-interactive options", (input, expected) => { expect(hopperLoaderArgsForTarget(input)).toEqual({ ok: true, @@ -63,54 +42,6 @@ describe("Hopper analysis profiles", () => { expect("loaderArgs" in input).toBe(false); }); - it("commits launcher build and configured overrides without target coupling", async () => { - directory = await createTestTempDirectory("rea-hopper-profile-"); - const launcher = join(directory, "hopper"); - await writeFile(launcher, "Hopper build A"); - const input = target("mach-o", "arm64"); - const derived = await resolveHopperAnalysisProfile(input, { - launcherPath: launcher, - loaderArgsOverride: [], - provider: HOPPER_PROVIDER_IDENTITY, - }); - const overridden = await resolveHopperAnalysisProfile(input, { - launcherPath: launcher, - loaderArgsOverride: ["-l", "Mach-O", "--intel-64"], - provider: HOPPER_PROVIDER_IDENTITY, - }); - if ( - !derived.ok || - derived.value.profile === null || - !overridden.ok || - overridden.value.profile === null - ) - throw new Error("fixture profiles did not resolve"); - expect(derived.value.compatibility).toEqual({ - loaderArgs: ["-l", "Mach-O", "--aarch64"], - }); - expect(overridden.value.compatibility).toEqual({ - loaderArgs: ["-l", "Mach-O", "--intel-64"], - }); - expect(overridden.value.profile.digest).not.toBe( - derived.value.profile.digest, - ); - expect(derived.value.profile.provider.version).toMatch( - /^launcher-sha256:[a-f0-9]{64}$/u, - ); - - await writeFile(launcher, "Hopper build B"); - const changedBuild = await resolveHopperAnalysisProfile(input, { - launcherPath: launcher, - loaderArgsOverride: [], - provider: HOPPER_PROVIDER_IDENTITY, - }); - if (!changedBuild.ok || changedBuild.value.profile === null) - throw new Error("changed fixture profile did not resolve"); - expect(changedBuild.value.profile.provider.version).not.toBe( - derived.value.profile.provider.version, - ); - }); - it("keeps compatibility output but declines cache identity when version is unresolved", async () => { const resolved = await resolveHopperAnalysisProfile( target("elf", "x86_64"), From 7d7b3f961af2dc9b7572e91aaec29e6c4eccf9d5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:07:47 +0800 Subject: [PATCH 021/791] fix(hopper): preserve typed string values and verify real searches --- bridge/hopper_bridge.py | 56 +++- docs/installation.md | 5 + docs/testing.md | 8 + scripts/lib/real-hopper-boundaries.mjs | 3 + scripts/lib/real-hopper-fixture.mjs | 16 +- scripts/lib/real-hopper-search.mjs | 205 +++++++++++++ scripts/verify-real-hopper.mjs | 17 ++ src/contracts/officialToolContracts.ts | 4 +- src/contracts/toolOutputSchemaGroups.ts | 16 +- src/hopper/HopperProviderCapabilities.ts | 6 + .../providers/hopper/bridgeSearch.test.ts | 285 ------------------ .../hopper/hopperBridgeFacade.test.ts | 31 -- tests/conformance/c/fixture.c | 12 + tests/conformance/objc/fixture.m | 3 + tests/fixtures/bridgeSearchProbe.py | 48 --- tests/fixtures/hopperBridgeFacadeProbe.py | 56 +--- 16 files changed, 338 insertions(+), 433 deletions(-) create mode 100644 scripts/lib/real-hopper-search.mjs delete mode 100644 tests/boundary/providers/hopper/bridgeSearch.test.ts delete mode 100644 tests/fixtures/bridgeSearchProbe.py diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index cb0e9c6dc67..0c1da8a57ec 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -454,11 +454,51 @@ def _procedure_map(document): def _strings(document): - result = {} - for segment in document.getSegmentsList(): - for value, address in segment.getStringsList(): - result[_hex(address)] = value - return result + return {address: record["value"] for address, record in _string_inventory(document).items()} + + +def _string_record(segment, address, display): + """Read one native typed object, retaining its shortened display separately.""" + length = segment.getObjectLength(address) + end = segment.getStartingAddress() + segment.getLength() + if isinstance(length, bool) or not isinstance(length, int) or length <= 0 or length > end - address: + raise CapabilityUnavailableError("Hopper reported an invalid string object extent at %s" % _hex(address)) + raw = segment.readBytes(address, length) + if not isinstance(raw, (bytes, bytearray)) or len(raw) != length: + raise CapabilityUnavailableError("Hopper could not read the complete string object at %s" % _hex(address)) + kind = segment.getTypeAtAddress(address) + encodings = ("utf-8", "latin-1") if kind == segment.TYPE_ASCII else ("utf-16-le", "utf-16-be") if kind == segment.TYPE_UNICODE else () + # Native display escaping is not invertible (a literal backslash-n and a + # newline can render alike). Decode byte strings directly; for UTF-16, + # compare rendered candidates to establish an unambiguous byte order. + candidates = [] + for encoding in encodings: + terminator = b"\x00\x00" if encoding.startswith("utf-16") else b"\x00" + terminated = raw.endswith(terminator) + payload = raw[:-len(terminator)] if terminated else raw + try: + value = payload.decode(encoding) + except UnicodeDecodeError: + continue + record = {"value": value, "provider_value": display, "string": {"encoding": encoding, "encoding_status": "inferred", "termination": "present_or_not_required" if terminated else "missing", "byte_length": length}} + if kind == segment.TYPE_ASCII: + return record + rendered = value.replace('"', '\\"').replace("\n", "\\n").replace("\r", "\\r").replace("\t", "\\t") + if rendered == display or (display.endswith("\u2026") and display[:-1] and rendered.startswith(display[:-1])): + candidates.append(record) + if candidates and len({candidate["value"] for candidate in candidates}) == 1: + return candidates[0] + raise CapabilityUnavailableError("Hopper's string display cannot be reconciled with its typed bytes at %s; inspect the bytes and encoding explicitly" % _hex(address)) + + +def _string_inventory(document): + key = (id(document), "string_records") + inventory = _search_inventory_cache.get(key) + if inventory is None: + records = {_hex(address): _string_record(segment, address, display) for segment in document.getSegmentsList() for display, address in segment.getStringsList()} + inventory = dict(sorted(records.items(), key=lambda item: int(item[0], 16))) + _search_inventory_cache[key] = inventory + return inventory def _invalidate_search_inventory(document): @@ -529,7 +569,7 @@ def _search_results(document, kind, params): if not matches(item[1]): continue selected.append(item) - return [ + return [{"address": address, **_string_inventory(document)[address]} for address, _ in selected] if kind == "string" else [ { "address": address, "value": value, @@ -894,12 +934,12 @@ def _dispatch(method, params): if method == "list_procedures": return [{"address": address, "value": value} for address, value in _search_inventory(document, "procedure")] if method == "list_strings": - values = dict(_search_inventory(document, "string")) + values = _string_inventory(document) requested = params.get("address") if requested is not None: key = _hex(_address(document, requested)) values = {key: values[key]} if key in values else {} - return [{"address": address, "value": value} for address, value in values.items()] + return [{"address": address, **record} for address, record in values.items()] if method == "list_names": result = dict(_search_inventory(document, "name")) requested = params.get("address") diff --git a/docs/installation.md b/docs/installation.md index 62e07ee768a..459d2d1ce4c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -341,6 +341,11 @@ code-unit limit. Oversized names fail before any batch edits; bookmarks and literal string results are not subject to that symbol-name limit. Rename success requires exact final readback. New bookmarks must point into mapped memory; existing legacy bookmarks outside it can still be removed. +String results read each native typed object's complete bytes, retain the original +provider display in `provider_value`, and report its encoding, byte length, and +termination. `encoding_status: inferred` distinguishes REA's decoding from an +observed source encoding. Hopper can split long literals into adjacent +unterminated objects; search matches each object's decoded bytes independently. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/docs/testing.md b/docs/testing.md index a58381590aa..2c2cbab541f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -203,6 +203,14 @@ and whitespace. Disposable binary copies prove that switching and closing actually removes the native document, and that CLI byte, function-dossier, literal-trace results and invalid-address diagnostics agree with MCP. No provider is mocked in this lane. +Real search probes cover Unicode names, literal metacharacters, case and regex +modes, annotation cache invalidation, complete native fragments of long literals +checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1 +decoding, and Hopper's UTF-16 symbol-name truncation boundary. When the macOS +Objective-C fixture is present, the lane also verifies native UTF-16 string objects +and their inferred encodings against their actual bytes. +Unmapped annotation destinations and +oversized later batch names fail before any earlier edit is applied. The Linux demo lane remains a separate `verify:hopper:linux` command. `verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index f98d25e8aeb..c53044bd475 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -9,6 +9,7 @@ import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; +import { verifyHopperSearch } from "./real-hopper-search.mjs"; /** Exercise real Hopper annotation and malformed-input boundaries over MCP. */ export async function verifyHopperBoundaryContracts( @@ -74,6 +75,7 @@ export async function verifyHopperBoundaryContracts( address, ); const workflows = await verifyHopperWorkflows(call, invalid, procedure); + const search = await verifyHopperSearch(call, invalid, procedure); try { if (foreignDocument !== undefined) { await invalid( @@ -278,6 +280,7 @@ export async function verifyHopperBoundaryContracts( foreignDocumentVerified: foreignDocument !== undefined, navigationAndText, workflows, + search, }; } diff --git a/scripts/lib/real-hopper-fixture.mjs b/scripts/lib/real-hopper-fixture.mjs index f6b9376990f..8d8ad67a545 100644 --- a/scripts/lib/real-hopper-fixture.mjs +++ b/scripts/lib/real-hopper-fixture.mjs @@ -21,12 +21,26 @@ export async function loadRealHopperFixtureTargets(manifestPath) { bindFixtureArtifact(root, large), ], ); - requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]); + const objc = manifest.fixtures.find((fixture) => fixture?.name === "objc"); + const unicodeArtifact = + objc === undefined + ? undefined + : await bindFixtureArtifact( + root, + fixtureEntry(manifest.fixtures, "objc"), + ); + requireDistinctArtifacts([ + primaryArtifact, + secondaryArtifact, + largeArtifact, + ...(unicodeArtifact === undefined ? [] : [unicodeArtifact]), + ]); return { manifestPath: canonicalManifest, primary: primaryArtifact, secondary: secondaryArtifact, large: largeArtifact, + ...(unicodeArtifact === undefined ? {} : { unicode: unicodeArtifact }), oracle, largeOracle, }; diff --git a/scripts/lib/real-hopper-search.mjs b/scripts/lib/real-hopper-search.mjs new file mode 100644 index 00000000000..349958814a7 --- /dev/null +++ b/scripts/lib/real-hopper-search.mjs @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; + +/** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */ +export async function verifyHopperSearch(call, invalid, procedure) { + const address = procedure.address; + const original = await call("address_name", { address }); + try { + await verifySymbolNames(call, invalid, address); + await verifySearchModes(call, address); + } finally { + await call("set_address_name", { address, name: original }); + } + assert.deepEqual( + await call("search_procedures", { + pattern: original, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: original }], + ); + return { + unicodeIdentity: true, + literalMetacharacters: true, + longValuePreserved: true, + symbolUtf16LimitValidatedBeforeMutation: true, + regexAndCaseModes: true, + annotationCacheInvalidation: true, + escapedUtf8AndByteStrings: true, + }; +} + +async function verifySymbolNames(call, invalid, address) { + const unicode = "REA_é_e\u0301_😀_Straße_ſ_İ"; + const extentNames = ["x".repeat(1024), "é".repeat(1024), "😀".repeat(512)]; + for (const name of ["REA_a.b[0]", unicode, ...extentNames]) { + assert.equal(await call("set_address_name", { address, name }), true); + assert.equal(await call("address_name", { address }), name); + assert.deepEqual(await call("list_names", { address }), [ + { address, value: name }, + ]); + assert.deepEqual( + await call("search_procedures", { + pattern: name, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: name }], + ); + assert.equal(await call("procedure_address", { procedure: name }), address); + } + assert.deepEqual( + await call("search_procedures", { pattern: unicode, mode: "literal" }), + [], + ); + const other = (await call("list_procedures")).find( + (item) => item.address !== address, + ); + assert.ok(other); + for (const name of ["x".repeat(1025), "é".repeat(1025), "😀".repeat(513)]) { + await invalid( + "set_address_name", + { address, name }, + /UTF-16 code-unit symbol-name limit/u, + ); + await invalid( + "set_addresses_names", + { names: { [other.address]: "must_not_apply", [address]: name } }, + /UTF-16 code-unit symbol-name limit/u, + ); + assert.equal(await call("address_name", { address }), extentNames.at(-1)); + assert.equal( + await call("address_name", { address: other.address }), + other.value, + ); + } +} + +async function verifySearchModes(call, address) { + await call("set_address_name", { address, name: "REA_Search_Upper" }); + assert.deepEqual( + await call("search_procedures", { + pattern: "rea_search_upper", + mode: "literal", + case_sensitive: false, + }), + [{ address, value: "REA_Search_Upper" }], + ); + assert.deepEqual( + await call("search_procedures", { + pattern: "rea_search_upper", + mode: "literal", + case_sensitive: true, + }), + [], + ); + assert.deepEqual( + await call("search_procedures", { + pattern: "^REA_Search_(?:Upper|Lower)$", + mode: "regex", + case_sensitive: true, + }), + [{ address, value: "REA_Search_Upper" }], + ); + const strings = await call("list_strings"); + const long = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle"; + const longItem = strings.find((item) => + item.value.startsWith("REA_LONG_LITERAL_"), + ); + assert.ok(longItem, "fixture omitted its real long literal"); + assert.ok(long.startsWith(longItem.value)); + assert.ok(longItem.provider_value.endsWith("…")); + assert.equal(longItem.string.termination, "missing"); + const fragments = strings.filter((item) => { + const offset = BigInt(item.address) - BigInt(longItem.address); + return offset >= 0n && offset < BigInt(Buffer.byteLength(long)); + }); + assert.equal(fragments.map((item) => item.value).join(""), long); + for (const fragment of fragments) { + assert.equal( + Buffer.byteLength(fragment.value), + fragment.string.byte_length - + (fragment.string.termination === "missing" ? 0 : 1), + ); + const bytes = await call("read_bytes", { + address: fragment.address, + length: fragment.string.byte_length, + }); + const expected = Buffer.from( + fragment.value + (fragment.string.termination === "missing" ? "" : "\0"), + ); + assert.equal(bytes.bytes_hex, expected.toString("hex")); + } + assert.deepEqual( + await call("search_strings", { + pattern: "needle", + mode: "literal", + case_sensitive: true, + }), + [fragments.at(-1)], + ); + const expected = strings.filter((item) => + /^REA_C_(?:ENTRY|LEAF)$/u.test(item.value), + ); + assert.equal(expected.length, 2); + assert.deepEqual( + await call("search_strings", { + pattern: "^REA_C_(?:ENTRY|LEAF)$", + mode: "regex", + case_sensitive: true, + }), + expected, + ); + await verifyHopperStringObjects(call, [ + { value: "REA_UTF8_é_😀", encoding: "utf-8" }, + { value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" }, + { value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" }, + { value: "REA_LITERAL_…", encoding: "utf-8" }, + { value: "REA_LATIN1_ÿ", encoding: "latin-1" }, + ]); +} + +/** Compare native string identities and values with the source fixture's bytes. */ +export async function verifyHopperStringObjects(call, expectations) { + const strings = await call("list_strings"); + for (const { value, encoding } of expectations) { + const item = strings.find((candidate) => candidate.value === value); + assert.ok( + item, + `Hopper omitted the decoded string ${JSON.stringify(value)}`, + ); + assert.equal(item.string.encoding, encoding); + assert.equal(item.string.encoding_status, "inferred"); + assert.equal(item.string.termination, "present_or_not_required"); + const expectedBytes = Buffer.from( + value + "\0", + encoding.replaceAll("-", ""), + ); + assert.equal(item.string.byte_length, expectedBytes.length); + const bytes = await call("read_bytes", { + address: item.address, + length: expectedBytes.length, + }); + assert.equal(bytes.bytes_hex, expectedBytes.toString("hex")); + assert.deepEqual(await call("list_strings", { address: item.address }), [ + item, + ]); + assert.deepEqual( + await call("search_strings", { + pattern: value, + mode: "literal", + case_sensitive: true, + }), + [item], + ); + } + assert.deepEqual( + await call("search_strings", { pattern: "\\xC3\\xA9", mode: "literal" }), + [], + ); + return { + sourceBytesVerified: true, + inferredEncodingReported: true, + displayEscapesExcludedFromMatching: true, + }; +} diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 6920f2657f2..4277ef1fa69 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -15,6 +15,7 @@ import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js"; import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js"; import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs"; +import { verifyHopperStringObjects } from "./lib/real-hopper-search.mjs"; import { requireBridgeDiagnostic, requireBridgeProgress, @@ -463,6 +464,21 @@ try { options, ); if (closed.isError === true) throw new Error(textValue(closed)); + let unicodeStrings = null; + if (fixtureTargets.unicode !== undefined) { + const call = async (name, args = {}) => + requireSuccessfulTool( + await client.callTool({ name, arguments: args }, options), + name, + ); + await call("open_binary", { path: fixtureTargets.unicode.path }); + unicodeStrings = await verifyHopperStringObjects(call, [ + { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, + { value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" }, + { value: "\nREA_UTF16_LEADING_é", encoding: "utf-16-le" }, + ]); + await call("close_binary"); + } const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { primary: targetA, secondary: targetB, @@ -492,6 +508,7 @@ try { segmentCount: firstOverview.segment_count, analyses: [firstAnalysis, secondAnalysis], fixtureAnalysis, + unicodeStrings, boundaryContracts, lifecycleAndCli, inventoryCounts, diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 0377f4b61cd..b45221311f5 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -126,7 +126,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "list_strings", - "List every provider-defined string, or filter to one address, as address/value pairs. Ghidra also reports encoding, terminator status, and byte length.", + "List every provider-defined string object, or filter to one address, as address/value pairs with encoding, terminator status, and byte length when available. Hopper reads complete typed objects and retains its original display text as provider_value; long literals may span adjacent unterminated objects.", z.object({ document, address: optionalAddress }), ), official( @@ -198,7 +198,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_strings", - "Search every analyzed string using literal matching by default or regex when requested. Results are deterministic and complete.", + "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper searches complete typed bytes rather than shortened displays; adjacent fragments of a long literal are matched independently, with their termination and original display retained.", z.object(analysisSearchInput), ), official( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index b60608ae16e..202c6b09784 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -97,9 +97,21 @@ const bookmarkFacetSchema = z.discriminatedUnion("state", [ const addressedString = z.object({ address: z.string(), value: z.string(), + provider_value: z + .string() + .exactOptional() + .describe( + "Original provider display text, which may shorten the typed object.", + ), string: z .object({ encoding: z.string().min(1), + encoding_status: z + .enum(["observed", "inferred"]) + .exactOptional() + .describe( + "Whether the provider declares the encoding or REA infers it from the typed bytes and native display. This does not establish the original source's intended encoding.", + ), termination: z.enum(["missing", "present_or_not_required"]), byte_length: z.number().int().min(0), }) @@ -197,9 +209,7 @@ export const officialOutputSchemas: Readonly> = { search_procedures: resultOf( z.array(z.object({ address: z.string(), value: z.string() })), ), - search_strings: resultOf( - z.array(z.object({ address: z.string(), value: z.string() })), - ), + search_strings: resultOf(z.array(addressedString)), set_address_name: resultOf(z.boolean()), set_addresses_names: resultOf(z.record(z.string(), z.boolean())), set_bookmark: resultOf(z.boolean()), diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 105bec87038..c8039629485 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,6 +79,12 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", + ...(operation === "list_strings" || operation === "search_strings" + ? [ + "String values cover native typed objects. Hopper may split long literals into adjacent unterminated fragments; string searches match each object independently.", + "Reported string encodings are inferred from typed bytes and native display; the original source's intended encoding is unknown.", + ] + : []), ]), }), ), diff --git a/tests/boundary/providers/hopper/bridgeSearch.test.ts b/tests/boundary/providers/hopper/bridgeSearch.test.ts deleted file mode 100644 index 2f0d0ce9865..00000000000 --- a/tests/boundary/providers/hopper/bridgeSearch.test.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; -import { fileURLToPath } from "node:url"; - -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { z } from "zod"; - -const bridgePath = fileURLToPath( - new URL("../../../../bridge/hopper_bridge.py", import.meta.url), -); -const probePath = fileURLToPath( - new URL("../../../fixtures/bridgeSearchProbe.py", import.meta.url), -); - -const searchResultSchema = z.object({ - action: z.literal("search"), - ok: z.literal(true), - result: z.array(z.object({ address: z.string(), value: z.string() })), -}); -const probeResultSchema = searchResultSchema; -type ProbeResult = z.infer; - -type ProbeInput = { - readonly action: "search"; - readonly items: readonly (readonly [string, string])[]; - readonly params: Readonly>; -}; - -interface PendingProbe { - readonly resolve: (result: ProbeResult) => void; - readonly reject: (error: Error) => void; - timeout: NodeJS.Timeout | undefined; -} - -const PROBE_TIMEOUT_MS = 3_000; -const PROBE_BUFFER_LIMIT = 1_024 * 1_024; -const pendingProbes: PendingProbe[] = []; -let probeProcess: ChildProcessWithoutNullStreams | undefined; -let probeStdout = ""; -let probeStderr = ""; -let probeFailure: Error | undefined; -let stoppingProbe = false; - -const failureWithStderr = (message: string): Error => { - const stderr = probeStderr.trim(); - return new Error(stderr.length === 0 ? message : `${message}: ${stderr}`); -}; - -const rejectPendingProbes = (error: Error): void => { - for (const pending of pendingProbes.splice(0)) { - if (pending.timeout !== undefined) clearTimeout(pending.timeout); - pending.reject(error); - } -}; - -const failProbeProcess = (error: Error): void => { - probeFailure ??= error; - rejectPendingProbes(probeFailure); - if ( - !stoppingProbe && - probeProcess?.exitCode === null && - probeProcess.signalCode === null - ) - probeProcess.kill("SIGKILL"); -}; - -const acceptProbeLine = (line: string): void => { - const pending = pendingProbes.shift(); - if (pending === undefined) { - failProbeProcess( - failureWithStderr("Python probe returned an extra result"), - ); - return; - } - if (pending.timeout !== undefined) clearTimeout(pending.timeout); - try { - const decoded: unknown = JSON.parse(line); - pending.resolve(probeResultSchema.parse(decoded)); - } catch (cause: unknown) { - const error = - cause instanceof Error - ? cause - : failureWithStderr("Python probe returned an invalid result"); - pending.reject(error); - failProbeProcess(error); - } -}; - -const acceptProbeOutput = (chunk: string): void => { - probeStdout += chunk; - if (Buffer.byteLength(probeStdout) > PROBE_BUFFER_LIMIT) { - failProbeProcess(failureWithStderr("Python probe output exceeded 1 MiB")); - return; - } - let newline = probeStdout.indexOf("\n"); - while (newline >= 0) { - const line = probeStdout.slice(0, newline).replace(/\r$/u, ""); - probeStdout = probeStdout.slice(newline + 1); - acceptProbeLine(line); - newline = probeStdout.indexOf("\n"); - } -}; - -const startProbeProcess = async (): Promise => { - stoppingProbe = false; - probeFailure = undefined; - probeStdout = ""; - probeStderr = ""; - const child = spawn("python3", [probePath, bridgePath], { - stdio: ["pipe", "pipe", "pipe"], - }); - probeProcess = child; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", acceptProbeOutput); - child.stderr.on("data", (chunk: string) => { - probeStderr += chunk; - if (Buffer.byteLength(probeStderr) > PROBE_BUFFER_LIMIT) - failProbeProcess(new Error("Python probe stderr exceeded 1 MiB")); - }); - child.stdin.on("error", (error) => { - if (!stoppingProbe) failProbeProcess(error); - }); - child.on("error", (error) => { - if (!stoppingProbe) failProbeProcess(error); - }); - child.on("exit", (code, signal) => { - if (!stoppingProbe) - failProbeProcess( - failureWithStderr( - `Python probe exited before responding (code ${String(code)}, signal ${String(signal)})`, - ), - ); - }); - await new Promise((resolve, reject) => { - const onSpawn = (): void => { - child.off("error", onError); - resolve(); - }; - const onError = (error: Error): void => { - child.off("spawn", onSpawn); - reject(error); - }; - child.once("spawn", onSpawn); - child.once("error", onError); - }); -}; - -const hasExited = (child: ChildProcessWithoutNullStreams): boolean => - child.exitCode !== null || child.signalCode !== null; - -const waitForExit = ( - child: ChildProcessWithoutNullStreams, - timeoutMs: number, -): Promise => { - if (hasExited(child)) return Promise.resolve(true); - return new Promise((resolve) => { - const onExit = (): void => { - clearTimeout(timeout); - resolve(true); - }; - const timeout = setTimeout(() => { - child.off("exit", onExit); - resolve(false); - }, timeoutMs); - child.once("exit", onExit); - }); -}; - -const stopProbeProcess = async (): Promise => { - const child = probeProcess; - if (child === undefined) return; - stoppingProbe = true; - rejectPendingProbes(new Error("Python probe stopped before responding")); - if (!hasExited(child)) child.stdin.end(); - if (!(await waitForExit(child, 500))) child.kill("SIGTERM"); - if (!(await waitForExit(child, 500))) child.kill("SIGKILL"); - if (!(await waitForExit(child, 500))) - throw new Error("Python probe did not stop after SIGKILL"); - probeProcess = undefined; -}; - -const probe = (input: ProbeInput): Promise => { - const child = probeProcess; - if (probeFailure !== undefined) return Promise.reject(probeFailure); - if (child === undefined || hasExited(child)) - return Promise.reject(new Error("Python probe is not running")); - return new Promise((resolve, reject) => { - const pending: PendingProbe = { resolve, reject, timeout: undefined }; - pending.timeout = setTimeout(() => { - failProbeProcess( - new Error( - `Python probe request timed out after ${String(PROBE_TIMEOUT_MS)} ms`, - ), - ); - }, PROBE_TIMEOUT_MS); - pendingProbes.push(pending); - child.stdin.write(`${JSON.stringify(input)}\n`, (error) => { - if (error !== null && error !== undefined) failProbeProcess(error); - }); - }); -}; - -beforeAll(startProbeProcess); -beforeEach(async () => { - if ( - probeFailure === undefined && - probeProcess !== undefined && - !hasExited(probeProcess) - ) - return; - await stopProbeProcess(); - await startProbeProcess(); -}); -afterAll(stopProbeProcess); - -describe("Hopper bridge search", () => { - it("returns matching literal values in deterministic order", async () => { - await expect( - probe({ - action: "search", - items: [ - ["0x1000", "REA_main_1"], - ["0x2000", "unrelated"], - ["0x3000", "rea_helper_42"], - ], - params: { - pattern: "REA", - mode: "literal", - case_sensitive: false, - }, - }), - ).resolves.toEqual({ - action: "search", - ok: true, - result: [ - { address: "0x1000", value: "REA_main_1" }, - { address: "0x3000", value: "rea_helper_42" }, - ], - }); - }); - - it("returns matching entries for regex searches", async () => { - await expect( - probe({ - action: "search", - items: [ - ["0x1000", "REA_GHIDRA_INVENTORY_ENTRY"], - ["0x2000", "unrelated"], - ["0x3000", "REA_GHIDRA_LEAF_VALUE"], - ], - params: { - pattern: "^REA_GHIDRA_(?:INVENTORY_ENTRY|LEAF_VALUE)$", - mode: "regex", - case_sensitive: true, - }, - }), - ).resolves.toEqual({ - action: "search", - ok: true, - result: [ - { address: "0x1000", value: "REA_GHIDRA_INVENTORY_ENTRY" }, - { address: "0x3000", value: "REA_GHIDRA_LEAF_VALUE" }, - ], - }); - }); - - it("returns the complete value for a long literal match", async () => { - const value = `${"x".repeat(4_096)}needle`; - await expect( - probe({ - action: "search", - items: [["0x1000", value]], - params: { - pattern: "needle", - mode: "literal", - case_sensitive: true, - }, - }), - ).resolves.toMatchObject({ - action: "search", - ok: true, - result: [{ address: "0x1000", value }], - }); - }); -}); diff --git a/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts b/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts index 5544ea53f96..62975437dff 100644 --- a/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts +++ b/tests/boundary/providers/hopper/hopperBridgeFacade.test.ts @@ -48,7 +48,6 @@ const probeResultSchema = z.strictObject({ reason: z.string(), }), }), - strings: z.strictObject({ "0x401234": z.literal("fixture string") }), procedure_references: z.strictObject({ procedure: z.strictObject({ address: z.literal("0x401000"), @@ -64,14 +63,6 @@ const probeResultSchema = z.strictObject({ unresolved_calls: z.array(z.unknown()), references: z.array(z.unknown()), }), - inventory_replies: z.array( - z.strictObject({ - id: z.number().int(), - result: z.array( - z.strictObject({ address: z.string(), value: z.string() }), - ), - }), - ), provider_faults: z.array( z.strictObject({ id: z.literal(1), @@ -198,28 +189,6 @@ describe("Hopper API facade", () => { }, { id: 9, result: "0x401000" }, ]); - expect(result.inventory_replies).toEqual([ - { - id: 1, - result: [ - { address: "0x2", value: "string-2" }, - { address: "0x10", value: "string-16" }, - { address: "0x100", value: "string-256" }, - ], - }, - { - id: 2, - result: [ - { address: "0x2", value: "name-2" }, - { address: "0x10", value: "name-16" }, - { address: "0x100", value: "name-256" }, - ], - }, - { id: 3, result: [{ address: "0x10", value: "string-16" }] }, - { id: 4, result: [{ address: "0x10", value: "name-16" }] }, - { id: 5, result: [] }, - { id: 6, result: [] }, - ]); expect(stdout).not.toContain("supersecret"); expect(result.analysis_guard.message).toContain( "requires completed Hopper background analysis", diff --git a/tests/conformance/c/fixture.c b/tests/conformance/c/fixture.c index 07dd3a57b76..66d48543661 100644 --- a/tests/conformance/c/fixture.c +++ b/tests/conformance/c/fixture.c @@ -2,6 +2,18 @@ volatile int rea_c_global = 7; +#define REA_LONG_CHUNK "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define REA_LONG_256 REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK REA_LONG_CHUNK +#define REA_LONG_1024 REA_LONG_256 REA_LONG_256 REA_LONG_256 REA_LONG_256 +__attribute__((noinline, used)) void rea_long_literal_procedure(void) { + puts("REA_LONG_LITERAL_" REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 REA_LONG_1024 "needle"); + puts("REA_UTF8_é_😀"); + puts("REA_ESCAPED_\"\\line\nend\t\r"); + puts("REA_LITERAL_BACKSLASH_\\n"); + puts("REA_LITERAL_…"); + puts("REA_LATIN1_\xff"); +} + __attribute__((noinline, used)) int rea_leaf(int value) { puts("REA_C_LEAF"); return value + rea_c_global; diff --git a/tests/conformance/objc/fixture.m b/tests/conformance/objc/fixture.m index bc9f5ea2d67..f3cc07b9799 100644 --- a/tests/conformance/objc/fixture.m +++ b/tests/conformance/objc/fixture.m @@ -16,6 +16,9 @@ int main(void) { @autoreleasepool { Protocol *protocol = @protocol(REAWidgetDelegate); NSLog(@"%@", NSStringFromProtocol(protocol)); + NSLog(@"REA_UTF16_é_😀"); + NSLog(@"REA_UTF16_ESCAPED_é_\"\\line\nend\t\r"); + NSLog(@"\nREA_UTF16_LEADING_é"); return [[[REAWidget alloc] init] performAction:@"run" error:0] ? 0 : 1; } } diff --git a/tests/fixtures/bridgeSearchProbe.py b/tests/fixtures/bridgeSearchProbe.py deleted file mode 100644 index e9f7c89e914..00000000000 --- a/tests/fixtures/bridgeSearchProbe.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Exercise REA's bridge inventory search without starting a Hopper socket.""" - -import json -from pathlib import Path -import sys - - -def _load_bridge(path): - source = Path(path).read_text(encoding="utf-8") - namespace = {"__file__": path, "__name__": "rea_hopper_bridge"} - exec(compile(source, path, "exec"), namespace) - return namespace - - -def _probe(namespace, payload): - action = payload["action"] - try: - if action == "search": - inventory = tuple(tuple(item) for item in payload["items"]) - namespace["_search_inventory"] = lambda _document, _kind: inventory - return { - "action": action, - "ok": True, - "result": namespace["_search_results"]( - object(), "string", payload["params"] - ), - } - raise ValueError("unknown probe action") - except Exception as error: - return { - "action": action, - "ok": False, - "type": type(error).__name__, - "diagnostic_type": namespace["_diagnostic_type"](error), - "message": str(error), - } - - -if __name__ == "__main__": - bridge = _load_bridge(sys.argv[1]) - if len(sys.argv) >= 3: - print(json.dumps(_probe(bridge, json.loads(sys.argv[2])), sort_keys=True)) - else: - for line in sys.stdin: - if not line.strip(): - continue - request = json.loads(line) - print(json.dumps(_probe(bridge, request), sort_keys=True), flush=True) diff --git a/tests/fixtures/hopperBridgeFacadeProbe.py b/tests/fixtures/hopperBridgeFacadeProbe.py index 6981c20733c..097be9f6b56 100644 --- a/tests/fixtures/hopperBridgeFacadeProbe.py +++ b/tests/fixtures/hopperBridgeFacadeProbe.py @@ -30,7 +30,7 @@ def getCurrentAddress(self): return 0x401000 def getSegmentsList(self): - return [FakeInventorySegment()] + return [] class FakeDocumentProvider: @@ -47,39 +47,6 @@ def getCurrentDocument(cls): return cls.current -class FakeStringSegment: - def getStringsList(self): - return [("fixture string", FakeStringAddress())] - - -class FakeStringAddress: - def __init__(self, value="0x401234"): - self.value = value - - def __str__(self): - return self.value - - -class FakeInventorySegment: - addresses = [FakeStringAddress("0x10"), 0x100, 0x2] - - def getStringsList(self): - return [ - ("string-" + str(self.number(address)), address) - for address in self.addresses - ] - - def getNamedAddresses(self): - return self.addresses - - def getNameAtAddress(self, address): - return "name-" + str(self.number(address)) - - @staticmethod - def number(address): - return address if isinstance(address, int) else int(str(address), 16) - - class FakeReferenceSegment: def getNameAtAddress(self, address): return "fixture-procedure" @@ -120,13 +87,6 @@ def getStartingAddress(self): def getEndingAddress(self): return 0x401004 - - -class FakeStringsDocument: - def getSegmentsList(self): - return [FakeStringSegment()] - - def load_bridge(path): namespace = { "__file__": path, @@ -181,7 +141,6 @@ def main(): bridge["REA_OWNS_PROCESS_LIFETIME"] = False bridge["_configure_hopper_api"](FakeDocumentProvider) bridge["_bind_session_document"]() - strings = bridge["_strings"](FakeStringsDocument()) current = bridge["_dispatch"]("current_document", {}) current_address = bridge["_dispatch"]("current_address", {}) selected = bridge["_session_document"]() is FakeDocumentProvider.current @@ -219,17 +178,6 @@ def main(): procedure_info = bridge["_dispatch"]( "procedure_info", {"procedure": "0x401000"} ) - inventories = bridge_replies(bridge, [ - request(method, params, index) - for index, (method, params) in enumerate([ - ("list_strings", {}), - ("list_names", {}), - ("list_strings", {"address": "0x10"}), - ("list_names", {"address": "0x10"}), - ("list_strings", {"address": "0xff"}), - ("list_names", {"address": "0xff"}), - ], 1) - ]) provider_faults = [] for error_type in (TypeError, ValueError, KeyError): FakeDocumentProvider.current.address_error = error_type("credential=supersecret") @@ -279,8 +227,6 @@ def main(): "imported_without_hopper": unavailable, "current_document": current, "current_address": current_address, - "strings": strings, - "inventory_replies": inventories, "containing_procedure": containing_procedure, "procedure_references": procedure_references, "procedure_info": procedure_info, From 458771cb583e38b848b3d8c58595d163143ff50a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:02:32 +0800 Subject: [PATCH 022/791] fix(hopper): preserve launcher evidence on readiness timeout --- src/domain/analysisErrorPresentation.ts | 12 ++-- src/domain/analysisErrorProjection.ts | 3 + src/domain/hopperErrors.ts | 1 + src/hopper/HopperClient.ts | 64 ++++++++++++------- .../hopper/hopperClient.helperFailure.test.ts | 45 +++++++++++++ 5 files changed, 99 insertions(+), 26 deletions(-) diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 56b0ad2736a..66cdb5caf07 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -52,9 +52,11 @@ export const analysisErrorRemediationAction = ( : "Review the reported worker memory limits and available host memory. Retry with sufficient memory or a smaller artifact; REA retains tighter inherited limits.") ); if (error instanceof HopperTimeoutError) - return error.providerState === "busy" - ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry." - : "Check binary_session for Hopper health, then retry the operation."; + return error.operation === undefined + ? "Inspect Hopper for a loader or license dialog and review details.launcher. Correct the loader configuration or complete Hopper setup, then open the target again." + : error.providerState === "busy" + ? "Check binary_session.analysis_activity, wait for the active Hopper request to finish, then retry." + : "Check binary_session for Hopper health, then retry the operation."; if (error instanceof HopperProcessError) return hopperProcessRemediation(error); if (error instanceof HopperStartError) @@ -201,7 +203,9 @@ export const analysisErrorUserMessage = (error: AnalysisError): string => { const hopperErrorUserMessage = (error: AnalysisError): string | undefined => { if (error instanceof HopperTimeoutError) { - const request = error.operation ?? "startup"; + if (error.operation === undefined) + return "REA timed out waiting for Hopper bridge readiness. Hopper may be waiting for a loader or license dialog; inspect its window and the captured launcher outcome before retrying."; + const request = error.operation; return error.providerState === "busy" ? `Hopper timed out during ${request} while the provider remained busy. Check binary_session.analysis_activity, wait for the active request to finish, then retry.` : `Hopper timed out during ${request} before it started. Check binary_session for provider health, then retry.`; diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 1e14bde9751..39509368a81 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -445,6 +445,9 @@ const lifecycleErrorDetails = ( return { stage: error.operation === undefined ? "startup" : error.stage, timeout_ms: error.timeoutMs, + ...(error.launcherOutcome === undefined + ? {} + : { launcher: error.launcherOutcome }), provider_state: error.providerState, retry_action: error.providerState === "busy" ? "wait" : "retry", ...(error.operation === undefined ? {} : { operation: error.operation }), diff --git a/src/domain/hopperErrors.ts b/src/domain/hopperErrors.ts index 2246bdec507..68c5b9ee47d 100644 --- a/src/domain/hopperErrors.ts +++ b/src/domain/hopperErrors.ts @@ -35,6 +35,7 @@ export class HopperTimeoutError extends HopperError { readonly stage: ProviderFailureStage = operation === undefined ? "launch" : "analysis", + readonly launcherOutcome?: HopperLauncherOutcome, ) { super( `Hopper ${operation === undefined ? "startup" : operation} timed out after ${String(timeoutMs)}ms`, diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index c8c49958cce..20430c16e67 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -502,7 +502,8 @@ export class HopperClient { deadline: ProviderStartupDeadline, ): Promise> { while (deadline.remainingMs() > 0) { - if (deadline.signal.aborted) return err(startupInterruption(deadline)); + if (deadline.signal.aborted) + return err(await this.#startupInterruption(deadline)); if (this.#closing) return err(new HopperProcessError(null)); if ( this.#launcherExitCode !== undefined && @@ -516,7 +517,7 @@ export class HopperClient { deadline, ); return deadline.interruption === "cancelled" - ? err(startupInterruption(deadline)) + ? err(await this.#startupInterruption(deadline)) : err(failure); } try { @@ -525,7 +526,7 @@ export class HopperClient { // best-effort cleanup: socket polling; absence means keep waiting. void cause; if ((await deadline.wait(50)) === "aborted") - return err(startupInterruption(deadline)); + return err(await this.#startupInterruption(deadline)); continue; } const attempt = await connectHopperSocketOnce( @@ -538,9 +539,44 @@ export class HopperClient { return ok(undefined); } if ((await deadline.wait(50)) === "aborted") - return err(startupInterruption(deadline)); + return err(await this.#startupInterruption(deadline)); } - return err(new HopperTimeoutError(this.#options.startupTimeoutMs)); + return err(await this.#startupInterruption(deadline)); + } + + async #startupInterruption( + deadline: ProviderStartupDeadline, + ): Promise { + if (deadline.interruption === "cancelled") + return new HopperCancelledError(); + return new HopperTimeoutError( + deadline.timeoutMs, + undefined, + undefined, + "not_started", + "launch", + this.#launcherOutcome( + (await this.#process?.waitForOutputClose(0)) ?? false, + ), + ); + } + + #launcherOutcome(outputClosed: boolean): HopperLauncherOutcome | undefined { + const snapshot = this.#process?.snapshot(); + const token = this.#token; + if (snapshot === undefined) return undefined; + const redact = (text: string): string => + token === undefined + ? text + : text.replaceAll(token, "[redacted transport credential]"); + return { + exit_code: snapshot.exitCode ?? null, + signal: snapshot.signal ?? null, + stdout: { ...snapshot.stdout, text: redact(snapshot.stdout.text) }, + stderr: { ...snapshot.stderr, text: redact(snapshot.stderr.text) }, + output_closed: outputClosed, + diagnostic_truncated: snapshot.diagnosticTruncated === true, + }; } async #launcherStartupFailure( @@ -549,29 +585,13 @@ export class HopperClient { deadline: ProviderStartupDeadline, ): Promise { const process = this.#process; - const token = this.#token; // Descendants may inherit the helper's pipes. Bound drainage independently // from readiness, and report whether producer output actually closed. const outputClosed = (await process?.waitForOutputClose( Math.min(1_000, deadline.remainingMs()), )) ?? false; - const snapshot = process?.snapshot(); - const redact = (text: string): string => - token === undefined - ? text - : text.replaceAll(token, "[redacted transport credential]"); - const launcherFailure: HopperLauncherOutcome | undefined = - snapshot === undefined - ? undefined - : { - exit_code: exitCode, - signal: snapshot.signal ?? null, - stdout: { ...snapshot.stdout, text: redact(snapshot.stdout.text) }, - stderr: { ...snapshot.stderr, text: redact(snapshot.stderr.text) }, - output_closed: outputClosed, - diagnostic_truncated: snapshot.diagnosticTruncated === true, - }; + const launcherFailure = this.#launcherOutcome(outputClosed); if (ownsProviderLifetime) return new HopperProcessError( exitCode, diff --git a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts index a99542fc405..b7be9ea19a2 100644 --- a/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts +++ b/tests/boundary/providers/hopper/hopperClient.helperFailure.test.ts @@ -134,3 +134,48 @@ it.each([0, 1])( }); }, ); + +it("preserves a successful helper's diagnostics when bridge readiness is never observed", async () => { + let token = ""; + const launcher: BridgeLauncher = { + launch(session) { + token = session.token; + return Promise.resolve( + ok({ + process: spawn( + process.execPath, + [ + "-e", + `process.stdout.write(${JSON.stringify("source=local-evidence ")} + ${JSON.stringify(session.token)}); process.stderr.write("loader callback pending");`, + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ), + ownsProcessLifetime: false as const, + providerLifetime: "external-application" as const, + shutdownMode: "bridge-request" as const, + }), + ); + }, + }; + const client = new HopperClient({ launcher, startupTimeoutMs: 1500 }); + onTestFinished(() => client.close()); + const result = await client.start(); + if (result.ok) + throw new Error("Expected missing bridge readiness to time out"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "provider_timeout", + details: { + stage: "startup", + launcher: { + exit_code: 0, + output_closed: true, + stdout: { text: expect.stringContaining("source=local-evidence") }, + stderr: { text: "loader callback pending" }, + }, + }, + }); + expect(projected.message).toContain("bridge readiness"); + expect(projected.message).not.toContain("before it started"); + expect(JSON.stringify(projected)).not.toContain(token); +}); From 0a49d5bfb931124d3d398dfbd64e9f6ae8a01405 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:03:50 +0800 Subject: [PATCH 023/791] fix(hopper): load verified FAT64 slices with native Mach-O semantics --- bridge/hopper_bridge.py | 2 +- docs/installation.md | 23 ++- docs/testing.md | 11 +- scripts/verify-real-hopper-fat.mjs | 234 +++++++++++++++++++++++---- src/hopper/BridgeLauncher.ts | 43 ++++- src/hopper/HopperAnalysisProfile.ts | 23 ++- src/hopper/HopperCleanup.test.ts | 47 ++++++ src/hopper/HopperCleanup.ts | 20 ++- src/hopper/HopperMachOImage.ts | 242 ++++++++++++++++++++++++++++ src/hopper/HopperProvider.ts | 39 ++++- 10 files changed, 630 insertions(+), 54 deletions(-) create mode 100644 src/hopper/HopperMachOImage.ts diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 0c1da8a57ec..993f43f2307 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -282,7 +282,7 @@ def _original_file_offset(document, offset): """Translate Hopper's image-relative mapping using the observed loaded header.""" if offset > 9007199254740991: raise CapabilityUnavailableError("Provider file offset exceeds the exact JSON integer range") - path = document.getExecutableFilePath() + path = globals().get("REA_ORIGINAL_EXECUTABLE_PATH") or document.getExecutableFilePath() if not path: raise CapabilityUnavailableError("Hopper did not identify the original executable for file-offset verification") try: diff --git a/docs/installation.md b/docs/installation.md index 459d2d1ce4c..2b0a97c0fd2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -323,14 +323,21 @@ memory has no original file offset. For FAT Mach-O, offsets refer to the origina container file. Results retain Hopper's image-relative offset, the observed slice base, and the source executable path. Loader selection reads the Mach-O container header, including FAT files with a -single architecture. FAT64 uses the same derived FAT/Mach-O loader chain; -loading depends on the installed Hopper build. Hopper 6.1.0-demo timed out on -a FAT64 fixture while FAT32 controls and its extracted thin image loaded. -This does not establish that other builds reject FAT64. If loading fails, use -`lipo -archs ` to read the exact architecture names, then -`lipo -thin -output ` and open the -resulting image. FAT64 is distinct from the CPU architecture: FAT32 containers -can contain 64-bit executable images. +single architecture. For FAT64, REA validates the architecture table and selected +Mach-O header, prepares a private thin image, and loads it with Hopper's native +Mach-O loader. It checks the entire source's SHA-256 while copying the slice; +source identity and reported offsets still refer to the original container. +An ambiguous architecture subtype requires an explicitly extracted thin image; +REA does not guess. Configured loader arguments remain explicit overrides. +FAT64 is distinct from the CPU architecture: FAT32 containers can contain 64-bit +executable images. This preparation avoids the Raw Binary loader dialog observed +with native FAT64 loading on Hopper 6.1.0-demo. +The prepared image and its owned document close together, including on MCP exit; +if document closure is unconfirmed, REA retains the backing image and reports +`cleanup_incomplete` with its path. Ordinary documents retain their existing +MCP-exit behavior. +Startup deadlines report missing bridge readiness and preserve the launcher +outcome; a successful helper exit does not prove that a loader dialog completed. Cursor navigation returns the observed object start when Hopper snaps an interior address; adjacent-object navigation rejects unmapped inputs and document ends. Native API text rejects NUL characters and unpaired Unicode surrogates before diff --git a/docs/testing.md b/docs/testing.md index 2c2cbab541f..e65cba07875 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -217,12 +217,11 @@ The Linux demo lane remains a separate `verify:hopper:linux` command. existing Xcode clang/lipo toolchain. It compiles arm64/x86-64 thin executables and one- and two-slice FAT32 containers, verifies exact/interior address mappings against bytes in the original files, checks CLI/MCP parity, and checks owned -runtime cleanup. `verify:hopper:fat64` additionally requires the installed Hopper -to load a compiled FAT64 container and verifies its original-file mappings. It -fails on a native load or analysis failure rather than treating a timeout as -proof of unsupported format. This additional lane remains unverified on Hopper -6.1.0-demo: FAT64 timed out while FAT32 controls and the extracted thin image -loaded successfully. +runtime cleanup. `verify:hopper:fat64` additionally checks FAT64 preparation through +Hopper's native Mach-O loader, source-container mappings, profile identity, +malformed and ambiguous slice rejection, and temporary-image lifetime. Both +lanes have been verified on Hopper 6.1.0-demo; this establishes REA's prepared +FAT64 workflow, not native FAT64-loader support. Cross-architecture fixture compilation is not required by `verify:hopper`. Golden tests use immutable captured text inputs with producer/source provenance diff --git a/scripts/verify-real-hopper-fat.mjs b/scripts/verify-real-hopper-fat.mjs index 516229cc487..d4709e357c5 100644 --- a/scripts/verify-real-hopper-fat.mjs +++ b/scripts/verify-real-hopper-fat.mjs @@ -1,6 +1,14 @@ import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; -import { access, mkdtemp, readFile, realpath, rm } from "node:fs/promises"; +import { + access, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { promisify } from "node:util"; @@ -10,7 +18,7 @@ import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; -import { requireMcpResult } from "./lib/mcp-verifier-results.mjs"; +import { mcpTextValue, requireMcpResult } from "./lib/mcp-verifier-results.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js"; import { parseConfig } from "../dist/config.js"; @@ -56,6 +64,9 @@ const fat = join(directory, "c-fat"); const fat64 = join(directory, "c-fat64"); const singleFat = join(directory, "c-single-fat"); const owned = new Set(); +const extraFat64 = []; +const invalidFat64 = []; +let preparedImagePath; const before = await snapshotHopperRuntime( "/tmp", HOPPER_TARGET_LEASE_DIRECTORY, @@ -114,6 +125,86 @@ try { "-output", singleFat, ]); + if (verifyFat64) { + const original = await readFile(fat64); + const swapped = Buffer.from(original); + swapped.writeUInt32BE(0xbfbafeca, 0); + swapped.writeUInt32LE(original.readUInt32BE(4), 4); + for (let index = 0; index < original.readUInt32BE(4); index++) { + const offset = 8 + index * 32; + for (const field of [0, 4, 24, 28]) + swapped.writeUInt32LE( + original.readUInt32BE(offset + field), + offset + field, + ); + for (const field of [8, 16]) + swapped.writeBigUInt64LE( + original.readBigUInt64BE(offset + field), + offset + field, + ); + } + const little = join(directory, "c-fat64-little-endian"); + await writeFile(little, swapped); + extraFat64.push(little); + const single = join(directory, "c-single-fat64"); + await run(toolPaths.lipo, [ + "-create", + process.arch === "arm64" ? arm : intel, + "-fat64", + "-output", + single, + ]); + extraFat64.push(single); + for (const [name, reason, mutate] of [ + [ + "reserved", + /nonzero reserved field/u, + (bytes) => bytes.writeUInt32BE(1, 36), + ], + [ + "range", + /extends beyond the file/u, + (bytes) => bytes.writeBigUInt64BE(BigInt(bytes.length), 24), + ], + [ + "identity", + /disagrees with its Mach-O header/u, + (bytes) => bytes.writeUInt32BE(bytes.readUInt32BE(12) + 1, 12), + ], + [ + "overlap", + /slices overlap/u, + (bytes) => { + const cpu = process.arch === "arm64" ? 0x100000c : 0x1000007; + const base = bytes.readUInt32BE(8) === cpu ? 8 : 40; + bytes.copy(bytes, base === 8 ? 40 : 8, base, base + 32); + }, + ], + ]) { + const bytes = Buffer.from(original); + mutate(bytes); + const path = join(directory, `c-fat64-invalid-${name}`); + await writeFile(path, bytes); + invalidFat64.push({ path, reason }); + } + const hostBytes = await readFile(process.arch === "arm64" ? arm : intel); + const secondOffset = Math.ceil((16384 + hostBytes.length) / 16384) * 16384; + const ambiguous = Buffer.alloc(secondOffset + hostBytes.length); + ambiguous.writeUInt32BE(0xcafebabf, 0); + ambiguous.writeUInt32BE(2, 4); + for (const [index, offset] of [16384, secondOffset].entries()) { + const base = 8 + index * 32; + ambiguous.writeUInt32BE(hostBytes.readUInt32LE(4), base); + ambiguous.writeUInt32BE(hostBytes.readUInt32LE(8), base + 4); + ambiguous.writeBigUInt64BE(BigInt(offset), base + 8); + ambiguous.writeBigUInt64BE(BigInt(hostBytes.length), base + 16); + ambiguous.writeUInt32BE(14, base + 24); + hostBytes.copy(ambiguous, offset); + } + const path = join(directory, "c-fat64-ambiguous"); + await writeFile(path, ambiguous); + invalidFat64.push({ path, reason: /selection.*ambiguous/u }); + } await client.connect(transport); assert.ok(transport.pid !== null); owned.add(transport.pid); @@ -135,18 +226,18 @@ try { ); return result; }; - let fatExpected; - let fatAddress; + const cliMappings = new Map(); for (const path of [ arm, intel, fat, singleFat, - ...(verifyFat64 ? [fat64] : []), + ...(verifyFat64 ? [fat64, ...extraFat64] : []), ]) { console.error(`Verifying Hopper source-file mappings for ${path}`); - await call("open_binary", { path }); const file = await readFile(path); + const sourceHash = createHash("sha256").update(file).digest("hex"); + await call("open_binary", { path }); const procedures = await call("list_procedures"); const entry = procedures.find((item) => item.value.endsWith("rea_entry")); assert.ok(entry, "compiler fixture omitted rea_entry"); @@ -168,7 +259,12 @@ try { .toString("hex"), bytes.bytes_hex, ); - if (path === fat || path === singleFat || path === fat64) { + if ( + path === fat || + path === singleFat || + path === fat64 || + extraFat64.includes(path) + ) { assert.ok( mapping.image_base_file_offset > 0, "FAT mapping omitted its physical slice offset", @@ -184,35 +280,109 @@ try { "fixture did not distinguish image-relative and original-file coordinates", ); } else assert.equal(mapping.image_base_file_offset, 0); - if (path === fat) { - fatExpected = mapping; - fatAddress = address; + if (path === fat || path === fat64) { + cliMappings.set(path, { mapping, address }); } observations.push({ path, address, ...mapping, matched_bytes: 16 }); } + if (path === fat64 || extraFat64.includes(path)) { + const session = await call("binary_session"); + assert.equal(session.path, path); + assert.equal(session.sha256, sourceHash); + const profile = session.analysis_provider_binding.analysis_profile; + assert.equal( + profile.parameters.prepared_image.method, + "fat64_thin_slice", + ); + assert.equal( + profile.parameters.prepared_image.source_offset, + observations.at(-1).image_base_file_offset, + ); + const document = await call("current_document"); + await call("set_comment", { + address: entry.address, + comment: "source-bound FAT64 preparation", + }); + await call("open_binary", { path }); + assert.deepEqual( + (await call("binary_session")).analysis_provider_binding + .analysis_profile, + profile, + ); + assert.equal(await call("current_document"), document); + assert.equal( + await call("comment", { address: entry.address }), + "source-bound FAT64 preparation", + ); + } + assert.equal( + createHash("sha256") + .update(await readFile(path)) + .digest("hex"), + sourceHash, + "native analysis modified source executable bytes", + ); await call("close_binary"); } - const pending = run( - process.execPath, - [ - "scripts/rea.mjs", - "address-to-file-offset", - fat, - fatAddress, - "--provider", - "hopper", - "--format", - "json", - ], - { - timeout: 180_000, - env: { ...process.env, HOPPER_LOADER_ARGS_JSON: "[]" }, - }, - ); - assert.ok(pending.child.pid !== undefined); - owned.add(pending.child.pid); - const cli = JSON.parse((await pending).stdout); - assert.deepEqual(cli.normalized_result, fatExpected); + for (const { path, reason } of invalidFat64) { + const reply = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + assert.equal(reply.isError, true, "malformed FAT64 unexpectedly launched"); + assert.match(mcpTextValue(reply), reason); + } + if (verifyFat64) { + await call("open_binary", { path: fat64 }); + await call("list_procedures"); + const resources = [ + ...(await snapshotHopperRuntime( + "/tmp", + HOPPER_TARGET_LEASE_DIRECTORY, + owned, + )), + ].filter((path) => !before.has(path)); + for (const root of resources) { + const image = join(root, "image.macho"); + if ( + await access(image).then( + () => true, + () => false, + ) + ) + preparedImagePath = image; + } + assert.ok( + preparedImagePath, + "prepared native image was not owned by this session", + ); + await client.close(); + await transport.close(); + await assert.rejects(access(preparedImagePath), { code: "ENOENT" }); + } + for (const [path, expected] of cliMappings) { + const pending = run( + process.execPath, + [ + "scripts/rea.mjs", + "address-to-file-offset", + path, + expected.address, + "--provider", + "hopper", + "--format", + "json", + ], + { + timeout: 180_000, + env: { ...process.env, HOPPER_LOADER_ARGS_JSON: "[]" }, + }, + ); + assert.ok(pending.child.pid !== undefined); + owned.add(pending.child.pid); + const cli = JSON.parse((await pending).stdout); + assert.deepEqual(cli.normalized_result, expected.mapping); + } } finally { await client.close(); await transport.close(); @@ -236,6 +406,8 @@ console.log( observations, cliMcpParity: true, fat64: verifyFat64 ? "verified" : "not_run", + malformedFat64Rejected: invalidFat64.length, + preparedImageRemovedOnMcpShutdown: preparedImagePath !== undefined, cleanShutdown: true, }, null, diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 8e4e7559bc9..19d50527ad8 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -23,6 +23,11 @@ import { } from "./LinuxPrivateDisplayProbe.js"; import writeFileAtomic from "write-file-atomic"; +import { + prepareHopperMachOImage, + type HopperMachOImage, +} from "./HopperMachOImage.js"; + import { acquireHopperTargetLease } from "./HopperTargetLease.js"; import type { HopperTargetLease } from "./HopperTargetLease.js"; @@ -41,11 +46,13 @@ export interface BridgeSession { export type BridgeLaunch = | (ProviderProcessLaunch & { readonly shutdownMode: "bridge-request"; + readonly preparedImagePath?: string; /** The owned helper may hand off to an external GUI application. */ readonly providerLifetime: "launcher-process" | "external-application"; }) | (Extract & { readonly shutdownMode: "process-cleanup"; + readonly preparedImagePath?: string; readonly providerLifetime: "launcher-process"; readonly cleanup: NonNullable; }); @@ -69,6 +76,10 @@ interface SharedHopperApplicationLauncherOptions { readonly targetKind: "executable" | "database"; readonly loaderArgs: readonly string[]; readonly bridgeScriptPath: string; + readonly preparedImage?: { + readonly image: HopperMachOImage; + readonly sourceSha256: string; + }; } /** Explicit launcher contract; the Linux adapter verifies its pinned Hopper build before execution. */ @@ -125,16 +136,30 @@ export class HopperApplicationLauncher implements BridgeLauncher { const lease = leaseResult.value; let leaseTransferred = false; try { + let loadedPath = this.options.targetPath; + if (this.options.preparedImage !== undefined) { + const prepared = await prepareHopperMachOImage( + this.options.targetPath, + this.options.preparedImage.sourceSha256, + this.options.preparedImage.image, + join(session.directory, "image.macho"), + options.signal, + ); + if (!prepared.ok) return prepared; + loadedPath = prepared.value; + } const ownsProcessLifetime = usesLinuxDemo(this.options); const bootstrap = await this.#writeBootstrap( session, ownsProcessLifetime, + loadedPath, ); if (!bootstrap.ok) return bootstrap; const launched = await this.#launchPreparedTarget({ session, signal: options.signal, bootstrapPath: bootstrap.value, + loadedPath, ownsProcessLifetime, lease, }); @@ -149,6 +174,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { readonly session: BridgeSession; readonly signal: AbortSignal | undefined; readonly bootstrapPath: string; + readonly loadedPath: string; readonly ownsProcessLifetime: boolean; readonly lease: HopperTargetLease | undefined; }): Promise< @@ -167,7 +193,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { "-Y", bootstrapPath, action, - this.options.targetPath, + input.loadedPath, ]; try { if (signal?.aborted === true) return err(new HopperCancelledError()); @@ -217,6 +243,9 @@ export class HopperApplicationLauncher implements BridgeLauncher { return err(new HopperStartError({ cause })); } const ownedLauncher = { + ...(this.options.preparedImage === undefined + ? {} + : { preparedImagePath: input.loadedPath }), process: started.process, ownsProcessLifetime: true as const, ownership: started.ownership, @@ -270,12 +299,18 @@ export class HopperApplicationLauncher implements BridgeLauncher { async #writeBootstrap( session: BridgeSession, ownsProcessLifetime: boolean, + loadedPath: string, ): Promise> { const bootstrapPath = `${session.directory}/bootstrap.py`; try { await writeFile( bootstrapPath, - bridgeBootstrapSource(session, this.options, ownsProcessLifetime), + bridgeBootstrapSource( + session, + this.options, + ownsProcessLifetime, + loadedPath, + ), { encoding: "utf8", mode: 0o600 }, ); await chmod(bootstrapPath, 0o600); @@ -321,12 +356,14 @@ const bridgeBootstrapSource = ( session: BridgeSession, options: SharedHopperApplicationLauncherOptions, ownsProcessLifetime: boolean, + loadedPath: string, ): string => [ `REA_SOCKET = ${JSON.stringify(session.socketPath)}`, `REA_TOKEN = ${JSON.stringify(session.token)}`, `REA_RUN_ID = ${JSON.stringify(session.runId)}`, - `REA_TARGET_PATH = ${JSON.stringify(options.targetPath)}`, + `REA_TARGET_PATH = ${JSON.stringify(loadedPath)}`, + `REA_ORIGINAL_EXECUTABLE_PATH = ${options.targetKind === "executable" ? JSON.stringify(options.targetPath) : "None"}`, `REA_OWNS_PROCESS_LIFETIME = ${ownsProcessLifetime ? "True" : "False"}`, `exec(compile(open(${JSON.stringify(options.bridgeScriptPath)}, 'rb').read(), ${JSON.stringify(options.bridgeScriptPath)}, 'exec'))`, "", diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index d168e0e3d47..c9590b83011 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -19,6 +19,11 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; +import { + resolveHopperMachOImage, + type HopperMachOImage, +} from "./HopperMachOImage.js"; + interface HopperProfileOptions { readonly launcherPath: string; readonly loaderArgsOverride: readonly string[]; @@ -43,7 +48,20 @@ export const resolveHopperAnalysisProfile = async ( ? "fat32" : "thin"; } - const derived = hopperLoaderArgsForTarget(target, machoContainer); + let preparedImage: HopperMachOImage | undefined; + if ( + target.kind === "executable" && + machoContainer === "fat64" && + options.loaderArgsOverride.length === 0 + ) { + const resolved = await resolveHopperMachOImage(target, options.signal); + if (!resolved.ok) return resolved; + preparedImage = resolved.value; + } + const derived = hopperLoaderArgsForTarget( + target, + preparedImage === undefined ? machoContainer : "thin", + ); if (!derived.ok) return derived; const loaderArgs = options.loaderArgsOverride.length === 0 @@ -70,6 +88,9 @@ export const resolveHopperAnalysisProfile = async ( ...(machoContainer === undefined ? {} : { macho_container: machoContainer }), + ...(preparedImage === undefined + ? {} + : { prepared_image: { ...preparedImage } }), loader: { source: options.loaderArgsOverride.length === 0 diff --git a/src/hopper/HopperCleanup.test.ts b/src/hopper/HopperCleanup.test.ts index 6e8018f9ff5..a26d1ac01b7 100644 --- a/src/hopper/HopperCleanup.test.ts +++ b/src/hopper/HopperCleanup.test.ts @@ -1,3 +1,8 @@ +import { access, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { PrivateRuntimeRoot } from "../process/PrivateRuntimeRoot.js"; + import { EventEmitter } from "node:events"; import { Socket } from "node:net"; @@ -120,3 +125,45 @@ describe("Hopper shutdown rejection diagnostics", () => { ); }); }); + +it("keeps a prepared backing image when native document closure cannot be confirmed", async () => { + const parent = await createTestTempDirectory("rea-hopper-backing-lifetime-"); + const runtimeRoot = await PrivateRuntimeRoot.create({ parent }); + const preparedImagePath = join(runtimeRoot.path, "image.macho"); + await writeFile(preparedImagePath, "owned backing bytes"); + const { logger } = loggerHarness(); + try { + const launch = { + ...processCleanupLaunch(), + preparedImagePath, + shutdownMode: "bridge-request" as const, + providerLifetime: "external-application" as const, + }; + const result = await cleanupHopperSession({ + socket: new Socket(), + launch, + runtimeRoot, + processSupervisor: undefined, + activeRequest: null, + retainDocument: true, + progress: undefined, + logger, + onDiagnostic: undefined, + request: () => Promise.reject(failure("document closure unobserved")), + releaseTransport: () => undefined, + }); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupResources: expect.arrayContaining([ + "hopper-document", + runtimeRoot.path, + ]), + }, + }); + await expect(access(preparedImagePath)).resolves.toBeUndefined(); + } finally { + await runtimeRoot.close(); + await rm(parent, { recursive: true, force: true }); + } +}); diff --git a/src/hopper/HopperCleanup.ts b/src/hopper/HopperCleanup.ts index 3d31884d882..a4409e29599 100644 --- a/src/hopper/HopperCleanup.ts +++ b/src/hopper/HopperCleanup.ts @@ -65,7 +65,7 @@ export const cleanupHopperSession = async ( await report( input.progress, 0, - input.retainDocument + input.retainDocument && input.launch?.preparedImagePath === undefined ? "detaching REA from the Hopper document" : "requesting Hopper document shutdown", ); @@ -75,7 +75,20 @@ export const cleanupHopperSession = async ( await stopProcess(input, state); recordUnconfirmedDocument(input, state); await report(input.progress, 0.75, "removing Hopper private runtime files"); - await closeRuntimeRoot(input.runtimeRoot, state); + if ( + input.launch?.preparedImagePath !== undefined && + !state.shutdownConfirmed + ) { + const resource = input.runtimeRoot?.path ?? input.launch.preparedImagePath; + state.resources.add(resource); + state.issues.push({ + resource, + reason: + "Prepared image retained because native document closure was not confirmed", + }); + } else { + await closeRuntimeRoot(input.runtimeRoot, state); + } return cleanupOutcome(input, state); }; @@ -99,7 +112,8 @@ const requestShutdown = async ( return; } const method = - input.retainDocument || input.launch?.shutdownMode === "process-cleanup" + (input.retainDocument && input.launch?.preparedImagePath === undefined) || + input.launch?.shutdownMode === "process-cleanup" ? "shutdown" : "shutdown_document"; const shutdown = await input diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts new file mode 100644 index 00000000000..0b6ac600e56 --- /dev/null +++ b/src/hopper/HopperMachOImage.ts @@ -0,0 +1,242 @@ +import { createHash } from "node:crypto"; +import { open, type FileHandle } from "node:fs/promises"; +import { z } from "zod"; + +import type { BinaryTarget } from "../domain/binaryTarget.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { + readFatSliceDeclarations, + validateFatSlice, +} from "../domain/apple/machoContainer.js"; +import { + HopperCancelledError, + HopperStartError, +} from "../domain/hopperErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; + +/** Stable source coordinates committed before preparing an owned thin image. */ +export const hopperMachOImageSchema = z.strictObject({ + method: z.literal("fat64_thin_slice"), + architecture_count: z.number().int().min(1).max(128), + container_table_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + source_offset: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + byte_length: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), + cpu_type: z.number().int().nonnegative(), + cpu_subtype: z.number().int().nonnegative(), +}); + +export type HopperMachOImage = z.infer; + +/** Validate the selected FAT64 slice without depending on Apple's lipo tool. */ +export const resolveHopperMachOImage = async ( + target: BinaryTarget & { readonly kind: "executable" }, + signal?: AbortSignal, +): Promise< + Result +> => { + const invalid = (reason: string) => + err(new BinaryTargetError(target.path, reason)); + if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); + let file: FileHandle; + try { + file = await open(target.path, "r"); + } catch (cause: unknown) { + return invalid( + `Cannot read FAT64 source for Hopper image preparation: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + try { + const read = async (offset: number, length: number): Promise => { + const bytes = Buffer.alloc(length); + const { bytesRead } = await file.read(bytes, 0, length, offset); + return bytes.subarray(0, bytesRead); + }; + const header = await read(0, 8); + const magic = header.length === 8 ? header.readUInt32BE(0) : 0; + if (magic !== 0xcafebabf && magic !== 0xbfbafeca) + return invalid( + "FAT64 source header changed before Hopper image preparation", + ); + const littleEndian = magic === 0xbfbafeca; + const count = littleEndian + ? header.readUInt32LE(4) + : header.readUInt32BE(4); + const size = (await file.stat()).size; + const tableEnd = 8 + count * 32; + if (count === 0 || count > 128 || tableEnd > size) + return invalid( + "FAT64 architecture table is truncated or exceeds the supported 128 entries", + ); + const table = await read(8, count * 32); + const parsed = readFatSliceDeclarations(table, { + count, + wide: true, + littleEndian, + }); + if (parsed.status === "malformed") return invalid(parsed.reason); + const cpuTypes = { x86: 7, x86_64: 0x1000007, arm: 12, arm64: 0x100000c }; + const matches = []; + const ranges = []; + for (const slice of parsed.slices) { + if (signal?.aborted) + return err(new AnalysisCancelledError("open_binary")); + const rangeError = validateFatSlice(slice, size, tableEnd); + if (rangeError !== null) return invalid(rangeError); + const reserved = littleEndian + ? table.readUInt32LE(slice.index * 32 + 28) + : table.readUInt32BE(slice.index * 32 + 28); + if (reserved !== 0) + return invalid( + `FAT64 architecture ${slice.index} has a nonzero reserved field`, + ); + if (slice.size < 12) + return invalid( + `FAT64 architecture ${slice.index} has a truncated Mach-O header`, + ); + const embedded = await read(slice.offset, 12); + if (embedded.length !== 12) + return invalid(`FAT64 architecture ${slice.index} became truncated`); + const thinMagic = embedded.readUInt32BE(0); + if (![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(thinMagic)) + return invalid( + `FAT64 architecture ${slice.index} does not contain a Mach-O image`, + ); + const little = thinMagic === 0xcefaedfe || thinMagic === 0xcffaedfe; + const identity = { + cpuType: little ? embedded.readUInt32LE(4) : embedded.readUInt32BE(4), + cpuSubtype: little + ? embedded.readUInt32LE(8) + : embedded.readUInt32BE(8), + }; + const identityError = validateFatSlice(slice, size, tableEnd, identity); + if (identityError !== null) return invalid(identityError); + ranges.push(slice); + if (slice.cpuType === cpuTypes[target.architecture]) matches.push(slice); + } + ranges.sort((a, b) => a.offset - b.offset); + for (let index = 1; index < ranges.length; index++) { + const previous = ranges[index - 1]; + const current = ranges[index]; + if ( + previous !== undefined && + current !== undefined && + current.offset < previous.offset + previous.size + ) + return invalid("FAT64 slices overlap"); + } + const selected = matches.length === 1 ? matches[0] : undefined; + if (selected === undefined) + return invalid( + `FAT64 selection for ${target.architecture} is ${matches.length === 0 ? "missing" : "ambiguous"}; extract the exact architecture into a thin image before opening it`, + ); + return ok({ + method: "fat64_thin_slice", + architecture_count: count, + container_table_sha256: createHash("sha256") + .update(header) + .update(table) + .digest("hex"), + source_offset: selected.offset, + byte_length: selected.size, + cpu_type: selected.cpuType, + cpu_subtype: selected.cpuSubtype, + }); + } catch (cause: unknown) { + return invalid( + `Cannot validate FAT64 image: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } finally { + await file.close(); + } +}; + +/** Copy a selected slice and verify the entire original source in the same pass. */ +export const prepareHopperMachOImage = async ( + sourcePath: string, + sourceSha256: string, + image: HopperMachOImage, + destination: string, + signal?: AbortSignal, +): Promise> => { + if (signal?.aborted) return err(new HopperCancelledError()); + try { + const source = await open(sourcePath, "r"); + try { + const output = await open(destination, "wx", 0o600); + try { + const hash = createHash("sha256"); + const tableHash = createHash("sha256"); + const tableEnd = 8 + image.architecture_count * 32; + const thinHeader: Buffer[] = []; + let position = 0; + let copied = 0; + for await (const chunk of source.createReadStream({ + autoClose: false, + })) { + if (signal?.aborted) return err(new HopperCancelledError()); + const bytes = Buffer.from(chunk); + hash.update(bytes); + if (position < tableEnd) + tableHash.update( + bytes.subarray(0, Math.min(bytes.length, tableEnd - position)), + ); + const start = Math.max(0, image.source_offset - position); + const end = Math.min( + bytes.length, + image.source_offset + image.byte_length - position, + ); + if (end > start) { + if (copied < 12) + thinHeader.push( + bytes.subarray(start, Math.min(end, start + 12 - copied)), + ); + await output.writeFile(bytes.subarray(start, end)); + copied += end - start; + } + position += bytes.length; + } + const observed = hash.digest("hex"); + const header = Buffer.concat(thinHeader); + const magic = header.length === 12 ? header.readUInt32BE(0) : 0; + const little = magic === 0xcefaedfe || magic === 0xcffaedfe; + const headerMatches = + header.length === 12 && + [0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic) && + (little ? header.readUInt32LE(4) : header.readUInt32BE(4)) === + image.cpu_type && + (little ? header.readUInt32LE(8) : header.readUInt32BE(8)) === + image.cpu_subtype; + if ( + tableHash.digest("hex") !== image.container_table_sha256 || + !headerMatches + ) + return err( + new HopperStartError({ + userMessage: `FAT64 source ${sourcePath} no longer agrees with its committed architecture table or selected Mach-O header. Open the stable source again.`, + }), + ); + if (observed !== sourceSha256 || copied !== image.byte_length) + return err( + new HopperStartError({ + userMessage: `FAT64 source ${sourcePath} changed before launch: expected SHA-256 ${sourceSha256}, observed ${observed}; copied ${copied} of ${image.byte_length} selected bytes. Open the stable source again.`, + }), + ); + return ok(destination); + } finally { + await output.close(); + } + } finally { + await source.close(); + } + } catch (cause: unknown) { + return signal?.aborted + ? err(new HopperCancelledError()) + : err( + new HopperStartError({ + cause, + userMessage: `Cannot prepare the selected FAT64 image from ${sourcePath}: ${cause instanceof Error ? cause.message : String(cause)}`, + }), + ); + } +}; diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index b03d73fd340..57817943fff 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -24,6 +24,7 @@ import { resolveHopperAnalysisProfile, } from "./HopperAnalysisProfile.js"; import { HopperClient } from "./HopperClient.js"; +import { hopperMachOImageSchema } from "./HopperMachOImage.js"; import { CAPABILITIES, @@ -158,7 +159,30 @@ export class HopperProvider implements AnalysisProviderCandidate { ), close: () => Promise.resolve(), }; - const container = profile?.parameters.macho_container; + const preparation = profile?.parameters.prepared_image; + const parsedImage = + preparation === undefined + ? undefined + : hopperMachOImageSchema.safeParse(preparation); + if (parsedImage?.success === false) + return { + execute: (operation) => + Promise.resolve( + err( + new AnalysisCapabilityUnavailableError( + IDENTITY.id, + operation, + "Hopper prepared-image profile is malformed; resolve the analysis profile again.", + ), + ), + ), + close: () => Promise.resolve(), + }; + const preparedImage = parsedImage?.data; + const container = + preparedImage === undefined + ? profile?.parameters.macho_container + : "thin"; const derivedLoaderArgs = hopperLoaderArgsForTarget( target, container === "thin" || container === "fat32" || container === "fat64" @@ -176,6 +200,14 @@ export class HopperProvider implements AnalysisProviderCandidate { launcherPath: this.config.hopperLauncherPath, targetPath: target.path, targetKind: target.kind, + ...(preparedImage === undefined + ? {} + : { + preparedImage: { + image: preparedImage, + sourceSha256: target.sha256, + }, + }), loaderArgs: this.config.hopperLoaderArgs.length > 0 ? this.config.hopperLoaderArgs @@ -206,6 +238,11 @@ export class HopperProvider implements AnalysisProviderCandidate { value: createAnalysisExecution(result.value, executionProvider, { ...(profile === undefined ? {} : { analysisProfile: profile }), limitations: [ + ...(preparedImage === undefined + ? [] + : [ + "REA loaded a verified thin Mach-O slice from the original FAT64 source. Identity and file offsets refer to the original file. The owned temporary image and its Hopper document close together, including on MCP shutdown.", + ]), ...(CAPABILITIES.find( (descriptor) => descriptor.operation === operation, )?.limitations ?? []), From 246170238ebb8669a20b57b559b03cfc26ee1b8c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:05:39 +0800 Subject: [PATCH 024/791] fix(hopper): preserve partial native call and string evidence --- bridge/hopper_bridge.py | 77 +++++++++++++++-------- docs/installation.md | 5 ++ docs/testing.md | 4 +- scripts/lib/real-hopper-search.mjs | 25 ++++++++ scripts/lib/real-hopper-semantic.mjs | 39 ++++++++++++ src/contracts/officialToolContracts.ts | 4 +- src/contracts/toolOutputSchemaGroups.ts | 46 +++----------- src/domain/hopperValues.ts | 55 +++++++++++++--- src/hopper/HopperProvider.ts | 2 +- src/hopper/HopperProviderCapabilities.ts | 5 +- tests/fixtures/hopperBridgeFacadeProbe.py | 3 + 11 files changed, 188 insertions(+), 77 deletions(-) diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 993f43f2307..635b27b10f6 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -408,6 +408,22 @@ def _instruction_addresses(procedure): return result +def _native_calls(procedure, direction): + references = procedure.getAllCallees() if direction == "outgoing" else procedure.getAllCallers() + kinds = {0: "none", 1: "unknown", 2: "direct", 3: "objective_c"} + calls = {} + unresolved = [] + for reference in references: + source, target, kind = reference.fromAddress(), reference.toAddress(), reference.type() + if source in BAD_ADDRESSES: + continue + if target in BAD_ADDRESSES: + unresolved.append({"address": _hex(source), "reason": "Native CallReference has no resolved target (type %s)" % kind}) + continue + calls[(source, target)] = {"classification": kinds.get(kind, "unknown"), "provider_type": kind, "provenance": "hopper-public-python-api:CallReference"} + return calls, sorted(unresolved, key=lambda item: int(item["address"], 16)) + + def _procedure_references(document, params): procedure = _procedure(document, params.get("procedure")) direction = params.get("direction", "outgoing") @@ -420,6 +436,8 @@ def _procedure_references(document, params): references = segment.getReferencesFromAddress(address) if direction == "outgoing" else segment.getReferencesOfAddress(address) for reference in references: edges.add((address, reference) if direction == "outgoing" else (reference, address)) + calls, unresolved = _native_calls(procedure, direction) + edges.update(calls) ordered = sorted(edges) items = [] for source, target in ordered: @@ -430,16 +448,15 @@ def _procedure_references(document, params): "target_address": _hex(target), "source_procedure": _procedure_identity(source_procedure) if source_procedure is not None else None, "target_procedure": _procedure_identity(target_procedure) if target_procedure is not None else None, - "kind": _unavailable("Hopper's public Python API does not classify reference kinds"), + "kind": _unavailable("Hopper exposes partial CallReference classification, but not detailed reference flags"), + **({"call": calls[(source, target)]} if (source, target) in calls else {}), }) return { "procedure": _procedure_identity(procedure), "direction": direction, "reference_kinds_available": False, - # The public Hopper API returns observed references but does not expose - # enough flow metadata to enumerate calls with no resolved target. - # Keep the required list field explicit; the provider records this as - # unknown coverage rather than claiming the list is exhaustive. - "unresolved_calls": [], + # These are native CallReference objects, not proof that every unresolved + # dispatch site has been enumerated by Hopper. + "unresolved_calls": unresolved, "references": items, } @@ -491,14 +508,24 @@ def _string_record(segment, address, display): raise CapabilityUnavailableError("Hopper's string display cannot be reconciled with its typed bytes at %s; inspect the bytes and encoding explicitly" % _hex(address)) -def _string_inventory(document): +def _string_inventory(document, addresses=None): + key = (id(document), "string_objects") + objects = _search_inventory_cache.get(key) + if objects is None: + objects = {address: (segment, display) for segment in document.getSegmentsList() for display, address in segment.getStringsList()} + _search_inventory_cache[key] = objects key = (id(document), "string_records") - inventory = _search_inventory_cache.get(key) - if inventory is None: - records = {_hex(address): _string_record(segment, address, display) for segment in document.getSegmentsList() for display, address in segment.getStringsList()} - inventory = dict(sorted(records.items(), key=lambda item: int(item[0], 16))) - _search_inventory_cache[key] = inventory - return inventory + records = _search_inventory_cache.setdefault(key, {}) + selected = sorted(objects if addresses is None else objects.keys() & addresses) + for address in selected: + name = _hex(address) + if name not in records: + segment, display = objects[address] + try: + records[name] = _string_record(segment, address, display) + except CapabilityUnavailableError as error: + records[name] = {"value": display, "provider_value": display, "decoding": _unavailable(str(error))} + return {_hex(address): records[_hex(address)] for address in selected} def _invalidate_search_inventory(document): @@ -690,6 +717,10 @@ def _analyze_function(document, params): incoming = [] outgoing = [] procedure_addresses = set(addresses) + outgoing_calls, unresolved_calls = _native_calls(procedure, "outgoing") + incoming_calls, _ = _native_calls(procedure, "incoming") + calls = {**incoming_calls, **outgoing_calls} + edges.update(calls) for source, target in sorted(edges): source_procedure, _ = _containing_procedure(document, source) target_procedure, _ = _containing_procedure(document, target) @@ -698,16 +729,14 @@ def _analyze_function(document, params): "target_address": _hex(target), "source_procedure": _procedure_identity(source_procedure) if source_procedure is not None else None, "target_procedure": _procedure_identity(target_procedure) if target_procedure is not None else None, - "kind": _unavailable("Hopper's public Python API does not classify reference kinds"), + "kind": _unavailable("Hopper exposes partial CallReference classification, but not detailed reference flags"), + **({"call": calls[(source, target)]} if (source, target) in calls else {}), } if target in procedure_addresses and source not in procedure_addresses: incoming.append(item) if source in procedure_addresses: outgoing.append(item) - string_map = { - int(address, 16): value - for address, value in _search_inventory(document, "string") - } + string_map = {int(address, 16): record for address, record in _string_inventory(document, {int(edge["target_address"], 16) for edge in outgoing}).items()} name_map = { int(address, 16): value for address, value in _search_inventory(document, "name") @@ -717,7 +746,7 @@ def _analyze_function(document, params): for edge in outgoing: target = int(edge["target_address"], 16) if target in string_map: - referenced_strings.append({"address": edge["target_address"], "value": string_map[target], "source_address": edge["source_address"]}) + referenced_strings.append({"address": edge["target_address"], **string_map[target], "source_address": edge["source_address"]}) if target in name_map: referenced_names.append({"address": edge["target_address"], "value": name_map[target], "source_address": edge["source_address"]}) comments.sort(key=lambda item: (int(item["address"], 16), item["kind"])) @@ -735,13 +764,14 @@ def _analyze_function(document, params): "callers": callers, "callees": callees, "incoming_references": incoming, "outgoing_references": outgoing, + "unresolved_calls": unresolved_calls, "referenced_strings": referenced_strings, "referenced_names": referenced_names, "basic_blocks": blocks, "limitations": [ - "Hopper's public Python API does not classify reference kinds.", + "Native CallReference classifications are observed where available; detailed reference flags remain unknown.", "Hopper's public Python API does not expose equivalent external or thunk classification in this dossier.", - "Unresolved indirect calls without reported target addresses are not represented as call edges.", + "Native calls without resolved target addresses are not represented as edges; enumeration beyond reported CallReference objects is unknown.", "Pseudocode and assembly are provider-specific representations, not original source.", ], } @@ -934,11 +964,8 @@ def _dispatch(method, params): if method == "list_procedures": return [{"address": address, "value": value} for address, value in _search_inventory(document, "procedure")] if method == "list_strings": - values = _string_inventory(document) requested = params.get("address") - if requested is not None: - key = _hex(_address(document, requested)) - values = {key: values[key]} if key in values else {} + values = _string_inventory(document, None if requested is None else {_address(document, requested)}) return [{"address": address, **record} for address, record in values.items()] if method == "list_names": result = dict(_search_inventory(document, "name")) diff --git a/docs/installation.md b/docs/installation.md index 2b0a97c0fd2..4553b2c474d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -353,6 +353,11 @@ provider display in `provider_value`, and report its encoding, byte length, and termination. `encoding_status: inferred` distinguishes REA's decoding from an observed source encoding. Hopper can split long literals into adjacent unterminated objects; search matches each object's decoded bytes independently. +Undecodable objects retain native display text with `decoding.available: false` +and a reason, so one uncertain object does not block unrelated inspections. +Function dossiers retain this same string evidence. Native call edges retain +Hopper's partial `CallReference` classification and exact endpoints; detailed +reference flags remain unavailable rather than being invented. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/docs/testing.md b/docs/testing.md index e65cba07875..3485fa3de05 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -206,7 +206,9 @@ No provider is mocked in this lane. Real search probes cover Unicode names, literal metacharacters, case and regex modes, annotation cache invalidation, complete native fragments of long literals checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1 -decoding, and Hopper's UTF-16 symbol-name truncation boundary. When the macOS +decoding, and Hopper's UTF-16 symbol-name truncation boundary. It verifies native +CallReference classifications across reference and dossier results, retains long +string fragment metadata in dossiers. When the macOS Objective-C fixture is present, the lane also verifies native UTF-16 string objects and their inferred encodings against their actual bytes. Unmapped annotation destinations and diff --git a/scripts/lib/real-hopper-search.mjs b/scripts/lib/real-hopper-search.mjs index 349958814a7..887128d7ca8 100644 --- a/scripts/lib/real-hopper-search.mjs +++ b/scripts/lib/real-hopper-search.mjs @@ -110,6 +110,31 @@ async function verifySearchModes(call, address) { assert.ok(long.startsWith(longItem.value)); assert.ok(longItem.provider_value.endsWith("…")); assert.equal(longItem.string.termination, "missing"); + const literalProcedure = (await call("list_procedures")).find((item) => + item.value.endsWith("rea_long_literal_procedure"), + ); + assert.ok(literalProcedure); + const dossier = await call("analyze_function", { + procedure: literalProcedure.address, + }); + const referenced = dossier.referenced_strings.find( + (item) => item.address === longItem.address, + ); + assert.ok(referenced, "function dossier omitted its long literal reference"); + const { source_address, ...stringRecord } = referenced; + assert.ok( + dossier.outgoing_references.some( + (edge) => + edge.source_address === source_address && + edge.target_address === longItem.address, + ), + ); + assert.deepEqual( + stringRecord, + longItem, + "function dossier discarded typed string provenance", + ); + const fragments = strings.filter((item) => { const offset = BigInt(item.address) - BigInt(longItem.address); return offset >= 0n && offset < BigInt(Buffer.byteLength(long)); diff --git a/scripts/lib/real-hopper-semantic.mjs b/scripts/lib/real-hopper-semantic.mjs index 6cf855f2ccc..efc8f00658b 100644 --- a/scripts/lib/real-hopper-semantic.mjs +++ b/scripts/lib/real-hopper-semantic.mjs @@ -1,3 +1,4 @@ +import assert from "node:assert/strict"; import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js"; /** Require truthful tri-state permissions from real Hopper memory regions. */ @@ -122,6 +123,44 @@ export async function verifyRealHopperFixture({ require_assembly: true, }, ); + const calls = entryDossier.outgoing_references.filter( + (edge) => edge.call?.classification === "direct", + ); + assert.ok( + calls.some((edge) => edge.target_address === branch.address), + "Hopper native direct-call classification was discarded", + ); + const observed = normalizedResult( + await client.callTool( + { + name: "procedure_references", + arguments: { procedure: entry.address, direction: "outgoing" }, + }, + options, + ), + "procedure_references native calls", + ); + assert.deepEqual(entryDossier.unresolved_calls, observed.unresolved_calls); + for (const call of calls) { + assert.equal(call.call.provider_type, 2); + assert.equal( + call.call.provenance, + "hopper-public-python-api:CallReference", + ); + assert.equal( + call.kind.available, + false, + "partial call metadata invented detailed reference flags", + ); + assert.deepEqual( + observed.references.find( + (edge) => + edge.source_address === call.source_address && + edge.target_address === call.target_address, + )?.call, + call.call, + ); + } const branchDossier = requireFunctionDossierOracle( await analyzeFixtureProcedure(client, options, branch, normalizedResult), { diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index b45221311f5..542b66b7bfd 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -174,7 +174,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "procedure_references", - "Return every raw incoming or outgoing reference edge for one procedure. Endpoint procedures are resolved only from provider containment; Ghidra preserves observed reference kinds while providers without kind authority mark them unavailable.", + "Return every raw incoming or outgoing reference edge for one procedure. Endpoint procedures are resolved only from provider containment. Ghidra preserves detailed reference flags. Hopper preserves native CallReference classifications and exact endpoints while leaving unavailable detailed flags explicit.", z.object({ procedure, direction: z.enum(["incoming", "outgoing"]).default("outgoing"), @@ -198,7 +198,7 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_strings", - "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper searches complete typed bytes rather than shortened displays; adjacent fragments of a long literal are matched independently, with their termination and original display retained.", + "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently.", z.object(analysisSearchInput), ), official( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index 202c6b09784..a5bfaa0796d 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -20,7 +20,9 @@ import { import { analysisBookmarkSchema, functionInstructionWindowSchema, - referenceKindSchema, + referenceEdgeSchema, + unresolvedCallSchema, + analysisStringSchema, } from "../domain/hopperValues.js"; import { nativeApiInspectionResultSchema } from "../domain/native/nativeApiBoundary.js"; import { @@ -94,30 +96,6 @@ const bookmarkFacetSchema = z.discriminatedUnion("state", [ }), ]); -const addressedString = z.object({ - address: z.string(), - value: z.string(), - provider_value: z - .string() - .exactOptional() - .describe( - "Original provider display text, which may shorten the typed object.", - ), - string: z - .object({ - encoding: z.string().min(1), - encoding_status: z - .enum(["observed", "inferred"]) - .exactOptional() - .describe( - "Whether the provider declares the encoding or REA infers it from the typed bytes and native display. This does not establish the original source's intended encoding.", - ), - termination: z.enum(["missing", "present_or_not_required"]), - byte_length: z.number().int().min(0), - }) - .optional(), -}); - /** Exact structured-content schemas shared by direct analysis providers. */ export const officialOutputSchemas: Readonly> = { annotate_native_function: resultOf(nativeFunctionAnnotationsSchema), @@ -137,7 +115,7 @@ export const officialOutputSchemas: Readonly> = { list_names: resultOf(z.array(addressedValue)), list_procedures: resultOf(z.array(addressedValue)), list_segments: segmentOutput, - list_strings: resultOf(z.array(addressedString)), + list_strings: resultOf(z.array(analysisStringSchema)), next_address: resultOf(z.string()), prev_address: resultOf(z.string()), procedure_address: resultOf(z.string()), @@ -190,18 +168,8 @@ export const officialOutputSchemas: Readonly> = { procedure: procedureIdentity, direction: z.enum(["incoming", "outgoing"]), reference_kinds_available: z.boolean().optional(), - unresolved_calls: z - .array(z.object({ address: z.string(), reason: z.string() })) - .default([]), - references: z.array( - z.object({ - source_address: z.string(), - target_address: z.string(), - source_procedure: procedureIdentity.nullable(), - target_procedure: procedureIdentity.nullable(), - kind: referenceKindSchema, - }), - ), + unresolved_calls: z.array(unresolvedCallSchema).default([]), + references: z.array(referenceEdgeSchema), }), ), procedure_pseudo_code: resultOf(nullableText), @@ -209,7 +177,7 @@ export const officialOutputSchemas: Readonly> = { search_procedures: resultOf( z.array(z.object({ address: z.string(), value: z.string() })), ), - search_strings: resultOf(z.array(addressedString)), + search_strings: resultOf(z.array(analysisStringSchema)), set_address_name: resultOf(z.boolean()), set_addresses_names: resultOf(z.record(z.string(), z.boolean())), set_bookmark: resultOf(z.boolean()), diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 9c3bbb208a2..36664003801 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -62,6 +62,36 @@ const segmentSchema = z.object({ const unavailableAnalysisFactSchema = z .object({ available: z.literal(false), reason: z.string() }) .strict(); +/** Typed string evidence shared by inventories and referenced function literals. */ +export const analysisStringSchema = z.object({ + address: z.string(), + value: z.string(), + provider_value: z + .string() + .exactOptional() + .describe( + "Original provider display text, which may shorten the typed object.", + ), + decoding: unavailableAnalysisFactSchema + .exactOptional() + .describe( + "When present, value retains native display text because typed bytes could not be decoded.", + ), + string: z + .object({ + encoding: z.string().min(1), + encoding_status: z + .enum(["observed", "inferred"]) + .exactOptional() + .describe( + "Whether the provider declares the encoding or REA infers it from the typed bytes and native display. This does not establish the original source's intended encoding.", + ), + termination: z.enum(["missing", "present_or_not_required"]), + byte_length: z.number().int().min(0), + }) + .exactOptional(), +}); + /** An analyzed bookmark retains a missing provider label as null. */ export const analysisBookmarkSchema = z.strictObject({ address: z.string(), @@ -256,13 +286,27 @@ export const referenceKindSchema = z.discriminatedUnion("available", [ unavailableAnalysisFactSchema, availableReferenceKindSchema, ]); -const referenceEdgeSchema = z +/** Reported call sites without resolved targets; unreported dispatch stays unknown. */ +export const unresolvedCallSchema = z.strictObject({ + address: z.string(), + reason: z.string(), +}); + +/** Observed endpoints with complete or partial reference classification evidence. */ +export const referenceEdgeSchema = z .object({ source_address: z.string(), target_address: z.string(), source_procedure: procedureIdentitySchema.nullable(), target_procedure: procedureIdentitySchema.nullable(), kind: referenceKindSchema, + call: z + .strictObject({ + classification: z.enum(["none", "unknown", "direct", "objective_c"]), + provider_type: z.number().int(), + provenance: z.string().min(1), + }) + .exactOptional(), }) .strict(); export const functionDossierSchema = z @@ -286,14 +330,9 @@ export const functionDossierSchema = z callees: z.array(procedureIdentitySchema), incoming_references: z.array(referenceEdgeSchema), outgoing_references: z.array(referenceEdgeSchema), + unresolved_calls: z.array(unresolvedCallSchema).exactOptional(), referenced_strings: z.array( - z - .object({ - address: z.string(), - value: z.string(), - source_address: z.string(), - }) - .strict(), + analysisStringSchema.extend({ source_address: z.string() }), ), referenced_names: z.array( z diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 57817943fff..01c89aa0df3 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -253,7 +253,7 @@ export class HopperProvider implements AnalysisProviderCandidate { : []), ...(operation === "procedure_references" ? [ - "Hopper's public Python API does not expose flow classification for calls without resolved targets; unresolved_calls is empty and its coverage is unknown.", + "Native CallReference classifications and unresolved targets are preserved. Detailed reference flags and coverage beyond reported CallReference objects remain unknown.", ] : []), ], diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index c8039629485..88a4d5287ef 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,9 +79,12 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", - ...(operation === "list_strings" || operation === "search_strings" + ...(operation === "list_strings" || + operation === "search_strings" || + operation === "analyze_function" ? [ "String values cover native typed objects. Hopper may split long literals into adjacent unterminated fragments; string searches match each object independently.", + "Undecodable typed objects retain native display text with an explicit decoding unknown; these values do not establish decoded source literals.", "Reported string encodings are inferred from typed bytes and native display; the original source's intended encoding is unknown.", ] : []), diff --git a/tests/fixtures/hopperBridgeFacadeProbe.py b/tests/fixtures/hopperBridgeFacadeProbe.py index 097be9f6b56..6fa24f42fe8 100644 --- a/tests/fixtures/hopperBridgeFacadeProbe.py +++ b/tests/fixtures/hopperBridgeFacadeProbe.py @@ -80,6 +80,9 @@ def signatureString(self): def getLocalVariableList(self): return [] + def getAllCallees(self): + return [] + class FakeBlock: def getStartingAddress(self): From f201b04fbbb29591b0d2bb3f3b24636526e4ec4e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:06:48 +0800 Subject: [PATCH 025/791] fix(hopper)!: isolate regex matching from the native API thread BREAKING CHANGE: Hopper regex mode uses ECMAScript Unicode syntax instead of Python re. Matching runs in a cancellable worker with a five-second deadline; literal mode keeps Unicode casefold matching. --- bridge/hopper_bridge.py | 7 +- docs/installation.md | 3 + docs/testing.md | 3 +- scripts/lib/real-hopper-boundaries.mjs | 10 +- scripts/lib/real-hopper-search.mjs | 57 ++++++ src/contracts/officialToolContracts.ts | 4 +- src/hopper/HopperProvider.ts | 19 +- src/hopper/HopperProviderCapabilities.ts | 5 + src/hopper/HopperRegexSearch.ts | 244 +++++++++++++++++++++++ src/hopper/HopperRegexWorker.ts | 27 +++ 10 files changed, 365 insertions(+), 14 deletions(-) create mode 100644 src/hopper/HopperRegexSearch.ts create mode 100644 src/hopper/HopperRegexWorker.ts diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 635b27b10f6..d99669a585d 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -8,7 +8,6 @@ import json import hmac import os -import re import socket import struct from typing import Any, Optional, Protocol, Sequence @@ -585,11 +584,7 @@ def _search_results(document, kind, params): needle = pattern if case_sensitive else pattern.casefold() matches = lambda value: needle in (value if case_sensitive else value.casefold()) else: - try: - expression = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE) - except (re.error, OverflowError) as error: - raise InvalidRequestError("Invalid regex pattern") from error - matches = lambda value: expression.search(value) is not None + raise CapabilityUnavailableError("Regex matching requires REA's supervised worker; the native bridge only executes literal searches") selected = [] for item in _search_inventory(document, kind): diff --git a/docs/installation.md b/docs/installation.md index 4553b2c474d..078b29013dd 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -358,6 +358,9 @@ and a reason, so one uncertain object does not block unrelated inspections. Function dossiers retain this same string evidence. Native call edges retain Hopper's partial `CallReference` classification and exact endpoints; detailed reference flags remain unavailable rather than being invented. +Regex searches use ECMAScript Unicode syntax in a cancellable worker with a +five-second matching deadline. Deadline or cancellation stops matching while +leaving the Hopper API available. Literal mode retains Unicode casefold matching. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/docs/testing.md b/docs/testing.md index 3485fa3de05..b86be70549d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -208,7 +208,8 @@ modes, annotation cache invalidation, complete native fragments of long literals checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1 decoding, and Hopper's UTF-16 symbol-name truncation boundary. It verifies native CallReference classifications across reference and dossier results, retains long -string fragment metadata in dossiers. When the macOS +string fragment metadata in dossiers, and exercises pathological regex deadline +and cancellation followed by successful requests in the same native session. When the macOS Objective-C fixture is present, the lane also verifies native UTF-16 string objects and their inferred encodings against their actual bytes. Unmapped annotation destinations and diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index c53044bd475..e54e718109f 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -9,7 +9,10 @@ import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; -import { verifyHopperSearch } from "./real-hopper-search.mjs"; +import { + verifyHopperSearch, + verifyHopperRegexIsolation, +} from "./real-hopper-search.mjs"; /** Exercise real Hopper annotation and malformed-input boundaries over MCP. */ export async function verifyHopperBoundaryContracts( @@ -75,7 +78,10 @@ export async function verifyHopperBoundaryContracts( address, ); const workflows = await verifyHopperWorkflows(call, invalid, procedure); - const search = await verifyHopperSearch(call, invalid, procedure); + const search = { + ...(await verifyHopperSearch(call, invalid, procedure)), + ...(await verifyHopperRegexIsolation(client, options, call, address)), + }; try { if (foreignDocument !== undefined) { await invalid( diff --git a/scripts/lib/real-hopper-search.mjs b/scripts/lib/real-hopper-search.mjs index 887128d7ca8..0d796f3d2d2 100644 --- a/scripts/lib/real-hopper-search.mjs +++ b/scripts/lib/real-hopper-search.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { mcpTextValue } from "./mcp-verifier-results.mjs"; /** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */ export async function verifyHopperSearch(call, invalid, procedure) { @@ -228,3 +229,59 @@ export async function verifyHopperStringObjects(call, expectations) { displayEscapesExcludedFromMatching: true, }; } + +/** A failed or cancelled pathological regex must leave the real Hopper API usable. */ +export async function verifyHopperRegexIsolation( + client, + options, + call, + address, +) { + const original = await call("address_name", { address }); + const pathological = "a".repeat(40) + "!"; + try { + await call("set_address_name", { address, name: pathological }); + const reply = await client.callTool( + { + name: "search_procedures", + arguments: { pattern: "(a+)+$", mode: "regex", case_sensitive: true }, + }, + options, + ); + assert.equal(reply.isError, true); + const { error } = JSON.parse(mcpTextValue(reply)); + assert.equal(error.code, "resource_constraint"); + assert.equal(error.details.operation, "search_procedures"); + assert.equal(error.details.resource, "cpu"); + assert.equal(await call("address_name", { address }), pathological); + const controller = new AbortController(); + const pending = client.callTool( + { + name: "search_procedures", + arguments: { pattern: "(a+)+$", mode: "regex", case_sensitive: true }, + }, + { ...options, signal: controller.signal }, + ); + const cancelled = pending.catch((cause) => cause); + // The same small inventory has already completed once; leave the pathological + // matching worker running before cancelling the second caller. + await new Promise((resolve) => setTimeout(resolve, 750)); + controller.abort(); + await cancelled; + assert.equal(await call("address_name", { address }), pathological); + assert.deepEqual( + await call("search_procedures", { + pattern: pathological, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: pathological }], + ); + } finally { + await call("set_address_name", { address, name: original }); + } + return { + regexDeadlinePreservesSession: true, + regexCancellationPreservesSession: true, + }; +} diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 542b66b7bfd..7d4d0715f58 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -193,12 +193,12 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_procedures", - "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete.", + "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", z.object(analysisSearchInput), ), official( "search_strings", - "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently.", + "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", z.object(analysisSearchInput), ), official( diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 01c89aa0df3..093f2633588 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -23,6 +23,7 @@ import { hopperLoaderArgsForTarget, resolveHopperAnalysisProfile, } from "./HopperAnalysisProfile.js"; +import { HopperRegexSearch } from "./HopperRegexSearch.js"; import { HopperClient } from "./HopperClient.js"; import { hopperMachOImageSchema } from "./HopperMachOImage.js"; @@ -229,9 +230,15 @@ export class HopperProvider implements AnalysisProviderCandidate { onDiagnostic: (diagnostic) => this.logger.info(diagnostic, "Hopper bridge reported a diagnostic"), }); + const regexSearch = new HopperRegexSearch(client); return { execute: async (operation, parameters, options) => { - const result = await client.callTool(operation, parameters, options); + const result = + (operation === "search_strings" || + operation === "search_procedures") && + parameters.mode === "regex" + ? await regexSearch.execute(operation, parameters, options) + : await client.callTool(operation, parameters, options); return result.ok ? { ok: true, @@ -286,8 +293,14 @@ export class HopperProvider implements AnalysisProviderCandidate { ]; }, operationHealthSnapshot: () => client.operationHealth(), - closeWithOutcome: (options) => client.closeWithOutcome(options), - close: () => client.close(), + closeWithOutcome: async (options) => { + await regexSearch.close(); + return client.closeWithOutcome(options); + }, + close: async () => { + await regexSearch.close(); + await client.close(); + }, }; } } diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 88a4d5287ef..1d237319101 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,6 +79,11 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", + ...(operation === "search_strings" || operation === "search_procedures" + ? [ + "Regex mode uses ECMAScript Unicode semantics in a supervised worker with a five-second matching deadline. Cancellation or deadline failure stops matching without blocking Hopper; literal mode retains native casefold semantics.", + ] + : []), ...(operation === "list_strings" || operation === "search_strings" || operation === "analyze_function" diff --git a/src/hopper/HopperRegexSearch.ts b/src/hopper/HopperRegexSearch.ts new file mode 100644 index 00000000000..585fb363159 --- /dev/null +++ b/src/hopper/HopperRegexSearch.ts @@ -0,0 +1,244 @@ +import { Worker } from "node:worker_threads"; +import { z } from "zod"; + +import type { ExecutionOptions } from "../application/AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { analysisStringSchema } from "../domain/hopperValues.js"; +import { HopperRemoteError } from "../domain/hopperErrors.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { analysisSearchInput } from "../contracts/analysisSearchContract.js"; +import type { HopperClient } from "./HopperClient.js"; + +const MATCHING_DEADLINE_MS = 5_000; +const inputSchema = z.strictObject({ + ...analysisSearchInput, + mode: z.literal("regex"), + document: z.string().nullish(), + case_sensitive: z + .boolean() + .nullish() + .transform((value) => value ?? false), +}); +const replySchema = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("matched"), + indices: z.array(z.number().int().nonnegative()), + }), + z.strictObject({ status: z.literal("invalid_pattern"), reason: z.string() }), +]); + +/** Match collected native facts off Hopper's API thread, with cancellable lifetime. */ +export class HopperRegexSearch { + readonly #active = new Map< + AbortController, + Promise> + >(); + + constructor(private readonly client: HopperClient) {} + + execute( + operation: "search_strings" | "search_procedures", + parameters: Readonly>, + options?: ExecutionOptions, + ): Promise> { + const controller = new AbortController(); + const signal = + options?.signal === undefined + ? controller.signal + : AbortSignal.any([controller.signal, options.signal]); + const pending = this.#execute(operation, parameters, { + ...options, + signal, + }); + this.#active.set(controller, pending); + void pending.then( + () => this.#active.delete(controller), + () => this.#active.delete(controller), + ); + return pending; + } + + /** Abort matching workers before their provider session closes. */ + async close(): Promise { + for (const controller of this.#active.keys()) controller.abort(); + await Promise.all(this.#active.values()); + } + + async #execute( + operation: "search_strings" | "search_procedures", + parameters: Readonly>, + options: ExecutionOptions, + ): Promise> { + const input = inputSchema.safeParse(parameters); + if (!input.success) + return err( + new HopperRemoteError( + -32000, + `Invalid search arguments: ${input.error.message}`, + { diagnosticType: "invalid_request", operation }, + ), + ); + const inventoryOptions: ExecutionOptions = { + ...options, + ...(options.progress === undefined + ? {} + : { + progress: { + report: (update) => + options.progress?.report({ + ...update, + completed: + update.total === null || update.total === 0 + ? 0 + : Math.min(0.5, update.completed / update.total / 2), + total: 1, + }) ?? Promise.resolve(), + }, + }), + }; + const inventory = await this.client.callTool( + operation === "search_strings" ? "list_strings" : "list_procedures", + input.data.document == null ? {} : { document: input.data.document }, + inventoryOptions, + ); + if (!inventory.ok) return inventory; + const parsed = z.array(analysisStringSchema).safeParse(inventory.value); + if (!parsed.success) + return err( + new AnalysisOutputError( + operation, + "Hopper search inventory is malformed", + ), + ); + if (options.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + await options.progress + ?.report({ + phase: "hopper_regex", + completed: 0.5, + total: 1, + message: "Matching inventory in cancellable regex worker", + }) + .catch(() => undefined); + if (options.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + let worker: Worker; + try { + worker = new Worker(new URL("./HopperRegexWorker.js", import.meta.url), { + workerData: { + pattern: input.data.pattern, + caseSensitive: input.data.case_sensitive, + values: parsed.data.map((item) => item.value), + }, + }); + } catch (cause: unknown) { + return err( + new AnalysisOutputError( + operation, + `Cannot start regex worker: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ), + ); + } + try { + return await new Promise>((resolve) => { + const finish = (result: Result): void => { + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + resolve(result); + }; + const onAbort = (): void => + finish(err(new AnalysisCancelledError(operation))); + const timer = setTimeout( + () => + finish( + err( + new AnalysisResourceConstraintError( + operation, + "cpu", + "Regex matching exceeded its supervised deadline; the Hopper API thread remains available.", + { matching_deadline_ms: MATCHING_DEADLINE_MS }, + { + remediationAction: + "Simplify the ECMAScript Unicode regex or use literal mode, then retry in the same session.", + }, + ), + ), + ), + MATCHING_DEADLINE_MS, + ); + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + worker.once("message", (message: unknown) => { + const reply = replySchema.safeParse(message); + if (!reply.success) + return finish( + err( + new AnalysisOutputError( + operation, + "Regex worker reply is malformed", + ), + ), + ); + if (reply.data.status === "invalid_pattern") + return finish( + err( + new HopperRemoteError( + -32000, + `Invalid regex pattern: ${reply.data.reason}`, + { diagnosticType: "invalid_request", operation }, + ), + ), + ); + const indices = reply.data.indices; + if ( + indices.some( + (index, position) => + index >= parsed.data.length || + (position > 0 && index <= (indices[position - 1] ?? -1)), + ) + ) + return finish( + err( + new AnalysisOutputError( + operation, + "Regex worker returned invalid inventory indices", + ), + ), + ); + const selected = new Set(indices); + finish(ok(parsed.data.filter((_, index) => selected.has(index)))); + }); + worker.once("error", (cause: Error) => + finish( + err( + new AnalysisOutputError( + operation, + `Regex worker failed: ${cause.message}`, + { cause }, + ), + ), + ), + ); + worker.once("exit", (code) => + finish( + err( + new AnalysisOutputError( + operation, + `Regex worker exited without a result (${code})`, + ), + ), + ), + ); + }); + } finally { + await worker.terminate(); + } + } +} diff --git a/src/hopper/HopperRegexWorker.ts b/src/hopper/HopperRegexWorker.ts new file mode 100644 index 00000000000..71fd79b5093 --- /dev/null +++ b/src/hopper/HopperRegexWorker.ts @@ -0,0 +1,27 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { z } from "zod"; + +const input = z + .strictObject({ + pattern: z.string().min(1), + caseSensitive: z.boolean(), + values: z.array(z.string()), + }) + .parse(workerData); + +let expression: RegExp | undefined; +try { + expression = new RegExp(input.pattern, input.caseSensitive ? "u" : "iu"); +} catch (cause: unknown) { + parentPort?.postMessage({ + status: "invalid_pattern", + reason: cause instanceof Error ? cause.message : String(cause), + }); +} +if (expression !== undefined) { + const indices = []; + for (const [index, value] of input.values.entries()) { + if (expression.test(value)) indices.push(index); + } + parentPort?.postMessage({ status: "matched", indices }); +} From 2b86ea9c7cd3cfc2141a021f8b531dbc84395c81 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:06:48 +0800 Subject: [PATCH 026/791] docs: require partial native facts at provider boundaries --- AGENTS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index aec33ebf1ca..0a2bc53b996 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,6 +60,8 @@ Keep portable evidence and scenario validation distinct from host-native executi Preserve meaningful failure reasons through application and adapter layers. Malformed input is distinct from an unsupported target, unavailable provider, or host operating-system permission denial. Diagnostics should identify the failed constraint and the target or lifecycle request it applies to. Recovery advice must address that reason and point to an available workflow; generic catches must not erase actionable validation details. Preserve collected observations on execution failure independently of whether cleanup succeeds. +Preserve partial native facts. A shared schema must not require one provider's full metadata before another provider can report the facts it observes. Keep observed classifications and endpoints alongside explicit unknowns for unsupported flags; never invent missing flags or discard known facts to satisfy an all-or-nothing availability shape. Before claiming a native API lacks a capability, inspect the installed API and probe representative producer objects. Exercise the resulting partial representation through CLI, MCP, and composed results such as function dossiers. + Leave meaningful target, action, capture, and output choices to the agent. A selected operation already expresses intent; do not require approval booleans or repeated permission declarations. Trace each setting to its consumer: remove ignored options and single-value confirmations, derive built-in lifecycle behavior, and supply defaults for omitted optional metadata. Report actual effects and limitations where they help interpret results rather than asking callers to restate them. Verify the representation consumed at the next boundary. Internal validator success does not establish JSON Schema validity or client compatibility: validate advertised input and output schemas against their declared dialect after SDK conversion, and keep generated contracts aligned. Use representative producer data and regressions that exercise the failed behavior. Platform and provider support claims require the corresponding real workflow; package startup, capability probes, and mock transport tests establish only their narrower claims. Report unverified coverage explicitly. From 4a45c14742e6868f687cead2ddf736c3f7c49acd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:40:47 +0800 Subject: [PATCH 027/791] fix(firmware): reject malformed UTF-8 producer reports --- docs/firmware-analysis.md | 5 ++- src/firmware/FirmwareFiles.ts | 6 ++- .../firmware/firmwareReportFiles.test.ts | 43 +++++++++++++++++++ 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 tests/boundary/firmware/firmwareReportFiles.test.ts diff --git a/docs/firmware-analysis.md b/docs/firmware-analysis.md index 1796ef43a3f..12886bd0d22 100644 --- a/docs/firmware-analysis.md +++ b/docs/firmware-analysis.md @@ -129,8 +129,9 @@ tree semantics, vulnerability scanning, emulation or hardware runtime capture. Unblob one worker. No automatic parallel extraction engines. - **120-second** deadline; inherited **1 GiB per-process address-space** limit, 120 CPU seconds and zero core-dump allowance through `prlimit`. -- JSON reports: at most **8 MiB**; oversized reports fail without a truncated - success. Diagnostic output is bounded with exact captured byte counts. +- JSON reports: at most **8 MiB** and valid UTF-8; malformed encoding and + oversized reports fail without repairing or truncating producer data. + Diagnostic output is bounded with exact captured byte counts. - Default staging budget: **256 MiB** and **10,000 entries**, configurable using `max_output_bytes` / `max_output_files` (CLI `--max-output-bytes` / `--max-output-files`). Depth defaults to 3, allowed range 1–10. diff --git a/src/firmware/FirmwareFiles.ts b/src/firmware/FirmwareFiles.ts index ad63b49dea5..a118984b811 100644 --- a/src/firmware/FirmwareFiles.ts +++ b/src/firmware/FirmwareFiles.ts @@ -153,7 +153,11 @@ export const readFirmwareReport = async ( `Report exceeded ${FIRMWARE_LIMITS.reportBytes} bytes`, ); try { - const value: unknown = JSON.parse(bytes.toString("utf8")); + const value: unknown = JSON.parse( + new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes, + ), + ); return value; } catch (cause: unknown) { throw new AnalysisOutputError( diff --git a/tests/boundary/firmware/firmwareReportFiles.test.ts b/tests/boundary/firmware/firmwareReportFiles.test.ts new file mode 100644 index 00000000000..f3a1816242a --- /dev/null +++ b/tests/boundary/firmware/firmwareReportFiles.test.ts @@ -0,0 +1,43 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; + +import { readFirmwareReport } from "../../../src/firmware/FirmwareFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.each([ + ["invalid leading byte", [0xff]], + ["incomplete sequence", [0xe2, 0x82]], + ["overlong sequence", [0xc0, 0xaf]], + ["encoded surrogate", [0xed, 0xa0, 0x80]], +])( + "rejects complete JSON containing %s without repairing producer bytes", + async (_name, bytes) => { + const root = await createTestTempDirectory("rea-firmware-report-"); + const path = join(root, "report.json"); + await writeFile( + path, + Buffer.concat([ + Buffer.from('{"description":"'), + Buffer.from(bytes), + Buffer.from('"}'), + ]), + ); + await expect( + readFirmwareReport(path, "inspect_firmware_regions"), + ).rejects.toMatchObject({ + _tag: "AnalysisOutputError", + reason: "Tool report is not valid complete JSON", + }); + }, +); + +it("preserves valid Unicode and a literal replacement character in producer JSON", async () => { + const root = await createTestTempDirectory("rea-firmware-report-"); + const path = join(root, "report.json"); + const report = { description: "固件 🧩 \ufffd", path: "/输入/firmware.bin" }; + await writeFile(path, JSON.stringify(report)); + expect(await readFirmwareReport(path, "inspect_firmware_regions")).toEqual( + report, + ); +}); From f2e9c491ea00bf79d5665102d95a3339b63cfe4f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:08:24 +0800 Subject: [PATCH 028/791] fix(native): share command supervision and retain failure output --- docs/native-investigation.md | 6 + src/domain/analysisErrorBase.ts | 4 + src/native/CommandRunner.ts | 248 +++++++++++----- src/native/NativeMacOSProvider.ts | 73 ++++- src/process/OwnedCommand.ts | 23 +- src/process/ProviderProcess.ts | 24 +- tests/boundary/process/ownedCommand.test.ts | 58 +++- .../providers/native/nativeProvider.test.ts | 266 +++++++++++++++++- 8 files changed, 604 insertions(+), 98 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 6255d5e4e17..e385ef9e568 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -12,6 +12,12 @@ input file and include the observed lipo slice offset for universal binaries. If that offset is unavailable, segment evidence locations are omitted with an explicit limitation; architecture inventory locations remain available. +Native Xcode command execution uses shared process supervision with a 60-second +deadline and a 64 MiB aggregate stdout/stderr budget. Timeout, cancellation, +stream failure, and output exhaustion retain the captured output, exit details, +and cleanup outcome in the error result. Truncated output is never parsed as a +complete observation. + - `trace_dylib_resolution` / `rea trace-dylib-resolution ` parses Mach-O load commands in TypeScript and follows dyld's path expansion for every executable in an app bundle, or for one Mach-O within its directory. diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index a6a0f7b2556..31ae207dbf7 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -37,6 +37,10 @@ export interface AnalysisCapturedOutput { readonly stdout: string; readonly stderr: string; readonly truncated: boolean; + readonly stdout_bytes?: number; + readonly stderr_bytes?: number; + readonly exit_code?: number | null; + readonly signal?: string | null; } /** Cleanup uncertainty attached to the original typed analysis failure. */ diff --git a/src/native/CommandRunner.ts b/src/native/CommandRunner.ts index 962535fa8cd..37794d15822 100644 --- a/src/native/CommandRunner.ts +++ b/src/native/CommandRunner.ts @@ -1,9 +1,18 @@ import { createHash, randomUUID } from "node:crypto"; import { createReadStream } from "node:fs"; import { realpath } from "node:fs/promises"; -import { spawn } from "node:child_process"; import { err, ok, type Result } from "../domain/result.js"; +import { + OwnedCommandFailure, + runOwnedCommand, +} from "../process/OwnedCommand.js"; +import type { ProviderProcessSnapshot } from "../process/ProviderProcess.js"; + +/** Maximum wall-clock duration for one native command. */ +export const NATIVE_COMMAND_TIMEOUT_MS = 60_000; +/** Retain complete command output up to a bounded aggregate size. */ +export const NATIVE_COMMAND_OUTPUT_BUDGET_BYTES = 64 * 1024 * 1024; export interface NativeCommandCapture { readonly tool: string; @@ -20,17 +29,38 @@ export interface NativeCommandCapture { readonly signal: string | null; } +/** Native command failure with retained partial output and cleanup status. */ export class NativeCommandFailure extends Error { constructor( readonly tool: string, - readonly reason: "unavailable" | "cancelled" | "nonzero-exit" | "io", + readonly reason: + | "unavailable" + | "cancelled" + | "timeout" + | "output-limit" + | "nonzero-exit" + | "io", readonly exitCode: number | null = null, options?: ErrorOptions, + readonly capture: NativeCommandFailureCapture | null = null, + readonly cleanupFailure: string | null = null, + readonly cleanupResources: readonly string[] = [], ) { super(`Native command ${tool} failed: ${reason}`, options); } } +/** Partial output and lifecycle details retained when a native command fails. */ +export interface NativeCommandFailureCapture { + readonly stdout: string; + readonly stderr: string; + readonly stdoutBytes: number; + readonly stderrBytes: number; + readonly exitCode: number | null; + readonly signal: string | null; + readonly truncated: boolean; +} + /** Cancellation and exit handling for one allowlisted native command. */ export interface NativeCommandOptions { readonly signal?: AbortSignal; @@ -136,20 +166,66 @@ export const resolveXcrunTool = async ( signal === undefined ? {} : { signal }, ); if (!found.ok) { - if (found.error.reason === "cancelled") - return err(new NativeCommandFailure(tool, "cancelled")); - return err(new NativeCommandFailure(tool, "unavailable")); + return err( + new NativeCommandFailure( + tool, + found.error.reason === "nonzero-exit" || + found.error.reason === "unavailable" + ? "unavailable" + : found.error.reason, + found.error.exitCode, + { cause: found.error }, + found.error.capture, + found.error.cleanupFailure, + found.error.cleanupResources, + ), + ); } if (found.value.exitCode !== 0) - return err(new NativeCommandFailure(tool, "unavailable")); + return err( + new NativeCommandFailure( + tool, + "unavailable", + found.value.exitCode, + undefined, + failureCapture(found.value, false), + ), + ); const candidate = found.value.stdout.trim(); if (!candidate.startsWith("/")) - return err(new NativeCommandFailure(tool, "unavailable")); + return err( + new NativeCommandFailure( + tool, + "unavailable", + found.value.exitCode, + undefined, + failureCapture(found.value, false), + ), + ); try { const path = await realpath(candidate); - return ok({ path, sha256: await hashFile(path) }); + return ok({ path, sha256: await hashFile(path, signal) }); } catch (cause: unknown) { - return err(new NativeCommandFailure(tool, "io", null, { cause })); + const capture = failureCapture(found.value, false); + if (signal?.aborted === true) + return err( + new NativeCommandFailure( + tool, + "cancelled", + capture.exitCode, + { cause }, + capture, + ), + ); + return err( + new NativeCommandFailure( + tool, + "io", + capture.exitCode, + { cause }, + capture, + ), + ); } }; @@ -163,7 +239,20 @@ type ProcessCapture = Omit< | "arguments" >; -const captureProcess = ( +const failureCapture = ( + capture: ProcessCapture, + truncated: boolean, +): NativeCommandFailureCapture => ({ + stdout: capture.stdout, + stderr: capture.stderr, + stdoutBytes: capture.stdoutBytes, + stderrBytes: capture.stderrBytes, + exitCode: capture.exitCode, + signal: capture.signal, + truncated, +}); + +const captureProcess = async ( executable: string, arguments_: readonly string[], tool: string, @@ -171,80 +260,83 @@ const captureProcess = ( readonly signal?: AbortSignal; readonly acceptNonZero?: boolean; }, -): Promise> => - new Promise((resolve) => { - if (options.signal?.aborted === true) { - resolve(err(new NativeCommandFailure(tool, "cancelled"))); - return; - } - const runToken = randomUUID(); - const child = spawn(executable, [...arguments_], { - shell: false, - stdio: ["ignore", "pipe", "pipe"], - env: { - PATH: "/usr/bin:/bin:/usr/sbin:/sbin", - LC_ALL: "C", - LANG: "C", - REA_NATIVE_RUN_TOKEN: runToken, +): Promise> => { + const runId = `rea-native-command-${randomUUID()}`; + try { + const snapshot = await runOwnedCommand( + { + command: executable, + arguments: [...arguments_], + runId, + env: { + PATH: "/usr/bin:/bin:/usr/sbin:/sbin", + LC_ALL: "C", + LANG: "C", + }, + hostEnvironment: {}, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }, + { + timeoutMs: NATIVE_COMMAND_TIMEOUT_MS, + diagnosticBytes: NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, + acceptNonZeroExit: options.acceptNonZero === true, }, - }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - let stdoutBytes = 0; - let stderrBytes = 0; - let settled = false; - let reason: NativeCommandFailure["reason"] | undefined; - const finish = ( - result: Result, - ): void => { - if (settled) return; - settled = true; - options.signal?.removeEventListener("abort", onAbort); - resolve(result); - }; - const stop = (failure: NativeCommandFailure["reason"]): void => { - reason ??= failure; - child.kill("SIGKILL"); - }; - const onAbort = (): void => stop("cancelled"); - options.signal?.addEventListener("abort", onAbort, { once: true }); - child.stdout.on("data", (chunk: Buffer) => { - stdoutBytes += chunk.length; - stdout.push(chunk); - }); - child.stderr.on("data", (chunk: Buffer) => { - stderrBytes += chunk.length; - stderr.push(chunk); - }); - child.once("error", (cause: unknown) => - finish(err(new NativeCommandFailure(tool, "io", null, { cause }))), ); - child.once("close", (code, signal) => { - if (reason !== undefined) { - finish(err(new NativeCommandFailure(tool, reason, code))); - return; - } - if (signal !== null || (code !== 0 && options.acceptNonZero !== true)) { - finish(err(new NativeCommandFailure(tool, "nonzero-exit", code))); - return; - } - finish( - ok({ - stdout: Buffer.concat(stdout).toString("utf8"), - stderr: Buffer.concat(stderr).toString("utf8"), - stdoutBytes, - stderrBytes, - exitCode: code, - signal, - }), + return ok(processCaptureFromSnapshot(snapshot)); + } catch (cause: unknown) { + if (cause instanceof OwnedCommandFailure) { + const capture = + cause.snapshot === null + ? null + : failureCapture( + processCaptureFromSnapshot(cause.snapshot), + cause.snapshot.diagnosticTruncated === true, + ); + return err( + new NativeCommandFailure( + tool, + cause.reason === "process" + ? processFailureReason(capture) + : cause.reason, + capture?.exitCode ?? null, + { cause }, + capture, + cause.cleanupFailure, + cause.resources, + ), ); - }); - }); + } + return err(new NativeCommandFailure(tool, "io", null, { cause })); + } +}; + +const processFailureReason = ( + capture: NativeCommandFailureCapture | null, +): "io" | "nonzero-exit" => + capture !== null && + (capture.signal !== null || + (capture.exitCode !== null && capture.exitCode !== 0)) + ? "nonzero-exit" + : "io"; -const hashFile = (path: string): Promise => +const processCaptureFromSnapshot = ( + snapshot: ProviderProcessSnapshot, +): ProcessCapture => ({ + stdout: snapshot.stdout.text, + stderr: snapshot.stderr.text, + stdoutBytes: snapshot.stdout.bytes, + stderrBytes: snapshot.stderr.bytes, + exitCode: snapshot.exitCode ?? null, + signal: snapshot.signal ?? null, +}); + +const hashFile = (path: string, signal?: AbortSignal): Promise => new Promise((resolve, reject) => { const hash = createHash("sha256"); - const stream = createReadStream(path); + const stream = createReadStream( + path, + signal === undefined ? undefined : { signal }, + ); stream.on("data", (chunk) => hash.update(chunk)); stream.once("error", reject); stream.once("end", () => resolve(hash.digest("hex"))); diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index d4806679053..d011509d6d6 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -31,6 +31,8 @@ import { AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisOutputError, + AnalysisResourceConstraintError, + AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; @@ -49,6 +51,8 @@ import { } from "../domain/native/nativeInspection.js"; import { err, ok, type Result } from "../domain/result.js"; import { + NATIVE_COMMAND_TIMEOUT_MS, + NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, NativeCommandFailure, XcrunCommandRunner, type NativeCommandCapture, @@ -641,15 +645,64 @@ const translateCommandFailure = ( operation: NativeToolName, failure: NativeCommandFailure, ): AnalysisError => { + const capturedOutput = + failure.capture === null + ? undefined + : { + stdout: failure.capture.stdout, + stderr: failure.capture.stderr, + truncated: failure.capture.truncated, + stdout_bytes: failure.capture.stdoutBytes, + stderr_bytes: failure.capture.stderrBytes, + exit_code: failure.capture.exitCode, + signal: failure.capture.signal, + }; + const cleanup = + failure.cleanupFailure === null + ? undefined + : { + reason: failure.cleanupFailure, + resources: [...failure.cleanupResources], + }; + const errorOptions = { + ...(capturedOutput === undefined ? {} : { capturedOutput }), + ...(cleanup === undefined ? {} : { cleanup }), + }; if (failure.reason === "unavailable") return new AnalysisCapabilityUnavailableError( IDENTITY.id, operation, `${failure.tool} is unavailable through xcrun.`, + errorOptions, ); if (failure.reason === "cancelled") - return new AnalysisCancelledError(operation); - return new ProviderAdapterError(IDENTITY.id, operation, { cause: failure }); + return new AnalysisCancelledError(operation, errorOptions); + if (failure.reason === "timeout") + return new AnalysisTimeoutError( + operation, + NATIVE_COMMAND_TIMEOUT_MS, + errorOptions, + ); + if (failure.reason === "output-limit") + return new AnalysisResourceConstraintError( + operation, + "memory", + `Native command output exceeded ${String(NATIVE_COMMAND_OUTPUT_BUDGET_BYTES)} bytes; complete output is required for analysis.`, + { max_output_bytes: NATIVE_COMMAND_OUTPUT_BUDGET_BYTES }, + errorOptions, + ); + return new ProviderAdapterError(IDENTITY.id, operation, { + cause: failure, + ...errorOptions, + diagnostics: { + tool: failure.tool, + reason: failure.reason, + exit_code: failure.capture?.exitCode ?? failure.exitCode, + signal: failure.capture?.signal ?? null, + stdout_bytes: failure.capture?.stdoutBytes ?? 0, + stderr_bytes: failure.capture?.stderrBytes ?? 0, + }, + }); }; const translateCodeSignExitFailure = ( @@ -657,7 +710,21 @@ const translateCodeSignExitFailure = ( ): AnalysisError => translateCommandFailure( "inspect_signature", - new NativeCommandFailure("codesign", "nonzero-exit", capture.exitCode), + new NativeCommandFailure( + "codesign", + "nonzero-exit", + capture.exitCode, + undefined, + { + stdout: capture.stdout, + stderr: capture.stderr, + stdoutBytes: capture.stdoutBytes, + stderrBytes: capture.stderrBytes, + exitCode: capture.exitCode, + signal: capture.signal, + truncated: false, + }, + ), ); const commandOutput = (capture: NativeCommandCapture): string => diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 1fa054c709e..b5de06b644e 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -30,7 +30,12 @@ export class OwnedCommandFailure extends Error { /** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ export const runOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, - limits: { readonly timeoutMs: number; readonly diagnosticBytes: number }, + limits: { + readonly timeoutMs: number; + readonly diagnosticBytes: number; + /** Accept an ordinary nonzero exit after output closes without other process errors. */ + readonly acceptNonZeroExit?: boolean; + }, options: { readonly signal?: AbortSignal; readonly launcher?: ( @@ -60,7 +65,11 @@ export const runOwnedCommand = async ( const collectOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, - limits: { readonly diagnosticBytes: number }, + limits: { + readonly diagnosticBytes: number; + /** Accept an ordinary nonzero exit after output closes without other process errors. */ + readonly acceptNonZeroExit?: boolean; + }, launcher: ( input: OwnedProviderProcessSpawnOptions, ) => Promise, @@ -143,10 +152,16 @@ const collectOwnedCommand = async ( "Command diagnostic output exceeded its complete-output budget.", snapshot, ); + const acceptedNonzeroExit = + limits.acceptNonZeroExit === true && + snapshot.exitCode !== null && + snapshot.exitCode !== undefined && + snapshot.exitCode !== 0 && + snapshot.signal === null; if ( processFailure !== undefined || - snapshot.exitCode !== 0 || - snapshot.signal !== null + ((snapshot.exitCode !== 0 || snapshot.signal !== null) && + !acceptedNonzeroExit) ) throw new OwnedCommandFailure( "process", diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index 896ae47e6f5..5bedea76e75 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -223,6 +223,12 @@ export class ProviderProcessSupervisor { readonly #onStderr = (chunk: Buffer | string): void => { this.#capture("stderr", this.#stderr, chunk); }; + readonly #onStdoutError = (cause: Error): void => { + this.#recordStreamError("stdout", cause); + }; + readonly #onStderrError = (cause: Error): void => { + this.#recordStreamError("stderr", cause); + }; readonly #onExit = ( code: number | null, signal: NodeJS.Signals | null, @@ -315,7 +321,9 @@ export class ProviderProcessSupervisor { if (this.#disposed) return; this.#disposed = true; this.launch.process.stdout?.off("data", this.#onStdout); + this.launch.process.stdout?.off("error", this.#onStdoutError); this.launch.process.stderr?.off("data", this.#onStderr); + this.launch.process.stderr?.off("error", this.#onStderrError); this.launch.process.off("exit", this.#onExit); this.launch.process.off("close", this.#onClose); this.launch.process.off("error", this.#onError); @@ -404,8 +412,20 @@ export class ProviderProcessSupervisor { } #attach(stream: Readable | null, name: "stdout" | "stderr"): void { - if (name === "stdout") stream?.on("data", this.#onStdout); - else stream?.on("data", this.#onStderr); + if (name === "stdout") { + stream?.on("data", this.#onStdout); + stream?.on("error", this.#onStdoutError); + } else { + stream?.on("data", this.#onStderr); + stream?.on("error", this.#onStderrError); + } + } + + #recordStreamError(stream: "stdout" | "stderr", cause: Error): void { + this.#options.onDiagnostic?.({ + type: "error", + message: `${stream} stream failed: ${cause.message}`, + }); } #capture( diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index e647263315c..05ccdb81cd4 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -1,6 +1,8 @@ import { randomUUID } from "node:crypto"; +import { EventEmitter } from "node:events"; +import { PassThrough } from "node:stream"; import { setTimeout as delay } from "node:timers/promises"; -import { beforeAll, expect, it } from "vitest"; +import { beforeAll, expect, it, vi } from "vitest"; import { OwnedCommandFailure, runOwnedCommand, @@ -93,6 +95,60 @@ it("collects a real owned command and verifies its exit before returning diagnos }); }); +it("fails an accepted nonzero command when an output stream errors and cleans it up", async () => { + const stdout = new PassThrough(); + const stderr = new PassThrough(); + class FakeProcess extends EventEmitter { + readonly stdout = stdout; + readonly stderr = stderr; + exitCode: number | null = null; + signalCode: NodeJS.Signals | null = null; + readonly pid = 41234; + kill(): boolean { + return true; + } + } + const child = new FakeProcess(); + const cleanup = vi.fn(async () => ({ + cleaned: true as const, + signaled: false, + })); + const request = command("unused fake launcher command"); + + const failure = runOwnedCommand( + request, + { timeoutMs: 2_000, diagnosticBytes: 1024, acceptNonZeroExit: true }, + { + launcher: async () => { + setImmediate(() => { + stdout.write("partial output"); + stdout.destroy(new Error("fixture read failure")); + stderr.end(); + child.exitCode = 7; + child.emit("exit", 7, null); + child.emit("close", 7, null); + }); + return { + process: child, + ownership: { + runId: request.runId, + leaderPid: child.pid, + processGroupId: child.pid, + }, + cleanup, + }; + }, + }, + ); + + await expect(failure).rejects.toMatchObject({ + reason: "process", + message: "stdout stream failed: fixture read failure", + snapshot: { stdout: { text: "partial output" }, exitCode: 7 }, + }); + expect(cleanup).toHaveBeenCalledOnce(); +}); + it("cancels a real acquired process and independently releases it", async () => { const controller = new AbortController(); let pid: number | undefined; diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index e0dca4d21d2..91b6d18d72b 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -1,12 +1,13 @@ import { mkdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, beforeAll, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js"; import { + NATIVE_COMMAND_OUTPUT_BUDGET_BYTES, NativeCommandFailure, XcrunCommandRunner, type NativeCommandRunner, @@ -14,6 +15,8 @@ import { import { err, ok } from "../../../../src/domain/result.js"; import { parseLipoArchitectures } from "../../../../src/native/parsers/lipo.js"; import { parseCodeSignature } from "../../../../src/native/parsers/codesign.js"; +import { prepareProcessOwnershipInspection } from "../../../../src/process/ProcessOwnershipObservation.js"; +import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { NativeFixtureRunner as FixtureRunner, @@ -22,6 +25,8 @@ import { nativeMachoTargetForFile, } from "../../../fixtures/nativeCommands.js"; +beforeAll(() => prepareProcessOwnershipInspection()); + let directory: string | undefined; afterEach(async () => { if (directory !== undefined) @@ -29,7 +34,7 @@ afterEach(async () => { directory = undefined; }); -describe("native macOS provider discovery and inspection", () => { +describe("native macOS provider discovery", () => { it.each(["ordinary.fixture", " spaced.fixture ", " leading", "trailing "])( "preserves the exact signing identifier %j reported by codesign", (identifier) => { @@ -87,6 +92,48 @@ describe("native macOS provider discovery and inspection", () => { expect(resolutions).toBe(2); }); + it.each(["timeout", "output-limit"] as const)( + "preserves %s resolution failures with their captured output and cleanup status", + async (reason) => { + const failure = new NativeCommandFailure( + "file", + reason, + 9, + undefined, + { + stdout: "partial xcrun output", + stderr: "partial xcrun diagnostic", + stdoutBytes: 20, + stderrBytes: 25, + exitCode: 9, + signal: null, + truncated: reason === "output-limit", + }, + "ownership cleanup was not confirmed", + ["process-group:1234"], + ); + const runner = new XcrunCommandRunner(() => + Promise.resolve(err(failure)), + ); + + const result = await runner.run("file", [], {}); + + expect(result).toEqual(err(failure)); + if (result.ok) throw new Error("Resolution failure unexpectedly passed"); + expect(result.error).toMatchObject({ + reason, + capture: { + stdout: "partial xcrun output", + truncated: reason === "output-limit", + }, + cleanupFailure: "ownership cleanup was not confirmed", + cleanupResources: ["process-group:1234"], + }); + }, + ); +}); + +describe("native command output collection", () => { it("retains the complete native command output", async () => { const runner = new XcrunCommandRunner(() => Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), @@ -101,25 +148,224 @@ describe("native macOS provider discovery and inspection", () => { expect(captured.ok && captured.value.stdout).toBe(output); }); - it("cancels and reaps a running native command without an operation timeout", async () => { + it("retains both output streams and exit status when a command fails", async () => { const runner = new XcrunCommandRunner(() => Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), ); - const controller = new AbortController(); - const pending = runner.run("file", ["-e", "setTimeout(() => {}, 10000)"], { - signal: controller.signal, + const result = await runner.run( + "file", + [ + "-e", + 'process.stdout.write("partial stdout"); process.stderr.write("partial stderr"); process.exitCode = 7', + ], + {}, + ); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + reason: "nonzero-exit", + exitCode: 7, + capture: { + stdout: "partial stdout", + stderr: "partial stderr", + stdoutBytes: 14, + stderrBytes: 14, + exitCode: 7, + signal: null, + truncated: false, + }, }); - setTimeout(() => controller.abort(), 30); + }); - const result = await pending; + it("retains accepted nonzero output as a complete command capture", async () => { + const runner = new XcrunCommandRunner(() => + Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), + ); + const result = await runner.run( + "file", + [ + "-e", + 'process.stdout.write("unsigned observation"); process.exitCode = 1', + ], + { acceptNonZero: true }, + ); expect(result).toMatchObject({ - ok: false, - error: { reason: "cancelled" }, + ok: true, + value: { + stdout: "unsigned observation", + stdoutBytes: 20, + exitCode: 1, + signal: null, + }, }); }); }); +describe("native command failure capture and limits", () => { + it("projects captured diagnostics from a failed native provider command", async () => { + const runner = new XcrunCommandRunner(() => + Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), + ); + const result = await new NativeMacOSProvider(runner, "darwin") + .createClient(machoTarget("/private/fixture")) + .execute("demangle_swift", { symbols: ["fixture-symbol"] }); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe("ProviderAdapterError"); + expect(result.error.capturedOutput?.stderr).toContain("Cannot find module"); + expect(result.error.capturedOutput?.stderr_bytes).toBeGreaterThan(0); + expect(result.error.capturedOutput?.exit_code).not.toBe(0); + expect(projectAnalysisError(result.error).details).toMatchObject({ + captured_output: { + stderr: expect.stringContaining("Cannot find module"), + stderr_bytes: expect.any(Number), + exit_code: expect.any(Number), + }, + diagnostics: { + reason: "nonzero-exit", + stderr_bytes: expect.any(Number), + }, + }); + }); + + it("retains output written before cancellation and reaps the stalled command", async () => { + const runner = new XcrunCommandRunner(() => + Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), + ); + const controller = new AbortController(); + const pending = runner.run( + "file", + [ + "-e", + 'process.stdout.write("started"); process.stderr.write("warning"); setInterval(() => {}, 1000)', + ], + { signal: controller.signal }, + ); + setTimeout(() => controller.abort(), 100); + + const result = await pending; + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + reason: "cancelled", + capture: { + stdout: "started", + stderr: "warning", + stdoutBytes: 7, + stderrBytes: 7, + truncated: false, + }, + }); + }); + + it("rejects output beyond the complete native-command budget", async () => { + const runner = new XcrunCommandRunner(() => + Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), + ); + const megabytesOverBudget = + Math.ceil(NATIVE_COMMAND_OUTPUT_BUDGET_BYTES / (1024 * 1024)) + 1; + const result = await runner.run( + "file", + [ + "-e", + `async function main() { const chunk = "x".repeat(1024 * 1024); for (let i = 0; i < ${String(megabytesOverBudget)}; i += 1) await new Promise((resolve) => process.stdout.write(chunk, resolve)); } void main()`, + ], + {}, + ); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.reason).toBe("output-limit"); + expect(result.error.capture?.truncated).toBe(true); + expect( + (result.error.capture?.stdoutBytes ?? 0) + + (result.error.capture?.stderrBytes ?? 0), + ).toBeLessThanOrEqual(NATIVE_COMMAND_OUTPUT_BUDGET_BYTES); + }); +}); + +describe("native command failure projection", () => { + it("projects native cancellation output and incomplete cleanup to callers", async () => { + const capture = { + stdout: "partial output", + stderr: "partial diagnostic", + stdoutBytes: 14, + stderrBytes: 19, + exitCode: null, + signal: "SIGKILL", + truncated: false, + }; + const runner: NativeCommandRunner = { + async run(tool) { + return err( + new NativeCommandFailure( + tool, + "cancelled", + null, + undefined, + capture, + "process group cleanup was not confirmed", + ["process-group:1234"], + ), + ); + }, + }; + const result = await new NativeMacOSProvider(runner, "darwin") + .createClient(machoTarget("/private/fixture")) + .execute("list_architectures", {}); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.capturedOutput).toMatchObject({ + stdout: "partial output", + stderr: "partial diagnostic", + stdout_bytes: 14, + stderr_bytes: 19, + signal: "SIGKILL", + truncated: false, + }); + expect(result.error.cleanup).toEqual({ + reason: "process group cleanup was not confirmed", + resources: ["process-group:1234"], + }); + expect(projectAnalysisError(result.error).details).toMatchObject({ + captured_output: { + stdout_bytes: 14, + stderr_bytes: 19, + signal: "SIGKILL", + }, + cleanup: "incomplete", + resources: ["process-group:1234"], + }); + }); + + it.each([ + ["timeout", "AnalysisTimeoutError", "provider_timeout"], + ["output-limit", "AnalysisResourceConstraintError", "resource_constraint"], + ] as const)( + "projects native command %s as a typed resource failure", + async (reason, tag, code) => { + const runner: NativeCommandRunner = { + async run(tool) { + return err(new NativeCommandFailure(tool, reason)); + }, + }; + const result = await new NativeMacOSProvider(runner, "darwin") + .createClient(machoTarget("/private/fixture")) + .execute("list_architectures", {}); + + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error._tag).toBe(tag); + expect(projectAnalysisError(result.error).code).toBe(code); + }, + ); +}); + describe("native macOS provider inspection", () => { it("normalizes comprehensive Mach-O inspection with exact bounded provenance", async () => { const runner = new FixtureRunner(); From dbcf8c33915d924fdbc881bf2cf9676587f3b96f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:15 +0800 Subject: [PATCH 029/791] fix(apple): budget Interface Builder decoding and projection --- docs/native-investigation.md | 11 +- .../apple/InterfaceBuilderAnalysis.ts | 233 +++++++++++++--- .../apple/InterfaceBuilderDecodeBudget.ts | 250 ++++++++++++++++++ src/artifacts/apple/NibArchive.test.ts | 47 ++++ src/artifacts/apple/NibArchive.ts | 166 +++++++++++- src/artifacts/apple/NibViewHierarchy.ts | 15 +- .../filesystem/interfaceBuilderBundle.test.ts | 188 +++++++++++++ 7 files changed, 861 insertions(+), 49 deletions(-) create mode 100644 src/artifacts/apple/InterfaceBuilderDecodeBudget.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index e385ef9e568..e9b0f1b0f6a 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -48,7 +48,16 @@ complete observation. BOM and rejects malformed byte sequences, unsupported encoding declarations, and declarations that disagree with the detected encoding. It never substitutes UTF-8 for a declared encoding it cannot process. Archive and evidence digests - still identify the original serialized bytes. + still identify the original serialized bytes. Interface Builder decoding + limits aggregate input archives to 32 MiB. + Before decoding, it also bounds the serialized structure: expanded binary + plist references, NIB table records, XML elements and text, plus their + projected representations must fit the 256 MiB aggregate decode budget. + This is a format-derived representation estimate, not a process-wide heap + ceiling. + Archives beyond either aggregate budget are omitted with their paths and an + explicitly partial `archive_decode` facet; previously decoded documents and + their identities remain available. - `inspect_native_dispatch_metadata` / `rea inspect-native-dispatch-metadata ` prefers a validated macOS Mach-O byte reader. It decodes 64-bit little-endian Objective-C class/metaclass records, superclass pointers, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index a61da0f718a..00b19fe15ed 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -8,6 +8,11 @@ import { parseXmlPropertyList, } from "../../domain/propertyListKeys.js"; import { decodeNibArchive, type NibArchiveDocument } from "./NibArchive.js"; +import { + estimatePropertyListDecodeBytes, + InterfaceBuilderDecodeBudget, + InterfaceBuilderDecodeBudgetExceeded, +} from "./InterfaceBuilderDecodeBudget.js"; import type { ArtifactEntry } from "../ArtifactReader.js"; import { buildInterfaceBuilderAnalysis, @@ -23,7 +28,9 @@ import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; import { decodeXmlPlistText } from "./XmlPropertyListText.js"; -const MAX_DOCUMENT_BYTES = 64 * 1024 * 1024; +const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024; +const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024; +const MAX_REPORTED_OMITTED_ARCHIVES = 64; /** Decode compiled Interface Builder archives from a local app bundle. */ export const analyzeInterfaceBuilderBundle = async (input: { @@ -36,28 +43,84 @@ export const analyzeInterfaceBuilderBundle = async (input: { const reader = new DirectoryArtifactReader(input.bundlePath); const documents: InterfaceBuilderDocumentInput[] = []; const invalid: string[] = []; + const omittedArchives: Array<{ path: string; reason: string }> = []; + let unreportedOmittedArchives = 0; const incompleteHierarchies = new Map(); const prototypeKeyOmissions = new Map(); let omitted = 0; let attempted = 0; + let candidates = 0; + let aggregateInputBytes = 0; + const decodeBudget = new InterfaceBuilderDecodeBudget( + MAX_AGGREGATE_DECODE_BYTES, + ); + const recordOmittedArchive = (path: string, reason: string): void => { + if (omittedArchives.length < MAX_REPORTED_OMITTED_ARCHIVES) + omittedArchives.push({ path, reason }); + else unreportedOmittedArchives += 1; + }; try { for await (const entry of reader.entries(input.signal)) { if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path)) continue; - if (attempted >= limits.max_documents) { + candidates += 1; + if (candidates > limits.max_documents) { omitted += 1; + recordOmittedArchive(entry.path, "max_documents"); continue; } - attempted += 1; try { - const bytes = await readEntry(reader, entry, input.signal); - const nib = - bytes.subarray(0, 10).toString("ascii") === "NIBArchive" - ? projectNibArchive(decodeNibArchive(bytes)) - : null; + const remainingInputBytes = + MAX_AGGREGATE_INPUT_BYTES - aggregateInputBytes; + if ( + entry.declaredSize !== null && + entry.declaredSize > remainingInputBytes + ) { + omitted += 1; + recordOmittedArchive(entry.path, "aggregate_decode_budget_exhausted"); + continue; + } + attempted += 1; + const bytes = await readEntry( + reader, + entry, + input.signal, + remainingInputBytes, + ); + aggregateInputBytes += bytes.length; + const isNibArchive = + bytes.subarray(0, 10).toString("ascii") === "NIBArchive"; + let nib: ReturnType | null = null; + let xmlText: string | undefined; + if (isNibArchive) { + const documentBudget = new InterfaceBuilderDecodeBudget( + decodeBudget.remainingBytes, + ); + nib = projectNibArchive( + decodeNibArchive(bytes, documentBudget), + documentBudget, + limits.max_objects, + ); + decodeBudget.reserve( + documentBudget.usedBytes, + "NIBArchive representation exceeds the aggregate Interface Builder decode budget", + ); + } else { + if (bytes.subarray(0, 8).toString("ascii") !== "bplist00") + xmlText = decodeXmlPlistText(bytes); + const decodedReservation = estimatePropertyListDecodeBytes( + bytes, + decodeBudget.remainingBytes, + xmlText, + ); + decodeBudget.reserve( + decodedReservation, + "plist representation exceeds the aggregate Interface Builder decode budget", + ); + } const { value: raw, omittedPrototypeKeys } = nib === null - ? decodePlist(bytes) + ? decodePlist(bytes, xmlText) : { value: nib.raw, omittedPrototypeKeys: 0 }; if (nib !== null && nib.omitted > 0) incompleteHierarchies.set(entry.path, nib.omitted); @@ -74,6 +137,11 @@ export const analyzeInterfaceBuilderBundle = async (input: { }); } catch (cause: unknown) { if (input.signal?.aborted === true) throw cause; + if (cause instanceof InterfaceBuilderDecodeBudgetExceeded) { + omitted += 1; + recordOmittedArchive(entry.path, cause.reason); + continue; + } invalid.push( `${entry.path}: ${cause instanceof Error ? cause.message : "archive decode failed"}`, ); @@ -92,6 +160,8 @@ export const analyzeInterfaceBuilderBundle = async (input: { incompleteHierarchies, prototypeKeyOmissions, invalid, + omittedArchives, + unreportedOmittedArchives, attempted, omitted, }); @@ -103,13 +173,23 @@ const finalizeHierarchyCoverage = ( incompleteHierarchies: ReadonlyMap; prototypeKeyOmissions: ReadonlyMap; invalid: readonly string[]; + omittedArchives: readonly { path: string; reason: string }[]; + unreportedOmittedArchives: number; attempted: number; omitted: number; }, ) => { - const { incompleteHierarchies, prototypeKeyOmissions, invalid } = counts; + const { + incompleteHierarchies, + prototypeKeyOmissions, + invalid, + omittedArchives, + unreportedOmittedArchives, + } = counts; const archiveDecodePartial = - invalid.length > 0 || prototypeKeyOmissions.size > 0; + invalid.length > 0 || + prototypeKeyOmissions.size > 0 || + omittedArchives.length > 0; return { ...result, documents: result.documents.map((document) => ({ @@ -139,12 +219,19 @@ const finalizeHierarchyCoverage = ( status: archiveDecodePartial ? ("partial" as const) : ("complete" as const), - reason: - invalid.length > 0 - ? "one_or_more_archives_invalid" - : prototypeKeyOmissions.size > 0 - ? "dictionary_entries_omitted" - : null, + reason: omittedArchives.some( + ({ reason }) => reason === "aggregate_decode_budget_exhausted", + ) + ? "aggregate_decode_budget_exhausted" + : omittedArchives.some(({ reason }) => reason === "max_documents") + ? "document_limit_reached" + : omittedArchives.length > 0 + ? "archive_decode_budget_exhausted" + : invalid.length > 0 + ? "one_or_more_archives_invalid" + : prototypeKeyOmissions.size > 0 + ? "dictionary_entries_omitted" + : null, examined: counts.attempted, omitted: counts.omitted, }, @@ -171,13 +258,70 @@ const finalizeHierarchyCoverage = ( ...[...prototypeKeyOmissions].map( ([path, count]) => `${path}: ${omittedPrototypeKeysLimitation(count)}`, ), + ...omittedArchives.map( + ({ path, reason }) => + `${path}: omitted because ${reason.replaceAll("_", " ")}.`, + ), + ...(unreportedOmittedArchives === 0 + ? [] + : [ + `${unreportedOmittedArchives} additional omitted archive path(s) and their reasons were not retained; reporting is bounded to ${MAX_REPORTED_OMITTED_ARCHIVES} paths.`, + ]), ], }; }; /** Project decoded NIB records into the same bounded graph input as ibtool. */ -const projectNibArchive = (archive: NibArchiveDocument) => { +const projectNibArchive = ( + archive: NibArchiveDocument, + budget: InterfaceBuilderDecodeBudget, + maxHierarchyNodes: number, +) => { const byId = new Map(archive.objects.map((object) => [object.id, object])); + const decodedStrings = new Map(); + const decodedString = ( + objectId: number, + object: NibArchiveDocument["objects"][number], + ): string | null => { + if (decodedStrings.has(objectId)) + return decodedStrings.get(objectId) ?? null; + const data = object.values["NS.bytes"]; + if (typeof data !== "object" || data === null || Array.isArray(data)) { + decodedStrings.set(objectId, null); + return null; + } + const encoded = data.$nib_data_base64; + if (typeof encoded !== "string") { + decodedStrings.set(objectId, null); + return null; + } + budget.reserve( + 64 + encoded.length * 2, + "NIBArchive string projection exceeds the aggregate Interface Builder decode budget", + ); + const value = Buffer.from(encoded, "base64").toString("utf8"); + decodedStrings.set(objectId, value); + return value; + }; + const reserveStringOccurrence = (value: string | null): string | null => { + if (value !== null) + budget.reserve( + 64 + value.length * 6, + "NIBArchive repeated string projection exceeds the aggregate Interface Builder decode budget", + ); + return value; + }; + let projectedFields = 0; + for (const object of archive.objects) + projectedFields += Object.keys(object.values).length; + budget.reserve( + archive.objects.length * 128 + projectedFields * 128, + "NIBArchive graph projection exceeds the aggregate Interface Builder decode budget", + ); + budget.reserve( + archive.classes.length * 96, + "NIBArchive class projection exceeds the aggregate Interface Builder decode budget", + ); const dereference = (value: JsonValue | undefined): JsonValue | undefined => { if (typeof value !== "object" || value === null || Array.isArray(value)) return value; @@ -186,12 +330,8 @@ const projectNibArchive = (archive: NibArchiveDocument) => { const target = byId.get(ref); if (target === undefined) return undefined; if (target.class_name.replace(/\0+$/u, "") === "NSString") { - const data = target.values["NS.bytes"]; - if (typeof data === "object" && data !== null && !Array.isArray(data)) { - const encoded = data.$nib_data_base64; - if (typeof encoded === "string") - return Buffer.from(encoded, "base64").toString("utf8"); - } + const string = decodedString(ref, target); + if (string !== null) return reserveStringOccurrence(string); } return { objectID: String(ref) }; }; @@ -201,14 +341,8 @@ const projectNibArchive = (archive: NibArchiveDocument) => { const ref = value.$nib_object_ref; if (typeof ref !== "number") return null; const target = byId.get(ref); - const data = target?.values["NS.bytes"]; if (target?.class_name.replace(/\0+$/u, "") !== "NSString") return null; - if (typeof data !== "object" || data === null || Array.isArray(data)) - return null; - const encoded = data.$nib_data_base64; - return typeof encoded === "string" - ? Buffer.from(encoded, "base64").toString("utf8") - : null; + return reserveStringOccurrence(decodedString(ref, target)); }; const runtimeClass = (objectId: number): string | null => { const object = byId.get(objectId); @@ -300,6 +434,7 @@ const projectNibArchive = (archive: NibArchiveDocument) => { }); } let hierarchyOmitted = 0; + let hierarchyNodeCount = 0; const hierarchyFor = ( objectId: number, seen: Set, @@ -309,11 +444,20 @@ const projectNibArchive = (archive: NibArchiveDocument) => { hierarchyOmitted += 1; return null; } + if (hierarchyNodeCount >= maxHierarchyNodes) { + hierarchyOmitted += 1; + return null; + } + if (!budget.tryReserve(128)) { + hierarchyOmitted += 1; + return null; + } const object = byId.get(objectId); if (object === undefined) { hierarchyOmitted += 1; return null; } + hierarchyNodeCount += 1; const nextSeen = new Set(seen).add(objectId); const children: NibHierarchyNode[] = []; for (const [key, value] of Object.entries(object.values)) { @@ -352,8 +496,15 @@ const projectNibArchive = (archive: NibArchiveDocument) => { const viewHierarchy = projectNibViewHierarchy( archive.objects, new Set(Object.keys(objects).map(Number)), + () => budget.tryReserve(128), + maxHierarchyNodes, + ); + const merged = mergeNibHierarchies( + hierarchy, + viewHierarchy.hierarchy ?? [], + () => budget.tryReserve(96), + maxHierarchyNodes, ); - const merged = mergeNibHierarchies(hierarchy, viewHierarchy.hierarchy ?? []); return { omitted: viewHierarchy.omitted + hierarchyOmitted + merged.omitted, raw: jsonValueSchema.parse({ @@ -376,10 +527,12 @@ const readEntry = async ( reader: DirectoryArtifactReader, entry: ArtifactEntry, signal?: AbortSignal, + maxInputBytes = MAX_AGGREGATE_INPUT_BYTES, ): Promise => { - if (entry.declaredSize !== null && entry.declaredSize > MAX_DOCUMENT_BYTES) - throw new RangeError( - "archive exceeds the 64 MiB per-document decode limit", + if (entry.declaredSize !== null && entry.declaredSize > maxInputBytes) + throw new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + "archive exceeds the aggregate Interface Builder decode budget", ); const stream = await reader.open(entry, signal); const chunks: Buffer[] = []; @@ -389,10 +542,11 @@ const readEntry = async ( ? chunk : Buffer.from(chunk as Uint8Array); length += bytes.length; - if (length > MAX_DOCUMENT_BYTES) { + if (length > maxInputBytes) { stream.destroy(); - throw new RangeError( - "archive exceeded the 64 MiB per-document decode limit", + throw new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + "archive exceeded the aggregate Interface Builder decode budget", ); } chunks.push(bytes); @@ -403,10 +557,11 @@ const readEntry = async ( /** Project decoded plist data and dates while retaining omitted-key coverage. */ const decodePlist = ( bytes: Buffer, + xmlText?: string, ): { readonly value: JsonValue; readonly omittedPrototypeKeys: number } => { const { value, omittedPrototypeKeys } = bytes.subarray(0, 8).toString("ascii") === "bplist00" ? { value: parseBinary(bytes), omittedPrototypeKeys: 0 } - : parseXmlPropertyList(decodeXmlPlistText(bytes)); + : parseXmlPropertyList(xmlText ?? decodeXmlPlistText(bytes)); return { value: projectPlistValue(value).value, omittedPrototypeKeys }; }; diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts new file mode 100644 index 00000000000..6ec1c82f15a --- /dev/null +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -0,0 +1,250 @@ +import { decodeXmlPlistText } from "./XmlPropertyListText.js"; + +/** Reason reported when an archive would exceed the shared decoding budget. */ +export type InterfaceBuilderBudgetReason = "aggregate_decode_budget_exhausted"; + +/** A named resource-limit failure that keeps budget exhaustion out of decode errors. */ +export class InterfaceBuilderDecodeBudgetExceeded extends Error { + constructor( + readonly reason: InterfaceBuilderBudgetReason, + message: string, + ) { + super(message); + this.name = "InterfaceBuilderDecodeBudgetExceeded"; + } +} + +/** Shared byte budget for decoder records and projected archive data. */ +export class InterfaceBuilderDecodeBudget { + readonly #initialBytes: number; + #remainingBytes: number; + + constructor(maxBytes: number) { + this.#initialBytes = maxBytes; + this.#remainingBytes = maxBytes; + } + + /** Representation bytes still available for the current inspection. */ + get remainingBytes(): number { + return this.#remainingBytes; + } + + /** Representation bytes reserved so far. */ + get usedBytes(): number { + return this.#initialBytes - this.#remainingBytes; + } + + /** Reserve representation bytes before allocation, or report exhaustion. */ + reserve(bytes: number, message: string): void { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + bytes > this.#remainingBytes + ) + throw new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + message, + ); + this.#remainingBytes -= bytes; + } + + /** Reserve bytes for an optional projection without throwing on exhaustion. */ + tryReserve(bytes: number): boolean { + if ( + !Number.isSafeInteger(bytes) || + bytes < 0 || + bytes > this.#remainingBytes + ) + return false; + this.#remainingBytes -= bytes; + return true; + } +} + +/** Preflight the representation created by the pinned plist decoder. */ +export const estimatePropertyListDecodeBytes = ( + bytes: Buffer, + maxBytes: number, + xmlText?: string, +): number => { + const estimate = + bytes.subarray(0, 8).toString("ascii") === "bplist00" + ? estimateBinaryPlistExpansion(bytes, maxBytes) + : estimateXmlPlistExpansion(bytes, maxBytes, xmlText); + if (estimate > maxBytes) + throw new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + "plist representation exceeds the aggregate Interface Builder decode budget", + ); + return estimate; +}; + +const estimateXmlPlistExpansion = ( + bytes: Buffer, + maxBytes: number, + xmlText?: string, +): number => { + const xml = xmlText ?? decodeXmlPlistText(bytes); + if (/]*\[/iu.test(xml)) + throw new InterfaceBuilderDecodeBudgetExceeded( + "aggregate_decode_budget_exhausted", + "XML plist internal entity expansion exceeds the aggregate Interface Builder decode budget", + ); + const elementPattern = + /||]*>|<\?[^]*?\?>|<\/?[A-Za-z][^>]*>/giu; + const decodedSourceBytes = xml.length * 2; + let elementCount = 0; + let xmlDepth = 0; + for (const match of xml.matchAll(elementPattern)) { + const token = match[0] ?? ""; + if ( + token.startsWith("`.repeat(11 * 1024); + const valid = (name: string, body = "") => + `document${name}${body}`; + // XML data expands to a Uint8Array and then to a base64 JSON projection; + // this first valid document exercises expansion beyond the source bytes. + const expandedData = Buffer.alloc(1024 * 1024, 7).toString("base64"); + await writeFile( + join(resources, "A.nib"), + valid("A", `payload${expandedData}`), + ); + await writeFile(join(resources, "B.nib"), valid("B", padding)); + await writeFile(join(resources, "C.nib"), valid("C", padding)); + await writeFile(join(resources, "D.nib"), valid("D", padding)); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "e".repeat(64), + }); + + expect(result.documents.map(({ relative_path }) => relative_path)).toEqual([ + "Contents/Resources/A.nib", + "Contents/Resources/B.nib", + "Contents/Resources/C.nib", + ]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "aggregate_decode_budget_exhausted", + examined: 3, + omitted: 1, + }), + ); + expect(result.limitations).toContain( + "Contents/Resources/D.nib: omitted because aggregate decode budget exhausted.", + ); + }); +}); + describe("bounded Interface Builder archive decoding", () => { it("counts malformed archives against the document limit", async () => { const root = await createTestTempDirectory("rea-ib-test-"); From 73b09f4c5c511ce80baa0c89717181bbe0ecba8f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:16 +0800 Subject: [PATCH 030/791] fix(managed): bind inspection evidence to artifact identity --- docs/managed-code-analysis.md | 6 ++ src/domain/managed/managedApplicationGraph.ts | 25 +++---- .../managed/managedInspectionEvidence.ts | 26 +++++++ .../managed/managedMemberComparisonMatch.ts | 14 ++-- .../managed/managedNativeVerification.ts | 26 +++---- .../managed/managedMemberComparison.test.ts | 70 ++++++++++++++++++- .../managedApplicationGraph.test.ts | 42 +++++++++++ .../managed/managedNativeVerification.test.ts | 48 +++++++++++++ 8 files changed, 222 insertions(+), 35 deletions(-) create mode 100644 src/domain/managed/managedInspectionEvidence.ts diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index b0a0df4c047..92bc2306095 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -137,6 +137,12 @@ function implements a specific managed method. ## Evidence record shape +Member comparison, application graph projection, and managed/native verification +authenticate supplied inspection Evidence and check that any subject SHA-256 +matches the normalized artifact SHA-256. A valid Evidence ID alone does not +establish that those two identities agree. Evidence without a subject remains +usable, with no subject digest available to cross-check. + Every planned operation returns a provider result and Evidence with four commitment groups: diff --git a/src/domain/managed/managedApplicationGraph.ts b/src/domain/managed/managedApplicationGraph.ts index 8158e205cf7..9074bc40cbe 100644 --- a/src/domain/managed/managedApplicationGraph.ts +++ b/src/domain/managed/managedApplicationGraph.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import canonicalize from "canonicalize"; import { z } from "zod"; -import { evidenceSchema, parseEvidence, type Evidence } from "../evidence.js"; +import { evidenceSchema, type Evidence } from "../evidence.js"; import { createJavaScriptApplicationGraph, javascriptApplicationGraphSchema, @@ -32,6 +32,7 @@ import { } from "./managedApplicationGraphNodes.js"; import { digestSchema } from "../digests.js"; import { prefixedDigestSchema } from "../digests.js"; +import { parseManagedInspectionEvidence } from "./managedInspectionEvidence.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const textSchema = z.string().min(1); @@ -194,7 +195,7 @@ const parseManagedInputs = ( artifact: input.managed_artifact === undefined ? null - : parseManagedEvidence( + : parseManagedInspectionEvidence( input.managed_artifact, "inspect_managed_artifact", managedArtifactInspectionSchema, @@ -202,7 +203,7 @@ const parseManagedInputs = ( members: input.managed_members === undefined ? null - : parseManagedEvidence( + : parseManagedInspectionEvidence( input.managed_members, "inspect_managed_members", managedMemberInspectionSchema, @@ -210,24 +211,13 @@ const parseManagedInputs = ( boundaries: input.managed_native_boundaries === undefined ? null - : parseManagedEvidence( + : parseManagedInspectionEvidence( input.managed_native_boundaries, "inspect_managed_native_boundaries", managedNativeBoundaryInspectionSchema, ), }); -const parseManagedEvidence = ( - rawEvidence: unknown, - operation: string, - schema: z.ZodType, -): { readonly evidence: Evidence; readonly result: Result } => { - const evidence = parseEvidence(rawEvidence); - if (evidence.operation !== operation) - throw new TypeError(`Evidence operation is not ${operation}`); - return { evidence, result: schema.parse(evidence.normalized_result) }; -}; - const chooseArtifact = ( parsed: ParsedManagedGraphInput, ): { readonly sha256: string; readonly path: string } => { @@ -272,6 +262,11 @@ const projectionLimitations = ( "Managed member Evidence was not supplied; type, method, and field nodes are absent.", ] : []), + ...(parsed.artifact === null + ? [ + "Managed artifact Evidence was not supplied; assembly identity observations are absent.", + ] + : []), ...(parsed.boundaries === null ? [ "Managed native-boundary Evidence was not supplied; P/Invoke and native-implementation nodes are absent.", diff --git a/src/domain/managed/managedInspectionEvidence.ts b/src/domain/managed/managedInspectionEvidence.ts new file mode 100644 index 00000000000..3bd35ea7b71 --- /dev/null +++ b/src/domain/managed/managedInspectionEvidence.ts @@ -0,0 +1,26 @@ +import { parseEvidence, type Evidence } from "../evidence.js"; +import type { z } from "zod"; + +/** + * Parse managed inspection Evidence and bind its artifact to any available subject. + * A null subject remains valid and means there is no digest to cross-check. + */ +export const parseManagedInspectionEvidence = < + Result extends { readonly artifact: { readonly sha256: string } }, +>( + rawEvidence: unknown, + operation: string, + schema: z.ZodType, +): { readonly evidence: Evidence; readonly result: Result } => { + const evidence = parseEvidence(rawEvidence); + if (evidence.operation !== operation) + throw new TypeError(`Evidence operation is not ${operation}`); + const result = schema.parse(evidence.normalized_result); + const artifactSha256 = result.artifact.sha256; + const subjectSha256 = evidence.subject?.digest.sha256; + if (subjectSha256 !== undefined && artifactSha256 !== subjectSha256) + throw new TypeError( + `Managed Evidence ${operation} (${evidence.evidence_id}) subject SHA-256 ${subjectSha256} does not match normalized artifact SHA-256 ${artifactSha256}`, + ); + return { evidence, result }; +}; diff --git a/src/domain/managed/managedMemberComparisonMatch.ts b/src/domain/managed/managedMemberComparisonMatch.ts index c9fd3aa43f3..229263c4f1e 100644 --- a/src/domain/managed/managedMemberComparisonMatch.ts +++ b/src/domain/managed/managedMemberComparisonMatch.ts @@ -2,11 +2,11 @@ import { createHash } from "node:crypto"; import canonicalize from "canonicalize"; -import { parseEvidence } from "../evidence.js"; import { managedMemberInspectionSchema, type ManagedMemberInspection, } from "./managedArtifact.js"; +import { parseManagedInspectionEvidence } from "./managedInspectionEvidence.js"; import type { ManagedMemberComparisonResult, ManagedMemberComparisonSide, @@ -275,11 +275,13 @@ export const parseManagedMemberEvidence = ( readonly evidenceId: string; readonly result: ManagedMemberInspection; } => { - const parsed = parseEvidence(evidence); - if (parsed.operation !== "inspect_managed_members") - throw new TypeError("Evidence operation is not inspect_managed_members"); + const parsed = parseManagedInspectionEvidence( + evidence, + "inspect_managed_members", + managedMemberInspectionSchema, + ); return { - evidenceId: parsed.evidence_id, - result: managedMemberInspectionSchema.parse(parsed.normalized_result), + evidenceId: parsed.evidence.evidence_id, + result: parsed.result, }; }; diff --git a/src/domain/managed/managedNativeVerification.ts b/src/domain/managed/managedNativeVerification.ts index 2263fd41201..0ba3cdef13b 100644 --- a/src/domain/managed/managedNativeVerification.ts +++ b/src/domain/managed/managedNativeVerification.ts @@ -2,11 +2,12 @@ import { createHash } from "node:crypto"; import canonicalize from "canonicalize"; -import { parseEvidence } from "../evidence.js"; +import type { Evidence } from "../evidence.js"; import { managedNativeBoundaryInspectionSchema, type ManagedNativeBoundaryInspection, } from "./managedArtifact.js"; +import { parseManagedInspectionEvidence } from "./managedInspectionEvidence.js"; import type { JsonValue } from "../jsonValue.js"; import { managedNativeVerificationResultSchema, @@ -33,25 +34,24 @@ const sha256 = (value: JsonValue): string => { return createHash("sha256").update(serialized).digest("hex"); }; -const managedBoundaryFromInput = ( +const parseManagedBoundary = ( input: ManagedNativeVerificationInput, -): ManagedNativeBoundaryInspection => { - const managedEvidence = parseEvidence(input.managed_boundaries); - if (managedEvidence.operation !== "inspect_managed_native_boundaries") - throw new TypeError( - "Evidence operation is not inspect_managed_native_boundaries", - ); - return managedNativeBoundaryInspectionSchema.parse( - managedEvidence.normalized_result, +): { + readonly evidence: Evidence; + readonly result: ManagedNativeBoundaryInspection; +} => + parseManagedInspectionEvidence( + input.managed_boundaries, + "inspect_managed_native_boundaries", + managedNativeBoundaryInspectionSchema, ); -}; /** Verify managed P/Invoke declarations against authenticated native Evidence. */ export const verifyManagedNativeBoundaries = ( input: ManagedNativeVerificationInput, ): ManagedNativeVerificationResult => { - const managed = managedBoundaryFromInput(input); - const managedEvidence = parseEvidence(input.managed_boundaries); + const { evidence: managedEvidence, result: managed } = + parseManagedBoundary(input); const native = collectNativeSymbols(input.native_observations); const verifiedPinvokes = managed.pinvoke_imports.map((item) => verifyPinvoke({ diff --git a/tests/boundary/providers/managed/managedMemberComparison.test.ts b/tests/boundary/providers/managed/managedMemberComparison.test.ts index 2ba2eb13f0d..4caf2b22ef1 100644 --- a/tests/boundary/providers/managed/managedMemberComparison.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparison.test.ts @@ -10,7 +10,10 @@ import { compareManagedMemberPaths, } from "../../../../src/application/managed/ManagedMemberComparisonService.js"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; -import { managedMemberComparisonResultSchema } from "../../../../src/domain/managed/managedMemberComparison.js"; +import { + managedMemberComparisonResultSchema, + parseManagedMemberEvidence, +} from "../../../../src/domain/managed/managedMemberComparison.js"; import { createEvidence } from "../../../../src/domain/evidence.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; import { jsonValueSchema } from "../../../../src/domain/jsonValue.js"; @@ -54,6 +57,71 @@ it.each([ }, ); +describe("managed member comparison subject identity", () => { + it("rejects authenticated member Evidence whose subject digest differs from its artifact", () => { + const bytes = buildManagedPeFixture(); + const target = managedPeFixtureTarget(bytes, "/fixture/mismatch.dll"); + const inspection = inspectManagedMembersBytes(bytes, target); + const otherDigest = "b".repeat(64); + const mismatched = createEvidence( + target, + { + id: "partial-fixture", + name: "Partial fixture", + version: "1", + }, + { + operation: "inspect_managed_members", + parameters: {}, + result: jsonValueSchema.parse({ + ...inspection, + artifact: { ...inspection.artifact, sha256: otherDigest }, + identity_scope: { + ...inspection.identity_scope, + requires_artifact_sha256: otherDigest, + }, + }), + }, + ); + const aligned = createEvidence( + target, + { + id: "partial-fixture", + name: "Partial fixture", + version: "1", + }, + { + operation: "inspect_managed_members", + parameters: {}, + result: inspection, + }, + ); + const unavailableSubject = createEvidence( + undefined, + { + id: "partial-fixture", + name: "Partial fixture", + version: "1", + }, + { + operation: "inspect_managed_members", + parameters: {}, + result: inspection, + }, + ); + + expect(() => parseManagedMemberEvidence(mismatched)).toThrow( + `Managed Evidence inspect_managed_members (${mismatched.evidence_id}) subject SHA-256 ${target.sha256} does not match normalized artifact SHA-256 ${otherDigest}`, + ); + expect(parseManagedMemberEvidence(aligned).result.artifact.sha256).toBe( + target.sha256, + ); + expect( + parseManagedMemberEvidence(unavailableSubject).result.artifact.sha256, + ).toBe(target.sha256); + }); +}); + describe("managed member comparison path workflow", () => { it("preserves undecoded signatures as unknown through content-addressed partial Evidence", () => { const observe = (signature: number, path: string) => { diff --git a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts index 1bc93d8a112..39d31b07bfd 100644 --- a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts +++ b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts @@ -167,6 +167,48 @@ describe("managed application graph request", () => { }); describe("managed application graph coverage", () => { + it("rejects managed Evidence digest mismatch and discloses missing artifact observations", () => { + const bytes = buildManagedPeFixture(); + const binary = managedPeFixtureTarget(bytes, "/fixture/Mismatch.dll"); + const members = inspectManagedMembersBytes(bytes, binary); + const mismatched = createEvidence(binary, MANAGED_STATIC_PROVIDER, { + operation: "inspect_managed_members", + parameters: {}, + result: { + ...members, + artifact: { ...members.artifact, sha256: "b".repeat(64) }, + identity_scope: { + ...members.identity_scope, + requires_artifact_sha256: "b".repeat(64), + }, + }, + }); + const rejected = projectManagedApplicationGraphEvidence({ + managed_members: mismatched, + }); + expect(rejected.ok).toBe(false); + if (rejected.ok) throw new Error("mismatched managed Evidence accepted"); + expect(JSON.stringify(rejected.error)).toContain( + `Managed Evidence inspect_managed_members (${mismatched.evidence_id}) subject SHA-256 ${binary.sha256} does not match normalized artifact SHA-256 ${"b".repeat(64)}`, + ); + + const aligned = createEvidence(binary, MANAGED_STATIC_PROVIDER, { + operation: "inspect_managed_members", + parameters: {}, + result: members, + }); + const projected = projectManagedApplicationGraphEvidence({ + managed_members: aligned, + }); + if (!projected.ok) throw projected.error; + const result = managedApplicationGraphResultSchema.parse( + parseEvidence(projected.value).normalized_result, + ); + expect(result.limitations).toContain( + "Managed artifact Evidence was not supplied; assembly identity observations are absent.", + ); + }); + it("preserves partial parser coverage in graph and per-fact coverage", () => { const bytes = buildManagedPeFixture(); const binary = managedPeFixtureTarget(bytes, "/fixture/ManagedInterop.exe"); diff --git a/tests/conformance/providers/managed/managedNativeVerification.test.ts b/tests/conformance/providers/managed/managedNativeVerification.test.ts index 4932ae11033..626705719c1 100644 --- a/tests/conformance/providers/managed/managedNativeVerification.test.ts +++ b/tests/conformance/providers/managed/managedNativeVerification.test.ts @@ -60,6 +60,54 @@ const nativeEvidenceWithExports = (names: readonly string[]) => { ); }; +describe("managed/native boundary Evidence identity", () => { + it("rejects managed boundary Evidence whose subject digest differs from its artifact", () => { + const input = exampleInput(); + const managed = input.managed_boundaries; + const result = managedNativeBoundaryInspectionSchema.parse( + managed.normalized_result, + ); + const otherDigest = "b".repeat(64); + const mismatched = createEvidence( + { + path: result.artifact.path, + sha256: result.artifact.sha256, + format: "pe", + }, + managed.provider, + { + operation: managed.operation, + parameters: managed.parameters, + result: { + ...result, + artifact: { ...result.artifact, sha256: otherDigest }, + identity_scope: { + ...result.identity_scope, + requires_artifact_sha256: otherDigest, + }, + }, + rawResult: managed.raw_result, + confidence: managed.confidence, + authority: managed.authority, + environment: managed.environment, + limitations: managed.limitations, + locations: managed.locations, + evidenceLinks: managed.evidence_links, + }, + ); + + expect(() => + verifyManagedNativeBoundaries({ + ...input, + managed_boundaries: mismatched, + }), + ).toThrow( + `Managed Evidence inspect_managed_native_boundaries (${mismatched.evidence_id}) subject SHA-256 ${result.artifact.sha256} does not match normalized artifact SHA-256 ${otherDigest}`, + ); + expect(verifyManagedNativeBoundaries(input).summary.verified).toBe(1); + }); +}); + describe("managed/native boundary verification", () => { it("verifies a P/Invoke declaration against native export Evidence", () => { const result = verifyManagedNativeBoundaries(exampleInput()); From 3d450a29fc91cef3e06ffce9a925ac70dcb24272 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:17 +0800 Subject: [PATCH 031/791] fix(application): budget bridge candidate projection by bytes --- docs/android-analysis.md | 9 +- docs/apple-application-analysis.md | 25 +++-- src/domain/android/androidApplication.ts | 32 ++++--- src/domain/apple/appleApplication.test.ts | 60 ++++++++++++ src/domain/apple/appleApplication.ts | 58 ++++++++--- src/domain/bridgeCandidateProjection.ts | 96 +++++++++++++++++++ tests/composition/android/application.test.ts | 56 +++++++++++ 7 files changed, 302 insertions(+), 34 deletions(-) create mode 100644 src/domain/bridgeCandidateProjection.ts diff --git a/docs/android-analysis.md b/docs/android-analysis.md index a1f6d27a90b..d72ba49776b 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -111,7 +111,14 @@ rea project-android-application-graph '{"inventory_evidence":[ { - const candidates = managed.flatMap((source) => - native.map((target) => ({ - managed_path: source.path, + return projectCartesianCandidates({ + groups: [{ left: managed, right: native }], + createCandidate: (managed_path, target) => ({ + managed_path, native_path: target.path, basis: bridgeBasis(target.path), - })), - ); - return { - candidates, - }; + }), + }); }; const bridgeBasis = ( diff --git a/src/domain/apple/appleApplication.test.ts b/src/domain/apple/appleApplication.test.ts index ea5419dc9c4..2e82c4fd737 100644 --- a/src/domain/apple/appleApplication.test.ts +++ b/src/domain/apple/appleApplication.test.ts @@ -494,4 +494,64 @@ describe("Apple application bridge candidates", () => { ], ]); }); + + it("reports candidate omission when one root expands beyond the bridge safety budget", () => { + const escapedDirectory = `${"é".repeat(100)}${'"'.repeat(120)}`; + const scripts = Array.from({ length: 129 }, (_, index) => ({ + path: `Payload/Fixture.app/${escapedDirectory}/script-${String(index).padStart(3, "0")}.js`, + format: "javascript-bundle", + })); + const native = Array.from({ length: 128 }, (_, index) => + macho( + `Payload/Fixture.app/${escapedDirectory}/lib${String(index).padStart(3, "0")}.dylib`, + ), + ); + const result = project( + inventoryEvidence("zip", "Fixture.zip", [...scripts, ...native]), + ); + + expect(result.components.javascript).toHaveLength(129); + expect(result.components.native_libraries).toHaveLength(128); + expect(result.bridge_candidate_coverage).toMatchObject({ + status: "partial", + total_candidates: 129 * 128, + emitted_candidates: result.bridge_candidates.length, + omitted_candidates: 129 * 128 - result.bridge_candidates.length, + }); + expect(result.bridge_candidates.length).toBeGreaterThan(0); + expect(result.bridge_candidates.length).toBeLessThan(129 * 128); + expect( + Buffer.byteLength(JSON.stringify(result.bridge_candidates)), + ).toBeLessThanOrEqual(2 * 1024 * 1024); + expect(result.coverage).toMatchObject({ + status: "partial", + inventory_complete: true, + }); + expect(result.limitations).toContain( + `Bridge candidate pairs exceeded the projection safety budget; ${result.bridge_candidate_coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, + ); + }); + + it("preserves candidate pairing for components without an application root", () => { + const result = project( + inventoryEvidence("ipa", "Fixture.ipa", [ + { path: "Payload/Fixture.app/Info.plist" }, + { + path: "Payload/Fixture.app/Resources/app.js", + format: "javascript-bundle", + }, + { path: "SwiftSupport/outside.js", format: "javascript-bundle" }, + { path: "SwiftSupport/liboutside.dylib", format: "mach-o" }, + ]), + ); + + expect(result.bridge_candidates).toContainEqual({ + source_path: "SwiftSupport/outside.js", + native_path: "SwiftSupport/liboutside.dylib", + basis: "javascript-and-native-content", + }); + expect(result.bridge_candidate_coverage).toMatchObject({ + status: "complete", + }); + }); }); diff --git a/src/domain/apple/appleApplication.ts b/src/domain/apple/appleApplication.ts index 9a275d3c9dd..dfda9f675e6 100644 --- a/src/domain/apple/appleApplication.ts +++ b/src/domain/apple/appleApplication.ts @@ -18,6 +18,10 @@ import { parseArtifactInventoryEvidence } from "../artifactInventoryEvidence.js" import { evidenceSchema } from "../evidence.js"; import { digestSchema } from "../digests.js"; import { prefixedDigestSchema } from "../digests.js"; +import { + bridgeCandidateCoverageSchema, + projectCartesianCandidates, +} from "../bridgeCandidateProjection.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const pathSchema = z.string().min(1); @@ -74,6 +78,7 @@ export const appleApplicationProjectionResultSchema = z.strictObject({ ]), }), ), + bridge_candidate_coverage: bridgeCandidateCoverageSchema, coverage: z.strictObject({ status: z.enum(["complete-within-inventory", "partial"]), inventory_complete: z.boolean(), @@ -162,6 +167,7 @@ export const projectAppleApplication = ( sidecars: entries.some(({ path }) => isSidecar(path)), symlinks: entries.some(({ kind }) => kind === "symlink"), unattributed, + bridgeCandidateCoverage: bridgeProjection.coverage, }); const withoutId = { root_sha256: inventory.manifest.root_sha256, @@ -179,10 +185,12 @@ export const projectAppleApplication = ( .sort(compare), runtime_families: runtimeFamilies, bridge_candidates: bridgeProjection.candidates, + bridge_candidate_coverage: bridgeProjection.coverage, coverage: { - status: complete - ? ("complete-within-inventory" as const) - : ("partial" as const), + status: + complete && bridgeProjection.coverage.status === "complete" + ? ("complete-within-inventory" as const) + : ("partial" as const), inventory_complete: inventory.complete, }, limitations, @@ -310,7 +318,7 @@ const identifyRuntimeFamilies = (all: readonly Component[]) => { return [...families].sort(compare); }; -/** Pair scripts and native code only within the same application root. */ +/** Pair scripts and native code within matching application-root groups. */ const identifyBridgeCandidates = ( scripts: readonly Component[], native: readonly Component[], @@ -318,18 +326,32 @@ const identifyBridgeCandidates = ( ) => { const owner = (path: string): string | null => roots.find((root) => isWithin(path, root)) ?? null; - return { - candidates: scripts.flatMap((script) => { - const root = owner(script.path); - return native - .filter(({ path }) => owner(path) === root) - .map((item) => ({ - source_path: script.path, - native_path: item.path, - basis: bridgeBasis(item.path), - })); + const scriptGroups: { root: string | null; left: Component[] }[] = []; + for (const script of scripts) { + const root = owner(script.path); + const last = scriptGroups.at(-1); + if (last?.root === root) last.left.push(script); + else scriptGroups.push({ root, left: [script] }); + } + const nativeByRoot = new Map(); + for (const item of native) { + const root = owner(item.path); + const scoped = nativeByRoot.get(root) ?? []; + scoped.push(item); + nativeByRoot.set(root, scoped); + } + const groups = scriptGroups.map(({ root, left }) => ({ + left, + right: nativeByRoot.get(root) ?? [], + })); + return projectCartesianCandidates({ + groups, + createCandidate: (source_path, item) => ({ + source_path, + native_path: item.path, + basis: bridgeBasis(item.path), }), - }; + }); }; const bridgeBasis = ( @@ -355,6 +377,7 @@ const projectionLimitations = (facts: { readonly sidecars: boolean; readonly symlinks: boolean; readonly unattributed: number; + readonly bridgeCandidateCoverage: AppleApplicationProjectionResult["bridge_candidate_coverage"]; }): string[] => [ ...(!facts.complete ? ["Source inventory pages are incomplete; absence is unknown."] @@ -392,6 +415,11 @@ const projectionLimitations = (facts: { "AppleDouble sidecar entries (._* files and __MACOSX/) describe neighbouring files and are excluded from bundle roles.", ] : []), + ...(facts.bridgeCandidateCoverage.status === "partial" + ? [ + `Bridge candidate pairs exceeded the projection safety budget; ${facts.bridgeCandidateCoverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, + ] + : []), "Bundle roles follow path conventions. Read each info_plist_path with inspect_plist for CFBundleExecutable, identifiers, and declared services.", "Bundle identifiers and signing claims require dedicated plist and CMS parsing; this projection reports only exact paths and hashes.", "Bridge candidates are path-based hypotheses, not observed runtime calls.", diff --git a/src/domain/bridgeCandidateProjection.ts b/src/domain/bridgeCandidateProjection.ts new file mode 100644 index 00000000000..bebe1001f1f --- /dev/null +++ b/src/domain/bridgeCandidateProjection.ts @@ -0,0 +1,96 @@ +import { z } from "zod"; + +/** Coverage metadata for the bounded projection of candidate relationships. */ +export const bridgeCandidateCoverageSchema = z.strictObject({ + status: z.enum(["complete", "partial"]), + total_candidates: z.number().int().nonnegative(), + emitted_candidates: z.number().int().nonnegative(), + omitted_candidates: z.number().int().nonnegative(), +}); + +// The serialized candidate array is the retained representation; accounting +// its exact UTF-8 JSON size bounds both array objects and repeated path strings. +/** Maximum UTF-8 JSON size of a retained candidate array. */ +export const MAX_BRIDGE_CANDIDATE_JSON_BYTES = 2 * 1024 * 1024; + +type PathComponent = { readonly path: string }; + +const jsonText = (value: unknown): string => { + const serialized = JSON.stringify(value); + if (serialized === undefined) + throw new TypeError("Bridge candidate is not JSON serializable"); + return serialized; +}; + +/** Retain a deterministic prefix of grouped Cartesian candidates within a byte budget. */ +export const projectCartesianCandidates = < + TLeft extends PathComponent, + TRight extends PathComponent, + TCandidate, +>(input: { + readonly groups: readonly { + readonly left: readonly TLeft[]; + readonly right: readonly TRight[]; + }[]; + readonly createCandidate: (leftPath: string, right: TRight) => TCandidate; +}) => { + const total = input.groups.reduce( + (sum, group) => sum + group.left.length * group.right.length, + 0, + ); + const preparedGroups = input.groups.map((group) => ({ + left: group.left.map((value) => ({ + value, + pathBytes: Buffer.byteLength(jsonText(value.path)), + })), + right: group.right.map((value) => { + const emptyLeftCandidate = jsonText(input.createCandidate("", value)); + return { + value, + rowOverhead: Buffer.byteLength(emptyLeftCandidate) - 2, + }; + }), + })); + let estimatedFullBytes = 2 + Math.max(0, total - 1); // JSON array brackets and commas. + for (const group of preparedGroups) { + estimatedFullBytes += + group.left.length * + group.right.reduce((sum, { rowOverhead }) => sum + rowOverhead, 0) + + group.right.length * + group.left.reduce((sum, { pathBytes }) => sum + pathBytes, 0); + } + + const shouldEmitAll = estimatedFullBytes <= MAX_BRIDGE_CANDIDATE_JSON_BYTES; + const candidates: TCandidate[] = []; + let estimatedRetainedBytes = 2; + candidateLoop: for (const group of preparedGroups) { + for (const left of group.left) { + for (const right of group.right) { + const candidateBytes = right.rowOverhead + left.pathBytes; + const additionalBytes = + candidateBytes + (candidates.length > 0 ? 1 : 0); + if ( + !shouldEmitAll && + estimatedRetainedBytes + additionalBytes > + MAX_BRIDGE_CANDIDATE_JSON_BYTES + ) + break candidateLoop; + candidates.push(input.createCandidate(left.value.path, right.value)); + estimatedRetainedBytes += additionalBytes; + } + } + } + + return { + candidates, + coverage: { + status: + candidates.length === total + ? ("complete" as const) + : ("partial" as const), + total_candidates: total, + emitted_candidates: candidates.length, + omitted_candidates: total - candidates.length, + }, + }; +}; diff --git a/tests/composition/android/application.test.ts b/tests/composition/android/application.test.ts index 0ce74967e3e..ac0318e75ad 100644 --- a/tests/composition/android/application.test.ts +++ b/tests/composition/android/application.test.ts @@ -142,7 +142,9 @@ describe("Android application projection", () => { "Runtime families are inferred from inventory formats and paths; filename suffixes do not establish valid DEX or JVM class bytes.", ); }); +}); +describe("Android bridge candidate coverage", () => { it("returns every component and bridge candidate from the inventory", async () => { const root = await createTestTempDirectory("rea-android-complete-"); const path = join(root, "Fixture.apk"); @@ -171,4 +173,58 @@ describe("Android application projection", () => { expect(projection.bridge_candidates).toHaveLength(1_001); expect(projection.coverage.status).toBe("complete-within-inventory"); }); + + it("marks a multidex by native projection partial when escaped UTF-8 candidate paths exceed its byte budget", async () => { + const root = await createTestTempDirectory("rea-android-bridge-budget-"); + const path = join(root, "Large.apk"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + const dex = Uint8Array.from([0x64, 0x65, 0x78, 0x0a, 0x30, 0x33, 0x35, 0]); + const escapedDirectory = `${"é".repeat(100)}${'"'.repeat(120)}`; + for (let index = 0; index < 129; index += 1) + await writer.add( + `${escapedDirectory}/classes${String(index + 1).padStart(3, "0")}.dex`, + new Uint8ArrayReader(dex), + ); + for (let index = 0; index < 128; index += 1) + await writer.add( + `lib/arm64-v8a/libnative${String(index).padStart(3, "0")}.so`, + new TextReader("native"), + ); + await writeFile(path, await writer.close()); + + const inventory = parseEvidence( + await runProviderAnalysis(path, "inventory_artifact", {}), + ); + const result = projectAndroidApplicationEvidence({ + inventory_evidence: [inventory], + }); + if (!result.ok) throw result.error; + const projection = androidApplicationProjectionResultSchema.parse( + result.value.normalized_result, + ); + + expect(projection.components.dex).toHaveLength(129); + expect(projection.components.native_libraries).toHaveLength(128); + expect(projection.bridge_candidate_coverage).toMatchObject({ + status: "partial", + total_candidates: 129 * 128, + emitted_candidates: projection.bridge_candidates.length, + omitted_candidates: 129 * 128 - projection.bridge_candidates.length, + }); + expect(projection.bridge_candidates.length).toBeGreaterThan(0); + expect(projection.bridge_candidates.length).toBeLessThan(129 * 128); + expect( + Buffer.byteLength(JSON.stringify(projection.bridge_candidates)), + ).toBeLessThanOrEqual(2 * 1024 * 1024); + expect(projection.coverage).toMatchObject({ + status: "partial", + inventory_complete: true, + }); + expect(projection.limitations).toContain( + "Bridge candidates are path-based hypotheses, not decoded JNI declarations or observed runtime calls.", + ); + expect(projection.limitations).toContain( + `Bridge candidate pairs exceeded the projection safety budget; ${projection.bridge_candidate_coverage.omitted_candidates} hypotheses are omitted. Component arrays still include every component from the supplied inventory pages.`, + ); + }); }); From f983c718c33789ee40aec43f84315f6b53e95bde Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:17 +0800 Subject: [PATCH 032/791] fix(browser): bound aggregate source-map decoding before expansion --- docs/web-source-location.md | 9 + src/browser/WebSourceMapFetcher.ts | 142 ++++++++++---- src/domain/webSourceLocation.ts | 1 + src/javascript/sourceMaps/SourceMapFormat.ts | 180 ++++++++++++++---- .../browser/webSourceMapFetcher.test.ts | 141 ++++++++++++++ 5 files changed, 394 insertions(+), 79 deletions(-) diff --git a/docs/web-source-location.md b/docs/web-source-location.md index dae31c3ad60..6e677b08f56 100644 --- a/docs/web-source-location.md +++ b/docs/web-source-location.md @@ -91,6 +91,15 @@ preceding browser mappings when a nested section starts with a gap or has no child leaves. Raw map text, original source identities and the dedicated decoder profile are preserved. +Browser collection accepts up to 64 MiB of map responses per inspection and +preflights indexed sections, sources, names, generated rows and mapping segments +against a 262144-record decode budget before upstream flattening. Indexed maps +are limited to 64 nested levels. These structural limits bound decoded object +growth while preserving large embedded `sourcesContent` strings within the +response-byte budget; exceeding a decoder limit returns a `fetch_failed` item. +Malformed UTF-8 map responses are reported as invalid rather than repaired with +replacement characters. + Focused tests cover duplicate mappings, indexed offsets/overlap/local indexes, 32-bit VLQ overflow, nullable content, UTF-16/CRLF positions, immutable artifact identity, cancellation, late output closure, cleanup failures, SDK schemas and diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 6adb381f39a..3cbed28f064 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -12,7 +12,6 @@ import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js"; import { analyzeParsedJavaScriptReferences } from "../domain/javascript/javascriptSemanticAnalysis.js"; import { traverseJavaScriptAst } from "../domain/javascript/javascriptSemanticTraversal.js"; import { parseJavaScriptSource } from "../domain/javascript/javascriptSourceParser.js"; -import { hasValidSourceMapContents } from "../domain/sourceMapContents.js"; import type { AnalyzeWebBundleInput, WebSourceMapItem, @@ -22,9 +21,9 @@ import { webSourceMapsSchema } from "../domain/webBundleAnalysis.js"; import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import { safeParseJson } from "../domain/safeJson.js"; import { - flattenSourceMapLeaves, - isVersion3Map, -} from "../domain/sourceMapEnvelope.js"; + SourceMapFormatFailure, + validateSourceMapStructure, +} from "../javascript/sourceMaps/SourceMapFormat.js"; export interface WebSourceMapRequest { readonly scriptKey: string; @@ -53,6 +52,11 @@ type ParsedSourceMapItem = Extract< SourceMapItem, { status: "included" | "partial" } >; +interface SourceMapDecodeContext { + readonly signal: AbortSignal | undefined; + readonly deadlineAt: number; + readonly budget: { records: number }; +} /** Fetch and validate approved source maps without browser credentials. */ export const fetchWebSourceMaps = async ( @@ -70,11 +74,17 @@ export const fetchWebSourceMaps = async ( const abortFromCaller = (): void => operationController.abort(signal?.reason); signal?.addEventListener("abort", abortFromCaller, { once: true }); const operationSignal = operationController.signal; - const budget = { retainedBytes: 0 }; + const deadlineAt = + Date.now() + (host.timeoutMs ?? SOURCE_MAP_FETCH_TIMEOUT_MS); + const budget = { + retainedBytes: 0, + deadlineAt, + decodedRecords: { records: 0 }, + }; try { for (const request of requests) { if (signal?.aborted === true) throw signal.reason; - if (operationSignal.aborted) { + if (operationSignal.aborted || Date.now() >= deadlineAt) { items.push( emptySourceMapItem( request, @@ -90,25 +100,28 @@ export const fetchWebSourceMaps = async ( clearTimeout(timeout); signal?.removeEventListener("abort", abortFromCaller); } - const retained = items.filter( - ({ status }) => status === "included" || status === "partial", - ).length; + if (signal?.aborted === true) throw signal.reason; return webSourceMapsSchema.parse({ - status: - items.length === 0 - ? "unavailable" - : retained === items.length && - !items.some(({ status }) => status === "partial") - ? "included" - : retained > 0 - ? "partial" - : "unavailable", + status: sourceMapsStatus(items), requested: requests.length, processed: items.length, items, }); }; +const sourceMapsStatus = ( + items: readonly SourceMapItem[], +): SourceMaps["status"] => { + const retained = items.filter( + ({ status }) => status === "included" || status === "partial", + ).length; + if (items.length === 0 || retained === 0) return "unavailable"; + return retained === items.length && + !items.some(({ status }) => status === "partial") + ? "included" + : "partial"; +}; + class SourceMapDeadlineError extends Error { constructor() { super("Source-map fetching exceeded its operation deadline."); @@ -123,12 +136,23 @@ class SourceMapSizeLimitError extends Error { } } +class SourceMapEncodingError extends Error { + constructor() { + super("Source-map response is not valid UTF-8."); + this.name = "SourceMapEncodingError"; + } +} + const fetchOne = async ( request: WebSourceMapRequest, input: AnalyzeWebBundleInput, signal: AbortSignal | undefined, host: SourceMapFetchHost, - budget: { retainedBytes: number }, + budget: { + retainedBytes: number; + deadlineAt: number; + decodedRecords: { records: number }; + }, ): Promise => { if (!approvedUrl(request.fetchUrl, input.allowed_origins)) return emptySourceMapItem( @@ -158,6 +182,7 @@ const fetchOne = async ( `Source-map server returned HTTP ${String(response.status)}.`, ); } + checkOperation(undefined, signal, budget.deadlineAt); return normalizeSourceMap( request, await readBoundedText( @@ -167,6 +192,11 @@ const fetchOne = async ( signal, ), fetchedUrl, + { + signal, + deadlineAt: budget.deadlineAt, + budget: budget.decodedRecords, + }, ); } catch (cause: unknown) { if ( @@ -174,6 +204,8 @@ const fetchOne = async ( !(signal.reason instanceof SourceMapDeadlineError) ) throw cause; + if (cause instanceof SourceMapEncodingError) + return emptySourceMapItem(request, "invalid", cause.message); return emptySourceMapItem( request, "fetch_failed", @@ -260,7 +292,11 @@ const readBoundedText = async ( joined.set(chunk, offset); offset += chunk.byteLength; } - return new TextDecoder().decode(joined); + try { + return new TextDecoder("utf-8", { fatal: true }).decode(joined); + } catch { + throw new SourceMapEncodingError(); + } }; const readWithAbort = ( @@ -346,13 +382,15 @@ type EncodedBrowserSourceMap = EncodedSourceMap | SectionedSourceMap; /** Compose indexed offsets before upstream flattening, retaining parent clipping. */ const sourceMapWithAbsoluteSections = ( - text: string, + value: unknown, + check: () => void, ): EncodedBrowserSourceMap => { - const root = JSON.parse(text) as EncodedBrowserSourceMap; + const root = value as EncodedBrowserSourceMap; if (!("sections" in root)) return root; const sections: Section[] = []; const pending: Section[] = [{ map: root, offset: { line: 0, column: 0 } }]; while (pending.length > 0) { + check(); const node = pending.pop(); if (node === undefined) break; if (!("sections" in node.map)) { @@ -387,17 +425,31 @@ const normalizeSourceMap = ( request: WebSourceMapRequest, text: string, fetchedUrl: string, + context: SourceMapDecodeContext, ): SourceMapItem => { - if (!validSourceMapEnvelope(text)) + const parsedJson = safeParseJson(text); + if (!parsedJson.ok) return emptySourceMapItem( request, "invalid", "Source-map JSON is not a version 3 map.", ); try { - const map = new AnyMap(sourceMapWithAbsoluteSections(text), fetchedUrl); + const check = (): void => + checkOperation(undefined, context.signal, context.deadlineAt); + validateSourceMapStructure(parsedJson.value, { + profile: "browser-collection", + budget: context.budget, + check, + }); + check(); + const map = new AnyMap( + sourceMapWithAbsoluteSections(parsedJson.value, check), + fetchedUrl, + ); const resolvedBySource = new Map(); const originalSources = map.sources.map((source, index) => { + if ((index & 255) === 0) check(); const content = map.sourcesContent?.[index]; const resolved = map.resolvedSources[index] ?? source ?? "[unknown-source]"; @@ -412,6 +464,7 @@ const normalizeSourceMap = ( }); const mappings: ParsedSourceMapItem["mappings"] = []; eachMapping(map, (mapping) => { + if ((mappings.length & 0x3fff) === 0) check(); if ( mapping.source === null || mapping.originalLine === null || @@ -429,7 +482,8 @@ const normalizeSourceMap = ( name: mapping.name ?? null, }); }); - const modules = originalModuleEdges(originalSources); + const modules = originalModuleEdges(originalSources, check); + check(); const parsed = { ...sourceMapContext(request), artifact: createWebTextArtifact(text, "application/source-map+json"), @@ -437,6 +491,7 @@ const normalizeSourceMap = ( original_module_edges: modules.edges, mappings, }; + check(); return modules.incomplete.length === 0 ? { ...parsed, status: "included", limitation: null } : { @@ -445,11 +500,17 @@ const normalizeSourceMap = ( limitation: `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, }; } catch (cause: unknown) { - void cause; + if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") + return emptySourceMapItem(request, "fetch_failed", cause.message); + if ( + context.signal?.aborted === true || + cause instanceof SourceMapDeadlineError + ) + throw cause; return emptySourceMapItem( request, "invalid", - "Source-map mappings could not be decoded safely.", + "Source-map JSON mappings could not be decoded safely.", ); } }; @@ -462,13 +523,16 @@ interface OriginalModuleEdges { const originalModuleEdges = ( sources: ParsedSourceMapItem["original_sources"], + check: () => void, ): OriginalModuleEdges => { const edges: ParsedSourceMapItem["original_module_edges"] = []; const seen = new Set(); const incomplete: string[] = []; for (const source of sources) { + check(); if (source.artifact === null) continue; const parsed = parseJavaScriptSource(source.artifact.text); + check(); if (parsed === null) { incomplete.push(source.source); continue; @@ -479,6 +543,7 @@ const originalModuleEdges = ( let unboundRequires: ReadonlySet | undefined; traverseJavaScriptAst(parsed, { enter: (node) => { + check(); const dependency = originalDependency(node); if (dependency === null) return; const { kind, specifier } = dependency; @@ -518,23 +583,16 @@ const originalModuleEdges = ( return { edges, incomplete }; }; -const validSourceMapEnvelope = (text: string): boolean => { - const parsedResult = safeParseJson(text); - if (!parsedResult.ok) return false; - const parsed: unknown = parsedResult.value; - if (isVersion3Map(parsed) && typeof parsed.mappings === "string") - return validSourceMapLeaf(parsed); - const leaves = flattenSourceMapLeaves(parsed, { validateOffsets: true }); - if (leaves === undefined) return false; - return leaves.every(validSourceMapLeaf); +const checkOperation = ( + callerSignal: AbortSignal | undefined, + operationSignal: AbortSignal | undefined, + deadlineAt: number, +): void => { + if (callerSignal?.aborted === true) throw callerSignal.reason; + if (operationSignal?.aborted === true) throw operationSignal.reason; + if (Date.now() >= deadlineAt) throw new SourceMapDeadlineError(); }; -const validSourceMapLeaf = (map: Readonly>): boolean => - typeof map.mappings === "string" && - Array.isArray(map.sources) && - Array.isArray(map.names) && - hasValidSourceMapContents(map.sources.length, map.sourcesContent); - const approvedUrl = ( value: string, allowedOrigins: readonly string[], diff --git a/src/domain/webSourceLocation.ts b/src/domain/webSourceLocation.ts index c4e0ee0707b..b93eae6e326 100644 --- a/src/domain/webSourceLocation.ts +++ b/src/domain/webSourceLocation.ts @@ -104,6 +104,7 @@ export const WEB_SOURCE_MAP_LIMITS = { outputBytes: 32 * 1024 * 1024, decodedRows: 262144, decodedSegments: 262144, + decodedRecords: 262144, sectionDepth: 64, decodeTimeoutMs: 20000, } as const; diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index 9c3e1db2738..f4907862853 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js"; +import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js"; const leafSchema = z.object({ version: z.literal(3), @@ -38,6 +39,17 @@ interface PendingMap { readonly stop: Offset | null; } +/** Shared structural constraints configurable only for stricter consumers. */ +export interface SourceMapStructureBudget { + records: number; +} + +export interface SourceMapStructureOptions { + readonly profile?: "browser-collection"; + readonly budget?: SourceMapStructureBudget; + readonly check?: () => void; +} + /** A decoder-profile failure with an actionable producer constraint. */ export class SourceMapFormatFailure extends Error { constructor( @@ -84,9 +96,32 @@ const addOffset = (base: Offset, offset: Offset): Offset => { return result; }; -const addSections = (node: PendingMap, pending: PendingMap[]): void => { +const consumeRecords = (budget: { records: number }, count: number): void => { + if (count > WEB_SOURCE_MAP_LIMITS.decodedRecords - budget.records) + throw new SourceMapFormatFailure( + "limit", + `Source-map structure exceeds the ${WEB_SOURCE_MAP_LIMITS.decodedRecords}-record decoder budget.`, + ); + budget.records += count; +}; + +const addSections = ( + node: PendingMap, + pending: PendingMap[], + budget: { records: number }, + options: SourceMapStructureOptions, +): void => { + options.check?.(); + const header = mapHeaderSchema.parse(node.value); + if (!Array.isArray(header.sections)) + throw new SourceMapFormatFailure( + "format", + "Indexed source-map sections must be an array.", + ); + consumeRecords(budget, header.sections.length); const sections = indexedSchema.parse(node.value).sections; for (const [index, section] of sections.entries()) { + if ((index & 255) === 0) options.check?.(); const previous = sections[index - 1]; if (previous !== undefined && compare(section.offset, previous.offset) <= 0) throw new SourceMapFormatFailure( @@ -133,6 +168,7 @@ const addSections = (node: PendingMap, pending: PendingMap[]): void => { const inspectMappingLayout = ( mappings: string, + check?: () => void, ): { lines: number; segments: number } => { let lines = 1; let segments = 0; @@ -152,7 +188,9 @@ const inspectMappingLayout = ( "Source-map segment requires one, four or five VLQ components.", ); }; + let index = 0; for (const character of mappings) { + if ((index++ & 0x3fff) === 0) check?.(); if (character === "," || character === ";") { finish(); inSegment = false; @@ -189,6 +227,58 @@ const inspectMappingLayout = ( return { lines, segments }; }; +const inspectLeaf = ( + node: PendingMap, + budget: { records: number }, + options: SourceMapStructureOptions, +): z.output => { + options.check?.(); + consumeRecords(budget, 1); + if ( + typeof node.value !== "object" || + node.value === null || + Array.isArray(node.value) + ) + return leafSchema.parse(node.value); + const rawMap = node.value as Readonly>; + const rawSources = rawMap.sources; + const rawNames = rawMap.names; + const rawContents = rawMap.sourcesContent; + consumeRecords(budget, Array.isArray(rawSources) ? rawSources.length : 0); + consumeRecords(budget, Array.isArray(rawNames) ? rawNames.length : 0); + consumeRecords(budget, Array.isArray(rawContents) ? rawContents.length : 0); + options.check?.(); + if (options.profile === "browser-collection" && !Array.isArray(rawNames)) + throw new SourceMapFormatFailure( + "format", + "Source-map leaf requires a names array.", + ); + if ( + options.profile === "browser-collection" && + !hasValidSourceMapContents( + Array.isArray(rawSources) ? rawSources.length : 0, + rawContents, + ) + ) + throw new SourceMapFormatFailure( + "format", + "Source-map sourcesContent must align with its source declarations.", + ); + options.check?.(); + const leaf = leafSchema.parse(node.value); + const layout = inspectMappingLayout(leaf.mappings, options.check); + consumeRecords(budget, layout.segments + layout.lines); + if ( + layout.segments > WEB_SOURCE_MAP_LIMITS.decodedSegments || + node.offset.line + layout.lines > WEB_SOURCE_MAP_LIMITS.decodedRows + ) + throw new SourceMapFormatFailure( + "limit", + "Source-map layout exceeds the 262144 segment/generated-row decoder budget.", + ); + return leaf; +}; + /** Validate structure before allocation, retaining raw declarations and section boundaries. */ export const inspectSourceMap = ( text: string, @@ -204,14 +294,51 @@ export const inspectSourceMap = ( ? withoutBom.slice(withoutBom.indexOf("\n") + 1) : withoutBom; const value: unknown = JSON.parse(jsonText); + const inspected = inspectSourceMapValue(value); + const normalized = + inspected.format === "regular" + ? inspected.leaves[0]?.map + : { + version: 3, + sections: inspected.leaves.map((leaf) => ({ + offset: leaf.offset, + map: leaf.map, + })), + }; + return { + jsonText: JSON.stringify(normalized), + ...inspected, + }; +}; + +/** Validate shared source-map structure before a consumer decodes mappings. */ +export const validateSourceMapStructure = ( + value: unknown, + options: SourceMapStructureOptions = {}, +): void => { + inspectSourceMapValue(value, options, false); +}; + +const inspectSourceMapValue = ( + value: unknown, + options: SourceMapStructureOptions = {}, + collect = true, +): { + readonly format: "regular" | "indexed"; + readonly reportedFile: string | null; + readonly declarations: readonly SourceMapSourceDeclaration[]; + readonly leaves: readonly SourceMapLeaf[]; +} => { + options.check?.(); const root = mapHeaderSchema.parse(value); const declarations: SourceMapSourceDeclaration[] = []; const leaves: SourceMapLeaf[] = []; const pending: PendingMap[] = [ { value, path: [], offset: { line: 0, column: 0 }, stop: null }, ]; - let segmentCount = 0; + const budget = options.budget ?? { records: 0 }; while (pending.length > 0) { + options.check?.(); const node = pending.pop(); if (node === undefined) break; if (node.path.length > WEB_SOURCE_MAP_LIMITS.sectionDepth) @@ -220,47 +347,26 @@ export const inspectSourceMap = ( "Indexed source-map nesting exceeds the 64-level decoder stack budget.", ); if (mapHeaderSchema.parse(node.value).sections !== undefined) { - addSections(node, pending); + addSections(node, pending, budget, options); continue; } - const leaf = leafSchema.parse(node.value); - const layout = inspectMappingLayout(leaf.mappings); - segmentCount += layout.segments; - if ( - segmentCount > WEB_SOURCE_MAP_LIMITS.decodedSegments || - node.offset.line + layout.lines > WEB_SOURCE_MAP_LIMITS.decodedRows - ) - throw new SourceMapFormatFailure( - "limit", - "Source-map layout exceeds the 262144 segment/generated-row decoder budget.", - ); - leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); + const leaf = inspectLeaf(node, budget, options); + if (collect) + leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); for (const [sourceIndex, source] of leaf.sources.entries()) - declarations.push({ - sectionPath: node.path, - sourceIndex, - source, - root: leaf.sourceRoot ?? null, - content: leaf.sourcesContent?.[sourceIndex] ?? null, - ignored: (leaf.ignoreList ?? leaf.x_google_ignoreList ?? []).includes( + if (collect) + declarations.push({ + sectionPath: node.path, sourceIndex, - ), - }); + source, + root: leaf.sourceRoot ?? null, + content: leaf.sourcesContent?.[sourceIndex] ?? null, + ignored: (leaf.ignoreList ?? leaf.x_google_ignoreList ?? []).includes( + sourceIndex, + ), + }); } - // Absolute leaves adapt nested first-line column offsets before upstream AnyMap. - // Raw map bytes and the original section/source identities are retained separately. - const normalized = - root.sections === undefined - ? leaves[0]?.map - : { - version: 3, - sections: leaves.map((leaf) => ({ - offset: leaf.offset, - map: leaf.map, - })), - }; return { - jsonText: JSON.stringify(normalized), format: root.sections === undefined ? "regular" : "indexed", reportedFile: root.file ?? null, declarations, diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 0a56be9affe..2a5d1e963ec 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -374,6 +374,52 @@ describe("web source-map fetching and validation: enforcing fetch limits and val }); }); +describe("source-map response encoding", () => { + it("rejects malformed UTF-8 source-map response bytes", async () => { + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => + Promise.resolve( + new Response(Uint8Array.of(0x7b, 0x22, 0xc3, 0x28, 0x22, 0x7d), { + status: 200, + }), + ), + }); + + expect(result.status).toBe("unavailable"); + expect(result.items[0]).toMatchObject({ + status: "invalid", + artifact: null, + limitation: "Source-map response is not valid UTF-8.", + }); + }); +}); + +describe("source-map operation deadline observations", () => { + it("keeps completed maps when a later request times out", async () => { + let calls = 0; + const requests = [ + request, + { ...request, scriptKey: `scr_${"2".repeat(64)}` }, + ]; + const result = await fetchWebSourceMaps(requests, input(), undefined, { + timeoutMs: 100, + fetch: () => { + calls += 1; + return calls === 1 + ? Promise.resolve(validMapResponse()) + : new Promise(() => undefined); + }, + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "fetch_failed", + ]); + expect(result.items[0]?.original_sources).toHaveLength(1); + }); +}); + describe("indexed source-map offsets", () => { it.each([ { line: 0, column: -1 }, @@ -621,6 +667,76 @@ describe("source-map dependency coverage of hard-to-parse sources", () => { }); describe("web source-map collection", () => { + it("rejects mapping expansion before the trace-mapping decoder allocates it", async () => { + const response = new Response( + JSON.stringify({ + version: 3, + names: [], + sources: [], + mappings: Array.from({ length: 262_145 }, () => "A").join(","), + }), + { status: 200 }, + ); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(response), + }); + + expect(result.status).toBe("unavailable"); + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + }); + expect(result.items[0]?.limitation).toContain("262144"); + }); + + it("accounts for decoded records across the complete fetch operation", async () => { + const mapText = JSON.stringify({ + version: 3, + names: [], + sources: [], + mappings: Array.from({ length: 150_000 }, () => "A").join(","), + }); + const requests = [ + request, + { ...request, scriptKey: `scr_${"2".repeat(64)}` }, + ]; + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => Promise.resolve(new Response(mapText, { status: 200 })), + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "fetch_failed", + ]); + expect(result.items[1]?.limitation).toContain("262144-record"); + }); + + it("rejects deeply nested indexed maps before recursive flattening", async () => { + let map: unknown = { + version: 3, + names: [], + sources: [], + mappings: "", + }; + for (let index = 0; index < 65; index += 1) + map = { + version: 3, + sections: [{ offset: { line: 0, column: 0 }, map }], + }; + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => + Promise.resolve(new Response(JSON.stringify(map), { status: 200 })), + }); + + expect(result.status).toBe("unavailable"); + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + }); + expect(result.items[0]?.limitation).toContain("64-level"); + }); + it("retains every mapping from a sectioned source map", async () => { const segmentCount = 10_001; const regular = { @@ -693,6 +809,31 @@ describe("web source-map collection", () => { }); }); +describe("raw source-map structure preflight", () => { + it("budgets source inventory before schema parsing its entries", async () => { + const response = new Response( + JSON.stringify({ + version: 3, + names: [], + sources: Array.from({ length: 262_145 }, (_, index) => + index === 0 ? 42 : "source.js", + ), + mappings: "", + }), + { status: 200 }, + ); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(response), + }); + + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + }); + expect(result.items[0]?.limitation).toContain("262144-record"); + }); +}); + const validMapResponse = () => new Response( JSON.stringify({ From e714fb027afdf11bfbb0838722523d3fa9ebf244 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:18 +0800 Subject: [PATCH 033/791] fix(inspector): bound retained metadata and preserve cancellation --- docs/javascript-runtime-observation.md | 18 ++- .../JavaScriptRuntimeObservationService.ts | 7 +- src/browser/CdpConnection.ts | 11 +- src/inspector/JavaScriptRuntimeScope.ts | 12 ++ .../V8InspectorCaptureProjection.test.ts | 46 +++++++ src/inspector/V8InspectorCaptureProjection.ts | 15 ++- src/inspector/V8InspectorProvider.ts | 71 +++++++--- .../javascriptRuntimeObservation.test.ts | 71 +++++++++- tests/fixtures/inspector/fakeV8Inspector.ts | 121 ++++++++++++------ 9 files changed, 296 insertions(+), 76 deletions(-) diff --git a/docs/javascript-runtime-observation.md b/docs/javascript-runtime-observation.md index 6553023af5a..ecfed708f97 100644 --- a/docs/javascript-runtime-observation.md +++ b/docs/javascript-runtime-observation.md @@ -119,12 +119,18 @@ targets belong to the same Electron application. ## Capture and determinism -Every observation records its time window and returns all valid script and -execution-context events received during it. Per-location protocol validation -remains in force. Scripts are validated after capture, deduplicated by stable -metadata, and canonically sorted. Wall-clock timestamps and protocol script IDs -are excluded from the durable result, so identical inputs and captured metadata -produce the same Evidence ID. +Capture retains valid script and execution-context metadata within an 8 MiB +aggregate budget, including conservative estimates for UTF-16 strings, record +and collection overhead, and projected URL fields. Each CDP message also has a +16 MiB transport limit. A message over that limit fails with `payload_limit`; +when the aggregate budget is reached, later metadata events are dropped and +the partial result reports `events_dropped`, `truncated: true`, and +`retained_metadata_budget_exceeded`. Repeated context transitions update the +current context record instead of accumulating duplicate objects. Scripts are +validated after capture, deduplicated by stable metadata, and canonically +sorted. Wall-clock timestamps and protocol script IDs are excluded from the +durable result, so identical inputs and captured metadata produce the same +Evidence ID. Inspector attach is inherently incomplete. Enabling `Debugger` reports known and uncollected scripts, but scripts collected before attachment may be diff --git a/src/application/javascript/JavaScriptRuntimeObservationService.ts b/src/application/javascript/JavaScriptRuntimeObservationService.ts index cfbdbb05ea4..655d16f2d4c 100644 --- a/src/application/javascript/JavaScriptRuntimeObservationService.ts +++ b/src/application/javascript/JavaScriptRuntimeObservationService.ts @@ -2,7 +2,10 @@ import type { ExecutionOptions } from "../AnalysisProvider.js"; import type { JavaScriptRuntimeObservationPort } from "./JavaScriptRuntimeObservationPort.js"; import { createJavaScriptRuntimeObservationEvidence } from "./JavaScriptRuntimeObservationEvidence.js"; import type { Evidence } from "../../domain/evidence.js"; -import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, +} from "../../domain/analysisErrorCore.js"; import { type AnalysisError } from "../../domain/analysisErrorBase.js"; import type { ListJavaScriptRuntimeTargetsInput, @@ -40,6 +43,8 @@ export const observeJavaScriptRuntime = async ( const ready = requireProvider(provider, "observe_javascript_runtime"); if (!ready.ok) return ready; const result = await ready.value.observe(input, options); + if (options.signal?.aborted === true) + return err(new AnalysisCancelledError("observe_javascript_runtime")); return result.ok ? ok( createJavaScriptRuntimeObservationEvidence( diff --git a/src/browser/CdpConnection.ts b/src/browser/CdpConnection.ts index fae6046ed27..3ab7f94762b 100644 --- a/src/browser/CdpConnection.ts +++ b/src/browser/CdpConnection.ts @@ -25,7 +25,9 @@ interface PendingCommand { export class CdpConnection { readonly #pending = new Map(); readonly #listeners = new Set<(event: CdpEvent) => void>(); - readonly #disconnectListeners = new Set<() => void>(); + readonly #disconnectListeners = new Set< + (error: BrowserObservationError) => void + >(); readonly #protocolFailureListeners = new Set< (error: BrowserObservationError) => void >(); @@ -74,9 +76,9 @@ export class CdpConnection { } /** Subscribe to an unexpected transport loss while an operation is active. */ - onDisconnect(listener: () => void): () => void { + onDisconnect(listener: (error: BrowserObservationError) => void): () => void { if (this.#closed) { - listener(); + listener(this.#transportError(this.#transportFailure)); return () => undefined; } this.#disconnectListeners.add(listener); @@ -281,7 +283,8 @@ export class CdpConnection { if (!wasClosed) this.#transportFailure = reason; this.#failPending(reason); if (wasClosed) return; - for (const listener of this.#disconnectListeners) listener(); + const error = this.#transportError(reason); + for (const listener of this.#disconnectListeners) listener(error); this.#disconnectListeners.clear(); } diff --git a/src/inspector/JavaScriptRuntimeScope.ts b/src/inspector/JavaScriptRuntimeScope.ts index 4396d7fe5da..47e501f0c25 100644 --- a/src/inspector/JavaScriptRuntimeScope.ts +++ b/src/inspector/JavaScriptRuntimeScope.ts @@ -9,6 +9,7 @@ import type { JavaScriptRuntimeLocation, JavaScriptRuntimeTargetLocation, } from "../domain/javascript/javascriptRuntimeObservation.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { authorizedElectronFile } from "../browser/ElectronFileScope.js"; export type RuntimeLocationDecision = @@ -18,6 +19,14 @@ export type RuntimeLocationDecision = readonly reason: BrowserExclusionReason; }; +/** Throw the operation's typed cancellation error whenever its signal is aborted. */ +export const throwIfRuntimeObservationCancelled = ( + signal?: AbortSignal, +): void => { + if (signal?.aborted === true) + throw new AnalysisCancelledError("observe_javascript_runtime"); +}; + /** * Wire bucket for inspector script exclusions. The V8 observation schemas * fix the `unsupported_location` key, so every canonical denial reason maps @@ -35,7 +44,9 @@ export const inspectorExclusionKey = ( /** Resolve a protocol location exposed by the explicitly selected Inspector endpoint. */ export const authorizeRuntimeLocation = async ( value: string, + signal?: AbortSignal, ): Promise => { + throwIfRuntimeObservationCancelled(signal); if (value.startsWith("node:") && value.length > 0) return { allowed: true, @@ -43,6 +54,7 @@ export const authorizeRuntimeLocation = async ( }; if (value.startsWith("file:")) { const filePath = await authorizedElectronFile(value); + throwIfRuntimeObservationCancelled(signal); return filePath === undefined ? { allowed: false, reason: "not_approved" } : { allowed: true, location: { kind: "file", file_path: filePath } }; diff --git a/src/inspector/V8InspectorCaptureProjection.test.ts b/src/inspector/V8InspectorCaptureProjection.test.ts index 124745e67a2..4a6eb6922a9 100644 --- a/src/inspector/V8InspectorCaptureProjection.test.ts +++ b/src/inspector/V8InspectorCaptureProjection.test.ts @@ -5,6 +5,7 @@ import { pathToFileURL } from "node:url"; import { afterAll, beforeAll, expect, it } from "vitest"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { finalizeInspectorCapture } from "./V8InspectorCaptureProjection.js"; import type { CaptureState } from "./V8InspectorProvider.js"; @@ -214,3 +215,48 @@ it("waits for active authorization workers to settle before returning a failure" expect(new Set(startedUrls)).toEqual(initialStarted); expect(active).toBe(0); }); + +it("stops finalization when cancellation arrives during location authorization", async () => { + const controller = new AbortController(); + let authorizationStarted = false; + const state = stateFor([ + { + rawUrl: "https://example.test/script.js", + executionContextKey: null, + cdpHash: null, + length: 0, + isModule: false, + }, + ]); + + const finalization = finalizeInspectorCapture({ + input: { + inspector_endpoint: "http://127.0.0.1:9222", + target_id: target.id, + observation_ms: 100, + }, + runtime: { + product: "Node.js/v24.18.0", + protocol_version: "1.3", + v8_version: null, + }, + target, + state, + signal: controller.signal, + authorizeLocation: async () => { + authorizationStarted = true; + controller.abort(); + return { + allowed: true, + location: { + kind: "url", + origin: "https://example.test", + sanitized_url: "https://example.test/script.js", + }, + }; + }, + }); + + await expect(finalization).rejects.toBeInstanceOf(AnalysisCancelledError); + expect(authorizationStarted).toBe(true); +}); diff --git a/src/inspector/V8InspectorCaptureProjection.ts b/src/inspector/V8InspectorCaptureProjection.ts index 102247afb8a..9c8d8ab70c7 100644 --- a/src/inspector/V8InspectorCaptureProjection.ts +++ b/src/inspector/V8InspectorCaptureProjection.ts @@ -9,6 +9,7 @@ import type { import { authorizeRuntimeLocation, inspectorExclusionKey, + throwIfRuntimeObservationCancelled, } from "./JavaScriptRuntimeScope.js"; import type { CaptureState, ScriptDraft } from "./V8InspectorProvider.js"; import type { AuthorizedV8InspectorTarget } from "./V8InspectorEndpoint.js"; @@ -38,7 +39,11 @@ interface FinalizeCaptureInput { readonly runtime: JavaScriptRuntimeTargetList["runtime"]; readonly target: AuthorizedV8InspectorTarget; readonly state: CaptureState; - readonly authorizeLocation?: typeof authorizeRuntimeLocation; + readonly authorizeLocation?: ( + rawUrl: string, + signal?: AbortSignal, + ) => ReturnType; + readonly signal?: AbortSignal; } const LOCATION_AUTHORIZATION_WORKERS = 8; @@ -55,7 +60,9 @@ export const finalizeInspectorCapture = async ({ target, state, authorizeLocation = authorizeRuntimeLocation, + signal, }: FinalizeCaptureInput): Promise => { + throwIfRuntimeObservationCancelled(signal); const exclusions = createInspectorExclusionCounts(); const scripts = new Map< string, @@ -74,6 +81,7 @@ export const finalizeInspectorCapture = async ({ const worker = async (): Promise => { try { for (;;) { + throwIfRuntimeObservationCancelled(signal); if (stopScheduling) return; const index = nextGroup; nextGroup += 1; @@ -82,8 +90,9 @@ export const finalizeInspectorCapture = async ({ const [rawUrl, drafts] = group; authorizedGroups[index] = { drafts, - decision: await authorizeLocation(rawUrl), + decision: await authorizeLocation(rawUrl, signal), }; + throwIfRuntimeObservationCancelled(signal); } } catch (cause: unknown) { stopScheduling = true; @@ -99,6 +108,7 @@ export const finalizeInspectorCapture = async ({ const failedWorker = workerResults.find( (result) => result.status === "rejected", ); + throwIfRuntimeObservationCancelled(signal); if (failedWorker?.status === "rejected") throw failedWorker.reason; for (const { drafts, decision } of authorizedGroups) { for (const draft of drafts) { @@ -131,6 +141,7 @@ export const finalizeInspectorCapture = async ({ ? 1 : 0, ); + throwIfRuntimeObservationCancelled(signal); return { runtime, target: { diff --git a/src/inspector/V8InspectorProvider.ts b/src/inspector/V8InspectorProvider.ts index 01189ec799a..6dc9f7ffc8c 100644 --- a/src/inspector/V8InspectorProvider.ts +++ b/src/inspector/V8InspectorProvider.ts @@ -42,6 +42,12 @@ import { export { V8_INSPECTOR_PROVIDER_IDENTITY } from "./providerIdentity.js"; import { V8_INSPECTOR_PROVIDER_IDENTITY } from "./providerIdentity.js"; +/** Maximum decoded CDP message accepted while observing one Inspector target. */ +export const INSPECTOR_MAX_PAYLOAD_BYTES = 16 * 1024 * 1024; +/** Aggregate retained metadata budget, including an estimate of JS object overhead. */ +export const INSPECTOR_MAX_RETAINED_METADATA_BYTES = 8 * 1024 * 1024; +const RETAINED_OBJECT_OVERHEAD_BYTES = 512; + export interface ScriptDraft { readonly rawUrl: string; readonly executionContextKey: string | null; @@ -147,6 +153,7 @@ export class V8InspectorProvider implements JavaScriptRuntimeObservationPort { target.webSocketUrl, "observe_javascript_runtime", options.signal, + { maxPayloadBytes: INSPECTOR_MAX_PAYLOAD_BYTES }, ); const state = emptyCaptureState(); const removeListener = connection.onEvent((event) => @@ -164,6 +171,7 @@ export class V8InspectorProvider implements JavaScriptRuntimeObservationPort { runtime: discovery.runtime, target, state, + ...(options.signal === undefined ? {} : { signal: options.signal }), }); outcome = ok(javascriptRuntimeObservationSchema.parse(result)); } catch (cause: unknown) { @@ -344,8 +352,8 @@ const ingestScript = ( length: nonnegativeInteger(value?.length), isModule: value?.isModule === true, }; - const bytes = metadataBytes(draft); - retainEvent(state, bytes); + const bytes = retainedMetadataBytes(draft); + if (!retainEvent(state, bytes)) return; state.scripts.push(draft); }; @@ -355,8 +363,16 @@ const ingestContext = ( state: CaptureState, ): void => { if (event.method === "Runtime.executionContextsCleared") { - for (const context of state.contexts.values()) context.state = "cleared"; - retainEvent(state, 0); + let byteDelta = 0; + for (const context of state.contexts.values()) { + byteDelta += + retainedMetadataBytes({ ...context, state: "cleared" }) - + retainedMetadataBytes(context); + } + if (replaceRetainedMetadata(state, byteDelta)) { + for (const context of state.contexts.values()) context.state = "cleared"; + retainEvent(state, 0); + } return; } const parameters = recordValue(event.params); @@ -385,14 +401,32 @@ const ingestContext = ( : "destroyed", origin: origin === "" ? null : origin, }; - const bytes = metadataBytes(draft); - retainEvent(state, bytes); + const previous = state.contexts.get(key); + const bytes = + retainedMetadataBytes(draft) - + (previous === undefined ? 0 : retainedMetadataBytes(previous)); + if (!retainEvent(state, bytes)) return; state.contexts.set(key, draft); }; -const retainEvent = (state: CaptureState, bytes: number): void => { - state.metadataBytes += bytes; +const retainEvent = (state: CaptureState, bytes: number): boolean => { + if (!replaceRetainedMetadata(state, bytes)) return false; state.eventsRetained += 1; + return true; +}; + +const replaceRetainedMetadata = ( + state: CaptureState, + byteDelta: number, +): boolean => { + if (state.metadataBytes + byteDelta > INSPECTOR_MAX_RETAINED_METADATA_BYTES) { + state.eventsDropped += 1; + state.truncated = true; + state.truncationReasons.add("retained_metadata_budget_exceeded"); + return false; + } + state.metadataBytes += byteDelta; + return true; }; const waitForCapture = async ( @@ -402,14 +436,7 @@ const waitForCapture = async ( ): Promise => { let removeDisconnect = (): void => undefined; const disconnected = new Promise((_resolve, reject) => { - removeDisconnect = connection.onDisconnect(() => - reject( - new BrowserObservationError( - "observe_javascript_runtime", - "disconnected", - ), - ), - ); + removeDisconnect = connection.onDisconnect((error) => reject(error)); }); try { await Promise.race([ @@ -425,8 +452,16 @@ const waitForCapture = async ( } }; -const metadataBytes = (value: object): number => - Buffer.byteLength(JSON.stringify(value)); +const retainedMetadataBytes = (value: ScriptDraft | ContextDraft): number => { + const strings = + "rawUrl" in value + ? [value.rawUrl, value.rawUrl, value.executionContextKey, value.cdpHash] + : [value.contextKey, value.origin]; + return ( + RETAINED_OBJECT_OVERHEAD_BYTES + + strings.reduce((total, text) => total + (text?.length ?? 0) * 2, 0) + ); +}; const nonnegativeInteger = (value: unknown): number => { const parsed = numberValue(value); diff --git a/tests/boundary/inspector/javascriptRuntimeObservation.test.ts b/tests/boundary/inspector/javascriptRuntimeObservation.test.ts index 314a1268219..bb4169956b9 100644 --- a/tests/boundary/inspector/javascriptRuntimeObservation.test.ts +++ b/tests/boundary/inspector/javascriptRuntimeObservation.test.ts @@ -175,9 +175,10 @@ describe("complete Inspector script hashes", () => { scriptHashes, }); try { - const result = await new V8InspectorProvider().observe( - observeInput(fake.endpoint, fake.targetId, "node"), - ); + const result = await new V8InspectorProvider().observe({ + ...observeInput(fake.endpoint, fake.targetId, "node"), + observation_ms: 1_000, + }); if (!result.ok) throw result.error; expect(result.value.scripts.items).toHaveLength(scriptHashes.length); expect( @@ -212,7 +213,7 @@ describe("undeclared V8 runtime role", () => { }); }); -describe("passive V8 Inspector evidence", () => { +describe("V8 Inspector script retention", () => { test("retains authorized script locations longer than the former byte ceiling", async () => { const longUrl = `https://example.test/${"a".repeat(20_000)}`; const fake = await startFakeV8Inspector({ @@ -261,7 +262,69 @@ describe("passive V8 Inspector evidence", () => { await fake.close(); } }); +}); + +describe("bounded V8 Inspector capture", () => { + test("bounds aggregate script and context metadata while retaining a useful partial capture", async () => { + const fixture = await runtimeFixture(); + const url = pathToFileURL(fixture.entry).href; + const scriptHashes = Array.from( + { length: 2_300 }, + (_, index) => `${"h".repeat(4_096)}${String(index)}`, + ); + const fake = await startFakeV8Inspector({ + targetUrl: url, + scriptUrls: scriptHashes.map(() => url), + scriptHashes, + contextTransitionCount: 2_000, + }); + try { + const result = await new V8InspectorProvider().observe({ + ...observeInput(fake.endpoint, fake.targetId, "node"), + observation_ms: 1_000, + }); + if (!result.ok) throw result.error; + expect(result.value.capture.truncated).toBe(true); + expect(result.value.capture.truncation_reasons).toEqual([ + "retained_metadata_budget_exceeded", + ]); + expect(result.value.capture.events_dropped).toBeGreaterThan(0); + expect(result.value.capture.metadata_bytes_retained).toBeLessThanOrEqual( + 8 * 1024 * 1024, + ); + expect(result.value.capture.events_observed).toBeGreaterThan( + result.value.capture.events_retained, + ); + expect(result.value.execution_contexts).toHaveLength(1); + } finally { + await fake.close(); + } + }); + test("rejects a CDP event larger than the configured transport payload budget", async () => { + const fixture = await runtimeFixture(); + const fake = await startFakeV8Inspector({ + targetUrl: pathToFileURL(fixture.entry).href, + oversizedEventBytes: 16 * 1024 * 1024, + }); + try { + const result = await new V8InspectorProvider().observe( + observeInput(fake.endpoint, fake.targetId, "node"), + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ + _tag: "BrowserObservationError", + reason: "payload_limit", + }); + expect(result.error.userMessage).toContain("protocol budget"); + } finally { + await fake.close(); + } + }); +}); + +describe("passive V8 Inspector evidence", () => { test("produces deterministic Evidence through the service without grants", async () => { const fixture = await runtimeFixture(); const fake = await startFakeV8Inspector({ diff --git a/tests/fixtures/inspector/fakeV8Inspector.ts b/tests/fixtures/inspector/fakeV8Inspector.ts index 13445313fcd..ac698d2da16 100644 --- a/tests/fixtures/inspector/fakeV8Inspector.ts +++ b/tests/fixtures/inspector/fakeV8Inspector.ts @@ -14,6 +14,8 @@ interface FakeV8InspectorOptions { readonly runtimeProduct?: string; readonly scriptUrls?: readonly string[]; readonly scriptHashes?: readonly string[]; + readonly oversizedEventBytes?: number; + readonly contextTransitionCount?: number; readonly additionalTargetUrl?: string; readonly additionalTargetCount?: number; readonly closeOnMethod?: string; @@ -94,47 +96,7 @@ export const startFakeV8Inspector = async ( ); }); webSockets.on("connection", (socket: WebSocket) => { - sockets.add(socket); - socket.on("close", () => sockets.delete(socket)); - socket.on("message", (raw) => { - const command = parseCommand(raw.toString()); - commands.push(command); - if (options.closeOnMethod === command.method) { - socket.close(); - return; - } - socket.send(JSON.stringify({ id: command.id, result: {} })); - if (command.method === "Runtime.enable") - socket.send( - JSON.stringify({ - method: "Runtime.executionContextCreated", - params: { - context: { - id: 1, - origin: "", - name: "node[fixture]", - }, - }, - }), - ); - if (command.method === "Debugger.enable") - for (const [index, url] of ( - options.scriptUrls ?? [options.targetUrl] - ).entries()) - socket.send( - JSON.stringify({ - method: "Debugger.scriptParsed", - params: { - scriptId: String(index + 1), - url, - executionContextId: 1, - hash: options.scriptHashes?.[index] ?? `hash-${String(index)}`, - length: 100 + index, - isModule: index % 2 === 0, - }, - }), - ); - }); + registerInspectorSocket(socket, options, commands, sockets); }); await new Promise((resolve, reject) => { http.once("error", reject); @@ -160,6 +122,83 @@ export const startFakeV8Inspector = async ( }; }; +const registerInspectorSocket = ( + socket: WebSocket, + options: FakeV8InspectorOptions, + commands: InspectorCommand[], + sockets: Set, +): void => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("message", (raw) => { + const command = parseCommand(raw.toString()); + commands.push(command); + if (options.closeOnMethod === command.method) { + socket.close(); + return; + } + socket.send(JSON.stringify({ id: command.id, result: {} })); + respondToInspectorCommand(socket, command, options); + }); +}; + +const respondToInspectorCommand = ( + socket: WebSocket, + command: InspectorCommand, + options: FakeV8InspectorOptions, +): void => { + if (command.method === "Runtime.enable") sendRuntimeEvents(socket, options); + if (command.method === "Debugger.enable") sendScriptEvents(socket, options); +}; + +const sendRuntimeEvents = ( + socket: WebSocket, + options: FakeV8InspectorOptions, +): void => { + socket.send( + JSON.stringify({ + method: "Runtime.executionContextCreated", + params: { context: { id: 1, origin: "", name: "node[fixture]" } }, + }), + ); + for (let index = 0; index < (options.contextTransitionCount ?? 0); index += 1) + socket.send( + JSON.stringify({ + method: "Runtime.executionContextDestroyed", + params: { executionContextId: 1 }, + }), + ); + if (options.oversizedEventBytes !== undefined) + socket.send( + JSON.stringify({ + method: "Runtime.noisyFixtureEvent", + params: { padding: "x".repeat(options.oversizedEventBytes) }, + }), + ); +}; + +const sendScriptEvents = ( + socket: WebSocket, + options: FakeV8InspectorOptions, +): void => { + for (const [index, url] of ( + options.scriptUrls ?? [options.targetUrl] + ).entries()) + socket.send( + JSON.stringify({ + method: "Debugger.scriptParsed", + params: { + scriptId: String(index + 1), + url, + executionContextId: 1, + hash: options.scriptHashes?.[index] ?? `hash-${String(index)}`, + length: 100 + index, + isModule: index % 2 === 0, + }, + }), + ); +}; + const target = ( id: string, url: string, From 02d07027617a5df014f92a351aa9418aade6f06f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:19 +0800 Subject: [PATCH 034/791] fix(electron): bound hook retention and report dropped coverage --- docs/electron-observation.md | 15 +- scripts/electron-active-hook-boundaries.cjs | 32 ++- scripts/electron-active-hook.cjs | 216 ++++++++++++++++-- .../PlaywrightElectronActiveActions.test.ts | 13 ++ .../PlaywrightElectronActiveActions.ts | 28 +++ ...PlaywrightElectronActiveProjection.test.ts | 129 +++++++++++ .../PlaywrightElectronActiveProjection.ts | 178 +++++++++++++++ .../PlaywrightElectronActiveProvider.ts | 154 +------------ .../electronActiveObservation.fixture.ts | 14 ++ .../javascript/electronActiveObservation.ts | 30 ++- .../browser/electronActiveHook.test.ts | 41 +++- 11 files changed, 658 insertions(+), 192 deletions(-) create mode 100644 src/browser/PlaywrightElectronActiveProjection.test.ts create mode 100644 src/browser/PlaywrightElectronActiveProjection.ts diff --git a/docs/electron-observation.md b/docs/electron-observation.md index 60ab80deb2e..62ade70954c 100644 --- a/docs/electron-observation.md +++ b/docs/electron-observation.md @@ -86,9 +86,18 @@ rea capture-electron-scenario scenario.json --json The result records action status and targets, correlated app/window/WebContents, preload, session, navigation, shell, permission, popup, download, protocol, -native-addon, process, and IPC timeline events. IPC channels are capped at -1,024 characters and argument-shape metadata at 32 entries; values are never -retained. Renderer crash/restart and deep-link actions are synthetic +native-addon, process, and IPC timeline events. Event retention has a 16 MiB +accounting budget for full channel strings, argument-shape arrays, and an +estimated object-graph cost. Admission computes an exact JSON byte length +incrementally, then charges twice that size plus 256 bytes per event and 8 bytes +per argument shape. The doubled JSON charge covers the timeline and IPC views; +the additional charge estimates their in-process object and array storage. This +is a resource estimate, not an exact V8 heap measurement. An event that does not +fit is dropped whole. The result reports the estimate, serialized-byte upper +bound, dropped event families, and +observed/dropped totals for IPC and runtime events. Retained sequence numbers +keep their original capture identities. Payload values are never retained. +Renderer crash/restart and deep-link actions are synthetic scenario controls; their attempted and observed outcomes remain in the timeline. The active hook blocks and records external shell/navigation, permission, download, popup, updater, and OS-integration effects. The timeline diff --git a/scripts/electron-active-hook-boundaries.cjs b/scripts/electron-active-hook-boundaries.cjs index a312ae2b497..81d199a188d 100644 --- a/scripts/electron-active-hook-boundaries.cjs +++ b/scripts/electron-active-hook-boundaries.cjs @@ -61,7 +61,7 @@ const createElectronActiveBoundaryPatches = ({ const target = typeof args[0] === "string" ? args[0] : null; recordRuntime("preload", "context-bridge-exposure", "attempted", { target, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, process_type: processType(), }); try { @@ -110,7 +110,7 @@ const createElectronActiveBoundaryPatches = ({ event === "will-download" ? "blocked" : "observed", { target: identity(args[0], "webContents"), - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: event === "will-download", }, ); @@ -124,7 +124,7 @@ const createElectronActiveBoundaryPatches = ({ (webContents, permission, callback) => { recordRuntime("permission", "permission-request", "blocked", { target: identity(webContents, "webContents"), - argument_shapes: [shape(permission)], + argument_shape_values: [permission], error: true, }); if (typeof callback === "function") callback(false); @@ -137,7 +137,7 @@ const createElectronActiveBoundaryPatches = ({ return originalPermissionCheck.call(this, (webContents, permission) => { recordRuntime("permission", "permission-check", "blocked", { target: identity(webContents, "webContents"), - argument_shapes: [shape(permission)], + argument_shape_values: [permission], error: true, }); return false; @@ -148,7 +148,7 @@ const createElectronActiveBoundaryPatches = ({ session.setCertificateVerifyProc = function blockedCertificate() { return originalCertificate.call(this, (request, callback) => { recordRuntime("permission", "certificate-error", "blocked", { - argument_shapes: [shape(request)], + argument_shape_values: [request], error: true, }); if (typeof callback === "function") callback(-2); @@ -168,14 +168,14 @@ const createElectronActiveBoundaryPatches = ({ } catch { recordRuntime("popup-attempt", "window-open-handler", "failed", { target: contentsId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: true, }); return { action: "deny" }; } recordRuntime("popup-attempt", "window-open-handler", "blocked", { target: contentsId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: true, }); return { action: "deny" }; @@ -223,7 +223,7 @@ const createElectronActiveBoundaryPatches = ({ method === "relaunch" ? "updater" : "protocol", method, "blocked", - { argument_shapes: args.map((value) => shape(value)), error: true }, + { argument_shape_values: args, error: true }, ); return method === "setAsDefaultProtocolClient" ? false : undefined; }; @@ -250,11 +250,7 @@ const createElectronActiveBoundaryPatches = ({ recordRuntime("native-addon", "dlopen", "attempted", { artifact_path: typeof filename === "string" ? filename : null, artifact_sha256: digest, - argument_shapes: [ - shape(module), - shape(filename), - ...args.map((value) => shape(value)), - ], + argument_shape_value_groups: [[module, filename], args], }); try { const result = original.call(this, module, filename, ...args); @@ -280,7 +276,7 @@ const createElectronActiveBoundaryPatches = ({ if (typeof original !== "function") return; utilityProcess.fork = function patchedFork(...args) { recordRuntime("process-lifecycle", "utility-process-fork", "attempted", { - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, process_type: "main", }); const child = original.apply(this, args); @@ -304,7 +300,7 @@ const createElectronActiveBoundaryPatches = ({ recordRuntime("process-lifecycle", `utility.${event}`, "observed", { target: childId, process_type: "utility", - argument_shapes: eventArgs.map((value) => shape(value)), + argument_shape_values: eventArgs, }); }); if (child && typeof child.postMessage === "function") { @@ -344,7 +340,7 @@ const createElectronActiveBoundaryPatches = ({ if (typeof original !== "function") continue; shell[method] = function blockedShellCall(...args) { recordRuntime("shell-attempt", method, "blocked", { - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: true, }); return Promise.reject(new Error("External shell effects are blocked")); @@ -359,7 +355,7 @@ const createElectronActiveBoundaryPatches = ({ if (typeof original !== "function") continue; childProcess[method] = function patchedChildProcess(...args) { recordRuntime("process-lifecycle", `child.${method}`, "attempted", { - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, }); const child = original.apply(this, args); const childId = identity(child, "child"); @@ -369,7 +365,7 @@ const createElectronActiveBoundaryPatches = ({ patchEmitter(child, (event, eventArgs) => recordRuntime("process-lifecycle", `child.${event}`, "observed", { target: childId, - argument_shapes: eventArgs.map((value) => shape(value)), + argument_shape_values: eventArgs, }), ); return child; diff --git a/scripts/electron-active-hook.cjs b/scripts/electron-active-hook.cjs index 4185ffc782a..8d7766c9560 100644 --- a/scripts/electron-active-hook.cjs +++ b/scripts/electron-active-hook.cjs @@ -16,9 +16,19 @@ const ipcEventKinds = new Set([ "ipc-renderer-post-message", ]); const events = []; +// Budget serialized data and estimated retained object/array storage together. +const retentionBudgetBytes = 16 * 1024 * 1024; +const snapshotEnvelopeReserveBytes = 1024; +let estimatedRetainedBytes = 0; +let serializedBytes = 0; let observed = 0; let observedIpc = 0; let observedRuntime = 0; +let dropped = 0; +let droppedIpc = 0; +let droppedRuntime = 0; +const droppedFamilies = new Map(); +const droppedRoles = new Map(); let sequence = 0; let correlationSequence = 0; @@ -43,6 +53,73 @@ const shape = (value, depth = 0) => { } }; +// Measure JSON strings without allocating an encoded copy of a caller value. +const jsonStringBytes = (value) => { + let bytes = 2; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code === 0x22 || code === 0x5c) bytes += 2; + else if ( + code === 0x08 || + code === 0x09 || + code === 0x0a || + code === 0x0c || + code === 0x0d + ) + bytes += 2; + else if (code <= 0x1f) bytes += 6; + else if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(index + 1); + if (next >= 0xdc00 && next <= 0xdfff) { + bytes += 4; + index += 1; + } else bytes += 6; + } else if (code >= 0xdc00 && code <= 0xdfff) bytes += 6; + else if (code <= 0x7f) bytes += 1; + else if (code <= 0x7ff) bytes += 2; + else bytes += 3; + if (bytes > retentionBudgetBytes) return bytes; + } + return bytes; +}; + +const jsonScalarBytes = (value) => { + if (value === null) return 4; + if (typeof value === "string") return jsonStringBytes(value); + if (typeof value === "boolean") return value ? 4 : 5; + return String(value).length; +}; + +const eventFamily = (kind, processTypeValue) => { + if (kind === "preload" && processTypeValue === "main") + return "preload-configuration"; + if (kind.startsWith("ipc-renderer")) + return processTypeValue === "renderer" ? "renderer-ipc" : "ipc"; + return ipcEventKinds.has(kind) ? "ipc" : kind; +}; + +const noteDropped = (kind, processTypeValue, isIpc) => { + dropped += 1; + if (isIpc) droppedIpc += 1; + else droppedRuntime += 1; + const family = eventFamily(kind, processTypeValue); + droppedFamilies.set(family, (droppedFamilies.get(family) ?? 0) + 1); + if (typeof processTypeValue === "string") + droppedRoles.set( + processTypeValue, + (droppedRoles.get(processTypeValue) ?? 0) + 1, + ); + if (kind === "window-lifecycle") + droppedRoles.set("window", (droppedRoles.get("window") ?? 0) + 1); + if (kind === "web-contents-lifecycle" || kind === "navigation") + droppedRoles.set( + "web_contents", + (droppedRoles.get("web_contents") ?? 0) + 1, + ); + if (kind === "preload" && processTypeValue === "preload") + droppedRoles.set("preload", (droppedRoles.get("preload") ?? 0) + 1); +}; + const isAllowedNavigation = (value) => { if (typeof value !== "string") return true; try { @@ -83,7 +160,8 @@ const frameIdentity = (event) => { const record = (event) => { observed += 1; - if (ipcEventKinds.has(event.kind)) observedIpc += 1; + const isIpc = ipcEventKinds.has(event.kind); + if (isIpc) observedIpc += 1; else observedRuntime += 1; const raw = { sequence: ++sequence, @@ -109,10 +187,15 @@ const record = (event) => { error: false, ...event, }; - const argumentShapes = Array.isArray(raw.argument_shapes) + const shapeGroups = Array.isArray(raw.argument_shape_value_groups) + ? raw.argument_shape_value_groups.filter(Array.isArray) + : Array.isArray(raw.argument_shape_values) + ? [raw.argument_shape_values] + : []; + const existingShapes = Array.isArray(raw.argument_shapes) ? raw.argument_shapes : []; - events.push({ + const retainedEvent = { ...raw, correlation_id: raw.correlation_id, event: typeof raw.event === "string" ? raw.event : null, @@ -121,9 +204,7 @@ const record = (event) => { receiver: typeof raw.receiver === "string" ? raw.receiver : null, frame: typeof raw.frame === "string" ? raw.frame : null, target: typeof raw.target === "string" ? raw.target : null, - argument_shapes: argumentShapes.map((value) => - typeof value === "string" ? value : "unknown", - ), + argument_shapes: [], result_shape: typeof raw.result_shape === "string" ? raw.result_shape : null, process_type: @@ -132,8 +213,91 @@ const record = (event) => { typeof raw.artifact_path === "string" ? raw.artifact_path : null, artifact_sha256: typeof raw.artifact_sha256 === "string" ? raw.artifact_sha256 : null, - }); - return events.at(-1); + }; + delete retainedEvent.argument_shape_values; + delete retainedEvent.argument_shape_value_groups; + + // All event values are JSON scalars except argument_shapes. Measure them + // incrementally so large channels, paths, or shape lists never require a + // full JSON.stringify temporary allocation before the budget decision. + let eventBytes = 2; + let argumentShapeCount = 0; + const eventKeys = Object.keys(retainedEvent); + for (let index = 0; index < eventKeys.length; index += 1) { + const key = eventKeys[index]; + const value = retainedEvent[key]; + if (index > 0) eventBytes += 1; + eventBytes += jsonStringBytes(key) + 1; + if (key === "argument_shapes") { + eventBytes += 2; + let shapeOverflow = false; + const countShape = (argument) => { + const encodedShapeBytes = jsonStringBytes(shape(argument)); + eventBytes += encodedShapeBytes + (argumentShapeCount === 0 ? 0 : 1); + argumentShapeCount += 1; + shapeOverflow = + eventBytes + estimatedRetainedBytes + snapshotEnvelopeReserveBytes > + retentionBudgetBytes; + }; + for (const group of shapeGroups) + for (const argument of group) { + countShape(argument); + if (shapeOverflow) break; + } + if (shapeOverflow) { + noteDropped(raw.kind, retainedEvent.process_type, isIpc); + return null; + } + for (const valueShape of existingShapes) + if (!shapeOverflow) { + eventBytes += + jsonStringBytes( + typeof valueShape === "string" ? valueShape : "unknown", + ) + (argumentShapeCount++ === 0 ? 0 : 1); + shapeOverflow = + eventBytes + estimatedRetainedBytes + snapshotEnvelopeReserveBytes > + retentionBudgetBytes; + } + if (shapeOverflow) { + noteDropped(raw.kind, retainedEvent.process_type, isIpc); + return null; + } + } else eventBytes += jsonScalarBytes(value); + if ( + eventBytes + estimatedRetainedBytes + snapshotEnvelopeReserveBytes > + retentionBudgetBytes + ) { + noteDropped(raw.kind, retainedEvent.process_type, isIpc); + return null; + } + } + // Reserve room for the only fields mutated after recording: result_shape and + // error. The shape vocabulary's longest value is "undefined". + const reservedMutationBytes = retainedEvent.result_shape === null ? 7 : 0; + // Include one byte for the event-array separator and reserve space for the + // snapshot's counters and JSON delimiters. + // One retained event, one defensive snapshot copy, and up to two serialized + // result views (timeline plus IPC), with per-object and per-array-slot costs. + const eventEstimateBytes = eventBytes * 3 + 512 + argumentShapeCount * 16; + const retainedEventBytes = eventEstimateBytes + reservedMutationBytes + 1; + if ( + estimatedRetainedBytes + retainedEventBytes + snapshotEnvelopeReserveBytes > + retentionBudgetBytes + ) { + noteDropped(raw.kind, retainedEvent.process_type, isIpc); + return null; + } + for (const group of shapeGroups) + for (const argument of group) + retainedEvent.argument_shapes.push(shape(argument)); + for (const valueShape of existingShapes) + retainedEvent.argument_shapes.push( + typeof valueShape === "string" ? valueShape : "unknown", + ); + events.push(retainedEvent); + estimatedRetainedBytes += retainedEventBytes; + serializedBytes += eventBytes + reservedMutationBytes + 1; + return retainedEvent; }; const recordRuntime = (kind, event, phase, details = {}) => @@ -145,7 +309,7 @@ const recordIpc = (kind, channel, args, details = {}) => return record({ kind, channel: typeof channel === "string" ? channel : null, - argument_shapes: values.map((value) => shape(value)), + argument_shape_values: values, ...details, }); })(); @@ -222,7 +386,7 @@ const patchNavigationMethods = (webContents, contentsId) => { webContents[method] = function patchedNavigation(...args) { recordRuntime("navigation", method, "attempted", { target: contentsId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, }); if (method === "loadURL" && !isAllowedNavigation(args[0])) { recordRuntime("navigation", method, "blocked", { @@ -354,7 +518,7 @@ const patchWebContents = (webContents, windowId) => { recordRuntime("navigation", event, "blocked", { target: contentsId, sender: windowId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: true, }); return; @@ -363,7 +527,7 @@ const patchWebContents = (webContents, windowId) => { recordRuntime(kind, event, "observed", { target: contentsId, sender: windowId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, error: kind === "error", }); if (event === "did-create-window") { @@ -388,7 +552,7 @@ const patchBrowserWindow = (electron) => { const targetId = identity(window, "window"); recordRuntime("window-lifecycle", "created", "completed", { target: targetId, - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, }); const preload = args[0]?.webPreferences?.preload; if (typeof preload === "string") @@ -427,7 +591,7 @@ try { "observed", { target: "app", - argument_shapes: args.map((value) => shape(value)), + argument_shape_values: args, process_type: "main", }, ), @@ -444,14 +608,14 @@ try { boundaries.patchChildProcess(); process.once("uncaughtException", (cause, origin) => { recordRuntime("process-lifecycle", "uncaught-exception", "observed", { - argument_shapes: [shape(cause), shape(origin)], + argument_shape_values: [cause, origin], error: true, }); throw cause; }); process.once("unhandledRejection", (reason, promise) => { recordRuntime("process-lifecycle", "unhandled-rejection", "observed", { - argument_shapes: [shape(reason), shape(promise)], + argument_shape_values: [reason, promise], error: true, }); throw reason; @@ -461,7 +625,25 @@ try { } globalThis.__reaElectronActiveSnapshot = () => ({ - events: events.slice(), + events: events.map((event) => ({ + ...event, + argument_shapes: [...event.argument_shapes], + })), + retention_budget_bytes: retentionBudgetBytes, + estimated_retained_bytes: estimatedRetainedBytes, + event_serialized_byte_upper_bound: serializedBytes, + retained: events.length, + dropped, + dropped_ipc: droppedIpc, + dropped_runtime: droppedRuntime, + dropped_event_families: [...droppedFamilies].map(([family, count]) => ({ + family, + count, + })), + dropped_event_roles: [...droppedRoles].map(([role, count]) => ({ + role, + count, + })), observed, observed_ipc: observedIpc, observed_runtime: observedRuntime, diff --git a/src/browser/PlaywrightElectronActiveActions.test.ts b/src/browser/PlaywrightElectronActiveActions.test.ts index c8890cd4579..5f78a29b8e3 100644 --- a/src/browser/PlaywrightElectronActiveActions.test.ts +++ b/src/browser/PlaywrightElectronActiveActions.test.ts @@ -9,6 +9,19 @@ import { import { createElectronActiveObservationFixtureResult } from "../domain/javascript/electronActiveObservation.fixture.js"; import { runElectronActions } from "./PlaywrightElectronActiveActions.js"; +it("keeps legacy Electron results explicit about unavailable retention data", () => { + const { retention: _retention, ...legacyResult } = + createElectronActiveObservationFixtureResult("/opt/app"); + const parsed = electronActiveObservationResultSchema.parse({ + ...legacyResult, + ipc: { ...legacyResult.ipc, dropped: undefined }, + timeline: { ...legacyResult.timeline, dropped: undefined }, + }); + expect(parsed.retention).toBeNull(); + expect(parsed.ipc.dropped).toBeNull(); + expect(parsed.timeline.dropped).toBeNull(); +}); + it("parses window, renderer, and deep-link actions for agents", () => { const input = electronActiveObservationInputSchema.parse({ executable_path: "/opt/electron", diff --git a/src/browser/PlaywrightElectronActiveActions.ts b/src/browser/PlaywrightElectronActiveActions.ts index 2c4a71183ff..bb1469e4650 100644 --- a/src/browser/PlaywrightElectronActiveActions.ts +++ b/src/browser/PlaywrightElectronActiveActions.ts @@ -68,6 +68,25 @@ const hookEventSchema = z.strictObject({ const hookSnapshotSchema = z.strictObject({ events: z.array(hookEventSchema), + retention_budget_bytes: z.number().int().min(0), + estimated_retained_bytes: z.number().int().min(0), + event_serialized_byte_upper_bound: z.number().int().min(0), + retained: z.number().int().min(0), + dropped: z.number().int().min(0), + dropped_ipc: z.number().int().min(0), + dropped_runtime: z.number().int().min(0), + dropped_event_families: z.array( + z.strictObject({ + family: z.string().min(1), + count: z.number().int().min(1), + }), + ), + dropped_event_roles: z.array( + z.strictObject({ + role: z.string().min(1), + count: z.number().int().min(1), + }), + ), observed: z.number().int().min(0), observed_ipc: z.number().int().min(0), observed_runtime: z.number().int().min(0), @@ -357,6 +376,15 @@ export const readApplicationState = async ( ? candidate() : { events: [], + retention_budget_bytes: 0, + estimated_retained_bytes: 0, + event_serialized_byte_upper_bound: 0, + retained: 0, + dropped: 0, + dropped_ipc: 0, + dropped_runtime: 0, + dropped_event_families: [], + dropped_event_roles: [], observed: 0, observed_ipc: 0, observed_runtime: 0, diff --git a/src/browser/PlaywrightElectronActiveProjection.test.ts b/src/browser/PlaywrightElectronActiveProjection.test.ts new file mode 100644 index 00000000000..baec9b21947 --- /dev/null +++ b/src/browser/PlaywrightElectronActiveProjection.test.ts @@ -0,0 +1,129 @@ +import { expect, it } from "vitest"; + +import { projectElectronActiveCapture } from "./PlaywrightElectronActiveProjection.js"; + +it("projects hook retention and dropped event families through the result schema", () => { + const event = { + sequence: 1, + correlation_id: "capture:123:1", + kind: "main-handler-invocation" as const, + event: null, + phase: "observed" as const, + channel: "readiness:echo", + direction: "renderer-to-main" as const, + sender: null, + receiver: "main", + frame: null, + target: null, + argument_shapes: [], + result_shape: null, + process_type: "main", + source: "electron-active-hook", + capture_method: "api-wrapper" as const, + artifact_path: null, + artifact_sha256: null, + error: false, + }; + const result = projectElectronActiveCapture( + { + executable: "/opt/electron", + application: "/opt/app/main.js", + }, + [], + { + windows: [], + metrics: [], + electronVersion: "test-electron", + hookSnapshot: { + events: [event], + retention_budget_bytes: 16 * 1024 * 1024, + estimated_retained_bytes: 1_024, + event_serialized_byte_upper_bound: 512, + retained: 1, + dropped: 1, + dropped_ipc: 0, + dropped_runtime: 1, + dropped_event_families: [{ family: "renderer-ipc", count: 1 }], + dropped_event_roles: [{ role: "renderer", count: 1 }], + observed: 2, + observed_ipc: 1, + observed_runtime: 1, + hook_error: false, + }, + }, + ); + + expect(result.retention).toMatchObject({ + budget_bytes: 16 * 1024 * 1024, + estimated_retained_bytes: 1_024, + event_serialized_byte_upper_bound: 512, + retained: 1, + dropped: 1, + dropped_event_families: [{ family: "renderer-ipc", count: 1 }], + dropped_event_roles: [{ role: "renderer", count: 1 }], + }); + expect(result.ipc).toMatchObject({ observed: 1, dropped: 0 }); + expect(result.timeline).toMatchObject({ observed: 2, dropped: 1 }); + expect(result.coverage.observed_event_families).toContain("renderer-ipc"); + expect(result.coverage.unavailable_event_families).not.toContain( + "renderer-ipc", + ); + expect(result.coverage.observed_roles).toContain("renderer"); + expect(result.coverage.unavailable_event_families).toContain("preload"); +}); + +it("keeps renderer coverage unavailable when only the main process was observed", () => { + const result = projectElectronActiveCapture( + { + executable: "/opt/electron", + application: "/opt/app/main.js", + }, + [], + { + windows: [], + metrics: [], + electronVersion: "test-electron", + hookSnapshot: { + events: [ + { + sequence: 1, + correlation_id: "capture:123:1", + kind: "app-lifecycle", + event: "ready", + phase: "observed", + channel: null, + direction: null, + sender: null, + receiver: null, + frame: null, + target: "app", + argument_shapes: [], + result_shape: null, + process_type: "main", + source: "electron-active-hook", + capture_method: "event-emitter", + artifact_path: null, + artifact_sha256: null, + error: false, + }, + ], + retention_budget_bytes: 16 * 1024 * 1024, + estimated_retained_bytes: 0, + event_serialized_byte_upper_bound: 0, + retained: 1, + dropped: 0, + dropped_ipc: 0, + dropped_runtime: 0, + dropped_event_families: [], + dropped_event_roles: [], + observed: 1, + observed_ipc: 0, + observed_runtime: 1, + hook_error: false, + }, + }, + ); + + expect(result.coverage.observed_roles).not.toContain("renderer"); + expect(result.coverage.unavailable_event_families).toContain("renderer-ipc"); +}); diff --git a/src/browser/PlaywrightElectronActiveProjection.ts b/src/browser/PlaywrightElectronActiveProjection.ts new file mode 100644 index 00000000000..d105d560635 --- /dev/null +++ b/src/browser/PlaywrightElectronActiveProjection.ts @@ -0,0 +1,178 @@ +import { + electronActiveObservationResultSchema, + type ElectronActiveObservationResult, +} from "../domain/javascript/electronActiveObservation.js"; +import type { + ElectronHookEvent, + ElectronHookSnapshot, + ElectronMetrics, +} from "./PlaywrightElectronActiveActions.js"; + +type ElectronCapturePaths = { + readonly executable: string; + readonly application: string; +}; + +type ElectronCaptureState = { + readonly windows: ReadonlyArray< + ElectronActiveObservationResult["windows"][number] + >; + readonly metrics: ElectronMetrics; + readonly electronVersion: string; + readonly hookSnapshot: ElectronHookSnapshot; +}; + +const observableEventFamilies = [ + "app-lifecycle", + "window-lifecycle", + "web-contents-lifecycle", + "navigation", + "shell-attempt", + "process-lifecycle", + "permission", + "popup-attempt", + "download", + "protocol", + "preload", + "preload-configuration", + "renderer-ipc", + "native-addon", + "updater", + "error", + "ipc", +] as const; + +const ipcEventKinds = new Set([ + "main-handler-invocation", + "main-event-invocation", + "utility-process-fork", + "utility-process-message", + "ipc-main-to-renderer", + "ipc-utility-to-main", + "ipc-renderer-send", + "ipc-renderer-invoke", + "ipc-renderer-post-message", +]); + +const eventFamily = ( + kind: ElectronHookEvent["kind"], + processType: string | null, +): string => { + if (kind === "preload" && processType === "main") + return "preload-configuration"; + if (kind.startsWith("ipc-renderer")) + return processType === "renderer" ? "renderer-ipc" : "ipc"; + return ipcEventKinds.has(kind) ? "ipc" : kind; +}; + +const createCoverage = (hookSnapshot: ElectronHookSnapshot) => { + const observedEventFamilies: string[] = [ + ...new Set([ + ...hookSnapshot.events.map(({ kind, process_type }) => + eventFamily(kind, process_type), + ), + ...hookSnapshot.dropped_event_families.map(({ family }) => family), + ]), + ].sort(); + const unavailableEventFamilies = new Set( + observableEventFamilies.filter( + (family) => !observedEventFamilies.includes(family), + ), + ); + const observedRoles = [ + ...new Set([ + ...hookSnapshot.events.flatMap(({ process_type, kind }) => [ + ...(process_type === null ? [] : [process_type]), + ...(kind === "window-lifecycle" ? ["window"] : []), + ...(kind === "web-contents-lifecycle" || kind === "navigation" + ? ["web_contents"] + : []), + ...(kind === "preload" && process_type === "preload" + ? ["preload"] + : []), + ...(kind.startsWith("ipc-renderer") && process_type === "renderer" + ? ["renderer"] + : []), + ]), + ...hookSnapshot.dropped_event_roles.map(({ role }) => role), + ]), + ].sort(); + if (!observedRoles.includes("preload")) + unavailableEventFamilies.add("preload"); + if (!observedRoles.includes("renderer")) + unavailableEventFamilies.add("renderer-ipc"); + return { + status: hookSnapshot.hook_error ? "hook_conflict" : "partial_attach", + observed_event_families: observedEventFamilies, + unavailable_event_families: [...unavailableEventFamilies].sort(), + observed_roles: observedRoles, + pre_capture_activity: "unavailable", + } as const; +}; + +/** Project one provider hook snapshot into the canonical public result. */ +export const projectElectronActiveCapture = ( + paths: ElectronCapturePaths, + actions: ElectronActiveObservationResult["actions"], + state: ElectronCaptureState, +): ElectronActiveObservationResult => { + const { hookSnapshot } = state; + const ipcEvents = hookSnapshot.events.filter(({ kind }) => + ipcEventKinds.has(kind), + ); + return electronActiveObservationResultSchema.parse({ + application: { + executable_path: paths.executable, + application_path: paths.application, + electron_version: state.electronVersion, + process_ownership: "provider-owned", + cleanup: "terminated-owned-process", + }, + actions, + windows: state.windows, + processes: { + items: state.metrics, + }, + ipc: { + events: ipcEvents, + observed: hookSnapshot.observed_ipc, + dropped: hookSnapshot.dropped_ipc, + }, + timeline: { + events: hookSnapshot.events, + observed: hookSnapshot.observed, + dropped: hookSnapshot.dropped, + }, + retention: { + budget_bytes: hookSnapshot.retention_budget_bytes, + estimated_retained_bytes: hookSnapshot.estimated_retained_bytes, + event_serialized_byte_upper_bound: + hookSnapshot.event_serialized_byte_upper_bound, + retained: hookSnapshot.retained, + dropped: hookSnapshot.dropped, + dropped_event_families: hookSnapshot.dropped_event_families, + dropped_event_roles: hookSnapshot.dropped_event_roles, + observed_ipc: hookSnapshot.observed_ipc, + dropped_ipc: hookSnapshot.dropped_ipc, + observed_runtime: hookSnapshot.observed_runtime, + dropped_runtime: hookSnapshot.dropped_runtime, + }, + coverage: createCoverage(hookSnapshot), + limitations: [ + "IPC payloads are represented by value shapes; payload values are never retained.", + "IPC direction and sender/receiver identifiers are observed only where Electron exposes them at the hooked boundary.", + "The runtime timeline records lifecycle, navigation, shell, permission, popup, download, protocol, preload, native-addon, process, and IPC events; activity before hook installation is unavailable.", + "The preload and renderer process contexts are not instrumented by the main-process -r hook; preload configuration and contextBridge API-shape events are main-boundary observations, not proof of renderer-side execution.", + "Process metrics are an Electron API snapshot and do not prove hostile-local-user isolation.", + ...(hookSnapshot.dropped > 0 + ? [ + `The active hook retained ${hookSnapshot.retained} of ${hookSnapshot.observed} observed events within a ${hookSnapshot.retention_budget_bytes}-byte retained-data budget; ${hookSnapshot.dropped} events were dropped (${hookSnapshot.dropped_ipc} IPC, ${hookSnapshot.dropped_runtime} runtime).`, + ] + : []), + "External shell opens, external navigation, permission grants, downloads, popup windows, updater relaunches, and OS integration are blocked and recorded by the active hook; other application filesystem and network behavior is not sandboxed by this provider.", + ...(hookSnapshot.hook_error + ? ["The active IPC hook could not be installed."] + : []), + ], + }); +}; diff --git a/src/browser/PlaywrightElectronActiveProvider.ts b/src/browser/PlaywrightElectronActiveProvider.ts index 5aa0ef88cbf..590376955be 100644 --- a/src/browser/PlaywrightElectronActiveProvider.ts +++ b/src/browser/PlaywrightElectronActiveProvider.ts @@ -10,10 +10,9 @@ import type { ProviderIdentity, } from "../application/AnalysisProvider.js"; import type { ElectronActiveObservationPort } from "../application/javascript/ElectronActiveObservationPort.js"; -import { - electronActiveObservationResultSchema, - type ElectronActiveObservationInput, - type ElectronActiveObservationResult, +import type { + ElectronActiveObservationInput, + ElectronActiveObservationResult, } from "../domain/javascript/electronActiveObservation.js"; import { AnalysisError } from "../domain/analysisErrorBase.js"; import { BrowserObservationError } from "../domain/browserObservationError.js"; @@ -38,10 +37,8 @@ import { runElectronActions, readApplicationState, runWithExecutionLimits, - type ElectronHookEvent, - type ElectronHookSnapshot, - type ElectronMetrics, } from "./PlaywrightElectronActiveActions.js"; +import { projectElectronActiveCapture } from "./PlaywrightElectronActiveProjection.js"; const OPERATION = "capture_electron_scenario" as const; const STARTUP_TIMEOUT_MS = 60_000; @@ -61,89 +58,6 @@ type ElectronPaths = { readonly root: string; }; -const observableEventFamilies = [ - "app-lifecycle", - "window-lifecycle", - "web-contents-lifecycle", - "navigation", - "shell-attempt", - "process-lifecycle", - "permission", - "popup-attempt", - "download", - "protocol", - "preload", - "preload-configuration", - "renderer-ipc", - "native-addon", - "updater", - "error", - "ipc", -] as const; - -const ipcEventKinds = new Set([ - "main-handler-invocation", - "main-event-invocation", - "utility-process-fork", - "utility-process-message", - "ipc-main-to-renderer", - "ipc-utility-to-main", - "ipc-renderer-send", - "ipc-renderer-invoke", - "ipc-renderer-post-message", -]); - -const createCoverage = (hookSnapshot: ElectronHookSnapshot) => { - const observedEventFamilies: string[] = [ - ...new Set( - hookSnapshot.events.map(({ kind, process_type }) => { - if (kind === "preload" && process_type === "main") - return "preload-configuration"; - if (kind.startsWith("ipc-renderer") && process_type !== "renderer") - return "ipc"; - return ipcEventKinds.has(kind) ? "ipc" : kind; - }), - ), - ].sort(); - const unavailableEventFamilies = new Set( - observableEventFamilies.filter( - (family) => !observedEventFamilies.includes(family), - ), - ); - const observedRoles = [ - ...new Set( - hookSnapshot.events.flatMap(({ process_type, kind }) => [ - ...(process_type === null ? [] : [process_type]), - ...(kind === "window-lifecycle" ? ["window"] : []), - ...(kind === "web-contents-lifecycle" || kind === "navigation" - ? ["web_contents"] - : []), - ...(kind === "preload" && process_type === "preload" - ? ["preload"] - : []), - ...(kind.startsWith("ipc-renderer") && process_type === "renderer" - ? ["renderer"] - : []), - ]), - ), - ].sort(); - const rendererContextObserved = hookSnapshot.events.some( - ({ process_type }) => - process_type === "renderer" || process_type === "preload", - ); - if (!rendererContextObserved) { - unavailableEventFamilies.add("preload"); - unavailableEventFamilies.add("renderer-ipc"); - } - return { - status: hookSnapshot.hook_error ? "hook_conflict" : "partial_attach", - observed_event_families: observedEventFamilies, - unavailable_event_families: [...unavailableEventFamilies].sort(), - observed_roles: observedRoles, - pre_capture_activity: "unavailable", - } as const; -}; - /** Launch an owned Electron application through the official Playwright API. */ export class PlaywrightElectronActiveProvider implements ElectronActiveObservationPort { identity(): ProviderIdentity { @@ -200,7 +114,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati readApplicationState(application), options.signal, ); - outcome = ok(createResult(paths, input, actions, state)); + outcome = ok(projectElectronActiveCapture(paths, actions, state)); } catch (cause: unknown) { outcome = err( options.signal?.aborted === true && @@ -339,64 +253,6 @@ const cleanupElectronProcesses = async ( }; }; -const createResult = ( - paths: ElectronPaths, - input: ElectronActiveObservationInput, - actions: ElectronActiveObservationResult["actions"], - state: { - readonly windows: ReadonlyArray<{ - readonly window_id: string; - readonly web_contents_id: string; - readonly url: string; - readonly title: string; - readonly visible: boolean | null; - readonly destroyed: boolean; - }>; - readonly metrics: ElectronMetrics; - readonly electronVersion: string; - readonly hookSnapshot: ElectronHookSnapshot; - }, -): ElectronActiveObservationResult => { - const { hookSnapshot } = state; - const ipcEvents = hookSnapshot.events.filter(({ kind }) => - ipcEventKinds.has(kind), - ); - return electronActiveObservationResultSchema.parse({ - application: { - executable_path: paths.executable, - application_path: paths.application, - electron_version: state.electronVersion, - process_ownership: "provider-owned", - cleanup: "terminated-owned-process", - }, - actions, - windows: state.windows, - processes: { - items: state.metrics, - }, - ipc: { - events: ipcEvents, - observed: hookSnapshot.observed_ipc, - }, - timeline: { - events: hookSnapshot.events, - observed: hookSnapshot.observed, - }, - coverage: createCoverage(hookSnapshot), - limitations: [ - "IPC payloads are represented by value shapes; payload values are never retained.", - "IPC direction and sender/receiver identifiers are observed only where Electron exposes them at the hooked boundary.", - "The runtime timeline records lifecycle, navigation, shell, permission, popup, download, protocol, preload, native-addon, process, and IPC events; activity before hook installation is unavailable.", - "The preload and renderer process contexts are not instrumented by the main-process -r hook; preload configuration and contextBridge API-shape events are main-boundary observations, not proof of renderer-side execution.", - "Process metrics are an Electron API snapshot and do not prove hostile-local-user isolation.", - "External shell opens, external navigation, permission grants, downloads, popup windows, updater relaunches, and OS integration are blocked and recorded by the active hook; other application filesystem and network behavior is not sandboxed by this provider.", - ...(hookSnapshot.hook_error - ? ["The active IPC hook could not be installed."] - : []), - ], - }); -}; - const canonicalPaths = async ( input: ElectronActiveObservationInput, ): Promise => { diff --git a/src/domain/javascript/electronActiveObservation.fixture.ts b/src/domain/javascript/electronActiveObservation.fixture.ts index 144dccca83e..aa217b57a21 100644 --- a/src/domain/javascript/electronActiveObservation.fixture.ts +++ b/src/domain/javascript/electronActiveObservation.fixture.ts @@ -49,6 +49,20 @@ export const createElectronActiveObservationFixtureResult = ( }, ], observed: 1, + dropped: 0, + }, + retention: { + budget_bytes: 16 * 1024 * 1024, + estimated_retained_bytes: 768, + event_serialized_byte_upper_bound: 512, + retained: 1, + dropped: 0, + dropped_event_families: [], + dropped_event_roles: [], + observed_ipc: 1, + dropped_ipc: 0, + observed_runtime: 0, + dropped_runtime: 0, }, limitations: [ "IPC payloads are represented by value shapes; payload values are never retained.", diff --git a/src/domain/javascript/electronActiveObservation.ts b/src/domain/javascript/electronActiveObservation.ts index 6c1391266ba..7a3fd9af6a3 100644 --- a/src/domain/javascript/electronActiveObservation.ts +++ b/src/domain/javascript/electronActiveObservation.ts @@ -262,13 +262,41 @@ export const electronActiveObservationResultSchema = z.strictObject({ ipc: z.strictObject({ events: z.array(ipcEventSchema), observed: z.number().int().min(0), + dropped: z.number().int().min(0).nullable().default(null), }), timeline: z .strictObject({ events: z.array(timelineEventSchema), observed: z.number().int().min(0), + dropped: z.number().int().min(0).nullable().default(null), }) - .default({ events: [], observed: 0 }), + .default({ events: [], observed: 0, dropped: null }), + retention: z + .strictObject({ + budget_bytes: z.number().int().min(0), + estimated_retained_bytes: z.number().int().min(0), + event_serialized_byte_upper_bound: z.number().int().min(0), + retained: z.number().int().min(0), + dropped: z.number().int().min(0), + dropped_event_families: z.array( + z.strictObject({ + family: z.string().min(1), + count: z.number().int().min(1), + }), + ), + dropped_event_roles: z.array( + z.strictObject({ + role: z.string().min(1), + count: z.number().int().min(1), + }), + ), + observed_ipc: z.number().int().min(0), + dropped_ipc: z.number().int().min(0), + observed_runtime: z.number().int().min(0), + dropped_runtime: z.number().int().min(0), + }) + .nullable() + .default(null), coverage: coverageSchema.default({ status: "partial_attach", observed_event_families: [], diff --git a/tests/boundary/browser/electronActiveHook.test.ts b/tests/boundary/browser/electronActiveHook.test.ts index 2afaebc7ad7..f4cd1e19fc2 100644 --- a/tests/boundary/browser/electronActiveHook.test.ts +++ b/tests/boundary/browser/electronActiveHook.test.ts @@ -102,8 +102,17 @@ electron.app.relaunch(); electron.contextBridge.exposeInMainWorld("api", { value: true }); window.webContents.loadURL("https://example.invalid").catch(() => undefined); for (let index = 0; index < 6000; index++) electron.app.relaunch(); +electron.ipcRenderer.send("x".repeat(9 * 1024 * 1024)); +electron.app.emit("r".repeat(9 * 1024 * 1024)); +const manyArguments = Array(30_000).fill(true); +for (let index = 0; index < 8; index++) electron.ipcRenderer.send("shape-budget", ...manyArguments); try { process.dlopen({}, "/missing/native.node"); } catch {} -process.stdout.write(JSON.stringify({ snapshot: globalThis.__reaElectronActiveSnapshot(), default_popup_decision: defaultPopupDecision, popup_decision: popupDecision }) + "\\n"); +const firstSnapshot = globalThis.__reaElectronActiveSnapshot(); +const firstIpc = firstSnapshot.events.find((event) => event.kind === "main-handler-invocation"); +if (firstIpc) { firstIpc.channel = "mutated"; firstIpc.argument_shapes.push("mutated"); } +firstSnapshot.events.length = 0; +const snapshot = globalThis.__reaElectronActiveSnapshot(); +process.stdout.write(JSON.stringify({ snapshot, default_popup_decision: defaultPopupDecision, popup_decision: popupDecision }) + "\\n"); })(); `; @@ -115,7 +124,7 @@ afterEach(async () => { ); }); -it("retains active hook lifecycle and IPC evidence without a count ceiling", async () => { +it("retains active hook evidence within its event-retention budget", async () => { const root = await createTestTempDirectory("rea-electron-hook-"); temporary.push(root); await writeFile(join(root, "electron.js"), electronModuleSource); @@ -123,7 +132,7 @@ it("retains active hook lifecycle and IPC evidence without a count ceiling", asy await writeFile(script, activeCaptureSource); const hook = join(process.cwd(), "scripts/electron-active-hook.cjs"); const execution = await runNode(hook, script, root, root); - expect(execution.code).toBe(0); + expect(execution.code, execution.stderr).toBe(0); const result = JSON.parse( execution.stdout.trim().split("\n").at(-1) ?? "null", ) as { @@ -133,13 +142,37 @@ it("retains active hook lifecycle and IPC evidence without a count ceiling", asy readonly hook_error: boolean; readonly events: readonly Record[]; readonly observed: number; + readonly retained: number; + readonly dropped: number; + readonly dropped_ipc: number; + readonly dropped_runtime: number; + readonly retention_budget_bytes: number; }; }; const { snapshot } = result; expect(snapshot.hook_error).toBe(false); - expect(snapshot.events).toHaveLength(snapshot.observed); + expect(snapshot.events).toHaveLength(snapshot.retained); + expect(snapshot.observed).toBe(snapshot.retained + snapshot.dropped); expect(snapshot.observed).toBeGreaterThan(5_000); + expect(snapshot.dropped_ipc).toBeGreaterThanOrEqual(2); + expect(snapshot.dropped_runtime).toBe(1); + expect(snapshot.dropped).toBe( + snapshot.dropped_ipc + snapshot.dropped_runtime, + ); + expect( + snapshot.events.some( + (event) => + event.kind === "ipc-renderer-send" && + Array.isArray(event.argument_shapes) && + event.argument_shapes.length === 30_000, + ), + ).toBe(true); + expect( + Buffer.byteLength(JSON.stringify(snapshot), "utf8"), + ).toBeLessThanOrEqual(snapshot.retention_budget_bytes); + const sequences = snapshot.events.map((event) => event.sequence as number); + expect(sequences).toEqual([...sequences].sort((left, right) => left - right)); expect(snapshot.events).toEqual( expect.arrayContaining([ expect.objectContaining({ kind: "window-lifecycle", event: "created" }), From 67252b23b9e35e9526fed7842dab47a2c7c9fe5f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:09:19 +0800 Subject: [PATCH 035/791] perf(javascript): index configuration and module lookups --- ...vascriptSemanticDataEffects.lookup.test.ts | 23 ++++ .../javascriptSemanticDataEffects.ts | 48 ++++++++- .../javascript/javascriptStaticAnalysis.ts | 31 ++++-- .../javascriptStaticAnalysisBundler.ts | 8 +- .../javascriptStaticAnalysisCalls.ts | 3 +- .../javascriptStaticAnalysisFindings.ts | 102 ++++++++++++++++-- ...avascriptStaticAnalysisModuleIndex.test.ts | 69 ++++++++++++ .../javascriptStaticAnalysisState.ts | 42 ++++---- 8 files changed, 276 insertions(+), 50 deletions(-) create mode 100644 src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts create mode 100644 src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts diff --git a/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts b/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts new file mode 100644 index 00000000000..e9536443f68 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts @@ -0,0 +1,23 @@ +import { expect, it } from "vitest"; + +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; + +it("keeps defaults linked to their own config reads across a large source", () => { + const count = 400; + const source = Array.from( + { length: count }, + (_, index) => + `const value${index} = process.env.KEY_${index} ?? "fallback-${index}";`, + ).join("\n"); + + const operations = analyzeJavaScriptSemantics(source).configurationOperations; + const defaults = operations.filter(({ kind }) => kind === "default"); + + expect(defaults).toHaveLength(count); + for (let index = 0; index < count; index += 1) { + expect(defaults.find(({ key }) => key === `KEY_${index}`)).toMatchObject({ + key: `KEY_${index}`, + value: `fallback-${index}`, + }); + } +}); diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 6dad44bf05a..18abeb532af 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -163,15 +163,24 @@ const collectDefaults = ( configurations: readonly ConfigurationCandidate[], ): JavaScriptSemanticConfigurationOperation[] => { const output: JavaScriptSemanticConfigurationOperation[] = []; + const configurationsByStart = configurations + .filter( + ({ node }) => + node.start !== null && + node.start !== undefined && + node.end !== null && + node.end !== undefined, + ) + .toSorted( + (left, right) => (left.node.start ?? 0) - (right.node.start ?? 0), + ); traverseWithContext(program, context, (node) => { if ( !t.isLogicalExpression(node) || (node.operator !== "??" && node.operator !== "||") ) return; - const sources = configurations.filter(({ node: source }) => - containsNode(node.left, source), - ); + const sources = configurationsIn(node.left, configurationsByStart); if (sources.length !== 1 || sources[0] === undefined) return; output.push({ configId: `config:default:${String(node.start ?? -1)}:${String(node.end ?? -1)}`, @@ -188,6 +197,39 @@ const collectDefaults = ( return output; }; +/** Find only configuration candidates whose source offsets can fit this left operand. */ +const configurationsIn = ( + expression: t.Node, + candidates: readonly ConfigurationCandidate[], +): readonly ConfigurationCandidate[] => { + if ( + expression.start === null || + expression.start === undefined || + expression.end === null || + expression.end === undefined + ) + return []; + let low = 0; + let high = candidates.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + const start = candidates[middle]?.node.start; + if (start !== null && start !== undefined && start < expression.start) + low = middle + 1; + else high = middle; + } + const sources: ConfigurationCandidate[] = []; + for (let index = low; index < candidates.length; index += 1) { + const candidate = candidates[index]; + if (candidate === undefined) continue; + const start = candidate.node.start; + if (start === null || start === undefined || start > expression.end) break; + if (containsNode(expression, candidate.node)) sources.push(candidate); + if (sources.length > 1) break; + } + return sources; +}; + const collectRequests = ( program: t.Program, context: TraversalContext, diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 0a18ceb7db6..af20eaa9264 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -56,7 +56,7 @@ export const analyzeParsedJavaScriptStaticSource = ( file: ParsedJavaScriptSource, openReceiverFacts?: ReadonlyMap, ): JavaScriptStaticAnalysis => { - const accumulator = createJavaScriptAnalysisAccumulator(); + const accumulator = createJavaScriptAnalysisAccumulator(source.length); traverseStaticSource(source, file, accumulator, openReceiverFacts); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); @@ -105,41 +105,50 @@ const finalizeStaticAnalysis = ( visited_ast_nodes: accumulator.visitedNodes, references: finalizeLocatedFindings( accumulator.references, - accumulator.modules, + accumulator.moduleRangeIndex, ), endpoints: finalizeLocatedFindings( accumulator.endpoints, - accumulator.modules, + accumulator.moduleRangeIndex, + ), + storage: finalizeLocatedFindings( + accumulator.storage, + accumulator.moduleRangeIndex, ), - storage: finalizeLocatedFindings(accumulator.storage, accumulator.modules), bundler_registrations: sortedUnique( accumulator.registrations, registrationKey, ), - role_paths: finalizeLocatedFindings(accumulator.roles, accumulator.modules), + role_paths: finalizeLocatedFindings( + accumulator.roles, + accumulator.moduleRangeIndex, + ), source_map_urls: accumulator.sourceMaps, vendors: detectVendors(source), electron: { browser_windows: finalizeLocatedFindings( accumulator.browserWindows, - accumulator.modules, + accumulator.moduleRangeIndex, ), context_bridge_apis: finalizeLocatedFindings( accumulator.contextBridgeApis, - accumulator.modules, + accumulator.moduleRangeIndex, + ), + ipc: finalizeLocatedFindings( + accumulator.ipc, + accumulator.moduleRangeIndex, ), - ipc: finalizeLocatedFindings(accumulator.ipc, accumulator.modules), sender_validations: finalizeLocatedFindings( accumulator.senderValidations, - accumulator.modules, + accumulator.moduleRangeIndex, ), utility_processes: finalizeLocatedFindings( accumulator.utilityProcesses, - accumulator.modules, + accumulator.moduleRangeIndex, ), native_addon_bindings: finalizeLocatedFindings( accumulator.nativeAddonBindings, - accumulator.modules, + accumulator.moduleRangeIndex, ), }, limitations, diff --git a/src/domain/javascript/javascriptStaticAnalysisBundler.ts b/src/domain/javascript/javascriptStaticAnalysisBundler.ts index 3e964f332f5..e82096f0194 100644 --- a/src/domain/javascript/javascriptStaticAnalysisBundler.ts +++ b/src/domain/javascript/javascriptStaticAnalysisBundler.ts @@ -164,13 +164,15 @@ const recoverBundlerModule = ( const exportsValue = collectJavaScriptExports(factory); const requireName = factoryRequireName(factory); const asyncChunks = collectBundlerAsyncChunkKeys(factory, requireName); - if (typeof factory.start === "number" && typeof factory.end === "number") - accumulator.modules.push({ + if (typeof factory.start === "number" && typeof factory.end === "number") { + const moduleRange = { start: factory.start, end: factory.end, key, requireName, - }); + }; + accumulator.moduleRangeIndex.add(moduleRange); + } return { module: { module_key: key, diff --git a/src/domain/javascript/javascriptStaticAnalysisCalls.ts b/src/domain/javascript/javascriptStaticAnalysisCalls.ts index 8addcb7cbb7..e984f77beed 100644 --- a/src/domain/javascript/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascript/javascriptStaticAnalysisCalls.ts @@ -4,7 +4,6 @@ import { sanitizeEndpointCandidate } from "../browserObservation.js"; import { addFindingOnce, addLocatedFinding, - moduleAtOffset, } from "./javascriptStaticAnalysisFindings.js"; import { STATIC_ROUTE_CALL_NAMES, @@ -42,7 +41,7 @@ export const inspectCall = ( const { accumulator } = context; const name = calleeName(node.callee); const first = stringValue(node.arguments[0]); - const module = moduleAtOffset(node.start, context.accumulator.modules); + const module = context.accumulator.moduleRangeIndex.find(node.start); const moduleRequireName = module?.requireName ?? null; const requireCall = requireCallSpecifier(node, name, moduleRequireName); if (requireCall !== null) diff --git a/src/domain/javascript/javascriptStaticAnalysisFindings.ts b/src/domain/javascript/javascriptStaticAnalysisFindings.ts index 0def3a5bd4c..4108e7e7262 100644 --- a/src/domain/javascript/javascriptStaticAnalysisFindings.ts +++ b/src/domain/javascript/javascriptStaticAnalysisFindings.ts @@ -6,6 +6,95 @@ import type { LocatedJavaScriptFindingInput, } from "./javascriptStaticAnalysisState.js"; +interface IndexedModuleRange { + readonly order: number; + readonly range: JavaScriptModuleRange; +} + +interface ModuleRangeTreeNode { + left?: ModuleRangeTreeNode; + right?: ModuleRangeTreeNode; + best?: IndexedModuleRange; +} + +/** Incremental point lookup preserving insertion-order precedence for overlaps. */ +export class JavaScriptModuleRangeIndex { + private readonly root: ModuleRangeTreeNode = {}; + private readonly end: number; + private nextOrder = 0; + + /** Create an empty offset index bounded by the analyzed source length. */ + constructor(sourceLength: number) { + this.end = Math.max(1, sourceLength + 1); + } + + /** Add a module range in the same discovery order used by the analysis pass. */ + add(range: JavaScriptModuleRange): void { + const indexed = { order: this.nextOrder, range }; + this.nextOrder += 1; + const start = Math.max(0, range.start); + const end = Math.min(this.end, range.end + 1); + if (start < end) this.insert(this.root, 0, this.end, start, end, indexed); + } + + /** Return the first discovered inclusive range containing an offset. */ + find(offset: number | null | undefined): JavaScriptModuleRange | undefined { + if ( + offset === null || + offset === undefined || + !Number.isSafeInteger(offset) || + offset < 0 || + offset >= this.end + ) + return undefined; + let node: ModuleRangeTreeNode | undefined = this.root; + let low = 0; + let high = this.end; + let best: IndexedModuleRange | undefined; + while (node !== undefined) { + if ( + node.best !== undefined && + (best === undefined || node.best.order < best.order) + ) + best = node.best; + if (high - low <= 1) break; + const middle = low + Math.floor((high - low) / 2); + if (offset < middle) { + node = node.left; + high = middle; + } else { + node = node.right; + low = middle; + } + } + return best?.range; + } + + private insert( + node: ModuleRangeTreeNode, + low: number, + high: number, + queryLow: number, + queryHigh: number, + value: IndexedModuleRange, + ): void { + if (queryLow <= low && high <= queryHigh) { + if (node.best === undefined || value.order < node.best.order) + node.best = value; + return; + } + const middle = low + Math.floor((high - low) / 2); + if (queryLow < middle) { + node.left ??= {}; + this.insert(node.left, low, middle, queryLow, queryHigh, value); + } + if (queryHigh > middle) { + node.right ??= {}; + this.insert(node.right, middle, high, queryLow, queryHigh, value); + } + } +} + /** Add one module-attributable finding once per source location. */ export const addLocatedFinding = < Value extends { readonly module_key: string | null }, @@ -39,18 +128,9 @@ export const finalizeLocatedFindings = < Value extends { readonly module_key: string | null }, >( values: readonly LocatedJavaScriptFinding[], - modules: readonly JavaScriptModuleRange[], + modules: JavaScriptModuleRangeIndex, ): Value[] => values.map(({ offset, value }) => ({ ...value, - module_key: moduleAtOffset(offset, modules)?.key ?? null, + module_key: modules.find(offset)?.key ?? null, })); - -/** Locate the recovered bundle factory containing one exact source offset. */ -export const moduleAtOffset = ( - offset: number | null | undefined, - modules: readonly JavaScriptModuleRange[], -): JavaScriptModuleRange | undefined => { - if (offset === null || offset === undefined) return undefined; - return modules.find(({ start, end }) => offset >= start && offset <= end); -}; diff --git a/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts new file mode 100644 index 00000000000..075597a703b --- /dev/null +++ b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts @@ -0,0 +1,69 @@ +import { expect, it } from "vitest"; + +import { JavaScriptModuleRangeIndex } from "./javascriptStaticAnalysisFindings.js"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +it("preserves first insertion precedence and inclusive interval bounds", () => { + const nested = new JavaScriptModuleRangeIndex(100); + expect(nested.find(0)).toBeUndefined(); + nested.add({ start: 10, end: 90, key: "outer-first", requireName: null }); + nested.add({ start: 20, end: 30, key: "inner-second", requireName: null }); + nested.add({ start: 30, end: 40, key: "later-overlap", requireName: null }); + expect(nested.find(9)).toBeUndefined(); + expect(nested.find(10)?.key).toBe("outer-first"); + expect(nested.find(20)?.key).toBe("outer-first"); + expect(nested.find(30)?.key).toBe("outer-first"); + expect(nested.find(90)?.key).toBe("outer-first"); + expect(nested.find(91)).toBeUndefined(); + expect(nested.find(100)).toBeUndefined(); +}); + +it("rejects missing and out-of-range lookup offsets", () => { + const index = new JavaScriptModuleRangeIndex(20); + index.add({ start: 5, end: 10, key: "bounded", requireName: null }); + expect(index.find(undefined)).toBeUndefined(); + expect(index.find(null)).toBeUndefined(); + expect(index.find(-1)).toBeUndefined(); + expect(index.find(0)).toBeUndefined(); + expect(index.find(11)).toBeUndefined(); + expect(index.find(20)).toBeUndefined(); + expect(index.find(Number.NaN)).toBeUndefined(); + expect(index.find(5)?.key).toBe("bounded"); + expect(index.find(10)?.key).toBe("bounded"); +}); + +it("keeps module references attributed across many real factory ranges", () => { + const count = 200; + const factories = Array.from({ length: count }, (_, index) => + index === 0 + ? '"module-0": function(module, exports, req) { globalThis.webpackChunkInner.push([[2], { "inner": function(innerModule, innerExports, req) { req("nested-dependency"); } }]); req("dependency-0"); }' + : `"module-${index}": function(module, exports, req) { req("dependency-${index}"); }`, + ).join(",\n"); + const analysis = analyzeJavaScriptStaticSource( + `require("outside");\nglobalThis.webpackChunkApp.push([[1], {\n${factories}\n}]);`, + ); + expect(analysis.references).toHaveLength(count + 2); + expect( + analysis.bundler_registrations.some( + ({ runtime, modules }) => + runtime === "webpackChunkInner" && + modules.some(({ module_key }) => module_key === "inner"), + ), + ).toBe(true); + expect( + analysis.references.find(({ specifier }) => specifier === "outside") + ?.module_key, + ).toBeNull(); + for (let index = 0; index < count; index += 1) { + expect( + analysis.references.find( + ({ specifier }) => specifier === `dependency-${index}`, + )?.module_key, + ).toBe(`module-${index}`); + } + expect( + analysis.references.find( + ({ specifier }) => specifier === "nested-dependency", + )?.module_key, + ).toBe("module-0"); +}); diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index b830b1bc4d8..23de6952bf0 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -16,6 +16,7 @@ import type { ElectronSenderValidationFinding, ElectronUtilityProcessFinding, } from "./electronStaticAnalysisTypes.js"; +import { JavaScriptModuleRangeIndex } from "./javascriptStaticAnalysisFindings.js"; /** Source offsets for one recovered bundle module factory. */ export interface JavaScriptModuleRange { @@ -45,7 +46,7 @@ export interface JavaScriptAnalysisAccumulator { readonly senderValidations: LocatedJavaScriptFinding[]; readonly utilityProcesses: LocatedJavaScriptFinding[]; readonly nativeAddonBindings: LocatedJavaScriptFinding[]; - readonly modules: JavaScriptModuleRange[]; + readonly moduleRangeIndex: JavaScriptModuleRangeIndex; readonly seen: Set; visitedNodes: number; unknownFindings: number; @@ -92,22 +93,23 @@ export interface LocatedJavaScriptFindingInput< } /** Create isolated mutable state for one source analysis. */ -export const createJavaScriptAnalysisAccumulator = - (): JavaScriptAnalysisAccumulator => ({ - references: [], - endpoints: [], - storage: [], - roles: [], - sourceMaps: [], - registrations: [], - browserWindows: [], - contextBridgeApis: [], - ipc: [], - senderValidations: [], - utilityProcesses: [], - nativeAddonBindings: [], - modules: [], - seen: new Set(), - visitedNodes: 0, - unknownFindings: 0, - }); +export const createJavaScriptAnalysisAccumulator = ( + sourceLength: number, +): JavaScriptAnalysisAccumulator => ({ + references: [], + endpoints: [], + storage: [], + roles: [], + sourceMaps: [], + registrations: [], + browserWindows: [], + contextBridgeApis: [], + ipc: [], + senderValidations: [], + utilityProcesses: [], + nativeAddonBindings: [], + moduleRangeIndex: new JavaScriptModuleRangeIndex(sourceLength), + seen: new Set(), + visitedNodes: 0, + unknownFindings: 0, +}); From 34f7c4fd661e07db66c560a4d54272b0163a52b6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:14:32 +0800 Subject: [PATCH 036/791] fix(docs): validate translated links to canonical product facts --- scripts/lib/docs-facts.mjs | 53 +++++++++----------- tests/boundary/cli/productCatalog.test.ts | 60 +++++++++++++++++------ 2 files changed, 68 insertions(+), 45 deletions(-) diff --git a/scripts/lib/docs-facts.mjs b/scripts/lib/docs-facts.mjs index d45d545aa75..45b947e995b 100644 --- a/scripts/lib/docs-facts.mjs +++ b/scripts/lib/docs-facts.mjs @@ -8,21 +8,14 @@ const README_PATHS = [ "README_ko.md", "README_ar.md", ]; -const tableCounts = (content, path, expectedCounts) => { - const lines = content.split(/\r?\n/u); - for (const [index, line] of lines.entries()) { - if (!/^\|\s*-/u.test(line)) continue; - const counts = []; - for (const row of lines.slice(index + 1)) { - if (!row.trim().startsWith("|")) break; - const count = Number(row.split("|")[2]?.trim()); - if (!Number.isInteger(count)) break; - counts.push(count); - } - if (counts.length === expectedCounts.length) return counts; - } - throw new Error(`Missing tool-family inventory table in ${path}`); -}; +const readmeLinkTargets = (content) => + [...content.matchAll(/\]\(([^)\s]+)\)/gu)] + .map((match) => match[1] ?? "") + .filter( + (target) => + !target.startsWith("#") && !/^README(?:_[a-z]+)?\.md$/u.test(target), + ) + .sort(); const requireText = (issues, path, content, expected) => { if (!content.includes(expected)) issues.push(`${path}: missing ${expected}`); @@ -71,32 +64,34 @@ export const documentationFactIssues = async (root, catalog) => { const issues = await skillReferenceIssues( join(root, "skills/reverse-engineer-anything"), ); - const expectedCounts = catalog.tools.families.map(({ count }) => count); + const readmes = new Map(); for (const path of README_PATHS) { const content = await readFile(join(root, path), "utf8"); + readmes.set(path, content); requireText(issues, path, content, "MCP-tool_catalog"); if (path === "README.md") { - requireText(issues, path, content, "(docs/installation.md"); + requireText( + issues, + path, + content, + "(docs/installation.md#supported-agents)", + ); + requireText(issues, path, content, "(docs/installation.md#mcp-registry)"); requireText( issues, path, content, "(docs/mcp-contracts.md#generated-catalog)", ); - continue; - } - for (const client of catalog.setup_clients) - requireText(issues, path, content, client.display_name); - try { - const actualCounts = tableCounts(content, path, expectedCounts); - if (JSON.stringify(actualCounts) !== JSON.stringify(expectedCounts)) - issues.push( - `${path}: tool family counts ${JSON.stringify(actualCounts)} do not match ${JSON.stringify(expectedCounts)}`, - ); - } catch (cause) { - issues.push(cause instanceof Error ? cause.message : String(cause)); } } + const canonicalLinks = readmeLinkTargets(readmes.get("README.md") ?? ""); + for (const path of README_PATHS.slice(1)) + if ( + JSON.stringify(readmeLinkTargets(readmes.get(path) ?? "")) !== + JSON.stringify(canonicalLinks) + ) + issues.push(`${path}: documentation links differ from README.md`); const installationPath = "docs/installation.md"; const installation = await readFile(join(root, installationPath), "utf8"); diff --git a/tests/boundary/cli/productCatalog.test.ts b/tests/boundary/cli/productCatalog.test.ts index 6dcc93b66d0..e5b25c1e877 100644 --- a/tests/boundary/cli/productCatalog.test.ts +++ b/tests/boundary/cli/productCatalog.test.ts @@ -1,7 +1,7 @@ import { EVMOLE_PROVIDER_IDENTITY } from "../../../src/evm/EvmoleRelease.js"; import { PWNTOOLS_PROVIDER_IDENTITY } from "../../../src/native/pwntools/PwntoolsRelease.js"; -import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { copyFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it } from "vitest"; @@ -273,19 +273,10 @@ describe("canonical product catalog drift", () => { ]); }); - it("reports tool-family and setup-client fact drift", async () => { + it("requires setup-client facts in the canonical installation guide", async () => { const catalog = await createProductCatalog(root); - const firstFamily = catalog.tools.families[0]; - if (firstFamily === undefined) throw new TypeError("Missing tool family"); const drifted = { ...catalog, - tools: { - ...catalog.tools, - families: [ - { ...firstFamily, count: firstFamily.count + 1 }, - ...catalog.tools.families.slice(1), - ], - }, setup_clients: [ ...catalog.setup_clients, { @@ -297,14 +288,51 @@ describe("canonical product catalog drift", () => { ], }; const issues = await documentationFactIssues(root, drifted); - expect(issues.some((issue) => issue.includes("tool family counts"))).toBe( - true, - ); - expect(issues.some((issue) => issue.includes("Future Client"))).toBe(true); expect(issues).toContain("docs/installation.md: missing Future Client"); expect(issues).not.toContain("README.md: missing Future Client"); }); + it("keeps translated readmes linked to canonical setup and tool details", async () => { + const directory = await createTestTempDirectory("rea-readme-facts-"); + temporaryRoots.push(directory); + const paths = [ + "README.md", + "README_zh.md", + "README_ja.md", + "README_ko.md", + "README_ar.md", + "docs/installation.md", + "AGENTS.md", + ".github/pull_request_template.md", + ]; + for (const path of paths) { + const destination = join(directory, path); + await mkdir(dirname(destination), { recursive: true }); + await copyFile(join(root, path), destination); + } + await mkdir(join(directory, "skills/reverse-engineer-anything"), { + recursive: true, + }); + const translatedPath = join(directory, "README_zh.md"); + const translated = await readFile(translatedPath, "utf8"); + await writeFile( + translatedPath, + translated.replace( + "(docs/installation.md#supported-agents)", + "(docs/installation.md#missing-agents)", + ), + "utf8", + ); + + const issues = await documentationFactIssues( + directory, + await createProductCatalog(root), + ); + expect(issues).toContain( + "README_zh.md: documentation links differ from README.md", + ); + }); + it("changes the provider projection digest when provider facts drift", async () => { const catalog = await createProductCatalog(root); const firstProvider = catalog.providers[0]; From 4458128a845c1448140b44d6e62d6ccfe611d37c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:27:35 +0800 Subject: [PATCH 037/791] fix(ghidra): make qualified function renames idempotent --- bridge/ghidra/ReaGhidraBridge.java | 22 ++- docs/native-investigation.md | 6 + docs/testing.md | 7 + scripts/lib/real-ghidra-boundaries.mjs | 9 ++ .../lib/real-ghidra-namespace-annotations.mjs | 151 ++++++++++++++++++ src/ghidra/GhidraProviderCapabilities.ts | 2 +- tests/conformance/ghidra/namespaces.c | 22 +++ 7 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 scripts/lib/real-ghidra-namespace-annotations.mjs create mode 100644 tests/conformance/ghidra/namespaces.c diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 326d36665f3..c6e61648655 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1011,6 +1011,8 @@ private JsonObject annotateNativeFunction(JsonObject params) throws Exception { for (String field : List.of("name", "comment", "inline_comment")) { if (params.has(field)) validateAnnotationText(requireText(params, field), field); } + String leafName = params.has("name") + ? annotationLeafName(function, requireString(params, "name")) : null; int transaction = currentProgram.startTransaction("REA function annotations"); boolean commit = false; try { @@ -1024,7 +1026,7 @@ private JsonObject annotateNativeFunction(JsonObject params) throws Exception { } } if (params.has("name")) - function.setName(requireString(params, "name"), SourceType.USER_DEFINED); + function.setName(leafName, SourceType.USER_DEFINED); monitor.checkCancelled(); invalidateAnalysisCaches(); JsonObject readback = annotationReadback(function); @@ -1032,7 +1034,7 @@ private JsonObject annotateNativeFunction(JsonObject params) throws Exception { if (!params.has(field)) continue; String requested = requireText(params, field); JsonElement measured = readback.get(field); - String expected = !field.equals("name") && requested.isEmpty() ? null : requested; + String expected = field.equals("name") ? leafName : requested.isEmpty() ? null : requested; String observed = field.equals("name") ? function.getName() : (measured.isJsonNull() ? null : measured.getAsString()); if (!java.util.Objects.equals(expected, observed)) @@ -1062,6 +1064,22 @@ private JsonObject annotateNativeFunction(JsonObject params) throws Exception { } } + private String annotationLeafName(Function function, String requested) { + // Fully qualified readback must be usable as an idempotent rename input. + if (requested.equals(procedureName(function))) return function.getName(); + String namespace = function.getParentNamespace().isGlobal() + ? "" : function.getParentNamespace().getName(true); + String prefix = namespace + "::"; + if (!namespace.isEmpty() && requested.startsWith(prefix)) { + String leaf = requested.substring(prefix.length()); + if (!leaf.isEmpty()) return leaf; + throw new RequestFailure("invalid_function_name", + "Function name has an empty leaf within namespace " + namespace + " at " + + canonicalAddress(function.getEntryPoint()) + ": " + requested); + } + return requested; + } + private static String requireText(JsonObject params, String field) { JsonElement value = params.get(field); if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString()) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index e9b0f1b0f6a..e4fdf9a5a22 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -317,6 +317,12 @@ unpaired Unicode surrogate; a rejection identifies the field and UTF-16 index and leaves every annotation unchanged. CRLF, supplementary Unicode characters, and combining characters are preserved. Windows P0 remains read-only. Ghidra has no GUI authority, and REA never falls back automatically to Hopper. +Ghidra name edits preserve the existing namespace. Supply either a leaf name +such as `renamed` or a fully qualified name in that namespace, such as +`alpha::renamed`. The returned qualified name can be reused as an idempotent +rename input. Edits retain the existing namespace; other namespace-like text +remains literal leaf-name text. A qualified name with an empty leaf is rejected +before any comment or name is changed. - `npm run verify:ghidra`: host-native debug/stripped targets, native type layout, instruction/call facts, value dependencies and process/project cleanup. diff --git a/docs/testing.md b/docs/testing.md index b86be70549d..f6076d26bec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -342,6 +342,13 @@ inputs, plus real read-permission denial on a non-root host. Root runs report that permission-denial check as unverified. A focused filesystem integration check retains selected-platform routing and exclusive creation; two copy/digest checks duplicated by the real workflow have been removed. +Namespace annotation probes compile a separate host C fixture with C++ ABI +symbols, avoiding a C++ runtime prerequisite. Real Ghidra demangling supplies +duplicate leaf names in two top-level namespaces and a nested namespace. The +workflow verifies leaf and qualified renames, repeated reuse of fully qualified +readback, lookup by the returned name, literal namespace-like leaf names, +rejection of empty qualified leaf names without changing comments, CLI +behavior, and independent CLI/MCP database ownership. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 0de39100230..988635fe0ec 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -11,6 +11,7 @@ import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot-e2e.mjs"; import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; import { verifyGhidraTargetAdmission } from "./real-ghidra-target-admission.mjs"; +import { verifyGhidraNamespaceAnnotations } from "./real-ghidra-namespace-annotations.mjs"; /** Probe real Ghidra location, annotation and error contracts through public adapters. */ export async function verifyGhidraBoundaries( @@ -656,6 +657,13 @@ export async function verifyGhidraBoundaries( entry: address, runtimeRoot: env.TMPDIR, }); + await verifyGhidraNamespaceAnnotations({ + call, + reject: invalid, + target, + entrypoint, + env, + }); await assert.rejects(access(socketRoot), { code: "ENOENT" }); await assert.rejects(access(runtimeRoot), { code: "ENOENT" }); assert.equal(await call("procedure_address", { procedure: name }), address); @@ -696,6 +704,7 @@ export async function verifyGhidraBoundaries( missing_and_nonregular_source_rejected: true, imported_source_identity_retained: true, equivalent_instruction_address_spellings: true, + qualified_annotation_name_roundtrip: true, source_immutable: true, reopen_discards_edits: true, long_tmpdir_private_socket_cleanup: true, diff --git a/scripts/lib/real-ghidra-namespace-annotations.mjs b/scripts/lib/real-ghidra-namespace-annotations.mjs new file mode 100644 index 00000000000..f37e0441506 --- /dev/null +++ b/scripts/lib/real-ghidra-namespace-annotations.mjs @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +/** Exercise qualified annotation names against real demangled C++ ABI symbols. */ +export async function verifyGhidraNamespaceAnnotations({ + call, + reject, + target, + entrypoint, + env, +}) { + const exec = promisify(execFile); + const compiler = process.env.REA_CC ?? "cc"; + try { + await exec(compiler, ["--version"], { env, timeout: 30000 }); + } catch (cause) { + throw new Error( + `Ghidra namespace annotation verification requires a working host C compiler '${compiler}'.`, + { cause }, + ); + } + const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-namespaces-")); + const source = fileURLToPath( + new URL("../../tests/conformance/ghidra/namespaces.c", import.meta.url), + ); + const path = join(workspace, "namespaces"); + try { + await exec(compiler, ["-O0", "-fno-inline", source, "-o", path], { + env, + timeout: 30000, + }); + const bytes = await readFile(path); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await call("close_binary"); + await call("open_binary", { path, provider_id: "ghidra" }); + const procedures = await call("list_procedures"); + const namespaces = ["alpha", "beta", "outer::inner"]; + const entries = namespaces.map((namespace) => { + const procedure = procedures.find( + ({ value }) => value === `${namespace}::same`, + ); + assert.ok( + procedure, + `Missing demangled fixture function ${namespace}::same`, + ); + return { namespace, address: procedure.address }; + }); + const ambiguous = await reject("procedure_address", { procedure: "same" }); + assert.equal(ambiguous.code, "invalid_request"); + for (const { address } of entries) + assert.ok(JSON.stringify(ambiguous.details.issues).includes(address)); + for (const { namespace, address } of entries) { + const leaf = await call("annotate_native_function", { + procedure: address, + name: "renamed", + comment: "Preserve this comment", + inline_comment: "Preserve this inline comment", + }); + assert.equal(leaf.annotations.name, `${namespace}::renamed`); + const qualified = await call("annotate_native_function", { + procedure: address, + name: `${namespace}::qualified`, + }); + assert.equal(qualified.annotations.name, `${namespace}::qualified`); + for (let repeat = 0; repeat < 2; repeat++) { + const replay = await call("annotate_native_function", { + procedure: qualified.annotations.name, + name: qualified.annotations.name, + }); + assert.deepEqual(replay.annotations, qualified.annotations); + assert.equal(replay.dossier.procedure.address, address); + } + assert.equal( + await call("procedure_address", { + procedure: qualified.annotations.name, + }), + address, + ); + for (const name of ["other::literal", `${namespace}::child::literal`]) { + const literal = await call("annotate_native_function", { + procedure: address, + name, + }); + const expected = name.startsWith(`${namespace}::`) + ? name + : `${namespace}::${name}`; + assert.equal(literal.annotations.name, expected); + const repeated = await call("annotate_native_function", { + procedure: expected, + name: expected, + }); + assert.deepEqual(repeated.annotations, literal.annotations); + } + await call("annotate_native_function", { + procedure: address, + name: qualified.annotations.name, + }); + const failure = await reject("annotate_native_function", { + procedure: address, + name: `${namespace}::`, + comment: "MUST NOT REPLACE", + inline_comment: "MUST NOT REPLACE", + }); + assert.equal(failure.code, "invalid_request"); + assert.deepEqual(failure.details.issues[0].path, ["name"]); + assert.match(failure.details.issues[0].message, /empty leaf/u); + const unchanged = await call("annotate_native_function", { + procedure: address, + name: qualified.annotations.name, + }); + assert.deepEqual(unchanged.annotations, qualified.annotations); + } + const { stdout } = await exec( + process.execPath, + [ + entrypoint, + "annotate-native-function", + path, + "alpha::same", + "--name", + "alpha::cli_renamed", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 72 * 1024 * 1024 }, + ); + assert.equal( + JSON.parse(stdout).normalized_result.annotations.name, + "alpha::cli_renamed", + ); + assert.equal( + (await call("analyze_function", { procedure: entries[0].address })) + .procedure.name, + "alpha::qualified", + "The independent CLI session must not alter the active MCP database", + ); + assert.deepEqual(await readFile(path), bytes); + assert.equal((await call("binary_session")).sha256, sha256); + await call("close_binary"); + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +} diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index 640444e0189..e8a6d18524e 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -46,7 +46,7 @@ export const limitationsFor = (operation: string): readonly string[] => { case "annotate_native_function": return [ ...common, - "Names use Ghidra USER_DEFINED source. Name writes preserve the existing namespace; readback uses the fully qualified name. Regular comments map to PRE and inline comments to EOL at the exact function entry. Changes commit together after readback and refreshed analysis; failure rolls them all back.", + "Names use Ghidra USER_DEFINED source. Name writes accept a leaf name or a fully qualified name within the existing namespace; readback uses the fully qualified name and can be reused without adding namespace prefixes. Edits retain the current namespace; other namespace-like text remains literal leaf-name text. Regular comments map to PRE and inline comments to EOL at the exact function entry. Changes commit together after readback and refreshed analysis; failure rolls them all back.", "Annotation text rejects NUL and unpaired Unicode surrogates before mutation, with the field and UTF-16 index in the error. Supported Unicode and line endings are preserved.", "Metadata edits invalidate immutable analysis snapshots and are discarded on close. CLI returns the updated dossier before session cleanup; this is not a saved Ghidra project.", ]; diff --git a/tests/conformance/ghidra/namespaces.c b/tests/conformance/ghidra/namespaces.c new file mode 100644 index 00000000000..3dc12faa0e8 --- /dev/null +++ b/tests/conformance/ghidra/namespaces.c @@ -0,0 +1,22 @@ +// Emit C++ ABI names with the host C compiler; no C++ runtime is required. +#if defined(__APPLE__) +#define REA_CPP_SYMBOL(name) "_" name +#else +#define REA_CPP_SYMBOL(name) name +#endif + +__attribute__((noinline, used)) int rea_namespace_alpha(int value) + __asm__(REA_CPP_SYMBOL("_ZN5alpha4sameEi")); +__attribute__((noinline, used)) int rea_namespace_beta(int value) + __asm__(REA_CPP_SYMBOL("_ZN4beta4sameEi")); +__attribute__((noinline, used)) int rea_namespace_nested(int value) + __asm__(REA_CPP_SYMBOL("_ZN5outer5inner4sameEi")); + +int rea_namespace_alpha(int value) { return value + 3; } +int rea_namespace_beta(int value) { return value * 2; } +int rea_namespace_nested(int value) { return value - 1; } + +int main(void) { + return rea_namespace_alpha(1) + rea_namespace_beta(2) + + rea_namespace_nested(3); +} From 2847311a579ce2da4c8ded4d89fcdd01545e6da5 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:31:25 +0800 Subject: [PATCH 038/791] fix(hopper): normalize native block endpoints and retain terminal evidence --- bridge/hopper_bridge.py | 41 ++++--- docs/testing.md | 4 +- scripts/lib/real-hopper-boundaries.mjs | 28 +++++ scripts/lib/real-hopper-function-basics.mjs | 115 ++++++++++++++++++++ scripts/verify-real-hopper.mjs | 11 +- tests/fixtures/hopperBridgeFacadeProbe.py | 3 + 6 files changed, 179 insertions(+), 23 deletions(-) diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index d99669a585d..55943217260 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -387,13 +387,26 @@ def _containing_procedure(document, address): return (procedure, None) if procedure is not None else (None, "not_in_procedure") +def _basic_block_end(procedure, block): + """Normalize native block endpoints to an exclusive byte address.""" + end = block.getEndingAddress() + # Hopper 6.1 returns the final instruction address, despite the Python + # documentation describing an exclusive end. Membership distinguishes + # that representation from an actual exclusive endpoint in other builds. + if procedure.getBasicBlockAtAddress(end) == block: + instruction = procedure.getSegment().getInstructionAtAddress(end) + if instruction is not None and instruction.getInstructionLength() > 0: + return end + instruction.getInstructionLength() + return end + + def _instruction_addresses(procedure): result = [] seen = set() segment = procedure.getSegment() for block in procedure.basicBlockIterator(): address = block.getStartingAddress() - end = block.getEndingAddress() + end = _basic_block_end(procedure, block) while address < end and address not in seen: seen.add(address) instruction = segment.getInstructionAtAddress(address) @@ -633,25 +646,11 @@ def _render_instruction(segment, address): def _assembly(procedure): """Render assembly while guarding against malformed instruction cycles.""" - lines = [] segment = procedure.getSegment() - seen = set() - for block in procedure.basicBlockIterator(): - address = block.getStartingAddress() - end = block.getEndingAddress() - while address < end and address not in seen: - seen.add(address) - instruction = segment.getInstructionAtAddress(address) - if instruction is None: - break - line = _render_instruction(segment, address) - if line is not None: - lines.append(line) - length = instruction.getInstructionLength() - if length <= 0: - break - address += length - return "\n".join(lines) + return "\n".join( + line for address in _instruction_addresses(procedure) + if (line := _render_instruction(segment, address)) is not None + ) def _read_function_instructions(document, params): @@ -688,7 +687,7 @@ def _analyze_function(document, params): successors.append(_hex(successor)) blocks.append({ "start": _hex(block.getStartingAddress()), - "end": _hex(block.getEndingAddress()), + "end": _hex(_basic_block_end(procedure, block)), "successors": sorted(set(successors), key=lambda value: int(value, 16)), }) pseudo = _pseudocode(document, procedure) or "" @@ -986,7 +985,7 @@ def _dispatch(method, params): return sorted((_hex(item.getEntryPoint()) for item in procedure.getAllCalleeProcedures()), key=lambda value: int(value, 16)) if method == "procedure_info": blocks = list(procedure.basicBlockIterator()) - length = sum(max(0, block.getEndingAddress() - block.getStartingAddress()) for block in blocks) + length = sum(max(0, _basic_block_end(procedure, block) - block.getStartingAddress()) for block in blocks) return { "name": _procedure_name(procedure), "entrypoint": _hex(procedure.getEntryPoint()), diff --git a/docs/testing.md b/docs/testing.md index f6076d26bec..224af2b91fe 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -211,7 +211,9 @@ CallReference classifications across reference and dossier results, retains long string fragment metadata in dossiers, and exercises pathological regex deadline and cancellation followed by successful requests in the same native session. When the macOS Objective-C fixture is present, the lane also verifies native UTF-16 string objects -and their inferred encodings against their actual bytes. +and their inferred encodings against their actual bytes. Native terminal calls +are checked across reference, instruction, assembly, block-range and procedure-length +projections; block endpoints are normalized using actual native membership. Unmapped annotation destinations and oversized later batch names fail before any earlier edit is applied. The Linux demo lane remains a separate `verify:hopper:linux` command. diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index e54e718109f..8414cbc45a5 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -394,6 +394,33 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { JSON.parse(analyzed.stdout).normalized_result, expectedFunction, ); + let cliTerminalFunctionParity = null; + if (targets.unicode !== undefined) { + const objc = join(directory, `objc-${suffix}`); + await copyFile(targets.unicode, objc); + await call("open_binary", { path: objc }); + const procedures = await call("list_procedures"); + const delegate = procedures.find( + (item) => item.value === "-[REAWidget delegate]", + ); + assert.ok(delegate, "Objective-C fixture omitted its tail-call method"); + const expected = await call("analyze_function", { + procedure: delegate.address, + }); + await call("close_binary"); + const analyzed = await runCli([ + dispatcher, + "function", + objc, + delegate.address, + "--provider", + "hopper", + "--format", + "json", + ]); + assert.deepEqual(JSON.parse(analyzed.stdout).normalized_result, expected); + cliTerminalFunctionParity = true; + } let failure; try { await runCli([ @@ -424,6 +451,7 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { cliInvalidAddress: true, cliLiteralTraceParity: true, cliFunctionDossierParity: true, + cliTerminalFunctionParity, callerCancellationRecovered: true, closedDocumentAbsent: true, }; diff --git a/scripts/lib/real-hopper-function-basics.mjs b/scripts/lib/real-hopper-function-basics.mjs index 998902f2432..b6384fcbc45 100644 --- a/scripts/lib/real-hopper-function-basics.mjs +++ b/scripts/lib/real-hopper-function-basics.mjs @@ -1,3 +1,4 @@ +import assert from "node:assert/strict"; import { requireMcpResult } from "./mcp-verifier-results.mjs"; export const verifyHopperFunctionBasics = async ( @@ -64,3 +65,117 @@ export const verifyHopperFunctionBasics = async ( instructionWindowCount: instructions.instructions.length, }; }; + +/** Check native terminal call sites across instruction, reference and dossier projections. */ +export async function verifyHopperTerminalInstructions(call) { + const procedures = await call("list_procedures"); + const selected = procedures.filter( + ({ value }) => + value === "-[REAWidget delegate]" || + value.startsWith("imp___stubs__") || + value.startsWith("_objc_msgSend$"), + ); + if (selected.length === 0) + throw new Error("Objective-C fixture omitted terminal-call procedures"); + let terminalCalls = 0; + for (const { address } of selected) { + const references = await call("procedure_references", { + procedure: address, + direction: "outgoing", + }); + const nativeCalls = references.references.filter( + (edge) => edge.call !== undefined, + ); + if (nativeCalls.length === 0) + throw new Error( + `Native fixture procedure ${address} omitted its terminal call`, + ); + const dossier = await call("analyze_function", { procedure: address }); + const instructions = await call("read_function_instructions", { + procedure: address, + }); + const assembly = await call("procedure_assembly", { procedure: address }); + const info = await call("procedure_info", { procedure: address }); + for (const edge of nativeCalls) { + const observed = dossier.outgoing_references.find( + (item) => + item.source_address === edge.source_address && + item.target_address === edge.target_address, + ); + assert.ok( + observed, + `Dossier discarded native call ${edge.source_address} -> ${edge.target_address}`, + ); + assert.deepEqual(observed.call, edge.call); + assert.ok( + instructions.instructions.some((line) => + line.startsWith(`${edge.source_address}:`), + ), + `Instruction inventory discarded native call site ${edge.source_address}`, + ); + assert.ok( + dossier.basic_blocks.some( + ({ start, end }) => + BigInt(start) <= BigInt(edge.source_address) && + BigInt(edge.source_address) < BigInt(end), + ), + `Normalized block excluded native call site ${edge.source_address}`, + ); + terminalCalls++; + } + assert.equal(assembly, instructions.instructions.join("\n")); + assert.deepEqual(dossier.assembly, instructions.instructions); + if (address === selected[0].address) + await verifyHopperTerminalComment( + call, + address, + nativeCalls.at(-1).source_address, + ); + const expectedLength = dossier.basic_blocks.reduce( + (total, { start, end }) => total + Number(BigInt(end) - BigInt(start)), + 0, + ); + assert.equal( + info.length, + expectedLength, + "Procedure length disagrees with normalized native blocks", + ); + } + if (terminalCalls === 0) + throw new Error("Objective-C fixture omitted native terminal calls"); + return { + procedures: selected.length, + terminalCalls, + normalizedBlockEndpoints: true, + terminalCommentPreserved: true, + }; +} + +async function verifyHopperTerminalComment(call, address, terminal) { + const original = await call("comment", { address: terminal }); + try { + await call("set_comment", { + address: terminal, + comment: "REA terminal instruction comment", + }); + const annotated = await call("analyze_function", { + procedure: address, + }); + if ( + !annotated.comments.some( + (item) => + item.address === terminal && + item.kind === "comment" && + item.text === "REA terminal instruction comment", + ) + ) + throw new Error( + "Dossier discarded a comment on the native terminal instruction", + ); + } finally { + await call("set_comment", { + address: terminal, + comment: original ?? "", + }); + } +} diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 4277ef1fa69..0fa4830368f 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -30,7 +30,10 @@ import { requireMcpResult, requireWorkflowEvidenceProvider, } from "./lib/mcp-verifier-results.mjs"; -import { verifyHopperFunctionBasics } from "./lib/real-hopper-function-basics.mjs"; +import { + verifyHopperTerminalInstructions, + verifyHopperFunctionBasics, +} from "./lib/real-hopper-function-basics.mjs"; import { openAndVerifyLargeFixture } from "./lib/real-hopper-exhaustive-search.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; import { requireHopperSelection } from "./lib/real-hopper-selection.mjs"; @@ -465,6 +468,7 @@ try { ); if (closed.isError === true) throw new Error(textValue(closed)); let unicodeStrings = null; + let terminalInstructions = null; if (fixtureTargets.unicode !== undefined) { const call = async (name, args = {}) => requireSuccessfulTool( @@ -472,6 +476,7 @@ try { name, ); await call("open_binary", { path: fixtureTargets.unicode.path }); + terminalInstructions = await verifyHopperTerminalInstructions(call); unicodeStrings = await verifyHopperStringObjects(call, [ { value: "REA_UTF16_é_😀", encoding: "utf-16-le" }, { value: 'REA_UTF16_ESCAPED_é_"\\line\nend\t\r', encoding: "utf-16-le" }, @@ -482,6 +487,9 @@ try { const lifecycleAndCli = await verifyHopperLifecycleAndCli(client, options, { primary: targetA, secondary: targetB, + ...(fixtureTargets.unicode === undefined + ? {} + : { unicode: fixtureTargets.unicode.path }), ownedProcessIds, }); const closedSession = requireMcpResult( @@ -509,6 +517,7 @@ try { analyses: [firstAnalysis, secondAnalysis], fixtureAnalysis, unicodeStrings, + terminalInstructions, boundaryContracts, lifecycleAndCli, inventoryCounts, diff --git a/tests/fixtures/hopperBridgeFacadeProbe.py b/tests/fixtures/hopperBridgeFacadeProbe.py index 6fa24f42fe8..195292136e8 100644 --- a/tests/fixtures/hopperBridgeFacadeProbe.py +++ b/tests/fixtures/hopperBridgeFacadeProbe.py @@ -71,6 +71,9 @@ def getSegment(self): def basicBlockIterator(self): return [FakeBlock()] + def getBasicBlockAtAddress(self, address): + return None + def getBasicBlockCount(self): return 1 From a61491ad00b21e912e16f68bff829927aed6e16f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:33:36 +0800 Subject: [PATCH 039/791] docs: add ten README translations and expand language navigation --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 285 +++++++++++++++++ README_es.md | 285 +++++++++++++++++ README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 285 +++++++++++++++++ README_pt-BR.md | 285 +++++++++++++++++ README_ru.md | 285 +++++++++++++++++ README_th.md | 285 +++++++++++++++++ README_tr.md | 285 +++++++++++++++++ README_uk.md | 285 +++++++++++++++++ README_vi.md | 285 +++++++++++++++++ README_zh-TW.md | 287 ++++++++++++++++++ README_zh.md | 2 +- release-please-config.json | 4 - scripts/lib/docs-facts.mjs | 17 +- tests/conformance/readmeConsistency.test.ts | 41 ++- .../conformance/releaseConfiguration.test.ts | 8 +- 19 files changed, 2900 insertions(+), 32 deletions(-) create mode 100644 README_de.md create mode 100644 README_es.md create mode 100644 README_pl.md create mode 100644 README_pt-BR.md create mode 100644 README_ru.md create mode 100644 README_th.md create mode 100644 README_tr.md create mode 100644 README_uk.md create mode 100644 README_vi.md create mode 100644 README_zh-TW.md diff --git a/README.md b/README.md index 4ffd6f4607d..b662aa6d3cc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-**English** · [简体中文](README_zh.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [العربية](README_ar.md) +**English** · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) # REA: Reverse Engineer Anything diff --git a/README_ar.md b/README_ar.md index e8cee241ce0..046602cbfd3 100644 --- a/README_ar.md +++ b/README_ar.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **العربية** +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · **العربية** # REA: هندسة عكسية لأي شيء diff --git a/README_de.md b/README_de.md new file mode 100644 index 00000000000..b544f7e5c99 --- /dev/null +++ b/README_de.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · **Deutsch** · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Alles reverse-engineeren + +### Ein MCP für das Reverse Engineering von Binärdateien, Anwendungen und Laufzeitverhalten. + +**Eine Funktion gefällt dir? Verstehe, wie sie funktioniert – bis hinunter auf die Binärebene.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Website](https://rea.tools/) · [Anleitungen](https://rea.tools/guides/) · [Fallstudien](https://rea.tools/showcase/)** + +[Schnellstart](#schnellstart) · [So funktioniert REA](#so-funktioniert-rea) · [Was du analysieren kannst](#was-du-analysieren-kannst) · [Fallstudien](#fallstudien) · [Häufige Fragen](#häufige-fragen) · [Dokumentation](#dokumentation) + +npx rea-agents setup + +
+ +REA startet seine Analysebrücke in Hopper und untersucht eine native Binärdatei + +
+ + + + + +
+ + Discord
+ Der Reverse-Engineering-Community beitreten +

+ Discord · Fragen und Antworten · Ergebnisse teilen +
+ +
+ +
+ +--- + +Du hast in einer App eine Funktion entdeckt, die du in dein eigenes Produkt übernehmen möchtest? Lass deinen Agenten sie mit REA untersuchen. Er kann die App ohne ihren Quellcode analysieren, die Funktionsweise erklären, Belege zeigen und eine passende Umsetzung für dein Projekt entwickeln. + +REA verbindet deinen Agenten mit Werkzeugen zum Untersuchen nativer Binärdateien, von JavaScript- und Electron-Apps, .NET-Assemblies und Websites. Dieselben Werkzeuge kannst du auch im Terminal nutzen. Die Analyse läuft lokal; die Ergebnisse enthalten die Belege und Einschränkungen hinter jeder Schlussfolgerung. + +Die Einrichtung registriert REA bei deinem Agenten und installiert passende Workflow-Anweisungen. Für native Analysen kannst du eine vorhandene Hopper- oder Ghidra-Installation verwenden. Mit deiner Zustimmung kann die Einrichtung auch Hopper installieren. Statische JavaScript-Analysen benötigen keines der beiden Programme. + +> **[Besuche die REA-Website](https://rea.tools/)** für Einrichtungshinweise, bebilderte Anleitungen und echte Fallstudien. + +## Schnellstart + +### Deinen Agenten einrichten + +Wenn Node.js und npm installiert sind, führe diesen Befehl aus: + +```bash +npx rea-agents setup +``` + +Wähle deine Agenten aus, prüfe die geplanten Änderungen und bestätige sie. Die Einrichtung fügt den MCP-Server von REA und passende Workflow-Anweisungen hinzu und sichert vorhandene Konfigurationen. Starte deinen Agenten anschließend neu. + +Die Einrichtung unterstützt Claude Code, Codex, Cursor, Gemini CLI und [weitere Agenten](docs/installation.md#supported-agents). Informationen zur Provider-Konfiguration und manuellen MCP-Registrierung findest du unter [Installation und Einrichtung](docs/installation.md). + +### Deinen Agenten fragen + +```text +Untersuche, wie die Suche in der Notes-App funktioniert, zeige mir die Belege und entwickle eine ähnliche Funktion für mein Projekt. +``` + +Ersetze Notes durch deine Ziel-App und benenne die Funktion, die du verstehen möchtest. + +### Das Terminal verwenden + +Untersuche ein entpacktes JavaScript-/Electron-App-Verzeichnis oder eine ASAR-Datei: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Das Ergebnis enthält Module, Imports, Electron-Grenzen und die zugehörigen Belege. Ersetze den Pfad durch dein Ziel, beispielsweise `"D:/apps/example"` unter Windows. + +Für die regelmäßige Nutzung kannst du den Befehl `rea` installieren: + +```bash +npm install --global rea-agents +rea --help +``` + +Konfiguriere für native Analysen zuerst einen Provider. Native Befehle, Provider-Auswahl, Snapshots und Skriptnutzung beschreibt der [CLI- und Evidence-Leitfaden](docs/cli.md). + +### REA aktualisieren + +REA entwickelt sich schnell weiter; neue Versionen enthalten häufig Fehlerbehebungen. Halte deine Installation aktuell. + +Für eine über npm installierte CLI: + +```bash +rea update +``` + +Um die Agentenregistrierungen und den Skill zu aktualisieren, führe den Einrichtungsbefehl aus, den das Update ausgibt. + +Wenn du `npx` verwendest, aktualisiere die Agenteneinrichtung mit: + +```bash +npx rea-agents@latest setup +``` + +Prüfe die Änderungen und starte deinen Agenten neu. Für einzelne CLI-Aufrufe verwende `npx rea-agents@latest`, gefolgt vom gewünschten Befehl. + +## So funktioniert REA + +Dein Agent ruft REA über MCP auf, um das Ziel zu untersuchen und relevanten Code zu verfolgen. REA liefert Ergebnisse mit ihren Belegen zurück. Der Agent nutzt sie für weitere Fragen, Erklärungen oder zum Schreiben und Testen einer Implementierung. CLI-Befehle verwenden dieselben Workflows. + +![REA-Untersuchungsablauf: Dein Agent fragt nach einem lokalen Ziel, REA untersucht und verfolgt es mit Analysewerkzeugen, und der Agent nutzt den zurückgegebenen Code, Verweise und offene Fragen zum Erklären, Implementieren und Testen.](website/public/assets/figures/rea-investigation-flow.svg) + +[Abbildung in voller Größe öffnen](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Was du analysieren kannst + +REA benötigt Node.js 22.x (>=22.19), 24.x (>=24.11) oder 26+ sowie npm. Zusätzliche Werkzeuge und unterstützte Hostsysteme hängen vom Ziel ab: + +| Ziel | Was REA zurückgibt | Voraussetzungen und Anleitung | +| ----------------------------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| Native Binärdateien | Pseudocode, Assemblercode, Zeichenketten, Symbole, Aufrufe und Verweise | Hopper, Ghidra oder IDA; [native Analyse](https://rea.tools/guides/native/) | +| Offline-ELF-Struktur | Sektionen, Segmente, ursprüngliche Symbole/Relokationen und mögliche Schutzmaßnahmen aus statischer Analyse | Vom Aufrufer bereitgestelltes pwntools unter Linux x64; [Binärdiagnostik](docs/binary-diagnostics.md) | +| EVM-Bytecode | Dispatch-Selektoren, Byte-Offsets, abgeleitete Argumente und Veränderbarkeit des Zustands | Lokale Eingabe mit Rohbytes oder Hexdarstellung; [Offline-EVM-Leitfaden](docs/evm-bytecode.md) | +| Aufgezeichnete Linux-Abstürze | Unverarbeitete Note-Einträge, Register/Signale aller aufgezeichneten Threads und optionale Zuordnungskandidaten | Vom Aufrufer bereitgestelltes pwntools; optional GDB/pwndbg; [aufgezeichnete Abstürze](docs/recorded-crashes.md) | +| JavaScript / Electron | Module, Imports, Source Maps, Routen, IPC und Beziehungen zu nativen Add-ons | Node.js und npm; [Anwendungsanalyse](https://rea.tools/guides/javascript/) | +| Websites | Seitenstruktur, Skripte, Netzwerkbeobachtungen und angeforderte Screenshots | Ein Browser aus der Chrome-Familie; [Browseranalyse](https://rea.tools/guides/browser/) | +| Gespeicherte Netzwerkaufzeichnungen | Anfragen, Antworten, zugängliche Nutzdaten und Quellpositionen | HAR; mitmdump unter Linux für Aufzeichnungen im nativen mitmproxy-Format; [Leitfaden zu Netzwerkaufzeichnungen](docs/web-network-captures.md) | +| .NET-Assemblies | Metadaten, CIL-Anweisungen, deklarierte native Abhängigkeiten und Build-Vergleiche | Statische Untersuchung; [Leitfaden für verwalteten Code](docs/managed-code-analysis.md) | +| Android-APKs | Manifest-Deklarationen, Klassen, dekompilierte Methoden und Verweise | JADX ohne grafische Oberfläche und ein vollständiges JDK unter Linux/macOS; [Android-Leitfaden](docs/android-analysis.md) | +| Firmware | Bereiche, Extraktionsergebnisse und Übergaben an die native Analyse | Binwalk / Unblob unter Linux; [Firmware-Leitfaden](docs/firmware-analysis.md) | +| Pakete und Ressourcen | Dateiinventare, Hashwerte, plists, Aufbau von Apple-Bundles und extrahierte Ressourcen | [Artefakt- und JavaScript-Leitfaden](docs/javascript-artifact-reconstruction.md), [Apple-Anwendungen](docs/apple-application-analysis.md) | +| Prozessverhalten | Terminalausgabe, Interaktionen, Beobachtungen zu Prozessende und Dateisystem sowie Laufvergleiche | Linux/macOS mit nativer PTY; [Prozessaufzeichnung](docs/process-capture.md) | + +Statische JavaScript- und .NET-Untersuchungen lesen die bereitgestellten Dateien, ohne die Anwendung auszuführen. Laufzeitaufzeichnungen führen das gewählte Ziel mit deinen Benutzerrechten aus oder interagieren damit; die jeweiligen Anleitungen beschreiben die Auswirkungen. + + + +Native Formate und unterstützte Hostsysteme unterscheiden sich je nach Provider. Siehe [Hopper- und Ghidra-Einrichtung](docs/installation.md#hopper), den [IDA-Leitfaden](docs/ida-provider.md) und die [experimentelle Ghidra-Unterstützung unter Windows](docs/windows-ghidra-p0.md). Ghidra unterstützt auch [16-Bit-DOS-Analyse](docs/ghidra-dos.md). Die Provider-Auswahl beschreibt der [CLI-Leitfaden](docs/cli.md#choose-a-provider). Prüfe die [Verfügbarkeit in Releases](docs/installation.md#released-package-and-main) für Funktionen, die seit der neuesten npm-Version hinzugekommen sind. + +## Fallstudien + +### DX-Ball: Eine Stereo-Panning-Berechnung rekonstruieren + +Verfolge einen Sound-Aufruf zur Hilfsfunktion, die Positionen in Panning-Werte umrechnet, untersuche die Anweisungen und überführe unvollständigen Pseudocode in C. Die Rekonstruktion besteht 3.205 Tests mit dem ursprünglichen x86-Code und reproduziert alle 63 Bytes der kompilierten Funktion. + +[Fallstudie lesen](https://rea.tools/showcase/dx-ball/) · [Rekonstruktionsrepository](https://github.com/N0zoM1z0/dx-ball) + +### Notion: Die Electron-Zwischenablagebrücke verfolgen + +Finde die Zwischenablage-API des Renderers, verfolge sie über preload und IPC bis zum Hauptprozess und untersuche das Zwischenablageformat samt Formatierungsdaten. + +[Fallstudie lesen](https://rea.tools/showcase/notion/) + +### TH04: Eine DOS-Geschossring-Berechnung wiederherstellen + +Untersuche die 16-Bit-Anweisungen des ursprünglichen PC-98-Spiels, rekonstruiere die festen und zielgerichteten Winkelberechnungen und vergleiche das rekonstruierte C++ mit der Ausgabe des damaligen Compilers. + +[Fallstudie lesen](https://rea.tools/showcase/th04/) · [Rekonstruktionsrepository](https://github.com/N0zoM1z0/th04) + +Wenn du mit REA etwas Interessantes untersucht hast, würden wir es gern sehen. Teile deine Fallstudie in einem [Issue](https://github.com/morluto/rea/issues) oder [Pull Request](https://github.com/morluto/rea/pulls) und beschreibe das Ziel, deine Frage, wie REA geholfen hat und was du herausgefunden hast. + +## Häufige Fragen + +
+Welche Agenten können REA verwenden? + +Jeder Agent, der lokale MCP-Server unterstützt. Die Einrichtung konfiguriert die [unterstützten Agenten](docs/installation.md#supported-agents); andere Clients können die [manuelle MCP-Registrierung](docs/installation.md#mcp-registry) nutzen. + +
+ +
+Brauche ich Hopper, Ghidra oder IDA? + +Für tiefgehende native Analysen wird eines dieser Programme verwendet. Statische JavaScript- und .NET-Untersuchungen funktionieren ohne native Analyse-Engine. Die Einrichtung kann Hopper nach Zustimmung installieren; Ghidra und IDA nutzen deine vorhandenen Installationen. Siehe [Provider-Einrichtung](docs/installation.md#hopper). + +
+ +
+Muss ich Hopper vorher starten? + +REA startet Hopper, wenn eine Operation es benötigt. Unter macOS kann beim ersten Start ein Dialog erscheinen, in dem du den Demomodus auswählst oder deine Lizenz aktivierst. Siehe [Hopper-Start und Fehlerbehebung](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+Was bewirkt die Installation des Skills über skills.sh? + +Der Skill stellt Untersuchungsanweisungen für deinen Agenten bereit. Verwende `rea setup`, um den MCP-Server von REA zu registrieren und passende Anweisungen zu installieren. Starte deinen Agenten anschließend neu. Siehe [Installation nur des Skills](docs/installation.md#skill-only-installation). + +
+ +
+Welchen Code liefert REA zurück? + +Native Analysen liefern Pseudocode und Assemblercode. JavaScript-/Electron-Analysen rekonstruieren Module und ihre Beziehungen. Dein Agent nutzt diese Ergebnisse zum Schreiben und Testen einer Implementierung; die [Fallstudien](#fallstudien) zeigen konkrete Beispiele. + +
+ +
+Lädt REA meine App hoch? + +REA analysiert Ziele lokal. Dein Agent erhält die Werkzeugergebnisse; sein Modellanbieter hat eine eigene Datenrichtlinie. + +
+ +
+Was soll ich tun, wenn ich einen Fehler finde? + +Aktualisiere zuerst; eine neuere Version könnte den Fehler bereits behoben haben. + +Für eine über npm installierte CLI: + +```bash +rea update +``` + +Für die Agenteneinrichtung über `npx`: + +```bash +npx rea-agents@latest setup +``` + +Wenn du einen Agenten verwendest, schließe die [Aktualisierung der Einrichtung](#rea-aktualisieren) ab und starte ihn neu. Wiederhole dieselbe Aufgabe. Besteht das Problem weiterhin, [öffne ein Issue](https://github.com/morluto/rea/issues) mit deiner REA-Version, dem Zieltyp, den Reproduktionsschritten und der Fehlerausgabe. + +
+ +## Dokumentation + +Beginne mit den [praxisnahen Anleitungen](https://rea.tools/guides/) auf der Website. Genaue Optionen, Voraussetzungen und Ergebniskontrakte findest du hier: + +- [Installation und Einrichtung](docs/installation.md): Agentenregistrierung, Provider-Konfiguration, Updates und Deinstallation. +- [Bereitschaft und Fehlerbehebung](docs/installation.md#check-readiness-for-your-task): Diagnose eines bestimmten Agenten oder einer Analyse-Engine. +- [CLI und Evidence](docs/cli.md): Befehle, Provider-Auswahl, Snapshots, Import/Export und Exit-Status. +- [MCP-Kontrakte](docs/mcp-contracts.md) und [Agenten-Prompts](docs/mcp-prompts.md): Werkzeugergebnisse, Sitzungen und geführte Untersuchungen. +- [Werkzeugkatalog](docs/mcp-contracts.md#generated-catalog): Beim Build erzeugtes Inventar von Werkzeugen, Providern und CLI-Befehlen. +- [Roadmap](docs/roadmap.md): Geplante Arbeiten und Fortschrittsübersichten zu Fähigkeiten. + +Melde Sicherheitslücken gemäß [SECURITY.md](SECURITY.md). + +## Mitwirken + +Wir freuen uns über deine Hilfe bei REA! [Öffne ein Issue](https://github.com/morluto/rea/issues), um einen Fehler zu melden oder eine Funktion vorzuschlagen, oder [sende einen Pull Request](https://github.com/morluto/rea/pulls), um Code oder Dokumentation zu verbessern. + +Die Entwicklungsumgebung und Prüfungen beschreibt [CONTRIBUTING.md](CONTRIBUTING.md), die Verifikationspfade der [Testleitfaden](docs/testing.md) und die Projektstruktur die [Architekturübersicht](docs/architecture.mermaid). + +## Projektlinks + +[Website](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Sicherheit](SECURITY.md) + +## Sternverlauf + +🎉 **20.000 GitHub-Sterne – vielen Dank!** + +Danke an alle, die REA nutzen, Fehler melden, Builds testen und Fehlerbehebungen beitragen. + + + + + + GitHub-Sternverlauf von REA + + + +## Haftungsausschluss + +REA stellt Werkzeuge für rechtmäßige Reverse-Engineering-Forschung, Analyse und Rekonstruktion bereit. Du bist dafür verantwortlich, erforderliche Genehmigungen einzuholen und geltendes Recht einzuhalten. Das Projekt unterstützt keine rechtswidrige oder unbefugte Nutzung. + +## Lizenz + +[MIT](LICENSE) diff --git a/README_es.md b/README_es.md new file mode 100644 index 00000000000..3b2c407419f --- /dev/null +++ b/README_es.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · **Español** · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Ingeniería inversa de cualquier cosa + +### Un solo MCP para la ingeniería inversa de binarios, aplicaciones y comportamiento en ejecución. + +**¿Ves una función que te gusta? Entiende cómo funciona, hasta el nivel binario.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Sitio web](https://rea.tools/) · [Guías](https://rea.tools/guides/) · [Casos prácticos](https://rea.tools/showcase/)** + +[Inicio rápido](#inicio-rápido) · [Cómo funciona REA](#cómo-funciona-rea) · [Qué puedes analizar](#qué-puedes-analizar) · [Casos prácticos](#casos-prácticos) · [Preguntas frecuentes](#preguntas-frecuentes) · [Documentación](#documentación) + +npx rea-agents setup + +
+ +REA inicia su puente de análisis dentro de Hopper mientras inspecciona un binario nativo + +
+ + + + + +
+ + Discord
+ Únete a la comunidad de ingeniería inversa +

+ Discord · Preguntas y respuestas · Comparte tus resultados +
+ +
+ +
+ +--- + +¿Has visto una función en una aplicación que quieres incorporar a tu producto? Pide a tu agente que la investigue con REA. Puede inspeccionar la aplicación sin su código fuente, explicar cómo funciona la función, mostrar las pruebas y crear una versión para tu proyecto. + +REA conecta tu agente con herramientas para inspeccionar binarios nativos, aplicaciones JavaScript y Electron, ensamblados .NET y sitios web. También puedes usar las mismas herramientas desde la terminal. El análisis se ejecuta localmente y los resultados incluyen las pruebas y limitaciones que sustentan cada conclusión. + +La configuración registra REA en tu agente e instala las instrucciones de trabajo correspondientes. El análisis nativo puede usar una instalación existente de Hopper o Ghidra; la configuración también puede instalar Hopper, si lo autorizas. El análisis estático de JavaScript no necesita ninguno de los dos motores. + +> **[Visita el sitio web de REA](https://rea.tools/)** para encontrar instrucciones de configuración, guías ilustradas y casos reales. + +## Inicio rápido + +### Configura tu agente + +Con Node.js y npm instalados, ejecuta: + +```bash +npx rea-agents setup +``` + +Elige tus agentes, revisa los cambios propuestos y apruébalos. La configuración añade el servidor MCP de REA y las instrucciones de trabajo correspondientes, con copias de seguridad de la configuración existente. Reinicia tu agente al terminar. + +La configuración admite Claude Code, Codex, Cursor, Gemini CLI y [otros agentes](docs/installation.md#supported-agents). Consulta [instalación y configuración](docs/installation.md) para configurar proveedores y registrar MCP manualmente. + +### Pregunta a tu agente + +```text +Investiga cómo funciona la búsqueda en la aplicación Notes, muéstrame las pruebas y crea una función similar para mi proyecto. +``` + +Sustituye Notes por la aplicación objetivo y especifica la función que quieres entender. + +### Usa la terminal + +Inspecciona un directorio extraído de una aplicación JavaScript/Electron o un archivo ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +El resultado incluye módulos, importaciones, límites de Electron y sus pruebas. Sustituye la ruta por la de tu objetivo, por ejemplo `"D:/apps/example"` en Windows. + +Para instalar el comando `rea` y usarlo habitualmente: + +```bash +npm install --global rea-agents +rea --help +``` + +Para el análisis nativo, configura primero un proveedor. Consulta la [guía de CLI y Evidence](docs/cli.md) para los comandos nativos, la selección de proveedores, las instantáneas y el uso en scripts. + +### Actualiza REA + +REA cambia rápidamente y las nuevas versiones incluyen correcciones frecuentes. Mantén tu instalación actualizada. + +Para una CLI instalada mediante npm: + +```bash +rea update +``` + +Para actualizar los registros de tus agentes y la skill, ejecuta el comando de configuración que muestra la actualización. + +Si usas `npx`, actualiza la configuración de tu agente con: + +```bash +npx rea-agents@latest setup +``` + +Revisa los cambios de configuración y reinicia tu agente. Para ejecutar comandos de CLI puntuales, usa `npx rea-agents@latest` seguido del comando. + +## Cómo funciona REA + +Tu agente llama a REA mediante MCP para inspeccionar el objetivo y seguir el código relevante. REA devuelve los hallazgos junto con sus pruebas. El agente los utiliza para hacer preguntas de seguimiento, explicar el comportamiento o escribir y probar una implementación. Los comandos de CLI usan los mismos flujos de trabajo. + +![Flujo de investigación de REA: tu agente pregunta sobre un objetivo local, REA lo inspecciona y rastrea con herramientas de análisis, y el agente utiliza el código, las referencias y los aspectos desconocidos devueltos para explicar, implementar y probar.](website/public/assets/figures/rea-investigation-flow.svg) + +[Abre la figura a tamaño completo](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Qué puedes analizar + +REA requiere Node.js 22.x (>=22.19), 24.x (>=24.11) o 26+, además de npm. Las herramientas adicionales y los sistemas anfitriones compatibles dependen del objetivo: + +| Objetivo | Qué devuelve REA | Requisitos y guía | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Binarios nativos | Pseudocódigo, ensamblador, cadenas, símbolos, llamadas y referencias | Hopper, Ghidra o IDA; [análisis nativo](https://rea.tools/guides/native/) | +| Estructura ELF sin ejecutar | Secciones, segmentos, símbolos/reubicaciones originales y posibles medidas de mitigación detectadas mediante análisis estático | pwntools proporcionado por quien llama, en Linux x64; [diagnóstico de binarios](docs/binary-diagnostics.md) | +| Bytecode EVM | Selectores de despacho, desplazamientos en bytes, argumentos inferidos y mutabilidad | Entrada local con bytes sin procesar o representación hexadecimal; [guía de EVM sin ejecutar](docs/evm-bytecode.md) | +| Fallos de Linux registrados | Registros note sin procesar, registros/señales de cada hilo registrado y posibles correspondencias de memoria opcionales | pwntools proporcionado por quien llama; GDB/pwndbg opcionales; [fallos registrados](docs/recorded-crashes.md) | +| JavaScript / Electron | Módulos, importaciones, mapas de código fuente, rutas, IPC y relaciones con complementos nativos | Node.js y npm; [análisis de aplicaciones](https://rea.tools/guides/javascript/) | +| Sitios web | Estructura de la página, scripts, observaciones de red y capturas de pantalla solicitadas | Un navegador de la familia Chrome; [análisis del navegador](https://rea.tools/guides/browser/) | +| Capturas de red guardadas | Solicitudes, respuestas, contenido accesible de las cargas útiles y ubicaciones de origen | HAR; mitmdump en Linux para capturas en formato nativo de mitmproxy; [guía de capturas](docs/web-network-captures.md) | +| Ensamblados .NET | Metadatos, instrucciones CIL, dependencias nativas declaradas y comparaciones de compilaciones | Inspección estática; [guía de código administrado](docs/managed-code-analysis.md) | +| APK de Android | Declaraciones del manifiesto, clases, métodos descompilados y referencias | JADX sin interfaz gráfica y un JDK completo en Linux/macOS; [guía de Android](docs/android-analysis.md) | +| Firmware | Regiones, resultados de extracción y derivaciones al análisis nativo | Binwalk / Unblob en Linux; [guía de firmware](docs/firmware-analysis.md) | +| Paquetes y recursos | Inventarios de archivos, resúmenes criptográficos, plists, estructura de paquetes de Apple y recursos extraídos | [Guía de artefactos y JavaScript](docs/javascript-artifact-reconstruction.md), [aplicaciones de Apple](docs/apple-application-analysis.md) | +| Comportamiento de procesos | Salida de la terminal, interacciones, observaciones de salida y del sistema de archivos, y comparaciones entre ejecuciones | Linux/macOS con una PTY nativa; [captura de procesos](docs/process-capture.md) | + +La inspección estática de JavaScript y .NET lee los archivos suministrados sin ejecutar la aplicación. La captura en ejecución inicia el objetivo seleccionado o interactúa con él con los permisos de tu usuario; cada guía describe sus efectos. + + + +Los formatos nativos y los sistemas anfitriones compatibles varían según el proveedor. Consulta [configuración de Hopper y Ghidra](docs/installation.md#hopper), la [guía de IDA](docs/ida-provider.md) y la [compatibilidad experimental de Ghidra en Windows](docs/windows-ghidra-p0.md). Ghidra también admite [análisis de DOS de 16 bits](docs/ghidra-dos.md). Para elegir un proveedor, consulta la [guía de CLI](docs/cli.md#choose-a-provider). Comprueba la [disponibilidad por versión](docs/installation.md#released-package-and-main) de las funciones añadidas después de la última versión publicada en npm. + +## Casos prácticos + +### DX-Ball: reconstruir un cálculo de panoramización de sonido + +Sigue una llamada de sonido hasta la función auxiliar que convierte la posición en panoramización, inspecciona las instrucciones y transforma el pseudocódigo incompleto en C. La reconstrucción supera 3.205 casos con el x86 original y reproduce los 63 bytes de la función compilada. + +[Lee el caso práctico](https://rea.tools/showcase/dx-ball/) · [Repositorio de reconstrucción](https://github.com/N0zoM1z0/dx-ball) + +### Notion: seguir el puente del portapapeles de Electron + +Encuentra la API del portapapeles del proceso de renderizado, síguela a través de preload e IPC hasta el proceso principal e inspecciona el formato enriquecido del portapapeles. + +[Lee el caso práctico](https://rea.tools/showcase/notion/) + +### TH04: recuperar el cálculo de un anillo de proyectiles en DOS + +Inspecciona las instrucciones de 16 bits del juego original para PC-98, recupera los cálculos de ángulos fijos y dirigidos, y compara el C++ reconstruido con la salida del compilador de la época. + +[Lee el caso práctico](https://rea.tools/showcase/th04/) · [Repositorio de reconstrucción](https://github.com/N0zoM1z0/th04) + +Si has usado REA para investigar algo interesante, nos gustaría verlo. Comparte tu caso en una [issue](https://github.com/morluto/rea/issues) o una [pull request](https://github.com/morluto/rea/pulls), incluyendo el objetivo, tu pregunta, cómo te ayudó REA y qué encontraste. + +## Preguntas frecuentes + +
+¿Qué agentes pueden usar REA? + +Cualquier agente que admita servidores MCP locales. La configuración prepara los [agentes compatibles](docs/installation.md#supported-agents); otros clientes pueden usar el [registro manual de MCP](docs/installation.md#mcp-registry). + +
+ +
+¿Necesito Hopper, Ghidra o IDA? + +El análisis nativo en profundidad utiliza uno de ellos. La inspección estática de JavaScript y .NET funciona sin un motor de análisis nativo. La configuración puede instalar Hopper después de tu aprobación; Ghidra e IDA usan las instalaciones que ya tienes. Consulta [configuración de proveedores](docs/installation.md#hopper). + +
+ +
+¿Tengo que iniciar Hopper primero? + +REA inicia Hopper cuando una operación lo necesita. En macOS, puede aparecer un diálogo al iniciarlo por primera vez para elegir el modo de demostración o activar tu licencia. Consulta [inicio de Hopper y resolución de problemas](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+¿Qué hace la instalación de la skill desde skills.sh? + +La skill proporciona instrucciones de investigación a tu agente. Usa `rea setup` para registrar el servidor MCP de REA e instalar las instrucciones correspondientes; después reinicia tu agente. Consulta [instalación solo de la skill](docs/installation.md#skill-only-installation). + +
+ +
+¿Qué código devuelve REA? + +El análisis nativo devuelve pseudocódigo y ensamblador. El análisis de JavaScript/Electron recupera módulos y sus relaciones. Tu agente utiliza estos hallazgos para escribir y probar una implementación; los [casos prácticos](#casos-prácticos) ofrecen ejemplos completos. + +
+ +
+¿REA sube mi aplicación? + +REA analiza los objetivos localmente. Tu agente recibe los resultados de las herramientas, y su proveedor de modelos tiene su propia política de datos. + +
+ +
+¿Qué debo hacer si encuentro un error? + +Actualiza primero; una versión reciente puede haber solucionado el problema. + +Para una CLI instalada mediante npm: + +```bash +rea update +``` + +Para configurar el agente mediante `npx`: + +```bash +npx rea-agents@latest setup +``` + +Si usas un agente, completa la [actualización de la configuración](#actualiza-rea) y reinícialo. Repite la misma tarea. Si el problema continúa, [abre una issue](https://github.com/morluto/rea/issues) con tu versión de REA, el tipo de objetivo, los pasos para reproducirlo y la salida del error. + +
+ +## Documentación + +Empieza con las [guías prácticas](https://rea.tools/guides/) del sitio web. Para opciones exactas, requisitos y contratos de resultados: + +- [Instalación y configuración](docs/installation.md): registro de agentes, configuración de proveedores, actualizaciones y desinstalación. +- [Preparación y resolución de problemas](docs/installation.md#check-readiness-for-your-task): diagnóstico de un agente o motor de análisis concreto. +- [CLI y Evidence](docs/cli.md): comandos, selección de proveedores, instantáneas, importación/exportación y estados de salida. +- [Contratos de MCP](docs/mcp-contracts.md) y [prompts para agentes](docs/mcp-prompts.md): resultados de herramientas, sesiones e investigaciones guiadas. +- [Catálogo de herramientas](docs/mcp-contracts.md#generated-catalog): inventario de herramientas, proveedores y comandos de CLI generado durante la compilación. +- [Hoja de ruta](docs/roadmap.md): trabajo previsto y seguimiento de capacidades. + +Informa de vulnerabilidades siguiendo [SECURITY.md](SECURITY.md). + +## Contribuir + +¡Nos encantaría que ayudaras con REA! [Abre una issue](https://github.com/morluto/rea/issues) para comunicar un error o proponer una función, o [envía una pull request](https://github.com/morluto/rea/pulls) para mejorar el código o la documentación. + +Consulta [CONTRIBUTING.md](CONTRIBUTING.md) para preparar el entorno de desarrollo y las comprobaciones, la [guía de pruebas](docs/testing.md) para los procesos de verificación y el [mapa de arquitectura](docs/architecture.mermaid) para la estructura del proyecto. + +## Enlaces del proyecto + +[Sitio web](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Seguridad](SECURITY.md) + +## Historial de estrellas + +🎉 **20.000 estrellas en GitHub: ¡gracias!** + +Gracias a quienes usan REA, informan de errores, prueban compilaciones y contribuyen con correcciones. + + + + + + Historial de estrellas de REA en GitHub + + + +## Aviso legal + +REA proporciona herramientas para la investigación, el análisis y la reconstrucción mediante ingeniería inversa conforme a la ley. Eres responsable de obtener las autorizaciones necesarias y cumplir las leyes aplicables. El proyecto no respalda usos ilegales ni no autorizados. + +## Licencia + +[MIT](LICENSE) diff --git a/README_ja.md b/README_ja.md index c25a0c13ccb..4c9e798252c 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · **日本語** · [한국어](README_ko.md) · [العربية](README_ar.md) +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · **日本語** · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) # REA:あらゆるものをリバースエンジニアリング diff --git a/README_ko.md b/README_ko.md index 9f6787042c3..2368c34485a 100644 --- a/README_ko.md +++ b/README_ko.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [日本語](README_ja.md) · **한국어** · [العربية](README_ar.md) +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · **한국어** · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) # REA: 무엇이든 리버스 엔지니어링 diff --git a/README_pl.md b/README_pl.md new file mode 100644 index 00000000000..f08507b803c --- /dev/null +++ b/README_pl.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · **Polski** · [العربية](README_ar.md) + +# REA: Reverse Engineer Anything + +### Jeden serwer MCP do inżynierii wstecznej plików binarnych, aplikacji i zachowania w czasie wykonywania. + +**Widzisz funkcję, która Ci się podoba? Dowiedz się, jak działa, aż do poziomu kodu binarnego.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Strona internetowa](https://rea.tools/) · [Przewodniki](https://rea.tools/guides/) · [Przykłady zastosowań](https://rea.tools/showcase/)** + +[Szybki start](#szybki-start) · [Jak działa REA](#jak-działa-rea) · [Co możesz analizować](#co-możesz-analizować) · [Przykłady zastosowań](#przykłady-zastosowań) · [FAQ](#faq) · [Dokumentacja](#dokumentacja) + +npx rea-agents setup + +
+ +REA uruchamia most analityczny w Hopperze podczas analizy natywnego pliku binarnego + +
+ + + + + +
+ + Discord
+ Dołącz do społeczności inżynierii wstecznej +

+ Discord · Pytania i odpowiedzi · Pokaż swój projekt +
+ +
+ +
+ +--- + +Widzisz w aplikacji funkcję, którą chcesz mieć we własnym produkcie? Poproś swojego agenta, aby zbadał ją za pomocą REA. Może przeanalizować aplikację bez jej kodu źródłowego, wyjaśnić działanie funkcji, pokazać dowody i stworzyć jej wersję dla Twojego projektu. + +REA łączy Twojego agenta z narzędziami do analizy natywnych plików binarnych, aplikacji JavaScript i Electron, zestawów .NET oraz stron internetowych. Z tych samych narzędzi możesz korzystać w terminalu. Analiza odbywa się lokalnie, a wyniki zawierają dowody i ograniczenia stojące za każdym wnioskiem. + +Konfigurator rejestruje REA w Twoim agencie i instaluje pasujące instrukcje pracy. Analiza kodu natywnego może korzystać z istniejącej instalacji Hoppera lub Ghidry; konfigurator może też zainstalować Hoppera po uzyskaniu zgody. Statyczna analiza JavaScript nie wymaga żadnego z tych silników. + +> **[Odwiedź stronę REA](https://rea.tools/)**, aby znaleźć instrukcje konfiguracji, ilustrowane przewodniki i rzeczywiste studia przypadków. + +## Szybki start + +### Skonfiguruj agenta + +Po zainstalowaniu Node.js i npm uruchom: + +```bash +npx rea-agents setup +``` + +Wybierz swoich agentów, sprawdź proponowane zmiany i zatwierdź je. Konfigurator dodaje serwer MCP REA i pasujące instrukcje pracy, tworząc kopie zapasowe istniejącej konfiguracji. Następnie uruchom agenta ponownie. + +Konfigurator obsługuje Claude Code, Codex, Cursor, Gemini CLI i [innych agentów](docs/installation.md#supported-agents). Konfigurację dostawców analizy i ręczną rejestrację MCP opisuje przewodnik [instalacji i konfiguracji](docs/installation.md). + +### Zapytaj agenta + +```text +Sprawdź, jak działa wyszukiwanie w aplikacji Notes, pokaż mi dowody i stwórz podobną funkcję dla mojego projektu. +``` + +Zastąp Notes wybraną aplikacją i wskaż funkcję, którą chcesz zrozumieć. + +### Korzystaj z terminala + +Przeanalizuj rozpakowany katalog aplikacji JavaScript/Electron lub archiwum ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Wynik obejmuje moduły, importy, granice komponentów Electron oraz dowody. Zastąp ścieżkę ścieżką do swojego celu, na przykład `"D:/apps/example"` w systemie Windows. + +Aby zainstalować polecenie `rea` do regularnego użytku: + +```bash +npm install --global rea-agents +rea --help +``` + +Przed analizą kodu natywnego skonfiguruj dostawcę analizy. Polecenia analizy natywnej, wybór dostawcy, migawki i użycie w skryptach opisuje [przewodnik CLI i Evidence](docs/cli.md). + +### Zaktualizuj REA + +REA szybko się rozwija, a nowe wydania często zawierają poprawki błędów. Dbaj o aktualność swojej instalacji. + +Dla CLI zainstalowanego przez npm: + +```bash +rea update +``` + +Aby odświeżyć rejestracje agentów i umiejętność, uruchom polecenie konfiguracji wyświetlone po aktualizacji. + +Jeśli używasz `npx`, zaktualizuj konfigurację agenta poleceniem: + +```bash +npx rea-agents@latest setup +``` + +Sprawdź zmiany konfiguracji i uruchom agenta ponownie. Do jednorazowych poleceń CLI używaj `npx rea-agents@latest`, a następnie nazwy polecenia. + +## Jak działa REA + +Twój agent wywołuje REA przez MCP, aby analizować cel i śledzić istotny kod. REA zwraca ustalenia wraz z dowodami. Agent wykorzystuje je do zadawania dalszych pytań, wyjaśniania zachowania lub pisania i testowania implementacji. Polecenia CLI korzystają z tych samych procedur. + +![Przebieg badania z REA: agent pyta o lokalny cel, REA analizuje go i śledzi za pomocą narzędzi analitycznych, a agent korzysta ze zwróconego kodu, odwołań i niewiadomych, aby wyjaśnić działanie, stworzyć implementację i ją przetestować.](website/public/assets/figures/rea-investigation-flow.svg) + +[Otwórz ilustrację w pełnym rozmiarze](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Co możesz analizować + +REA wymaga Node.js 22.x (>=22.19), 24.x (>=24.11) lub 26+ oraz npm. Dodatkowe narzędzia i obsługiwane platformy zależą od celu analizy: + +| Cel | Co zwraca REA | Wymagania i przewodnik | +| --------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | +| Natywne pliki binarne | Pseudokod, kod asemblera, ciągi znaków, symbole, wywołania i odwołania | Hopper, Ghidra lub IDA; [analiza kodu natywnego](https://rea.tools/guides/native/) | +| Układ pliku ELF w trybie offline | Sekcje, segmenty, oryginalne symbole/relokacje i potencjalne mechanizmy zabezpieczeń wskazane przez analizę statyczną | pwntools dostarczone przez użytkownika w systemie Linux x64; [diagnostyka plików binarnych](docs/binary-diagnostics.md) | +| Kod bajtowy EVM | Selektory wywołań, przesunięcia bajtowe, wywnioskowane argumenty i mutowalność | Lokalny plik z surowymi bajtami lub zapisem szesnastkowym; [przewodnik EVM offline](docs/evm-bytecode.md) | +| Zarejestrowane awarie systemu Linux | Surowe rekordy note, rejestry/sygnały każdego zarejestrowanego wątku i opcjonalni kandydaci na mapowania | pwntools dostarczone przez użytkownika; opcjonalnie GDB/pwndbg; [zarejestrowane awarie](docs/recorded-crashes.md) | +| JavaScript / Electron | Moduły, importy, mapy źródeł, trasy, IPC i powiązania z natywnymi dodatkami | Node.js i npm; [analiza aplikacji](https://rea.tools/guides/javascript/) | +| Strony internetowe | Struktura strony, skrypty, obserwacje sieciowe i zamówione zrzuty ekranu | Przeglądarka z rodziny Chrome; [analiza w przeglądarce](https://rea.tools/guides/browser/) | +| Zapisane przechwycenia ruchu sieciowego | Żądania, odpowiedzi, dostępne dane przesyłane i lokalizacje źródłowe | HAR; mitmdump w systemie Linux dla natywnych przechwyceń mitmproxy; [przewodnik przechwytywania](docs/web-network-captures.md) | +| Zestawy .NET | Metadane, instrukcje CIL, zadeklarowane zależności natywne i porównania kompilacji | Analiza statyczna; [przewodnik kodu zarządzanego](docs/managed-code-analysis.md) | +| Pakiety APK dla Androida | Deklaracje manifestu, klasy, zdekompilowane metody i odwołania | JADX bez interfejsu graficznego i pełny JDK w systemie Linux/macOS; [przewodnik Androida](docs/android-analysis.md) | +| Oprogramowanie układowe | Regiony, wyniki wyodrębniania i przekazanie do analizy kodu natywnego | Binwalk / Unblob w systemie Linux; [przewodnik oprogramowania układowego](docs/firmware-analysis.md) | +| Pakiety i zasoby | Spisy plików, skróty, pliki plist, struktura pakietów Apple i wyodrębnione zasoby | [Przewodnik artefaktów i JavaScript](docs/javascript-artifact-reconstruction.md), [aplikacje Apple](docs/apple-application-analysis.md) | +| Zachowanie procesów | Wyjście terminala, interakcje, obserwacje zakończenia i systemu plików oraz porównania uruchomień | Linux/macOS z natywnym PTY; [przechwytywanie procesów](docs/process-capture.md) | + +Statyczna analiza JavaScript i .NET odczytuje dostarczone pliki bez uruchamiania aplikacji. Przechwytywanie w czasie wykonywania uruchamia wybrany cel lub wchodzi z nim w interakcje z uprawnieniami Twojego użytkownika; skutki opisano w odpowiednich przewodnikach. + + + +Obsługiwane formaty natywne i platformy zależą od dostawcy analizy. Zobacz [konfigurację Hoppera i Ghidry](docs/installation.md#hopper), [przewodnik IDA](docs/ida-provider.md) oraz [eksperymentalną obsługę Ghidry w systemie Windows](docs/windows-ghidra-p0.md). Ghidra obsługuje też [analizę 16-bitowego DOS-a](docs/ghidra-dos.md). Wybór dostawcy opisuje [przewodnik CLI](docs/cli.md#choose-a-provider). Sprawdź [dostępność w wydaniach](docs/installation.md#released-package-and-main) dla funkcji dodanych po najnowszym wydaniu npm. + +## Przykłady zastosowań + +### DX-Ball: odtworzenie obliczania panoramy dźwięku + +Prześledź wywołanie dźwięku do funkcji pomocniczej przeliczającej pozycję na panoramę, sprawdź instrukcje i przekształć niepełny pseudokod w C. Odtworzona implementacja przechodzi 3 205 przypadków dla oryginalnego x86 i odtwarza wszystkie 63 bajty skompilowanej funkcji. + +[Przeczytaj studium przypadku](https://rea.tools/showcase/dx-ball/) · [Repozytorium odtworzonej implementacji](https://github.com/N0zoM1z0/dx-ball) + +### Notion: śledzenie mostu schowka w Electron + +Znajdź API schowka w procesie renderującym, prześledź je przez preload i IPC do procesu głównego, a następnie sprawdź format schowka wraz z danymi formatowania. + +[Przeczytaj studium przypadku](https://rea.tools/showcase/notion/) + +### TH04: odtworzenie obliczania pierścienia pocisków w DOS-ie + +Sprawdź 16-bitowe instrukcje oryginalnej gry na PC-98, odtwórz obliczenia kątów stałych i skierowanych na cel oraz porównaj odtworzony C++ z wynikiem historycznego kompilatora. + +[Przeczytaj studium przypadku](https://rea.tools/showcase/th04/) · [Repozytorium odtworzonej implementacji](https://github.com/N0zoM1z0/th04) + +Jeśli użyłeś REA do czegoś ciekawego, chętnie to zobaczymy. Opisz swój przypadek w [zgłoszeniu](https://github.com/morluto/rea/issues) lub [pull requeście](https://github.com/morluto/rea/pulls), podając cel analizy, pytanie, sposób, w jaki REA pomogło, i swoje ustalenia. + +## FAQ + +
+Którzy agenci mogą korzystać z REA? + +Każdy agent obsługujący lokalne serwery MCP. Konfigurator konfiguruje [obsługiwanych agentów](docs/installation.md#supported-agents); inne klienty mogą użyć [ręcznej rejestracji MCP](docs/installation.md#mcp-registry). + +
+ +
+Czy potrzebuję Hoppera, Ghidry lub IDA? + +Dogłębna analiza kodu natywnego używa jednego z nich. Statyczna analiza JavaScript i .NET działa bez silnika analizy natywnej. Konfigurator może zainstalować Hoppera po uzyskaniu zgody; Ghidra i IDA korzystają z istniejących instalacji. Zobacz [konfigurację dostawców analizy](docs/installation.md#hopper). + +
+ +
+Czy muszę najpierw uruchomić Hoppera? + +REA uruchamia Hoppera, gdy wymaga tego operacja. W systemie macOS przy pierwszym uruchomieniu okno dialogowe może poprosić o wybór trybu demonstracyjnego lub aktywację licencji. Zobacz [uruchamianie Hoppera i rozwiązywanie problemów](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+Co daje zainstalowanie umiejętności z skills.sh? + +Umiejętność dostarcza agentowi instrukcje prowadzenia analizy. Użyj `rea setup`, aby zarejestrować serwer MCP REA i zainstalować pasujące instrukcje, a następnie uruchom agenta ponownie. Zobacz [instalację samej umiejętności](docs/installation.md#skill-only-installation). + +
+ +
+Jaki kod zwraca REA? + +Analiza kodu natywnego zwraca pseudokod i kod asemblera. Analiza JavaScript/Electron odzyskuje moduły i ich powiązania. Agent wykorzystuje te ustalenia do napisania i przetestowania implementacji; [przykłady zastosowań](#przykłady-zastosowań) pokazują ten proces w praktyce. + +
+ +
+Czy REA przesyła moją aplikację? + +REA analizuje cele lokalnie. Twój agent otrzymuje wyniki narzędzi, a dostawca jego modelu ma własną politykę dotyczącą danych. + +
+ +
+Co zrobić, gdy napotkam błąd? + +Najpierw zaktualizuj REA; najnowsze wydanie mogło już naprawić ten problem. + +Dla CLI zainstalowanego przez npm: + +```bash +rea update +``` + +Dla konfiguracji agenta przez `npx`: + +```bash +npx rea-agents@latest setup +``` + +Jeśli używasz agenta, wykonaj [odświeżenie konfiguracji](#zaktualizuj-rea) i uruchom go ponownie. Ponów tę samą operację. Jeśli problem pozostaje, [otwórz zgłoszenie](https://github.com/morluto/rea/issues), podając wersję REA, typ celu analizy, kroki odtworzenia problemu i komunikaty błędu. + +
+ +## Dokumentacja + +Zacznij od [przewodników z przykładami](https://rea.tools/guides/) na stronie internetowej. Dokładne opcje, wymagania wstępne i kontrakty wyników opisano tutaj: + +- [Instalacja i konfiguracja](docs/installation.md): rejestracja agentów, konfiguracja dostawców analizy, aktualizacje i odinstalowanie. +- [Gotowość i rozwiązywanie problemów](docs/installation.md#check-readiness-for-your-task): diagnostyka jednego agenta lub silnika analizy. +- [CLI i Evidence](docs/cli.md): polecenia, wybór dostawcy, migawki, import/eksport i kody zakończenia. +- [Kontrakty MCP](docs/mcp-contracts.md) i [prompty dla agentów](docs/mcp-prompts.md): wyniki narzędzi, sesje i prowadzone analizy. +- [Katalog narzędzi](docs/mcp-contracts.md#generated-catalog): generowany podczas kompilacji wykaz narzędzi, dostawców i poleceń CLI. +- [Plan rozwoju](docs/roadmap.md): planowane prace i śledzenie funkcjonalności. + +Zgłaszaj podatności zgodnie z [SECURITY.md](SECURITY.md). + +## Współtworzenie + +Chętnie przyjmiemy Twoją pomoc w rozwoju REA! [Otwórz zgłoszenie](https://github.com/morluto/rea/issues), aby poinformować o błędzie lub zaproponować funkcję, albo [wyślij pull request](https://github.com/morluto/rea/pulls) z ulepszeniami kodu lub dokumentacji. + +Zobacz [CONTRIBUTING.md](CONTRIBUTING.md), aby poznać konfigurację środowiska programistycznego i kontrole, [testowanie](docs/testing.md), aby poznać ścieżki weryfikacji, oraz [mapę architektury](docs/architecture.mermaid), aby poznać strukturę projektu. + +## Linki projektu + +[Strona internetowa](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Zgłoszenia](https://github.com/morluto/rea/issues) · [Bezpieczeństwo](SECURITY.md) + +## Historia gwiazdek + +🎉 **20 000 gwiazdek na GitHubie — dziękujemy!** + +Dziękujemy wszystkim, którzy używają REA, zgłaszają błędy, testują kompilacje i przesyłają poprawki. + + + + + + Historia gwiazdek REA na GitHubie + + + +## Zastrzeżenie + +REA dostarcza narzędzia do zgodnych z prawem badań z zakresu inżynierii wstecznej, analizy i odtwarzania. Odpowiadasz za uzyskanie wymaganych zezwoleń i przestrzeganie obowiązującego prawa. Projekt nie popiera działań nielegalnych ani nieautoryzowanych. + +## Licencja + +[MIT](LICENSE) diff --git a/README_pt-BR.md b/README_pt-BR.md new file mode 100644 index 00000000000..f4c61592c28 --- /dev/null +++ b/README_pt-BR.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · **Português (Brasil)** · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Faça engenharia reversa de qualquer coisa + +### Um único MCP para engenharia reversa de binários, aplicativos e comportamento em execução. + +**Viu uma funcionalidade de que gostou? Entenda como ela funciona, até o nível binário.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Site](https://rea.tools/) · [Guias](https://rea.tools/guides/) · [Casos de uso](https://rea.tools/showcase/)** + +[Início rápido](#início-rápido) · [Como o REA funciona](#como-o-rea-funciona) · [O que você pode analisar](#o-que-você-pode-analisar) · [Casos de uso](#casos-de-uso) · [Perguntas frequentes](#perguntas-frequentes) · [Documentação](#documentação) + +npx rea-agents setup + +
+ +REA inicia sua ponte de análise dentro do Hopper enquanto inspeciona um binário nativo + +
+ + + + + +
+ + Discord
+ Participe da comunidade de engenharia reversa +

+ Discord · Perguntas e respostas · Compartilhe seus resultados +
+ +
+ +
+ +--- + +Viu uma funcionalidade em um aplicativo que gostaria de ter no seu próprio produto? Peça ao seu agente para investigá-la com o REA. Ele pode inspecionar o aplicativo sem o código-fonte, explicar como a funcionalidade funciona, apresentar as evidências e criar uma versão para o seu projeto. + +O REA conecta seu agente a ferramentas para inspecionar binários nativos, aplicativos JavaScript e Electron, assemblies .NET e sites. Você também pode usar as mesmas ferramentas pelo terminal. A análise acontece localmente, e os resultados incluem as evidências e limitações por trás de cada conclusão. + +A configuração registra o REA no seu agente e instala as instruções de fluxo de trabalho correspondentes. A análise nativa pode usar uma instalação existente do Hopper ou do Ghidra; a configuração também pode instalar o Hopper opcionalmente, mediante aprovação. A análise estática de JavaScript não precisa de nenhum dos dois motores. + +> **[Visite o site do REA](https://rea.tools/)** para instruções de configuração, guias ilustrados e estudos de caso reais. + +## Início rápido + +### Configure seu agente + +Com Node.js e npm instalados, execute: + +```bash +npx rea-agents setup +``` + +Escolha seus agentes, revise as alterações propostas e aprove-as. A configuração adiciona o servidor MCP do REA e as instruções de fluxo de trabalho correspondentes, com backups das configurações existentes. Reinicie seu agente depois. + +A configuração oferece suporte ao Claude Code, Codex, Cursor, Gemini CLI e [outros agentes](docs/installation.md#supported-agents). Consulte [instalação e configuração](docs/installation.md) para configurar provedores e registrar o MCP manualmente. + +### Peça ao seu agente + +```text +Investigue como a busca funciona no aplicativo Notes, mostre as evidências e crie uma funcionalidade semelhante para o meu projeto. +``` + +Substitua Notes pelo aplicativo alvo e indique a funcionalidade que deseja entender. + +### Use o terminal + +Inspecione um diretório extraído de um aplicativo JavaScript/Electron ou um arquivo ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +O resultado inclui módulos, importações, limites do Electron e suas evidências. Substitua o caminho pelo do seu alvo, como `"D:/apps/example"` no Windows. + +Para instalar o comando `rea` e usá-lo regularmente: + +```bash +npm install --global rea-agents +rea --help +``` + +Para análise nativa, configure primeiro um provedor. Consulte o [guia de CLI e Evidence](docs/cli.md) para comandos nativos, seleção de provedores, snapshots e uso em scripts. + +### Atualize o REA + +O REA muda rapidamente, e novas versões incluem correções frequentes de bugs. Mantenha sua instalação atualizada. + +Para uma CLI instalada pelo npm: + +```bash +rea update +``` + +Para atualizar os registros dos agentes e a skill, execute o comando de configuração exibido pela atualização. + +Se você usa `npx`, atualize a configuração do agente com: + +```bash +npx rea-agents@latest setup +``` + +Revise as alterações de configuração e reinicie seu agente. Para comandos avulsos da CLI, use `npx rea-agents@latest` seguido do comando. + +## Como o REA funciona + +Seu agente chama o REA por MCP para inspecionar o alvo e rastrear o código relevante. O REA retorna os resultados com suas evidências. O agente usa esses resultados para fazer novas perguntas, explicar o comportamento ou escrever e testar uma implementação. Os comandos da CLI usam os mesmos fluxos de trabalho. + +![Fluxo de investigação do REA: seu agente pergunta sobre um alvo local, o REA o inspeciona e rastreia com ferramentas de análise, e o agente usa o código, as referências e os pontos desconhecidos retornados para explicar, implementar e testar.](website/public/assets/figures/rea-investigation-flow.svg) + +[Abra a figura em tamanho completo](website/public/assets/figures/rea-investigation-flow.svg). + + + +## O que você pode analisar + +O REA requer Node.js 22.x (>=22.19), 24.x (>=24.11) ou 26+, além do npm. As ferramentas adicionais e os sistemas anfitriões compatíveis dependem do alvo: + +| Alvo | O que o REA retorna | Requisitos e guia | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| Binários nativos | Pseudocódigo, assembly, strings, símbolos, chamadas e referências | Hopper, Ghidra ou IDA; [análise nativa](https://rea.tools/guides/native/) | +| Estrutura ELF sem execução | Seções, segmentos, símbolos/relocações originais e possíveis mecanismos de mitigação identificados por análise estática | pwntools fornecido pelo chamador no Linux x64; [diagnóstico de binários](docs/binary-diagnostics.md) | +| Bytecode EVM | Seletores de despacho, offsets de bytes, argumentos inferidos e mutabilidade | Entrada local com bytes brutos ou representação hexadecimal; [guia de EVM sem execução](docs/evm-bytecode.md) | +| Falhas do Linux registradas | Registros note brutos, registradores/sinais de cada thread registrada e candidatos opcionais a mapeamentos | pwntools fornecido pelo chamador; GDB/pwndbg opcionais; [falhas registradas](docs/recorded-crashes.md) | +| JavaScript / Electron | Módulos, importações, source maps, rotas, IPC e relações com complementos nativos | Node.js e npm; [análise de aplicativos](https://rea.tools/guides/javascript/) | +| Sites | Estrutura da página, scripts, observações de rede e capturas de tela solicitadas | Um navegador da família Chrome; [análise do navegador](https://rea.tools/guides/browser/) | +| Capturas de rede salvas | Requisições, respostas, payloads acessíveis e localizações de origem | HAR; mitmdump no Linux para capturas no formato nativo do mitmproxy; [guia de capturas](docs/web-network-captures.md) | +| Assemblies .NET | Metadados, instruções CIL, dependências nativas declaradas e comparações de builds | Inspeção estática; [guia de código gerenciado](docs/managed-code-analysis.md) | +| APKs Android | Declarações do manifesto, classes, métodos descompilados e referências | JADX sem interface gráfica e um JDK completo no Linux/macOS; [guia de Android](docs/android-analysis.md) | +| Firmware | Regiões, resultados de extração e encaminhamentos à análise nativa | Binwalk / Unblob no Linux; [guia de firmware](docs/firmware-analysis.md) | +| Pacotes e recursos | Inventários de arquivos, digests, plists, estrutura de bundles Apple e recursos extraídos | [Guia de artefatos e JavaScript](docs/javascript-artifact-reconstruction.md), [aplicativos Apple](docs/apple-application-analysis.md) | +| Comportamento de processos | Saída do terminal, interações, observações de encerramento e do sistema de arquivos, e comparações entre execuções | Linux/macOS com PTY nativo; [captura de processos](docs/process-capture.md) | + +A inspeção estática de JavaScript e .NET lê os arquivos fornecidos sem executar o aplicativo. A captura em execução inicia o alvo selecionado ou interage com ele usando as permissões do seu usuário; cada guia descreve seus efeitos. + + + +Os formatos nativos e os sistemas anfitriões compatíveis variam por provedor. Consulte [configuração do Hopper e Ghidra](docs/installation.md#hopper), o [guia do IDA](docs/ida-provider.md) e o [suporte experimental ao Ghidra no Windows](docs/windows-ghidra-p0.md). O Ghidra também oferece [análise de DOS de 16 bits](docs/ghidra-dos.md). Para selecionar um provedor, consulte o [guia da CLI](docs/cli.md#choose-a-provider). Verifique a [disponibilidade nas versões](docs/installation.md#released-package-and-main) das funcionalidades adicionadas após a versão mais recente no npm. + +## Casos de uso + +### DX-Ball: reconstruir um cálculo de panorâmica de áudio + +Siga uma chamada de áudio até a função auxiliar que converte posição em panorâmica, inspecione as instruções e transforme pseudocódigo incompleto em C. A reconstrução passa em 3.205 casos com o x86 original e reproduz todos os 63 bytes da função compilada. + +[Leia o estudo de caso](https://rea.tools/showcase/dx-ball/) · [Repositório da reconstrução](https://github.com/N0zoM1z0/dx-ball) + +### Notion: rastrear a ponte da área de transferência do Electron + +Encontre a API da área de transferência do renderizador, siga-a por preload e IPC até o processo principal e inspecione o formato da área de transferência com seus dados de formatação. + +[Leia o estudo de caso](https://rea.tools/showcase/notion/) + +### TH04: recuperar o cálculo de um anel de projéteis no DOS + +Inspecione as instruções de 16 bits do jogo original para PC-98, recupere os cálculos de ângulos fixos e direcionados e compare o C++ reconstruído com a saída do compilador da época. + +[Leia o estudo de caso](https://rea.tools/showcase/th04/) · [Repositório da reconstrução](https://github.com/N0zoM1z0/th04) + +Se você usou o REA em algo interessante, gostaríamos de conhecer. Compartilhe seu caso em uma [issue](https://github.com/morluto/rea/issues) ou um [pull request](https://github.com/morluto/rea/pulls), incluindo o alvo, sua pergunta, como o REA ajudou e o que descobriu. + +## Perguntas frequentes + +
+Quais agentes podem usar o REA? + +Qualquer agente que ofereça suporte a servidores MCP locais. A configuração prepara os [agentes compatíveis](docs/installation.md#supported-agents); outros clientes podem usar o [registro manual do MCP](docs/installation.md#mcp-registry). + +
+ +
+Preciso do Hopper, Ghidra ou IDA? + +A análise nativa aprofundada usa um desses motores. A inspeção estática de JavaScript e .NET funciona sem um motor de análise nativa. A configuração pode instalar o Hopper após aprovação; Ghidra e IDA usam as instalações existentes. Consulte [configuração dos provedores](docs/installation.md#hopper). + +
+ +
+Preciso iniciar o Hopper antes? + +O REA inicia o Hopper quando uma operação precisa dele. No macOS, um diálogo na primeira execução pode pedir que você escolha o modo de demonstração ou ative a licença. Consulte [inicialização do Hopper e solução de problemas](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+O que faz a instalação da skill pelo skills.sh? + +A skill fornece instruções de investigação ao seu agente. Use `rea setup` para registrar o servidor MCP do REA e instalar as instruções correspondentes, depois reinicie seu agente. Consulte [instalação apenas da skill](docs/installation.md#skill-only-installation). + +
+ +
+Que código o REA retorna? + +A análise nativa retorna pseudocódigo e assembly. A análise de JavaScript/Electron recupera módulos e suas relações. Seu agente usa esses resultados para escrever e testar uma implementação; os [casos de uso](#casos-de-uso) apresentam exemplos completos. + +
+ +
+O REA envia meu aplicativo para algum servidor? + +O REA analisa os alvos localmente. Seu agente recebe os resultados das ferramentas, e o provedor do modelo tem sua própria política de dados. + +
+ +
+O que devo fazer se encontrar um bug? + +Atualize primeiro; uma versão recente pode já ter corrigido o problema. + +Para uma CLI instalada pelo npm: + +```bash +rea update +``` + +Para configurar o agente por `npx`: + +```bash +npx rea-agents@latest setup +``` + +Se você usa um agente, conclua a [atualização da configuração](#atualize-o-rea) e reinicie-o. Tente executar a mesma tarefa novamente. Se o problema persistir, [abra uma issue](https://github.com/morluto/rea/issues) com sua versão do REA, o tipo de alvo, os passos para reproduzir e a saída do erro. + +
+ +## Documentação + +Comece pelos [guias práticos](https://rea.tools/guides/) do site. Para opções exatas, pré-requisitos e contratos dos resultados: + +- [Instalação e configuração](docs/installation.md): registro de agentes, configuração de provedores, atualizações e desinstalação. +- [Prontidão e solução de problemas](docs/installation.md#check-readiness-for-your-task): diagnóstico de um agente ou motor de análise específico. +- [CLI e Evidence](docs/cli.md): comandos, seleção de provedores, snapshots, importação/exportação e status de saída. +- [Contratos MCP](docs/mcp-contracts.md) e [prompts para agentes](docs/mcp-prompts.md): resultados das ferramentas, sessões e investigações guiadas. +- [Catálogo de ferramentas](docs/mcp-contracts.md#generated-catalog): inventário de ferramentas, provedores e comandos da CLI gerado durante o build. +- [Roadmap](docs/roadmap.md): trabalho planejado e acompanhamento das capacidades. + +Relate vulnerabilidades conforme [SECURITY.md](SECURITY.md). + +## Contribuir + +Sua ajuda com o REA é bem-vinda! [Abra uma issue](https://github.com/morluto/rea/issues) para relatar um bug ou sugerir uma funcionalidade, ou [envie um pull request](https://github.com/morluto/rea/pulls) para melhorar o código ou a documentação. + +Consulte [CONTRIBUTING.md](CONTRIBUTING.md) para configurar o ambiente de desenvolvimento e as verificações, o [guia de testes](docs/testing.md) para os fluxos de verificação e o [mapa da arquitetura](docs/architecture.mermaid) para a estrutura do projeto. + +## Links do projeto + +[Site](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Segurança](SECURITY.md) + +## Histórico de estrelas + +🎉 **20.000 estrelas no GitHub — muito obrigado!** + +Obrigado a todos que usam o REA, relatam bugs, testam builds e contribuem com correções. + + + + + + Histórico de estrelas do REA no GitHub + + + +## Aviso legal + +O REA fornece ferramentas para pesquisa, análise e reconstrução por engenharia reversa dentro da lei. Você é responsável por obter as autorizações necessárias e cumprir as leis aplicáveis. O projeto não apoia usos ilegais ou não autorizados. + +## Licença + +[MIT](LICENSE) diff --git a/README_ru.md b/README_ru.md new file mode 100644 index 00000000000..e8a3681ce89 --- /dev/null +++ b/README_ru.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · **Русский** · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Реверс-инжиниринг чего угодно + +### Один MCP для реверс-инжиниринга бинарных файлов, приложений и поведения во время выполнения. + +**Нашли интересную функцию? Разберитесь, как она работает, вплоть до двоичного кода.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Сайт](https://rea.tools/) · [Руководства](https://rea.tools/guides/) · [Примеры](https://rea.tools/showcase/)** + +[Быстрый старт](#быстрый-старт) · [Как работает REA](#как-работает-rea) · [Что можно анализировать](#что-можно-анализировать) · [Примеры](#примеры) · [Частые вопросы](#частые-вопросы) · [Документация](#документация) + +npx rea-agents setup + +
+ +REA запускает мост анализа внутри Hopper при исследовании нативного бинарного файла + +
+ + + + + +
+ + Discord
+ Присоединяйтесь к сообществу реверс-инжиниринга +

+ Discord · Вопросы и ответы · Обмен результатами +
+ +
+ +
+ +--- + +Увидели в приложении функцию, которую хотите добавить в свой продукт? Попросите агента исследовать её с помощью REA. Он может изучить приложение без исходного кода, объяснить работу функции, показать подтверждающие данные и создать подходящую реализацию для вашего проекта. + +REA подключает вашего агента к инструментам для исследования нативных бинарных файлов, приложений JavaScript и Electron, сборок .NET и сайтов. Те же инструменты доступны в терминале. Анализ выполняется локально, а результаты содержат данные и ограничения, на которых основан каждый вывод. + +Настройка регистрирует REA в вашем агенте и устанавливает соответствующие инструкции по работе. Для нативного анализа можно использовать уже установленный Hopper или Ghidra; настройка также может установить Hopper с вашего согласия. Для статического анализа JavaScript не нужен ни один из этих движков. + +> **[Посетите сайт REA](https://rea.tools/)**: там есть инструкции по настройке, иллюстрированные руководства и реальные исследования. + +## Быстрый старт + +### Настройте агента + +Если Node.js и npm установлены, выполните: + +```bash +npx rea-agents setup +``` + +Выберите агентов, проверьте предлагаемые изменения и подтвердите их. Настройка добавляет MCP-сервер REA и соответствующие рабочие инструкции, сохраняя резервные копии существующих настроек. После этого перезапустите агента. + +Настройка поддерживает Claude Code, Codex, Cursor, Gemini CLI и [других агентов](docs/installation.md#supported-agents). Настройка провайдеров и ручная регистрация MCP описаны в разделе [установка и настройка](docs/installation.md). + +### Задайте вопрос агенту + +```text +Исследуй, как работает поиск в приложении Notes, покажи подтверждающие данные и создай похожую функцию для моего проекта. +``` + +Замените Notes нужным приложением и укажите функцию, которую хотите понять. + +### Используйте терминал + +Исследуйте распакованный каталог приложения JavaScript/Electron или файл ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Результат включает модули, импорты, границы Electron и подтверждающие данные. Замените путь своим, например `"D:/apps/example"` в Windows. + +Для регулярного использования установите команду `rea`: + +```bash +npm install --global rea-agents +rea --help +``` + +Для нативного анализа сначала настройте провайдер. Нативные команды, выбор провайдера, снимки и работу со скриптами описывает [руководство по CLI и Evidence](docs/cli.md). + +### Обновите REA + +REA быстро развивается, и новые версии часто содержат исправления ошибок. Регулярно обновляйте установленную версию. + +Для CLI, установленного через npm: + +```bash +rea update +``` + +Чтобы обновить регистрации агентов и skill, выполните команду настройки, которую выведет обновление. + +Если используете `npx`, обновите настройку агента командой: + +```bash +npx rea-agents@latest setup +``` + +Проверьте изменения настроек и перезапустите агента. Для разовых команд CLI используйте `npx rea-agents@latest`, а затем нужную команду. + +## Как работает REA + +Агент вызывает REA через MCP, чтобы исследовать цель и проследить связанный код. REA возвращает результаты вместе с подтверждающими данными. Агент использует их для уточняющих вопросов, объяснения поведения или написания и тестирования реализации. Команды CLI используют те же рабочие процессы. + +![Процесс исследования с REA: агент задаёт вопрос о локальной цели, REA исследует её инструментами анализа, а агент использует возвращённый код, ссылки и неизвестные сведения для объяснения, реализации и тестирования.](website/public/assets/figures/rea-investigation-flow.svg) + +[Открыть рисунок в полном размере](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Что можно анализировать + +REA требует Node.js 22.x (>=22.19), 24.x (>=24.11) или 26+, а также npm. Дополнительные инструменты и поддерживаемые системы зависят от цели: + +| Цель | Что возвращает REA | Требования и руководство | +| --------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | +| Нативные бинарные файлы | Псевдокод, ассемблер, строки, символы, вызовы и ссылки | Hopper, Ghidra или IDA; [нативный анализ](https://rea.tools/guides/native/) | +| Структура ELF без запуска | Секции, сегменты, исходные символы/релокации и возможные механизмы защиты по данным статического анализа | pwntools, предоставленный вызывающей стороной, в Linux x64; [диагностика бинарных файлов](docs/binary-diagnostics.md) | +| Байт-код EVM | Селекторы диспетчеризации, смещения байтов, предполагаемые аргументы и изменяемость состояния | Локальные необработанные байты или шестнадцатеричное представление; [руководство по EVM без запуска](docs/evm-bytecode.md) | +| Записанные сбои Linux | Необработанные записи note, регистры/сигналы каждого записанного потока и необязательные кандидаты сопоставления с памятью | pwntools, предоставленный вызывающей стороной; необязательные GDB/pwndbg; [записанные сбои](docs/recorded-crashes.md) | +| JavaScript / Electron | Модули, импорты, карты исходного кода, маршруты, IPC и связи с нативными дополнениями | Node.js и npm; [анализ приложений](https://rea.tools/guides/javascript/) | +| Сайты | Структура страницы, скрипты, наблюдения за сетью и запрошенные снимки экрана | Браузер семейства Chrome; [анализ браузера](https://rea.tools/guides/browser/) | +| Сохранённые сетевые захваты | Запросы, ответы, доступные полезные данные и позиции в источнике | HAR; mitmdump в Linux для захватов в собственном формате mitmproxy; [руководство по сетевым захватам](docs/web-network-captures.md) | +| Сборки .NET | Метаданные, инструкции CIL, объявленные нативные зависимости и сравнения сборок | Статическое исследование; [руководство по управляемому коду](docs/managed-code-analysis.md) | +| Android APK | Объявления манифеста, классы, декомпилированные методы и ссылки | JADX без графического интерфейса и полный JDK в Linux/macOS; [руководство по Android](docs/android-analysis.md) | +| Прошивки | Области, результаты извлечения и передача в нативный анализ | Binwalk / Unblob в Linux; [руководство по прошивкам](docs/firmware-analysis.md) | +| Пакеты и ресурсы | Списки файлов, хеши, plist, структура пакетов Apple и извлечённые ресурсы | [Руководство по артефактам и JavaScript](docs/javascript-artifact-reconstruction.md), [приложения Apple](docs/apple-application-analysis.md) | +| Поведение процессов | Вывод терминала, взаимодействия, наблюдения за завершением и файловой системой, сравнения запусков | Linux/macOS с нативным PTY; [захват процессов](docs/process-capture.md) | + +Статическое исследование JavaScript и .NET читает предоставленные файлы, не запуская приложение. Захват во время выполнения запускает выбранную цель или взаимодействует с ней с правами вашего пользователя; соответствующие руководства описывают последствия. + + + +Нативные форматы и поддерживаемые системы различаются у разных провайдеров. См. [настройку Hopper и Ghidra](docs/installation.md#hopper), [руководство по IDA](docs/ida-provider.md) и [экспериментальную поддержку Ghidra в Windows](docs/windows-ghidra-p0.md). Ghidra также поддерживает [анализ 16-битного DOS](docs/ghidra-dos.md). Выбор провайдера описан в [руководстве по CLI](docs/cli.md#choose-a-provider). Для функций, добавленных после последней версии в npm, проверьте [доступность в релизах](docs/installation.md#released-package-and-main). + +## Примеры + +### DX-Ball: восстановление расчёта звуковой панорамы + +Проследите звуковой вызов до функции, преобразующей координату в значение панорамы, изучите инструкции и превратите неполный псевдокод в C. Восстановленная реализация проходит 3 205 тестов с исходным x86-кодом и воспроизводит все 63 байта скомпилированной функции. + +[Прочитать исследование](https://rea.tools/showcase/dx-ball/) · [Репозиторий реконструкции](https://github.com/N0zoM1z0/dx-ball) + +### Notion: исследование моста буфера обмена Electron + +Найдите API буфера обмена в процессе рендеринга, проследите его через preload и IPC до основного процесса и изучите формат буфера обмена с форматированием. + +[Прочитать исследование](https://rea.tools/showcase/notion/) + +### TH04: восстановление расчёта кольца пуль в DOS + +Изучите 16-битные инструкции оригинальной игры для PC-98, восстановите расчёты фиксированных и прицельных углов и сравните восстановленный C++ с выводом компилятора того времени. + +[Прочитать исследование](https://rea.tools/showcase/th04/) · [Репозиторий реконструкции](https://github.com/N0zoM1z0/th04) + +Если вы исследовали с REA что-то интересное, поделитесь результатом. Создайте [issue](https://github.com/morluto/rea/issues) или [pull request](https://github.com/morluto/rea/pulls) и укажите цель, ваш вопрос, как помог REA и что удалось узнать. + +## Частые вопросы + +
+Какие агенты могут использовать REA? + +Любой агент с поддержкой локальных MCP-серверов. Настройка конфигурирует [поддерживаемых агентов](docs/installation.md#supported-agents); другие клиенты могут использовать [ручную регистрацию MCP](docs/installation.md#mcp-registry). + +
+ +
+Нужны ли мне Hopper, Ghidra или IDA? + +Углублённый нативный анализ использует один из этих инструментов. Статическое исследование JavaScript и .NET работает без движка нативного анализа. Настройка может установить Hopper после подтверждения; Ghidra и IDA используют существующие установки. См. [настройку провайдеров](docs/installation.md#hopper). + +
+ +
+Нужно ли сначала запускать Hopper? + +REA запускает Hopper, когда он нужен для операции. В macOS при первом запуске может появиться диалог выбора деморежима или активации лицензии. См. [запуск Hopper и устранение неполадок](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+Что даёт установка skill с skills.sh? + +Skill предоставляет агенту инструкции по исследованию. Используйте `rea setup`, чтобы зарегистрировать MCP-сервер REA и установить соответствующие инструкции, затем перезапустите агента. См. [установку только skill](docs/installation.md#skill-only-installation). + +
+ +
+Какой код возвращает REA? + +Нативный анализ возвращает псевдокод и ассемблер. Анализ JavaScript/Electron восстанавливает модули и их связи. Агент использует эти результаты, чтобы написать и протестировать реализацию; в [примерах](#примеры) показаны практические исследования. + +
+ +
+REA загружает моё приложение на сервер? + +REA анализирует цели локально. Ваш агент получает результаты инструментов, а у его поставщика моделей есть собственная политика обработки данных. + +
+ +
+Что делать, если я столкнулся с ошибкой? + +Сначала обновитесь: недавний релиз мог уже исправить ошибку. + +Для CLI, установленного через npm: + +```bash +rea update +``` + +Для настройки агента через `npx`: + +```bash +npx rea-agents@latest setup +``` + +Если используете агента, завершите [обновление настройки](#обновите-rea) и перезапустите его. Повторите ту же задачу. Если проблема сохраняется, [откройте issue](https://github.com/morluto/rea/issues), указав версию REA, тип цели, шаги воспроизведения и вывод ошибки. + +
+ +## Документация + +Начните с [практических руководств](https://rea.tools/guides/) на сайте. Точные параметры, требования и контракты результатов описаны здесь: + +- [Установка и настройка](docs/installation.md): регистрация агентов, настройка провайдеров, обновление и удаление. +- [Готовность и устранение неполадок](docs/installation.md#check-readiness-for-your-task): диагностика конкретного агента или движка анализа. +- [CLI и Evidence](docs/cli.md): команды, выбор провайдера, снимки, импорт/экспорт и коды завершения. +- [Контракты MCP](docs/mcp-contracts.md) и [промпты агентов](docs/mcp-prompts.md): результаты инструментов, сеансы и исследования с подсказками. +- [Каталог инструментов](docs/mcp-contracts.md#generated-catalog): создаваемый при сборке перечень инструментов, провайдеров и команд CLI. +- [План развития](docs/roadmap.md): запланированные работы и отслеживание возможностей. + +Сообщайте об уязвимостях согласно [SECURITY.md](SECURITY.md). + +## Участие в проекте + +Будем рады вашей помощи с REA! [Откройте issue](https://github.com/morluto/rea/issues), чтобы сообщить об ошибке или предложить функцию, либо [отправьте pull request](https://github.com/morluto/rea/pulls) с улучшениями кода или документации. + +Настройка среды разработки и проверки описаны в [CONTRIBUTING.md](CONTRIBUTING.md), направления проверки — в [руководстве по тестированию](docs/testing.md), а структура проекта — на [схеме архитектуры](docs/architecture.mermaid). + +## Ссылки проекта + +[Сайт](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Безопасность](SECURITY.md) + +## История звёзд + +🎉 **20 000 звёзд на GitHub — спасибо!** + +Спасибо всем, кто использует REA, сообщает об ошибках, тестирует сборки и помогает с исправлениями. + + + + + + История звёзд REA на GitHub + + + +## Отказ от ответственности + +REA предоставляет инструменты для законных исследований, анализа и реконструкции с помощью реверс-инжиниринга. Вы отвечаете за получение необходимых разрешений и соблюдение применимых законов. Проект не поддерживает незаконное или несанкционированное использование. + +## Лицензия + +[MIT](LICENSE) diff --git a/README_th.md b/README_th.md new file mode 100644 index 00000000000..e26c9843732 --- /dev/null +++ b/README_th.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · **ไทย** · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Reverse Engineer Anything + +### MCP เดียวสำหรับวิศวกรรมย้อนกลับ ทั้งไบนารี แอปพลิเคชัน และพฤติกรรมขณะทำงาน + +**เห็นฟีเจอร์ที่ชอบแล้วอยากรู้ว่าทำงานอย่างไร? เข้าใจได้ลึกถึงระดับไบนารี** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[เว็บไซต์](https://rea.tools/) · [คู่มือ](https://rea.tools/guides/) · [ตัวอย่างการใช้งาน](https://rea.tools/showcase/)** + +[เริ่มต้นอย่างรวดเร็ว](#เริ่มต้นอย่างรวดเร็ว) · [REA ทำงานอย่างไร](#rea-ทำงานอย่างไร) · [สิ่งที่วิเคราะห์ได้](#สิ่งที่วิเคราะห์ได้) · [ตัวอย่างการใช้งาน](#ตัวอย่างการใช้งาน) · [คำถามที่พบบ่อย](#คำถามที่พบบ่อย) · [เอกสาร](#เอกสาร) + +npx rea-agents setup + +
+ +REA เริ่มบริดจ์การวิเคราะห์ภายใน Hopper ขณะตรวจสอบไบนารีเนทีฟ + +
+ + + + + +
+ + Discord
+ เข้าร่วมชุมชนวิศวกรรมย้อนกลับ +

+ Discord · ถามตอบ · แบ่งปันผลงาน +
+ +
+ +
+ +--- + +เห็นฟีเจอร์ในแอปที่อยากนำมาใช้ในผลิตภัณฑ์ของคุณเองหรือไม่? ขอให้เอเจนต์ของคุณตรวจสอบด้วย REA เอเจนต์สามารถวิเคราะห์แอปโดยไม่ต้องมีซอร์สโค้ด อธิบายการทำงานของฟีเจอร์ แสดงหลักฐาน และสร้างเวอร์ชันสำหรับโปรเจกต์ของคุณได้ + +REA เชื่อมเอเจนต์ของคุณกับเครื่องมือสำหรับตรวจสอบไบนารีเนทีฟ แอป JavaScript และ Electron แอสเซมบลี .NET และเว็บไซต์ คุณใช้เครื่องมือเดียวกันจากเทอร์มินัลได้ การวิเคราะห์ทำงานบนเครื่องของคุณ และผลลัพธ์มีทั้งหลักฐานและข้อจำกัดที่ประกอบแต่ละข้อสรุป + +ขั้นตอนตั้งค่าจะลงทะเบียน REA กับเอเจนต์และติดตั้งคำแนะนำการทำงานที่ตรงกัน การวิเคราะห์เนทีฟใช้ Hopper หรือ Ghidra ที่ติดตั้งอยู่แล้วได้ และขั้นตอนตั้งค่าสามารถติดตั้ง Hopper เพิ่มเติมเมื่อได้รับอนุมัติ การวิเคราะห์ JavaScript แบบสถิตไม่ต้องใช้เอนจินทั้งสองนี้ + +> **[เยี่ยมชมเว็บไซต์ REA](https://rea.tools/)** เพื่อดูคำแนะนำการตั้งค่า คู่มือพร้อมภาพประกอบ และกรณีศึกษาจริง + +## เริ่มต้นอย่างรวดเร็ว + +### ตั้งค่าเอเจนต์ + +เมื่อติดตั้ง Node.js และ npm แล้ว ให้เรียกใช้: + +```bash +npx rea-agents setup +``` + +เลือกเอเจนต์ ตรวจสอบการเปลี่ยนแปลงที่เสนอ แล้วอนุมัติ ขั้นตอนตั้งค่าจะเพิ่มเซิร์ฟเวอร์ MCP ของ REA และคำแนะนำการทำงานที่ตรงกัน พร้อมสำรองการตั้งค่าที่มีอยู่ จากนั้นเริ่มเอเจนต์ใหม่ + +ขั้นตอนตั้งค่ารองรับ Claude Code, Codex, Cursor, Gemini CLI และ[เอเจนต์อื่น ๆ](docs/installation.md#supported-agents) ดู[การติดตั้งและตั้งค่า](docs/installation.md) สำหรับการตั้งค่าผู้ให้บริการวิเคราะห์และการลงทะเบียน MCP ด้วยตนเอง + +### ถามเอเจนต์ + +```text +ตรวจสอบว่าการค้นหาในแอป Notes ทำงานอย่างไร แสดงหลักฐานให้ฉันดู และสร้างฟีเจอร์ที่คล้ายกันสำหรับโปรเจกต์ของฉัน +``` + +แทนที่ Notes ด้วยแอปเป้าหมายและฟีเจอร์ที่คุณต้องการเข้าใจ + +### ใช้เทอร์มินัล + +ตรวจสอบไดเรกทอรีแอป JavaScript/Electron ที่แตกไฟล์แล้ว หรือไฟล์ ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +ผลลัพธ์ประกอบด้วยโมดูล การนำเข้า ขอบเขตของ Electron และหลักฐาน แทนที่พาธด้วยเป้าหมายของคุณ เช่น `"D:/apps/example"` บน Windows + +หากต้องการติดตั้งคำสั่ง `rea` สำหรับใช้งานเป็นประจำ: + +```bash +npm install --global rea-agents +rea --help +``` + +สำหรับการวิเคราะห์เนทีฟ ให้ตั้งค่าผู้ให้บริการวิเคราะห์ก่อน ดู[คู่มือ CLI และ Evidence](docs/cli.md) สำหรับคำสั่งเนทีฟ การเลือกผู้ให้บริการ สแนปช็อต และการใช้งานในสคริปต์ + +### อัปเดต REA + +REA เปลี่ยนแปลงอย่างรวดเร็ว และรุ่นใหม่มีการแก้ไขบั๊กอยู่บ่อยครั้ง โปรดอัปเดตการติดตั้งของคุณให้เป็นปัจจุบัน + +สำหรับ CLI ที่ติดตั้งผ่าน npm: + +```bash +rea update +``` + +หากต้องการปรับปรุงการลงทะเบียนเอเจนต์และสกิลให้เป็นปัจจุบัน ให้เรียกใช้คำสั่งตั้งค่าที่แสดงหลังการอัปเดต + +หากใช้ `npx` ให้อัปเดตการตั้งค่าเอเจนต์ด้วย: + +```bash +npx rea-agents@latest setup +``` + +ตรวจสอบการเปลี่ยนแปลงการตั้งค่าและเริ่มเอเจนต์ใหม่ สำหรับคำสั่ง CLI ที่ใช้ครั้งเดียว ให้ใช้ `npx rea-agents@latest` ตามด้วยคำสั่งที่ต้องการ + +## REA ทำงานอย่างไร + +เอเจนต์เรียก REA ผ่าน MCP เพื่อตรวจสอบเป้าหมายและติดตามโค้ดที่เกี่ยวข้อง REA ส่งข้อค้นพบกลับมาพร้อมหลักฐาน เอเจนต์ใช้ข้อมูลนี้เพื่อถามคำถามต่อ อธิบายพฤติกรรม หรือเขียนและทดสอบการนำไปใช้งาน คำสั่ง CLI ใช้กระบวนการเดียวกัน + +![กระบวนการตรวจสอบด้วย REA: เอเจนต์ถามเกี่ยวกับเป้าหมายบนเครื่อง REA ตรวจสอบและติดตามด้วยเครื่องมือวิเคราะห์ แล้วเอเจนต์ใช้โค้ด การอ้างอิง และสิ่งที่ยังไม่ทราบที่ได้รับกลับมาเพื่ออธิบาย สร้าง และทดสอบ](website/public/assets/figures/rea-investigation-flow.svg) + +[เปิดภาพขนาดเต็ม](website/public/assets/figures/rea-investigation-flow.svg) + + + +## สิ่งที่วิเคราะห์ได้ + +REA ต้องใช้ Node.js 22.x (>=22.19), 24.x (>=24.11) หรือ 26+ และ npm เครื่องมือเพิ่มเติมและแพลตฟอร์มที่รองรับขึ้นอยู่กับเป้าหมาย: + +| เป้าหมาย | สิ่งที่ REA ส่งกลับ | ข้อกำหนดและคู่มือ | +| ---------------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| ไบนารีเนทีฟ | โค้ดเทียม แอสเซมบลี สตริง สัญลักษณ์ การเรียก และการอ้างอิง | Hopper, Ghidra หรือ IDA; [การวิเคราะห์เนทีฟ](https://rea.tools/guides/native/) | +| โครงสร้าง ELF แบบออฟไลน์ | เซกชัน เซกเมนต์ สัญลักษณ์/รีโลเคชันต้นฉบับ และตัวเลือกกลไกป้องกันที่พบจากการวิเคราะห์แบบสถิต | pwntools ที่ผู้ใช้จัดเตรียมบน Linux x64; [การวินิจฉัยไบนารี](docs/binary-diagnostics.md) | +| ไบต์โค้ด EVM | ตัวเลือกฟังก์ชันสำหรับการเรียก ออฟเซ็ตไบต์ อาร์กิวเมนต์และความสามารถในการเปลี่ยนสถานะที่อนุมานได้ | ไฟล์บนเครื่องที่มีไบต์ดิบหรือข้อมูลเลขฐานสิบหก; [คู่มือ EVM แบบออฟไลน์](docs/evm-bytecode.md) | +| ข้อมูลการแครชของ Linux ที่บันทึกไว้ | ระเบียน note ดิบ รีจิสเตอร์/สัญญาณของทุกเธรดที่บันทึกไว้ และตัวเลือกการแมปเพิ่มเติม | pwntools ที่ผู้ใช้จัดเตรียม; เลือกใช้ GDB/pwndbg เพิ่มเติมได้; [การแครชที่บันทึกไว้](docs/recorded-crashes.md) | +| JavaScript / Electron | โมดูล การนำเข้า ซอร์สแมป เส้นทาง IPC และความสัมพันธ์กับส่วนเสริมเนทีฟ | Node.js และ npm; [การวิเคราะห์แอปพลิเคชัน](https://rea.tools/guides/javascript/) | +| เว็บไซต์ | โครงสร้างหน้า สคริปต์ ข้อมูลเครือข่ายที่สังเกตได้ และภาพหน้าจอที่ร้องขอ | เบราว์เซอร์ในตระกูล Chrome; [การวิเคราะห์ผ่านเบราว์เซอร์](https://rea.tools/guides/browser/) | +| ข้อมูลเครือข่ายที่บันทึกไว้ | คำขอ คำตอบ เพย์โหลดที่ปรากฏ และตำแหน่งต้นทาง | HAR; mitmdump บน Linux สำหรับข้อมูลที่บันทึกในรูปแบบเนทีฟของ mitmproxy; [คู่มือการบันทึกข้อมูล](docs/web-network-captures.md) | +| แอสเซมบลี .NET | เมทาดาทา คำสั่ง CIL การพึ่งพาเนทีฟที่ประกาศไว้ และการเปรียบเทียบผลการบิลด์ | การตรวจสอบแบบสถิต; [คู่มือโค้ดแบบจัดการ](docs/managed-code-analysis.md) | +| APK ของ Android | การประกาศใน manifest คลาส เมธอดที่ดีคอมไพล์ และการอ้างอิง | JADX แบบไม่มี GUI และ JDK แบบเต็มบน Linux/macOS; [คู่มือ Android](docs/android-analysis.md) | +| เฟิร์มแวร์ | พื้นที่ข้อมูล ผลการแตกไฟล์ และการส่งต่อไปวิเคราะห์เนทีฟ | Binwalk / Unblob บน Linux; [คู่มือเฟิร์มแวร์](docs/firmware-analysis.md) | +| แพ็กเกจและทรัพยากร | รายการไฟล์ ค่าแฮช plist โครงสร้างบันเดิลของ Apple และทรัพยากรที่แตกออกมา | [คู่มืออาร์ติแฟกต์และ JavaScript](docs/javascript-artifact-reconstruction.md), [แอปพลิเคชัน Apple](docs/apple-application-analysis.md) | +| พฤติกรรมของโปรเซส | เอาต์พุตเทอร์มินัล การโต้ตอบ ข้อมูลการสิ้นสุดและระบบไฟล์ที่สังเกตได้ และการเปรียบเทียบการรัน | Linux/macOS ที่มี PTY เนทีฟ; [การบันทึกโปรเซส](docs/process-capture.md) | + +การตรวจสอบ JavaScript และ .NET แบบสถิตอ่านไฟล์ที่ให้มาโดยไม่รันแอปพลิเคชัน การบันทึกขณะทำงานจะรันหรือโต้ตอบกับเป้าหมายที่เลือกโดยใช้สิทธิ์ของผู้ใช้คุณ คู่มือการทำงานแต่ละฉบับอธิบายผลที่เกิดขึ้นไว้ + + + +รูปแบบไบนารีเนทีฟและแพลตฟอร์มที่รองรับแตกต่างกันตามผู้ให้บริการวิเคราะห์ ดู[การตั้งค่า Hopper และ Ghidra](docs/installation.md#hopper), [คู่มือ IDA](docs/ida-provider.md) และ[การรองรับ Ghidra บน Windows แบบทดลอง](docs/windows-ghidra-p0.md) Ghidra ยังรองรับ[การวิเคราะห์ DOS แบบ 16 บิต](docs/ghidra-dos.md) สำหรับการเลือกผู้ให้บริการ ดู[คู่มือ CLI](docs/cli.md#choose-a-provider) ตรวจสอบ[ความพร้อมใช้งานในแต่ละรุ่น](docs/installation.md#released-package-and-main) สำหรับฟีเจอร์ที่เพิ่มหลังรุ่นล่าสุดบน npm + +## ตัวอย่างการใช้งาน + +### DX-Ball: สร้างการคำนวณแพนเสียงขึ้นใหม่ + +ติดตามการเรียกเสียงไปยังฟังก์ชันช่วยแปลงตำแหน่งเป็นค่าแพน ตรวจสอบคำสั่ง และแปลงโค้ดเทียมที่ยังไม่สมบูรณ์เป็น C การสร้างขึ้นใหม่นี้ผ่านการทดสอบ 3,205 กรณีเทียบกับ x86 ต้นฉบับ และสร้างไบต์ทั้ง 63 ไบต์ของฟังก์ชันที่คอมไพล์แล้วได้ตรงกัน + +[อ่านกรณีศึกษา](https://rea.tools/showcase/dx-ball/) · [รีโพซิทอรีการสร้างขึ้นใหม่](https://github.com/N0zoM1z0/dx-ball) + +### Notion: ติดตามบริดจ์คลิปบอร์ดของ Electron + +ค้นหา API คลิปบอร์ดของ renderer ติดตามผ่าน preload และ IPC ไปยังโปรเซสหลัก แล้วตรวจสอบรูปแบบคลิปบอร์ดที่มีข้อมูลการจัดรูปแบบ + +[อ่านกรณีศึกษา](https://rea.tools/showcase/notion/) + +### TH04: กู้คืนการคำนวณวงแหวนกระสุนของ DOS + +ตรวจสอบคำสั่ง 16 บิตของเกมต้นฉบับบน PC-98 กู้คืนการคำนวณมุมคงที่และมุมเล็งเป้าหมาย แล้วเปรียบเทียบ C++ ที่สร้างขึ้นใหม่กับผลลัพธ์จากคอมไพเลอร์ในยุคนั้น + +[อ่านกรณีศึกษา](https://rea.tools/showcase/th04/) · [รีโพซิทอรีการสร้างขึ้นใหม่](https://github.com/N0zoM1z0/th04) + +หากคุณใช้ REA กับสิ่งที่น่าสนใจ เราอยากเห็นผลงานของคุณ แบ่งปันกรณีของคุณผ่าน [issue](https://github.com/morluto/rea/issues) หรือ [pull request](https://github.com/morluto/rea/pulls) พร้อมระบุเป้าหมาย คำถามของคุณ REA ช่วยอย่างไร และสิ่งที่คุณค้นพบ + +## คำถามที่พบบ่อย + +
+เอเจนต์ใดใช้ REA ได้บ้าง? + +เอเจนต์ใดก็ตามที่รองรับเซิร์ฟเวอร์ MCP บนเครื่อง ขั้นตอนตั้งค่าจะกำหนดค่าให้กับ[เอเจนต์ที่รองรับ](docs/installation.md#supported-agents) ส่วนไคลเอนต์อื่นใช้[การลงทะเบียน MCP ด้วยตนเอง](docs/installation.md#mcp-registry) ได้ + +
+ +
+จำเป็นต้องมี Hopper, Ghidra หรือ IDA หรือไม่? + +การวิเคราะห์เนทีฟเชิงลึกต้องใช้หนึ่งในเครื่องมือเหล่านี้ การตรวจสอบ JavaScript และ .NET แบบสถิตทำงานได้โดยไม่มีเอนจินวิเคราะห์เนทีฟ ขั้นตอนตั้งค่าสามารถติดตั้ง Hopper หลังได้รับอนุมัติ ส่วน Ghidra และ IDA ใช้การติดตั้งที่คุณมีอยู่แล้ว ดู[การตั้งค่าผู้ให้บริการวิเคราะห์](docs/installation.md#hopper) + +
+ +
+ต้องเริ่ม Hopper ก่อนหรือไม่? + +REA เริ่ม Hopper เมื่อการดำเนินการต้องใช้ บน macOS อาจมีหน้าต่างเมื่อรันครั้งแรกให้เลือกโหมดทดลองหรือเปิดใช้ใบอนุญาต ดู[การเริ่ม Hopper และการแก้ไขปัญหา](docs/installation.md#launcher-paths-and-troubleshooting) + +
+ +
+การติดตั้งสกิลจาก skills.sh ทำอะไร? + +สกิลให้คำแนะนำการตรวจสอบแก่เอเจนต์ ใช้ `rea setup` เพื่อลงทะเบียนเซิร์ฟเวอร์ MCP ของ REA และติดตั้งคำแนะนำที่ตรงกัน แล้วเริ่มเอเจนต์ใหม่ ดู[การติดตั้งเฉพาะสกิล](docs/installation.md#skill-only-installation) + +
+ +
+REA ส่งกลับโค้ดแบบใด? + +การวิเคราะห์เนทีฟส่งกลับโค้ดเทียมและแอสเซมบลี การวิเคราะห์ JavaScript/Electron กู้คืนโมดูลและความสัมพันธ์ระหว่างโมดูล เอเจนต์ใช้ข้อค้นพบเหล่านี้เพื่อเขียนและทดสอบการนำไปใช้งาน ดูตัวอย่างที่ทำจริงใน[ตัวอย่างการใช้งาน](#ตัวอย่างการใช้งาน) + +
+ +
+REA อัปโหลดแอปของฉันหรือไม่? + +REA วิเคราะห์เป้าหมายบนเครื่องของคุณ เอเจนต์ได้รับผลลัพธ์ของเครื่องมือ และผู้ให้บริการโมเดลของเอเจนต์มีนโยบายข้อมูลของตนเอง + +
+ +
+ควรทำอย่างไรเมื่อพบข้อผิดพลาด? + +อัปเดตก่อน รุ่นล่าสุดอาจแก้ไขปัญหานี้แล้ว + +สำหรับ CLI ที่ติดตั้งผ่าน npm: + +```bash +rea update +``` + +สำหรับการตั้งค่าเอเจนต์ผ่าน `npx`: + +```bash +npx rea-agents@latest setup +``` + +หากใช้เอเจนต์ ให้ทำ[การปรับปรุงการตั้งค่า](#อัปเดต-rea) ให้เสร็จแล้วเริ่มเอเจนต์ใหม่ ลองงานเดิมอีกครั้ง หากยังพบปัญหา ให้[เปิด issue](https://github.com/morluto/rea/issues) พร้อมระบุรุ่น REA ประเภทเป้าหมาย ขั้นตอนทำให้เกิดปัญหา และข้อความข้อผิดพลาด + +
+ +## เอกสาร + +เริ่มจาก[คู่มือพร้อมตัวอย่าง](https://rea.tools/guides/) บนเว็บไซต์ สำหรับตัวเลือก ข้อกำหนดเบื้องต้น และสัญญาผลลัพธ์ที่แน่นอน: + +- [การติดตั้งและตั้งค่า](docs/installation.md): การลงทะเบียนเอเจนต์ การตั้งค่าผู้ให้บริการ การอัปเดต และการถอนการติดตั้ง +- [ความพร้อมใช้งานและการแก้ไขปัญหา](docs/installation.md#check-readiness-for-your-task): ตรวจวินิจฉัยเอเจนต์หรือเอนจินวิเคราะห์หนึ่งตัว +- [CLI และ Evidence](docs/cli.md): คำสั่ง การเลือกผู้ให้บริการ สแนปช็อต การนำเข้า/ส่งออก และรหัสการสิ้นสุด +- [สัญญา MCP](docs/mcp-contracts.md) และ[พรอมป์ต์สำหรับเอเจนต์](docs/mcp-prompts.md): ผลลัพธ์เครื่องมือ เซสชัน และการตรวจสอบตามคำแนะนำ +- [แค็ตตาล็อกเครื่องมือ](docs/mcp-contracts.md#generated-catalog): รายการเครื่องมือ ผู้ให้บริการ และคำสั่ง CLI ที่สร้างระหว่างการบิลด์ +- [แผนพัฒนา](docs/roadmap.md): งานที่วางแผนไว้และรายการติดตามความสามารถ + +รายงานช่องโหว่ผ่าน [SECURITY.md](SECURITY.md) + +## การมีส่วนร่วม + +เรายินดีรับความช่วยเหลือในการพัฒนา REA! [เปิด issue](https://github.com/morluto/rea/issues) เพื่อรายงานบั๊กหรือเสนอฟีเจอร์ หรือ[ส่ง pull request](https://github.com/morluto/rea/pulls) เพื่อปรับปรุงโค้ดหรือเอกสาร + +ดู [CONTRIBUTING.md](CONTRIBUTING.md) สำหรับการตั้งค่าสภาพแวดล้อมพัฒนาและการตรวจสอบ ดู[การทดสอบ](docs/testing.md) สำหรับแนวทางการตรวจยืนยัน และ[แผนผังสถาปัตยกรรม](docs/architecture.mermaid) สำหรับโครงสร้างโปรเจกต์ + +## ลิงก์ของโปรเจกต์ + +[เว็บไซต์](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [ความปลอดภัย](SECURITY.md) + +## ประวัติดาว + +🎉 **20,000 ดาวบน GitHub — ขอบคุณทุกคน!** + +ขอบคุณทุกคนที่ใช้ REA รายงานบั๊ก ทดสอบบิลด์ และช่วยแก้ไข + + + + + + ประวัติดาว GitHub ของ REA + + + +## ข้อสงวนสิทธิ์ + +REA มีเครื่องมือสำหรับการวิจัยวิศวกรรมย้อนกลับ การวิเคราะห์ และการสร้างขึ้นใหม่อย่างถูกกฎหมาย คุณมีหน้าที่ขออนุญาตตามที่จำเป็นและปฏิบัติตามกฎหมายที่เกี่ยวข้อง โปรเจกต์นี้ไม่สนับสนุนการใช้งานที่ผิดกฎหมายหรือไม่ได้รับอนุญาต + +## สัญญาอนุญาต + +[MIT](LICENSE) diff --git a/README_tr.md b/README_tr.md new file mode 100644 index 00000000000..69a9d6304ed --- /dev/null +++ b/README_tr.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **Türkçe** · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Her şeyi tersine mühendislikle inceleyin + +### İkili dosyaların, uygulamaların ve çalışma zamanı davranışının tersine mühendisliği için tek bir MCP. + +**Beğendiğiniz bir özellik mi gördünüz? İkili kod düzeyine kadar nasıl çalıştığını anlayın.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Web sitesi](https://rea.tools/) · [Rehberler](https://rea.tools/guides/) · [Örnek çalışmalar](https://rea.tools/showcase/)** + +[Hızlı başlangıç](#hızlı-başlangıç) · [REA nasıl çalışır](#rea-nasıl-çalışır) · [Neleri analiz edebilirsiniz](#neleri-analiz-edebilirsiniz) · [Örnek çalışmalar](#örnek-çalışmalar) · [Sık sorulan sorular](#sık-sorulan-sorular) · [Belgeler](#belgeler) + +npx rea-agents setup + +
+ +REA, yerel bir ikili dosyayı incelerken Hopper içinde analiz köprüsünü başlatıyor + +
+ + + + + +
+ + Discord
+ Tersine mühendislik topluluğuna katılın +

+ Discord · Soru-cevap · Sonuç paylaşımı +
+ +
+ +
+ +--- + +Bir uygulamada kendi ürününüze eklemek istediğiniz bir özellik mi gördünüz? Ajanınızdan REA ile araştırmasını isteyin. Kaynak kodu olmadan uygulamayı inceleyebilir, özelliğin nasıl çalıştığını açıklayabilir, kanıtları gösterebilir ve projenize uygun bir sürümünü geliştirebilir. + +REA, ajanınızı yerel makine kodu içeren ikili dosyaları, JavaScript ve Electron uygulamalarını, .NET derlemelerini ve web sitelerini inceleyen araçlara bağlar. Aynı araçları terminalden de kullanabilirsiniz. Analiz yerel olarak çalışır; sonuçlar her çıkarımın dayandığı kanıtları ve sınırlamaları içerir. + +Kurulum, REA'yı ajanınıza kaydeder ve uyumlu iş akışı talimatlarını yükler. Yerel makine kodu analizi mevcut bir Hopper veya Ghidra kurulumunu kullanabilir; kurulum işlemi, onayınızla isteğe bağlı olarak Hopper da yükleyebilir. Statik JavaScript analizi bu iki motora da ihtiyaç duymaz. + +> Kurulum talimatları, görselli rehberler ve gerçek örnek çalışmalar için **[REA web sitesini ziyaret edin](https://rea.tools/)**. + +## Hızlı başlangıç + +### Ajanınızı yapılandırın + +Node.js ve npm yüklüyken şu komutu çalıştırın: + +```bash +npx rea-agents setup +``` + +Ajanlarınızı seçin, önerilen değişiklikleri inceleyin ve onaylayın. Kurulum, mevcut yapılandırmaları yedekleyerek REA'nın MCP sunucusunu ve uyumlu iş akışı talimatlarını ekler. Ardından ajanınızı yeniden başlatın. + +Kurulum Claude Code, Codex, Cursor, Gemini CLI ve [diğer ajanları](docs/installation.md#supported-agents) destekler. Sağlayıcı yapılandırması ve elle MCP kaydı için [yükleme ve kurulum](docs/installation.md) belgesine bakın. + +### Ajanınıza sorun + +```text +Notes uygulamasında aramanın nasıl çalıştığını araştır, kanıtları göster ve projem için benzer bir özellik geliştir. +``` + +Notes yerine hedef uygulamanızı ve anlamak istediğiniz özelliği belirtin. + +### Terminali kullanın + +Çıkarılmış bir JavaScript/Electron uygulama dizinini veya ASAR dosyasını inceleyin: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Sonuç; modülleri, içe aktarmaları, Electron sınırlarını ve bunlara ilişkin kanıtları içerir. Yolu hedefinizle değiştirin; Windows'ta örneğin `"D:/apps/example"` kullanabilirsiniz. + +Düzenli kullanım için `rea` komutunu yüklemek üzere: + +```bash +npm install --global rea-agents +rea --help +``` + +Yerel makine kodu analizi için önce bir sağlayıcı yapılandırın. Yerel analiz komutları, sağlayıcı seçimi, anlık görüntüler ve betik kullanımı için [CLI ve Evidence rehberine](docs/cli.md) bakın. + +### REA'yı güncelleyin + +REA hızla değişir ve yeni sürümler sık sık hata düzeltmeleri içerir. Kurulumunuzu güncel tutun. + +npm ile yüklenen CLI için: + +```bash +rea update +``` + +Ajan kayıtlarını ve skill'i güncellemek için güncellemenin çıktısında gösterilen kurulum komutunu çalıştırın. + +`npx` kullanıyorsanız ajan kurulumunu şu komutla güncelleyin: + +```bash +npx rea-agents@latest setup +``` + +Kurulum değişikliklerini inceleyin ve ajanınızı yeniden başlatın. Tek seferlik CLI işlemleri için `npx rea-agents@latest` komutunun ardından istediğiniz komutu yazın. + +## REA nasıl çalışır + +Ajanınız, hedefi incelemek ve ilgili kodu izlemek için MCP üzerinden REA'yı çağırır. REA bulguları kanıtlarıyla birlikte döndürür. Ajan bunları takip soruları sormak, davranışı açıklamak veya bir uygulama yazıp test etmek için kullanır. CLI komutları aynı iş akışlarını kullanır. + +![REA araştırma akışı: ajanınız yerel bir hedef hakkında soru sorar, REA analiz araçlarıyla hedefi inceler ve izler; ajan döndürülen kodu, referansları ve bilinmeyenleri açıklama, geliştirme ve test için kullanır.](website/public/assets/figures/rea-investigation-flow.svg) + +[Şekli tam boyutuyla açın](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Neleri analiz edebilirsiniz + +REA; Node.js 22.x (>=22.19), 24.x (>=24.11) veya 26+ ve npm gerektirir. Ek araçlar ve desteklenen ana bilgisayar sistemleri hedefe bağlıdır: + +| Hedef | REA'nın döndürdükleri | Gereksinimler ve rehber | +| --------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Yerel makine kodu içeren ikili dosyalar | Sözde kod, assembly kodu, dizeler, semboller, çağrılar ve referanslar | Hopper, Ghidra veya IDA; [yerel kod analizi](https://rea.tools/guides/native/) | +| Çevrimdışı ELF yapısı | Bölümler, segmentler, özgün semboller/yeniden konumlandırmalar ve statik analizden olası güvenlik önlemleri | Linux x64 üzerinde çağıranın sağladığı pwntools; [ikili dosya tanılama](docs/binary-diagnostics.md) | +| EVM bayt kodu | Yönlendirme seçicileri, bayt ofsetleri, çıkarımla belirlenen argümanlar ve durum değiştirilebilirliği | Yerel ham bayt/onaltılık girdi; [çevrimdışı EVM rehberi](docs/evm-bytecode.md) | +| Kaydedilmiş Linux çökmeleri | Ham note kayıtları, kaydedilen her iş parçacığının yazmaçları/sinyalleri ve isteğe bağlı eşleme adayları | Çağıranın sağladığı pwntools; isteğe bağlı GDB/pwndbg; [kaydedilmiş çökmeler](docs/recorded-crashes.md) | +| JavaScript / Electron | Modüller, içe aktarmalar, kaynak haritaları, rotalar, IPC ve yerel eklenti ilişkileri | Node.js ve npm; [uygulama analizi](https://rea.tools/guides/javascript/) | +| Web siteleri | Sayfa yapısı, betikler, ağ gözlemleri ve istenen ekran görüntüleri | Chrome ailesinden bir tarayıcı; [tarayıcı analizi](https://rea.tools/guides/browser/) | +| Kaydedilmiş ağ yakalamaları | İstekler, yanıtlar, erişilebilir yük içerikleri ve kaynak konumları | HAR; mitmproxy'nin kendi biçimindeki yakalamalar için Linux üzerinde mitmdump; [ağ yakalama rehberi](docs/web-network-captures.md) | +| .NET derlemeleri | Meta veriler, CIL talimatları, bildirilmiş yerel bağımlılıklar ve derleme karşılaştırmaları | Statik inceleme; [yönetilen kod rehberi](docs/managed-code-analysis.md) | +| Android APK'leri | Manifest bildirimleri, sınıflar, tersine derlenmiş metotlar ve referanslar | Linux/macOS üzerinde arayüzsüz JADX ve tam bir JDK; [Android rehberi](docs/android-analysis.md) | +| Donanım yazılımı | Bölgeler, çıkarma sonuçları ve yerel kod analizine devirler | Linux üzerinde Binwalk / Unblob; [donanım yazılımı rehberi](docs/firmware-analysis.md) | +| Paketler ve kaynaklar | Dosya envanterleri, özet değerleri, plist'ler, Apple paket yapısı ve çıkarılmış kaynaklar | [Artefakt ve JavaScript rehberi](docs/javascript-artifact-reconstruction.md), [Apple uygulamaları](docs/apple-application-analysis.md) | +| Süreç davranışı | Terminal çıktısı, etkileşimler, sonlanma ve dosya sistemi gözlemleri, çalıştırma karşılaştırmaları | Yerel PTY ile Linux/macOS; [süreç yakalama](docs/process-capture.md) | + +Statik JavaScript ve .NET incelemesi, uygulamayı çalıştırmadan sağlanan dosyaları okur. Çalışma zamanı yakalaması, seçilen hedefi kullanıcı izinlerinizle çalıştırır veya onunla etkileşime girer; her çalışma zamanı rehberi etkilerini açıklar. + + + +Yerel makine kodu içeren ikili dosya biçimleri ve ana bilgisayar desteği sağlayıcıya göre değişir. [Hopper ve Ghidra kurulumu](docs/installation.md#hopper), [IDA rehberi](docs/ida-provider.md) ve [Windows için deneysel Ghidra desteği](docs/windows-ghidra-p0.md) belgelerine bakın. Ghidra ayrıca [16 bit DOS analizini](docs/ghidra-dos.md) destekler. Sağlayıcı seçimi için [CLI rehberine](docs/cli.md#choose-a-provider) bakın. En son npm sürümünden sonra eklenen özellikler için [sürümlerde kullanılabilirliği](docs/installation.md#released-package-and-main) kontrol edin. + +## Örnek çalışmalar + +### DX-Ball: Sesin stereo konumlandırma hesabını yeniden oluşturma + +Bir ses çağrısından konumu stereo konumlandırmaya dönüştüren yardımcı fonksiyona ilerleyin, talimatları inceleyin ve eksik sözde kodu C'ye dönüştürün. Yeniden oluşturulan kod, özgün x86 üzerinde 3.205 test durumunu geçer ve derlenmiş fonksiyonun 63 baytının tamamını yeniden üretir. + +[Örnek çalışmayı okuyun](https://rea.tools/showcase/dx-ball/) · [Yeniden oluşturma deposu](https://github.com/N0zoM1z0/dx-ball) + +### Notion: Electron pano köprüsünü izleme + +İşleyici sürecin pano API'sini bulun, preload ve IPC üzerinden ana sürece kadar izleyin ve zengin pano biçimini inceleyin. + +[Örnek çalışmayı okuyun](https://rea.tools/showcase/notion/) + +### TH04: DOS mermi halkası hesabını geri çıkarma + +Özgün PC-98 oyununun 16 bit talimatlarını inceleyin, sabit ve hedefe yönelik açı hesaplarını geri çıkarın ve yeniden oluşturulan C++ kodunu dönemin derleyici çıktısıyla karşılaştırın. + +[Örnek çalışmayı okuyun](https://rea.tools/showcase/th04/) · [Yeniden oluşturma deposu](https://github.com/N0zoM1z0/th04) + +REA'yı ilginç bir konuda kullandıysanız görmek isteriz. Hedefi, sorunuzu, REA'nın nasıl yardımcı olduğunu ve bulgularınızı bir [issue](https://github.com/morluto/rea/issues) veya [pull request](https://github.com/morluto/rea/pulls) ile paylaşın. + +## Sık sorulan sorular + +
+Hangi ajanlar REA'yı kullanabilir? + +Yerel MCP sunucularını destekleyen her ajan REA'yı kullanabilir. Kurulum, [desteklenen ajanları](docs/installation.md#supported-agents) yapılandırır; diğer istemciler [elle MCP kaydını](docs/installation.md#mcp-registry) kullanabilir. + +
+ +
+Hopper, Ghidra veya IDA gerekiyor mu? + +Derin yerel kod analizi bunlardan birini kullanır. Statik JavaScript ve .NET incelemesi yerel analiz motoru olmadan çalışır. Kurulum onaydan sonra Hopper yükleyebilir; Ghidra ve IDA mevcut kurulumlarınızı kullanır. [Sağlayıcı kurulumuna](docs/installation.md#hopper) bakın. + +
+ +
+Önce Hopper'ı başlatmam gerekiyor mu? + +REA, bir işlem gerektirdiğinde Hopper'ı başlatır. macOS'ta ilk çalıştırmada demo modunu seçmenizi veya lisansınızı etkinleştirmenizi isteyen bir iletişim kutusu açılabilir. [Hopper başlatma ve sorun giderme](docs/installation.md#launcher-paths-and-troubleshooting) belgesine bakın. + +
+ +
+skills.sh üzerinden skill yüklemek ne yapar? + +Skill, ajanınıza araştırma talimatları sağlar. REA'nın MCP sunucusunu kaydetmek ve uyumlu talimatları yüklemek için `rea setup` kullanın, ardından ajanınızı yeniden başlatın. [Yalnızca skill yükleme](docs/installation.md#skill-only-installation) belgesine bakın. + +
+ +
+REA hangi kodu döndürür? + +Yerel kod analizi sözde kod ve assembly kodu döndürür. JavaScript/Electron analizi modülleri ve aralarındaki ilişkileri geri çıkarır. Ajanınız bu bulgularla bir uygulama yazar ve test eder; [örnek çalışmalar](#örnek-çalışmalar) uygulamalı örnekler sunar. + +
+ +
+REA uygulamamı yükler mi? + +REA hedefleri yerel olarak analiz eder. Ajanınız araç sonuçlarını alır ve model sağlayıcısının kendi veri politikası vardır. + +
+ +
+Bir hatayla karşılaşırsam ne yapmalıyım? + +Önce güncelleyin; yakın tarihli bir sürüm hatayı düzeltmiş olabilir. + +npm ile yüklenen CLI için: + +```bash +rea update +``` + +`npx` üzerinden ajan kurulumu için: + +```bash +npx rea-agents@latest setup +``` + +Ajan kullanıyorsanız [kurulum yenilemesini](#reayı-güncelleyin) tamamlayın ve ajanı yeniden başlatın. Aynı görevi tekrar deneyin. Sorun sürerse REA sürümünüzü, hedef türünü, yeniden üretme adımlarını ve hata çıktısını içeren bir [issue açın](https://github.com/morluto/rea/issues). + +
+ +## Belgeler + +Web sitesindeki [uygulamalı rehberlerden](https://rea.tools/guides/) başlayın. Kesin seçenekler, ön koşullar ve sonuç sözleşmeleri için: + +- [Yükleme ve kurulum](docs/installation.md): ajan kaydı, sağlayıcı yapılandırması, güncellemeler ve kaldırma. +- [Hazırlık ve sorun giderme](docs/installation.md#check-readiness-for-your-task): belirli bir ajanı veya analiz motorunu tanılama. +- [CLI ve Evidence](docs/cli.md): komutlar, sağlayıcı seçimi, anlık görüntüler, içe/dışa aktarma ve çıkış durumları. +- [MCP sözleşmeleri](docs/mcp-contracts.md) ve [ajan istemleri](docs/mcp-prompts.md): araç sonuçları, oturumlar ve yönlendirmeli araştırmalar. +- [Araç kataloğu](docs/mcp-contracts.md#generated-catalog): derleme sırasında oluşturulan araç, sağlayıcı ve CLI komutu envanteri. +- [Yol haritası](docs/roadmap.md): planlanan çalışmalar ve yetenek takibi. + +Güvenlik açıklarını [SECURITY.md](SECURITY.md) üzerinden bildirin. + +## Katkıda bulunma + +REA'ya katkınızı bekliyoruz! Hata bildirmek veya özellik önermek için bir [issue açın](https://github.com/morluto/rea/issues); kodu veya belgeleri geliştirmek için bir [pull request gönderin](https://github.com/morluto/rea/pulls). + +Geliştirme ortamı ve kontroller için [CONTRIBUTING.md](CONTRIBUTING.md), doğrulama süreçleri için [test rehberi](docs/testing.md) ve proje yapısı için [mimari harita](docs/architecture.mermaid) belgelerine bakın. + +## Proje bağlantıları + +[Web sitesi](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Güvenlik](SECURITY.md) + +## Yıldız geçmişi + +🎉 **GitHub'da 20.000 yıldız — teşekkürler!** + +REA'yı kullanan, hata bildiren, derlemeleri test eden ve düzeltmelere katkıda bulunan herkese teşekkürler. + + + + + + REA GitHub yıldız geçmişi + + + +## Sorumluluk reddi + +REA, yasal tersine mühendislik araştırması, analizi ve yeniden oluşturma için araçlar sağlar. Gerekli izinleri almak ve geçerli yasalara uymak sizin sorumluluğunuzdadır. Proje yasa dışı veya yetkisiz kullanımı desteklemez. + +## Lisans + +[MIT](LICENSE) diff --git a/README_uk.md b/README_uk.md new file mode 100644 index 00000000000..59063869da9 --- /dev/null +++ b/README_uk.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Українська** · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Реверс-інжиніринг будь-чого + +### Один MCP для реверс-інжинірингу бінарних файлів, застосунків і поведінки під час виконання. + +**Побачили цікаву функцію? Зрозумійте, як вона працює, аж до рівня двійкового коду.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Сайт](https://rea.tools/) · [Посібники](https://rea.tools/guides/) · [Приклади](https://rea.tools/showcase/)** + +[Швидкий старт](#швидкий-старт) · [Як працює REA](#як-працює-rea) · [Що можна аналізувати](#що-можна-аналізувати) · [Приклади](#приклади) · [Поширені запитання](#поширені-запитання) · [Документація](#документація) + +npx rea-agents setup + +
+ +REA запускає міст аналізу всередині Hopper під час дослідження нативного бінарного файла + +
+ + + + + +
+ + Discord
+ Приєднуйтеся до спільноти реверс-інжинірингу +

+ Discord · Запитання й відповіді · Обмін результатами +
+ +
+ +
+ +--- + +Побачили в застосунку функцію, яку хочете додати до свого продукту? Попросіть агента дослідити її за допомогою REA. Він може перевірити застосунок без вихідного коду, пояснити роботу функції, показати підтвердження та створити реалізацію для вашого проєкту. + +REA підключає вашого агента до інструментів дослідження нативних бінарних файлів, застосунків JavaScript і Electron, збірок .NET та сайтів. Ті самі інструменти можна використовувати з термінала. Аналіз виконується локально, а результати містять підтвердження й обмеження, на яких ґрунтується кожен висновок. + +Налаштування реєструє REA у вашому агенті та встановлює відповідні інструкції робочих процесів. Для нативного аналізу можна використати вже встановлений Hopper або Ghidra; налаштування також може встановити Hopper за вашою згодою. Статичному аналізу JavaScript не потрібен жоден із цих рушіїв. + +> **[Відвідайте сайт REA](https://rea.tools/)**, щоб знайти інструкції налаштування, ілюстровані посібники та реальні дослідження. + +## Швидкий старт + +### Налаштуйте агента + +Якщо Node.js і npm встановлено, виконайте: + +```bash +npx rea-agents setup +``` + +Виберіть агентів, перевірте запропоновані зміни та схваліть їх. Налаштування додає MCP-сервер REA та відповідні інструкції робочих процесів, створюючи резервні копії наявної конфігурації. Після цього перезапустіть агента. + +Налаштування підтримує Claude Code, Codex, Cursor, Gemini CLI та [інших агентів](docs/installation.md#supported-agents). Конфігурацію провайдерів і ручну реєстрацію MCP описано в розділі [встановлення й налаштування](docs/installation.md). + +### Запитайте агента + +```text +Досліди, як працює пошук у застосунку Notes, покажи підтвердження й створи подібну функцію для мого проєкту. +``` + +Замініть Notes цільовим застосунком і вкажіть функцію, яку хочете зрозуміти. + +### Використовуйте термінал + +Перевірте розпакований каталог застосунку JavaScript/Electron або файл ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Результат містить модулі, імпорти, межі Electron і підтвердження для них. Замініть шлях своїм, наприклад `"D:/apps/example"` у Windows. + +Для регулярного використання встановіть команду `rea`: + +```bash +npm install --global rea-agents +rea --help +``` + +Для нативного аналізу спочатку налаштуйте провайдера. Нативні команди, вибір провайдера, знімки та використання у скриптах описує [посібник CLI та Evidence](docs/cli.md). + +### Оновіть REA + +REA швидко розвивається, а нові випуски часто містять виправлення помилок. Підтримуйте встановлену версію актуальною. + +Для CLI, встановленого через npm: + +```bash +rea update +``` + +Щоб оновити реєстрації агентів і skill, виконайте команду налаштування, яку виведе оновлення. + +Якщо використовуєте `npx`, оновіть налаштування агента командою: + +```bash +npx rea-agents@latest setup +``` + +Перевірте зміни налаштувань і перезапустіть агента. Для разових команд CLI використовуйте `npx rea-agents@latest`, а потім потрібну команду. + +## Як працює REA + +Агент викликає REA через MCP, щоб перевірити ціль і простежити пов'язаний код. REA повертає знахідки разом із підтвердженнями. Агент використовує їх для уточнювальних запитань, пояснення поведінки або написання й тестування реалізації. Команди CLI використовують ті самі робочі процеси. + +![Процес дослідження з REA: агент запитує про локальну ціль, REA перевіряє й простежує її інструментами аналізу, а агент використовує повернений код, посилання та невідомі дані для пояснення, реалізації й тестування.](website/public/assets/figures/rea-investigation-flow.svg) + +[Відкрити зображення в повному розмірі](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Що можна аналізувати + +REA потребує Node.js 22.x (>=22.19), 24.x (>=24.11) або 26+, а також npm. Додаткові інструменти й підтримувані системи залежать від цілі: + +| Ціль | Що повертає REA | Вимоги й посібник | +| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| Нативні бінарні файли | Псевдокод, асемблерний код, рядки, символи, виклики й посилання | Hopper, Ghidra або IDA; [нативний аналіз](https://rea.tools/guides/native/) | +| Структура ELF без запуску | Секції, сегменти, початкові символи/релокації та можливі захисні механізми за даними статичного аналізу | pwntools, наданий стороною, що викликає інструмент, у Linux x64; [діагностика бінарних файлів](docs/binary-diagnostics.md) | +| Байт-код EVM | Селектори диспетчеризації, зміщення байтів, виведені припущення про аргументи та змінюваність стану | Локальні необроблені байти або шістнадцяткове представлення; [посібник EVM без запуску](docs/evm-bytecode.md) | +| Записані аварійні завершення в Linux | Необроблені записи note, регістри/сигнали кожного записаного потоку й необов'язкові кандидати зіставлення пам'яті | pwntools, наданий стороною, що викликає інструмент; необов'язкові GDB/pwndbg; [записані аварійні завершення](docs/recorded-crashes.md) | +| JavaScript / Electron | Модулі, імпорти, карти вихідного коду, маршрути, IPC і зв'язки з нативними доповненнями | Node.js і npm; [аналіз застосунків](https://rea.tools/guides/javascript/) | +| Сайти | Структура сторінки, скрипти, мережеві спостереження й запитані знімки екрана | Браузер сімейства Chrome; [аналіз браузера](https://rea.tools/guides/browser/) | +| Збережені мережеві захоплення | Запити, відповіді, доступні корисні дані та місця в джерелі | HAR; mitmdump у Linux для захоплень у власному форматі mitmproxy; [посібник мережевих захоплень](docs/web-network-captures.md) | +| Збірки .NET | Метадані, інструкції CIL, оголошені нативні залежності й порівняння збірок | Статична перевірка; [посібник керованого коду](docs/managed-code-analysis.md) | +| Android APK | Оголошення маніфесту, класи, декомпільовані методи й посилання | JADX без графічного інтерфейсу й повний JDK у Linux/macOS; [посібник Android](docs/android-analysis.md) | +| Прошивки | Області, результати видобування й передавання до нативного аналізу | Binwalk / Unblob у Linux; [посібник прошивок](docs/firmware-analysis.md) | +| Пакети й ресурси | Переліки файлів, хеші, plist, структура пакетів Apple та видобуті ресурси | [Посібник артефактів і JavaScript](docs/javascript-artifact-reconstruction.md), [застосунки Apple](docs/apple-application-analysis.md) | +| Поведінка процесів | Вивід термінала, взаємодії, спостереження за завершенням і файловою системою та порівняння запусків | Linux/macOS із нативним PTY; [захоплення процесів](docs/process-capture.md) | + +Статична перевірка JavaScript і .NET читає надані файли, не запускаючи застосунок. Захоплення під час виконання запускає вибрану ціль або взаємодіє з нею з правами вашого користувача; відповідні посібники описують його вплив. + + + +Нативні формати й підтримка систем відрізняються залежно від провайдера. Дивіться [налаштування Hopper і Ghidra](docs/installation.md#hopper), [посібник IDA](docs/ida-provider.md) та [експериментальну підтримку Ghidra у Windows](docs/windows-ghidra-p0.md). Ghidra також підтримує [аналіз 16-бітного DOS](docs/ghidra-dos.md). Для вибору провайдера дивіться [посібник CLI](docs/cli.md#choose-a-provider). Для функцій, доданих після найновішої версії в npm, перевірте [доступність у випусках](docs/installation.md#released-package-and-main). + +## Приклади + +### DX-Ball: відновлення розрахунку звукової панорами + +Простежте звуковий виклик до допоміжної функції, що перетворює координату на панорамування, перевірте інструкції та перетворіть неповний псевдокод на C. Відновлена реалізація проходить 3 205 тестів з оригінальним x86-кодом і відтворює всі 63 байти скомпільованої функції. + +[Прочитати дослідження](https://rea.tools/showcase/dx-ball/) · [Репозиторій реконструкції](https://github.com/N0zoM1z0/dx-ball) + +### Notion: відстеження мосту буфера обміну Electron + +Знайдіть API буфера обміну в процесі рендерингу, простежте його через preload та IPC до головного процесу й перевірте формат буфера обміну з даними форматування. + +[Прочитати дослідження](https://rea.tools/showcase/notion/) + +### TH04: відновлення розрахунку кільця куль у DOS + +Перевірте 16-бітні інструкції оригінальної гри для PC-98, відновіть розрахунки фіксованих і прицільних кутів та порівняйте відновлений C++ із виводом тогочасного компілятора. + +[Прочитати дослідження](https://rea.tools/showcase/th04/) · [Репозиторій реконструкції](https://github.com/N0zoM1z0/th04) + +Якщо ви дослідили з REA щось цікаве, поділіться результатом. Створіть [issue](https://github.com/morluto/rea/issues) або [pull request](https://github.com/morluto/rea/pulls) і вкажіть ціль, ваше запитання, як допоміг REA та що ви з'ясували. + +## Поширені запитання + +
+Які агенти можуть використовувати REA? + +Будь-який агент, що підтримує локальні MCP-сервери. Налаштування конфігурує [підтримуваних агентів](docs/installation.md#supported-agents); інші клієнти можуть використати [ручну реєстрацію MCP](docs/installation.md#mcp-registry). + +
+ +
+Чи потрібні мені Hopper, Ghidra або IDA? + +Поглиблений нативний аналіз використовує один із цих інструментів. Статична перевірка JavaScript і .NET працює без рушія нативного аналізу. Налаштування може встановити Hopper після схвалення; Ghidra й IDA використовують наявні встановлення. Дивіться [налаштування провайдерів](docs/installation.md#hopper). + +
+ +
+Чи потрібно спочатку запускати Hopper? + +REA запускає Hopper, коли він потрібен для операції. У macOS під час першого запуску може з'явитися діалог вибору деморежиму або активації ліцензії. Дивіться [запуск Hopper та усунення неполадок](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+Що дає встановлення skill із skills.sh? + +Skill надає агенту інструкції дослідження. Використовуйте `rea setup`, щоб зареєструвати MCP-сервер REA та встановити відповідні інструкції, а потім перезапустіть агента. Дивіться [встановлення лише skill](docs/installation.md#skill-only-installation). + +
+ +
+Який код повертає REA? + +Нативний аналіз повертає псевдокод і асемблерний код. Аналіз JavaScript/Electron відновлює модулі та їхні зв'язки. Агент використовує ці результати, щоб написати й протестувати реалізацію; [приклади](#приклади) містять практичні дослідження. + +
+ +
+Чи завантажує REA мій застосунок на сервер? + +REA аналізує цілі локально. Ваш агент отримує результати інструментів, а постачальник його моделі має власну політику даних. + +
+ +
+Що робити, якщо я зіткнувся з помилкою? + +Спочатку оновіться: нещодавній випуск уже міг виправити помилку. + +Для CLI, встановленого через npm: + +```bash +rea update +``` + +Для налаштування агента через `npx`: + +```bash +npx rea-agents@latest setup +``` + +Якщо використовуєте агента, завершіть [оновлення налаштувань](#оновіть-rea) і перезапустіть його. Повторіть те саме завдання. Якщо проблема залишається, [відкрийте issue](https://github.com/morluto/rea/issues) із версією REA, типом цілі, кроками відтворення та виводом помилки. + +
+ +## Документація + +Почніть із [практичних посібників](https://rea.tools/guides/) на сайті. Точні параметри, передумови й контракти результатів описано тут: + +- [Встановлення й налаштування](docs/installation.md): реєстрація агентів, конфігурація провайдерів, оновлення та видалення. +- [Готовність та усунення неполадок](docs/installation.md#check-readiness-for-your-task): діагностика конкретного агента або рушія аналізу. +- [CLI та Evidence](docs/cli.md): команди, вибір провайдера, знімки, імпорт/експорт і статуси завершення. +- [Контракти MCP](docs/mcp-contracts.md) і [промпти агентів](docs/mcp-prompts.md): результати інструментів, сеанси й керовані дослідження. +- [Каталог інструментів](docs/mcp-contracts.md#generated-catalog): створюваний під час збірки перелік інструментів, провайдерів і команд CLI. +- [План розвитку](docs/roadmap.md): заплановані роботи й відстеження можливостей. + +Повідомляйте про вразливості згідно з [SECURITY.md](SECURITY.md). + +## Участь у проєкті + +Будемо раді вашій допомозі з REA! [Відкрийте issue](https://github.com/morluto/rea/issues), щоб повідомити про помилку чи запропонувати функцію, або [надішліть pull request](https://github.com/morluto/rea/pulls) з поліпшеннями коду чи документації. + +Налаштування середовища розробки й перевірки описано в [CONTRIBUTING.md](CONTRIBUTING.md), шляхи верифікації — у [посібнику тестування](docs/testing.md), а структуру проєкту — на [схемі архітектури](docs/architecture.mermaid). + +## Посилання проєкту + +[Сайт](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Безпека](SECURITY.md) + +## Історія зірок + +🎉 **20 000 зірок на GitHub — дякуємо!** + +Дякуємо всім, хто використовує REA, повідомляє про помилки, тестує збірки й допомагає з виправленнями. + + + + + + Історія зірок REA на GitHub + + + +## Відмова від відповідальності + +REA надає інструменти для законних досліджень, аналізу й реконструкції за допомогою реверс-інжинірингу. Ви відповідаєте за отримання необхідних дозволів і дотримання застосовних законів. Проєкт не підтримує незаконне або несанкціоноване використання. + +## Ліцензія + +[MIT](LICENSE) diff --git a/README_vi.md b/README_vi.md new file mode 100644 index 00000000000..a5d20ab3726 --- /dev/null +++ b/README_vi.md @@ -0,0 +1,285 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · **Tiếng Việt** · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA: Kỹ thuật dịch ngược cho mọi thứ + +### Một MCP để dịch ngược tệp nhị phân, ứng dụng và hành vi khi chạy. + +**Thấy một tính năng bạn thích? Hiểu cách nó hoạt động, đến tận mức mã nhị phân.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[Trang web](https://rea.tools/) · [Hướng dẫn](https://rea.tools/guides/) · [Các nghiên cứu điển hình](https://rea.tools/showcase/)** + +[Bắt đầu nhanh](#bắt-đầu-nhanh) · [Cách REA hoạt động](#cách-rea-hoạt-động) · [Những gì bạn có thể phân tích](#những-gì-bạn-có-thể-phân-tích) · [Các nghiên cứu điển hình](#các-nghiên-cứu-điển-hình) · [Câu hỏi thường gặp](#câu-hỏi-thường-gặp) · [Tài liệu](#tài-liệu) + +npx rea-agents setup + +
+ +REA khởi chạy cầu nối phân tích bên trong Hopper khi kiểm tra tệp nhị phân mã máy + +
+ + + + + +
+ + Discord
+ Tham gia cộng đồng kỹ thuật dịch ngược +

+ Discord · Hỏi đáp · Chia sẻ kết quả +
+ +
+ +
+ +--- + +Bạn thấy một tính năng trong ứng dụng và muốn đưa nó vào sản phẩm của mình? Hãy nhờ tác nhân của bạn điều tra bằng REA. Tác nhân có thể kiểm tra ứng dụng mà không cần mã nguồn, giải thích cách tính năng hoạt động, đưa ra bằng chứng và xây dựng một phiên bản cho dự án của bạn. + +REA kết nối tác nhân của bạn với các công cụ kiểm tra tệp nhị phân mã máy, ứng dụng JavaScript và Electron, assembly .NET và trang web. Bạn cũng có thể dùng các công cụ đó từ terminal. Việc phân tích chạy cục bộ; kết quả bao gồm bằng chứng và giới hạn làm cơ sở cho từng kết luận. + +Quá trình thiết lập đăng ký REA với tác nhân và cài đặt các hướng dẫn quy trình tương ứng. Phân tích mã máy có thể sử dụng Hopper hoặc Ghidra đã cài sẵn; quá trình thiết lập cũng có thể cài Hopper nếu bạn chấp thuận. Phân tích tĩnh JavaScript không cần công cụ nào trong hai công cụ này. + +> **[Truy cập trang web REA](https://rea.tools/)** để xem cách thiết lập, hướng dẫn có hình minh họa và các nghiên cứu điển hình thực tế. + +## Bắt đầu nhanh + +### Thiết lập tác nhân + +Sau khi cài Node.js và npm, chạy: + +```bash +npx rea-agents setup +``` + +Chọn các tác nhân, xem lại những thay đổi được đề xuất và chấp thuận. Quá trình thiết lập thêm máy chủ MCP của REA cùng hướng dẫn quy trình tương ứng, đồng thời sao lưu cấu hình hiện có. Sau đó khởi động lại tác nhân. + +Quá trình thiết lập hỗ trợ Claude Code, Codex, Cursor, Gemini CLI và [các tác nhân khác](docs/installation.md#supported-agents). Xem [cài đặt và thiết lập](docs/installation.md) để cấu hình trình phân tích và đăng ký MCP thủ công. + +### Hỏi tác nhân + +```text +Tìm hiểu cách tính năng tìm kiếm trong ứng dụng Notes hoạt động, cho tôi xem bằng chứng và xây dựng tính năng tương tự cho dự án của tôi. +``` + +Thay Notes bằng ứng dụng mục tiêu và chỉ rõ tính năng bạn muốn hiểu. + +### Sử dụng terminal + +Kiểm tra thư mục ứng dụng JavaScript/Electron đã giải nén hoặc tệp ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +Kết quả bao gồm mô-đun, các khai báo import, ranh giới Electron và bằng chứng tương ứng. Thay đường dẫn bằng mục tiêu của bạn, chẳng hạn `"D:/apps/example"` trên Windows. + +Để cài lệnh `rea` cho việc sử dụng thường xuyên: + +```bash +npm install --global rea-agents +rea --help +``` + +Để phân tích mã máy, trước tiên hãy cấu hình trình phân tích. Xem [hướng dẫn CLI và Evidence](docs/cli.md) về các lệnh phân tích mã máy, lựa chọn trình phân tích, ảnh chụp trạng thái và sử dụng trong script. + +### Cập nhật REA + +REA thay đổi nhanh và các bản phát hành mới thường xuyên có bản sửa lỗi. Hãy giữ bản cài đặt của bạn được cập nhật. + +Với CLI được cài bằng npm: + +```bash +rea update +``` + +Để cập nhật đăng ký tác nhân và skill, chạy lệnh thiết lập được in ra sau khi cập nhật. + +Nếu dùng `npx`, cập nhật thiết lập tác nhân bằng: + +```bash +npx rea-agents@latest setup +``` + +Xem lại thay đổi cấu hình và khởi động lại tác nhân. Với lệnh CLI dùng một lần, dùng `npx rea-agents@latest` rồi thêm lệnh cần chạy. + +## Cách REA hoạt động + +Tác nhân gọi REA qua MCP để kiểm tra mục tiêu và lần theo mã liên quan. REA trả về các phát hiện kèm bằng chứng. Tác nhân dùng chúng để đặt câu hỏi tiếp theo, giải thích hành vi hoặc viết và kiểm thử một cách triển khai. Các lệnh CLI sử dụng cùng quy trình. + +![Quy trình điều tra của REA: tác nhân hỏi về mục tiêu cục bộ, REA kiểm tra và lần theo mục tiêu bằng công cụ phân tích, rồi tác nhân dùng mã, tham chiếu và những điểm chưa biết được trả về để giải thích, triển khai và kiểm thử.](website/public/assets/figures/rea-investigation-flow.svg) + +[Mở hình ở kích thước đầy đủ](website/public/assets/figures/rea-investigation-flow.svg). + + + +## Những gì bạn có thể phân tích + +REA yêu cầu Node.js 22.x (>=22.19), 24.x (>=24.11) hoặc 26+, cùng với npm. Công cụ bổ sung và nền tảng chạy REA được hỗ trợ tùy thuộc vào mục tiêu: + +| Mục tiêu | REA trả về gì | Yêu cầu và hướng dẫn | +| ----------------------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | +| Tệp nhị phân mã máy | Mã giả, hợp ngữ, chuỗi, ký hiệu, lời gọi và tham chiếu | Hopper, Ghidra hoặc IDA; [phân tích mã máy](https://rea.tools/guides/native/) | +| Bố cục ELF ngoại tuyến | Các section, segment, ký hiệu/thông tin tái định vị gốc và các biện pháp giảm thiểu có thể có từ phân tích tĩnh | pwntools do bên gọi cung cấp trên Linux x64; [chẩn đoán tệp nhị phân](docs/binary-diagnostics.md) | +| Bytecode EVM | Bộ chọn điều phối, độ lệch byte, tham số suy luận và khả năng thay đổi trạng thái | Đầu vào cục bộ chứa byte thô hoặc dạng thập lục phân; [hướng dẫn EVM ngoại tuyến](docs/evm-bytecode.md) | +| Sự cố Linux đã ghi lại | Bản ghi note thô, thanh ghi/tín hiệu của mọi luồng được ghi lại và các ứng viên ánh xạ tùy chọn | pwntools do bên gọi cung cấp; GDB/pwndbg tùy chọn; [sự cố đã ghi lại](docs/recorded-crashes.md) | +| JavaScript / Electron | Mô-đun, các khai báo import, source map, tuyến, IPC và quan hệ với tiện ích mở rộng mã máy | Node.js và npm; [phân tích ứng dụng](https://rea.tools/guides/javascript/) | +| Trang web | Cấu trúc trang, script, quan sát mạng và ảnh chụp màn hình được yêu cầu | Trình duyệt thuộc họ Chrome; [phân tích trình duyệt](https://rea.tools/guides/browser/) | +| Bản ghi lưu lượng mạng đã lưu | Yêu cầu, phản hồi, payload có thể truy cập và vị trí nguồn | HAR; mitmdump trên Linux cho bản ghi định dạng gốc của mitmproxy; [hướng dẫn bản ghi mạng](docs/web-network-captures.md) | +| Assembly .NET | Siêu dữ liệu, lệnh CIL, các phụ thuộc mã máy được khai báo và so sánh bản dựng | Kiểm tra tĩnh; [hướng dẫn mã được quản lý](docs/managed-code-analysis.md) | +| APK Android | Khai báo manifest, lớp, phương thức được dịch ngược và tham chiếu | JADX không giao diện đồ họa và JDK đầy đủ trên Linux/macOS; [hướng dẫn Android](docs/android-analysis.md) | +| Phần sụn | Các vùng, kết quả trích xuất và dữ liệu chuyển sang phân tích mã máy | Binwalk / Unblob trên Linux; [hướng dẫn phần sụn](docs/firmware-analysis.md) | +| Gói và tài nguyên | Danh sách tệp, giá trị băm, plist, cấu trúc bundle Apple và tài nguyên được trích xuất | [Hướng dẫn hiện vật phần mềm và JavaScript](docs/javascript-artifact-reconstruction.md), [ứng dụng Apple](docs/apple-application-analysis.md) | +| Hành vi tiến trình | Đầu ra terminal, tương tác, quan sát khi thoát và hệ thống tệp, cùng so sánh các lần chạy | Linux/macOS có PTY gốc; [ghi lại tiến trình](docs/process-capture.md) | + +Kiểm tra tĩnh JavaScript và .NET đọc các tệp được cung cấp mà không chạy ứng dụng. Ghi lại khi chạy sẽ khởi chạy hoặc tương tác với mục tiêu đã chọn bằng quyền người dùng của bạn; từng hướng dẫn mô tả tác động của nó. + + + +Định dạng mã máy và nền tảng chạy REA được hỗ trợ khác nhau tùy trình phân tích. Xem [thiết lập Hopper và Ghidra](docs/installation.md#hopper), [hướng dẫn IDA](docs/ida-provider.md) và [hỗ trợ Ghidra thử nghiệm trên Windows](docs/windows-ghidra-p0.md). Ghidra cũng hỗ trợ [phân tích DOS 16 bit](docs/ghidra-dos.md). Để chọn trình phân tích, xem [hướng dẫn CLI](docs/cli.md#choose-a-provider). Kiểm tra [khả năng sử dụng theo bản phát hành](docs/installation.md#released-package-and-main) đối với tính năng được bổ sung sau phiên bản npm mới nhất. + +## Các nghiên cứu điển hình + +### DX-Ball: tái tạo phép tính phân bố âm thanh trái phải + +Lần theo lời gọi âm thanh đến hàm hỗ trợ chuyển vị trí thành giá trị phân bố âm thanh, kiểm tra các lệnh và chuyển mã giả chưa đầy đủ sang C. Bản tái tạo vượt qua 3.205 trường hợp kiểm thử với x86 gốc và tái tạo đủ 63 byte của hàm đã biên dịch. + +[Đọc nghiên cứu điển hình](https://rea.tools/showcase/dx-ball/) · [Kho mã tái tạo](https://github.com/N0zoM1z0/dx-ball) + +### Notion: lần theo cầu nối clipboard của Electron + +Tìm API clipboard trong tiến trình kết xuất, lần theo qua preload và IPC đến tiến trình chính, rồi kiểm tra định dạng clipboard có dữ liệu định dạng phong phú. + +[Đọc nghiên cứu điển hình](https://rea.tools/showcase/notion/) + +### TH04: khôi phục phép tính vòng đạn trong DOS + +Kiểm tra các lệnh 16 bit của trò chơi PC-98 gốc, khôi phục phép tính góc cố định và góc nhắm mục tiêu, rồi so sánh C++ được tái tạo với đầu ra của trình biên dịch thời đó. + +[Đọc nghiên cứu điển hình](https://rea.tools/showcase/th04/) · [Kho mã tái tạo](https://github.com/N0zoM1z0/th04) + +Nếu bạn dùng REA cho một mục tiêu thú vị, chúng tôi muốn biết. Chia sẻ trường hợp của bạn qua [issue](https://github.com/morluto/rea/issues) hoặc [pull request](https://github.com/morluto/rea/pulls), nêu mục tiêu, câu hỏi, cách REA giúp bạn và những gì tìm được. + +## Câu hỏi thường gặp + +
+Tác nhân nào có thể dùng REA? + +Bất kỳ tác nhân nào hỗ trợ máy chủ MCP cục bộ. Quá trình thiết lập cấu hình các [tác nhân được hỗ trợ](docs/installation.md#supported-agents); ứng dụng khách khác có thể dùng [đăng ký MCP thủ công](docs/installation.md#mcp-registry). + +
+ +
+Tôi có cần Hopper, Ghidra hoặc IDA không? + +Phân tích mã máy chuyên sâu sử dụng một trong những công cụ đó. Kiểm tra tĩnh JavaScript và .NET hoạt động mà không cần bộ máy phân tích mã máy. Quá trình thiết lập có thể cài Hopper sau khi được chấp thuận; Ghidra và IDA dùng bản cài đặt sẵn có. Xem [thiết lập trình phân tích](docs/installation.md#hopper). + +
+ +
+Tôi có cần khởi chạy Hopper trước không? + +REA khởi chạy Hopper khi một thao tác cần đến nó. Trên macOS, hộp thoại lần chạy đầu có thể yêu cầu chọn chế độ dùng thử hoặc kích hoạt giấy phép. Xem [khởi động Hopper và khắc phục sự cố](docs/installation.md#launcher-paths-and-troubleshooting). + +
+ +
+Cài skill từ skills.sh có tác dụng gì? + +Skill cung cấp hướng dẫn điều tra cho tác nhân. Dùng `rea setup` để đăng ký máy chủ MCP của REA và cài hướng dẫn tương ứng, sau đó khởi động lại tác nhân. Xem [chỉ cài skill](docs/installation.md#skill-only-installation). + +
+ +
+REA trả về mã gì? + +Phân tích mã máy trả về mã giả và hợp ngữ. Phân tích JavaScript/Electron khôi phục mô-đun cùng quan hệ giữa chúng. Tác nhân dùng các phát hiện này để viết và kiểm thử cách triển khai; các [nghiên cứu điển hình](#các-nghiên-cứu-điển-hình) cung cấp ví dụ thực hành. + +
+ +
+REA có tải ứng dụng của tôi lên không? + +REA phân tích mục tiêu cục bộ. Tác nhân nhận kết quả công cụ, và nhà cung cấp mô hình của tác nhân có chính sách dữ liệu riêng. + +
+ +
+Tôi nên làm gì khi gặp lỗi? + +Hãy cập nhật trước; một bản phát hành gần đây có thể đã sửa lỗi. + +Với CLI được cài bằng npm: + +```bash +rea update +``` + +Với thiết lập tác nhân qua `npx`: + +```bash +npx rea-agents@latest setup +``` + +Nếu dùng tác nhân, hãy hoàn tất [làm mới thiết lập](#cập-nhật-rea) và khởi động lại nó. Thử lại cùng tác vụ. Nếu vấn đề vẫn còn, [mở issue](https://github.com/morluto/rea/issues) với phiên bản REA, loại mục tiêu, các bước tái hiện và đầu ra lỗi. + +
+ +## Tài liệu + +Bắt đầu với [hướng dẫn thực hành](https://rea.tools/guides/) trên trang web. Để biết tùy chọn cụ thể, điều kiện tiên quyết và hợp đồng kết quả: + +- [Cài đặt và thiết lập](docs/installation.md): đăng ký tác nhân, cấu hình trình phân tích, cập nhật và gỡ cài đặt. +- [Kiểm tra mức sẵn sàng và khắc phục sự cố](docs/installation.md#check-readiness-for-your-task): chẩn đoán một tác nhân hoặc bộ máy phân tích cụ thể. +- [CLI và Evidence](docs/cli.md): lệnh, lựa chọn trình phân tích, ảnh chụp trạng thái, nhập/xuất và trạng thái thoát. +- [Hợp đồng MCP](docs/mcp-contracts.md) và [prompt cho tác nhân](docs/mcp-prompts.md): kết quả công cụ, phiên làm việc và điều tra có hướng dẫn. +- [Danh mục công cụ](docs/mcp-contracts.md#generated-catalog): danh sách công cụ, trình phân tích và lệnh CLI được tạo khi dựng dự án. +- [Lộ trình](docs/roadmap.md): công việc dự kiến và theo dõi các khả năng. + +Báo cáo lỗ hổng theo [SECURITY.md](SECURITY.md). + +## Đóng góp + +Chúng tôi hoan nghênh bạn giúp phát triển REA! [Mở issue](https://github.com/morluto/rea/issues) để báo lỗi hoặc đề xuất tính năng, hoặc [gửi pull request](https://github.com/morluto/rea/pulls) để cải thiện mã hay tài liệu. + +Xem [CONTRIBUTING.md](CONTRIBUTING.md) về thiết lập môi trường phát triển và các bước kiểm tra, [hướng dẫn kiểm thử](docs/testing.md) về các luồng xác minh và [sơ đồ kiến trúc](docs/architecture.mermaid) về cấu trúc dự án. + +## Liên kết dự án + +[Trang web](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Bảo mật](SECURITY.md) + +## Lịch sử sao + +🎉 **20.000 sao trên GitHub — xin cảm ơn!** + +Cảm ơn mọi người đã sử dụng REA, báo lỗi, kiểm thử bản dựng và đóng góp bản sửa lỗi. + + + + + + Lịch sử sao GitHub của REA + + + +## Miễn trừ trách nhiệm + +REA cung cấp công cụ cho nghiên cứu dịch ngược, phân tích và tái tạo hợp pháp. Bạn có trách nhiệm có được các quyền cho phép cần thiết và tuân thủ luật áp dụng. Dự án không ủng hộ việc sử dụng trái pháp luật hoặc không được cho phép. + +## Giấy phép + +[MIT](LICENSE) diff --git a/README_zh-TW.md b/README_zh-TW.md new file mode 100644 index 00000000000..26c21f27de0 --- /dev/null +++ b/README_zh-TW.md @@ -0,0 +1,287 @@ +
+ +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · **繁體中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) + +# REA:逆向分析一切 + +### 通過一個 MCP 服務,逆向分析二進位檔案、應用程式和執行階段行為。 + +**看到喜歡的功能,弄清它的工作原理,深入到二進位層面。** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[網站](https://rea.tools/) · [指南](https://rea.tools/guides/) · [案例](https://rea.tools/showcase/)** + +[快速開始](#快速開始) · [REA 的工作原理](#rea-的工作原理) · [可以分析什麼](#可以分析什麼) · [案例](#案例) · [常見問題](#常見問題) · [文件](#文件) + +npx rea-agents setup + +
+ +REA 在 Hopper 中啟動分析橋,檢查原生二進位檔案 + +
+ + + + + +
+ + Discord
+ 加入逆向工程社群 +

+ Discord · 問答 · 成果分享 +
+ +
+ +
+ +--- + +在應用中看到一個想加入自己產品的功能?讓 AI 代理用 REA 調查它。即使沒有原始碼,AI 代理也可以檢查應用、解釋功能的工作方式、展示證據,並為你的專案實現類似功能。 + +REA 將 AI 代理連接到分析工具,用於檢查原生二進位檔案、JavaScript 和 Electron 應用、.NET 程式集以及網站。你也可以在終端中使用同樣的工具。分析在本機執行,結果包含支撐各項結論的證據和相關限制。 + +設定流程會向 AI 代理註冊 REA,並安裝對應的工作流程指引。原生分析可以使用已有的 Hopper 或 Ghidra;設定流程也可以在你核准後安裝 Hopper。靜態 JavaScript 分析不需要這兩個引擎。 + +> **[訪問 REA 網站](https://rea.tools/)**,查看設定說明、圖解指南和真實案例。 + +## 快速開始 + +### 設定 AI 代理 + +安裝 Node.js 和 npm 後,執行: + +```bash +npx rea-agents setup +``` + +選擇 AI 代理,檢查計劃中的變更並核准。設定流程會添加 REA 的 MCP 服務和對應的工作流程指引,並備份已有組態。完成後重新啟動 AI 代理。 + +設定流程支援 Claude Code、Codex、Cursor、Gemini CLI 和[其他 AI 代理](docs/installation.md#supported-agents)。提供者組態和手動註冊 MCP 的方法見[安裝與設定](docs/installation.md)。 + +### 詢問 AI 代理 + +```text +弄清 Notes 應用的搜尋功能如何工作,展示證據,並為我的專案實現類似功能。 +``` + +把 Notes 替換為目標應用,並描述你想瞭解的功能。 + +### 使用終端 + +檢查已解包的 JavaScript/Electron 應用目錄或 ASAR 檔案: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +結果包含模組、匯入、Electron 邊界及相關證據。將路徑替換為你的目標路徑,例如 Windows 上的 `"D:/apps/example"`。 + +如果需要經常使用,可以安裝 `rea` 命令: + +```bash +npm install --global rea-agents +rea --help +``` + +原生分析需要先設定分析提供者。原生命令、提供者選擇、快照和腳本用法見 [CLI 與 Evidence 指南](docs/cli.md)。 + +### 更新 REA + +REA 更新較快,新版本經常包含錯誤修復。請將安裝的版本保持最新。 + +通過 npm 安裝的 CLI 使用: + +```bash +rea update +``` + +要更新 AI 代理註冊和 skill,請執行更新命令輸出的設定命令。 + +如果使用 `npx`,通過以下命令更新 AI 代理設定: + +```bash +npx rea-agents@latest setup +``` + +檢查設定變更並重新啟動 AI 代理。單次 CLI 操作可使用 `npx rea-agents@latest`,後接所需命令。 + +## REA 的工作原理 + +AI 代理通過 MCP 呼叫 REA,檢查目標並追蹤相關程式碼。REA 返回發現及對應證據。AI 代理據此提出後續問題、解釋行為,或編寫並測試實現。CLI 命令使用同樣的工作流程。 + +![REA 調查流程:AI 代理詢問本地目標的行為,REA 使用分析工具檢查和追蹤目標,AI 代理再利用返回的程式碼、引用和未知項進行解釋、實現與測試。](website/public/assets/figures/rea-investigation-flow.svg) + +[打開完整尺寸的圖示](website/public/assets/figures/rea-investigation-flow.svg)。 + + + +## 可以分析什麼 + +REA 需要 Node.js 22.x(>=22.19)、24.x(>=24.11)或 26+,以及 npm。額外工具和主機支援取決於分析目標: + +| 目標 | REA 返回的內容 | 要求與指南 | +| --------------------- | --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | +| 原生二進位檔案 | 偽程式碼、組合語言、字串、符號、呼叫和引用 | Hopper、Ghidra 或 IDA;[原生分析](https://rea.tools/guides/native/) | +| 離線 ELF 佈局 | 節、段、原始符號/重定位資訊及靜態防護機制候選項 | Linux x64 上由呼叫方提供的 pwntools;[二進位診斷](docs/binary-diagnostics.md) | +| EVM 位元組碼 | 分派選擇器、位元組偏移、推斷的參數和狀態可變性 | 本地原始位元組/十六進位輸入載體;[離線 EVM 指南](docs/evm-bytecode.md) | +| 已記錄的 Linux 崩潰 | 原始 note 記錄、每個已記錄執行緒的暫存器/訊號及可選的對映候選項 | 呼叫方提供的 pwntools;可選的 GDB/pwndbg;[已記錄崩潰](docs/recorded-crashes.md) | +| JavaScript / Electron | 模組、匯入、source map、路由、IPC 和原生附加元件關係 | Node.js 和 npm;[應用分析](https://rea.tools/guides/javascript/) | +| 網站 | 頁面結構、腳本、網路觀察結果和按請求獲取的螢幕擷取畫面 | Chrome 系瀏覽器;[瀏覽器分析](https://rea.tools/guides/browser/) | +| 已保存的網路擷取 | 請求、回應、可訪問的酬載和來源位置 | HAR;原生 mitmproxy 擷取需要 Linux 上的 mitmdump;[擷取指南](docs/web-network-captures.md) | +| .NET 程式集 | 中繼資料、CIL 指令、聲明的原生依賴和建置對比 | 靜態檢查;[托管程式碼指南](docs/managed-code-analysis.md) | +| Android APK | 清單聲明、類別、反編譯的方法和引用 | Linux/macOS 上的無界面 JADX 和完整 JDK;[Android 指南](docs/android-analysis.md) | +| 韌體 | 區域、提取結果和轉交原生分析的內容 | Linux 上的 Binwalk / Unblob;[韌體指南](docs/firmware-analysis.md) | +| 軟體包與資源 | 檔案清單、摘要、plist、Apple bundle 結構和提取的資源 | [製品與 JavaScript 指南](docs/javascript-artifact-reconstruction.md)、[Apple 應用](docs/apple-application-analysis.md) | +| 處理程序行為 | 終端輸出、互動、退出和檔案系統觀察結果,以及執行對比 | 支援原生 PTY 的 Linux/macOS;[處理程序擷取](docs/process-capture.md) | + +靜態 JavaScript 和 .NET 檢查讀取提供的檔案,不執行應用。執行階段擷取會以你的使用者權限執行目標或與之互動;各執行階段指南說明瞭具體影響。 + + + +不同提供者支援的原生格式和主機系統有所不同。請參閱 [Hopper 與 Ghidra 設定](docs/installation.md#hopper)、[IDA 指南](docs/ida-provider.md)和[實驗性 Windows Ghidra 支援](docs/windows-ghidra-p0.md)。Ghidra 也支援 [16 位 DOS 分析](docs/ghidra-dos.md)。提供者選擇見 [CLI 指南](docs/cli.md#choose-a-provider)。對於最新 npm 版本發佈後新增的功能,請查看[發佈可用性](docs/installation.md#released-package-and-main)。 + +## 案例 + +### DX-Ball:重建聲像計算 + +沿聲音呼叫追蹤到根據位置計算聲像的輔助函數,檢查指令,將不完整的偽程式碼轉為 C。重建結果通過了 3,205 個原始 x86 測試用例,並復現了編譯後函數的全部 63 個位元組。 + +[閱讀案例](https://rea.tools/showcase/dx-ball/) · +[重建專案儲存庫](https://github.com/N0zoM1z0/dx-ball) + +### Notion:追蹤 Electron 剪貼簿橋 + +找到渲染處理程序的剪貼簿 API,沿 preload 和 IPC 追蹤到主處理程序,並檢查富格式剪貼簿資料。 + +[閱讀案例](https://rea.tools/showcase/notion/) + +### TH04:恢復 DOS 環形子彈的計算 + +檢查原始 PC-98 遊戲的 16 位指令,恢復固定角度和瞄準角度的計算,並將重建的 C++ 與當年的編譯器輸出進行對比。 + +[閱讀案例](https://rea.tools/showcase/th04/) · +[重建專案儲存庫](https://github.com/N0zoM1z0/th04) + +如果你用 REA 分析了有趣的目標,我們很想瞭解。歡迎通過 [issue](https://github.com/morluto/rea/issues) 或 [pull request](https://github.com/morluto/rea/pulls) 分享案例,說明目標、你的問題、REA 如何提供幫助,以及你的發現。 + +## 常見問題 + +
+哪些 AI 代理可以使用 REA? + +任何支援本地 MCP 服務的 AI 代理都可以使用。設定流程會設定[受支援的 AI 代理](docs/installation.md#supported-agents);其他客戶端可通過[手動註冊 MCP](docs/installation.md#mcp-registry) 使用。 + +
+ +
+需要 Hopper、Ghidra 或 IDA 嗎? + +深入的原生分析需要其中一個引擎。靜態 JavaScript 和 .NET 檢查不需要原生分析引擎。設定流程可以在你核准後安裝 Hopper;Ghidra 和 IDA 則使用已有安裝。見[提供者設定](docs/installation.md#hopper)。 + +
+ +
+需要先啟動 Hopper 嗎? + +REA 會在操作需要時啟動 Hopper。在 macOS 上,首次執行時可能會出現對話框,讓你選擇演示模式或啟用授權。見 [Hopper 啟動與故障排查](docs/installation.md#launcher-paths-and-troubleshooting)。 + +
+ +
+從 skills.sh 安裝 skill 有什麼作用? + +skill 為 AI 代理提供調查指引。使用 `rea setup` 註冊 REA 的 MCP 服務並安裝對應的指引,然後重新啟動 AI 代理。見[僅安裝 skill](docs/installation.md#skill-only-installation)。 + +
+ +
+REA 返回什麼程式碼? + +原生分析返回偽程式碼和組合語言。JavaScript/Electron 分析恢復模組及其關係。AI 代理利用這些發現編寫並測試實現;[案例](#案例)提供了完整示例。 + +
+ +
+REA 會上傳我的應用嗎? + +REA 在本機分析目標。AI 代理會接收工具結果,其模型提供商有自己的資料政策。 + +
+ +
+遇到錯誤時該怎麼辦? + +先更新;最近的版本可能已經修復了問題。 + +通過 npm 安裝的 CLI 使用: + +```bash +rea update +``` + +通過 `npx` 設定 AI 代理時使用: + +```bash +npx rea-agents@latest setup +``` + +如果正在使用 AI 代理,請完成[設定更新](#更新-rea)並重新啟動 AI 代理。重試同一任務。如果問題仍然存在,請[提交 issue](https://github.com/morluto/rea/issues),附上 REA 版本、目標類型、復現步驟和錯誤輸出。 + +
+ +## 文件 + +可以先閱讀網站上的[實操指南](https://rea.tools/guides/)。具體選項、前提條件和結果契約見: + +- [安裝與設定](docs/installation.md):AI 代理註冊、提供者組態、更新和卸載。 +- [就緒檢查與故障排查](docs/installation.md#check-readiness-for-your-task):診斷特定 AI 代理或分析引擎。 +- [CLI 與 Evidence](docs/cli.md):命令、提供者選擇、快照、匯入/匯出和退出狀態。 +- [MCP 契約](docs/mcp-contracts.md)和[AI 代理提示詞](docs/mcp-prompts.md):工具結果、會話和引導式調查。 +- [工具目錄](docs/mcp-contracts.md#generated-catalog):建置時生成的工具、提供者和 CLI 命令清單。 +- [路線圖](docs/roadmap.md):計劃中的工作和能力跟蹤。 + +請按照 [SECURITY.md](SECURITY.md) 報告漏洞。 + +## 參與貢獻 + +歡迎幫助改進 REA!你可以[提交 issue](https://github.com/morluto/rea/issues) 報告錯誤或建議功能,也可以[提交 pull request](https://github.com/morluto/rea/pulls) 改進程式碼或文件。 + +開發設定和檢查見 [CONTRIBUTING.md](CONTRIBUTING.md),驗證流程見[測試指南](docs/testing.md),專案結構見[架構圖](docs/architecture.mermaid)。 + +## 專案連結 + +[網站](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [安全](SECURITY.md) + +## Star 歷史 + +🎉 **GitHub Star 達到 20,000 個,感謝大家!** + +感謝每一位使用 REA、報告錯誤、測試建置和貢獻修復的朋友。 + + + + + + REA 的 GitHub Star 歷史 + + + +## 免責聲明 + +REA 為合法的逆向工程研究、分析和重建提供工具。你有責任取得所需授權並遵守適用法律。本專案不認同非法或未經授權的使用。 + +## 授權條款 + +[MIT](LICENSE) diff --git a/README_zh.md b/README_zh.md index c9dd4a0a04b..86ab20c80a6 100644 --- a/README_zh.md +++ b/README_zh.md @@ -1,6 +1,6 @@
-[English](README.md) · **简体中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [العربية](README_ar.md) +[English](README.md) · [Português (Brasil)](README_pt-BR.md) · **简体中文** · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) # REA:逆向分析一切 diff --git a/release-please-config.json b/release-please-config.json index f68b92ec299..2a7a80b7a4b 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -17,10 +17,6 @@ ".": { "package-name": "rea-agents", "extra-files": [ - { "type": "generic", "path": "README_zh.md" }, - { "type": "generic", "path": "README_ja.md" }, - { "type": "generic", "path": "README_ko.md" }, - { "type": "generic", "path": "README_ar.md" }, { "type": "generic", "path": "docs/installation.md" }, { "type": "generic", diff --git a/scripts/lib/docs-facts.mjs b/scripts/lib/docs-facts.mjs index 45b947e995b..c9c46d594fc 100644 --- a/scripts/lib/docs-facts.mjs +++ b/scripts/lib/docs-facts.mjs @@ -1,19 +1,13 @@ import { readFile, readdir, stat } from "node:fs/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; -const README_PATHS = [ - "README.md", - "README_zh.md", - "README_ja.md", - "README_ko.md", - "README_ar.md", -]; const readmeLinkTargets = (content) => [...content.matchAll(/\]\(([^)\s]+)\)/gu)] .map((match) => match[1] ?? "") .filter( (target) => - !target.startsWith("#") && !/^README(?:_[a-z]+)?\.md$/u.test(target), + !target.startsWith("#") && + !/^README(?:_[A-Za-z0-9-]+)?\.md$/u.test(target), ) .sort(); @@ -64,8 +58,11 @@ export const documentationFactIssues = async (root, catalog) => { const issues = await skillReferenceIssues( join(root, "skills/reverse-engineer-anything"), ); + const readmePaths = (await readdir(root)) + .filter((path) => /^README(?:_[A-Za-z0-9-]+)?\.md$/u.test(path)) + .sort(); const readmes = new Map(); - for (const path of README_PATHS) { + for (const path of readmePaths) { const content = await readFile(join(root, path), "utf8"); readmes.set(path, content); requireText(issues, path, content, "MCP-tool_catalog"); @@ -86,7 +83,7 @@ export const documentationFactIssues = async (root, catalog) => { } } const canonicalLinks = readmeLinkTargets(readmes.get("README.md") ?? ""); - for (const path of README_PATHS.slice(1)) + for (const path of readmePaths.filter((path) => path !== "README.md")) if ( JSON.stringify(readmeLinkTargets(readmes.get(path) ?? "")) !== JSON.stringify(canonicalLinks) diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index e1f7029c5d2..8a5c0d2185c 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -5,13 +5,28 @@ import { describe, expect, it } from "vitest"; import { SUPPORTED_CLIENT_DEFINITIONS } from "../../src/application/SupportedClients.js"; import { PRODUCT_IDENTITY } from "../../src/identity.js"; -const translatedReadmes = [ - "README_zh.md", - "README_ja.md", - "README_ko.md", - "README_ar.md", +const readmeLanguages = [ + { path: "README.md", label: "English" }, + { path: "README_pt-BR.md", label: "Português (Brasil)" }, + { path: "README_zh.md", label: "简体中文" }, + { path: "README_zh-TW.md", label: "繁體中文" }, + { path: "README_ja.md", label: "日本語" }, + { path: "README_ko.md", label: "한국어" }, + { path: "README_tr.md", label: "Türkçe" }, + { path: "README_ru.md", label: "Русский" }, + { path: "README_vi.md", label: "Tiếng Việt" }, + { path: "README_th.md", label: "ไทย" }, + { path: "README_de.md", label: "Deutsch" }, + { path: "README_es.md", label: "Español" }, + { path: "README_uk.md", label: "Українська" }, + { path: "README_pl.md", label: "Polski" }, + { path: "README_ar.md", label: "العربية" }, ] as const; +const translatedReadmes = readmeLanguages + .map((language) => language.path) + .filter((path) => path !== "README.md"); + const normalizedProse = (content: string): string => content.replace(/\s+/gu, " ").trim(); @@ -30,11 +45,25 @@ const linkedTargets = (content: string): string[] => .map((match) => match[1] ?? "") .filter( (target) => - !target.startsWith("#") && !/^README(?:_[a-z]+)?\.md$/u.test(target), + !target.startsWith("#") && + !/^README(?:_[A-Za-z0-9-]+)?\.md$/u.test(target), ) .sort(); describe("onboarding documentation product facts", () => { + it.each(readmeLanguages)( + "links every available language from $path", + async ({ path, label }) => { + const content = await readFile(resolve(path), "utf8"); + const selector = content.split("\n\n")[1] ?? ""; + expect(selector).toContain(`**${label}**`); + for (const language of readmeLanguages) { + if (language.path !== path) + expect(selector).toContain(`[${language.label}](${language.path})`); + } + }, + ); + it("keeps detailed requirements and versioned MCP configuration aligned in the installation guide", async () => { const content = await readFile(resolve("docs/installation.md"), "utf8"); expect(jsonExamples(content)).toContainEqual( diff --git a/tests/conformance/releaseConfiguration.test.ts b/tests/conformance/releaseConfiguration.test.ts index 1c5b09abe79..46c88b6cb54 100644 --- a/tests/conformance/releaseConfiguration.test.ts +++ b/tests/conformance/releaseConfiguration.test.ts @@ -7,13 +7,7 @@ import { SUPPORTED_NODE_VERSION_RANGE } from "../../src/domain/runtimeVersion.js const readJson = async (path: string): Promise => JSON.parse(await readFile(path, "utf8")) as unknown; -const versionedDocumentation = [ - "README_zh.md", - "README_ja.md", - "README_ko.md", - "README_ar.md", - "docs/installation.md", -] as const; +const versionedDocumentation = ["docs/installation.md"] as const; describe("release configuration", () => { it("keeps doctor and setup runtime support aligned with package engines", async () => { From 2a4979b7cdc1a517865573d6c36654c022347d0a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:35:05 +0800 Subject: [PATCH 040/791] docs: move Portuguese toward the end of language navigation --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 2 +- README_es.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 2 +- README_pt-BR.md | 2 +- README_ru.md | 2 +- README_th.md | 2 +- README_tr.md | 2 +- README_uk.md | 2 +- README_vi.md | 2 +- README_zh-TW.md | 2 +- README_zh.md | 2 +- tests/conformance/readmeConsistency.test.ts | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b662aa6d3cc..115d6cc6d30 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-**English** · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +**English** · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Reverse Engineer Anything diff --git a/README_ar.md b/README_ar.md index 046602cbfd3..24c3e148cd2 100644 --- a/README_ar.md +++ b/README_ar.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · **العربية** +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · **العربية** # REA: هندسة عكسية لأي شيء diff --git a/README_de.md b/README_de.md index b544f7e5c99..dc2c2b87823 100644 --- a/README_de.md +++ b/README_de.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · **Deutsch** · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · **Deutsch** · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Alles reverse-engineeren diff --git a/README_es.md b/README_es.md index 3b2c407419f..e9df0c5c7a2 100644 --- a/README_es.md +++ b/README_es.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · **Español** · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · **Español** · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Ingeniería inversa de cualquier cosa diff --git a/README_ja.md b/README_ja.md index 4c9e798252c..f51f9d7c12d 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · **日本語** · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · **日本語** · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA:あらゆるものをリバースエンジニアリング diff --git a/README_ko.md b/README_ko.md index 2368c34485a..4180c38e292 100644 --- a/README_ko.md +++ b/README_ko.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · **한국어** · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · **한국어** · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: 무엇이든 리버스 엔지니어링 diff --git a/README_pl.md b/README_pl.md index f08507b803c..baa8e3f1cc3 100644 --- a/README_pl.md +++ b/README_pl.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · **Polski** · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · **Polski** · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Reverse Engineer Anything diff --git a/README_pt-BR.md b/README_pt-BR.md index f4c61592c28..bad0348639e 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -1,6 +1,6 @@
-[English](README.md) · **Português (Brasil)** · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · **Português (Brasil)** · [العربية](README_ar.md) # REA: Faça engenharia reversa de qualquer coisa diff --git a/README_ru.md b/README_ru.md index e8a3681ce89..a20e077e1bf 100644 --- a/README_ru.md +++ b/README_ru.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · **Русский** · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · **Русский** · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Реверс-инжиниринг чего угодно diff --git a/README_th.md b/README_th.md index e26c9843732..da34fb80c72 100644 --- a/README_th.md +++ b/README_th.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · **ไทย** · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · **ไทย** · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Reverse Engineer Anything diff --git a/README_tr.md b/README_tr.md index 69a9d6304ed..cf449c2e72e 100644 --- a/README_tr.md +++ b/README_tr.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **Türkçe** · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **Türkçe** · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Her şeyi tersine mühendislikle inceleyin diff --git a/README_uk.md b/README_uk.md index 59063869da9..45e9c8265fd 100644 --- a/README_uk.md +++ b/README_uk.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Українська** · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Українська** · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Реверс-інжиніринг будь-чого diff --git a/README_vi.md b/README_vi.md index a5d20ab3726..252faf85e1f 100644 --- a/README_vi.md +++ b/README_vi.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · **Tiếng Việt** · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · **Tiếng Việt** · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA: Kỹ thuật dịch ngược cho mọi thứ diff --git a/README_zh-TW.md b/README_zh-TW.md index 26c21f27de0..c8a8eeadc84 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · [简体中文](README_zh.md) · **繁體中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · **繁體中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA:逆向分析一切 diff --git a/README_zh.md b/README_zh.md index 86ab20c80a6..218ca5d8097 100644 --- a/README_zh.md +++ b/README_zh.md @@ -1,6 +1,6 @@
-[English](README.md) · [Português (Brasil)](README_pt-BR.md) · **简体中文** · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [العربية](README_ar.md) +[English](README.md) · **简体中文** · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) # REA:逆向分析一切 diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index 8a5c0d2185c..db61e5dd765 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -7,7 +7,6 @@ import { PRODUCT_IDENTITY } from "../../src/identity.js"; const readmeLanguages = [ { path: "README.md", label: "English" }, - { path: "README_pt-BR.md", label: "Português (Brasil)" }, { path: "README_zh.md", label: "简体中文" }, { path: "README_zh-TW.md", label: "繁體中文" }, { path: "README_ja.md", label: "日本語" }, @@ -20,6 +19,7 @@ const readmeLanguages = [ { path: "README_es.md", label: "Español" }, { path: "README_uk.md", label: "Українська" }, { path: "README_pl.md", label: "Polski" }, + { path: "README_pt-BR.md", label: "Português (Brasil)" }, { path: "README_ar.md", label: "العربية" }, ] as const; From c3136064c7f8f72b2f6c6799ec41ea6424111bb7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:45:40 +0800 Subject: [PATCH 041/791] fix(cli): accept exact native names through explicit selectors --- docs/native-investigation.md | 12 ++ docs/testing.md | 3 + scripts/lib/real-hopper-boundaries.mjs | 10 ++ scripts/lib/real-hopper-cli-selectors.mjs | 88 ++++++++++++ src/cli/analysisSelector.ts | 29 ++++ src/cli/coreBinaryCommands.ts | 125 ++++++++++++++++-- .../boundary/cli/analysisSelectorCli.test.ts | 62 +++++++++ 7 files changed, 317 insertions(+), 12 deletions(-) create mode 100644 scripts/lib/real-hopper-cli-selectors.mjs create mode 100644 src/cli/analysisSelector.ts create mode 100644 tests/boundary/cli/analysisSelectorCli.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index e4fdf9a5a22..e23b94a4f08 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -83,6 +83,18 @@ complete observation. partial coverage. Other providers retain the existing symbol-based inventory with its narrower coverage. +## Exact CLI selectors + +`function`, `instructions` and `decompile` accept `--procedure=`; +`xrefs` accepts `--address=`. `search` accepts `--pattern=` +and `trace` accepts `--query=`. These alternatives preserve names and text +that begin with a dash, including Objective-C names such as +`rea function ./app '--procedure=-[REAWidget delegate]' --provider hopper --format json`. +Use the equals form when the value resembles a CLI flag, for example +`rea trace ./app --query=--help --provider hopper --format json`. +The existing positional forms remain available. Missing selectors and conflicting +positional/named values fail before analysis starts; identical selections agree. + ## Native instruction, call and type primitives Ghidra supplies three exact-object operations: diff --git a/docs/testing.md b/docs/testing.md index 224af2b91fe..2635552f183 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -214,6 +214,9 @@ Objective-C fixture is present, the lane also verifies native UTF-16 string obje and their inferred encodings against their actual bytes. Native terminal calls are checked across reference, instruction, assembly, block-range and procedure-length projections; block endpoints are normalized using actual native membership. +Exact Objective-C names also exercise named CLI selectors for function, instruction, +decompilation, reference and search operations. A literal `--help` trace query proves +that selector data is preserved independently of global CLI flags. Unmapped annotation destinations and oversized later batch names fail before any earlier edit is applied. The Linux demo lane remains a separate `verify:hopper:linux` command. diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index 8414cbc45a5..b094cf41233 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -7,6 +7,7 @@ import { promisify } from "node:util"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; +import { verifyHopperCliSelectors } from "./real-hopper-cli-selectors.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; import { @@ -395,6 +396,7 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { expectedFunction, ); let cliTerminalFunctionParity = null; + let cliNamedSelectorParity = null; if (targets.unicode !== undefined) { const objc = join(directory, `objc-${suffix}`); await copyFile(targets.unicode, objc); @@ -420,6 +422,13 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { ]); assert.deepEqual(JSON.parse(analyzed.stdout).normalized_result, expected); cliTerminalFunctionParity = true; + await call("open_binary", { path: objc }); + cliNamedSelectorParity = await verifyHopperCliSelectors( + call, + runCli, + dispatcher, + objc, + ); } let failure; try { @@ -452,6 +461,7 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { cliLiteralTraceParity: true, cliFunctionDossierParity: true, cliTerminalFunctionParity, + cliNamedSelectorParity, callerCancellationRecovered: true, closedDocumentAbsent: true, }; diff --git a/scripts/lib/real-hopper-cli-selectors.mjs b/scripts/lib/real-hopper-cli-selectors.mjs new file mode 100644 index 00000000000..7bfa4d1270f --- /dev/null +++ b/scripts/lib/real-hopper-cli-selectors.mjs @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; + +/** Compare exact native symbol and literal selectors through MCP and the real CLI. */ +export async function verifyHopperCliSelectors(call, runCli, dispatcher, path) { + const procedures = await call("list_procedures"); + const delegate = procedures.find( + (item) => item.value === "-[REAWidget delegate]", + ); + assert.ok(delegate, "Objective-C fixture omitted its tail-call method"); + const cases = [ + { + command: "function", + option: "procedure", + operation: "analyze_function", + parameters: { procedure: delegate.value }, + }, + { + command: "instructions", + option: "procedure", + operation: "read_function_instructions", + parameters: { procedure: delegate.value }, + }, + { + command: "decompile", + option: "procedure", + operation: "procedure_pseudo_code", + parameters: { procedure: delegate.value }, + }, + { + command: "xrefs", + option: "address", + operation: "xrefs", + parameters: { address: delegate.value }, + }, + { + command: "search", + option: "pattern", + operation: "search_procedures", + parameters: { + pattern: delegate.value, + mode: "literal", + case_sensitive: true, + }, + flags: ["--kind", "procedures", "--case-sensitive"], + }, + { + command: "trace", + option: "query", + operation: "trace_feature", + parameters: { query: "--help" }, + }, + ]; + const expected = []; + for (const item of cases) + expected.push(await call(item.operation, item.parameters)); + await call("close_binary"); + for (let i = 0; i < cases.length; i++) { + const item = cases[i]; + const selector = item.parameters[item.option]; + const { stdout } = await runCli([ + dispatcher, + item.command, + path, + `--${item.option}=${selector}`, + ...(item.flags ?? []), + "--provider", + "hopper", + "--format", + "json", + ]); + const evidence = JSON.parse(stdout); + assert.deepEqual( + evidence.normalized_result, + expected[i], + `${item.command} named selector differs from MCP`, + ); + assert.equal( + evidence.parameters[item.option], + selector, + `${item.command} altered its selector`, + ); + } + return { + commands: cases.map(({ command }) => command), + exactObjectiveCNames: true, + literalGlobalFlagPreserved: true, + }; +} diff --git a/src/cli/analysisSelector.ts b/src/cli/analysisSelector.ts new file mode 100644 index 00000000000..f6e550cd0fb --- /dev/null +++ b/src/cli/analysisSelector.ts @@ -0,0 +1,29 @@ +import { AnalysisInputError } from "../domain/analysisErrorCore.js"; + +/** Preserve a CLI selector supplied positionally or through its named option. */ +export const resolveCliAnalysisSelector = ( + positional: string | undefined, + named: string | undefined, + option: string, + operation: string, +): string => { + if (positional !== undefined && named !== undefined && positional !== named) + throw new AnalysisInputError(operation, undefined, [ + { + path: [option], + reason: "invalid_value", + message: `The positional selector and --${option} select different objects; supply one selector.`, + }, + ]); + const selected = named ?? positional; + if (selected === undefined) + throw new AnalysisInputError(operation, undefined, [ + { + path: [option], + reason: "missing_argument", + message: `Supply a positional selector or --${option}=. Use the named option for values beginning with a dash.`, + expected: "string", + }, + ]); + return selected; +}; diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.ts index 441d0e1474b..718f55d6d6f 100644 --- a/src/cli/coreBinaryCommands.ts +++ b/src/cli/coreBinaryCommands.ts @@ -2,6 +2,7 @@ import { z } from "incur"; import { stat } from "node:fs/promises"; import { resolve } from "node:path"; +import { resolveCliAnalysisSelector } from "./analysisSelector.js"; import { runDirectAnalysis } from "../composition/directAnalysis.js"; import { CLI_COMMANDS } from "../cliCommandNames.js"; import { logCliCommand } from "../cliLogging.js"; @@ -25,6 +26,7 @@ export const registerCoreBinaryCommands = ( resultOutput?: CliResultOutput, ): void => { registerOverviewCommands(cli, logger, resultOutput); + registerDecompileCommand(cli, logger); registerFunctionCommand(cli, logger); registerInstructionsCommand(cli, logger); registerSearchCommand(cli, logger); @@ -85,13 +87,27 @@ const registerOverviewCommands = ( ), ), }); +}; + +const registerDecompileCommand = (cli: CliInstance, logger: Logger): void => { cli.command(CLI_COMMANDS.decompile, { description: "Read one part of an app as code", args: z.object({ path: z.string().describe("App or program path"), - address: z.string().describe("Procedure address"), + address: z + .string() + .min(1) + .optional() + .describe("Procedure name or address"), }), options: z.object({ + procedure: z + .string() + .min(1) + .optional() + .describe( + "Procedure name or address; use --procedure= for leading dashes", + ), snapshot: z .string() .min(1) @@ -105,7 +121,14 @@ const registerOverviewCommands = ( runDirectAnalysis( args.path, "procedure_pseudo_code", - { procedure: args.address }, + { + procedure: resolveCliAnalysisSelector( + args.address, + options.procedure, + "procedure", + "procedure_pseudo_code", + ), + }, directAnalysisOptions( logger, options.snapshot, @@ -169,9 +192,16 @@ const registerXrefsCommand = (cli: CliInstance, logger: Logger): void => { description: "List bounded references to an analyzed address", args: z.object({ path: z.string().describe("App or program path"), - address: z.string().describe("Hexadecimal address"), + address: z.string().min(1).optional().describe("Address or symbol name"), }), options: z.object({ + address: z + .string() + .min(1) + .optional() + .describe( + "Address or symbol name; use --address= for leading dashes", + ), snapshot: z .string() .min(1) @@ -185,7 +215,14 @@ const registerXrefsCommand = (cli: CliInstance, logger: Logger): void => { runDirectAnalysis( args.path, "xrefs", - { address: args.address }, + { + address: resolveCliAnalysisSelector( + args.address, + options.address, + "address", + "xrefs", + ), + }, directAnalysisOptions( logger, options.snapshot, @@ -202,9 +239,16 @@ const registerTraceCommand = (cli: CliInstance, logger: Logger): void => { description: "Trace a literal feature through analyzed references", args: z.object({ path: z.string().describe("App or program path"), - query: z.string().min(1).describe("Literal feature query"), + query: z.string().min(1).optional().describe("Literal feature query"), }), options: z.object({ + query: z + .string() + .min(1) + .optional() + .describe( + "Complete literal feature query; use --query= for leading dashes", + ), caseSensitive: z .boolean() .default(false) @@ -226,7 +270,12 @@ const registerTraceCommand = (cli: CliInstance, logger: Logger): void => { args.path, "trace_feature", { - query: args.query, + query: resolveCliAnalysisSelector( + args.query, + options.query, + "query", + "trace_feature", + ), case_sensitive: options.caseSensitive, }, directAnalysisOptions( @@ -245,9 +294,20 @@ const registerFunctionCommand = (cli: CliInstance, logger: Logger): void => { description: "Analyze one complete function with evidence", args: z.object({ path: z.string().describe("App or program path"), - address: z.string().describe("Procedure name or address"), + address: z + .string() + .min(1) + .optional() + .describe("Procedure name or address"), }), options: z.object({ + procedure: z + .string() + .min(1) + .optional() + .describe( + "Procedure name or address; use --procedure= for leading dashes", + ), snapshot: z .string() .min(1) @@ -261,7 +321,14 @@ const registerFunctionCommand = (cli: CliInstance, logger: Logger): void => { runDirectAnalysis( args.path, "analyze_function", - { procedure: args.address }, + { + procedure: resolveCliAnalysisSelector( + args.address, + options.procedure, + "procedure", + "analyze_function", + ), + }, directAnalysisOptions( logger, options.snapshot, @@ -281,9 +348,20 @@ const registerInstructionsCommand = ( description: "Read every raw instruction without decompiling", args: z.object({ path: z.string().describe("App or program path"), - address: z.string().describe("Procedure name or address"), + address: z + .string() + .min(1) + .optional() + .describe("Procedure name or address"), }), options: z.object({ + procedure: z + .string() + .min(1) + .optional() + .describe( + "Procedure name or address; use --procedure= for leading dashes", + ), snapshot: z .string() .min(1) @@ -298,7 +376,12 @@ const registerInstructionsCommand = ( args.path, "read_function_instructions", { - procedure: args.address, + procedure: resolveCliAnalysisSelector( + args.address, + options.procedure, + "procedure", + "read_function_instructions", + ), }, directAnalysisOptions( logger, @@ -316,9 +399,20 @@ const registerSearchCommand = (cli: CliInstance, logger: Logger): void => { description: "Search every analyzed string or procedure name", args: z.object({ path: z.string().describe("App or program path"), - pattern: z.string().min(1).describe("Literal text or regex pattern"), + pattern: z + .string() + .min(1) + .optional() + .describe("Literal text or regex pattern"), }), options: z.object({ + pattern: z + .string() + .min(1) + .optional() + .describe( + "Complete literal text or regex pattern; use --pattern= for leading dashes", + ), kind: z .enum(["strings", "procedures"]) .default("strings") @@ -346,7 +440,14 @@ const registerSearchCommand = (cli: CliInstance, logger: Logger): void => { args.path, options.kind === "strings" ? "search_strings" : "search_procedures", { - pattern: args.pattern, + pattern: resolveCliAnalysisSelector( + args.pattern, + options.pattern, + "pattern", + options.kind === "strings" + ? "search_strings" + : "search_procedures", + ), mode: options.mode, case_sensitive: options.caseSensitive, }, diff --git a/tests/boundary/cli/analysisSelectorCli.test.ts b/tests/boundary/cli/analysisSelectorCli.test.ts new file mode 100644 index 00000000000..404cad6a2cc --- /dev/null +++ b/tests/boundary/cli/analysisSelectorCli.test.ts @@ -0,0 +1,62 @@ +import { describe, expect } from "vitest"; + +import { cliTest } from "../../support/cli/cliFixture.js"; + +const selectors = [ + ["function", "procedure", "analyze_function"], + ["instructions", "procedure", "read_function_instructions"], + ["decompile", "procedure", "procedure_pseudo_code"], + ["xrefs", "address", "xrefs"], + ["search", "pattern", "search_strings"], + ["trace", "query", "trace_feature"], +] as const; + +describe("CLI analysis selector boundaries", () => { + cliTest( + "rejects missing and conflicting selectors before opening a target", + async ({ cli }) => { + for (const [command, option, operation] of selectors) { + for (const [selection, reason] of [ + [[], "missing_argument"], + [["first", `--${option}=-second`], "invalid_value"], + ] as const) { + const result = await cli.run({ + arguments: [ + command, + "does-not-exist", + ...selection, + "--format", + "json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).not.toBe(0); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { operation, issues: [{ path: [option], reason }] }, + }); + } + } + }, + ); + + cliTest( + "keeps named dash values out of global-option parsing", + async ({ cli }) => { + for (const [command, option] of selectors) { + const result = await cli.run({ + arguments: [ + command, + "does-not-exist", + `--${option}=--help`, + "--format", + "json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).not.toBe(0); + expect(result.json).toMatchObject({ code: "target_unavailable" }); + } + }, + ); +}); From 826583dd43fc918c70a87dfe148c2fdbc8612f3b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:53:52 +0800 Subject: [PATCH 042/791] fix(verify): remove stale mocked bridge package probe --- scripts/verify-package-pack.mjs | 2 +- scripts/verify-packed-bridge.mjs | 75 +++++--------------------------- 2 files changed, 13 insertions(+), 64 deletions(-) diff --git a/scripts/verify-package-pack.mjs b/scripts/verify-package-pack.mjs index 2d5ebed08e6..f3e2d81bdb8 100644 --- a/scripts/verify-package-pack.mjs +++ b/scripts/verify-package-pack.mjs @@ -71,6 +71,6 @@ export async function verifyPackagePack({ root, workspace }) { ) { throw new Error("package contained generated Python bytecode"); } - await verifyPackedBridge({ root, workspace, tarball, packedFiles }); + await verifyPackedBridge({ workspace, tarball, packedFiles }); return { tarball }; } diff --git a/scripts/verify-packed-bridge.mjs b/scripts/verify-packed-bridge.mjs index a2ce508428f..c6baac949d3 100644 --- a/scripts/verify-packed-bridge.mjs +++ b/scripts/verify-packed-bridge.mjs @@ -1,5 +1,4 @@ import { execFile } from "node:child_process"; -import { readFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -7,67 +6,17 @@ const exec = promisify(execFile); const packedHopperBridge = "package/bridge/hopper_bridge.py"; const packedGhidraBridge = "package/bridge/ghidra/ReaGhidraBridge.java"; -/** Verify regex search through the packaged production bridge. */ -export async function verifyPackedBridge({ - root, - workspace, - tarball, - packedFiles, -}) { - if (!packedFiles.includes(packedHopperBridge)) - throw new Error("package omitted the Hopper bridge"); - if (!packedFiles.includes(packedGhidraBridge)) - throw new Error("package omitted the Ghidra bridge"); +/** Check shipped bridge files; real-provider lanes verify runtime behavior. */ +export async function verifyPackedBridge({ workspace, tarball, packedFiles }) { + for (const path of [packedHopperBridge, packedGhidraBridge]) { + if (!packedFiles.includes(path)) throw new Error(`package omitted ${path}`); + } await exec("tar", ["-xf", tarball, "-C", workspace]); - const ghidraSource = await readFile( - join(workspace, packedGhidraBridge), - "utf8", - ); - for (const commitment of [ - "extends HeadlessScript", - 'request.method.equals("ping")', - 'request.method.equals("shutdown")', - 'boolean readOnly = !descriptor.transport.equals("unix-socket")', - 'result.addProperty("read_only", readOnly)', - 'case "annotate_native_function"', - 'currentProgram.startTransaction("REA function annotations")', - "currentProgram.endTransaction(transaction, commit)", - "analysisTimeoutOccurred()", - ]) - if (!ghidraSource.includes(commitment)) - throw new Error(`packaged Ghidra bridge omitted ${commitment}`); - const probe = JSON.parse( - ( - await exec("python3", [ - join(root, "tests/fixtures/bridgeSearchProbe.py"), - join(workspace, packedHopperBridge), - JSON.stringify({ - action: "search", - items: [ - ["0x1000", "REA_GHIDRA_INVENTORY_ENTRY"], - ["0x2000", "unrelated"], - ["0x3000", "REA_GHIDRA_LEAF_VALUE"], - ], - params: { - pattern: "^REA_GHIDRA_(?:INVENTORY_ENTRY|LEAF_VALUE)$", - mode: "regex", - case_sensitive: true, - }, - }), - ]) - ).stdout, - ); - const matches = probe.result; - if ( - probe.ok !== true || - !Array.isArray(matches) || - matches.length !== 2 || - matches[0]?.address !== "0x1000" || - matches[0]?.value !== "REA_GHIDRA_INVENTORY_ENTRY" || - matches[1]?.address !== "0x3000" || - matches[1]?.value !== "REA_GHIDRA_LEAF_VALUE" - ) - throw new Error( - `packaged Hopper bridge regex search drifted: ${JSON.stringify(probe)}`, - ); + // Parse the actual packed Python source without loading Hopper globals or + // generating bytecode. This check has no dependency on test-only fixtures. + await exec("python3", [ + "-c", + "import ast, pathlib, sys; p = pathlib.Path(sys.argv[1]); ast.parse(p.read_text(encoding='utf-8'), filename=str(p))", + join(workspace, packedHopperBridge), + ]); } From 0639a541a3b712940119f59e8189bc962b267726 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:54:03 +0800 Subject: [PATCH 043/791] fix(mcp): preserve retained evidence on oversized delivery --- docs/testing.md | 11 +- scripts/lib/real-ghidra-boundaries.mjs | 19 ++ scripts/lib/real-ghidra-large-results.mjs | 178 ++++++++++++++++++ src/domain/analysisErrorPresentation.ts | 2 +- src/server/registerAndroidTools.ts | 6 +- .../registerApplicationTools/helpers.ts | 12 +- src/server/registerArtifactComparisonTool.ts | 7 +- src/server/registerBinaryDiagnosticsTools.ts | 6 +- src/server/registerBrowserScenarioTool.ts | 6 +- src/server/registerBrowserTools.ts | 12 +- src/server/registerBundleComparisonTool.ts | 5 +- src/server/registerElectronTools.ts | 12 +- src/server/registerEnhancedTools.ts | 16 +- src/server/registerEvidenceTools.ts | 6 +- src/server/registerEvmTools.ts | 6 +- src/server/registerFirmwareTools.ts | 6 +- src/server/registerFunctionComparisonTool.ts | 4 +- src/server/registerInvestigationTools.ts | 16 +- src/server/registerJavaScriptRecoveryTool.ts | 6 +- ...gisterJavaScriptRuntimeObservationTools.ts | 6 +- .../compareManagedMembers.ts | 9 +- .../importManagedReconstruction.ts | 10 +- .../projectManagedApplicationGraph.ts | 6 +- .../verifyManagedNativeBoundaries.ts | 10 +- src/server/registerOfficialTools.ts | 6 +- src/server/registerProcessComparisonTool.ts | 7 +- src/server/registerRecordedCrashTools.ts | 6 +- src/server/registerSessionTools.ts | 4 +- src/server/registerWebModuleTool.ts | 6 +- src/server/registerWebNetworkCaptureTool.ts | 6 +- src/server/registerWebRuntimeTools.ts | 10 +- src/server/registerWebScriptTool.ts | 6 +- src/server/registerWebSourceLocationTool.ts | 6 +- src/server/toolResult.ts | 21 ++- tests/boundary/mcp/largeToolResult.test.ts | 59 ++---- tests/conformance/ghidra/large-results.c | 9 + 36 files changed, 339 insertions(+), 184 deletions(-) create mode 100644 scripts/lib/real-ghidra-large-results.mjs create mode 100644 tests/conformance/ghidra/large-results.c diff --git a/docs/testing.md b/docs/testing.md index 2635552f183..02b9b7b19f3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -184,7 +184,9 @@ MCP SDK transport tests with recording providers remain integration tests. They are useful for schema drift and failure projection but do not prove that Hopper, Ghidra or another substituted engine works. `verify:package` proves packaging/install behavior and fake-provider integration; use the corresponding -real-provider lanes for engine claims. Real Apple dispatch and Interface +real-provider lanes for engine claims. Packed-bridge checks verify shipped files +and Python syntax without deleted mock fixtures or source-spelling assertions. +Real Apple dispatch and Interface Builder verifiers currently prove format integration through production readers. `verify:hopper` exercises an installed Hopper through the production stdio MCP @@ -354,6 +356,13 @@ workflow verifies leaf and qualified renames, repeated reuse of fully qualified readback, lookup by the returned name, literal namespace-like leaf names, rejection of empty qualified leaf names without changing comments, CLI behavior, and independent CLI/MCP database ownership. +Large-result probes compile initialized host-native data sized from the pinned +MCP SDK receive budget. Real byte reads, annotation edits and function dossiers +exceed that budget while preserving the connection and active analysis run. +Each delivery error must identify its successfully retained Evidence record; +export must recover every source byte and complete annotation, with CLI parity +and an unchanged executable. The focused formatter check retains only the +separate case where recording was not acknowledged. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 988635fe0ec..833a3422204 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -7,10 +7,12 @@ import { setTimeout as delay } from "node:timers/promises"; import { promisify } from "node:util"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; +import { analysisErrorProjectionSchema } from "../../dist/contracts/errorSchemas.js"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot-e2e.mjs"; import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; import { verifyGhidraTargetAdmission } from "./real-ghidra-target-admission.mjs"; +import { verifyGhidraLargeResults } from "./real-ghidra-large-results.mjs"; import { verifyGhidraNamespaceAnnotations } from "./real-ghidra-namespace-annotations.mjs"; /** Probe real Ghidra location, annotation and error contracts through public adapters. */ @@ -53,6 +55,15 @@ export async function verifyGhidraBoundaries( true, `${name} accepted ${JSON.stringify(args)}`, ); + // SDK input-schema rejections are text-only; application errors use the + // canonical structured error projection, separately from success schemas. + if (reply.structuredContent !== undefined) { + analysisErrorProjectionSchema.parse(reply.structuredContent.error); + assert.deepEqual( + reply.structuredContent, + JSON.parse(mcpTextValue(reply)), + ); + } if (diagnostic !== undefined) { const error = reply.structuredContent?.error; assert.equal(error?.code, "invalid_request"); @@ -664,6 +675,13 @@ export async function verifyGhidraBoundaries( entrypoint, env, }); + await verifyGhidraLargeResults({ + call, + reject: invalid, + target, + entrypoint, + env, + }); await assert.rejects(access(socketRoot), { code: "ENOENT" }); await assert.rejects(access(runtimeRoot), { code: "ENOENT" }); assert.equal(await call("procedure_address", { procedure: name }), address); @@ -705,6 +723,7 @@ export async function verifyGhidraBoundaries( imported_source_identity_retained: true, equivalent_instruction_address_spellings: true, qualified_annotation_name_roundtrip: true, + oversized_result_retention_and_complete_export: true, source_immutable: true, reopen_discards_edits: true, long_tmpdir_private_socket_cleanup: true, diff --git a/scripts/lib/real-ghidra-large-results.mjs b/scripts/lib/real-ghidra-large-results.mjs new file mode 100644 index 00000000000..6185904a906 --- /dev/null +++ b/scripts/lib/real-ghidra-large-results.mjs @@ -0,0 +1,178 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; + +/** Exercise complete oversized results and recovery through real CLI/MCP adapters. */ +export async function verifyGhidraLargeResults({ + call, + reject, + target, + entrypoint, + env, +}) { + const exec = promisify(execFile); + const compiler = process.env.REA_CC ?? "cc"; + try { + await exec(compiler, ["--version"], { env, timeout: 30000 }); + } catch (cause) { + throw new Error( + `Ghidra large-result verification requires a working host C compiler '${compiler}'.`, + { cause }, + ); + } + // Hex encoding, Evidence projection and the two MCP representations multiply + // the source size. Derive the fixture from the pinned SDK's receive budget. + const length = Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 4); + const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-large-results-")); + const path = join(workspace, "large-results"); + const source = fileURLToPath( + new URL("../../tests/conformance/ghidra/large-results.c", import.meta.url), + ); + try { + await exec( + compiler, + ["-O0", `-DREA_PAYLOAD_BYTES=${length}`, source, "-o", path], + { env, timeout: 30000 }, + ); + const bytes = await readFile(path); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await call("close_binary"); + await call("open_binary", { path, provider_id: "ghidra" }); + const names = await call("list_names"); + const payload = names.find(({ value }) => + /(?:^|_)rea_large_payload$/u.test(value), + ); + const entry = names.find(({ value }) => + /(?:^|_)rea_large_entry$/u.test(value), + ); + assert.ok(payload); + assert.ok(entry); + const mapping = await call("address_to_file_offset", { + address: payload.address, + }); + const expected = bytes.subarray( + mapping.file_offset, + mapping.file_offset + length, + ); + assert.equal(expected.length, length); + assert.equal(expected[0], 0x5a); + assert.equal(expected.at(-1), 0xa5); + const before = await call("binary_session"); + const readError = await reject("read_bytes", { + address: payload.address, + length, + }); + const readId = retainedReference(readError, "read_bytes"); + const comment = + "REA_BIG_COMMENT:" + + "x".repeat(Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 10)); + const annotationError = await reject("annotate_native_function", { + procedure: entry.address, + name: "rea_frame_probe", + comment, + inline_comment: comment, + }); + const annotationId = retainedReference( + annotationError, + "annotate_native_function", + ); + assert.equal( + await call("procedure_address", { procedure: "rea_frame_probe" }), + entry.address, + "A delivery constraint must preserve the successfully committed edit", + ); + const dossierError = await reject("analyze_function", { + procedure: "rea_frame_probe", + }); + const dossierId = retainedReference(dossierError, "analyze_function"); + const after = await call("binary_session"); + assert.equal(after.analysis_run.run_id, before.analysis_run.run_id); + assert.equal(after.sha256, sha256); + assert.equal( + (await call("read_bytes", { address: payload.address, length: 1 })) + .bytes_hex, + "5a", + ); + const lastAddress = `0x${(BigInt(payload.address) + BigInt(length - 1)).toString(16)}`; + assert.equal( + (await call("read_bytes", { address: lastAddress, length: 1 })).bytes_hex, + "a5", + ); + const bundlePath = join(workspace, "bundle.json"); + await call("export_evidence_bundle", { path: bundlePath }); + const bundle = JSON.parse(await readFile(bundlePath, "utf8")); + const retained = (id) => { + const record = bundle.records.find( + ({ evidence_id }) => evidence_id === id, + ); + assert.ok(record, `Missing exported retained record ${id}`); + assert.equal(record.provider.id, "ghidra"); + assert.equal(record.subject.digest.sha256, sha256); + return record; + }; + const readRecord = retained(readId); + const read = readRecord.normalized_result; + assert.equal(read.requested_bytes, length); + assert.equal(read.returned_bytes, length); + assert.equal(read.complete, true); + assert.equal(read.bytes_hex, expected.toString("hex")); + assert.equal(readRecord.raw_result.bytes_hex, read.bytes_hex); + const annotationRecord = retained(annotationId); + const annotation = annotationRecord.normalized_result; + assert.equal(annotation.annotations.name, "rea_frame_probe"); + assert.equal(annotation.annotations.comment, comment); + assert.equal(annotation.annotations.inline_comment, comment); + assert.equal(annotationRecord.raw_result.annotations.comment, comment); + assert.equal( + annotationRecord.raw_result.annotations.inline_comment, + comment, + ); + const dossierRecord = retained(dossierId); + const dossier = dossierRecord.normalized_result; + assert.equal(dossier.procedure.name, "rea_frame_probe"); + assert.ok(dossier.comments.some(({ text }) => text === comment)); + assert.ok( + dossierRecord.raw_result.comments.some(({ text }) => text === comment), + ); + const { stdout } = await exec( + process.execPath, + [ + entrypoint, + "read-bytes", + path, + payload.address, + "--length", + String(length), + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 72 * 1024 * 1024 }, + ); + assert.deepEqual(JSON.parse(stdout).normalized_result, read); + assert.deepEqual(await readFile(path), bytes); + await call("close_binary"); + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +} + +function retainedReference(error, operation) { + assert.equal(error.code, "resource_constraint"); + assert.equal(error.details.operation, operation); + assert.equal(error.details.resource, "transport"); + const limits = error.details.reported_limits; + assert.equal(limits.boundary, "mcp-response"); + assert.equal(limits.constraint, "receive-buffer"); + assert.ok(limits.response_bytes_at_least > limits.result_budget_bytes); + assert.equal(limits.evidence_reference.kind, "retained-evidence"); + assert.match(limits.evidence_reference.evidence_id, /^ev_[a-f0-9]{64}$/u); + return limits.evidence_reference.evidence_id; +} diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 66cdb5caf07..215cf58bda9 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -44,7 +44,7 @@ export const analysisErrorRemediationAction = ( return ( error.remediationAction ?? (error.resource === "transport" - ? "Use the reported Evidence reference with a focused analysis tool when retained in this session, or export the session through export_evidence_bundle to a caller-selected path. Complete CLI JSON output is also available. The connection remains usable." + ? "Export retained session evidence through export_evidence_bundle to a caller-selected path, or use complete CLI JSON output. Tools that accept retained-evidence references can inspect compatible records. The connection remains usable." : error.resource === "cpu" ? "Review the reported worker CPU limits and observed signal. Retry with sufficient CPU time or a smaller artifact; REA retains tighter inherited limits." : error.resource === "file-size" diff --git a/src/server/registerAndroidTools.ts b/src/server/registerAndroidTools.ts index b58cf251770..3040fb4cdee 100644 --- a/src/server/registerAndroidTools.ts +++ b/src/server/registerAndroidTools.ts @@ -7,7 +7,7 @@ import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind Android handlers to their exact named contracts through shared workflows. */ export const registerAndroidTools = ( @@ -24,9 +24,7 @@ export const registerAndroidTools = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }; }; const packageContract = toolContract("inspect_android_package"); diff --git a/src/server/registerApplicationTools/helpers.ts b/src/server/registerApplicationTools/helpers.ts index 0a68d97f0d8..032c6103865 100644 --- a/src/server/registerApplicationTools/helpers.ts +++ b/src/server/registerApplicationTools/helpers.ts @@ -1,6 +1,6 @@ import { APPLICATION_TOOL_CONTRACTS } from "../../contracts/applicationToolContracts.js"; import type { Evidence } from "../../domain/evidence.js"; -import { toCallToolResult } from "../toolResult.js"; +import { toEvidenceToolResult } from "../toolResult.js"; import type { ApplicationToolRegistration } from "./types.js"; export const recordResult = ( @@ -19,15 +19,7 @@ export const recordResult = ( evidence, unknownRegistration(evidence, unknownKind), ); - if (recorded !== undefined && !recorded.ok) - return toCallToolResult(recorded, contract); - return toCallToolResult( - { ok: true, value: evidence }, - contract, - recorded === undefined - ? undefined - : { retainedEvidenceId: evidence.evidence_id }, - ); + return toEvidenceToolResult(evidence, contract, recorded); }; const unknownRegistration = ( diff --git a/src/server/registerArtifactComparisonTool.ts b/src/server/registerArtifactComparisonTool.ts index f42cca0e033..a832a1a561d 100644 --- a/src/server/registerArtifactComparisonTool.ts +++ b/src/server/registerArtifactComparisonTool.ts @@ -10,7 +10,7 @@ import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { ARTIFACT_COMPARISON_PROVIDER } from "./sessionToolPolicies.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Register Evidence-backed deterministic artifact comparison. */ export const registerArtifactComparisonTool = ( @@ -50,13 +50,14 @@ export const registerArtifactComparisonTool = ( limitations: comparison.limitations, evidenceLinks: [...leftEvidenceIds, ...rightEvidenceIds], }); - return toCallToolResult( + return toEvidenceToolResult( + evidence, + contract, recordDerivedEvidence( session, evidence, artifactUnknownInput(left, right, comparison.status), ), - contract, ); }, ); diff --git a/src/server/registerBinaryDiagnosticsTools.ts b/src/server/registerBinaryDiagnosticsTools.ts index 949e24624a1..8c7de7cf8d7 100644 --- a/src/server/registerBinaryDiagnosticsTools.ts +++ b/src/server/registerBinaryDiagnosticsTools.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind offline layout inspection to its named contract and session Evidence owner. */ export const registerBinaryDiagnosticsTools = ( @@ -24,9 +24,7 @@ export const registerBinaryDiagnosticsTools = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerBrowserScenarioTool.ts b/src/server/registerBrowserScenarioTool.ts index b1a5bb3dee1..b72e3637e8e 100644 --- a/src/server/registerBrowserScenarioTool.ts +++ b/src/server/registerBrowserScenarioTool.ts @@ -13,7 +13,7 @@ import { browserScenarioSchema } from "../domain/browserScenario.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; interface BrowserScenarioToolRegistration { readonly logger: Logger; @@ -45,9 +45,7 @@ export const registerBrowserScenarioTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = options.recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult({ ok: true, value: result.value }, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerBrowserTools.ts b/src/server/registerBrowserTools.ts index 55a065a03cb..988863d57e2 100644 --- a/src/server/registerBrowserTools.ts +++ b/src/server/registerBrowserTools.ts @@ -28,7 +28,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; interface BrowserToolRegistration { readonly logger: Logger; @@ -206,13 +206,5 @@ const evidenceResult = ( evidence: Evidence, ) => { const recorded = options.recordEvidence?.(evidence); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult( - { ok: true, value: evidence }, - contract, - recorded === undefined - ? undefined - : { retainedEvidenceId: evidence.evidence_id }, - ); + return toEvidenceToolResult(evidence, contract, recorded); }; diff --git a/src/server/registerBundleComparisonTool.ts b/src/server/registerBundleComparisonTool.ts index 7ec2662f8d3..ed918a94994 100644 --- a/src/server/registerBundleComparisonTool.ts +++ b/src/server/registerBundleComparisonTool.ts @@ -6,11 +6,10 @@ import { toolContract } from "../contracts/toolContracts.js"; import { compareBundles } from "../domain/bundleComparison.js"; import { createEvidence } from "../domain/evidence.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; -import { ok } from "../domain/result.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { BUNDLE_COMPARISON_PROVIDER } from "./sessionToolPolicies.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Register canonical Evidence bundle comparison. */ export const registerBundleComparisonTool = ( @@ -47,7 +46,7 @@ export const registerBundleComparisonTool = ( limitations: comparison.limitations, }); const recorded = session.recordEvidence(evidence); - return toCallToolResult(recorded.ok ? ok(evidence) : recorded, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); }; diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index e338eff4abf..3dbf8e05998 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -25,7 +25,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; interface ElectronToolRegistration { readonly logger: Logger; @@ -166,13 +166,5 @@ const evidenceResult = ( evidence: Evidence, ) => { const recorded = options.recordEvidence?.(evidence); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult( - { ok: true, value: evidence }, - contract, - recorded === undefined - ? undefined - : { retainedEvidenceId: evidence.evidence_id }, - ); + return toEvidenceToolResult(evidence, contract, recorded); }; diff --git a/src/server/registerEnhancedTools.ts b/src/server/registerEnhancedTools.ts index b6d075bb657..e7a76037e28 100644 --- a/src/server/registerEnhancedTools.ts +++ b/src/server/registerEnhancedTools.ts @@ -27,7 +27,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; import { executeFunctionAnalysisEvidence } from "../application/FunctionAnalysisEvidence.js"; /** Optional session services used by enhanced tool registration. */ @@ -271,7 +271,7 @@ const executeEnhancedTool = async ( recordUnknown: registration.recordUnknown, }); if (!unknowns.ok) return toCallToolResult(unknowns, contract); - return toCallToolResult({ ok: true, value: evidence }, contract); + return toEvidenceToolResult(evidence, contract, recorded); } return toCallToolResult(result, contract); }; @@ -308,12 +308,12 @@ const executeFunctionTool = async ( message: result.ok ? "completed" : "failed", terminal: true, }); - if (result.ok) { - const recorded = registration.recordEvidence?.(result.value); - if (recorded !== undefined && !recorded.ok) - return toCallToolResult(recorded, contract); - } - return toCallToolResult(result, contract); + if (!result.ok) return toCallToolResult(result, contract); + return toEvidenceToolResult( + result.value, + contract, + registration.recordEvidence?.(result.value), + ); }; const jsonParameters = ( diff --git a/src/server/registerEvidenceTools.ts b/src/server/registerEvidenceTools.ts index be678179b28..5793006846c 100644 --- a/src/server/registerEvidenceTools.ts +++ b/src/server/registerEvidenceTools.ts @@ -13,7 +13,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; import { createArtifactExtractionDestination } from "../application/artifacts/ArtifactExtractionDestination.js"; interface EvidenceToolRegistration { @@ -94,9 +94,7 @@ export const registerEvidenceTools = ( return toCallToolResult(sourceRecorded, contract); } const recorded = options.recordEvidence?.(evidence); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult({ ok: true, value: evidence }, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); } diff --git a/src/server/registerEvmTools.ts b/src/server/registerEvmTools.ts index 09af2dc26c7..ee2c93b2e42 100644 --- a/src/server/registerEvmTools.ts +++ b/src/server/registerEvmTools.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind the named EVM contract to shared validation and session Evidence recording. */ export const registerEvmTools = ( @@ -24,9 +24,7 @@ export const registerEvmTools = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerFirmwareTools.ts b/src/server/registerFirmwareTools.ts index 66c763bffcd..4cb6ba6973a 100644 --- a/src/server/registerFirmwareTools.ts +++ b/src/server/registerFirmwareTools.ts @@ -7,7 +7,7 @@ import type { FirmwareOperation } from "../domain/firmware/firmwareAnalysis.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind firmware handlers to their exact named schemas. */ export const registerFirmwareTools = ( @@ -26,9 +26,7 @@ export const registerFirmwareTools = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }; const inspect = toolContract("inspect_firmware_regions"); server.registerTool( diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 84a45068495..8fff00abad6 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -17,7 +17,7 @@ import { recordSessionEvidenceSources } from "./sessionEvidence.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { FUNCTION_COMPARISON_PROVIDER } from "./sessionToolPolicies.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Register explicit Evidence-backed function comparison. */ export const registerFunctionComparisonTool = ( @@ -89,7 +89,7 @@ export const registerFunctionComparisonTool = ( rightIds, }), ); - return toCallToolResult(recorded, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); }; diff --git a/src/server/registerInvestigationTools.ts b/src/server/registerInvestigationTools.ts index 2e2dee834a1..e185bc02f5b 100644 --- a/src/server/registerInvestigationTools.ts +++ b/src/server/registerInvestigationTools.ts @@ -23,7 +23,7 @@ import { STATIC_RUNTIME_PROVIDER, } from "./sessionToolPolicies.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Register Evidence-composed differential investigation workflows. */ export const registerInvestigationTools = ( @@ -101,7 +101,7 @@ const registerChangedBehavior = ( ], }, ); - return toCallToolResult(recorded, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); }; @@ -158,7 +158,7 @@ const registerCallPath = ( ], }, ); - return toCallToolResult(recorded, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); }; @@ -198,7 +198,9 @@ const registerStaticRuntime = ( limitations: result.limitations, evidenceLinks: links, }); - return toCallToolResult( + return toEvidenceToolResult( + evidence, + contract, recordWorkflowEvidence( session, evidence, @@ -218,7 +220,6 @@ const registerStaticRuntime = ( ], }, ), - contract, ); }, ); @@ -255,7 +256,9 @@ const registerReconstruction = ( limitations: result.limitations, evidenceLinks: links, }); - return toCallToolResult( + return toEvidenceToolResult( + evidence, + contract, recordWorkflowEvidence(session, evidence, result.status === "unknown", { question: "Does the reconstruction satisfy every declared claim?", domain: "reconstruction-verification", @@ -266,7 +269,6 @@ const registerReconstruction = ( rationale, })), }), - contract, ); }, ); diff --git a/src/server/registerJavaScriptRecoveryTool.ts b/src/server/registerJavaScriptRecoveryTool.ts index d70a4431489..032aa8405e6 100644 --- a/src/server/registerJavaScriptRecoveryTool.ts +++ b/src/server/registerJavaScriptRecoveryTool.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind optional recovery to its exact named contract without acquiring an engine. */ export const registerJavaScriptRecoveryTool = ( @@ -24,9 +24,7 @@ export const registerJavaScriptRecoveryTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerJavaScriptRuntimeObservationTools.ts b/src/server/registerJavaScriptRuntimeObservationTools.ts index 6581ce75b4b..f03a3a5737b 100644 --- a/src/server/registerJavaScriptRuntimeObservationTools.ts +++ b/src/server/registerJavaScriptRuntimeObservationTools.ts @@ -19,7 +19,7 @@ import { observeJavaScriptRuntimeInputSchema } from "../domain/javascript/javasc import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; interface RuntimeToolRegistration { readonly logger: Logger; @@ -83,7 +83,5 @@ const runRuntimeTool = async ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = options.recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult({ ok: true, value: result.value }, contract); + return toEvidenceToolResult(result.value, contract, recorded); }; diff --git a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts index c744551037e..7715f1a46b3 100644 --- a/src/server/registerManagedWorkflowTools/compareManagedMembers.ts +++ b/src/server/registerManagedWorkflowTools/compareManagedMembers.ts @@ -5,7 +5,7 @@ import { compareManagedMembersEvidenceValidated } from "../../application/manage import { managedMemberComparisonResultSchema } from "../../domain/managed/managedMemberComparison.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { toCallToolResult } from "../toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "../toolResult.js"; import { managedWorkflowContract } from "./contract.js"; import { resolveManagedEvidence } from "./evidence.js"; import type { ManagedWorkflowToolRegistration } from "./types.js"; @@ -68,12 +68,7 @@ export const registerCompareManagedMembers = ( relationships: [], }) : options.recordEvidence?.(result.value); - if (output !== undefined && !output.ok) - return toCallToolResult(output, compareContract); - return toCallToolResult( - { ok: true, value: result.value }, - compareContract, - ); + return toEvidenceToolResult(result.value, compareContract, output); }, ); }; diff --git a/src/server/registerManagedWorkflowTools/importManagedReconstruction.ts b/src/server/registerManagedWorkflowTools/importManagedReconstruction.ts index d9ddecb6bd7..8aecfe3ad7b 100644 --- a/src/server/registerManagedWorkflowTools/importManagedReconstruction.ts +++ b/src/server/registerManagedWorkflowTools/importManagedReconstruction.ts @@ -3,7 +3,7 @@ import type { McpServer } from "@modelcontextprotocol/server"; import { importManagedReconstructionEvidenceValidated } from "../../application/managed/ManagedReconstructionService.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { toCallToolResult } from "../toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "../toolResult.js"; import { managedWorkflowContract } from "./contract.js"; import { resolveManagedEvidence } from "./evidence.js"; import type { ManagedWorkflowToolRegistration } from "./types.js"; @@ -41,9 +41,11 @@ export const registerImportManagedReconstruction = ( if (recordedSource !== undefined && !recordedSource.ok) return toCallToolResult(recordedSource, reconstructionContract); const recorded = options.recordEvidence?.(result.value); - if (recorded !== undefined && !recorded.ok) - return toCallToolResult(recorded, reconstructionContract); - return toCallToolResult(result, reconstructionContract); + return toEvidenceToolResult( + result.value, + reconstructionContract, + recorded, + ); }, ); }; diff --git a/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts b/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts index 2e7ccc5fd6a..ee3510c9fb7 100644 --- a/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts +++ b/src/server/registerManagedWorkflowTools/projectManagedApplicationGraph.ts @@ -4,7 +4,7 @@ import type { McpServer } from "@modelcontextprotocol/server"; import { projectManagedApplicationGraphEvidence } from "../../application/managed/ManagedApplicationGraphService.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { toCallToolResult } from "../toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "../toolResult.js"; import { managedWorkflowContract } from "./contract.js"; import { resolveManagedArtifactEvidence, @@ -63,9 +63,7 @@ export const registerProjectManagedApplicationGraph = ( if (!recordedSources.ok) return toCallToolResult(recordedSources, graphContract); const recorded = options.recordEvidence?.(result.value); - if (recorded !== undefined && !recorded.ok) - return toCallToolResult(recorded, graphContract); - return toCallToolResult(result, graphContract); + return toEvidenceToolResult(result.value, graphContract, recorded); }, ); }; diff --git a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts index 3bdffe8c1b4..b569d7b7cc1 100644 --- a/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts +++ b/src/server/registerManagedWorkflowTools/verifyManagedNativeBoundaries.ts @@ -5,7 +5,7 @@ import { verifyManagedNativeBoundariesEvidence } from "../../application/managed import { managedNativeVerificationResultSchema } from "../../domain/managed/managedNativeVerification.js"; import { logToolExecution } from "../toolLogging.js"; import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; -import { toCallToolResult } from "../toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "../toolResult.js"; import { managedWorkflowContract } from "./contract.js"; import { resolveManagedBoundaryEvidence, @@ -90,9 +90,11 @@ export const registerVerifyManagedNativeBoundaries = ( relationships: [], }) : options.recordEvidence?.(result.value); - if (output !== undefined && !output.ok) - return toCallToolResult(output, nativeVerificationContract); - return toCallToolResult(result, nativeVerificationContract); + return toEvidenceToolResult( + result.value, + nativeVerificationContract, + output, + ); }, ); }; diff --git a/src/server/registerOfficialTools.ts b/src/server/registerOfficialTools.ts index 2e73af71541..8f56e6b0b28 100644 --- a/src/server/registerOfficialTools.ts +++ b/src/server/registerOfficialTools.ts @@ -22,7 +22,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Optional session services used by direct tool registration. */ export interface OfficialToolRegistration { @@ -91,9 +91,7 @@ const registerOfficialTool = ( }, ); const recorded = registration.recordEvidence?.(evidence); - if (recorded !== undefined && !recorded.ok) - return toCallToolResult(recorded, contract); - return toCallToolResult({ ok: true, value: evidence }, contract); + return toEvidenceToolResult(evidence, contract, recorded); }, ); }; diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts index bff0492cb05..931086bc1c0 100644 --- a/src/server/registerProcessComparisonTool.ts +++ b/src/server/registerProcessComparisonTool.ts @@ -25,7 +25,7 @@ import { recordSessionEvidenceSources } from "./sessionEvidence.js"; import { runDerivedOperation } from "./runDerivedOperation.js"; import { PROCESS_PROVIDER } from "./sessionToolPolicies.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; const PROCESS_CAPTURE_EVIDENCE = { operation: "capture_process_scenario", @@ -143,7 +143,9 @@ export const registerProcessComparisonTool = ( ); if (!recordedSources.ok) return toCallToolResult(recordedSources, contract); - return toCallToolResult( + return toEvidenceToolResult( + evidence, + contract, recordDerivedEvidence( session, evidence, @@ -156,7 +158,6 @@ export const registerProcessComparisonTool = ( comparison, ), ), - contract, ); }, ); diff --git a/src/server/registerRecordedCrashTools.ts b/src/server/registerRecordedCrashTools.ts index 0381689a0fc..eac5037c0c6 100644 --- a/src/server/registerRecordedCrashTools.ts +++ b/src/server/registerRecordedCrashTools.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind recorded crash inspection to its named contract and session Evidence owner. */ export const registerRecordedCrashTools = ( @@ -24,9 +24,7 @@ export const registerRecordedCrashTools = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index fb6feb548f2..fab581dab6f 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -35,7 +35,7 @@ import { } from "./sessionAvailabilityPolicy.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; const recordProcessResidualUnknowns = ( session: BinarySessionPort, @@ -122,7 +122,7 @@ const registerProcessTools = ({ captured.value.residual_unknowns, ); if (!unknowns.ok) return toCallToolResult(unknowns, captureContract); - return toCallToolResult(ok(evidence), captureContract); + return toEvidenceToolResult(evidence, captureContract, recorded); }, ); }; diff --git a/src/server/registerWebModuleTool.ts b/src/server/registerWebModuleTool.ts index aba48648027..ad20af90d14 100644 --- a/src/server/registerWebModuleTool.ts +++ b/src/server/registerWebModuleTool.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind the module relationship workflow to its exact canonical contract. */ export const registerWebModuleTool = ( @@ -24,9 +24,7 @@ export const registerWebModuleTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerWebNetworkCaptureTool.ts b/src/server/registerWebNetworkCaptureTool.ts index 2c1a9e5c58e..0752e81668b 100644 --- a/src/server/registerWebNetworkCaptureTool.ts +++ b/src/server/registerWebNetworkCaptureTool.ts @@ -8,7 +8,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind historical inspection to its named contract and caller-owned Evidence writer. */ export const registerWebNetworkCaptureTool = ( @@ -38,9 +38,7 @@ export const registerWebNetworkCaptureTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerWebRuntimeTools.ts b/src/server/registerWebRuntimeTools.ts index 482231e9cde..56935d2d411 100644 --- a/src/server/registerWebRuntimeTools.ts +++ b/src/server/registerWebRuntimeTools.ts @@ -6,7 +6,7 @@ import type { Logger } from "../logger.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind distinct runtime operations to named contracts and the session's Evidence owner. */ export const registerWebRuntimeTools = ( @@ -29,9 +29,7 @@ export const registerWebRuntimeTools = ( ); if (!result.ok) return toCallToolResult(result, execution); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, execution) - : toCallToolResult(result, execution); + return toEvidenceToolResult(result.value, execution, recorded); }, ); server.registerTool( @@ -43,9 +41,7 @@ export const registerWebRuntimeTools = ( ); if (!result.ok) return toCallToolResult(result, listeners); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, listeners) - : toCallToolResult(result, listeners); + return toEvidenceToolResult(result.value, listeners, recorded); }, ); }; diff --git a/src/server/registerWebScriptTool.ts b/src/server/registerWebScriptTool.ts index c2445a838ae..d3f670aeb8c 100644 --- a/src/server/registerWebScriptTool.ts +++ b/src/server/registerWebScriptTool.ts @@ -6,7 +6,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Register local script export without requiring a live browser provider. */ export const registerWebScriptTool = ( @@ -26,9 +26,7 @@ export const registerWebScriptTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = options.recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/registerWebSourceLocationTool.ts b/src/server/registerWebSourceLocationTool.ts index 2091de83e9a..f3ed840f65f 100644 --- a/src/server/registerWebSourceLocationTool.ts +++ b/src/server/registerWebSourceLocationTool.ts @@ -5,7 +5,7 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { Logger } from "../logger.js"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; -import { toCallToolResult } from "./toolResult.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; /** Bind source point tracing to its exact named contract and session Evidence owner. */ export const registerWebSourceLocationTool = ( @@ -24,9 +24,7 @@ export const registerWebSourceLocationTool = ( ); if (!result.ok) return toCallToolResult(result, contract); const recorded = recordEvidence?.(result.value); - return recorded !== undefined && !recorded.ok - ? toCallToolResult(recorded, contract) - : toCallToolResult(result, contract); + return toEvidenceToolResult(result.value, contract, recorded); }, ); }; diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 6beb1e54d13..9313f6dfa50 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -2,6 +2,7 @@ import type { CallToolResult } from "@modelcontextprotocol/server"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import type { ToolContract } from "../contracts/toolContracts.js"; +import type { Evidence } from "../domain/evidence.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -28,6 +29,22 @@ export const toCallToolResult = ( ? successResult(result.value, contract, context) : errorResult(result.error); +/** Deliver Evidence using the producer's explicit recording acknowledgment. */ +export const toEvidenceToolResult = ( + evidence: Evidence, + contract: ToolContract, + recorded: Result | undefined, +): CallToolResult => + recorded !== undefined && !recorded.ok + ? errorResult(recorded.error) + : toCallToolResult( + { ok: true, value: evidence }, + contract, + recorded === undefined + ? undefined + : { retainedEvidenceId: evidence.evidence_id }, + ); + const errorResult = (error: AnalysisError): CallToolResult => { const projected = projectAnalysisError(error); const structuredContent = { error: projected }; @@ -66,8 +83,8 @@ const successResult = ( contract.name, "transport", encoded.constraint === "string-length" - ? "The complete MCP response exceeds Node's single-string representation limit. Use a focused retained-Evidence query or export to consume the complete analysis." - : "The complete MCP response exceeds the stdio response budget. The analysis result was not truncated; use a focused retained-Evidence query or export to consume it.", + ? "The operation completed, but its complete MCP response exceeds Node's single-string representation limit. Export retained evidence to consume the complete analysis, or use complete CLI JSON output." + : "The operation completed, but its complete MCP response exceeds the stdio response budget. The analysis result was not truncated; export retained evidence to consume it, or use complete CLI JSON output.", { boundary: "mcp-response", default_receive_buffer_bytes: STDIO_DEFAULT_MAX_BUFFER_SIZE, diff --git a/tests/boundary/mcp/largeToolResult.test.ts b/tests/boundary/mcp/largeToolResult.test.ts index 0bf82e4c007..4f8daa87aaf 100644 --- a/tests/boundary/mcp/largeToolResult.test.ts +++ b/tests/boundary/mcp/largeToolResult.test.ts @@ -1,14 +1,13 @@ import { describe, expect, it } from "vitest"; +import { z } from "zod"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; import { toolContract } from "../../../src/contracts/toolContracts.js"; -import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; -import { ok } from "../../../src/domain/result.js"; -import { toCallToolResult } from "../../../src/server/toolResult.js"; -import { EvidenceLedger } from "../../../src/application/investigation/EvidenceLedger.js"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import { toEvidenceToolResult } from "../../../src/server/toolResult.js"; describe("large complete Evidence MCP delivery", () => { - it("preserves retained content and returns an actionable framing constraint", () => { + it("does not claim retention without a recording acknowledgment", () => { const normalized = { payload: "x".repeat(Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 3)), }; @@ -21,49 +20,29 @@ describe("large complete Evidence MCP delivery", () => { result: normalized, }, ); - const ledger = new EvidenceLedger(); - expect(ledger.record(evidence).ok).toBe(true); - const result = toCallToolResult( - ok(evidence), + const detached = toEvidenceToolResult( + evidence, toolContract("analyze_javascript_application"), - { retainedEvidenceId: evidence.evidence_id }, + undefined, ); - expect(result.isError).toBe(true); - expect(result.structuredContent).toMatchObject({ - error: { - code: "resource_constraint", - category: "resource_constraint", - details: { - resource: "transport", - reported_limits: { - boundary: "mcp-response", - constraint: "receive-buffer", - evidence_reference: { - kind: "retained-evidence", - evidence_id: evidence.evidence_id, - }, - }, - }, - }, - }); - expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThan( + expect(detached.isError).toBe(true); + expect(Buffer.byteLength(JSON.stringify(detached))).toBeLessThan( STDIO_DEFAULT_MAX_BUFFER_SIZE, ); - expect(result.content).toEqual([ - { type: "text", text: JSON.stringify(result.structuredContent) }, - ]); - const retained = ledger.get(evidence.evidence_id); - expect(parseEvidence(retained)).toEqual(evidence); - expect(retained?.normalized_result).toEqual(normalized); - const detached = toCallToolResult( - ok(evidence), - toolContract("analyze_javascript_application"), - ); expect(detached.structuredContent).toMatchObject({ error: { details: { reported_limits: { evidence_id: evidence.evidence_id } }, }, }); - expect(JSON.stringify(detached)).not.toContain("retained-evidence"); + const limits = z + .object({ + error: z.object({ + details: z.object({ + reported_limits: z.record(z.string(), z.unknown()), + }), + }), + }) + .parse(detached.structuredContent).error.details.reported_limits; + expect(limits).not.toHaveProperty("evidence_reference"); }); }); diff --git a/tests/conformance/ghidra/large-results.c b/tests/conformance/ghidra/large-results.c new file mode 100644 index 00000000000..d1406fc9c9f --- /dev/null +++ b/tests/conformance/ghidra/large-results.c @@ -0,0 +1,9 @@ +// A real initialized data range, with independently recognizable endpoints. +volatile unsigned char rea_large_payload[REA_PAYLOAD_BYTES] = { + [0] = 0x5a, [REA_PAYLOAD_BYTES - 1] = 0xa5}; + +__attribute__((noinline, used)) int rea_large_entry(void) { + return rea_large_payload[0] + rea_large_payload[REA_PAYLOAD_BYTES - 1]; +} + +int main(void) { return rea_large_entry(); } From 08561e0eb7a4a01a256f3693a97061fbea7fb8fa Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:08:29 +0800 Subject: [PATCH 044/791] feat(clients): register Grok Build and Grok Bot during setup (#1134) * feat(clients): register Grok Build and Grok Bot during setup Detect both clients and install the local REA MCP registration the same way setup already handles the other agents. Grok Build edits only the REA tables in its TOML config and sets startup_timeout_sec to 30. Grok Bot writes the mcpServers.rea entry into mcp.json. That file is the registration to add to the signed-in account; setup does not call the account API. Filed by Grok. * fix(clients): clear a disabled Grok Build registration during setup Grok Build keeps a server suppressed when its name stays in disabled_mcp_servers, and setup was treating that file as already aligned. Setup now removes only that name, doctor reports the registration stale until it is gone, and uninstall keeps comments in an inline table whose only server was REA. Split the lifecycle suite and reflow the get-started page so static checks pass. * fix(verify): check packed Hopper search as literal-only The package verifier still called a regex probe that main deleted. The native bridge refuses regex and joins string hits with typed records, so the restored probe stubs that inventory and the check asserts the literal matches and the regex refusal. The new README translations also name Grok Build and Grok Bot. * fix(clients): stop treating a Grok Bot data directory as registered Grok Bot stores connectors in the signed-in account and runs them on its hosted computer. Setup no longer writes mcp.json or reports that file as aligned. Doctor and setup tell the caller to add the server from the Grok Bot chat. * fix(clients): drop multiline Grok disable-list entries Decode disabled_mcp_servers strings with the document parser so a multiline basic or literal string that loads as rea is removed. --------- Co-authored-by: Grok Co-authored-by: Cursor Agent Co-authored-by: morluto --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 2 +- README_es.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 2 +- README_pt-BR.md | 2 +- README_ru.md | 2 +- README_th.md | 2 +- README_tr.md | 2 +- README_uk.md | 2 +- README_vi.md | 2 +- README_zh-TW.md | 2 +- README_zh.md | 2 +- docs/installation.md | 30 +- docs/roadmap.md | 2 +- scripts/verify-package-environment.mjs | 2 + scripts/verify-package-update.mjs | 2 + scripts/verify-published-package.mjs | 9 + scripts/verify-windows-package.mjs | 4 + skill-src/reverse-engineer-anything/SKILL.md | 4 +- .../ClientConfigurationDocument.ts | 985 +++++++++++++- src/application/ClientRegistrationStatus.ts | 43 +- src/application/Doctor.test.ts | 51 + src/application/Doctor.ts | 2 +- src/application/DoctorScope.ts | 15 +- src/application/Setup.planning.test.ts | 32 + src/application/Setup.ts | 38 +- src/application/SetupClientConfiguration.ts | 14 +- src/application/SetupHost.ts | 8 +- src/application/SetupPlan.ts | 8 +- src/application/SupportedClients.ts | 48 +- src/application/Uninstall.ts | 11 +- .../clientConfigurationDialects.test.ts | 36 + .../filesystem/clientLifecycle.test.ts | 1129 ++++++++++++++++- .../clientRegistrationStatus.test.ts | 56 +- .../process-global/updateMaintenance.test.ts | 2 + website/public/faq/index.html | 3 +- website/public/get-started/index.html | 5 +- 40 files changed, 2501 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index 115d6cc6d30..51d21e4d613 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Choose your agents, review the proposed changes, and approve them. Setup adds REA's MCP server and matching workflow instructions, with backups of existing configuration. Restart your agent afterward. -Setup supports Claude Code, Codex, Cursor, Gemini CLI and +Setup supports Claude Code, Codex, Cursor, Gemini CLI, Grok Build and [other agents](docs/installation.md#supported-agents). See [installation and setup](docs/installation.md) for provider configuration and manual MCP registration. diff --git a/README_ar.md b/README_ar.md index 24c3e148cd2..8e977c3260a 100644 --- a/README_ar.md +++ b/README_ar.md @@ -68,7 +68,7 @@ npx rea-agents setup اختر الوكلاء، وراجع التغييرات المقترحة، ثم وافق عليها. يضيف الإعداد خادم MCP الخاص بـ REA وتعليمات سير العمل المطابقة، مع الاحتفاظ بنسخ احتياطية من الإعدادات الموجودة. أعد تشغيل وكيلك بعد ذلك. -يدعم الإعداد Claude Code وCodex وCursor وGemini CLI و[وكلاء آخرين](docs/installation.md#supported-agents). راجع [التثبيت والإعداد](docs/installation.md) لتهيئة موفّري التحليل وتسجيل MCP يدويًا. +يدعم الإعداد Claude Code وCodex وCursor وGemini CLI وGrok Build و[وكلاء آخرين](docs/installation.md#supported-agents). راجع [التثبيت والإعداد](docs/installation.md) لتهيئة موفّري التحليل وتسجيل MCP يدويًا. ### اطلب من وكيلك diff --git a/README_de.md b/README_de.md index dc2c2b87823..a6342dc0f6b 100644 --- a/README_de.md +++ b/README_de.md @@ -68,7 +68,7 @@ npx rea-agents setup Wähle deine Agenten aus, prüfe die geplanten Änderungen und bestätige sie. Die Einrichtung fügt den MCP-Server von REA und passende Workflow-Anweisungen hinzu und sichert vorhandene Konfigurationen. Starte deinen Agenten anschließend neu. -Die Einrichtung unterstützt Claude Code, Codex, Cursor, Gemini CLI und [weitere Agenten](docs/installation.md#supported-agents). Informationen zur Provider-Konfiguration und manuellen MCP-Registrierung findest du unter [Installation und Einrichtung](docs/installation.md). +Die Einrichtung unterstützt Claude Code, Codex, Cursor, Gemini CLI, Grok Build und [weitere Agenten](docs/installation.md#supported-agents). Informationen zur Provider-Konfiguration und manuellen MCP-Registrierung findest du unter [Installation und Einrichtung](docs/installation.md). ### Deinen Agenten fragen diff --git a/README_es.md b/README_es.md index e9df0c5c7a2..4d0a06d4f1c 100644 --- a/README_es.md +++ b/README_es.md @@ -68,7 +68,7 @@ npx rea-agents setup Elige tus agentes, revisa los cambios propuestos y apruébalos. La configuración añade el servidor MCP de REA y las instrucciones de trabajo correspondientes, con copias de seguridad de la configuración existente. Reinicia tu agente al terminar. -La configuración admite Claude Code, Codex, Cursor, Gemini CLI y [otros agentes](docs/installation.md#supported-agents). Consulta [instalación y configuración](docs/installation.md) para configurar proveedores y registrar MCP manualmente. +La configuración admite Claude Code, Codex, Cursor, Gemini CLI, Grok Build y [otros agentes](docs/installation.md#supported-agents). Consulta [instalación y configuración](docs/installation.md) para configurar proveedores y registrar MCP manualmente. ### Pregunta a tu agente diff --git a/README_ja.md b/README_ja.md index f51f9d7c12d..61efe98257b 100644 --- a/README_ja.md +++ b/README_ja.md @@ -68,7 +68,7 @@ npx rea-agents setup 使うエージェントを選び、予定されている変更を確認して承認してください。セットアップは REA の MCP サーバーと対応するワークフローの手順を追加し、既存の設定をバックアップします。完了後にエージェントを再起動してください。 -セットアップは Claude Code、Codex、Cursor、Gemini CLI、および[その他のエージェント](docs/installation.md#supported-agents)に対応しています。プロバイダーの設定と MCP の手動登録については、[インストールとセットアップ](docs/installation.md)を参照してください。 +セットアップは Claude Code、Codex、Cursor、Gemini CLI、Grok Build、および[その他のエージェント](docs/installation.md#supported-agents)に対応しています。プロバイダーの設定と MCP の手動登録については、[インストールとセットアップ](docs/installation.md)を参照してください。 ### エージェントに依頼する diff --git a/README_ko.md b/README_ko.md index 4180c38e292..bc41bcc0c0a 100644 --- a/README_ko.md +++ b/README_ko.md @@ -68,7 +68,7 @@ npx rea-agents setup 에이전트를 선택하고, 예정된 변경 사항을 검토한 뒤 승인하세요. 설정 과정은 기존 설정을 백업하고 REA의 MCP 서버와 버전에 맞는 워크플로 지침을 추가합니다. 완료 후 에이전트를 다시 시작하세요. -설정은 Claude Code, Codex, Cursor, Gemini CLI 및 [다른 에이전트](docs/installation.md#supported-agents)를 지원합니다. 제공자 설정과 수동 MCP 등록은 [설치 및 설정](docs/installation.md)을 참고하세요. +설정은 Claude Code, Codex, Cursor, Gemini CLI, Grok Build 및 [다른 에이전트](docs/installation.md#supported-agents)를 지원합니다. 제공자 설정과 수동 MCP 등록은 [설치 및 설정](docs/installation.md)을 참고하세요. ### 에이전트에게 요청 diff --git a/README_pl.md b/README_pl.md index baa8e3f1cc3..404166822e4 100644 --- a/README_pl.md +++ b/README_pl.md @@ -68,7 +68,7 @@ npx rea-agents setup Wybierz swoich agentów, sprawdź proponowane zmiany i zatwierdź je. Konfigurator dodaje serwer MCP REA i pasujące instrukcje pracy, tworząc kopie zapasowe istniejącej konfiguracji. Następnie uruchom agenta ponownie. -Konfigurator obsługuje Claude Code, Codex, Cursor, Gemini CLI i [innych agentów](docs/installation.md#supported-agents). Konfigurację dostawców analizy i ręczną rejestrację MCP opisuje przewodnik [instalacji i konfiguracji](docs/installation.md). +Konfigurator obsługuje Claude Code, Codex, Cursor, Gemini CLI, Grok Build i [innych agentów](docs/installation.md#supported-agents). Konfigurację dostawców analizy i ręczną rejestrację MCP opisuje przewodnik [instalacji i konfiguracji](docs/installation.md). ### Zapytaj agenta diff --git a/README_pt-BR.md b/README_pt-BR.md index bad0348639e..a83a1b83308 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -68,7 +68,7 @@ npx rea-agents setup Escolha seus agentes, revise as alterações propostas e aprove-as. A configuração adiciona o servidor MCP do REA e as instruções de fluxo de trabalho correspondentes, com backups das configurações existentes. Reinicie seu agente depois. -A configuração oferece suporte ao Claude Code, Codex, Cursor, Gemini CLI e [outros agentes](docs/installation.md#supported-agents). Consulte [instalação e configuração](docs/installation.md) para configurar provedores e registrar o MCP manualmente. +A configuração oferece suporte ao Claude Code, Codex, Cursor, Gemini CLI, Grok Build e [outros agentes](docs/installation.md#supported-agents). Consulte [instalação e configuração](docs/installation.md) para configurar provedores e registrar o MCP manualmente. ### Peça ao seu agente diff --git a/README_ru.md b/README_ru.md index a20e077e1bf..07051c98fd9 100644 --- a/README_ru.md +++ b/README_ru.md @@ -68,7 +68,7 @@ npx rea-agents setup Выберите агентов, проверьте предлагаемые изменения и подтвердите их. Настройка добавляет MCP-сервер REA и соответствующие рабочие инструкции, сохраняя резервные копии существующих настроек. После этого перезапустите агента. -Настройка поддерживает Claude Code, Codex, Cursor, Gemini CLI и [других агентов](docs/installation.md#supported-agents). Настройка провайдеров и ручная регистрация MCP описаны в разделе [установка и настройка](docs/installation.md). +Настройка поддерживает Claude Code, Codex, Cursor, Gemini CLI, Grok Build и [других агентов](docs/installation.md#supported-agents). Настройка провайдеров и ручная регистрация MCP описаны в разделе [установка и настройка](docs/installation.md). ### Задайте вопрос агенту diff --git a/README_th.md b/README_th.md index da34fb80c72..e2f4d7454cc 100644 --- a/README_th.md +++ b/README_th.md @@ -68,7 +68,7 @@ npx rea-agents setup เลือกเอเจนต์ ตรวจสอบการเปลี่ยนแปลงที่เสนอ แล้วอนุมัติ ขั้นตอนตั้งค่าจะเพิ่มเซิร์ฟเวอร์ MCP ของ REA และคำแนะนำการทำงานที่ตรงกัน พร้อมสำรองการตั้งค่าที่มีอยู่ จากนั้นเริ่มเอเจนต์ใหม่ -ขั้นตอนตั้งค่ารองรับ Claude Code, Codex, Cursor, Gemini CLI และ[เอเจนต์อื่น ๆ](docs/installation.md#supported-agents) ดู[การติดตั้งและตั้งค่า](docs/installation.md) สำหรับการตั้งค่าผู้ให้บริการวิเคราะห์และการลงทะเบียน MCP ด้วยตนเอง +ขั้นตอนตั้งค่ารองรับ Claude Code, Codex, Cursor, Gemini CLI, Grok Build และ[เอเจนต์อื่น ๆ](docs/installation.md#supported-agents) ดู[การติดตั้งและตั้งค่า](docs/installation.md) สำหรับการตั้งค่าผู้ให้บริการวิเคราะห์และการลงทะเบียน MCP ด้วยตนเอง ### ถามเอเจนต์ diff --git a/README_tr.md b/README_tr.md index cf449c2e72e..8966e6a8665 100644 --- a/README_tr.md +++ b/README_tr.md @@ -68,7 +68,7 @@ npx rea-agents setup Ajanlarınızı seçin, önerilen değişiklikleri inceleyin ve onaylayın. Kurulum, mevcut yapılandırmaları yedekleyerek REA'nın MCP sunucusunu ve uyumlu iş akışı talimatlarını ekler. Ardından ajanınızı yeniden başlatın. -Kurulum Claude Code, Codex, Cursor, Gemini CLI ve [diğer ajanları](docs/installation.md#supported-agents) destekler. Sağlayıcı yapılandırması ve elle MCP kaydı için [yükleme ve kurulum](docs/installation.md) belgesine bakın. +Kurulum Claude Code, Codex, Cursor, Gemini CLI, Grok Build ve [diğer ajanları](docs/installation.md#supported-agents) destekler. Sağlayıcı yapılandırması ve elle MCP kaydı için [yükleme ve kurulum](docs/installation.md) belgesine bakın. ### Ajanınıza sorun diff --git a/README_uk.md b/README_uk.md index 45e9c8265fd..52bdd78a048 100644 --- a/README_uk.md +++ b/README_uk.md @@ -68,7 +68,7 @@ npx rea-agents setup Виберіть агентів, перевірте запропоновані зміни та схваліть їх. Налаштування додає MCP-сервер REA та відповідні інструкції робочих процесів, створюючи резервні копії наявної конфігурації. Після цього перезапустіть агента. -Налаштування підтримує Claude Code, Codex, Cursor, Gemini CLI та [інших агентів](docs/installation.md#supported-agents). Конфігурацію провайдерів і ручну реєстрацію MCP описано в розділі [встановлення й налаштування](docs/installation.md). +Налаштування підтримує Claude Code, Codex, Cursor, Gemini CLI, Grok Build та [інших агентів](docs/installation.md#supported-agents). Конфігурацію провайдерів і ручну реєстрацію MCP описано в розділі [встановлення й налаштування](docs/installation.md). ### Запитайте агента diff --git a/README_vi.md b/README_vi.md index 252faf85e1f..0edf0b48a50 100644 --- a/README_vi.md +++ b/README_vi.md @@ -68,7 +68,7 @@ npx rea-agents setup Chọn các tác nhân, xem lại những thay đổi được đề xuất và chấp thuận. Quá trình thiết lập thêm máy chủ MCP của REA cùng hướng dẫn quy trình tương ứng, đồng thời sao lưu cấu hình hiện có. Sau đó khởi động lại tác nhân. -Quá trình thiết lập hỗ trợ Claude Code, Codex, Cursor, Gemini CLI và [các tác nhân khác](docs/installation.md#supported-agents). Xem [cài đặt và thiết lập](docs/installation.md) để cấu hình trình phân tích và đăng ký MCP thủ công. +Quá trình thiết lập hỗ trợ Claude Code, Codex, Cursor, Gemini CLI, Grok Build và [các tác nhân khác](docs/installation.md#supported-agents). Xem [cài đặt và thiết lập](docs/installation.md) để cấu hình trình phân tích và đăng ký MCP thủ công. ### Hỏi tác nhân diff --git a/README_zh-TW.md b/README_zh-TW.md index c8a8eeadc84..9b0915116d2 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -68,7 +68,7 @@ npx rea-agents setup 選擇 AI 代理,檢查計劃中的變更並核准。設定流程會添加 REA 的 MCP 服務和對應的工作流程指引,並備份已有組態。完成後重新啟動 AI 代理。 -設定流程支援 Claude Code、Codex、Cursor、Gemini CLI 和[其他 AI 代理](docs/installation.md#supported-agents)。提供者組態和手動註冊 MCP 的方法見[安裝與設定](docs/installation.md)。 +設定流程支援 Claude Code、Codex、Cursor、Gemini CLI、Grok Build 和[其他 AI 代理](docs/installation.md#supported-agents)。提供者組態和手動註冊 MCP 的方法見[安裝與設定](docs/installation.md)。 ### 詢問 AI 代理 diff --git a/README_zh.md b/README_zh.md index 218ca5d8097..0e7624e0d9d 100644 --- a/README_zh.md +++ b/README_zh.md @@ -68,7 +68,7 @@ npx rea-agents setup 选择智能体,检查计划中的变更并批准。设置流程会添加 REA 的 MCP 服务和匹配的工作流指引,并备份已有配置。完成后重启智能体。 -设置流程支持 Claude Code、Codex、Cursor、Gemini CLI 和[其他智能体](docs/installation.md#supported-agents)。提供方配置和手动注册 MCP 的方法见[安装与设置](docs/installation.md)。 +设置流程支持 Claude Code、Codex、Cursor、Gemini CLI、Grok Build 和[其他智能体](docs/installation.md#supported-agents)。提供方配置和手动注册 MCP 的方法见[安装与设置](docs/installation.md)。 ### 询问智能体 diff --git a/docs/installation.md b/docs/installation.md index 078b29013dd..5dcf4d99a11 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -119,7 +119,8 @@ readiness for one task instead of auditing every integration, see ## Supported agents -Setup can configure these clients for REA's local MCP server: +Setup can configure these clients for REA's local MCP server. Grok Bot is +listed after the table because its connector is not one of these files: | Client | `--client` value | | ------------------ | ---------------- | @@ -135,12 +136,37 @@ Setup can configure these clients for REA's local MCP server: | GitHub Copilot CLI | `copilot_cli` | | Command Code | `commandcode` | | VS Code | `vscode` | +| Grok Build | `grok_build` | For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2 both load. If the configuration already uses OpenCode V2's native `mcp.servers` table, setup registers REA there instead and replaces any earlier `mcp.rea` entry from REA. +Grok Build loads `[mcp_servers.rea]` from `$GROK_HOME/config.toml`, or from +`~/.grok/config.toml` when `GROK_HOME` is unset. Setup edits that server +table, `[mcp_servers.rea.env]`, and a root `disabled_mcp_servers` entry that +names `rea`. It sets `startup_timeout_sec = 30` and leaves every other name +in that list. The shared skill installed under `~/.agents/skills` is already +on Grok Build's skill path. + +Grok Bot (`grok_bot`) is detected from `~/.grokbot`, or from `SAND_DATA_ROOT` +when that value is an absolute path. A relative or empty `SAND_DATA_ROOT` +stays on `~/.grokbot`. That directory is not the connector store. Grok Bot +keeps connectors in the signed-in account and runs them on its hosted +computer. It does not import `mcp.json` from the data directory, and it does +not attach a stdio server running on this machine. Setup does not call the +account connector API, does not write a registration file, and does not report +the data directory as aligned. Ask the Grok Bot chat to add a custom MCP +server named `rea` that runs on the Bot's computer: + +```bash +npx -y rea-agents@ mcp +``` + +Do not put credentials in that command or its arguments. `rea doctor --client grok_bot` +reports this manual step. `rea uninstall` does not remove the account connector. + ## Review setup changes `rea setup` first offers the supported agents in a multi-select. Existing REA @@ -209,7 +235,7 @@ rea setup --yes --all-detected --install-hopper --json Setup pins package-runner MCP registrations to the exact installed REA version, installs the matching skill and on-demand references in the same plan, and adds -`startup_timeout_sec = 30` for Codex. `rea update` installs the exact resolved +`startup_timeout_sec = 30` for Codex and Grok Build. `rea update` installs the exact resolved release into the npm prefix that owns the running package, then checks the new executable's version before reporting success. It does not reopen onboarding. Release lookup and installation both use npm's configured registry. diff --git a/docs/roadmap.md b/docs/roadmap.md index 3832bf23882..62323d591cc 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -67,7 +67,7 @@ REA setup lets you select agent integration and optional Hopper installation. It installs the bundled workflow, configures detected agents, and can save verified paths for an existing Ghidra installation. It configures Claude Code, Claude Desktop, Codex, Cursor, Gemini CLI, Windsurf, Devin, OpenCode, Antigravity, -GitHub Copilot CLI, Command Code, and VS Code using each client's configuration format. +GitHub Copilot CLI, Command Code, VS Code, and Grok Build using each client's configuration format. Grok Bot is detected locally, but its connector is added from the Grok Bot chat because the signed-in account store is not a data-directory file. Ghidra analysis supports Linux x64 and macOS x64/arm64 with Ghidra 12.1.x and the 64-bit full JDK that installation declares. Current 12.1 releases require JDK 21 diff --git a/scripts/verify-package-environment.mjs b/scripts/verify-package-environment.mjs index 73a37039830..90955d59329 100644 --- a/scripts/verify-package-environment.mjs +++ b/scripts/verify-package-environment.mjs @@ -79,6 +79,8 @@ export async function verifyPackageEnvironment({ CLAUDE_CONFIG_DIR: home, CODEX_HOME: codexDir, COPILOT_HOME: join(home, ".copilot"), + GROK_HOME: join(home, ".grok"), + SAND_DATA_ROOT: join(home, ".grokbot"), OPENCODE_CONFIG: undefined, XDG_CONFIG_HOME: join(home, ".config"), PATH: `${fakeBin}:${process.env.PATH ?? ""}`, diff --git a/scripts/verify-package-update.mjs b/scripts/verify-package-update.mjs index fc12965464b..fcc27b09148 100644 --- a/scripts/verify-package-update.mjs +++ b/scripts/verify-package-update.mjs @@ -101,6 +101,8 @@ export async function verifyPackageUpdate({ CLAUDE_CONFIG_DIR: home, CODEX_HOME: join(home, ".codex"), COPILOT_HOME: join(home, ".copilot"), + GROK_HOME: join(home, ".grok"), + SAND_DATA_ROOT: join(home, ".grokbot"), XDG_CONFIG_HOME: join(home, ".config"), OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"), npm_config_prefix: prefix, diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index aa1c05b386d..07c28d76bd3 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -99,6 +99,15 @@ async function verifyPublishedUpdate({ version: targetVersion, canaryRoot }) { const environment = { ...process.env, HOME: home, + USERPROFILE: home, + APPDATA: join(home, "AppData", "Roaming"), + CLAUDE_CONFIG_DIR: home, + CODEX_HOME: join(home, ".codex"), + COPILOT_HOME: join(home, ".copilot"), + GROK_HOME: join(home, ".grok"), + SAND_DATA_ROOT: join(home, ".grokbot"), + XDG_CONFIG_HOME: join(home, ".config"), + OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.json"), npm_config_cache: join(canaryRoot, "update-cache"), npm_config_prefix: prefix, PATH: `${join(prefix, "bin")}${delimiter}${process.env.PATH ?? ""}`, diff --git a/scripts/verify-windows-package.mjs b/scripts/verify-windows-package.mjs index 3d3fabf3531..8ece0dde37e 100644 --- a/scripts/verify-windows-package.mjs +++ b/scripts/verify-windows-package.mjs @@ -108,9 +108,13 @@ try { HOME: home, USERPROFILE: home, APPDATA: join(home, "AppData", "Roaming"), + CLAUDE_CONFIG_DIR: home, + CODEX_HOME: join(home, ".codex"), XDG_CONFIG_HOME: join(home, ".config"), OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.jsonc"), COPILOT_HOME: join(home, ".copilot"), + GROK_HOME: join(home, ".grok"), + SAND_DATA_ROOT: join(home, ".grokbot"), }; await mkdir(join(home, ".config", "opencode"), { recursive: true }); await writeFile( diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/skill-src/reverse-engineer-anything/SKILL.md index d1b3690605d..404ad45ea8a 100644 --- a/skill-src/reverse-engineer-anything/SKILL.md +++ b/skill-src/reverse-engineer-anything/SKILL.md @@ -31,7 +31,9 @@ When tools are absent or registration is stale: `npx -y rea-agents@latest doctor --client codex --json`. Substitute the current supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`, - `commandcode`, or `vscode`. If the client is unknown, use `doctor --json` and + `commandcode`, `vscode`, `grok_build`, or `grok_bot`. `grok_bot` has no local + file registration: doctor reports the Grok Bot chat step, and setup does not + write `mcp.json`. If the client is unknown, use `doctor --json` and inspect its registration results before choosing a setup scope. 2. Distinguish the reason. Missing, malformed, or stale registration needs a scoped configuration repair. An aligned registration with no tools in the diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index f12db600b65..0b96a79d37c 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -2,13 +2,16 @@ import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; import { isDeepStrictEqual } from "node:util"; import { applyEdits, + findNodeAtLocation, modify, parse as parseJsonc, + parseTree, printParseErrorCode, type ParseError, } from "jsonc-parser"; import { z } from "zod"; +import { PRODUCT_IDENTITY } from "../identity.js"; import type { SetupClient } from "./SupportedClients.js"; export type ClientConfigurationFormat = NonNullable; @@ -66,6 +69,7 @@ export const clientConfigurationServersKey = ( ): ClientServersKey => { switch (format) { case "toml": + case "grok": return "mcp_servers"; case "opencode": return "mcp"; @@ -80,7 +84,8 @@ const parseDocument = ( text: string, format: ClientConfigurationFormat, ): Record => { - if (format === "toml") return objectSchema.parse(parseToml(text)); + if (format === "toml" || format === "grok") + return objectSchema.parse(parseToml(text)); const errors: ParseError[] = []; // Accept a UTF-8 BOM without shifting diagnostics or editing the original text. const jsonText = text.startsWith("\uFEFF") ? ` ${text.slice(1)}` : text; @@ -178,6 +183,959 @@ const valueAt = ( return current; }; +type TomlStringMode = "none" | "ml-basic" | "ml-literal"; + +const TOML_BASIC_ESCAPES: Readonly> = { + b: "\b", + t: "\t", + n: "\n", + f: "\f", + r: "\r", + e: "\u001b", + '"': '"', + "\\": "\\", +}; + +/** A `[table]` or `[[array]]` header outside strings, with decoded key segments. */ +interface TomlTableHeader { + readonly path: readonly string[]; +} + +const skipHorizontalSpace = (line: string, index: number): number => { + let next = index; + while (line[next] === " " || line[next] === "\t") next += 1; + return next; +}; + +const decodeBasicEscape = ( + line: string, + slash: number, +): { readonly value: string; readonly next: number } | undefined => { + const escape = line[slash + 1]; + if (escape === undefined) return undefined; + const simple = TOML_BASIC_ESCAPES[escape]; + if (simple !== undefined) return { value: simple, next: slash + 2 }; + // smol-toml accepts `\xHH` even though TOML 1.0 does not. + if (escape === "x") { + const hex = line.slice(slash + 2, slash + 4); + if (!/^[0-9A-Fa-f]{2}$/u.test(hex)) return undefined; + return { + value: String.fromCodePoint(Number.parseInt(hex, 16)), + next: slash + 4, + }; + } + if (escape !== "u" && escape !== "U") return undefined; + const length = escape === "u" ? 4 : 8; + const hex = line.slice(slash + 2, slash + 2 + length); + if (!new RegExp(`^[0-9A-Fa-f]{${String(length)}}$`, "u").test(hex)) + return undefined; + const code = Number.parseInt(hex, 16); + if (code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return undefined; + return { value: String.fromCodePoint(code), next: slash + 2 + length }; +}; + +/** Read one single-line quoted key. Multiline quotes are not a header key. */ +const parseQuotedKey = ( + line: string, + start: number, +): { readonly value: string; readonly next: number } | undefined => { + const quote = line[start]; + if ( + (quote !== '"' && quote !== "'") || + line.startsWith(quote === '"' ? '"""' : "'''", start) + ) + return undefined; + if (quote === "'") { + const end = line.indexOf("'", start + 1); + if (end < 0 || line.slice(start + 1, end).includes("\n")) return undefined; + return { value: line.slice(start + 1, end), next: end + 1 }; + } + let value = ""; + let index = start + 1; + while (index < line.length) { + const char = line[index]; + if (char === "\n" || char === "\r") return undefined; + if (char === '"') return { value, next: index + 1 }; + if (char !== "\\") { + value += char ?? ""; + index += 1; + continue; + } + const decoded = decodeBasicEscape(line, index); + if (decoded === undefined) return undefined; + value += decoded.value; + index = decoded.next; + } + return undefined; +}; + +const parseBareKey = ( + line: string, + start: number, +): { readonly value: string; readonly next: number } | undefined => { + let index = start; + while (/[A-Za-z0-9_-]/u.test(line[index] ?? "")) index += 1; + if (index === start) return undefined; + return { value: line.slice(start, index), next: index }; +}; + +/** + * Parse a table header. Quoted segments stay one key, so `mcp_servers."rea.env"` + * is not the dotted path `mcp_servers.rea.env`. + */ +const parseTomlTableHeader = (line: string): TomlTableHeader | undefined => { + let index = skipHorizontalSpace(line, 0); + const array = line.startsWith("[[", index); + if (!array && line[index] !== "[") return undefined; + index += array ? 2 : 1; + const path: string[] = []; + for (;;) { + index = skipHorizontalSpace(line, index); + const key = + line[index] === '"' || line[index] === "'" + ? parseQuotedKey(line, index) + : parseBareKey(line, index); + if (key === undefined) return undefined; + path.push(key.value); + index = skipHorizontalSpace(line, key.next); + if (line[index] !== ".") break; + index += 1; + } + const close = array ? "]]" : "]"; + if (!line.startsWith(close, index)) return undefined; + index = skipHorizontalSpace(line, index + close.length); + if (index < line.length && line[index] !== "#") return undefined; + return { path }; +}; + +/** + * String state after one physical line, starting at `start`. A multiline + * closer does not end the scan: `""", """` closes one string and opens another. + */ +const tomlStringModeAfter = ( + line: string, + mode: TomlStringMode, + start = 0, +): TomlStringMode => { + if (mode === "ml-basic" || mode === "ml-literal") { + const delimiter = mode === "ml-basic" ? '"""' : "'''"; + const closed = closeMultiline(line, start, delimiter, mode === "ml-basic"); + if (closed === undefined) return mode; + return tomlStringModeAfter(line, "none", closed); + } + let index = start; + while (index < line.length) { + if (line[index] === "#") return "none"; + if (line.startsWith('"""', index)) { + const closed = closeMultiline(line, index + 3, '"""', true); + if (closed === undefined) return "ml-basic"; + index = closed; + continue; + } + if (line.startsWith("'''", index)) { + const closed = closeMultiline(line, index + 3, "'''", false); + if (closed === undefined) return "ml-literal"; + index = closed; + continue; + } + const quote = line[index]; + if (quote === '"' || quote === "'") { + const quoted = parseQuotedKey(line, index); + if (quoted === undefined) return "none"; + index = quoted.next; + continue; + } + index += 1; + } + return "none"; +}; + +/** Index after a multiline delimiter, or undefined when this line does not close it. */ +const closeMultiline = ( + line: string, + start: number, + delimiter: '"""' | "'''", + escapes: boolean, +): number | undefined => { + let index = start; + while (index < line.length) { + if (escapes && line[index] === "\\") { + if (line[index + 1] === undefined) return undefined; + index += 2; + continue; + } + if (line.startsWith(delimiter, index)) { + // One or two quotes before the closer belong to the string: """keep"""" + let end = index + delimiter.length; + const quote = delimiter[0]; + let extra = 0; + while (extra < 2 && line[end] === quote) { + extra += 1; + end += 1; + } + return end; + } + index += 1; + } + return undefined; +}; + +const isBlankOrComment = (line: string): boolean => { + const trimmed = line.trimStart(); + return trimmed === "" || trimmed.startsWith("#"); +}; + +const looksLikeTomlHeader = (line: string): boolean => + line.trimStart().startsWith("["); + +/** `mcp_servers.` and tables nested under it, compared by key segment. */ +const isGrokServerTable = ( + path: readonly string[], + serverKey: string, +): boolean => path[0] === "mcp_servers" && path[1] === serverKey; + +const parseKeyPath = ( + text: string, + start: number, +): + | { readonly segments: readonly string[]; readonly next: number } + | undefined => { + const segments: string[] = []; + let index = start; + for (;;) { + index = skipHorizontalSpace(text, index); + const current = text[index]; + if (current === undefined || current === "\n" || current === "\r") + return undefined; + const key = + current === '"' || current === "'" + ? parseQuotedKey(text, index) + : parseBareKey(text, index); + if (key === undefined) return undefined; + segments.push(key.value); + index = skipHorizontalSpace(text, key.next); + if (text[index] !== ".") return { segments, next: index }; + index += 1; + } +}; + +const parseAssignment = ( + line: string, +): + | { readonly segments: readonly string[]; readonly equals: number } + | undefined => { + const key = parseKeyPath(line, 0); + if (key === undefined) return undefined; + const equals = skipHorizontalSpace(line, key.next); + if (line[equals] !== "=") return undefined; + return { segments: key.segments, equals }; +}; + +const skipWsAndComments = (text: string, start: number): number => { + let index = start; + while (index < text.length) { + const char = text[index]; + if (char === " " || char === "\t" || char === "\n" || char === "\r") { + index += 1; + continue; + } + if (char === "#") { + while (index < text.length && text[index] !== "\n") index += 1; + continue; + } + return index; + } + return index; +}; + +/** Index after one TOML value. Containers and multiline strings may cross lines. */ +const endOfTomlValue = (text: string, start: number): number => { + const index = skipWsAndComments(text, start); + if (index >= text.length) return index; + if (text.startsWith('"""', index)) { + const closed = closeMultiline(text, index + 3, '"""', true); + return closed ?? text.length; + } + if (text.startsWith("'''", index)) { + const closed = closeMultiline(text, index + 3, "'''", false); + return closed ?? text.length; + } + const quote = text[index]; + if (quote === '"' || quote === "'") { + const quoted = parseQuotedKey(text, index); + return quoted === undefined ? text.length : quoted.next; + } + if (quote === "{" || quote === "[") return endOfContainer(text, index); + let cursor = index; + while (cursor < text.length) { + const char = text[cursor]; + if ( + char === "\n" || + char === "#" || + char === "," || + char === "}" || + char === "]" + ) + break; + cursor += 1; + } + return cursor; +}; + +const endOfContainer = (text: string, open: number): number => { + const closers: string[] = [text[open] === "{" ? "}" : "]"]; + let index = open + 1; + while (index < text.length && closers.length > 0) { + if (text.startsWith('"""', index)) { + const closed = closeMultiline(text, index + 3, '"""', true); + if (closed === undefined) return text.length; + index = closed; + continue; + } + if (text.startsWith("'''", index)) { + const closed = closeMultiline(text, index + 3, "'''", false); + if (closed === undefined) return text.length; + index = closed; + continue; + } + const quote = text[index]; + if (quote === '"' || quote === "'") { + const quoted = parseQuotedKey(text, index); + if (quoted === undefined) return text.length; + index = quoted.next; + continue; + } + if (quote === "#") { + while (index < text.length && text[index] !== "\n") index += 1; + continue; + } + if (quote === "{" || quote === "[") { + closers.push(quote === "{" ? "}" : "]"); + index += 1; + continue; + } + if (quote === closers[closers.length - 1]) { + closers.pop(); + index += 1; + continue; + } + index += 1; + } + return index; +}; + +/** TOML basic string. JSON leaves U+007F raw, which TOML rejects. */ +const tomlBasicString = (value: string): string => + JSON.stringify(value).replaceAll("\u007f", "\\u007f"); + +const tomlInlineKey = (key: string): string => + /^[A-Za-z0-9_-]+$/u.test(key) ? key : tomlBasicString(key); + +const tomlInlineValue = (value: unknown): string => { + if (typeof value === "string") return tomlBasicString(value); + if (typeof value === "boolean") return value ? "true" : "false"; + if (typeof value === "number" && Number.isFinite(value)) return String(value); + if (Array.isArray(value)) + return `[${value.map((item) => tomlInlineValue(item)).join(", ")}]`; + if (typeof value === "object" && value !== null) { + const body = Object.entries(value) + .map( + ([key, nested]) => `${tomlInlineKey(key)} = ${tomlInlineValue(nested)}`, + ) + .join(", "); + return `{ ${body} }`; + } + return '""'; +}; + +interface InlinePair { + readonly leading: string; + readonly text: string; + readonly drop: boolean; +} + +const readTrivia = ( + text: string, + start: number, + limit: number, +): { readonly text: string; readonly next: number } => { + let index = start; + while (index < limit) { + const char = text[index]; + if ( + char === " " || + char === "\t" || + char === "\n" || + char === "\r" || + char === "," + ) { + index += 1; + continue; + } + if (char === "#") { + while (index < limit && text[index] !== "\n") index += 1; + continue; + } + break; + } + return { text: text.slice(start, index), next: index }; +}; + +const commasOutsideComments = (trivia: string): string => + trivia + .split("\n") + .map((part) => { + const comment = part.indexOf("#"); + if (comment < 0) return part.replaceAll(",", ""); + return `${part.slice(0, comment).replaceAll(",", "")}${part.slice(comment)}`; + }) + .join("\n"); + +const hasSeparatorComma = (trivia: string): boolean => + trivia.split("\n").some((part) => { + const comment = part.indexOf("#"); + const code = comment < 0 ? part : part.slice(0, comment); + return code.includes(","); + }); + +/** + * Replace or remove `serverKey` inside one `{ ... }` table. A closed inline + * table cannot later grow a `[mcp_servers.rea]` header. + */ +const editGrokInlineTable = ( + table: string, + serverKey: string, + entry: unknown, +): string | undefined => { + if (!table.startsWith("{")) return undefined; + const end = endOfContainer(table, 0); + if (table[end - 1] !== "}") return undefined; + const pairs: InlinePair[] = []; + let index = 1; + let trailing = ""; + while (index < end - 1) { + const trivia = readTrivia(table, index, end - 1); + if (trivia.next >= end - 1) { + trailing = trivia.text; + break; + } + const key = parseKeyPath(table, trivia.next); + if (key === undefined) return undefined; + const equals = skipWsAndComments(table, key.next); + if (table[equals] !== "=") return undefined; + const valueEnd = endOfTomlValue(table, equals + 1); + if (valueEnd > end - 1) return undefined; + pairs.push({ + leading: trivia.text, + text: table.slice(trivia.next, valueEnd), + drop: key.segments[0] === serverKey, + }); + index = valueEnd; + } + const insertion = + entry === undefined + ? undefined + : `${tomlInlineKey(serverKey)} = ${tomlInlineValue(entry)}`; + if (insertion === undefined && pairs.every((pair) => !pair.drop)) + return table; + return rebuildInlineTable(pairs, trailing, insertion); +}; + +const rebuildInlineTable = ( + pairs: readonly InlinePair[], + trailing: string, + insertion: string | undefined, +): string => rebuildDelimitedEntries(pairs, trailing, insertion, "{", "}"); + +/** + * Rebuild `{ ... }` or `[ ... ]` after dropping entries. An empty container + * keeps comment trivia; a table with no comments stays `{}`. + */ +const rebuildDelimitedEntries = ( + pairs: readonly InlinePair[], + trailing: string, + insertion: string | undefined, + open: "{" | "[", + close: "}" | "]", +): string => { + const parts: string[] = [open]; + let emitted = false; + // A comma already emitted in a dropped pair's comment trivia. + let separatorPending = false; + for (const pair of pairs) { + if (pair.drop) { + if (!pair.leading.includes("#")) continue; + // A separator already emitted must not be repeated for the next field. + const dropped: string = + !emitted || separatorPending + ? commasOutsideComments(pair.leading) + : pair.leading; + parts.push(dropped); + separatorPending = separatorPending || hasSeparatorComma(dropped); + continue; + } + const kept = emitted ? pair.leading : commasOutsideComments(pair.leading); + parts.push(separatorPending ? commasOutsideComments(kept) : kept); + if (emitted && !separatorPending && !pair.leading.includes(",")) + parts.push(", "); + separatorPending = false; + parts.push(pair.text); + emitted = true; + } + if (insertion !== undefined) { + if (!(emitted && separatorPending)) parts.push(emitted ? ", " : " "); + parts.push(insertion); + emitted = true; + } + if (!emitted) { + const retained = `${parts.slice(1).join("")}${commasOutsideComments(trailing)}`; + return retained.includes("#") + ? `${open}${retained}${close}` + : `${open}${close}`; + } + const tail = commasOutsideComments(trailing); + parts.push(tail); + if (tail === "" && open === "{") parts.push(" "); + parts.push(close); + return parts.join(""); +}; + +/** + * Decoded TOML string, or undefined when the token is not a string. + * The document parser accepts multiline basic and literal strings. + */ +const tomlStringValue = (text: string): string | undefined => { + try { + const value = parseToml(`value = ${text}`)["value"]; + return typeof value === "string" ? value : undefined; + } catch (cause: unknown) { + void cause; + return undefined; + } +}; + +/** + * Array text with `serverKey` removed. `""` means the assignment itself can + * be deleted. Undefined when the array cannot be edited safely. + */ +const tomlArrayWithoutString = ( + array: string, + serverKey: string, +): string | undefined => { + if (!array.startsWith("[")) return undefined; + const end = endOfContainer(array, 0); + if (end !== array.length || array[end - 1] !== "]") return undefined; + const pairs: InlinePair[] = []; + let index = 1; + let trailing = ""; + while (index < end - 1) { + const trivia = readTrivia(array, index, end - 1); + if (trivia.next >= end - 1) { + trailing = trivia.text; + break; + } + const valueEnd = endOfTomlValue(array, trivia.next); + if (valueEnd > end - 1) return undefined; + const text = array.slice(trivia.next, valueEnd); + pairs.push({ + leading: trivia.text, + text, + drop: tomlStringValue(text) === serverKey, + }); + index = valueEnd; + } + if (pairs.every((pair) => !pair.drop)) return array; + const rebuilt = rebuildDelimitedEntries(pairs, trailing, undefined, "[", "]"); + return rebuilt === "[]" ? "" : rebuilt; +}; + +type DisabledServerListEdit = + | { readonly kind: "ignore" } + | { readonly kind: "keep" } + | { readonly kind: "delete"; readonly comment: string | undefined } + | { readonly kind: "replace"; readonly text: string }; + +interface DisabledServerListInput { + readonly text: string; + readonly lineStart: number; + readonly valueOffset: number; + readonly valueEnd: number; + readonly statementEnd: number; + readonly tablePath: readonly string[]; + readonly segments: readonly string[]; + readonly serverKey: string; + readonly entry: unknown; +} + +/** Drop `serverKey` from a root `disabled_mcp_servers` array while installing. */ +const disabledServerListEdit = ( + input: DisabledServerListInput, +): DisabledServerListEdit => { + if ( + input.entry === undefined || + input.tablePath.length !== 0 || + input.segments.length !== 1 || + input.segments[0] !== "disabled_mcp_servers" + ) + return { kind: "ignore" }; + const open = skipWsAndComments(input.text, input.valueOffset); + if (input.text[open] !== "[") return { kind: "ignore" }; + const arrayText = input.text.slice(open, input.valueEnd); + const edited = tomlArrayWithoutString(arrayText, input.serverKey); + if (edited === undefined || edited === arrayText) return { kind: "keep" }; + if (edited === "") { + const suffix = input.text.slice(input.valueEnd, input.statementEnd); + const hash = suffix.indexOf("#"); + return { + kind: "delete", + comment: hash < 0 ? undefined : suffix.slice(hash), + }; + } + return { + kind: "replace", + text: `${input.text.slice(input.lineStart, open)}${edited}${input.text.slice(input.valueEnd, input.statementEnd)}`, + }; +}; + +/** Lines that replace a disabled-server assignment, or undefined to leave it. */ +const disabledServerListLines = ( + edit: DisabledServerListEdit, + lines: readonly string[], + lineIndex: number, + endLine: number, +): readonly string[] | undefined => { + switch (edit.kind) { + case "ignore": + return undefined; + case "keep": + return lines.slice(lineIndex, endLine + 1); + case "replace": + return edit.text.split("\n"); + case "delete": + return edit.comment === undefined ? [] : [edit.comment]; + default: { + const unreachable: never = edit; + return unreachable; + } + } +}; + +/** + * Replace REA's Grok server tables and assignments. Comments that introduce a + * following table stay. Text inside strings is not treated as a header. + */ +const upsertGrokServerSection = ( + originalText: string, + serverKey: string, + entry: unknown, +): string => { + // smol-toml accepts a leading BOM. Leave it in place or the first header is missed. + const bom = originalText.startsWith("\uFEFF"); + const source = bom ? originalText.slice(1) : originalText; + // Keep a preceding \r on each line. Rewriting \r\n in the whole file would + // change string values. A file whose every break is CRLF gets a CRLF section. + const crlf = + source.includes("\r\n") && !source.replaceAll("\r\n", "").includes("\n"); + const text = source; + const lines = text.split("\n"); + const lineStarts: number[] = []; + let cursor = 0; + for (const line of lines) { + lineStarts.push(cursor); + cursor += line.length + 1; + } + const kept: string[] = []; + const pending: string[] = []; + let skipping = false; + let inlineServerTable = false; + let mode: TomlStringMode = "none"; + let tablePath: readonly string[] = []; + const discardPending = (): void => { + pending.length = 0; + }; + const flushPending = (): void => { + kept.push(...pending); + pending.length = 0; + }; + const lineIndexAt = (offset: number): number => { + let line = 0; + for (let index = 0; index < lineStarts.length; index += 1) + if ((lineStarts[index] ?? 0) <= offset) line = index; + return line; + }; + let lineIndex = 0; + while (lineIndex < lines.length) { + const physical = lines[lineIndex] ?? ""; + const line = physical.endsWith("\r") ? physical.slice(0, -1) : physical; + if (mode !== "none") { + mode = tomlStringModeAfter(line, mode); + if (skipping) discardPending(); + else kept.push(physical); + lineIndex += 1; + continue; + } + const header = parseTomlTableHeader(line); + if (header !== undefined) { + tablePath = header.path; + if (isGrokServerTable(header.path, serverKey)) { + discardPending(); + skipping = true; + lineIndex += 1; + continue; + } + skipping = false; + flushPending(); + kept.push(physical); + lineIndex += 1; + continue; + } + const assignment = parseAssignment(line); + if (assignment !== undefined) { + const lineStart = lineStarts[lineIndex] ?? 0; + const valueOffset = lineStart + assignment.equals + 1; + const valueEnd = endOfTomlValue(text, valueOffset); + let statementEnd = valueEnd; + while (statementEnd < text.length && text[statementEnd] !== "\n") + statementEnd += 1; + const endLine = lineIndexAt(Math.max(statementEnd - 1, lineStart)); + if (skipping) { + discardPending(); + lineIndex = endLine + 1; + mode = "none"; + continue; + } + const disabledLines = disabledServerListLines( + disabledServerListEdit({ + text, + lineStart, + valueOffset, + valueEnd, + statementEnd, + tablePath, + segments: assignment.segments, + serverKey, + entry, + }), + lines, + lineIndex, + endLine, + ); + if (disabledLines !== undefined) { + flushPending(); + kept.push(...disabledLines); + lineIndex = endLine + 1; + mode = "none"; + continue; + } + const absolute = [...tablePath, ...assignment.segments]; + const open = skipWsAndComments(text, valueOffset); + const closesServerTable = + absolute.length === 1 && + absolute[0] === "mcp_servers" && + text[open] === "{"; + if (closesServerTable || isGrokServerTable(absolute, serverKey)) { + if (closesServerTable) { + flushPending(); + const edited = editGrokInlineTable( + text.slice(open, valueEnd), + serverKey, + entry, + ); + const prefix = text.slice(lineStart, open); + const suffix = text.slice(valueEnd, statementEnd); + const replacement = + edited === undefined + ? text.slice(lineStart, statementEnd) + : `${prefix}${edited}${suffix}`; + kept.push(...replacement.split("\n")); + inlineServerTable = true; + } else discardPending(); + } else { + flushPending(); + for (let index = lineIndex; index <= endLine; index += 1) + kept.push(lines[index] ?? ""); + } + lineIndex = endLine + 1; + mode = "none"; + continue; + } + if (isBlankOrComment(line)) { + pending.push(physical); + lineIndex += 1; + continue; + } + mode = tomlStringModeAfter(line, "none"); + if (skipping) { + if (looksLikeTomlHeader(line)) { + flushPending(); + skipping = false; + tablePath = ["\0"]; + kept.push(physical); + } else discardPending(); + } else { + flushPending(); + kept.push(physical); + } + lineIndex += 1; + } + if (!skipping) flushPending(); + let body = kept.join("\n"); + // The last kept CRLF line has no following LF from the join. + if (body.endsWith("\r")) body += "\n"; + if (entry !== undefined && !inlineServerTable) { + let fragment = stringifyToml({ + mcp_servers: { [serverKey]: entry }, + }).replace(/\n+$/u, ""); + if (crlf) fragment = fragment.replaceAll("\n", "\r\n"); + const nl = crlf ? "\r\n" : "\n"; + const endedWithNewline = body.endsWith("\n"); + const base = body.replace(/\n+$/u, ""); + const trimmed = + endedWithNewline && base.endsWith("\r") ? base.slice(0, -1) : base; + body = + trimmed.length === 0 + ? `${fragment}${nl}` + : `${trimmed}${nl}${nl}${fragment}${nl}`; + } + return bom ? `\uFEFF${body}` : body; +}; + +const grokServerEntry = ( + document: Record, + serverKey: string, +): unknown => { + const servers = objectSchema.safeParse(document.mcp_servers); + if (!servers.success || !Object.hasOwn(servers.data, serverKey)) + return undefined; + return servers.data[serverKey]; +}; + +/** Whether Grok Build's root disable list names `serverKey`. */ +export const grokServerListedDisabled = ( + document: Record, + serverKey: string, +): boolean => + Array.isArray(document.disabled_mcp_servers) && + document.disabled_mcp_servers.includes(serverKey); + +/** Keep an existing Grok document intact aside from the REA server tables. */ +const serializeGrokConfiguration = ( + document: Record, + originalText: string | undefined, +): string => { + const entry = grokServerEntry(document, PRODUCT_IDENTITY.mcpServerKey); + if (originalText === undefined) return stringifyToml(document); + return upsertGrokServerSection( + originalText, + PRODUCT_IDENTITY.mcpServerKey, + entry, + ); +}; + +/** Comments in the trivia between two properties. `undefined` when there are none. */ +const commentsBetweenProperties = (gap: string): string | undefined => { + const pieces: string[] = []; + let index = 0; + while (index < gap.length) { + const line = gap.indexOf("//", index); + const block = gap.indexOf("/*", index); + if (line < 0 && block < 0) break; + const start = line < 0 ? block : block < 0 ? line : Math.min(line, block); + const lineComment = start === line; + const close = lineComment + ? gap.indexOf("\n", start) + : gap.indexOf("*/", start + 2); + if (!lineComment && close < 0) return undefined; + const end = lineComment ? (close < 0 ? gap.length : close) : close + 2; + const comment = gap.slice(start, end); + const lineBreak = gap.lastIndexOf("\n", start - 1); + if (lineBreak < 0) pieces.push(` ${comment}`); + else { + const prefix = gap.slice(lineBreak + 1, start); + let indentEnd = 0; + while (prefix[indentEnd] === " " || prefix[indentEnd] === "\t") + indentEnd += 1; + const newline = + lineBreak > 0 && gap[lineBreak - 1] === "\r" ? "\r\n" : "\n"; + pieces.push(`${newline}${prefix.slice(0, indentEnd)}${comment}`); + } + index = end; + } + return pieces.length === 0 ? undefined : pieces.join(""); +}; + +/** + * Delete an object property without taking a comment from the gap before it. + * jsonc-parser's removal span includes that gap, so a trailing comment on the + * previous property would disappear. Returns `undefined` to keep the stock edit. + */ +const deleteJsonPropertyPreservingComments = ( + text: string, + path: readonly string[], +): string | undefined => { + const root = parseTree(text, [], { allowTrailingComma: true }); + const value = + root === undefined ? undefined : findNodeAtLocation(root, [...path]); + const property = value?.parent; + const parent = property?.parent; + if ( + property === undefined || + parent?.type !== "object" || + property.type !== "property" || + parent.children === undefined + ) + return undefined; + const propertyIndex = parent.children.indexOf(property); + const previous = parent.children[propertyIndex - 1]; + if (propertyIndex <= 0 || previous === undefined) return undefined; + const comments = commentsBetweenProperties( + text.slice(previous.offset + previous.length, property.offset), + ); + if (comments === undefined) return undefined; + let removeEnd = property.offset + property.length; + const hasNext = parent.children[propertyIndex + 1] !== undefined; + if (hasNext) { + let comma = removeEnd; + while (comma < text.length) { + const char = text[comma]; + if (char === " " || char === "\t" || char === "\r" || char === "\n") { + comma += 1; + continue; + } + if (text.startsWith("/*", comma)) { + const close = text.indexOf("*/", comma + 2); + if (close < 0) return undefined; + comma = close + 2; + continue; + } + if (text.startsWith("//", comma)) { + while ( + comma < text.length && + text[comma] !== "\n" && + text[comma] !== "\r" + ) + comma += 1; + continue; + } + break; + } + if (text[comma] !== ",") return undefined; + removeEnd = comma + 1; + } + // A line comment runs to the next break. Keep the following token outside it. + const remainder = text.slice(removeEnd); + let suffix = `${hasNext ? "," : ""}${comments}`; + if ( + suffix.includes("//") && + !suffix.endsWith("\n") && + !suffix.endsWith("\r") && + !remainder.startsWith("\n") && + !remainder.startsWith("\r") + ) + suffix += "\n"; + return `${text.slice(0, previous.offset + previous.length)}${suffix}${remainder}`; +}; + /** Serialize a validated client document, retaining JSONC comments. */ export const serializeClientConfiguration = ( document: Record, @@ -188,17 +1146,22 @@ export const serializeClientConfiguration = ( if (format === undefined || format === "unsupported") throw new TypeError("client does not have a supported MCP config format"); if (format === "toml") return stringifyToml(document); + if (format === "grok") + return serializeGrokConfiguration(document, originalText); if (originalText !== undefined && editedPaths !== undefined) - return editedPaths.reduce( - (text, path) => - applyEdits( - text, - modify(text, [...path], valueAt(document, path), { - formattingOptions: { insertSpaces: true, tabSize: 2, eol: "\n" }, - }), - ), - originalText, - ); + return editedPaths.reduce((text, path) => { + const value = valueAt(document, path); + if (value === undefined) { + const preserved = deleteJsonPropertyPreservingComments(text, path); + if (preserved !== undefined) return preserved; + } + return applyEdits( + text, + modify(text, [...path], value, { + formattingOptions: { insertSpaces: true, tabSize: 2, eol: "\n" }, + }), + ); + }, originalText); return `${JSON.stringify(document, null, 2)}\n`; }; diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index 91acc429393..8c2194ac8ce 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -1,5 +1,6 @@ import { effectiveClientServer, + grokServerListedDisabled, parseClientConfiguration, } from "./ClientConfigurationDocument.js"; import { access, readFile } from "node:fs/promises"; @@ -13,7 +14,10 @@ import { isOwnedClientRegistrationCommand, npxRegistrationCommand, } from "./ClientRegistrationIdentity.js"; -import { supportedClients } from "./SupportedClients.js"; +import { + manualRegistrationRemediation, + supportedClients, +} from "./SupportedClients.js"; import type { SetupClient } from "./SupportedClients.js"; interface ClientRegistrationStatusBase { @@ -37,7 +41,7 @@ export type ClientRegistrationStatus = ClientRegistrationStatusBase & } | { readonly command: readonly []; - readonly state: "missing" | "invalid"; + readonly state: "missing" | "invalid" | "manual"; readonly remediation: string; } ); @@ -80,21 +84,33 @@ export const readClientRegistrationStatuses = async ( CLAUDE_CONFIG_DIR: options.environment.CLAUDE_CONFIG_DIR, CODEX_HOME: options.environment.CODEX_HOME, COPILOT_HOME: options.environment.COPILOT_HOME, + GROK_HOME: options.environment.GROK_HOME, OPENCODE_CONFIG: options.environment.OPENCODE_CONFIG, + SAND_DATA_ROOT: options.environment.SAND_DATA_ROOT, XDG_CONFIG_HOME: options.environment.XDG_CONFIG_HOME, }, )) { if ( - client.format === "unsupported" || - (!(await exists(client.markerPath)) && !(await exists(client.configPath))) + !(await exists(client.markerPath)) && + !(await exists(client.configPath)) ) continue; + const manualRemediation = manualRegistrationRemediation(client.name); + if (client.format === "unsupported") { + if (manualRemediation !== undefined) + statuses.push({ + client: client.name, + config_path: client.markerPath ?? client.configPath, + command: [], + state: "manual", + remediation: manualRemediation, + }); + continue; + } try { const content = await readFile(client.configPath, "utf8"); - const raw = effectiveClientServer( - parseClientConfiguration(content, client.format), - PRODUCT_IDENTITY.mcpServerKey, - ); + const parsed = parseClientConfiguration(content, client.format); + const raw = effectiveClientServer(parsed, PRODUCT_IDENTITY.mcpServerKey); if (raw === undefined) { statuses.push( unavailableStatus(client.name, client.configPath, "missing"), @@ -116,7 +132,14 @@ export const readClientRegistrationStatuses = async ( client, currentCommandPath, options.platform ?? process.platform, - ) + ) && + !( + client.format === "grok" && + grokServerListedDisabled( + parsed.document, + PRODUCT_IDENTITY.mcpServerKey, + ) + ) ? "aligned" : "stale", ), @@ -148,7 +171,7 @@ const registrationAligned = ( if (!isOwnedClientRegistrationCommand(command, currentCommandPath)) return false; if ( - client.name === "codex" && + (client.name === "codex" || client.name === "grok_build") && registration.startup_timeout_sec !== MCP_STARTUP_POLICY.codexStartupTimeoutSeconds ) diff --git a/src/application/Doctor.test.ts b/src/application/Doctor.test.ts index d6845245a19..5c8dbc10edf 100644 --- a/src/application/Doctor.test.ts +++ b/src/application/Doctor.test.ts @@ -286,7 +286,58 @@ describe("doctor scoped readiness", () => { ); expect(staleSelected.healthy).toBe(false); }); +}); + +describe("Grok Bot readiness", () => { + it("does not treat a Grok Bot data directory as an aligned registration", async () => { + const registrations: readonly ClientRegistrationStatus[] = [ + { + client: "grok_bot", + config_path: "/home/user/.grokbot", + command: [], + state: "manual", + remediation: "Add the connector from the Grok Bot chat.", + }, + ]; + const audit = await runDoctor( + undefined, + host({ clientRegistrations: () => Promise.resolve(registrations) }), + ); + expect(audit.checks.map(({ name }) => name)).not.toContain( + "registration:grok_bot", + ); + expect(audit.identity?.registrations).toEqual(registrations); + + const scoped = await runDoctor( + undefined, + host({ clientRegistrations: () => Promise.resolve(registrations) }), + { clients: ["grok_bot"] }, + ); + expect(scoped.healthy).toBe(false); + expect(scoped.scope_checks).toContainEqual( + expect.objectContaining({ + name: "registration:grok_bot", + ok: false, + remediation: "Add the connector from the Grok Bot chat.", + }), + ); + const absent = await runDoctor( + undefined, + host({ clientRegistrations: () => Promise.resolve([]) }), + { clients: ["grok_bot"] }, + ); + expect(absent.scope_checks).toContainEqual( + expect.objectContaining({ + name: "registration:grok_bot", + ok: false, + remediation: expect.stringContaining("hosted computer"), + }), + ); + }); +}); + +describe("doctor scoped readiness", () => { it("keeps agent readiness independent of Hopper host requirements", async () => { const result = await runDoctor( undefined, diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 078db0d73dc..9ff752f3a14 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -215,7 +215,7 @@ const collectDoctorIdentity = async ( ...registrations .filter( (registration): registration is UnhealthyClientRegistrationStatus => - registration.state !== "aligned", + registration.state !== "aligned" && registration.state !== "manual", ) .map(registrationCheck), ], diff --git a/src/application/DoctorScope.ts b/src/application/DoctorScope.ts index bd9af069b91..a92cd2863a2 100644 --- a/src/application/DoctorScope.ts +++ b/src/application/DoctorScope.ts @@ -3,6 +3,7 @@ import type { ClientRegistrationStatus, UnhealthyClientRegistrationStatus, } from "./ClientRegistrationStatus.js"; +import { manualRegistrationRemediation } from "./SupportedClients.js"; import type { DoctorCheck, DoctorProviderInspection, @@ -135,8 +136,17 @@ const selectedProviderChecksFor = ( const selectedRegistrationCheck = ( client: string, status: ClientRegistrationStatus | undefined, -): DoctorCheck => - status === undefined +): DoctorCheck => { + const manual = manualRegistrationRemediation(client); + if (status === undefined && manual !== undefined) + return { + name: `registration:${client}`, + ok: false, + classification: "config_drift", + detail: `${client} is not registered through a local configuration file.`, + remediation: manual, + }; + return status === undefined ? { name: `registration:${client}`, ok: false, @@ -152,6 +162,7 @@ const selectedRegistrationCheck = ( detail: status.config_path, } : registrationCheck(status); +}; const registrationCheck = ( status: UnhealthyClientRegistrationStatus, diff --git a/src/application/Setup.planning.test.ts b/src/application/Setup.planning.test.ts index dbcd96f6f31..41b2a717871 100644 --- a/src/application/Setup.planning.test.ts +++ b/src/application/Setup.planning.test.ts @@ -254,4 +254,36 @@ describe("setup workflow lifecycle and provider planning", () => { expect(host.configurations).toBe(0); expect(host.checkedHopperPaths).toEqual(["/manual/Hopper"]); }); + + it("does not write a file when Grok Bot is selected", async () => { + const host = new FakeSetupHost(); + host.doctorHealthy = true; + host.skill = "unchanged"; + host.clients = [ + { + name: "grok_bot", + displayName: "Grok Bot", + configPath: "/home/user/.grokbot", + format: "unsupported", + }, + ]; + + const planned = await runSetup( + { ...options(true), clientIds: ["grok_bot"], dryRun: true }, + host, + ); + expect(planned.status).toBe("planned"); + expect(planned.plannedActions).toEqual([]); + expect(planned.remediation).toContain("hosted computer"); + expect(host.configurations).toBe(0); + + const applied = await runSetup( + { ...options(true), clientIds: ["grok_bot"] }, + host, + ); + expect(applied.status).toBe("needs_human"); + expect(applied.appliedActions).toEqual([]); + expect(applied.remediation).toContain("hosted computer"); + expect(host.configurations).toBe(0); + }); }); diff --git a/src/application/Setup.ts b/src/application/Setup.ts index 9032d79e804..a22484efe34 100644 --- a/src/application/Setup.ts +++ b/src/application/Setup.ts @@ -11,7 +11,10 @@ import type { SetupProviderEnvironment, SetupResult, } from "./SetupTypes.js"; -import type { SetupClient } from "./SupportedClients.js"; +import { + manualRegistrationRemediation, + type SetupClient, +} from "./SupportedClients.js"; import type { DoctorReport, DoctorScope } from "./Doctor.js"; import { configureDetectedClients } from "./SetupClients.js"; import { discoverSetupState, planSetupActions } from "./SetupPlan.js"; @@ -163,6 +166,7 @@ export const runSetup = async ( planSelection, ); + const manualRemediation = manualClientsRemediation(planned.selectedClients); if (options.dryRun === true) return { status: "planned", @@ -171,6 +175,9 @@ export const runSetup = async ( clients, doctor: summarizeDoctor(discovery.initialDoctor), clientStates, + ...(manualRemediation === undefined + ? {} + : { remediation: manualRemediation }), }; let approved = options.approved || plannedActions.length === 0; @@ -250,11 +257,14 @@ export const runSetup = async ( "REA analysis skill could not be installed or verified. Check permissions for `~/.agents/skills`, then rerun setup.", ); const doctor = summarizeDoctor(await host.doctor(readinessScope)); - const remediation = finalSetupRemediation( - host.platform, - appliedActions.includes("installed_hopper"), - doctor.healthy, - hopperPath, + const remediation = combineRemediation( + finalSetupRemediation( + host.platform, + appliedActions.includes("installed_hopper"), + doctor.healthy, + hopperPath, + ), + manualRemediation, ); return { status: remediation === undefined ? "ready" : "needs_human", @@ -383,6 +393,22 @@ const setupFailure = async ( const emitProgress = (options: SetupOptions, event: SetupProgressEvent): void => options.onProgress?.(event); +const manualClientsRemediation = ( + clients: readonly SetupClient[], +): string | undefined => + clients + .map((client) => manualRegistrationRemediation(client.name)) + .find((remediation) => remediation !== undefined); + +const combineRemediation = ( + ...parts: readonly (string | undefined)[] +): string | undefined => { + const present = parts.filter( + (part): part is string => part !== undefined && part !== "", + ); + return present.length === 0 ? undefined : present.join(" "); +}; + const finalSetupRemediation = ( platform: NodeJS.Platform, installedHopper: boolean, diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index 7487e46bdcd..2d5343a58fd 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -2,6 +2,7 @@ import { clientRegistrationEntry, clientConfigurationValuesEqual, clientServerPath, + grokServerListedDisabled, legacyClientServerPath, parseClientConfiguration, serializeClientConfiguration, @@ -58,7 +59,7 @@ const configureClientDocument = async ( let parsed: ClientConfigurationDocument; try { parsed = parseClientConfiguration( - original ?? (format === "toml" ? "" : "{}"), + original ?? (format === "toml" || format === "grok" ? "" : "{}"), format, ); } catch (cause: unknown) { @@ -234,7 +235,7 @@ const restoreConfig = async ( } }; -/** Whether REA's entry matches and no conflicting legacy entry remains. */ +/** Whether REA's entry matches, no legacy entry remains, and Grok is not suppressing it. */ const registrationCurrent = ( parsed: ClientConfigurationDocument, desired: unknown, @@ -242,7 +243,12 @@ const registrationCurrent = ( clientConfigurationValuesEqual( parsed.servers[PRODUCT_IDENTITY.mcpServerKey], desired, - ) && !Object.hasOwn(parsed.legacyServers, PRODUCT_IDENTITY.mcpServerKey); + ) && + !Object.hasOwn(parsed.legacyServers, PRODUCT_IDENTITY.mcpServerKey) && + !( + parsed.dialect === "grok" && + grokServerListedDisabled(parsed.document, PRODUCT_IDENTITY.mcpServerKey) + ); const clientConfigurationDesired = ( client: SetupClient, @@ -262,7 +268,7 @@ const clientConfigurationDesired = ( ); return { ...registration, - ...(client.name === "codex" + ...(client.name === "codex" || client.name === "grok_build" ? { startup_timeout_sec: MCP_STARTUP_POLICY.codexStartupTimeoutSeconds, } diff --git a/src/application/SetupHost.ts b/src/application/SetupHost.ts index b4a8a494463..29416428670 100644 --- a/src/application/SetupHost.ts +++ b/src/application/SetupHost.ts @@ -140,9 +140,11 @@ export const systemSetupHost = ( ? Promise.resolve({ status: "skipped" }) : configureClientConfiguration(client, providerEnvironment, command), clientNeedsConfigure: (client, providerEnvironment, command) => - clientConfigurationAligned(client, providerEnvironment, command).then( - (aligned) => !aligned, - ), + client.format === "unsupported" + ? Promise.resolve(false) + : clientConfigurationAligned(client, providerEnvironment, command).then( + (aligned) => !aligned, + ), inspectClientConfiguration: inspectClientConfiguration, skillNeedsInstall: () => canonicalSkillNeedsInstall(homedir()), installSkill: () => installCanonicalSkill(homedir()), diff --git a/src/application/SetupPlan.ts b/src/application/SetupPlan.ts index fcea6271d91..9891cf322bf 100644 --- a/src/application/SetupPlan.ts +++ b/src/application/SetupPlan.ts @@ -168,9 +168,11 @@ export const planSetupActions = async (input: { }; } const clientPlans = inspections.flatMap(({ client, inspection }) => - inspection.status === "create" || - inspection.status === "update" || - (input.discovery.installHopper && inspection.status === "already_current") + client.format !== "unsupported" && + (inspection.status === "create" || + inspection.status === "update" || + (input.discovery.installHopper && + inspection.status === "already_current")) ? [ { client, diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 399efb926b4..7db4cd42565 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -1,4 +1,4 @@ -import { join } from "node:path"; +import { isAbsolute, join } from "node:path"; import { lstatSync } from "node:fs"; /** One supported client configuration location. */ @@ -14,6 +14,7 @@ export interface SetupClient { | "copilot_cli" | "opencode" | "commandcode" + | "grok" | "unsupported"; } @@ -27,7 +28,9 @@ interface ClientPathContext { readonly CLAUDE_CONFIG_DIR?: string | undefined; readonly CODEX_HOME?: string | undefined; readonly COPILOT_HOME?: string | undefined; + readonly GROK_HOME?: string | undefined; readonly OPENCODE_CONFIG?: string | undefined; + readonly SAND_DATA_ROOT?: string | undefined; readonly XDG_CONFIG_HOME?: string | undefined; }; } @@ -77,6 +80,32 @@ const claudeCodeMarkerDirectory = ({ home, env }: ClientPathContext): string => const codexDirectory = ({ home, env }: ClientPathContext): string => env.CODEX_HOME ?? join(home, ".codex"); +const grokDirectory = ({ home, env }: ClientPathContext): string => + env.GROK_HOME ?? join(home, ".grok"); + +/** Grok Bot uses an absolute SAND_DATA_ROOT; anything else stays ~/.grokbot. */ +const grokBotDirectory = ({ home, env }: ClientPathContext): string => { + const root = env.SAND_DATA_ROOT; + return root !== undefined && root !== "" && isAbsolute(root) + ? root + : join(home, ".grokbot"); +}; + +/** + * Grok Bot stores connectors in the signed-in account and runs them on its + * hosted computer. The data directory is only a detection marker. + */ +export const GROK_BOT_MANUAL_REGISTRATION_REMEDIATION = + "Grok Bot keeps connectors in the signed-in account and runs them on its hosted computer. Setup does not write that account store, and a data-directory mcp.json is not a registration. In the Grok Bot chat, add a custom MCP server named rea that runs on the Bot's computer with `npx -y rea-agents@ mcp`. Do not put credentials in the command or arguments. A stdio server on this machine is not attached."; + +/** Remediation for a client whose connector is not a local configuration file. */ +export const manualRegistrationRemediation = ( + clientName: string, +): string | undefined => + clientName === "grok_bot" + ? GROK_BOT_MANUAL_REGISTRATION_REMEDIATION + : undefined; + const copilotDirectory = ({ home, env }: ClientPathContext): string => env.COPILOT_HOME ?? join(home, ".copilot"); @@ -212,6 +241,21 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ markerPath: vscodeUserDirectory, format: "vscode", }, + { + name: "grok_build", + displayName: "Grok Build", + configPath: (context: ClientPathContext) => + join(grokDirectory(context), "config.toml"), + markerPath: grokDirectory, + format: "grok", + }, + { + name: "grok_bot", + displayName: "Grok Bot", + configPath: grokBotDirectory, + markerPath: grokBotDirectory, + format: "unsupported", + }, ] as const satisfies readonly ClientDefinition[]; const resolvePath = (path: ClientPath, context: ClientPathContext): string => @@ -226,7 +270,9 @@ export const supportedClients = ( CLAUDE_CONFIG_DIR: process.env.CLAUDE_CONFIG_DIR, CODEX_HOME: process.env.CODEX_HOME, COPILOT_HOME: process.env.COPILOT_HOME, + GROK_HOME: process.env.GROK_HOME, OPENCODE_CONFIG: process.env.OPENCODE_CONFIG, + SAND_DATA_ROOT: process.env.SAND_DATA_ROOT, XDG_CONFIG_HOME: process.env.XDG_CONFIG_HOME, }, ): readonly SetupClient[] => { diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index 6482094886d..d0203ae2f50 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -16,7 +16,11 @@ import { z } from "zod"; import { PRODUCT_IDENTITY } from "../identity.js"; import { isOwnedClientRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js"; -import { supportedClients, type SetupClient } from "./SupportedClients.js"; +import { + manualRegistrationRemediation, + supportedClients, + type SetupClient, +} from "./SupportedClients.js"; interface ManagedPathStats { readonly uid?: number; @@ -127,7 +131,8 @@ const removeClient = async ( return item( client.name, "skipped", - "This client has no documented local MCP configuration boundary.", + manualRegistrationRemediation(client.name) ?? + "This client has no documented local MCP configuration boundary.", ); const resolved = await resolveUninstallConfigPath(client, fileSystem); if (typeof resolved !== "string") return resolved; @@ -152,7 +157,7 @@ const removeClient = async ( return item( client.name, "failed", - `Configuration is not valid ${client.format === "toml" ? "TOML" : client.format === "opencode" ? "JSONC" : "JSON"} and was not changed. Repair it, then rerun uninstall.`, + `Configuration is not valid ${client.format === "toml" || client.format === "grok" ? "TOML" : client.format === "opencode" ? "JSONC" : "JSON"} and was not changed. Repair it, then rerun uninstall.`, ); } const name = PRODUCT_IDENTITY.mcpServerKey; diff --git a/tests/boundary/filesystem/clientConfigurationDialects.test.ts b/tests/boundary/filesystem/clientConfigurationDialects.test.ts index 32db9c7570d..afea88e6f2c 100644 --- a/tests/boundary/filesystem/clientConfigurationDialects.test.ts +++ b/tests/boundary/filesystem/clientConfigurationDialects.test.ts @@ -25,7 +25,9 @@ beforeEach(() => { "CLAUDE_CONFIG_DIR", "CODEX_HOME", "COPILOT_HOME", + "GROK_HOME", "OPENCODE_CONFIG", + "SAND_DATA_ROOT", "XDG_CONFIG_HOME", ]) vi.stubEnv(name, undefined); @@ -398,6 +400,40 @@ describe("platform-aware client config paths", () => { }, ); + it("honors Grok Build and Grok Bot configuration roots", () => { + expect( + supportedClients("/home/a", "linux", { + GROK_HOME: "/custom/grok", + SAND_DATA_ROOT: "/custom/grokbot", + }).filter(({ name }) => name === "grok_build" || name === "grok_bot"), + ).toEqual([ + expect.objectContaining({ + name: "grok_build", + displayName: "Grok Build", + configPath: "/custom/grok/config.toml", + markerPath: "/custom/grok", + format: "grok", + }), + expect.objectContaining({ + name: "grok_bot", + displayName: "Grok Bot", + configPath: "/custom/grokbot", + markerPath: "/custom/grokbot", + format: "unsupported", + }), + ]); + expect( + supportedClients("/home/a", "linux", { + SAND_DATA_ROOT: "relative/sand", + }).find(({ name }) => name === "grok_bot")?.configPath, + ).toBe("/home/a/.grokbot"); + expect( + supportedClients("/home/a", "darwin", {}).find( + ({ name }) => name === "grok_build", + )?.configPath, + ).toBe("/home/a/.grok/config.toml"); + }); + it("honors Copilot CLI and Devin platform configuration roots", () => { expect( supportedClients("/home/a", "linux", { diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 1a25e810089..8ea5a99454b 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -9,13 +9,19 @@ import { writeFile, } from "node:fs/promises"; import { dirname, join } from "node:path"; +import { parse as parseToml } from "smol-toml"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { parse as parseJsonc, type ParseError } from "jsonc-parser"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { supportedClients } from "../../../src/application/SupportedClients.js"; +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; -import { detectClients } from "../../../src/application/SetupHost.js"; +import { + detectClients, + systemSetupHost, +} from "../../../src/application/SetupHost.js"; import { runUninstall, systemUninstallHost, @@ -29,7 +35,9 @@ beforeEach(() => { "CLAUDE_CONFIG_DIR", "CODEX_HOME", "COPILOT_HOME", + "GROK_HOME", "OPENCODE_CONFIG", + "SAND_DATA_ROOT", "XDG_CONFIG_HOME", ]) vi.stubEnv(name, undefined); @@ -62,6 +70,8 @@ describe("client configuration filesystem lifecycle", () => { "copilot_cli", "commandcode", "vscode", + "grok_build", + "grok_bot", ]); expect( detected.find(({ name }) => name === "claude_code")?.configPath, @@ -183,6 +193,1123 @@ describe("client configuration filesystem lifecycle", () => { }); }); +describe("Grok Build server table edits", () => { + it("edits only the Grok Build REA tables and removes them on uninstall", async () => { + const home = await createTestTempDirectory("rea-grok-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + "# keep this comment", + 'disabled_mcp_servers = ["xapi"]', + "", + "[cli]", + 'default_model = "grok-4"', + "", + "[mcp_servers.blender]", + 'command = "blender"', + 'args = ["--mcp"]', + "", + "[mcp_servers.rea]", + 'command = "old"', + 'args = ["mcp"]', + "", + "[mcp_servers.rea.env]", + 'OLD = "1"', + "", + "[[marketplace.sources]]", + 'name = "official"', + 'url = "https://example.test/market"', + "", + "[privacy]", + "share = false", + "", + ].join("\r\n"); + await writeFile(configPath, original); + const environment = { + HOPPER_LAUNCHER_PATH: "/Hopper Path", + GHIDRA_INSTALL_DIR: "/opt/ghidra", + JAVA_HOME: "/opt/jdk-21", + }; + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + environment, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("\r\n"); + expect(configured).toContain("# keep this comment"); + expect(configured).toContain('disabled_mcp_servers = ["xapi"]'); + expect(configured).toContain('default_model = "grok-4"'); + expect(configured).toContain("[mcp_servers.blender]"); + expect(configured).toContain('command = "blender"'); + expect(configured).toContain("[[marketplace.sources]]"); + expect(configured).toContain('name = "official"'); + expect(configured).toContain('url = "https://example.test/market"'); + expect(configured).toContain("[privacy]"); + expect(configured).toContain("share = false"); + expect(configured).toContain("[mcp_servers.rea]"); + expect(configured).toContain("startup_timeout_sec = 30"); + expect(configured).not.toContain('OLD = "1"'); + expect(configured).not.toContain('command = "old"'); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + blender: { command: "blender", args: ["--mcp"] }, + rea: { + command: "rea", + args: ["mcp"], + startup_timeout_sec: 30, + env: { + JAVA_HOME: "/opt/jdk-21", + HOPPER_LAUNCHER_PATH: "/Hopper Path", + GHIDRA_INSTALL_DIR: "/opt/ghidra", + }, + }, + }, + marketplace: { sources: [{ name: "official" }] }, + privacy: { share: false }, + }); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + environment, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + expect(await readFile(configPath, "utf8")).toBe(configured); + + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("# keep this comment"); + expect(removed).toContain("[mcp_servers.blender]"); + expect(removed).toContain("[[marketplace.sources]]"); + expect(removed).toContain('url = "https://example.test/market"'); + expect(removed).toContain("[privacy]"); + expect(removed).not.toContain("[mcp_servers.rea]"); + expect(removed).not.toContain("startup_timeout_sec"); + expect(removed).not.toContain("JAVA_HOME"); + }); +}); + +describe("Grok Build disabled server list", () => { + it("removes rea from disabled_mcp_servers and leaves other names", async () => { + const home = await createTestTempDirectory("rea-grok-disabled-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + "disabled_mcp_servers = [", + ' "other",', + ' "r\\u0065a",', + ' "rea", # was off', + ' "github",', + "]", + "[mcp_servers.rea]", + "enabled = false", + 'command = "rea"', + 'args = ["mcp"]', + "startup_timeout_sec = 30", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("# was off"); + expect(configured).not.toContain("enabled"); + expect(parseToml(configured)).toMatchObject({ + disabled_mcp_servers: ["other", "github"], + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + await writeFile(configPath, 'disabled_mcp_servers = ["rea"] # note\n'); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const created = await readFile(configPath, "utf8"); + expect(created).toContain("# note"); + expect(created).not.toContain("disabled_mcp_servers"); + expect(parseToml(created)).toMatchObject({ + mcp_servers: { rea: { command: "rea", args: ["mcp"] } }, + }); + }); + + it("removes multiline rea entries from disabled_mcp_servers", async () => { + const home = await createTestTempDirectory("rea-grok-disabled-multiline-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + "disabled_mcp_servers = [", + ' """rea""",', + " '''rea''',", + ' """', + 'rea""",', + ' """rea', + '""",', + ' "github",', + "]", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(parseToml(configured)).toMatchObject({ + disabled_mcp_servers: ["rea\n", "github"], + mcp_servers: { rea: { command: "rea", args: ["mcp"] } }, + }); + await writeFile(configPath, 'disabled_mcp_servers = ["""rea"""] # note\n'); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const created = await readFile(configPath, "utf8"); + expect(created).toContain("# note"); + expect(created).not.toContain("disabled_mcp_servers"); + }); +}); + +describe("Grok Build text that resembles a server table", () => { + it("preserves Grok text that only looks like an REA table", async () => { + const home = await createTestTempDirectory("rea-grok-preserve-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + "banner = '''", + "welcome", + "[mcp_servers.rea]", + "keep this text", + "[privacy]", + "'''", + "", + "[mcp_servers.rea]", + 'command = "old"', + 'args = ["mcp"]', + "", + "# Production server: preserve this configuration", + '[mcp_servers."rea.env"]', + 'command = "other"', + 'args = ["serve"]', + "", + '[mcp_servers."other[prod]"]', + 'command = "other"', + 'args = ["serve"]', + "", + '[[marketplace."sources[prod]"]]', + 'name = "official"', + "", + ].join("\n"); + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("keep this text"); + expect(configured).toContain( + "# Production server: preserve this configuration", + ); + expect(configured).toContain('[mcp_servers."rea.env"]'); + expect(configured).toContain('[mcp_servers."other[prod]"]'); + expect(configured).toContain('[[marketplace."sources[prod]"]]'); + expect(configured).not.toContain('command = "old"'); + expect(parseToml(configured)).toMatchObject({ + banner: "welcome\n[mcp_servers.rea]\nkeep this text\n[privacy]\n", + mcp_servers: { + "rea.env": { command: "other", args: ["serve"] }, + "other[prod]": { command: "other", args: ["serve"] }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + marketplace: { "sources[prod]": [{ name: "official" }] }, + }); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("keep this text"); + expect(removed).toContain( + "# Production server: preserve this configuration", + ); + expect(removed).toContain('[mcp_servers."other[prod]"]'); + expect(parseToml(removed)).toMatchObject({ + mcp_servers: { + "rea.env": { command: "other" }, + "other[prod]": { command: "other" }, + }, + }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); + + it("replaces spaced and escaped Grok REA headers without duplicating them", async () => { + const home = await createTestTempDirectory("rea-grok-alias-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + "[ mcp_servers . rea ]", + 'command = "old"', + 'args = ["mcp"]', + "", + '[mcp_servers."\\u0072ea".env]', + 'OLD = "1"', + "", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).not.toContain("mcp_servers . rea"); + expect(configured).not.toContain("\\u0072ea"); + expect(configured).not.toContain('OLD = "1"'); + expect( + configured + .split("\n") + .filter((line) => line.trim() === "[mcp_servers.rea]"), + ).toHaveLength(1); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); +}); + +describe("Grok Build multiline and escaped headers", () => { + it("preserves a multiline string opened on the line that closed the previous one", async () => { + const home = await createTestTempDirectory("rea-grok-multiline-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + 'banner = ["""', + "first", + '""", """', + "[mcp_servers.rea]", + 'command = "keep this text"', + "[privacy]", + "keep this too", + '"""]', + "", + "marker = ['''", + "first", + "''', '''", + "[mcp_servers.rea]", + 'command = "keep literal"', + "''']", + "", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain('command = "keep this text"'); + expect(configured).toContain("keep this too"); + expect(configured).toContain('command = "keep literal"'); + expect(parseToml(configured)).toMatchObject({ + banner: [ + "first\n", + '[mcp_servers.rea]\ncommand = "keep this text"\n[privacy]\nkeep this too\n', + ], + marker: ["first\n", '[mcp_servers.rea]\ncommand = "keep literal"\n'], + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + }); + + it("replaces a hex-escaped Grok REA header and keeps a hex-escaped foreign name", async () => { + const home = await createTestTempDirectory("rea-grok-hex-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + '[mcp_servers."\\x72ea"]', + 'command = "rea"', + 'args = ["mcp"]', + "", + '[mcp_servers."rea\\x2Eenv"]', + 'command = "other"', + 'args = ["serve"]', + "", + ].join("\n"); + await writeFile(configPath, original); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain('[mcp_servers."rea\\x2Eenv"]'); + expect(removed).not.toContain("\\x72"); + expect(parseToml(removed)).toMatchObject({ + mcp_servers: { "rea.env": { command: "other", args: ["serve"] } }, + }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).not.toContain("\\x72"); + expect(configured).toContain('[mcp_servers."rea\\x2Eenv"]'); + expect( + configured + .split("\n") + .filter((line) => line.trim() === "[mcp_servers.rea]"), + ).toHaveLength(1); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + "rea.env": { command: "other", args: ["serve"] }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); +}); + +describe("Grok Build inline and dotted assignments", () => { + it("updates and removes inline and dotted Grok REA assignments", async () => { + const home = await createTestTempDirectory("rea-grok-assign-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const inline = [ + "[mcp_servers]", + 'rea = """', + "[mcp_servers.rea]", + 'command = "keep? no, this is the old value"', + '"""', + 'other = { command = "other" }', + "", + "# next table", + "[privacy]", + "share = false", + "", + ].join("\n"); + await writeFile(configPath, inline); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configuredInline = await readFile(configPath, "utf8"); + expect(configuredInline).not.toContain("keep? no, this is the old value"); + expect(configuredInline).toContain('other = { command = "other" }'); + expect(configuredInline).toContain("# next table"); + expect(configuredInline).toContain("[privacy]"); + expect(parseToml(configuredInline)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + privacy: { share: false }, + }); + expect(configuredInline).not.toContain("enabled"); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removedInline = await readFile(configPath, "utf8"); + expect(removedInline).toContain('other = { command = "other" }'); + expect(removedInline).toContain("# next table"); + expect(parseToml(removedInline)).not.toHaveProperty("mcp_servers.rea"); + + const dotted = [ + 'mcp_servers.rea = { command = "rea", args = ["mcp"] }', + 'mcp_servers.other.command = "other"', + "", + "# next table", + "[privacy]", + "share = false", + "", + ].join("\n"); + await writeFile(configPath, dotted); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removedDotted = await readFile(configPath, "utf8"); + expect(removedDotted).toContain('mcp_servers.other.command = "other"'); + expect(removedDotted).toContain("# next table"); + expect(parseToml(removedDotted)).toMatchObject({ + mcp_servers: { other: { command: "other" } }, + privacy: { share: false }, + }); + expect(parseToml(removedDotted)).not.toHaveProperty("mcp_servers.rea"); + + await writeFile( + configPath, + [ + "[mcp_servers]", + 'rea.command = "old"', + 'rea.args = ["old"]', + 'other.command = "other"', + "", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configuredDotted = await readFile(configPath, "utf8"); + expect(configuredDotted).toContain('other.command = "other"'); + expect(configuredDotted).not.toContain('rea.command = "old"'); + expect(parseToml(configuredDotted)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); +}); + +describe("Grok Build closed inline tables", () => { + it("splices REA into a closed mcp_servers inline table", async () => { + const home = await createTestTempDirectory("rea-grok-inline-table-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + "# preamble", + 'mcp_servers = { other = { command = "other" }, "\\x72ea" = { command = "old", args = ["old"] } }', + "", + "# next table", + "[privacy]", + "share = false", + "", + ].join("\n"); + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("# preamble"); + expect(configured).toContain("# next table"); + expect(configured).toContain("[privacy]"); + expect(configured).toContain("mcp_servers ="); + expect(configured).not.toMatch(/^\[mcp_servers\.rea\]$/mu); + expect(configured).not.toContain("\\x72"); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + privacy: { share: false }, + }); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toEqual({ status: "unchanged" }); + + await writeFile( + configPath, + [ + 'mcp_servers = { other = { command = "other" }, rea = { command = "rea", args = ["mcp"] } }', + "", + ].join("\n"), + ); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain('other = { command = "other" }'); + expect(removed).toContain("mcp_servers ="); + expect(parseToml(removed)).toMatchObject({ + mcp_servers: { other: { command: "other" } }, + }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + + await writeFile(configPath, "mcp_servers = { }\n"); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const created = await readFile(configPath, "utf8"); + expect(created).toContain("mcp_servers ="); + expect(created).not.toMatch(/^\[mcp_servers\.rea\]$/mu); + expect(parseToml(created)).toMatchObject({ + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); + + it("keeps a quote that belongs to a multiline string closer", async () => { + const home = await createTestTempDirectory("rea-grok-quotes-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + 'banner = ["""keep""""]', + "marker = ['''keep'''']", + "[mcp_servers.rea]", + 'command = "rea"', + 'args = ["mcp"]', + "", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(parseToml(configured)).toMatchObject({ + banner: ['keep"'], + marker: ["keep'"], + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + expect( + configured + .split("\n") + .filter((line) => line.trim() === "[mcp_servers.rea]"), + ).toHaveLength(1); + }); +}); + +describe("Grok Build scalar and nested values", () => { + it("preserves a basic string that uses the escape smol-toml accepts", async () => { + const home = await createTestTempDirectory("rea-grok-escape-e-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + [ + 'banner = "\\e[31m"', + "[mcp_servers.rea]", + 'command = "rea"', + 'args = ["mcp"]', + "", + ].join("\n"), + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain('banner = "\\e[31m"'); + expect(parseToml(configured)).toMatchObject({ + banner: "\u001b[31m", + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); + + it("updates a closed inline table whose last value is a scalar", async () => { + const home = await createTestTempDirectory("rea-grok-scalar-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + 'mcp_servers = { rea.command = "rea", rea.args = ["mcp"], rea.startup_timeout_sec = 10 }\n', + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).not.toMatch(/^\[mcp_servers\.rea\]$/mu); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); + + it("removes a nested array inside a Grok REA table", async () => { + const home = await createTestTempDirectory("rea-grok-nested-array-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + "[mcp_servers.rea]", + 'command = "rea"', + 'args = ["mcp"]', + "extra = [", + " [1, 2],", + "]", + "", + "[privacy]", + "share = false", + "", + ].join("\n"); + await writeFile(configPath, original); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("[privacy]"); + expect(removed).not.toContain("[1, 2]"); + expect(parseToml(removed)).toMatchObject({ privacy: { share: false } }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("[privacy]"); + expect(configured).not.toContain("[1, 2]"); + expect(parseToml(configured)).toMatchObject({ + privacy: { share: false }, + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); +}); + +describe("Grok Build newlines and leading marks", () => { + it("edits a BOM-prefixed Grok document without losing the first table", async () => { + const home = await createTestTempDirectory("rea-grok-bom-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + '\uFEFF[profile]\nmcp_servers.rea = "keep this setting"\n', + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured.startsWith("\uFEFF")).toBe(true); + expect(configured).toContain('mcp_servers.rea = "keep this setting"'); + expect(parseToml(configured)).toMatchObject({ + profile: { mcp_servers: { rea: "keep this setting" } }, + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + + await writeFile( + configPath, + '\uFEFF[mcp_servers.rea]\ncommand = "rea"\nargs = ["mcp"]\n', + ); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed.startsWith("\uFEFF")).toBe(true); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); + + it("keeps CRLF string values in a Grok file that also uses LF", async () => { + const home = await createTestTempDirectory("rea-grok-crlf-string-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = + 'banner = """a\r\nb"""\n[mcp_servers.rea]\ncommand = "rea"\nargs = ["mcp"]\n'; + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + expect(parseToml(await readFile(configPath, "utf8"))).toMatchObject({ + banner: "a\r\nb", + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + + await writeFile(configPath, original); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(parseToml(removed)).toMatchObject({ banner: "a\r\nb" }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); + + it("keeps LF string values in a Grok file that also uses CRLF", async () => { + const home = await createTestTempDirectory("rea-grok-mixed-newline-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + 'banner = """keep\nthis LF"""\n[mcp_servers.rea]\r\ncommand = "rea"\r\nargs = ["mcp"]\r\n', + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + expect(parseToml(await readFile(configPath, "utf8"))).toMatchObject({ + banner: "keep\nthis LF", + mcp_servers: { + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + }); +}); + +describe("Grok Build inline comments", () => { + it("keeps comments on dotted Grok fields without repeating a comma", async () => { + const home = await createTestTempDirectory("rea-grok-dotted-comment-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = [ + 'mcp_servers = { other = {command="other"}, # keep other note', + ' rea.command = "rea", # command note', + ' rea.args = ["mcp"] }', + "", + ].join("\n"); + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("# keep other note"); + expect(configured).toContain("# command note"); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + + await writeFile(configPath, original); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("# keep other note"); + expect(removed).toContain("# command note"); + expect(parseToml(removed)).toMatchObject({ + mcp_servers: { other: { command: "other" } }, + }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); + + it("keeps an inline comment that sits before the Grok REA pair", async () => { + const home = await createTestTempDirectory("rea-grok-inline-before-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + const original = + 'mcp_servers = { other = {command="other"}, # important other-server note\n rea = {command="rea",args=["mcp"]} }\n'; + await writeFile(configPath, original); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + undefined, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("# important other-server note"); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { command: "rea", args: ["mcp"], startup_timeout_sec: 30 }, + }, + }); + + await writeFile(configPath, original); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("# important other-server note"); + expect(parseToml(removed)).toMatchObject({ + mcp_servers: { other: { command: "other" } }, + }); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); + + it("keeps inline-table comments and encodes a DEL in an inline value", async () => { + const home = await createTestTempDirectory("rea-grok-inline-comment-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + 'mcp_servers = { rea = {command="rea",args=["mcp"]}, # other: one, two\n other = {command="other"} }\n', + ); + expect( + await configureClientConfiguration( + { name: "grok_build", configPath, format: "grok" }, + { HOPPER_LAUNCHER_PATH: "/tmp/path\u007fwith-del" }, + ["rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured).toContain("# other: one, two"); + expect(parseToml(configured)).toMatchObject({ + mcp_servers: { + other: { command: "other" }, + rea: { + command: "rea", + args: ["mcp"], + startup_timeout_sec: 30, + env: { HOPPER_LAUNCHER_PATH: "/tmp/path\u007fwith-del" }, + }, + }, + }); + }); + + it("keeps comments when uninstall removes the only inline Grok server", async () => { + const home = await createTestTempDirectory("rea-grok-inline-empty-"); + roots.push(home); + const configPath = join(home, ".grok/config.toml"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + 'mcp_servers = { # policy\n rea = { command = "rea", args = ["mcp"] } # owned\n}\n', + ); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("# policy"); + expect(removed).toContain("# owned"); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + }); +}); + +describe("JSON client configuration comments", () => { + it("keeps a same-line JSONC comment when uninstall removes Cursor", async () => { + const home = await createTestTempDirectory("rea-cursor-jsonc-comment-"); + roots.push(home); + const configPath = join(home, ".cursor/mcp.json"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + '{\n "mcpServers": {\n "other": {"command":"other"}, // important other-server note\n "rea": {"command":"rea","args":["mcp"]}\n }\n}\n', + ); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("// important other-server note"); + const errors: ParseError[] = []; + const document = parseJsonc(removed, errors, { allowTrailingComma: true }); + expect(errors).toEqual([]); + expect(document).toEqual({ + mcpServers: { other: { command: "other" } }, + }); + }); + + it("keeps a JSONC comment when a block comment precedes the comma", async () => { + const home = await createTestTempDirectory("rea-cursor-jsonc-block-"); + roots.push(home); + const configPath = join(home, ".cursor/mcp.json"); + await mkdir(dirname(configPath), { recursive: true }); + await writeFile( + configPath, + '{"mcpServers":{\n "other":{"command":"other"}, // keep other note\n "rea":{"command":"rea","args":["mcp"]} /**/,\n "keep":{"command":"keep"}\n}}\n', + ); + expect((await runUninstall(false, systemUninstallHost(home))).status).toBe( + "complete", + ); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("// keep other note"); + const errors: ParseError[] = []; + const document = parseJsonc(removed, errors, { allowTrailingComma: true }); + expect(errors).toEqual([]); + expect(document).toEqual({ + mcpServers: { + other: { command: "other" }, + keep: { command: "keep" }, + }, + }); + }); + + it("keeps a JSONC comment without swallowing the next server or closers", async () => { + const home = await createTestTempDirectory("rea-cursor-jsonc-next-"); + roots.push(home); + const configPath = join(home, ".cursor/mcp.json"); + await mkdir(dirname(configPath), { recursive: true }); + const files = [ + '{\n "mcpServers": {\n "other": {"command":"other"}, // important other-server note\n "rea": {"command":"rea","args":["mcp"]}, "keep": {"command":"keep"}\n }\n}\n', + '{\n "mcpServers": {\n "other": {"command":"other"}, // important other-server note\n "rea": {"command":"rea","args":["mcp"]}}}\n', + ]; + for (const original of files) { + await writeFile(configPath, original); + expect( + (await runUninstall(false, systemUninstallHost(home))).status, + ).toBe("complete"); + const removed = await readFile(configPath, "utf8"); + expect(removed).toContain("// important other-server note"); + const errors: ParseError[] = []; + const document = parseJsonc(removed, errors, { + allowTrailingComma: true, + }); + expect(errors).toEqual([]); + expect(document).toMatchObject({ + mcpServers: { other: { command: "other" } }, + }); + if (original.includes('"keep"')) + expect(document).toMatchObject({ + mcpServers: { keep: { command: "keep" } }, + }); + } + }); +}); + +describe("Grok Bot registration", () => { + it("does not write or align a data-directory mcp.json", async () => { + const home = await createTestTempDirectory("rea-grokbot-"); + roots.push(home); + const directory = join(home, ".grokbot"); + const configPath = join(directory, "mcp.json"); + await mkdir(directory, { recursive: true }); + const original = + '{"mcpServers":{"other":{"command":"other"},"rea":{"command":"rea","args":["mcp"]}}}\n'; + await writeFile(configPath, original); + const client = supportedClients(home).find( + ({ name }) => name === "grok_bot", + ); + if (client === undefined) throw new Error("missing grok_bot"); + expect(client.format).toBe("unsupported"); + expect(client.configPath).toBe(directory); + expect( + await systemSetupHost().configureClient(client, {}, ["rea", "mcp"]), + ).toEqual({ status: "skipped" }); + expect(await readFile(configPath, "utf8")).toBe(original); + expect( + await readClientRegistrationStatuses(home, "/current/rea", { + environment: {}, + }), + ).toEqual([ + expect.objectContaining({ + client: "grok_bot", + config_path: directory, + state: "manual", + command: [], + }), + ]); + const removed = await runUninstall(false, systemUninstallHost(home)); + expect(removed.status).toBe("complete"); + expect(removed.items).toContainEqual( + expect.objectContaining({ name: "grok_bot", status: "skipped" }), + ); + expect(await readFile(configPath, "utf8")).toBe(original); + }); +}); + describe("client configuration filesystem removal", () => { it("uninstalls only owned entries and refuses purge symlinks", async () => { const home = await createTestTempDirectory("rea-uninstall-"); diff --git a/tests/boundary/filesystem/clientRegistrationStatus.test.ts b/tests/boundary/filesystem/clientRegistrationStatus.test.ts index ea9d046432c..05279dae3ce 100644 --- a/tests/boundary/filesystem/clientRegistrationStatus.test.ts +++ b/tests/boundary/filesystem/clientRegistrationStatus.test.ts @@ -1,13 +1,30 @@ import { mkdir, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; import { PRODUCT_IDENTITY } from "../../../src/identity.js"; +beforeEach(() => { + for (const name of [ + "APPDATA", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", + "COPILOT_HOME", + "GROK_HOME", + "OPENCODE_CONFIG", + "SAND_DATA_ROOT", + "XDG_CONFIG_HOME", + ]) + vi.stubEnv(name, undefined); +}); +afterEach(() => { + vi.unstubAllEnvs(); +}); + describe("client registration status", () => { it("discovers Claude Code from its config file without a marker directory", async () => { const home = await createTestTempDirectory("rea-claude-config-only-"); @@ -206,6 +223,43 @@ describe("Node-wrapped registration policy", () => { }, ); + it.each([1, 30])( + "checks Grok Build startup timeout %d independently of launcher", + async (timeout) => { + const home = await createTestTempDirectory("rea-grok-registration-"); + await mkdir(join(home, ".grok")); + const entry = resolve("scripts/rea.mjs"); + await writeFile( + join(home, ".grok/config.toml"), + `[mcp_servers.rea]\ncommand = ${JSON.stringify(process.execPath)}\nargs = [${JSON.stringify(entry)}, "mcp"]\nstartup_timeout_sec = ${String(timeout)}\n`, + ); + const statuses = await readClientRegistrationStatuses(home, entry, { + environment: {}, + }); + expect(statuses).toEqual([ + expect.objectContaining({ + client: "grok_build", + state: timeout === 30 ? "aligned" : "stale", + }), + ]); + }, + ); + + it("reports Grok Build stale when rea is listed in disabled_mcp_servers", async () => { + const home = await createTestTempDirectory("rea-grok-disabled-status-"); + await mkdir(join(home, ".grok")); + const entry = resolve("scripts/rea.mjs"); + await writeFile( + join(home, ".grok/config.toml"), + `disabled_mcp_servers = ["rea"]\n[mcp_servers.rea]\ncommand = ${JSON.stringify(process.execPath)}\nargs = [${JSON.stringify(entry)}, "mcp"]\nstartup_timeout_sec = 30\n`, + ); + expect( + await readClientRegistrationStatuses(home, entry, { environment: {} }), + ).toEqual([ + expect.objectContaining({ client: "grok_build", state: "stale" }), + ]); + }); + it("checks a direct-launcher Codex startup timeout without Node wrapping", async () => { const home = await createTestTempDirectory("rea-direct-registration-"); await mkdir(join(home, ".codex")); diff --git a/tests/process-global/updateMaintenance.test.ts b/tests/process-global/updateMaintenance.test.ts index a7c66dacd0f..83a6c8929b6 100644 --- a/tests/process-global/updateMaintenance.test.ts +++ b/tests/process-global/updateMaintenance.test.ts @@ -27,6 +27,8 @@ beforeEach(async () => { CLAUDE_CONFIG_DIR: home, CODEX_HOME: join(home, ".codex"), COPILOT_HOME: join(home, ".copilot"), + GROK_HOME: join(home, ".grok"), + SAND_DATA_ROOT: join(home, ".grokbot"), XDG_CONFIG_HOME: join(home, ".config"), OPENCODE_CONFIG: join(home, ".config", "opencode", "opencode.jsonc"), })) diff --git a/website/public/faq/index.html b/website/public/faq/index.html index 9ef107adc3a..174f74c6ea2 100644 --- a/website/public/faq/index.html +++ b/website/public/faq/index.html @@ -107,7 +107,8 @@

Setup and updates

Any agent that supports local MCP servers. Setup includes Claude Code, Claude Desktop, Codex, Cursor, Gemini CLI, Windsurf, Devin, OpenCode, Antigravity, GitHub Copilot CLI, - Command Code and VS Code. + Command Code, Grok Build, and VS Code. Grok Bot's connector is + added from its chat.

Connect your coding agent. diff --git a/website/public/get-started/index.html b/website/public/get-started/index.html index 25b3e57c609..a9ba773f06e 100644 --- a/website/public/get-started/index.html +++ b/website/public/get-started/index.html @@ -75,8 +75,9 @@

Set up REA.

Connect your coding agent

Run setup to connect REA to your coding agent. Client choices - include Claude Code, Codex, Cursor, Gemini CLI, Windsurf and other - supported MCP clients. + include Claude Code, Codex, Cursor, Gemini CLI, Windsurf, Grok + Build, and other supported MCP clients. Grok Bot's connector is + added from its chat.

From ae06dcf83f65b06abb390d2e0b0ea33543278d6e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:12:12 +0800 Subject: [PATCH 045/791] test: prune resource-recovery wording assertions --- src/domain/analysisErrorProjection.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index 536c027f516..8177b08abc0 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -116,16 +116,6 @@ it.each([ captured_output: retainedOutput, }, }); - expect(projected.remediation.action).toContain( - resource === "cpu" - ? "CPU" - : resource === "transport" - ? "Evidence reference" - : resource === "file-size" - ? "file-size" - : "memory", - ); - expect(projected.remediation.action).not.toContain("doctor"); expect(analysisErrorProjectionSchema.safeParse(projected).success).toBe( true, ); From 82626c4499a53c227187575d40a845fadd5318e2 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:12:55 +0800 Subject: [PATCH 046/791] fix(hopper): bind file mappings to verified source bytes Read source digests, FAT tables and embedded headers from one opened source stream before translating native offsets. Preserve native mapping observations through changed, missing, nonregular and access-denied source failures; keep filesystem work outside the Hopper API thread. Exercise source drift, relocated slices, missing files, permissions and recovery through real Hopper CLI/MCP verification for thin, FAT32 and FAT64 inputs. --- bridge/hopper_bridge.py | 84 +---- docs/native-investigation.md | 15 + docs/testing.md | 9 +- scripts/lib/real-hopper-source-binding.mjs | 121 +++++++ scripts/verify-real-hopper-fat.mjs | 22 ++ src/contracts/hopperMemoryContracts.ts | 2 +- src/domain/analysisErrorBase.ts | 4 +- src/domain/analysisErrorCore.ts | 4 +- src/domain/configurationErrors.ts | 7 +- .../native/fileOffsetPartialObservation.ts | 12 + src/hopper/BridgeLauncher.ts | 1 - src/hopper/HopperFileOffset.ts | 322 ++++++++++++++++++ src/hopper/HopperProvider.ts | 13 +- src/hopper/HopperProviderCapabilities.ts | 5 + 14 files changed, 536 insertions(+), 85 deletions(-) create mode 100644 scripts/lib/real-hopper-source-binding.mjs create mode 100644 src/domain/native/fileOffsetPartialObservation.ts create mode 100644 src/hopper/HopperFileOffset.ts diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 55943217260..660ff52ea23 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -9,7 +9,6 @@ import hmac import os import socket -import struct from typing import Any, Optional, Protocol, Sequence BAD_ADDRESSES = (-1, 0xFFFFFFFFFFFFFFFF, None) @@ -277,76 +276,6 @@ def _procedure(document, value=None): return result -def _original_file_offset(document, offset): - """Translate Hopper's image-relative mapping using the observed loaded header.""" - if offset > 9007199254740991: - raise CapabilityUnavailableError("Provider file offset exceeds the exact JSON integer range") - path = globals().get("REA_ORIGINAL_EXECUTABLE_PATH") or document.getExecutableFilePath() - if not path: - raise CapabilityUnavailableError("Hopper did not identify the original executable for file-offset verification") - try: - with open(path, "rb") as source: - size = os.fstat(source.fileno()).st_size - header = source.read(8) - formats = { - b"\xca\xfe\xba\xbe": (">", False), - b"\xbe\xba\xfe\xca": ("<", False), - b"\xca\xfe\xba\xbf": (">", True), - b"\xbf\xba\xfe\xca": ("<", True), - } - fat = formats.get(header[:4]) - if fat is None: - if offset >= size: - raise InvalidRequestError("Mapped offset is outside the original executable file at %s" % path) - return offset, path, 0 - if len(header) != 8: - raise CapabilityUnavailableError("Original FAT executable has a truncated header at %s" % path) - byte_order, wide = fat - count = struct.unpack(byte_order + "I", header[4:])[0] - stride = 32 if wide else 20 - table_end = 8 + count * stride - if count == 0 or table_end > size: - raise CapabilityUnavailableError("Original FAT executable has an invalid architecture table at %s" % path) - header_address = document.getAddressFromFileOffset(0) - reader = getattr(document, "readBytes", None) - if header_address in BAD_ADDRESSES or document.getSegmentAtAddress(header_address) is None or not callable(reader): - raise CapabilityUnavailableError("Hopper did not expose a readable loaded Mach-O header to identify its FAT slice") - loaded = reader(header_address, 12) - if not isinstance(loaded, (bytes, bytearray)) or len(loaded) != 12: - raise CapabilityUnavailableError("Hopper did not expose the loaded Mach-O header to identify its FAT slice") - loaded_order = {b"\xce\xfa\xed\xfe": "<", b"\xcf\xfa\xed\xfe": "<", b"\xfe\xed\xfa\xce": ">", b"\xfe\xed\xfa\xcf": ">"}.get(bytes(loaded[:4])) - if loaded_order is None: - raise CapabilityUnavailableError("Hopper's loaded header is not a supported Mach-O image") - identity = struct.unpack(loaded_order + "II", loaded[4:12]) - selected = None - for index in range(count): - source.seek(8 + index * stride) - entry = source.read(stride) - if len(entry) != stride: - raise CapabilityUnavailableError("Original FAT executable architecture table changed during verification") - cpu, subtype, start, length, alignment = struct.unpack(byte_order + ("IIQQI" if wide else "IIIII"), entry[:28] if wide else entry) - if length < 12 or start < table_end or start > size or length > size - start or alignment > 52 or start % (1 << alignment): - raise CapabilityUnavailableError("Original FAT executable has an invalid slice range at index %s" % index) - if (cpu, subtype) != identity: - continue - source.seek(start) - if source.read(12) != bytes(loaded): - raise CapabilityUnavailableError("FAT slice declaration disagrees with Hopper's loaded header") - if selected is not None: - raise CapabilityUnavailableError("Multiple FAT slices match Hopper's loaded header; original-file mapping is ambiguous") - selected = (start, length) - if selected is None: - raise CapabilityUnavailableError("No original FAT slice matches Hopper's loaded header") - start, length = selected - if offset >= length: - raise InvalidRequestError("Mapped offset is outside Hopper's loaded FAT slice") - if start + offset > 9007199254740991: - raise CapabilityUnavailableError("Original file offset exceeds the exact JSON integer range") - return start + offset, path, start - except OSError as error: - raise CapabilityUnavailableError("Cannot read the original executable for file-offset verification at %s: %s" % (path, error)) from error - - def _procedure_name(procedure): entry = procedure.getEntryPoint() return procedure.getSegment().getNameAtAddress(entry) or _hex(entry) @@ -873,8 +802,17 @@ def _dispatch(method, params): # file header. An integer alone does not prove source-file provenance. if inverse(offset) != address: raise InvalidRequestError("Address has no authoritative file-offset mapping: reverse lookup disagrees at %s" % _hex(address)) - original_offset, source_path, image_base = _original_file_offset(document, offset) - return {"address": _hex(address), "file_offset": original_offset, "provider_file_offset": offset, "image_base_file_offset": image_base, "source_path": source_path} + if offset > 9007199254740991: + raise CapabilityUnavailableError("Provider file offset exceeds the exact JSON integer range") + header_address = inverse(0) + reader = getattr(document, "readBytes", None) + header = reader(header_address, 12) if header_address not in BAD_ADDRESSES and document.getSegmentAtAddress(header_address) is not None and callable(reader) else None + return { + "address": _hex(address), + "provider_file_offset": offset, + "provider_source_path": document.getExecutableFilePath(), + "provider_image_header_hex": bytes(header).hex() if isinstance(header, (bytes, bytearray)) and len(header) == 12 else None, + } if method == "resolve_containing_procedure": address = _address(document, params.get("address")) procedure, reason = _containing_procedure(document, address) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index e23b94a4f08..4a98b90d8b1 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -12,6 +12,21 @@ input file and include the observed lipo slice offset for universal binaries. If that offset is unavailable, segment evidence locations are omitted with an explicit limitation; architecture inventory locations remain available. +Hopper's `address_to_file_offset` combines the native image-relative mapping with +original-file coordinates only after checking the source against the session's +SHA-256. FAT tables and embedded Mach-O headers come from that same hashing pass. +A changed or nonregular source reports `artifact_changed`; an unreadable or +missing source reports its access or filesystem failure. Those errors retain the native offset, +loaded header and provider source path in `details.partial_observation`, with the +original-file coordinate explicitly unavailable. Restore the original bytes or +reopen the changed executable to obtain verified coordinates. Snapshot replay +remains historical evidence for its recorded artifact identity. + +A selected Hopper database's digest identifies the database rather than its +original executable. Original-file mapping is unavailable for that selection; +native mapping facts remain in partial evidence, and `read_bytes` still reads +the loaded database. Open the original executable to verify source coordinates. + Native Xcode command execution uses shared process supervision with a 60-second deadline and a 64 MiB aggregate stdout/stderr budget. Timeout, cancellation, stream failure, and output exhaustion retain the captured output, exit details, diff --git a/docs/testing.md b/docs/testing.md index 02b9b7b19f3..4610c4fa0ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -227,8 +227,13 @@ The Linux demo lane remains a separate `verify:hopper:linux` command. existing Xcode clang/lipo toolchain. It compiles arm64/x86-64 thin executables and one- and two-slice FAT32 containers, verifies exact/interior address mappings against bytes in the original files, checks CLI/MCP parity, and checks owned -runtime cleanup. `verify:hopper:fat64` additionally checks FAT64 preparation through -Hopper's native Mach-O loader, source-container mappings, profile identity, +runtime cleanup. Source byte changes, removal, permission denial (for non-root +callers), and nonregular replacement must retain native partial mapping facts, +reject unverified original-file coordinates, +and recover after restoration. Single-slice FAT cases also relocate the slice +without changing its loaded bytes. `verify:hopper:fat64` additionally checks +FAT64 preparation through Hopper's native Mach-O loader, source-container +mappings, profile identity, malformed and ambiguous slice rejection, and temporary-image lifetime. Both lanes have been verified on Hopper 6.1.0-demo; this establishes REA's prepared FAT64 workflow, not native FAT64-loader support. diff --git a/scripts/lib/real-hopper-source-binding.mjs b/scripts/lib/real-hopper-source-binding.mjs new file mode 100644 index 00000000000..0fa6868cc46 --- /dev/null +++ b/scripts/lib/real-hopper-source-binding.mjs @@ -0,0 +1,121 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, rename, rm, stat, writeFile } from "node:fs/promises"; +import { mcpTextValue } from "./mcp-verifier-results.mjs"; + +/** Exercise source replacement, removal, and restoration against a loaded native document. */ +export async function verifyHopperSourceBinding({ + client, + call, + path, + file, + address, + mapping, + relocate, +}) { + const originalHash = createHash("sha256").update(file).digest("hex"); + const changed = relocate === undefined ? Buffer.from(file) : relocate(file); + if (relocate === undefined) changed[changed.length - 1] ^= 1; + const changedHash = createHash("sha256").update(changed).digest("hex"); + const at = (delta) => `0x${(BigInt(address) + BigInt(delta)).toString(16)}`; + const observeFailure = async (delta) => { + const reply = await client.callTool( + { name: "address_to_file_offset", arguments: { address: at(delta) } }, + { timeout: 180_000 }, + ); + assert.equal( + reply.isError, + true, + "mapping unexpectedly accepted an unbound source", + ); + const error = JSON.parse(mcpTextValue(reply)).error; + const partial = error.details.partial_observation; + assert.equal(partial.operation, "address_to_file_offset"); + assert.equal(partial.address, at(delta)); + assert.equal( + partial.provider_file_offset, + mapping.provider_file_offset + delta, + ); + assert.equal(partial.original_file_offset.available, false); + assert.equal(typeof partial.original_file_offset.reason, "string"); + assert.match(partial.provider_image_header_hex, /^[a-f0-9]{24}$/u); + return error; + }; + try { + await writeFile(path, changed); + const error = await observeFailure(2); + assert.equal(error.code, "artifact_changed"); + assert.ok(error.details.reason.includes(originalHash)); + assert.ok(error.details.reason.includes(changedHash)); + assert.equal((await call("binary_session")).sha256, originalHash); + assert.equal( + (await call("read_bytes", { address: at(2), length: 1 })).bytes_hex, + file + .subarray(mapping.file_offset + 2, mapping.file_offset + 3) + .toString("hex"), + ); + } finally { + await writeFile(path, file); + } + const renamed = `${path}.temporarily-moved`; + await rename(path, renamed); + try { + const error = await observeFailure(3); + assert.equal(error.code, "target_unavailable"); + assert.match(error.details.reason, /ENOENT/u); + } finally { + await rename(renamed, path); + } + await rename(path, renamed); + try { + await mkdir(path); + const error = await observeFailure(4); + assert.equal(error.code, "artifact_changed"); + assert.match(error.details.reason, /no longer a regular file/u); + } finally { + await rm(path, { recursive: true, force: true }); + await rename(renamed, path); + } + const permissionDenied = process.getuid() !== 0; + if (permissionDenied) { + const mode = (await stat(path)).mode & 0o777; + try { + await chmod(path, 0); + const error = await observeFailure(5); + assert.equal(error.code, "access_denied"); + assert.ok(["EACCES", "EPERM"].includes(error.details.system_code)); + } finally { + await chmod(path, mode); + } + } + for (const delta of [2, 3, 4, ...(permissionDenied ? [5] : [])]) { + const recovered = await call("address_to_file_offset", { + address: at(delta), + }); + assert.equal(recovered.file_offset, mapping.file_offset + delta); + } + return { + replacement: relocate === undefined ? "same_size_bytes" : "relocated_slice", + missingSource: "verified", + nonregularSource: "verified", + permissionDenied: permissionDenied ? "verified" : "not_run_root", + nativePartialEvidence: "verified", + recovery: "verified", + }; +} + +/** Relocate the sole FAT slice while retaining its exact loaded bytes and CPU identity. */ +export function relocateSingleFatSlice(file) { + const wide = file.readUInt32BE(0) === 0xcafebabf; + assert.equal(file.readUInt32BE(4), 1); + const offset = wide + ? Number(file.readBigUInt64BE(16)) + : file.readUInt32BE(16); + const size = wide ? Number(file.readBigUInt64BE(24)) : file.readUInt32BE(20); + const relocated = Buffer.alloc(offset * 2 + size); + file.subarray(0, offset).copy(relocated); + file.subarray(offset, offset + size).copy(relocated, offset * 2); + if (wide) relocated.writeBigUInt64BE(BigInt(offset * 2), 16); + else relocated.writeUInt32BE(offset * 2, 16); + return relocated; +} diff --git a/scripts/verify-real-hopper-fat.mjs b/scripts/verify-real-hopper-fat.mjs index d4709e357c5..b068b884069 100644 --- a/scripts/verify-real-hopper-fat.mjs +++ b/scripts/verify-real-hopper-fat.mjs @@ -22,6 +22,10 @@ import { mcpTextValue, requireMcpResult } from "./lib/mcp-verifier-results.mjs"; import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs"; import { HOPPER_TARGET_LEASE_DIRECTORY } from "../dist/hopper/HopperTargetLease.js"; import { parseConfig } from "../dist/config.js"; +import { + verifyHopperSourceBinding, + relocateSingleFatSlice, +} from "./lib/real-hopper-source-binding.mjs"; const run = promisify(execFile); const verifyFat64 = process.argv[2] === "--fat64"; @@ -86,6 +90,7 @@ transport.stderr?.on("data", (chunk) => process.stderr.write(chunk)); const ajv = new Ajv({ strict: false }); addFormats(ajv); const observations = []; +const sourceBinding = []; try { await run(toolPaths.clang, [ "-isysroot", @@ -241,11 +246,13 @@ try { const procedures = await call("list_procedures"); const entry = procedures.find((item) => item.value.endsWith("rea_entry")); assert.ok(entry, "compiler fixture omitted rea_entry"); + let entryMapping; for (const address of [ entry.address, `0x${(BigInt(entry.address) + 1n).toString(16)}`, ]) { const mapping = await call("address_to_file_offset", { address }); + if (address === entry.address) entryMapping = mapping; const bytes = await call("read_bytes", { address, length: 16 }); assert.equal( mapping.file_offset, @@ -322,6 +329,20 @@ try { sourceHash, "native analysis modified source executable bytes", ); + sourceBinding.push({ + path, + ...(await verifyHopperSourceBinding({ + client, + call, + path, + file, + address: entry.address, + mapping: entryMapping, + ...(path === singleFat || path.endsWith("c-single-fat64") + ? { relocate: relocateSingleFatSlice } + : {}), + })), + }); await call("close_binary"); } for (const { path, reason } of invalidFat64) { @@ -405,6 +426,7 @@ console.log( { observations, cliMcpParity: true, + sourceBinding, fat64: verifyFat64 ? "verified" : "not_run", malformedFat64Rejected: invalidFat64.length, preparedImageRemovedOnMcpShutdown: preparedImagePath !== undefined, diff --git a/src/contracts/hopperMemoryContracts.ts b/src/contracts/hopperMemoryContracts.ts index 5e5a84e1126..e4c49788a17 100644 --- a/src/contracts/hopperMemoryContracts.ts +++ b/src/contracts/hopperMemoryContracts.ts @@ -16,7 +16,7 @@ export const HOPPER_MEMORY_TOOL_DEFINITIONS = [ { name: "address_to_file_offset", description: - "Map one provider-normalized virtual address to its original nonnegative file offset. Hopper translates FAT Mach-O image-relative offsets using the observed loaded header, preserving its provider coordinate, image base, and source path. Unmapped addresses fail explicitly, and unavailable or ambiguous original-file mappings return typed capability unavailability.", + "Map one provider-normalized virtual address to its original nonnegative file offset. Hopper verifies the original bytes against the selected executable digest and translates FAT image-relative offsets using the loaded header. Results preserve the provider coordinate, image base, and source path. Unmapped addresses fail explicitly; changed or unavailable sources return typed errors retaining native mapping facts as partial evidence.", inputSchema: z.object({ address, document }), }, ] as const; diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 31ae207dbf7..4ba3bd99126 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -84,9 +84,11 @@ import type { PartialProcessCaptureObservation, ProcessCaptureCleanupReport, } from "./process/processCapture.js"; +import type { FileOffsetPartialObservation } from "./native/fileOffsetPartialObservation.js"; import type { NativeCallPartialObservation } from "./native/nativeCallPartialObservation.js"; /** Provider-neutral evidence collected before a typed analysis failure. */ export type AnalysisPartialObservation = | PartialProcessCaptureObservation - | NativeCallPartialObservation; + | NativeCallPartialObservation + | FileOffsetPartialObservation; diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index 22ddc4921c5..e0ba8982057 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -136,7 +136,7 @@ export class AnalysisAccessDeniedError extends AnalysisError { readonly operation: string, readonly path: string, readonly systemCode: "EACCES" | "EPERM", - options?: ErrorOptions, + options?: AnalysisErrorOptions, ) { super( `Host filesystem read access denied (${systemCode}) for ${path} during ${operation}`, @@ -152,7 +152,7 @@ export class AnalysisArtifactChangedError extends AnalysisError { readonly operation: string, readonly path: string, readonly reason: string, - options?: ErrorOptions, + options?: AnalysisErrorOptions, ) { super(`Selected artifact changed during acquisition: ${reason}`, options); } diff --git a/src/domain/configurationErrors.ts b/src/domain/configurationErrors.ts index 444693b993b..c7c07f75e6b 100644 --- a/src/domain/configurationErrors.ts +++ b/src/domain/configurationErrors.ts @@ -1,4 +1,7 @@ -import { AnalysisError } from "./analysisErrorBase.js"; +import { + AnalysisError, + type AnalysisErrorOptions, +} from "./analysisErrorBase.js"; /** Runtime configuration could not be parsed safely. */ export class ConfigurationError extends AnalysisError { @@ -16,7 +19,7 @@ export class NoBinaryOpenError extends AnalysisError { } /** Additional constraints identified while admitting a filesystem target. */ -export interface BinaryTargetErrorOptions extends ErrorOptions { +export interface BinaryTargetErrorOptions extends AnalysisErrorOptions { readonly constraint?: "directory_requires_file"; } diff --git a/src/domain/native/fileOffsetPartialObservation.ts b/src/domain/native/fileOffsetPartialObservation.ts new file mode 100644 index 00000000000..5376790e7a5 --- /dev/null +++ b/src/domain/native/fileOffsetPartialObservation.ts @@ -0,0 +1,12 @@ +/** Native mapping facts retained when original-file coordinates cannot be verified. */ +export interface FileOffsetPartialObservation { + readonly operation: "address_to_file_offset"; + readonly address: string; + readonly provider_file_offset: number; + readonly provider_source_path: string | null; + readonly provider_image_header_hex: string | null; + readonly original_file_offset: { + readonly available: false; + readonly reason: string; + }; +} diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 19d50527ad8..436d5f0b54d 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -363,7 +363,6 @@ const bridgeBootstrapSource = ( `REA_TOKEN = ${JSON.stringify(session.token)}`, `REA_RUN_ID = ${JSON.stringify(session.runId)}`, `REA_TARGET_PATH = ${JSON.stringify(loadedPath)}`, - `REA_ORIGINAL_EXECUTABLE_PATH = ${options.targetKind === "executable" ? JSON.stringify(options.targetPath) : "None"}`, `REA_OWNS_PROCESS_LIFETIME = ${ownsProcessLifetime ? "True" : "False"}`, `exec(compile(open(${JSON.stringify(options.bridgeScriptPath)}, 'rb').read(), ${JSON.stringify(options.bridgeScriptPath)}, 'exec'))`, "", diff --git a/src/hopper/HopperFileOffset.ts b/src/hopper/HopperFileOffset.ts new file mode 100644 index 00000000000..3856dbccdd8 --- /dev/null +++ b/src/hopper/HopperFileOffset.ts @@ -0,0 +1,322 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { open, type FileHandle } from "node:fs/promises"; +import { z } from "zod"; + +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisAccessDeniedError, + AnalysisArtifactChangedError, + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { BinaryTargetError } from "../domain/configurationErrors.js"; +import type { BinaryTarget } from "../domain/binaryTarget.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import type { FileOffsetPartialObservation } from "../domain/native/fileOffsetPartialObservation.js"; +import { + readFatSliceDeclarations, + type FatSliceDeclaration, + validateFatSlice, +} from "../domain/apple/machoContainer.js"; +import { err, ok, type Result } from "../domain/result.js"; + +const OPERATION = "address_to_file_offset"; +const MAX_FAT_ARCHITECTURES = 128; +const PREFIX_BYTES = 8 + MAX_FAT_ARCHITECTURES * 32; +const observationSchema = z.strictObject({ + address: z.string().regex(/^0x[0-9a-f]+$/u), + provider_file_offset: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), + provider_source_path: z.string().nullable(), + provider_image_header_hex: z + .string() + .regex(/^[a-f0-9]{24}$/u) + .nullable(), +}); +type Observation = z.infer; + +/** Bind native image-relative offsets to coordinates in the admitted source bytes. */ +export async function mapHopperFileOffset( + target: BinaryTarget, + value: JsonValue, + signal?: AbortSignal, +): Promise> { + const parsed = observationSchema.safeParse(value); + if (!parsed.success) + return err( + new AnalysisOutputError( + OPERATION, + "Hopper's native file mapping observation is malformed", + ), + ); + const observation = parsed.data; + const partial = (reason: string): FileOffsetPartialObservation => ({ + operation: OPERATION, + ...observation, + original_file_offset: { available: false, reason }, + }); + const unavailable = (reason: string) => + err( + new AnalysisCapabilityUnavailableError("hopper", OPERATION, reason, { + partialObservation: partial(reason), + }), + ); + if (target.kind !== "executable") + return unavailable( + "The selected database digest does not identify its original executable bytes. Open that executable to verify original-file coordinates; read_bytes still exposes the database's native bytes.", + ); + if (signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { + partialObservation: partial("Source verification was cancelled"), + }), + ); + try { + const captured = await snapshotFromPath(target.path, signal); + if (!captured.ok) + return err( + new AnalysisArtifactChangedError( + OPERATION, + target.path, + captured.error, + { + partialObservation: partial(captured.error), + }, + ), + ); + const snapshot = captured.value; + if (signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { + partialObservation: partial("Source verification was cancelled"), + }), + ); + if (snapshot.sha256 !== target.sha256) { + const reason = `Source ${target.path} differs from the admitted artifact: expected SHA-256 ${target.sha256}, observed ${snapshot.sha256}. Reopen the changed source for a new analysis identity.`; + return err( + new AnalysisArtifactChangedError(OPERATION, target.path, reason, { + partialObservation: partial(reason), + }), + ); + } + const mapped = originalOffset(snapshot, observation); + if (!mapped.ok) return unavailable(mapped.error); + return ok({ + address: observation.address, + file_offset: mapped.value + observation.provider_file_offset, + provider_file_offset: observation.provider_file_offset, + image_base_file_offset: mapped.value, + source_path: target.path, + }); + } catch (cause: unknown) { + if (signal?.aborted) + return err( + new AnalysisCancelledError(OPERATION, { + cause, + partialObservation: partial("Source verification was cancelled"), + }), + ); + const reason = `Cannot verify original source bytes: ${cause instanceof Error ? cause.message : String(cause)}`; + const code = + typeof cause === "object" && cause !== null && "code" in cause + ? cause.code + : undefined; + if (code === "EACCES" || code === "EPERM") + return err( + new AnalysisAccessDeniedError(OPERATION, target.path, code, { + cause, + partialObservation: partial(reason), + }), + ); + return err( + new BinaryTargetError(target.path, reason, { + cause, + partialObservation: partial(reason), + }), + ); + } +} + +async function snapshotFromPath(path: string, signal?: AbortSignal) { + const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK); + try { + if (!(await file.stat()).isFile()) + return err( + `Source ${path} is no longer a regular file. Restore the admitted executable or reopen a regular source.`, + ); + return ok(await readSourceSnapshot(file, signal)); + } finally { + await file.close(); + } +} + +interface FatLayout { + readonly slices: readonly FatSliceDeclaration[]; + readonly tableEnd: number; +} + +function fatLayout(prefix: Buffer): Result { + const magic = prefix.length < 4 ? 0 : prefix.readUInt32BE(0); + if (![0xcafebabe, 0xbebafeca, 0xcafebabf, 0xbfbafeca].includes(magic)) + return ok(null); + if (prefix.length < 8) return err("The original FAT header is truncated"); + const littleEndian = magic === 0xbebafeca || magic === 0xbfbafeca; + const wide = magic === 0xcafebabf || magic === 0xbfbafeca; + const count = littleEndian ? prefix.readUInt32LE(4) : prefix.readUInt32BE(4); + if (count === 0 || count > MAX_FAT_ARCHITECTURES) + return err("FAT architecture count is outside the supported 1..128 range"); + const tableEnd = 8 + count * (wide ? 32 : 20); + const table = prefix.subarray(8, tableEnd); + const parsed = readFatSliceDeclarations(table, { count, wide, littleEndian }); + if (parsed.status !== "parsed") return err(parsed.reason); + if (wide) { + for (const slice of parsed.slices) + if ( + (littleEndian + ? table.readUInt32LE(slice.index * 32 + 28) + : table.readUInt32BE(slice.index * 32 + 28)) !== 0 + ) + return err( + `FAT64 architecture ${slice.index} has a nonzero reserved field`, + ); + } + return ok({ slices: parsed.slices, tableEnd }); +} + +async function readSourceSnapshot(file: FileHandle, signal?: AbortSignal) { + const prefix = Buffer.alloc(PREFIX_BYTES); + const hash = createHash("sha256"); + let size = 0; + let layout: Result | undefined; + const headers = new Map(); + const capture = (bytes: Buffer, position: number) => { + if (layout?.ok !== true || layout.value === null) return; + for (const slice of layout.value.slices) { + const start = Math.max(position, slice.offset); + const end = Math.min(position + bytes.length, slice.offset + 12); + if (end <= start) continue; + let header = headers.get(slice.index); + if (header === undefined) { + header = Buffer.alloc(12); + headers.set(slice.index, header); + } + bytes.copy( + header, + start - slice.offset, + start - position, + end - position, + ); + } + }; + for await (const chunk of file.createReadStream({ + start: 0, + autoClose: false, + ...(signal === undefined ? {} : { signal }), + })) { + const bytes = Buffer.from(chunk); + hash.update(bytes); + if (size < prefix.length) + bytes.copy(prefix, size, 0, Math.min(bytes.length, prefix.length - size)); + if (layout === undefined && size + bytes.length >= prefix.length) { + layout = fatLayout(prefix); + capture(prefix, 0); + } + capture(bytes, size); + size += bytes.length; + } + if (layout === undefined) { + layout = fatLayout(prefix.subarray(0, size)); + capture(prefix.subarray(0, size), 0); + } + return { layout, headers, size, sha256: hash.digest("hex") }; +} + +type SourceSnapshot = Awaited>; + +function originalOffset( + snapshot: SourceSnapshot, + observation: Observation, +): Result { + const { layout, size, headers } = snapshot; + if (!layout.ok) return layout; + if (layout.value === null) + return observation.provider_file_offset < size + ? ok(0) + : err("Provider mapping lies outside the admitted executable bytes"); + const header = + observation.provider_image_header_hex === null + ? null + : Buffer.from(observation.provider_image_header_hex, "hex"); + const loadedMagic = header?.readUInt32BE(0); + if ( + header === null || + loadedMagic === undefined || + ![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(loadedMagic) + ) + return err( + "Hopper did not expose the loaded Mach-O header needed to identify its FAT slice", + ); + const little = loadedMagic === 0xcefaedfe || loadedMagic === 0xcffaedfe; + const cpu = little ? header.readUInt32LE(4) : header.readUInt32BE(4); + const subtype = little ? header.readUInt32LE(8) : header.readUInt32BE(8); + const matches = []; + const ranges = [...layout.value.slices].sort((a, b) => a.offset - b.offset); + for (let index = 1; index < ranges.length; index++) { + const previous = ranges[index - 1]; + const current = ranges[index]; + if ( + previous !== undefined && + current !== undefined && + current.offset < previous.offset + previous.size + ) + return err("Original FAT slices overlap"); + } + for (const slice of layout.value.slices) { + const reason = validateFatSlice(slice, size, layout.value.tableEnd); + if (reason !== null) return err(reason); + const embedded = headers.get(slice.index); + if (slice.size < 12 || embedded === undefined) + return err( + `FAT architecture ${slice.index} has a truncated Mach-O header`, + ); + const embeddedMagic = embedded.readUInt32BE(0); + if ( + ![0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(embeddedMagic) + ) + return err( + `FAT architecture ${slice.index} does not contain a Mach-O image`, + ); + const embeddedLittle = + embeddedMagic === 0xcefaedfe || embeddedMagic === 0xcffaedfe; + const identityError = validateFatSlice(slice, size, layout.value.tableEnd, { + cpuType: embeddedLittle + ? embedded.readUInt32LE(4) + : embedded.readUInt32BE(4), + cpuSubtype: embeddedLittle + ? embedded.readUInt32LE(8) + : embedded.readUInt32BE(8), + }); + if (identityError !== null) return err(identityError); + if (slice.cpuType === cpu && slice.cpuSubtype === subtype) + matches.push(slice); + } + const selected = matches.length === 1 ? matches[0] : undefined; + if (selected === undefined) + return err( + `Original FAT slice matching Hopper's loaded header is ${matches.length === 0 ? "missing" : "ambiguous"}`, + ); + if (headers.get(selected.index)?.equals(header) !== true) + return err( + "Original FAT slice header disagrees with Hopper's loaded header", + ); + if (observation.provider_file_offset >= selected.size) + return err("Provider mapping lies outside the loaded FAT slice"); + if (!Number.isSafeInteger(selected.offset + observation.provider_file_offset)) + return err("Original file offset exceeds the exact JSON integer range"); + return ok(selected.offset); +} diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 093f2633588..44b09b417f6 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -23,6 +23,7 @@ import { hopperLoaderArgsForTarget, resolveHopperAnalysisProfile, } from "./HopperAnalysisProfile.js"; +import { mapHopperFileOffset } from "./HopperFileOffset.js"; import { HopperRegexSearch } from "./HopperRegexSearch.js"; import { HopperClient } from "./HopperClient.js"; import { hopperMachOImageSchema } from "./HopperMachOImage.js"; @@ -239,10 +240,16 @@ export class HopperProvider implements AnalysisProviderCandidate { parameters.mode === "regex" ? await regexSearch.execute(operation, parameters, options) : await client.callTool(operation, parameters, options); - return result.ok + if (!result.ok) return result; + const mapped = + operation === "address_to_file_offset" + ? await mapHopperFileOffset(target, result.value, options?.signal) + : result; + return mapped.ok ? { ok: true, - value: createAnalysisExecution(result.value, executionProvider, { + value: createAnalysisExecution(mapped.value, executionProvider, { + rawResult: result.value, ...(profile === undefined ? {} : { analysisProfile: profile }), limitations: [ ...(preparedImage === undefined @@ -266,7 +273,7 @@ export class HopperProvider implements AnalysisProviderCandidate { ], }), } - : result; + : mapped; }, runtimeLineageSnapshots: () => { const observation = client.runtimeLineage(); diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 1d237319101..00f89fcdbf6 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,6 +79,11 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", + ...(operation === "address_to_file_offset" + ? [ + "Original-file coordinates require source bytes matching the selected executable digest. A changed, unavailable, or database-only source retains native mapping facts in partial error evidence.", + ] + : []), ...(operation === "search_strings" || operation === "search_procedures" ? [ "Regex mode uses ECMAScript Unicode semantics in a supervised worker with a five-second matching deadline. Cancellation or deadline failure stops matching without blocking Hopper; literal mode retains native casefold semantics.", From 567a82f8d4feb629a7c10d2cc9180377eb1f6f66 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:29:29 +0800 Subject: [PATCH 047/791] fix(update): accept npm 12 release metadata (#1138) * fix(update): accept npm 12 release metadata (cherry picked from commit aa400d18c95e36029c5c654c396ec66924e1358f) * fix(update): classify npm release metadata without a spawn mock Keep the npm 11 string and npm 12 one-element array accepted by the cherry-picked lookup, and separate invalid metadata from a failed registry command. --------- Co-authored-by: Dapo Olaomo --- src/application/Update.test.ts | 37 +++++++- src/application/Update.ts | 33 ++++++- src/application/UpdateRuntime.release.test.ts | 88 +++++++++++++++++++ src/application/UpdateRuntime.ts | 83 ++++++++++++----- src/cliLogging.test.ts | 5 +- 5 files changed, 215 insertions(+), 31 deletions(-) create mode 100644 src/application/UpdateRuntime.release.test.ts diff --git a/src/application/Update.test.ts b/src/application/Update.test.ts index 12b9c0c0933..8e1ce064249 100644 --- a/src/application/Update.test.ts +++ b/src/application/Update.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { err, ok, type Result } from "../domain/result.js"; import { runUpdate, + type ReleaseLookupFailure, type UpdateHost, type UpdateOutput, type NpmInstallation, @@ -14,7 +15,7 @@ import { import type { IntegrationMaintenance } from "./UpdateMaintenance.js"; class FakeUpdateHost implements UpdateHost { - release: Result = ok("3.3.0"); + release: Result = ok("3.3.0"); installationResult: NpmInstallation | undefined = { prefix: "/fixture", packageRoot: "/fixture/lib/node_modules/rea-agents", @@ -35,7 +36,7 @@ class FakeUpdateHost implements UpdateHost { this.installationInspected = true; return Promise.resolve(this.installationResult); }; - latestVersion = (): Promise> => + latestVersion = (): Promise> => Promise.resolve(this.release); installVersion = ( _installation: NpmInstallation, @@ -180,7 +181,10 @@ describe("update failures and version boundaries", () => { it("preserves registry errors without installation", async () => { const host = new FakeUpdateHost(); - host.release = err("npm registry returned HTTP 503"); + host.release = err({ + kind: "unavailable", + detail: "npm registry returned HTTP 503", + }); const result = await runUpdate("3.2.1", host); expect(result).toMatchObject({ status: "failed", @@ -191,6 +195,10 @@ describe("update failures and version boundaries", () => { "remediation", expect.stringContaining("HTTP 503"), ); + expect(result).toHaveProperty( + "remediation", + expect.stringContaining("registry access"), + ); expect(host.installedVersions).toEqual([]); }); @@ -207,6 +215,29 @@ describe("update failures and version boundaries", () => { }, ); + it("describes invalid release metadata without blaming registry access", async () => { + const host = new FakeUpdateHost(); + host.release = err({ + kind: "invalid-metadata", + detail: "expected one version string", + }); + const result = await runUpdate("3.2.1", host); + expect(result).toMatchObject({ + status: "failed", + reason: "version-check", + latestVersion: null, + }); + expect(result).toHaveProperty( + "remediation", + expect.stringContaining("expected one version string"), + ); + expect(result).toHaveProperty( + "remediation", + expect.not.stringContaining("registry access"), + ); + expect(host.installedVersions).toEqual([]); + }); + it.each([ ["3.4.0", "3.3.0"], ["3.3.0", "3.3.0-rc.1"], diff --git a/src/application/Update.ts b/src/application/Update.ts index e0b60b25e95..16b4b6a3a51 100644 --- a/src/application/Update.ts +++ b/src/application/Update.ts @@ -13,12 +13,17 @@ export interface NpmInstallation { /** Output policy for the package manager subprocess. */ export type UpdateOutput = "human" | "structured"; +/** Why release lookup failed before a version could be compared. */ +export type ReleaseLookupFailure = + | { readonly kind: "unavailable"; readonly detail: string } + | { readonly kind: "invalid-metadata"; readonly detail: string }; + /** Effects used to update and verify one identified REA installation. */ export interface UpdateHost { installation(): Promise; latestVersion( installation: NpmInstallation | undefined, - ): Promise>; + ): Promise>; installVersion( installation: NpmInstallation, version: string, @@ -95,6 +100,28 @@ export const updateInstallCommand = ( `${PRODUCT_IDENTITY.packageName}@${version}`, ]; +const versionCheckRemediation = ( + release: Result, +): string => { + if (release.ok) { + return "REA received invalid version metadata. Check the installed version and npm registry, then rerun rea update."; + } + const failure = release.error; + const kind = failure.kind; + switch (kind) { + case "unavailable": + return `REA could not resolve the latest release: ${failure.detail}. Retry rea update when registry access is available.`; + case "invalid-metadata": + return `REA could not resolve the latest release: ${failure.detail}. Check the npm release metadata, then rerun rea update.`; + default: { + const exhaustive: never = kind; + throw new TypeError( + `Unhandled release lookup failure: ${String(exhaustive)}`, + ); + } + } +}; + /** Identify, resolve, install, and verify REA without applying setup changes. */ export const runUpdate = async ( currentVersion: string, @@ -111,9 +138,7 @@ export const runUpdate = async ( currentVersion, latestVersion, reason: "version-check", - remediation: release.ok - ? "REA received invalid version metadata. Check the installed version and npm registry, then rerun rea update." - : `REA could not resolve the latest release: ${release.error}. Retry rea update when registry access is available.`, + remediation: versionCheckRemediation(release), }; if (!gt(targetVersion, currentVersion)) return { diff --git a/src/application/UpdateRuntime.release.test.ts b/src/application/UpdateRuntime.release.test.ts new file mode 100644 index 00000000000..0625d3719dc --- /dev/null +++ b/src/application/UpdateRuntime.release.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from "vitest"; + +import { err, ok } from "../domain/result.js"; +import { safeParseJson } from "../domain/safeJson.js"; +import { PRODUCT_IDENTITY } from "../identity.js"; +import { + npmLatestVersionCommand, + npmReleaseLookup, + parseNpmLatestVersion, +} from "./UpdateRuntime.js"; + +const installation = { + prefix: "/opt/rea", + packageRoot: "/opt/rea/lib/node_modules/rea-agents", +}; + +describe("npm latest release metadata", () => { + it("accepts npm 11's scalar JSON response", () => { + expect(parseNpmLatestVersion('"6.0.0"')).toEqual(ok("6.0.0")); + }); + + it("accepts npm 12's singleton-array JSON response", () => { + expect(parseNpmLatestVersion('[\n "6.0.0"\n]\n')).toEqual(ok("6.0.0")); + }); + + it.each([ + ["empty array", "[]"], + ["multiple versions", '["6.0.0", "6.0.1"]'], + ["empty version", '[""]'], + ["number", "6"], + ["object", '{"version":"6.0.0"}'], + ["null", "null"], + ])("rejects %s response data", (_description, metadata) => { + expect(parseNpmLatestVersion(metadata)).toEqual( + err( + "Invalid npm release metadata: expected a version string or a single-element version array.", + ), + ); + }); + + it("preserves malformed JSON separately from a wrong metadata shape", () => { + const metadata = '"6.0.0'; + const decoded = safeParseJson(metadata); + expect(decoded.ok).toBe(false); + if (decoded.ok) return; + expect(parseNpmLatestVersion(metadata)).toEqual(err(decoded.error)); + expect(decoded.error).not.toContain("single-element version array"); + }); + + it("keeps command failures distinct from rejected metadata", () => { + expect(npmReleaseLookup(err("npm registry returned HTTP 503"))).toEqual( + err({ kind: "unavailable", detail: "npm registry returned HTTP 503" }), + ); + expect(npmReleaseLookup(ok("[]"))).toEqual( + err({ + kind: "invalid-metadata", + detail: + "Invalid npm release metadata: expected a version string or a single-element version array.", + }), + ); + expect(npmReleaseLookup(ok('["6.0.0"]'))).toEqual(ok("6.0.0")); + }); + + it("keeps the global registry query and adds the owning prefix", () => { + expect(npmLatestVersionCommand(installation)).toEqual([ + "npm", + "view", + "--global", + "--prefix", + installation.prefix, + PRODUCT_IDENTITY.packageName, + "dist-tags.latest", + "--json", + "--fetch-retries=0", + "--fetch-timeout=10000", + ]); + expect(npmLatestVersionCommand(undefined)).toEqual([ + "npm", + "view", + "--global", + PRODUCT_IDENTITY.packageName, + "dist-tags.latest", + "--json", + "--fetch-retries=0", + "--fetch-timeout=10000", + ]); + }); +}); diff --git a/src/application/UpdateRuntime.ts b/src/application/UpdateRuntime.ts index 3486078901e..caefba91acb 100644 --- a/src/application/UpdateRuntime.ts +++ b/src/application/UpdateRuntime.ts @@ -3,13 +3,14 @@ import { homedir } from "node:os"; import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import spawn from "cross-spawn"; -import { z } from "zod"; import { PRODUCT_IDENTITY } from "../identity.js"; import { err, ok, type Result } from "../domain/result.js"; +import { safeParseJson } from "../domain/safeJson.js"; import { updateInstallCommand, type NpmInstallation, + type ReleaseLookupFailure, type UpdateHost, type UpdateOutput, } from "./Update.js"; @@ -116,6 +117,60 @@ export const detectNpmInstallation = async ( return { prefix: dirname(library), packageRoot: canonicalPackageRoot }; }; +const INVALID_NPM_RELEASE_METADATA = + "Invalid npm release metadata: expected a version string or a single-element version array."; + +/** npm view arguments for dist-tags.latest, scoped to the owning prefix when known. */ +export const npmLatestVersionCommand = ( + installation: NpmInstallation | undefined, +): readonly string[] => [ + "npm", + "view", + "--global", + ...(installation === undefined ? [] : ["--prefix", installation.prefix]), + PRODUCT_IDENTITY.packageName, + "dist-tags.latest", + "--json", + "--fetch-retries=0", + "--fetch-timeout=10000", +]; + +/** + * Read one latest version from `npm view --json`. + * npm 11 emits a JSON string. npm 12 emits that string as a one-element array. + */ +export const parseNpmLatestVersion = ( + metadata: string, +): Result => { + const decoded = safeParseJson(metadata); + if (!decoded.ok) return err(decoded.error); + const version = readNpmLatestVersion(decoded.value); + return version === undefined + ? err(INVALID_NPM_RELEASE_METADATA) + : ok(version); +}; + +const readNpmLatestVersion = (value: unknown): string | undefined => { + if (typeof value === "string") return nonemptyVersion(value); + if (!Array.isArray(value) || value.length !== 1) return undefined; + const only = value[0]; + return typeof only === "string" ? nonemptyVersion(only) : undefined; +}; + +const nonemptyVersion = (value: string): string | undefined => + value.length === 0 ? undefined : value; + +/** Classify npm view output as a version or a lookup failure. */ +export const npmReleaseLookup = ( + response: Result, +): Result => { + if (!response.ok) return err({ kind: "unavailable", detail: response.error }); + const parsed = parseNpmLatestVersion(response.value); + return parsed.ok + ? ok(parsed.value) + : err({ kind: "invalid-metadata", detail: parsed.error }); +}; + /** Create npm, registry, verification, and read-only maintenance effects. */ export const systemUpdateHost = ( packageRoot = fileURLToPath(new URL("../..", import.meta.url)), @@ -127,28 +182,10 @@ export const systemUpdateHost = ( globalRoot: () => requireCommandOutput(["npm", "root", "--global"]), globalPrefix: () => requireCommandOutput(["npm", "prefix", "--global"]), }), - latestVersion: async (installation) => { - const response = await runUpdateCommand([ - "npm", - "view", - "--global", - ...(installation === undefined ? [] : ["--prefix", installation.prefix]), - PRODUCT_IDENTITY.packageName, - "dist-tags.latest", - "--json", - "--fetch-retries=0", - "--fetch-timeout=10000", - ]); - if (!response.ok) return response; - try { - const parsed = z.string().min(1).safeParse(JSON.parse(response.value)); - return parsed.success - ? ok(parsed.data) - : err(`Invalid npm release metadata: ${parsed.error.message}`); - } catch (cause: unknown) { - return err(cause instanceof Error ? cause.message : String(cause)); - } - }, + latestVersion: async (installation) => + npmReleaseLookup( + await runUpdateCommand(npmLatestVersionCommand(installation)), + ), installVersion: async (installation, version, output) => { const result = await runUpdateCommand( updateInstallCommand(installation, version), diff --git a/src/cliLogging.test.ts b/src/cliLogging.test.ts index b37a6f183e9..e275519e6d0 100644 --- a/src/cliLogging.test.ts +++ b/src/cliLogging.test.ts @@ -88,7 +88,10 @@ describe("CLI command outcome classification", () => { }; const updateHost: UpdateHost = { installation: () => Promise.resolve(installation), - latestVersion: () => Promise.resolve(err("registry unavailable")), + latestVersion: () => + Promise.resolve( + err({ kind: "unavailable", detail: "registry unavailable" }), + ), installVersion: () => Promise.resolve(err("not called")), installedVersion: () => Promise.resolve(err("not called")), planMaintenance: () => From 9fd65ce0711ce9257aee15e62a5ecf9832960035 Mon Sep 17 00:00:00 2001 From: lalo adrian morales <146933685+lalomorales22@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:30:46 -0700 Subject: [PATCH 048/791] fix(setup): treat empty client config files as new documents (#1117) * fix(setup): treat empty client config files as new documents An empty MCP config file (for example Antigravity's mcp_config.json, which the client can create with zero bytes) was reported as malformed, which blocked rea setup from configuring every other detected agent. Parse a whitespace- or BOM-only JSON/JSONC file as an empty document and write a fresh document over it. Co-Authored-By: Claude Opus 5.5 * test(setup): use explicit unicode escape for BOM-only empty config case --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../ClientConfigurationDocument.ts | 18 ++++++++++- .../filesystem/clientConfigurationBom.test.ts | 30 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index 0b96a79d37c..2b8a83a2258 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -80,12 +80,22 @@ export const clientConfigurationServersKey = ( } }; +/** + * Whether raw configuration text holds no settings to preserve. Some clients + * create a zero-byte config file before any server is added. + */ +const isEmptyClientConfigurationText = (text: string): boolean => + text.replace(/^\uFEFF/, "").trim() === ""; + const parseDocument = ( text: string, format: ClientConfigurationFormat, ): Record => { if (format === "toml" || format === "grok") return objectSchema.parse(parseToml(text)); + // An empty file, which some clients create before any server is added, + // holds no settings to preserve. + if (isEmptyClientConfigurationText(text)) return {}; const errors: ParseError[] = []; // Accept a UTF-8 BOM without shifting diagnostics or editing the original text. const jsonText = text.startsWith("\uFEFF") ? ` ${text.slice(1)}` : text; @@ -1148,7 +1158,13 @@ export const serializeClientConfiguration = ( if (format === "toml") return stringifyToml(document); if (format === "grok") return serializeGrokConfiguration(document, originalText); - if (originalText !== undefined && editedPaths !== undefined) + // An empty original holds no comments or settings to preserve. Writing a + // fresh document also keeps a lone BOM from shifting past the closing brace. + if ( + originalText !== undefined && + !isEmptyClientConfigurationText(originalText) && + editedPaths !== undefined + ) return editedPaths.reduce((text, path) => { const value = valueAt(document, path); if (value === undefined) { diff --git a/tests/boundary/filesystem/clientConfigurationBom.test.ts b/tests/boundary/filesystem/clientConfigurationBom.test.ts index 92b5d9628c1..7dcf9ae04fc 100644 --- a/tests/boundary/filesystem/clientConfigurationBom.test.ts +++ b/tests/boundary/filesystem/clientConfigurationBom.test.ts @@ -149,3 +149,33 @@ it("restores original BOM-prefixed JSONC bytes after update and removal", async }); expect(await readFile(client.configPath, "utf8")).toBe(original); }); + +it.each(["", "\n \n", "\uFEFF"])( + "configures an empty client file %j as a new document", + async (original) => { + const home = await createTestTempDirectory("rea-client-config-empty-"); + const client = supportedClients(home, "linux", {}).find( + (candidate) => candidate.name === "antigravity", + ); + if (client?.format !== "json") + throw new Error("missing Antigravity JSON config"); + await mkdir(dirname(client.configPath), { recursive: true }); + await writeFile(client.configPath, original); + const command = [join(home, "rea"), "mcp"]; + + expect(await inspectClientConfiguration(client, {}, command)).toEqual({ + status: "update", + backupPath: `${client.configPath}.rea.backup`, + }); + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "configured", + backupPath: `${client.configPath}.rea.backup`, + }); + expect(JSON.parse(await readFile(client.configPath, "utf8"))).toEqual({ + mcpServers: { rea: clientRegistrationEntry(client.format, command, {}) }, + }); + expect(await inspectClientConfiguration(client, {}, command)).toEqual({ + status: "already_current", + }); + }, +); From 14b19a0b8a5af401212c552173e761eac53bfe46 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:50:16 +0800 Subject: [PATCH 049/791] fix(native): resample tool identity for each invocation --- src/native/CommandRunner.ts | 21 ++---- .../providers/native/nativeProvider.test.ts | 65 +++++++++++++------ 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/src/native/CommandRunner.ts b/src/native/CommandRunner.ts index 37794d15822..7d11cda3e28 100644 --- a/src/native/CommandRunner.ts +++ b/src/native/CommandRunner.ts @@ -83,8 +83,6 @@ export type NativeToolResolver = ( /** Run allowlisted Xcode tools directly without a shell. */ export class XcrunCommandRunner implements NativeCommandRunner { - readonly #resolved = new Map(); - constructor( private readonly resolveTool: NativeToolResolver = (tool, signal) => resolveXcrunTool(tool, signal), @@ -99,7 +97,7 @@ export class XcrunCommandRunner implements NativeCommandRunner { return Promise.resolve( err(new NativeCommandFailure(tool, "unavailable")), ); - return this.#resolve(tool, options.signal).then(async (resolved) => { + return this.resolveTool(tool, options.signal).then(async (resolved) => { if (!resolved.ok) return resolved; const captured = await captureProcess( resolved.value.path, @@ -115,26 +113,15 @@ export class XcrunCommandRunner implements NativeCommandRunner { executableSha256: resolved.value.sha256, toolVersion: null, versionReason: - "Tool exposes no uniform stable version flag; executable digest identifies it.", + "Tool exposes no uniform stable version flag; executable SHA-256 is a prelaunch file sample and does not prove atomic OS image binding.", arguments: [...arguments_], }) : captured; }); } - - async #resolve( - tool: string, - signal?: AbortSignal, - ): Promise> { - const existing = this.#resolved.get(tool); - if (existing !== undefined) return ok(existing); - const resolved = await this.resolveTool(tool, signal); - if (resolved.ok) this.#resolved.set(tool, resolved.value); - return resolved; - } } -/** Immutable executable identity returned by native tool discovery. */ +/** Executable path and SHA-256 sampled during one tool resolution. */ export interface ResolvedTool { readonly path: string; readonly sha256: string; @@ -154,7 +141,7 @@ const ALLOWED_TOOLS = new Set([ "vtool", ]); -/** Locate an Xcode tool through `xcrun --find` and pin its executable digest. */ +/** Locate an Xcode tool through `xcrun --find` and sample its executable digest. */ export const resolveXcrunTool = async ( tool: string, signal?: AbortSignal, diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index 91b6d18d72b..9260e7fd179 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -1,4 +1,5 @@ -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { afterEach, beforeAll, describe, expect, it } from "vitest"; @@ -74,24 +75,6 @@ describe("native macOS provider discovery", () => { }); }); - it("retries a failed native tool resolution and caches only success", async () => { - let resolutions = 0; - const runner = new XcrunCommandRunner((tool) => { - resolutions += 1; - return Promise.resolve( - resolutions === 1 - ? err(new NativeCommandFailure(tool, "unavailable")) - : ok({ path: "/usr/bin/true", sha256: "a".repeat(64) }), - ); - }); - const options = {}; - - expect((await runner.run("file", [], options)).ok).toBe(false); - expect((await runner.run("file", [], options)).ok).toBe(true); - expect((await runner.run("file", [], options)).ok).toBe(true); - expect(resolutions).toBe(2); - }); - it.each(["timeout", "output-limit"] as const)( "preserves %s resolution failures with their captured output and cleanup status", async (reason) => { @@ -134,6 +117,50 @@ describe("native macOS provider discovery", () => { }); describe("native command output collection", () => { + it("retries discovery and resamples the executable after its tool path is replaced", async () => { + directory = await createTestTempDirectory("rea-native-tool-identity-"); + const executable = join(directory, "fixture-tool"); + const source = (value: string): string => + `#!/bin/sh\nprintf '%s\\n' '${value}'\n`; + const install = async (value: string): Promise => { + const contents = source(value); + await writeFile(executable, contents, { mode: 0o700 }); + await chmod(executable, 0o700); + return createHash("sha256").update(contents).digest("hex"); + }; + let unavailable = true; + const runner = new XcrunCommandRunner(async (tool) => { + if (unavailable) { + unavailable = false; + return err(new NativeCommandFailure(tool, "unavailable")); + } + const digest = createHash("sha256") + .update(await readFile(executable)) + .digest("hex"); + return ok({ path: executable, sha256: digest }); + }); + + const originalDigest = await install("original-tool-output"); + expect(await runner.run("file", [], {})).toMatchObject({ + ok: false, + error: { reason: "unavailable" }, + }); + const original = await runner.run("file", [], {}); + if (!original.ok) throw original.error; + expect(original.value.stdout).toBe("original-tool-output\n"); + expect(original.value.executableSha256).toBe(originalDigest); + + const replacementDigest = await install("replacement-tool-output"); + const replacement = await runner.run("file", [], {}); + if (!replacement.ok) throw replacement.error; + expect(replacement.value.stdout).toBe("replacement-tool-output\n"); + expect(replacement.value.executableSha256).toBe(replacementDigest); + expect(replacement.value.versionReason).toContain("prelaunch file sample"); + expect(replacement.value.versionReason).toContain( + "does not prove atomic OS image binding", + ); + }); + it("retains the complete native command output", async () => { const runner = new XcrunCommandRunner(() => Promise.resolve(ok({ path: process.execPath, sha256: "a".repeat(64) })), From 649517181268a0b729970b7cbbda15e04b199e32 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:50:23 +0800 Subject: [PATCH 050/791] fix(javascript): classify direct defaults and argv elements --- .../javascriptSemanticDataEffectHelpers.ts | 13 --- ...vascriptSemanticDataEffects.lookup.test.ts | 23 ---- .../javascriptSemanticDataEffects.ts | 73 ++++-------- .../javascriptBuiltinDefaultsCli.test.ts | 104 ++++++++++++++++++ 4 files changed, 128 insertions(+), 85 deletions(-) delete mode 100644 src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts diff --git a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts index 3fa77761006..f98b791dcab 100644 --- a/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts +++ b/src/domain/javascript/javascriptSemanticDataEffectHelpers.ts @@ -287,16 +287,3 @@ export const dataEffectMemberObject = ( (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) ? node.object : null; - -/** Compare Babel source offsets for exact containment. */ -export const containsSemanticNode = (outer: t.Node, inner: t.Node): boolean => - outer.start !== null && - outer.start !== undefined && - outer.end !== null && - outer.end !== undefined && - inner.start !== null && - inner.start !== undefined && - inner.end !== null && - inner.end !== undefined && - outer.start <= inner.start && - outer.end >= inner.end; diff --git a/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts b/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts deleted file mode 100644 index e9536443f68..00000000000 --- a/src/domain/javascript/javascriptSemanticDataEffects.lookup.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { expect, it } from "vitest"; - -import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; - -it("keeps defaults linked to their own config reads across a large source", () => { - const count = 400; - const source = Array.from( - { length: count }, - (_, index) => - `const value${index} = process.env.KEY_${index} ?? "fallback-${index}";`, - ).join("\n"); - - const operations = analyzeJavaScriptSemantics(source).configurationOperations; - const defaults = operations.filter(({ kind }) => kind === "default"); - - expect(defaults).toHaveLength(count); - for (let index = 0; index < count; index += 1) { - expect(defaults.find(({ key }) => key === `KEY_${index}`)).toMatchObject({ - key: `KEY_${index}`, - value: `fallback-${index}`, - }); - } -}); diff --git a/src/domain/javascript/javascriptSemanticDataEffects.ts b/src/domain/javascript/javascriptSemanticDataEffects.ts index 18abeb532af..2819ea4bd3b 100644 --- a/src/domain/javascript/javascriptSemanticDataEffects.ts +++ b/src/domain/javascript/javascriptSemanticDataEffects.ts @@ -20,7 +20,6 @@ import { } from "./javascriptSemanticState.js"; import { builtinDataEffectMethod as builtinMethod, - containsSemanticNode as containsNode, dataEffectArgumentBindingId as argumentBindingId, dataEffectLiteralString as literalString, dataEffectMemberCallee as memberCallee, @@ -150,6 +149,7 @@ const processConfigurationMember = ( if (family !== "env" && family !== "argv") return null; // `process.env[""]` names an exact (empty) key; only a dynamic key is unknown. const key = semanticStaticPropertyKey(node.property, node.computed); + if (family === "argv" && key !== null && !isArrayIndexKey(key)) return null; return { kind: family === "env" ? "environment" : "argv", key, @@ -157,39 +157,47 @@ const processConfigurationMember = ( }; }; +const isArrayIndexKey = (key: string): boolean => { + if (!/^(?:0|[1-9][0-9]*)$/u.test(key)) return false; + const index = Number(key); + return Number.isSafeInteger(index) && index < 2 ** 32 - 1; +}; + const collectDefaults = ( program: t.Program, context: TraversalContext, configurations: readonly ConfigurationCandidate[], ): JavaScriptSemanticConfigurationOperation[] => { const output: JavaScriptSemanticConfigurationOperation[] = []; - const configurationsByStart = configurations - .filter( - ({ node }) => - node.start !== null && - node.start !== undefined && - node.end !== null && - node.end !== undefined, - ) - .toSorted( - (left, right) => (left.node.start ?? 0) - (right.node.start ?? 0), - ); + const configurationsByNode = new Map( + configurations.map((candidate) => [candidate.node, candidate]), + ); traverseWithContext(program, context, (node) => { if ( !t.isLogicalExpression(node) || (node.operator !== "??" && node.operator !== "||") ) return; - const sources = configurationsIn(node.left, configurationsByStart); - if (sources.length !== 1 || sources[0] === undefined) return; + let source = node.left; + while ( + t.isParenthesizedExpression(source) || + t.isTSAsExpression(source) || + t.isTSTypeAssertion(source) || + t.isTypeCastExpression(source) || + t.isTSSatisfiesExpression(source) || + t.isTSNonNullExpression(source) + ) + source = source.expression; + const candidate = configurationsByNode.get(source); + if (candidate === undefined) return; output.push({ configId: `config:default:${String(node.start ?? -1)}:${String(node.end ?? -1)}`, kind: "default", location: range(node.right), ownerCallableId: context.callableStack.at(-1) ?? null, resultBindingId: outerAssignedBinding(node, context), - key: sources[0].operation.key, - sourceConfigId: sources[0].operation.configId, + key: candidate.operation.key, + sourceConfigId: candidate.operation.configId, value: primitiveValue(node.right), resolution: primitiveValue(node.right) === null ? "partial" : "complete", }); @@ -197,39 +205,6 @@ const collectDefaults = ( return output; }; -/** Find only configuration candidates whose source offsets can fit this left operand. */ -const configurationsIn = ( - expression: t.Node, - candidates: readonly ConfigurationCandidate[], -): readonly ConfigurationCandidate[] => { - if ( - expression.start === null || - expression.start === undefined || - expression.end === null || - expression.end === undefined - ) - return []; - let low = 0; - let high = candidates.length; - while (low < high) { - const middle = low + Math.floor((high - low) / 2); - const start = candidates[middle]?.node.start; - if (start !== null && start !== undefined && start < expression.start) - low = middle + 1; - else high = middle; - } - const sources: ConfigurationCandidate[] = []; - for (let index = low; index < candidates.length; index += 1) { - const candidate = candidates[index]; - if (candidate === undefined) continue; - const start = candidate.node.start; - if (start === null || start === undefined || start > expression.end) break; - if (containsNode(expression, candidate.node)) sources.push(candidate); - if (sources.length > 1) break; - } - return sources; -}; - const collectRequests = ( program: t.Program, context: TraversalContext, diff --git a/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts b/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts index 729a504fc03..6dbf855fc4a 100644 --- a/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts +++ b/tests/acceptance/applications/javascriptBuiltinDefaultsCli.test.ts @@ -53,3 +53,107 @@ it.each([ }, 20_000, ); + +it("keeps only direct logical defaults and actual argv elements in the public CLI graph", async () => { + const root = await createTestTempDirectory("rea-builtin-default-cli-"); + const manyDefaults = Array.from( + { length: 400 }, + (_, index) => + `const value${index} = process.env.KEY_${index} ?? "fallback-${index}";`, + ).join("\n"); + await writeFile( + join(root, "app.js"), + [ + `const chained = process.env.CHAINED ?? "fallback" ?? "unreachable"; +const wrapped = ((process.env.WRAPPED)) ?? "wrapped"; +const argument = process.argv[2]; +const argumentLength = process.argv.length; +const namedArgument = process.argv["02"]; +const negativeArgument = process.argv["-1"]; +const method = process.argv.map; +const lastArrayIndex = process.argv[4294967294]; +const outOfRangeArgument = process.argv[4294967295]; +const dynamicArgument = process.argv[process.env.ARGV_INDEX]; +function shadowed(process) { return process.argv[3]; }`, + manyDefaults, + ].join("\n"), + ); + const { stdout } = await execute( + process.execPath, + ["scripts/rea.mjs", "analyze-javascript-application", root, "--json"], + { cwd: process.cwd(), maxBuffer: 16 * 1024 * 1024 }, + ); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(JSON.parse(stdout)).normalized_result, + ); + const configSources = result.semantic_graph.nodes.filter( + ({ kind }) => kind === "config-source", + ); + const defaults = configSources.filter( + ({ properties }) => properties.source_kind === "default", + ); + expect(defaults).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + key: "CHAINED", + value: "fallback", + }), + }), + expect.objectContaining({ + properties: expect.objectContaining({ + key: "WRAPPED", + value: "wrapped", + }), + }), + ]), + ); + expect( + defaults.filter(({ properties }) => properties.key === "CHAINED"), + ).toHaveLength(1); + expect(defaults).toHaveLength(402); + const defaultsByKey = new Map( + defaults.map(({ properties }) => [properties.key, properties.value]), + ); + for (let index = 0; index < 400; index += 1) + expect(defaultsByKey.get(`KEY_${index}`)).toBe(`fallback-${index}`); + const argvSources = configSources.filter( + ({ properties }) => properties.source_kind === "argv", + ); + expect(argvSources).toHaveLength(3); + expect(argvSources).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ key: "2" }), + }), + expect.objectContaining({ + properties: expect.objectContaining({ key: "4294967294" }), + }), + expect.objectContaining({ + properties: expect.objectContaining({ key: null }), + }), + ]), + ); + expect( + result.semantic_graph.relations + .filter(({ relation }) => relation === "reads-argv") + .map(({ resolution }) => resolution), + ).toEqual(expect.arrayContaining(["resolved", "candidate"])); + expect( + result.semantic_graph.relations.filter( + ({ relation, source_node_id, target_node_id }) => { + if (relation !== "overrides") return false; + const source = result.semantic_graph.nodes.find( + ({ node_id }) => node_id === source_node_id, + ); + const target = result.semantic_graph.nodes.find( + ({ node_id }) => node_id === target_node_id, + ); + return ( + source?.properties.key === "CHAINED" && + target?.properties.key === "CHAINED" + ); + }, + ), + ).toHaveLength(1); +}, 20_000); From d575a7340bb19b84733c79b14de9f4df7181c6b3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:50:49 +0800 Subject: [PATCH 051/791] fix(apple): decode bundle plist bytes before executable lookup --- docs/native-investigation.md | 7 +- src/application/AppBundleExecutable.ts | 5 +- .../apple/InterfaceBuilderAnalysis.ts | 2 +- .../apple/InterfaceBuilderDecodeBudget.ts | 2 +- src/artifacts/apple/KeyedArchiveReader.ts | 2 +- .../propertyListXmlText.ts} | 4 +- .../applications/appBundleEncodingCli.test.ts | 188 ++++++++++++++++++ 7 files changed, 200 insertions(+), 10 deletions(-) rename src/{artifacts/apple/XmlPropertyListText.ts => domain/propertyListXmlText.ts} (90%) create mode 100644 tests/acceptance/applications/appBundleEncodingCli.test.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 4a98b90d8b1..753ebdb3ec3 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -59,11 +59,14 @@ complete observation. a source-owned AppKit XIB with `ibtool`. Storyboards require an installed iOS platform; unsupported archive forms remain explicit. XML plist archives accept UTF-8 and BOM-marked UTF-16 in either byte order. - The shared keyed-archive and Interface Builder decoder consumes an initial + App bundle `Info.plist` executable resolution uses the same decoder as keyed + archives and Interface Builder. The decoder consumes an initial BOM and rejects malformed byte sequences, unsupported encoding declarations, and declarations that disagree with the detected encoding. It never substitutes UTF-8 for a declared encoding it cannot process. Archive and evidence digests - still identify the original serialized bytes. Interface Builder decoding + still identify the original serialized bytes. Bundle resolution reports + malformed bytes and unsupported encodings against the original plist path + before looking up the executable. Interface Builder decoding limits aggregate input archives to 32 MiB. Before decoding, it also bounds the serialized structure: expanded binary plist references, NIB table records, XML elements and text, plus their diff --git a/src/application/AppBundleExecutable.ts b/src/application/AppBundleExecutable.ts index 06d2251a684..7df3ca03aa5 100644 --- a/src/application/AppBundleExecutable.ts +++ b/src/application/AppBundleExecutable.ts @@ -8,6 +8,7 @@ import { z } from "zod"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { isPathWithinRoot } from "../domain/localPath.js"; import { parseXmlPropertyList } from "../domain/propertyListKeys.js"; +import { decodeXmlPlistText } from "../domain/propertyListXmlText.js"; import { err, ok, type Result } from "../domain/result.js"; const execFileAsync = promisify(execFile); @@ -194,7 +195,7 @@ const resolveLayoutExecutable = async ( ? await (fileSystem.decodeBinaryPlist ?? readBinaryPlistExecutable)( plistPath, ) - : parseXmlPlistExecutable(plist.toString("utf8")); + : parseXmlPlistExecutable(decodeXmlPlistText(plist)); } catch (cause: unknown) { return err( new BinaryTargetError( @@ -203,7 +204,7 @@ const resolveLayoutExecutable = async ( ? `permission denied decoding app Info.plist ${plistPath}${codeDescription(cause)}` : isSystemErrorCode(cause) ? `could not decode app Info.plist ${plistPath}${codeDescription(cause)}` - : `app Info.plist is malformed or lacks CFBundleExecutable: ${plistPath}`, + : `app Info.plist is malformed, unsupported, or lacks CFBundleExecutable: ${plistPath}${cause instanceof Error ? ` (${cause.message})` : ""}`, { cause }, ), ); diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index 00b19fe15ed..f8358d3a82c 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -26,7 +26,7 @@ import { } from "./NibViewHierarchy.js"; import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js"; -import { decodeXmlPlistText } from "./XmlPropertyListText.js"; +import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; const MAX_AGGREGATE_INPUT_BYTES = 32 * 1024 * 1024; const MAX_AGGREGATE_DECODE_BYTES = 256 * 1024 * 1024; diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index 6ec1c82f15a..58a9b1ce3be 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -1,4 +1,4 @@ -import { decodeXmlPlistText } from "./XmlPropertyListText.js"; +import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; /** Reason reported when an archive would exceed the shared decoding budget. */ export type InterfaceBuilderBudgetReason = "aggregate_decode_budget_exhausted"; diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts index 795f5b1269a..54401c4a1ed 100644 --- a/src/artifacts/apple/KeyedArchiveReader.ts +++ b/src/artifacts/apple/KeyedArchiveReader.ts @@ -16,7 +16,7 @@ import { import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; -import { decodeXmlPlistText } from "./XmlPropertyListText.js"; +import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js"; const MAX_BYTES = 64 * 1024 * 1024; diff --git a/src/artifacts/apple/XmlPropertyListText.ts b/src/domain/propertyListXmlText.ts similarity index 90% rename from src/artifacts/apple/XmlPropertyListText.ts rename to src/domain/propertyListXmlText.ts index 8fb472d048a..2d6820120f6 100644 --- a/src/artifacts/apple/XmlPropertyListText.ts +++ b/src/domain/propertyListXmlText.ts @@ -1,7 +1,5 @@ -import { TextDecoder } from "node:util"; - /** Decode BOM-aware XML plist text with fatal byte validation. */ -export const decodeXmlPlistText = (bytes: Buffer): string => { +export const decodeXmlPlistText = (bytes: Uint8Array): string => { const encoding = bytes[0] === 0xff && bytes[1] === 0xfe ? "utf-16le" diff --git a/tests/acceptance/applications/appBundleEncodingCli.test.ts b/tests/acceptance/applications/appBundleEncodingCli.test.ts new file mode 100644 index 00000000000..e19ac7333cd --- /dev/null +++ b/tests/acceptance/applications/appBundleEncodingCli.test.ts @@ -0,0 +1,188 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { describe, expect, onTestFinished } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const encodingFixture = () => { + const xml = + 'CFBundleExecutableAppCFBundleIdentifierdev.rea.encoding'; + const utf16 = Buffer.from(xml, "utf16le"); + const utf8 = xml.replace("UTF-16", "UTF-8"); + const cases = [ + { + name: "UTF-8 BOM", + bytes: Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from(utf8), + ]), + valid: true, + }, + { + name: "UTF-16 LE", + bytes: Buffer.concat([Buffer.from([0xff, 0xfe]), utf16]), + valid: true, + }, + { + name: "UTF-16 BE", + bytes: Buffer.concat([ + Buffer.from([0xfe, 0xff]), + Buffer.from(utf16).swap16(), + ]), + valid: true, + }, + { + name: "unsupported declaration", + bytes: Buffer.from(utf8.replace("UTF-8", "Shift_JIS")), + valid: false, + }, + { + name: "declaration mismatch", + bytes: Buffer.from(xml), + valid: false, + }, + { + name: "truncated UTF-16", + bytes: Buffer.concat([Buffer.from([0xff, 0xfe]), utf16.subarray(0, -1)]), + valid: false, + }, + { + name: "malformed UTF-8 executable", + bytes: Buffer.concat([ + Buffer.from(utf8.split("App")[0] + ""), + Buffer.from([0xff]), + Buffer.from("" + utf8.split("App")[1]), + ]), + valid: false, + }, + ]; + return { utf16, cases }; +}; + +describe.skipIf(process.platform !== "darwin")( + "app bundle XML byte boundaries", + () => { + cliTest( + "resolves real compiled bundles without changing executable names during decoding", + async ({ cli, processes }) => { + const directory = await createTestTempDirectory("rea-bundle-encoding-"); + const app = join(directory, "Example.app"); + const contents = join(app, "Contents"); + await mkdir(join(contents, "MacOS"), { recursive: true }); + const compiled = await processes.run("/usr/bin/xcrun", [ + "clang", + resolve("tests/conformance/interface-builder/fixture.c"), + "-o", + join(contents, "MacOS", "App"), + ]); + expect(compiled.exitCode, compiled.stderr).toBe(0); + const { utf16, cases } = encodingFixture(); + for (const { name, bytes, valid } of cases) { + await writeFile(join(contents, "Info.plist"), bytes); + const result = await cli.run({ + arguments: ["inspect-plist", app, "--json"], + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + }); + expect( + result.exitCode, + `${name}: ${result.stdout}\n${result.stderr}`, + ).toBe(valid ? 0 : 1); + if (valid) + expect(result.json).toMatchObject({ + normalized_result: { + value: { + CFBundleExecutable: "App", + CFBundleIdentifier: "dev.rea.encoding", + }, + bundle: { executable: "App" }, + }, + }); + else + expect(result.json).toMatchObject({ + error: "Analysis failed", + details: { + path: join(contents, "Info.plist"), + reason: expect.stringContaining( + "app Info.plist is malformed, unsupported, or lacks CFBundleExecutable", + ), + }, + }); + } + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + env: { + PATH: process.env.PATH ?? "", + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + stderr: "pipe", + }); + const client = new Client({ + name: "bundle-encoding-parity", + version: "1", + }); + onTestFinished(async () => { + try { + await client.close(); + } finally { + await transport.close(); + } + }); + await client.connect(transport); + await writeFile( + join(contents, "Info.plist"), + Buffer.concat([ + Buffer.from([0xfe, 0xff]), + Buffer.from(utf16).swap16(), + ]), + ); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: app }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_plist", + arguments: {}, + }); + expect(inspected.isError, JSON.stringify(inspected)).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + result: { + value: { + CFBundleExecutable: "App", + CFBundleIdentifier: "dev.rea.encoding", + }, + }, + }); + await writeFile( + join(contents, "Info.plist"), + Buffer.concat([Buffer.from([0xff, 0xfe]), utf16.subarray(0, -1)]), + ); + const invalid = await client.callTool({ + name: "open_binary", + arguments: { path: app }, + }); + expect(invalid.isError).toBe(true); + expect(invalid.structuredContent).toMatchObject({ + error: { + details: { + path: join(contents, "Info.plist"), + reason: expect.stringContaining( + "app Info.plist is malformed, unsupported, or lacks CFBundleExecutable", + ), + }, + }, + }); + }, + 45_000, + ); + }, +); From e1df5e85551125b27ef0debb7573de5507b149e1 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:53:00 +0800 Subject: [PATCH 052/791] fix(apple): account for repeated archive projections --- docs/native-investigation.md | 8 +- docs/testing.md | 5 +- scripts/verify/apple/interface-builder.mjs | 32 +- .../apple/InterfaceBuilderAnalysis.ts | 105 ++++--- .../apple/InterfaceBuilderDecodeBudget.ts | 29 +- src/artifacts/apple/NibArchive.ts | 2 +- .../filesystem/interfaceBuilderBundle.test.ts | 280 +++++++++++++++++- 7 files changed, 391 insertions(+), 70 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 753ebdb3ec3..5d97e7c6ffc 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -71,8 +71,12 @@ complete observation. Before decoding, it also bounds the serialized structure: expanded binary plist references, NIB table records, XML elements and text, plus their projected representations must fit the 256 MiB aggregate decode budget. - This is a format-derived representation estimate, not a process-wide heap - ceiling. + NIB string values, data, property names and class names are charged at each + projected JSON occurrence, including when the archive reuses one source + value. Binary plist data projection also accounts for the temporary byte + copy made by the pinned decoder projection. These are format-derived + representation reservations, not exact heap measurements or a process-wide + heap ceiling. Archives beyond either aggregate budget are omitted with their paths and an explicitly partial `archive_decode` facet; previously decoded documents and their identities remain available. diff --git a/docs/testing.md b/docs/testing.md index 4610c4fa0ec..bbfc70a3c86 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -526,8 +526,9 @@ runtime or PC-98 device execution. ## Apple Interface Builder archives `npm run verify:interface-builder` compiles the source-owned AppKit XIB into a -real `.nib` with Xcode `ibtool`, wraps it in a temporary app bundle, and checks -the decoded view hierarchy, outlet, action, evidence coverage, and truncation +real `.nib` with Xcode `ibtool`, wraps it in a temporary app bundle, and runs +the compiled CLI and stdio MCP server with production providers. It compares +their decoded results and checks the view hierarchy, outlet, action, evidence coverage, and truncation status. Storyboard compilation additionally requires an installed iOS platform. Keep the provider-specific acceptance path independent from optional diff --git a/scripts/verify/apple/interface-builder.mjs b/scripts/verify/apple/interface-builder.mjs index fb3346f738c..a27bf656b77 100644 --- a/scripts/verify/apple/interface-builder.mjs +++ b/scripts/verify/apple/interface-builder.mjs @@ -8,8 +8,11 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { parseBinaryTarget } from "../../../dist/application/BinaryTargetResolver.js"; -import { ArtifactProvider } from "../../../dist/artifacts/ArtifactProvider.js"; +import { + artifactCliEvidence, + artifactMcpResult, + withArtifactMcp, +} from "../../lib/artifact-e2e.mjs"; const exec = promisify(execFile); const fixtureRoot = await mkdtemp(join(tmpdir(), "rea-interface-builder-")); @@ -41,14 +44,19 @@ try { ]); await exec("cc", [join(sourceRoot, "fixture.c"), "-o", executablePath]); - const target = await parseBinaryTarget(appPath); - if (!target.ok) throw target.error; - const execution = await new ArtifactProvider() - .createClient(target.value) - .execute("decode_interface_builder", {}, {}); - if (!execution.ok) throw execution.error; - - const decoded = execution.value.result; + const evidence = await artifactCliEvidence( + "decode-interface-builder", + appPath, + ); + const decoded = evidence.normalized_result; + await withArtifactMcp(appPath, async (client) => { + const mcp = await artifactMcpResult(client, "decode_interface_builder"); + assert.deepEqual( + mcp, + decoded, + "CLI and MCP compiled archive results differ", + ); + }); assert.equal(decoded.documents.length, 1); assert.equal( decoded.documents[0]?.relative_path, @@ -87,7 +95,9 @@ try { report = { ok: true, fixture: "compiled-appkit-nib", - provider: execution.value.provider, + provider: evidence.provider, + public_cli: true, + public_stdio_mcp: true, target_sha256: decoded.target_sha256, archive_sha256: decoded.documents[0].archive_sha256, objects: decoded.documents[0].object_count, diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index f8358d3a82c..f47c5c56c99 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -278,6 +278,14 @@ const projectNibArchive = ( maxHierarchyNodes: number, ) => { const byId = new Map(archive.objects.map((object) => [object.id, object])); + const normalizedClassNames = new Map(); + const normalizeClassName = (value: string): string => { + const known = normalizedClassNames.get(value); + if (known !== undefined) return known; + const normalized = value.replace(/\0+$/u, ""); + normalizedClassNames.set(value, normalized); + return normalized; + }; const decodedStrings = new Map(); const decodedString = ( objectId: number, @@ -303,14 +311,6 @@ const projectNibArchive = ( decodedStrings.set(objectId, value); return value; }; - const reserveStringOccurrence = (value: string | null): string | null => { - if (value !== null) - budget.reserve( - 64 + value.length * 6, - "NIBArchive repeated string projection exceeds the aggregate Interface Builder decode budget", - ); - return value; - }; let projectedFields = 0; for (const object of archive.objects) projectedFields += Object.keys(object.values).length; @@ -325,13 +325,21 @@ const projectNibArchive = ( const dereference = (value: JsonValue | undefined): JsonValue | undefined => { if (typeof value !== "object" || value === null || Array.isArray(value)) return value; + const encodedData = value.$nib_data_base64; + if (typeof encodedData === "string") { + budget.reserveBase64JsonStringOccurrence(encodedData); + return value; + } const ref = value.$nib_object_ref; if (typeof ref !== "number") return value; const target = byId.get(ref); if (target === undefined) return undefined; - if (target.class_name.replace(/\0+$/u, "") === "NSString") { + if (normalizeClassName(target.class_name) === "NSString") { const string = decodedString(ref, target); - if (string !== null) return reserveStringOccurrence(string); + if (string !== null) { + budget.reserveJsonStringOccurrence(string); + return string; + } } return { objectID: String(ref) }; }; @@ -341,18 +349,31 @@ const projectNibArchive = ( const ref = value.$nib_object_ref; if (typeof ref !== "number") return null; const target = byId.get(ref); - if (target?.class_name.replace(/\0+$/u, "") !== "NSString") return null; - return reserveStringOccurrence(decodedString(ref, target)); + if ( + target === undefined || + normalizeClassName(target.class_name) !== "NSString" + ) + return null; + const string = decodedString(ref, target); + if (string === null) return null; + budget.reserveJsonStringOccurrence(string); + return string; }; + const runtimeClasses = new Map(); const runtimeClass = (objectId: number): string | null => { + if (runtimeClasses.has(objectId)) + return runtimeClasses.get(objectId) ?? null; const object = byId.get(objectId); if (object === undefined) return null; - const storedClass = object.class_name.replace(/\0+$/u, ""); - if (storedClass === "NSClassSwapper") - return referencedString(object.values.NSClassName) ?? storedClass; - return storedClass === "UIClassSwapper" - ? (referencedString(object.values.UIClassName) ?? storedClass) - : storedClass; + const storedClass = normalizeClassName(object.class_name); + const resolvedClass = + storedClass === "NSClassSwapper" + ? (referencedString(object.values.NSClassName) ?? storedClass) + : storedClass === "UIClassSwapper" + ? (referencedString(object.values.UIClassName) ?? storedClass) + : storedClass; + runtimeClasses.set(objectId, resolvedClass); + return resolvedClass; }; const objects = Object.fromEntries( archive.objects @@ -364,19 +385,22 @@ const projectNibArchive = ( }) .map((object) => { const className = - runtimeClass(object.id) ?? object.class_name.replace(/\0+$/u, ""); - const strings = Object.entries(object.values).flatMap( - ([key, value]) => { - const resolved = dereference(value); - return typeof resolved === "string" ? [[key, resolved]] : []; - }, - ); - const name = - strings.find(([key]) => - /(?:title|label|identifier|accessibility|name|contents)/iu.test( - key ?? "", - ), - )?.[1] ?? className; + runtimeClass(object.id) ?? normalizeClassName(object.class_name); + budget.reserveJsonStringOccurrence(className); + let name = className; + for (const [key, value] of Object.entries(object.values)) { + if ( + !/(?:title|label|identifier|accessibility|name|contents)/iu.test( + key, + ) + ) + continue; + const resolved = dereference(value); + if (typeof resolved !== "string") continue; + name = resolved; + break; + } + budget.reserveJsonStringOccurrence(name); return [ String(object.id), { @@ -384,10 +408,10 @@ const projectNibArchive = ( label: name, objectID: String(object.id), nibValues: Object.fromEntries( - Object.entries(object.values).map(([key, value]) => [ - key, - dereference(value) ?? null, - ]), + Object.entries(object.values).map(([key, value]) => { + budget.reserveJsonPropertyNameOccurrence(key); + return [key, dereference(value) ?? null]; + }), ), }, ]; @@ -403,7 +427,7 @@ const projectNibArchive = ( : null; }; for (const object of archive.objects) { - const className = object.class_name.replace(/\0+$/u, ""); + const className = normalizeClassName(object.class_name); // AppKit connectors use NS-prefixed keys; UIKit's runtime outlet and // event connections use UI-prefixed keys with the same roles. const uiKit = /^UIRuntime\w*Connection$/u.test(className); @@ -420,6 +444,7 @@ const projectNibArchive = ( : className.includes("Control") || className.includes("Event") ? "action" : className; + budget.reserveJsonStringOccurrence(type); const eventMask = values.UIEventMask; // The source is the outlet owner or the sending control; the destination // is the outlet value or the action target, and a nil target is the @@ -485,7 +510,7 @@ const projectNibArchive = ( return { objectID: String(objectId), children }; }; const ibData = archive.objects.find( - ({ class_name }) => class_name.replace(/\0+$/u, "") === "NSIBObjectData", + ({ class_name }) => normalizeClassName(class_name) === "NSIBObjectData", ); const rootId = reference(ibData?.values.NSRoot); const hierarchyRoots = rootId === null ? [] : [rootId]; @@ -512,7 +537,11 @@ const projectNibArchive = ( "com.apple.ibtool.document.connections": connections, "com.apple.ibtool.document.hierarchy": merged.hierarchy, "com.apple.ibtool.document.classes": Object.fromEntries( - archive.classes.map((name) => [name.replace(/\0+$/u, ""), {}]), + archive.classes.map((name) => { + const className = normalizeClassName(name); + budget.reserveJsonPropertyNameOccurrence(className); + return [className, {}]; + }), ), }), }; diff --git a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts index 58a9b1ce3be..d8b6f3a9322 100644 --- a/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts +++ b/src/artifacts/apple/InterfaceBuilderDecodeBudget.ts @@ -48,6 +48,33 @@ export class InterfaceBuilderDecodeBudget { this.#remainingBytes -= bytes; } + /** Reserve an upper bound for one JSON-serialized string occurrence. */ + reserveJsonStringOccurrence(value: string): void { + // JSON escaping uses at most six ASCII bytes per UTF-16 code unit, plus + // quotes. Count occurrences even when the source string is shared. + this.reserve( + value.length * 6 + 2, + "projected JSON strings exceed the aggregate Interface Builder decode budget", + ); + } + + /** Reserve the exact JSON string length for a base64 value from Buffer. */ + reserveBase64JsonStringOccurrence(value: string): void { + // Buffer's base64 alphabet contains no JSON-escaped characters. + this.reserve( + value.length + 2, + "projected NIB base64 values exceed the aggregate Interface Builder decode budget", + ); + } + + /** Reserve an upper bound for one JSON object property name occurrence. */ + reserveJsonPropertyNameOccurrence(value: string): void { + this.reserve( + value.length * 6 + 3, + "projected JSON property names exceed the aggregate Interface Builder decode budget", + ); + } + /** Reserve bytes for an optional projection without throwing on exhaustion. */ tryReserve(bytes: number): boolean { if ( @@ -214,7 +241,7 @@ const estimateBinaryPlistExpansion = ( const byteLength = type === 6 ? size * 2 : size; if (cursor + byteLength > offsetTable) throw new TypeError("binary plist scalar object is truncated"); - estimate += type === 4 ? Math.ceil((size * 4) / 3) : byteLength * 2; + estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2; if (estimate > maxBytes) throw new InterfaceBuilderDecodeBudgetExceeded( "aggregate_decode_budget_exhausted", diff --git a/src/artifacts/apple/NibArchive.ts b/src/artifacts/apple/NibArchive.ts index 8ed48b76564..79b8a0d055d 100644 --- a/src/artifacts/apple/NibArchive.ts +++ b/src/artifacts/apple/NibArchive.ts @@ -395,7 +395,7 @@ const readValue = ( `value ${index} data`, ); budget.reserve( - 32 + Math.ceil((length * 4) / 3), + 32 + 4 * Math.ceil(length / 3), `NIBArchive data value ${index} exceeds the aggregate Interface Builder decode budget`, ); const encoded = bytes.subarray(cursor.offset, next).toString("base64"); diff --git a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts index 42d7b10a3d2..b5fabfcb09d 100644 --- a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts +++ b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import { analyzeInterfaceBuilderBundle } from "../../../src/artifacts/apple/InterfaceBuilderAnalysis.js"; import { encodeNibArchiveFixture } from "../../../src/artifacts/apple/NibArchive.fixture.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js"; @@ -96,6 +97,96 @@ const binaryPlistWithRepeatedReferences = (depth: number): Buffer => { return Buffer.concat([header, ...encodedObjects, offsetTable, trailer]); }; +const binaryPlistWithRepeatedData = ( + referenceCount: number, + dataLength: number, +): Buffer => { + const header = Buffer.from("bplist00", "ascii"); + const array = Buffer.concat([ + Buffer.from([0xaf, 0x10, referenceCount]), + Buffer.alloc(referenceCount, 1), + ]); + const data = Buffer.concat([ + Buffer.from([0x4f, 0x12]), + Buffer.from( + new Uint8Array([ + (dataLength >>> 24) & 0xff, + (dataLength >>> 16) & 0xff, + (dataLength >>> 8) & 0xff, + dataLength & 0xff, + ]), + ), + Buffer.alloc(dataLength, 7), + ]); + const offsets = Buffer.alloc(8); + offsets.writeUInt32BE(header.length, 0); + offsets.writeUInt32BE(header.length + array.length, 4); + const offsetTable = header.length + array.length + data.length; + const trailer = Buffer.alloc(32); + trailer[6] = 4; + trailer[7] = 1; + trailer.writeBigUInt64BE(2n, 8); + trailer.writeBigUInt64BE(0n, 16); + trailer.writeBigUInt64BE(BigInt(offsetTable), 24); + return Buffer.concat([header, array, data, offsets, trailer]); +}; + +const encodeNibArchiveWithSharedDataValue = (input: { + readonly className: string; + readonly fieldName: string; + readonly payload: string; + readonly objectCount: number; +}): Buffer => { + const varint = (value: number): Buffer => { + const bytes: number[] = []; + let remaining = value; + do { + let byte = remaining & 0x7f; + remaining >>>= 7; + if (remaining === 0) byte |= 0x80; + bytes.push(byte); + } while (remaining > 0); + return Buffer.from(bytes); + }; + const objects = Buffer.concat( + Array.from({ length: input.objectCount }, () => + Buffer.concat([varint(0), varint(0), varint(1)]), + ), + ); + const fieldBytes = Buffer.from(input.fieldName); + const keys = Buffer.concat([varint(fieldBytes.length), fieldBytes]); + const payloadBytes = Buffer.from(input.payload); + const values = Buffer.concat([ + varint(0), + Buffer.from([8]), + varint(payloadBytes.length), + payloadBytes, + ]); + const classBytes = Buffer.from(input.className); + const classes = Buffer.concat([ + varint(classBytes.length), + varint(0), + classBytes, + ]); + const objectsOffset = 50; + const keysOffset = objectsOffset + objects.length; + const valuesOffset = keysOffset + keys.length; + const classesOffset = valuesOffset + values.length; + const header = Buffer.alloc(50); + header.write("NIBArchive", 0, "ascii"); + header.writeUInt32LE(1, 10); + header.writeUInt32LE(10, 14); + header.writeUInt32LE(input.objectCount, 18); + header.writeUInt32LE(objectsOffset, 22); + header.writeUInt32LE(1, 26); + header.writeUInt32LE(keysOffset, 30); + header.writeUInt32LE(1, 34); + header.writeUInt32LE(valuesOffset, 38); + header.writeUInt32LE(1, 42); + header.writeUInt32LE(classesOffset, 46); + return Buffer.concat([header, objects, keys, values, classes]); +}; + describe("compiled Interface Builder bundle reader", () => { it("reads nib plist archives and reports provenance", async () => { const root = await createTestTempDirectory("rea-ib-test-"); @@ -483,24 +574,16 @@ describe("bounded Interface Builder archive decoding", () => { "RepeatedReferences.nib: omitted because aggregate decode budget exhausted.", ); }); +}); - it("budgets repeated large NSString projections before retaining each field", async () => { - const root = await createTestTempDirectory("rea-ib-repeated-string-test-"); +describe("binary plist data expansion budget", () => { + it("reserves copied binary plist data for every expanded reference", async () => { + const root = await createTestTempDirectory("rea-ib-binary-data-budget-"); const bundle = join(root, "Example.app"); await mkdir(bundle, { recursive: true }); - const largeString = "x".repeat(1024 * 1024); - const references = Object.fromEntries( - Array.from({ length: 50 }, (_, index) => [`title${index}`, { ref: 1 }]), - ); await writeFile( - join(bundle, "RepeatedString.nib"), - encodeNibArchiveFixture({ - classes: ["UIView", "NSString"], - objects: [ - { classIndex: 0, values: references }, - { classIndex: 1, values: { "NS.bytes": largeString } }, - ], - }), + join(bundle, "RepeatedData.nib"), + binaryPlistWithRepeatedData(200, 700 * 1024), ); const result = await analyzeInterfaceBuilderBundle({ @@ -519,11 +602,178 @@ describe("bounded Interface Builder archive decoding", () => { }), ); expect(result.limitations).toContain( - "RepeatedString.nib: omitted because aggregate decode budget exhausted.", + "RepeatedData.nib: omitted because aggregate decode budget exhausted.", + ); + }); +}); + +describe("NIB projection byte amplification", () => { + it("omits NIB projections whose shared strings or class names amplify beyond budget", async () => { + const repeatedString = "x".repeat(1024 * 1024); + const stringReferences = Object.fromEntries( + Array.from({ length: 50 }, (_, index) => [`title${index}`, { ref: 1 }]), + ); + const longName = "n".repeat(512 * 1024); + const longNameObjects = Array.from({ length: 700 }, () => ({ + classIndex: 0, + values: { field: true }, + })); + const cases = [ + { + name: "RepeatedString", + archive: encodeNibArchiveFixture({ + classes: ["UIView", "NSString"], + objects: [ + { classIndex: 0, values: stringReferences }, + { classIndex: 1, values: { "NS.bytes": repeatedString } }, + ], + }), + }, + { + name: "RepeatedClass", + archive: encodeNibArchiveFixture({ + classes: [longName], + objects: longNameObjects, + }), + }, + ]; + + for (const { name, archive } of cases) { + const root = await createTestTempDirectory(`rea-ib-${name}-budget-`); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + await writeFile(join(bundle, `${name}.nib`), archive); + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "f".repeat(64), + }); + + expect(result.documents, name).toEqual([]); + expect(result.graph.coverage, name).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "aggregate_decode_budget_exhausted", + examined: 1, + omitted: 1, + }), + ); + expect(result.limitations, name).toContain( + `${name}.nib: omitted because aggregate decode budget exhausted.`, + ); + } + }); + + it("retains a single large base64 data field within the representation budget", async () => { + const root = await createTestTempDirectory("rea-ib-large-data-positive-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + const archive = encodeNibArchiveFixture({ + classes: ["UIView"], + objects: [ + { + classIndex: 0, + values: { payload: "x".repeat(29 * 1024 * 1024) }, + }, + ], + }); + await writeFile(join(bundle, "LargePayload.nib"), archive); + + const result = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "f".repeat(64), + }); + + expect(result.documents.map(({ relative_path }) => relative_path)).toEqual([ + "LargePayload.nib", + ]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ facet: "archive_decode", status: "complete" }), ); + const serialized = JSON.stringify(result); + expect(serialized).toContain("$nib_data_base64"); + expect(Buffer.byteLength(serialized)).toBeLessThan(256 * 1024 * 1024); }); }); +cliTest( + "compiled CLI preserves a good archive and reports repeated key and data omissions", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-ib-cli-amplification-"); + const bundle = join(root, "Example.app"); + const contents = join(bundle, "Contents"); + const resources = join(bundle, "Contents", "Resources"); + await mkdir(resources, { recursive: true }); + await mkdir(join(contents, "MacOS"), { recursive: true }); + await writeFile( + join(contents, "MacOS", "App"), + thinMach(0xfeedfacf, 0x0100000c), + ); + await writeFile( + join(contents, "Info.plist"), + "CFBundleExecutableApp", + ); + await writeFile( + join(resources, "A-Good.nib"), + encodeNibArchiveFixture({ + classes: ["UIView"], + objects: [{ classIndex: 0, values: { title: "Good" } }], + }), + ); + const longKey = "k".repeat(512 * 1024); + await writeFile( + join(resources, "B-RepeatedKey.nib"), + encodeNibArchiveFixture({ + classes: ["UIView"], + objects: Array.from({ length: 700 }, () => ({ + classIndex: 0, + values: { [longKey]: true }, + })), + }), + ); + await writeFile( + join(resources, "C-RepeatedData.nib"), + encodeNibArchiveWithSharedDataValue({ + className: "UIView", + fieldName: "payload", + payload: "d".repeat(384 * 1024), + objectCount: 700, + }), + ); + + const output = await cli.run({ + arguments: ["decode-interface-builder", bundle, "--json"], + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + }); + + expect(output.exitCode, `${output.stderr}\n${output.stdout}`).toBe(0); + const result = interfaceBuilderAnalysisSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect(result.documents.map(({ relative_path }) => relative_path)).toEqual([ + "Contents/Resources/A-Good.nib", + ]); + expect(result.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "archive_decode", + status: "partial", + reason: "aggregate_decode_budget_exhausted", + examined: 3, + omitted: 2, + }), + ); + expect(result.limitations).toContain( + "Contents/Resources/B-RepeatedKey.nib: omitted because aggregate decode budget exhausted.", + ); + expect(result.limitations).toContain( + "Contents/Resources/C-RepeatedData.nib: omitted because aggregate decode budget exhausted.", + ); + }, +); + describe("aggregate Interface Builder archive retention budget", () => { it("keeps earlier documents when the aggregate decode reservation is exhausted", async () => { const root = await createTestTempDirectory("rea-ib-budget-test-"); From 0a0174b9ac8a73b61094e857a2950f4e5a06f544 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:17:10 +0800 Subject: [PATCH 053/791] fix(browser): validate and bound source-map evidence --- docs/web-source-location.md | 13 +- scripts/fixtures/browser-verifier-site.mjs | 68 +++ scripts/lib/browser-verifier-assertions.mjs | 37 +- scripts/verify-real-browser.mjs | 59 ++- src/browser/WebSourceMapFetcher.ts | 438 ++++++++++++++---- src/domain/webBundleAnalysis.ts | 31 +- src/domain/webBundleAnalyzerResult.ts | 1 + src/javascript/sourceMaps/DecodedSourceMap.ts | 82 ++++ src/javascript/sourceMaps/SourceMapFormat.ts | 19 +- src/javascript/sourceMaps/TraceSourceMap.ts | 65 +-- .../browser/webSourceMapFetcher.test.ts | 277 +++++++++++ 11 files changed, 915 insertions(+), 175 deletions(-) create mode 100644 src/javascript/sourceMaps/DecodedSourceMap.ts diff --git a/docs/web-source-location.md b/docs/web-source-location.md index 6e677b08f56..f1736ec0fc2 100644 --- a/docs/web-source-location.md +++ b/docs/web-source-location.md @@ -96,7 +96,18 @@ preflights indexed sections, sources, names, generated rows and mapping segments against a 262144-record decode budget before upstream flattening. Indexed maps are limited to 64 nested levels. These structural limits bound decoded object growth while preserving large embedded `sourcesContent` strings within the -response-byte budget; exceeding a decoder limit returns a `fetch_failed` item. +response-byte budget. It also bounds the expanded inline evidence to 32 MiB +across the inspection, checking resolved source identities and mapping rows +before publishing them, and bounds raw resolved-source identities to 64 MiB +before upstream materializes their arrays. Exceeding either decoder budget +returns a `fetch_failed` item while retaining other maps. Decoded coordinates +and source/name indexes are validated per map; a malformed map is reported as +`invalid` without discarding earlier or later maps. +Collection reserves each requested map's failure context before fetching it. +When the aggregate inline-result budget cannot reserve another item, remaining +requests are left unprocessed and reported as unknown through the top-level +`limitation`; `requested` retains the full count and `processed` counts returned +items. Malformed UTF-8 map responses are reported as invalid rather than repaired with replacement characters. diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs index 30a7b36e768..841e7f90092 100644 --- a/scripts/fixtures/browser-verifier-site.mjs +++ b/scripts/fixtures/browser-verifier-site.mjs @@ -27,6 +27,50 @@ export async function startBrowserVerifierSite() { response.end(sourceMap()); return; } + if (request.url === "/bad.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + names: [], + sources: ["../src/bad.ts"], + mappings: "AAAAD", + }), + ); + return; + } + if (request.url === "/expanded.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + names: [], + sources: ["../src/" + "x".repeat(300000) + ".ts"], + mappings: "AAAA" + ",CAAA".repeat(127), + }), + ); + return; + } + if (request.url === "/overflow.js.map") { + response.setHeader("content-type", "application/source-map+json"); + response.end( + JSON.stringify({ + version: 3, + sections: [ + { + offset: { line: 0, column: Number.MAX_SAFE_INTEGER }, + map: { + version: 3, + names: [], + sources: ["overflow.ts"], + mappings: "CAAA", + }, + }, + ], + }), + ); + return; + } if (request.url === "/session-generation") { response.setHeader("content-type", "application/json"); response.end(JSON.stringify({ generation: sessionGeneration })); @@ -37,6 +81,27 @@ export async function startBrowserVerifierSite() { response.end(browserScript(port)); return; } + if (request.url?.startsWith("/bad.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const badMapProbe = true;\n//# sourceMappingURL=/bad.js.map", + ); + return; + } + if (request.url?.startsWith("/expanded.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const expandedMapProbe = true;\n//# sourceMappingURL=/expanded.js.map", + ); + return; + } + if (request.url?.startsWith("/overflow.js") === true) { + response.setHeader("content-type", "text/javascript"); + response.end( + "export const overflowMapProbe = true;\n//# sourceMappingURL=/overflow.js.map", + ); + return; + } if (request.url?.startsWith("/api") === true) { response.setHeader("content-type", "application/json"); response.setHeader( @@ -113,6 +178,9 @@ const browserScript = (port) => ` socket.addEventListener("open", () => socket.send(JSON.stringify({ token: "websocket-secret-value" }))); socket.addEventListener("message", () => socket.close()); }; + void import("/bad.js"); + void import("/expanded.js"); + void import("/overflow.js"); observe(); setInterval(observe, 150); let observedGeneration = 0; diff --git a/scripts/lib/browser-verifier-assertions.mjs b/scripts/lib/browser-verifier-assertions.mjs index 7c26e1d9870..514fbf26786 100644 --- a/scripts/lib/browser-verifier-assertions.mjs +++ b/scripts/lib/browser-verifier-assertions.mjs @@ -150,7 +150,7 @@ export function assertBundleAnalysis(result) { ) throw new Error("Real Chrome static bundle findings were missing"); if ( - result.observations.source_maps.status !== "included" || + result.observations.source_maps.status !== "partial" || !result.observations.source_maps.items.some( ({ status, original_sources, original_module_edges, mappings }) => status === "included" && @@ -166,6 +166,41 @@ export function assertBundleAnalysis(result) { throw new Error( `Real Chrome approved source-map reconstruction was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, artifact, mappings }) => + declared_url.endsWith("/bad.js.map") && + status === "invalid" && + artifact === null && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome malformed-map isolation was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, limitation, mappings }) => + declared_url.endsWith("/expanded.js.map") && + status === "fetch_failed" && + limitation.includes("Expanded source-map evidence") && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome expanded-map omission was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); + if ( + !result.observations.source_maps.items.some( + ({ declared_url, status, mappings }) => + declared_url.endsWith("/overflow.js.map") && + status === "invalid" && + mappings.length === 0, + ) + ) + throw new Error( + `Real Chrome overflowing-map isolation was missing: ${JSON.stringify(sourceMapSummary(result.observations.source_maps))}`, + ); } /** Assert explicitly selected console text and JSON/WebSocket shape capture. */ diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index b1cf21006ac..d35756de586 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -1,9 +1,12 @@ #!/usr/bin/env node import { spawn } from "node:child_process"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { access, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { CdpBrowserProvider } from "../dist/browser/CdpBrowserProvider.js"; import { waitForBrowserDevtoolsPort } from "../dist/browser/BrowserProcessStartup.js"; @@ -12,7 +15,6 @@ import { inspectWebPageInputSchema, listBrowserTargetsInputSchema, } from "../dist/domain/browserObservation.js"; -import { analyzeWebBundleInputSchema } from "../dist/domain/webBundleAnalysis.js"; import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js"; import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js"; import { @@ -39,6 +41,7 @@ import { verifyBrowserNetworkEvidence } from "./lib/browser-network-e2e.mjs"; import { verifyBrowserScriptExport } from "./lib/browser-script-export-e2e.mjs"; import { verifyBrowserModules } from "./lib/browser-module-e2e.mjs"; import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs"; +import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs"; const REAL_BROWSER_STARTUP_TIMEOUT_MS = 60_000; const SCENARIO_SECRET_VALUE = "rea-browser-verifier-secret"; @@ -160,17 +163,7 @@ try { ); assertSensitiveShapes(withSource.value); - const bundle = await provider.analyzeBundle( - analyzeWebBundleInputSchema.parse({ - cdp_endpoint: endpoint, - allowed_origins: [site.origin], - target_id: target, - observation_ms: 200, - fetch_source_maps: true, - }), - ); - if (!bundle.ok) throw bundle.error; - assertBundleAnalysis(bundle.value); + const bundle = await verifyPublicBundle(endpoint, target, site.origin); const captureDiff = await provider.compareCaptures( compareWebCapturesInputSchema.parse({ @@ -349,8 +342,9 @@ try { networkRequests: observed.value.network.requests.length, consoleEvents: observed.value.console.events.length, websocketEvents: observed.value.network.websocket_events.length, - bundleScripts: bundle.value.capture.scripts_analyzed, - sourceMaps: bundle.value.observations.source_maps.processed, + bundleScripts: bundle.capture.scripts_analyzed, + sourceMaps: bundle.observations.source_maps.processed, + bundle_cli_and_stdio_mcp: true, sessionEvents: session.value.timeline.length, pageScopedTransport: true, screenshotBytes: screenshot.value.artifact.bytes, @@ -383,6 +377,43 @@ process.stdout.write( })}\n`, ); +async function verifyPublicBundle(endpoint, target, origin) { + const evidence = await artifactCliEvidence("analyze-web-bundle", endpoint, [ + target, + "--allowed-origins", + origin, + "--fetch-source-maps", + "--observation-ms", + "200", + ]); + assertBundleAnalysis(evidence.normalized_result); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [fileURLToPath(new URL("./rea.mjs", import.meta.url)), "mcp"], + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + stderr: "pipe", + }); + const client = new Client({ name: "browser-bundle-e2e", version: "1" }); + try { + await client.connect(transport); + const result = await artifactMcpResult(client, "analyze_web_bundle", { + cdp_endpoint: endpoint, + target_id: target, + allowed_origins: [origin], + fetch_source_maps: true, + observation_ms: 200, + }); + assertBundleAnalysis(result); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + return evidence.normalized_result; +} + async function verifyPageScopedTransport(provider, proxy, origin) { const target = await pageTarget(provider, proxy.endpoint); const input = inspectWebPageInputSchema.parse({ diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 3cbed28f064..5464fcc0f87 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -1,11 +1,9 @@ import * as t from "@babel/types"; import { - AnyMap, - eachMapping, - type EncodedSourceMap, - type Section, - type SectionedSourceMap, -} from "@jridgewell/trace-mapping"; + decodeValidatedSourceMapLeaves, + isBeforeSourceMapLeafStop, + resolveSourceMapSource, +} from "../javascript/sourceMaps/DecodedSourceMap.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { isUrlLikeModuleSpecifier } from "../domain/webBundleAnalyzerAst.js"; @@ -20,10 +18,13 @@ import type { import { webSourceMapsSchema } from "../domain/webBundleAnalysis.js"; import { createWebTextArtifact } from "../domain/webContentArtifact.js"; import { safeParseJson } from "../domain/safeJson.js"; +import { jsonParts } from "../domain/jsonSerialization.js"; import { + inspectSourceMapValue, SourceMapFormatFailure, - validateSourceMapStructure, + type SourceMapLeaf, } from "../javascript/sourceMaps/SourceMapFormat.js"; +import { WEB_SOURCE_MAP_LIMITS } from "../domain/webSourceLocation.js"; export interface WebSourceMapRequest { readonly scriptKey: string; @@ -45,6 +46,9 @@ interface SourceMapFetchHost { // hitting either produces an explicit fetch_failed item, never truncated data. const SOURCE_MAP_FETCH_TIMEOUT_MS = 30_000; const SOURCE_MAP_RESPONSE_BYTES = 64 * 1024 * 1024; +const SOURCE_MAP_OUTPUT_BYTES = WEB_SOURCE_MAP_LIMITS.outputBytes; +const SOURCE_MAP_FAILURE_LIMITATION_RESERVE = 512; +const SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE = 1_024; type SourceMaps = WebSourceMaps; type SourceMapItem = WebSourceMapItem; @@ -55,7 +59,11 @@ type ParsedSourceMapItem = Extract< interface SourceMapDecodeContext { readonly signal: AbortSignal | undefined; readonly deadlineAt: number; - readonly budget: { records: number }; + readonly budget: { + records: number; + expandedBytes: number; + resolvedSourceBytes: number; + }; } /** Fetch and validate approved source maps without browser credentials. */ @@ -79,11 +87,27 @@ export const fetchWebSourceMaps = async ( const budget = { retainedBytes: 0, deadlineAt, - decodedRecords: { records: 0 }, + decodedRecords: { + records: 0, + expandedBytes: SOURCE_MAP_COLLECTION_ENVELOPE_RESERVE, + resolvedSourceBytes: 0, + }, }; + let limitation: string | undefined; try { for (const request of requests) { if (signal?.aborted === true) throw signal.reason; + const reservation = sourceMapFailureReservation(request) + 1; + if ( + reservation > + SOURCE_MAP_OUTPUT_BYTES - budget.decodedRecords.expandedBytes + ) { + const unprocessed = requests.length - items.length; + limitation = `Source-map output budget was exhausted before processing ${String(unprocessed)} requested map${unprocessed === 1 ? "" : "s"}. Their results are unknown.`; + break; + } + const expandedBytesBeforeItem = budget.decodedRecords.expandedBytes; + budget.decodedRecords.expandedBytes += reservation; if (operationSignal.aborted || Date.now() >= deadlineAt) { items.push( emptySourceMapItem( @@ -94,7 +118,17 @@ export const fetchWebSourceMaps = async ( ); continue; } - items.push(await fetchOne(request, input, operationSignal, host, budget)); + const item = await fetchOne( + request, + input, + operationSignal, + host, + budget, + ); + if (item.artifact === null) + budget.decodedRecords.expandedBytes = + expandedBytesBeforeItem + reservation; + items.push(item); } } finally { clearTimeout(timeout); @@ -102,20 +136,23 @@ export const fetchWebSourceMaps = async ( } if (signal?.aborted === true) throw signal.reason; return webSourceMapsSchema.parse({ - status: sourceMapsStatus(items), + status: sourceMapsStatus(items, limitation !== undefined), requested: requests.length, processed: items.length, items, + ...(limitation === undefined ? {} : { limitation }), }); }; const sourceMapsStatus = ( items: readonly SourceMapItem[], + omitted: boolean, ): SourceMaps["status"] => { const retained = items.filter( ({ status }) => status === "included" || status === "partial", ).length; if (items.length === 0 || retained === 0) return "unavailable"; + if (omitted) return "partial"; return retained === items.length && !items.some(({ status }) => status === "partial") ? "included" @@ -151,7 +188,11 @@ const fetchOne = async ( budget: { retainedBytes: number; deadlineAt: number; - decodedRecords: { records: number }; + decodedRecords: { + records: number; + expandedBytes: number; + resolvedSourceBytes: number; + }; }, ): Promise => { if (!approvedUrl(request.fetchUrl, input.allowed_origins)) @@ -378,49 +419,6 @@ const promiseWithAbort = ( }); }; -type EncodedBrowserSourceMap = EncodedSourceMap | SectionedSourceMap; - -/** Compose indexed offsets before upstream flattening, retaining parent clipping. */ -const sourceMapWithAbsoluteSections = ( - value: unknown, - check: () => void, -): EncodedBrowserSourceMap => { - const root = value as EncodedBrowserSourceMap; - if (!("sections" in root)) return root; - const sections: Section[] = []; - const pending: Section[] = [{ map: root, offset: { line: 0, column: 0 } }]; - while (pending.length > 0) { - check(); - const node = pending.pop(); - if (node === undefined) break; - if (!("sections" in node.map)) { - sections.push(node); - continue; - } - // A parent's boundary clips the preceding leaf even when its first child - // starts later or it has no children. Empty maps retain that boundary - // without adding source/name identities or point mappings. - sections.push({ - offset: node.offset, - map: { version: 3, sources: [], names: [], mappings: "" }, - }); - for (let index = node.map.sections.length - 1; index >= 0; index -= 1) { - const child = node.map.sections[index]; - if (child === undefined) continue; - pending.push({ - map: child.map, - offset: { - line: node.offset.line + child.offset.line, - column: - child.offset.column + - (child.offset.line === 0 ? node.offset.column : 0), - }, - }); - } - } - return { ...root, sections }; -}; - const normalizeSourceMap = ( request: WebSourceMapRequest, text: string, @@ -434,56 +432,153 @@ const normalizeSourceMap = ( "invalid", "Source-map JSON is not a version 3 map.", ); + const retainedBytesBeforeMap = context.budget.expandedBytes; try { const check = (): void => checkOperation(undefined, context.signal, context.deadlineAt); - validateSourceMapStructure(parsedJson.value, { + const inspected = inspectSourceMapValue(parsedJson.value, { profile: "browser-collection", budget: context.budget, check, }); + context.budget.expandedBytes = + retainedBytesBeforeMap - + sourceMapFailureReservation(request) + + sourceMapSuccessReservation(request); + consumeExpandedBytes( + context.budget, + jsonBytes(text) + + artifactMetadataBytes("application/source-map+json", text) - + 4, + ); + const sourcesByRaw = new Map(); + const resolvedByRoot = new Map>(); + preflightResolvedSourceBytes( + inspected.leaves, + fetchedUrl, + context.budget, + resolvedByRoot, + sourcesByRaw, + check, + ); check(); - const map = new AnyMap( - sourceMapWithAbsoluteSections(parsedJson.value, check), + const decoded = decodeValidatedSourceMapLeaves( + inspected.leaves, fetchedUrl, + check, ); - const resolvedBySource = new Map(); - const originalSources = map.sources.map((source, index) => { - if ((index & 255) === 0) check(); - const content = map.sourcesContent?.[index]; - const resolved = - map.resolvedSources[index] ?? source ?? "[unknown-source]"; - if (source !== null) resolvedBySource.set(source, resolved); - return { - source: sanitizeSource(resolved), - artifact: - typeof content === "string" - ? createWebTextArtifact(content, sourceMediaType(source)) - : null, - }; - }); + const sourceJsonBytes = new Map(); + const nameJsonBytes = new Map(); + const originalSources: ParsedSourceMapItem["original_sources"] = []; + let sourceBytes = 0; + for (const { leaf, resolvedSources } of decoded) { + for (const [index, rawSource] of leaf.map.sources.entries()) { + if ((index & 255) === 0) check(); + const resolved = + resolvedSources[index] ?? rawSource ?? "[unknown-source]"; + const sanitized = cachedSanitized(sourcesByRaw, resolved); + sourceBytes += + cachedJsonBytes(sourceJsonBytes, sanitized) + + sourceEntryOverheadBytes(); + const content = leaf.map.sourcesContent?.[index] ?? null; + if (content !== null) + sourceBytes += + jsonBytes(content) + + artifactMetadataBytes(sourceMediaType(rawSource), content) - + 4; + } + } + consumeExpandedBytes( + context.budget, + sourceBytes + originalSourcesCount(decoded), + ); + for (const { leaf, resolvedSources } of decoded) { + for (const [index, rawSource] of leaf.map.sources.entries()) { + if ((index & 255) === 0) check(); + const resolved = + resolvedSources[index] ?? rawSource ?? "[unknown-source]"; + const sanitized = cachedSanitized(sourcesByRaw, resolved); + const content = leaf.map.sourcesContent?.[index] ?? null; + originalSources.push({ + source: sanitized, + artifact: + content === null + ? null + : createWebTextArtifact(content, sourceMediaType(rawSource)), + }); + } + } + let mappingBytes = 0; + let mappingCount = 0; + for (const { leaf, rows, resolvedSources } of decoded) + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check(); + for (const segment of row) { + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; + if (segment.length === 1) continue; + const rawSource = leaf.map.sources[segment[1]]; + const source = cachedSanitized( + sourcesByRaw, + resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]", + ); + const name = + segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; + mappingBytes += + cachedJsonBytes(sourceJsonBytes, source) + + (name === null ? 4 : cachedJsonBytes(nameJsonBytes, name)) + + mappingRowBytes( + leaf.offset.line + line + 1, + segment[0] + (line === 0 ? leaf.offset.column : 0), + segment[2] + 1, + segment[3], + ); + mappingCount += 1; + if ( + mappingBytes > + WEB_SOURCE_MAP_LIMITS.outputBytes - context.budget.expandedBytes + ) + throw expandedOutputFailure(); + } + } + consumeExpandedBytes(context.budget, mappingBytes + mappingCount); const mappings: ParsedSourceMapItem["mappings"] = []; - eachMapping(map, (mapping) => { - if ((mappings.length & 0x3fff) === 0) check(); - if ( - mapping.source === null || - mapping.originalLine === null || - mapping.originalColumn === null - ) - return; - mappings.push({ - generated_line: mapping.generatedLine, - generated_column: mapping.generatedColumn, - source: sanitizeSource( - resolvedBySource.get(mapping.source) ?? mapping.source, - ), - original_line: mapping.originalLine, - original_column: mapping.originalColumn, - name: mapping.name ?? null, - }); - }); - const modules = originalModuleEdges(originalSources, check); + for (const { leaf, rows, resolvedSources } of decoded) + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check(); + for (const segment of row) { + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; + if (segment.length === 1) continue; + const rawSource = leaf.map.sources[segment[1]]; + const resolved = + resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]"; + const source = cachedSanitized(sourcesByRaw, resolved); + const name = + segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; + mappings.push({ + generated_line: leaf.offset.line + line + 1, + generated_column: + segment[0] + (line === 0 ? leaf.offset.column : 0), + source, + original_line: segment[2] + 1, + original_column: segment[3], + name, + }); + } + } + const modules = originalModuleEdges(originalSources, check, (edge) => + consumeExpandedBytes( + context.budget, + Buffer.byteLength(JSON.stringify(edge)) + 1, + ), + ); check(); + if (modules.incomplete.length > 0) + consumeExpandedBytes( + context.budget, + jsonBytes( + `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, + ) - 4, + ); const parsed = { ...sourceMapContext(request), artifact: createWebTextArtifact(text, "application/source-map+json"), @@ -500,6 +595,9 @@ const normalizeSourceMap = ( limitation: `Module edges are incomplete: ${modules.incomplete.length} of ${originalSources.filter(({ artifact }) => artifact !== null).length} original sources could not be parsed in full (${modules.incomplete.join(", ")}).`, }; } catch (cause: unknown) { + // Failed maps publish no artifact or mapping rows. Release their output + // reservation while keeping cumulative fetch and decoder-work accounting. + context.budget.expandedBytes = retainedBytesBeforeMap; if (cause instanceof SourceMapFormatFailure && cause.reason === "limit") return emptySourceMapItem(request, "fetch_failed", cause.message); if ( @@ -524,6 +622,9 @@ interface OriginalModuleEdges { const originalModuleEdges = ( sources: ParsedSourceMapItem["original_sources"], check: () => void, + beforeEdge: ( + edge: ParsedSourceMapItem["original_module_edges"][number], + ) => void, ): OriginalModuleEdges => { const edges: ParsedSourceMapItem["original_module_edges"] = []; const seen = new Set(); @@ -571,12 +672,14 @@ const originalModuleEdges = ( const key = `${source.source}\0${kind}\0${specifier}`; if (seen.has(key)) return; seen.add(key); - edges.push({ + const edge = { from_source: source.source, kind, specifier, resolved_source: resolveOriginalSource(specifier, source.source), - }); + }; + beforeEdge(edge); + edges.push(edge); }, }); } @@ -641,6 +744,151 @@ const sanitizeSource = (value: string): string => { } }; +const preflightResolvedSourceBytes = ( + leaves: readonly SourceMapLeaf[], + mapUrl: string, + budget: { readonly expandedBytes: number; resolvedSourceBytes: number }, + resolvedByRoot: Map>, + sanitizedByResolved: Map, + check: () => void, +): void => { + let estimate = budget.expandedBytes; + let resolvedBytes = budget.resolvedSourceBytes; + for (const { map } of leaves) { + let sourcesForRoot = resolvedByRoot.get(map.sourceRoot ?? null); + if (sourcesForRoot === undefined) { + sourcesForRoot = new Map(); + resolvedByRoot.set(map.sourceRoot ?? null, sourcesForRoot); + } + for (const source of map.sources) { + check(); + const sourceKey = source ?? ""; + let resolved = sourcesForRoot.get(sourceKey); + if (resolved === undefined) + resolved = resolveSourceMapSource(source, map.sourceRoot, mapUrl); + const bytes = Buffer.byteLength(resolved, "utf8"); + if (bytes > SOURCE_MAP_RESPONSE_BYTES - resolvedBytes) + throw resolvedSourceLimitFailure(); + resolvedBytes += bytes; + sourcesForRoot.set(sourceKey, resolved); + const sanitized = cachedSanitized(sanitizedByResolved, resolved); + estimate += jsonBytes(sanitized); + if (estimate > WEB_SOURCE_MAP_LIMITS.outputBytes) + throw expandedOutputFailure(); + } + } + budget.resolvedSourceBytes = resolvedBytes; +}; + +const consumeExpandedBytes = ( + budget: { expandedBytes: number }, + bytes: number, +): void => { + if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - budget.expandedBytes) + throw expandedOutputFailure(); + budget.expandedBytes += bytes; +}; + +const expandedOutputFailure = (): SourceMapFormatFailure => + new SourceMapFormatFailure( + "limit", + `Expanded source-map evidence exceeds the ${WEB_SOURCE_MAP_LIMITS.outputBytes / (1024 * 1024)} MiB public representation budget.`, + ); + +const resolvedSourceLimitFailure = (): SourceMapFormatFailure => + new SourceMapFormatFailure( + "limit", + `Resolved source-map identities exceed the ${SOURCE_MAP_RESPONSE_BYTES / (1024 * 1024)} MiB decoder representation budget.`, + ); + +const jsonBytes = (value: string): number => { + let bytes = 0; + for (const part of jsonParts(value)) bytes += Buffer.byteLength(part, "utf8"); + return bytes; +}; + +const artifactMetadataBytes = (mediaType: string, text: string): number => + Buffer.byteLength( + JSON.stringify({ + sha256: "0".repeat(64), + bytes: Buffer.byteLength(text), + media_type: mediaType, + charset: "utf-8", + text: "", + }), + ) - 2; + +const sourceMapFailureReservation = (request: WebSourceMapRequest): number => { + const envelope = emptySourceMapItem( + { ...request, declaredUrl: "", scriptKey: "" }, + "fetch_failed", + "x".repeat(SOURCE_MAP_FAILURE_LIMITATION_RESERVE), + ); + return ( + Buffer.byteLength(JSON.stringify(envelope)) - + 4 + + jsonBytes(request.declaredUrl) + + jsonBytes(request.scriptKey) + ); +}; + +const sourceMapSuccessReservation = (request: WebSourceMapRequest): number => { + const context = sourceMapContext({ + ...request, + declaredUrl: "", + scriptKey: "", + }); + const envelope = { + ...context, + status: "included", + artifact: null, + original_sources: [], + original_module_edges: [], + mappings: [], + limitation: null, + }; + return ( + Buffer.byteLength(JSON.stringify(envelope)) - + 4 + + jsonBytes(request.declaredUrl) + + jsonBytes(request.scriptKey) + ); +}; + +const sourceEntryOverheadBytes = (): number => + Buffer.byteLength('{"source":,"artifact":null}'); + +const originalSourcesCount = ( + decoded: ReturnType, +): number => + decoded.reduce((count, { leaf }) => count + leaf.map.sources.length, 0); + +const cachedJsonBytes = (cache: Map, value: string): number => { + const cached = cache.get(value); + if (cached !== undefined) return cached; + const bytes = jsonBytes(value); + cache.set(value, bytes); + return bytes; +}; + +const cachedSanitized = (cache: Map, value: string): string => { + const cached = cache.get(value); + if (cached !== undefined) return cached; + const sanitized = sanitizeSource(value); + cache.set(value, sanitized); + return sanitized; +}; + +const mappingRowBytes = ( + generatedLine: number, + generatedColumn: number, + originalLine: number, + originalColumn: number, +): number => + Buffer.byteLength( + `{"generated_line":${String(generatedLine)},"generated_column":${String(generatedColumn)},"source":,"original_line":${String(originalLine)},"original_column":${String(originalColumn)},"name":}`, + ); + const resolveOriginalSource = ( specifier: string, base: string, diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index f04f180946c..c3bbcf35241 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -99,29 +99,44 @@ export const webSourceMapsSchema = z requested: z.number().int().min(0), processed: z.number().int().min(0), items: z.array(sourceMapSchema), + limitation: z.string().min(1).optional(), }) .superRefine((sourceMaps, context) => { if ( - sourceMaps.requested !== sourceMaps.processed || + sourceMaps.processed > sourceMaps.requested || sourceMaps.processed !== sourceMaps.items.length ) context.addIssue({ code: "custom", message: "Source-map coverage counts are inconsistent", }); + if ( + sourceMaps.processed < sourceMaps.requested && + sourceMaps.limitation === undefined + ) + context.addIssue({ + code: "custom", + message: + "Unprocessed source maps require an explicit coverage limitation", + path: ["limitation"], + }); const statuses = sourceMaps.items.map(({ status }) => status); const retained = statuses.filter( (status) => status === "included" || status === "partial", ).length; const allowedStatuses = - sourceMaps.items.length === 0 - ? ["not_requested", "unavailable"] - : retained === 0 + sourceMaps.processed < sourceMaps.requested + ? retained === 0 ? ["unavailable"] - : retained === sourceMaps.items.length && - !statuses.includes("partial") - ? ["included"] - : ["partial"]; + : ["partial"] + : sourceMaps.items.length === 0 + ? ["not_requested", "unavailable"] + : retained === 0 + ? ["unavailable"] + : retained === sourceMaps.items.length && + !statuses.includes("partial") + ? ["included"] + : ["partial"]; if (!allowedStatuses.includes(sourceMaps.status)) context.addIssue({ code: "custom", diff --git a/src/domain/webBundleAnalyzerResult.ts b/src/domain/webBundleAnalyzerResult.ts index f11a3a1b994..b4ac67a29de 100644 --- a/src/domain/webBundleAnalyzerResult.ts +++ b/src/domain/webBundleAnalyzerResult.ts @@ -113,6 +113,7 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ { dimension: "source_maps" as const, reason: + input.sourceMaps.limitation ?? "One or more requested source maps were unavailable or incomplete", affected_script_keys: input.sourceMaps.items .filter(({ status }) => status !== "included") diff --git a/src/javascript/sourceMaps/DecodedSourceMap.ts b/src/javascript/sourceMaps/DecodedSourceMap.ts new file mode 100644 index 00000000000..b329cd68e73 --- /dev/null +++ b/src/javascript/sourceMaps/DecodedSourceMap.ts @@ -0,0 +1,82 @@ +import { TraceMap, decodedMappings } from "@jridgewell/trace-mapping"; +import { + SourceMapFormatFailure, + type SourceMapLeaf, +} from "./SourceMapFormat.js"; + +/** Determine whether a leaf-local mapping falls before its enclosing boundary. */ +export const isBeforeSourceMapLeafStop = ( + leaf: SourceMapLeaf, + line: number, + column: number, +): boolean => { + if (leaf.stop === null) return true; + const generatedLine = leaf.offset.line + line; + const generatedColumn = column + (line === 0 ? leaf.offset.column : 0); + return ( + generatedLine < leaf.stop.line || + (generatedLine === leaf.stop.line && generatedColumn < leaf.stop.column) + ); +}; + +/** Resolve one declaration with the same pinned upstream implementation used by full maps. */ +export const resolveSourceMapSource = ( + source: string | null, + sourceRoot: string | null | undefined, + mapUrl: string, +): string => { + const map = new TraceMap( + { + version: 3, + names: [], + sources: [source ?? ""], + ...(sourceRoot == null ? {} : { sourceRoot }), + mappings: "", + }, + mapUrl, + ); + const resolved = map.resolvedSources[0]; + if (resolved === undefined) + throw new SourceMapFormatFailure( + "format", + "Source-map resolver did not return the requested source identity.", + ); + return resolved; +}; + +/** Decode leaf mappings and validate producer indices before any consumer publishes them. */ +export const decodeValidatedSourceMapLeaves = ( + leaves: readonly SourceMapLeaf[], + mapUrl: string, + check?: () => void, +): readonly { + readonly leaf: SourceMapLeaf; + readonly rows: ReturnType; + readonly resolvedSources: readonly string[]; +}[] => + leaves.map((leaf) => { + check?.(); + const map = new TraceMap(JSON.stringify(leaf.map), mapUrl); + const rows = decodedMappings(map); + for (const [line, row] of rows.entries()) { + if ((line & 255) === 0) check?.(); + for (const segment of row) { + const generatedLine = leaf.offset.line + line; + const generatedColumn = + segment[0] + (line === 0 ? leaf.offset.column : 0); + if ( + !Number.isSafeInteger(generatedLine) || + !Number.isSafeInteger(generatedColumn) || + ![1, 4, 5].includes(segment.length) || + segment.some((value) => !Number.isSafeInteger(value) || value < 0) || + (segment.length !== 1 && segment[1] >= leaf.map.sources.length) || + (segment.length === 5 && segment[4] >= leaf.map.names.length) + ) + throw new SourceMapFormatFailure( + "format", + "Decoded local or absolute position, source index or name index is invalid.", + ); + } + } + return { leaf, rows, resolvedSources: map.resolvedSources }; + }); diff --git a/src/javascript/sourceMaps/SourceMapFormat.ts b/src/javascript/sourceMaps/SourceMapFormat.ts index f4907862853..c1816d51e6a 100644 --- a/src/javascript/sourceMaps/SourceMapFormat.ts +++ b/src/javascript/sourceMaps/SourceMapFormat.ts @@ -311,18 +311,10 @@ export const inspectSourceMap = ( }; }; -/** Validate shared source-map structure before a consumer decodes mappings. */ -export const validateSourceMapStructure = ( +/** Return validated leaves when a consumer needs decoded rows after preflight. */ +export const inspectSourceMapValue = ( value: unknown, options: SourceMapStructureOptions = {}, -): void => { - inspectSourceMapValue(value, options, false); -}; - -const inspectSourceMapValue = ( - value: unknown, - options: SourceMapStructureOptions = {}, - collect = true, ): { readonly format: "regular" | "indexed"; readonly reportedFile: string | null; @@ -351,10 +343,9 @@ const inspectSourceMapValue = ( continue; } const leaf = inspectLeaf(node, budget, options); - if (collect) - leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); - for (const [sourceIndex, source] of leaf.sources.entries()) - if (collect) + leaves.push({ map: leaf, offset: node.offset, stop: node.stop }); + if (options.profile !== "browser-collection") + for (const [sourceIndex, source] of leaf.sources.entries()) declarations.push({ sectionPath: node.path, sourceIndex, diff --git a/src/javascript/sourceMaps/TraceSourceMap.ts b/src/javascript/sourceMaps/TraceSourceMap.ts index 2fb5ef55995..14755d2ec87 100644 --- a/src/javascript/sourceMaps/TraceSourceMap.ts +++ b/src/javascript/sourceMaps/TraceSourceMap.ts @@ -20,6 +20,10 @@ import { SourceMapFormatFailure, type SourceMapSourceDeclaration, } from "./SourceMapFormat.js"; +import { + decodeValidatedSourceMapLeaves, + isBeforeSourceMapLeafStop, +} from "./DecodedSourceMap.js"; /** Decode with the pinned upstream codec; preserve every equal-position candidate. */ export const traceSourceMap = ( @@ -28,7 +32,21 @@ export const traceSourceMap = ( position: WebSourcePosition, ): WebSourceMapReport => { const inspected = inspectSourceMap(text); - const { map, rows } = decodeValidatedMap(inspected, url); + const map = new AnyMap(inspected.jsonText, url); + const decodedLeaves = decodeValidatedSourceMapLeaves(inspected.leaves, url); + for (const { leaf, rows } of decodedLeaves) + for (const [line, row] of rows.entries()) + for (const segment of row) + if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) + throw new SourceMapFormatFailure( + "format", + "Embedded source-map mappings overlap the following section boundary.", + ); + const rows = + inspected.format === "regular" + ? (decodedLeaves[0]?.rows ?? []) + : decodedMappings(map); + validateFlattenedRows(inspected, rows, map); const matched = greatestPosition(rows, position); const segments = matched === null @@ -77,49 +95,14 @@ export const traceSourceMap = ( }); }; -const decodeValidatedMap = ( +const validateFlattenedRows = ( inspected: ReturnType, - url: string, -) => { - let regularRows: ReturnType | undefined; - for (const leaf of inspected.leaves) { - const decoded = decodedMappings( - new TraceMap(JSON.stringify(leaf.map), url), - ); - if (inspected.format === "regular") regularRows = decoded; - for (const [line, row] of decoded.entries()) - for (const segment of row) { - if ( - segment.some((value) => !Number.isSafeInteger(value) || value < 0) || - (segment.length !== 1 && segment[1] >= leaf.map.sources.length) || - (segment.length === 5 && segment[4] >= leaf.map.names.length) - ) - throw new SourceMapFormatFailure( - "format", - "Decoded section-local position, source index or name index is invalid.", - ); - const generatedLine = leaf.offset.line + line; - const generatedColumn = - segment[0] + (line === 0 ? leaf.offset.column : 0); - if ( - leaf.stop !== null && - (generatedLine > leaf.stop.line || - (generatedLine === leaf.stop.line && - generatedColumn >= leaf.stop.column)) - ) - throw new SourceMapFormatFailure( - "format", - "Embedded source-map mappings overlap the following section boundary.", - ); - } - } - const map = new AnyMap(inspected.jsonText, url); - const rows = regularRows ?? decodedMappings(map); + rows: ReturnType, + map: ReturnType, +): void => { for (const row of rows) for (const segment of row) { if ( - ![1, 4, 5].includes(segment.length) || - segment.some((v) => !Number.isSafeInteger(v) || v < 0) || (segment.length !== 1 && segment[1] >= inspected.declarations.length) || (segment.length === 5 && segment[4] >= map.names.length) ) @@ -128,8 +111,6 @@ const decodeValidatedMap = ( "Decoded mapping has invalid component count, position, source index or name index.", ); } - - return { map, rows }; }; const greatestPosition = ( diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 2a5d1e963ec..7d251b41ad5 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -374,6 +374,57 @@ describe("web source-map fetching and validation: enforcing fetch limits and val }); }); +describe("browser map decoded validation", () => { + it("keeps decoded-index and coordinate failures local to their map", async () => { + const maps = [ + { version: 3, names: [], sources: ["ok.js"], mappings: "AAAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "ACAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "AADA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "DAAA" }, + { version: 3, names: [], sources: ["bad.js"], mappings: "AAAAD" }, + { + version: 3, + sections: [ + { + offset: { line: 0, column: Number.MAX_SAFE_INTEGER }, + map: { + version: 3, + names: [], + sources: ["bad.js"], + mappings: "CAAA", + }, + }, + ], + }, + { version: 3, names: [], sources: ["ok.js"], mappings: "AAAA" }, + ]; + let index = 0; + const requests = maps.map((_, requestIndex) => ({ + ...request, + scriptKey: `scr_${String(requestIndex + 1).padStart(64, "0")}`, + })); + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => + Promise.resolve( + new Response(JSON.stringify(maps[index++]), { status: 200 }), + ), + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "invalid", + "invalid", + "invalid", + "invalid", + "invalid", + "included", + ]); + expect(result.items[0]?.mappings).toHaveLength(1); + expect(result.items[6]?.mappings).toHaveLength(1); + }); +}); + describe("source-map response encoding", () => { it("rejects malformed UTF-8 source-map response bytes", async () => { const result = await fetchWebSourceMaps([request], input(), undefined, { @@ -809,6 +860,232 @@ describe("web source-map collection", () => { }); }); +describe("source-map output admission", () => { + it("admits a complete maximum-size map set when its real output fits", async () => { + const localOrigin = "http://127.0.0.1:9222"; + const mapUrl = `${localOrigin}/m`; + const requests = Array.from({ length: 50_000 }, (_, index) => ({ + ...request, + scriptKey: `scr_${index.toString(16).padStart(64, "0")}`, + declaredUrl: mapUrl, + fetchUrl: mapUrl, + })); + const map = JSON.stringify({ + version: 3, + names: [], + sources: ["x"], + mappings: "AAAA", + }); + const result = await fetchWebSourceMaps( + requests, + input({ allowed_origins: [localOrigin] }), + undefined, + { + timeoutMs: 120_000, + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }, + ); + + expect(result.status).toBe("included"); + expect(result.limitation).toBeUndefined(); + expect(result.items).toHaveLength(requests.length); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual( + 32 * 1_024 * 1_024, + ); + }, 120_000); + + it("reserves large failure contexts and reports unprocessed requests as unknown", async () => { + const largeDeclaredUrl = `${origin}/${"u".repeat(8 * 1_024 * 1_024)}`; + const requests = [ + request, + ...Array.from({ length: 5 }, (_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 2).padStart(64, "0")}`, + declaredUrl: largeDeclaredUrl, + fetchUrl: `${origin}/invalid-${String(index)}.map`, + })), + ]; + let calls = 0; + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => { + calls += 1; + return Promise.resolve( + calls === 1 + ? validMapResponse() + : new Response("not-json", { status: 200 }), + ); + }, + }); + + expect(calls).toBe(4); + expect(result).toMatchObject({ + status: "partial", + requested: 6, + processed: 4, + limitation: expect.stringContaining("2 requested maps"), + }); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "invalid", + "invalid", + "invalid", + ]); + expect( + result.items.slice(1).map(({ declared_url }) => declared_url), + ).toEqual([largeDeclaredUrl, largeDeclaredUrl, largeDeclaredUrl]); + expect(Buffer.byteLength(JSON.stringify(result))).toBeLessThanOrEqual( + 32 * 1_024 * 1_024, + ); + }); +}); + +describe("expanded source-map evidence budget", () => { + it("retains a large declared source whose exact public representation fits the budget", async () => { + const source = "x".repeat(12 * 1024 * 1024); + const map = JSON.stringify({ + version: 3, + names: [], + sources: [source], + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + expect(result.items[0]?.status).toBe("included"); + expect(result.items[0]?.original_sources[0]?.source).toBe( + `${origin}/assets/${source}`, + ); + }); + + it("releases discarded malformed maps before reserving a later valid map", async () => { + const malformed = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "a/../".repeat(3_300_000), + sources: ["main.ts"], + mappings: "ACAA", + }); + const valid = JSON.stringify({ + version: 3, + names: [], + sources: ["main.js"], + sourcesContent: [`export default "${"x".repeat(2 * 1024 * 1024)}";`], + mappings: "AAAA", + }); + const responses = [malformed, malformed, valid]; + const result = await fetchWebSourceMaps( + responses.map((_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, + })), + input(), + undefined, + { + fetch: () => + Promise.resolve(new Response(responses.shift(), { status: 200 })), + }, + ); + expect(result.items.map(({ status }) => status)).toEqual([ + "invalid", + "invalid", + "included", + ]); + expect(result.items[2]?.artifact?.text).toBe(valid); + }, 15_000); + + it("bounds expanded evidence across individually valid maps", async () => { + const expanded = JSON.stringify({ + version: 3, + names: [], + sources: ["x".repeat(60_000)], + mappings: Array.from({ length: 280 }, () => "AAAA").join(","), + }); + const responses = [expanded, expanded]; + const requests = Array.from({ length: 2 }, (_, index) => ({ + ...request, + scriptKey: `scr_${String(index + 1).padStart(64, "0")}`, + })); + const result = await fetchWebSourceMaps(requests, input(), undefined, { + fetch: () => + Promise.resolve(new Response(responses.shift() ?? "", { status: 200 })), + }); + + expect(result.status).toBe("partial"); + expect(result.items.map(({ status }) => status)).toEqual([ + "included", + "fetch_failed", + ]); + expect(result.items[0]?.mappings).toHaveLength(280); + expect(result.items[1]?.limitation).toContain( + "Expanded source-map evidence", + ); + expect(result.items[1]?.mappings).toEqual([]); + }); + + it("accepts a large raw source root that the pinned resolver normalizes away", async () => { + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "a/../".repeat(2_300_000), + sources: ["main.ts"], + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]?.status).toBe("included"); + expect(result.items[0]?.original_sources[0]?.source).toBe( + `${origin}/assets/main.ts`, + ); + }); + + it("rejects a large resolved source product before allocating mapping rows", async () => { + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: "x".repeat(8 * 1_024 * 1_024), + sources: ["main.ts"], + mappings: "AAAA,AAAA,AAAA", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + mappings: [], + }); + expect(result.items[0]?.limitation).toContain( + "Expanded source-map evidence", + ); + }); + + it("bounds raw resolved identities even when URL sanitization removes credentials", async () => { + const userinfo = "u".repeat(1 * 1_024 * 1_024); + const map = JSON.stringify({ + version: 3, + names: [], + sourceRoot: `https://${userinfo}@sources.example.test/`, + sources: Array.from({ length: 65 }, (_, index) => `source-${index}.ts`), + mappings: "", + }); + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => Promise.resolve(new Response(map, { status: 200 })), + }); + + expect(result.items[0]).toMatchObject({ + status: "fetch_failed", + artifact: null, + original_sources: [], + }); + expect(result.items[0]?.limitation).toContain( + "Resolved source-map identities exceed the 64 MiB decoder representation budget", + ); + }); +}); + describe("raw source-map structure preflight", () => { it("budgets source inventory before schema parsing its entries", async () => { const response = new Response( From faf209e50d9faab03a185c1f044de3582552d11e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:17:20 +0800 Subject: [PATCH 054/791] fix(browser): bound script metadata and preserve omissions --- docs/browser-observation.md | 9 +- docs/testing.md | 2 + .../browser-capture-metadata-budget-e2e.mjs | 209 ++++++++++++++++++ scripts/verify-real-browser.mjs | 3 + src/browser/CdpCaptureCompleteness.ts | 1 + src/browser/CdpCaptureEventHandlers.ts | 112 +++++++++- src/browser/CdpCaptureEventState.ts | 5 + src/browser/CdpCaptureEventTypes.ts | 6 + src/browser/CdpCaptureEvents.ts | 34 +++ src/browser/CdpPageCapture.ts | 1 + src/domain/browserCompleteness.ts | 1 + src/domain/webBundleAnalyzerResult.ts | 53 ++++- .../browser/cdpCaptureScriptMetadata.test.ts | 120 ++++++++++ 13 files changed, 541 insertions(+), 15 deletions(-) create mode 100644 scripts/lib/browser-capture-metadata-budget-e2e.mjs create mode 100644 tests/boundary/browser/cdpCaptureScriptMetadata.test.ts diff --git a/docs/browser-observation.md b/docs/browser-observation.md index c0840111de6..dae77e06602 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -138,7 +138,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex - Console observations with a stack source retain call type, argument types, timestamp, and source location. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded. - WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained. - Storage observations always redact values. Key names, IndexedDB names, and cache names require `include_storage_keys`; stable content fingerprints require the additional `include_storage_fingerprints` selection. Cookie fingerprints cover cookies applicable to the current main-frame URL, including its path, rather than every cookie path on that origin. Other storage remains scoped to the origin. Cache bodies above 64 KiB and partial IndexedDB remote objects make fingerprint coverage incomplete, so identical observations remain `unknown`. -- Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest. +- Script metadata is included only when CDP supplies a URL on an allowed origin. Stable keys exclude transient CDP script IDs, and exact transient raw URLs are used only during script/resource reconciliation. URL-less scripts are excluded because their origin cannot be established. When CDP supplies an execution-context association, the accepted script retains its authorized frame ID for later attribution. Script metadata admission bounds retained identity and source-map strings to 8 MiB and 50,000 records. When a source-map URL cannot fit, the script identity remains and source-map coverage reports `resource_budget_exhausted`; rejected script observations are counted without retaining their IDs. Source content is omitted unless explicitly requested and is returned inline as a self-verifying artifact with its SHA-256 digest. Frames, resources, scripts, events, and workers outside the selected origin scope are excluded. An explicit `allowed_origins` list can include additional origins; excluded target details are counted without being exposed. Retained workers include validated opener-target and parent-frame IDs when CDP @@ -148,20 +148,21 @@ prove static module ownership. ## Completeness and limits The default inspection window is 500 ms; callers choose its duration. The -capture returns all in-scope frames, DOM and accessibility nodes, scripts, +capture returns in-scope frames, DOM and accessibility nodes, admitted scripts, resources, workers, storage names, and observed network, console, and WebSocket events. Full selected text, script sources, and JSON shapes are returned inline. The result distinguishes `complete_within_window`, `policy_filtered`, `attach_limited`, and `truncated` coverage. It reports excluded items and any values omitted or truncated by -validation; it does not drop records to meet an internal collection quota. +validation and resource bounds. Omitted script metadata makes its affected +section explicitly truncated. Disallowed-origin entries are filtered before retention. CDP discovery, WebSocket connection, and commands continue until the peer responds or the caller cancels the operation; REA has no internal byte, message-count, or pending-command quota for these exchanges. Without caller cancellation, a stalled peer can leave an operation waiting. On cancellation, REA closes only its own WebSocket, waiting up to one second before terminating it. Malformed unsolicited events and malformed replies to pending commands poison the connection so later commands fail closed. When a correlated reply reports a session ID, it must match the command's selected session. An optional method is reported unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. -Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Every requested map, mapping, original source, AST node, and finding from parsed content is returned inline. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. +Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. ## Non-goals and threat model diff --git a/docs/testing.md b/docs/testing.md index bbfc70a3c86..f56710fd41e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -246,6 +246,8 @@ Do not call handcrafted utility output or synthetic binary builders real-data goldens. Keep unsupported binary layouts and malformed boundaries as targeted regressions until a real fixture establishes equivalent coverage. +`verify:browser` exercises source-map failure isolation and expanded-output limits through the compiled CLI and stdio MCP with real Chrome. It also submits five 2 MiB source-map annotations, checks retained script identities and explicit map omissions, and closes its owned fixture target. These public workflows replace redundant metadata-admission assertions; replacement and document-reset boundaries remain focused producer regressions. + `verify:browser` also captures a source-owned noise canvas as a real PNG above 8 MiB through the CLI and stdio MCP, with complete byte/digest parity and real PNG decoding. Its SDK client explicitly permits the larger inline JSON response; diff --git a/scripts/lib/browser-capture-metadata-budget-e2e.mjs b/scripts/lib/browser-capture-metadata-budget-e2e.mjs new file mode 100644 index 00000000000..26b695371d3 --- /dev/null +++ b/scripts/lib/browser-capture-metadata-budget-e2e.mjs @@ -0,0 +1,209 @@ +import { createServer } from "node:http"; +import { fileURLToPath } from "node:url"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { artifactCliEvidence, artifactMcpResult } from "./artifact-e2e.mjs"; + +const ANNOTATION_BYTES = 2 * 1024 * 1024; +const SCRIPT_COUNT = 5; + +/** Verify real Chrome scriptParsed metadata is bounded before public capture retention. */ +export async function verifyBrowserCaptureMetadataBudget(endpoint) { + const marker = "rea-capture-metadata-budget"; + let page = ""; + let pageReady = false; + let fixtureOrigin = ""; + const server = createServer( + { maxHeaderSize: 3 * 1024 * 1024 }, + (request, response) => { + if (request.url?.startsWith("/maps/") === true) { + response.writeHead(200, { + "content-type": "application/source-map+json", + }); + response.end( + JSON.stringify({ + version: 3, + names: [], + sources: [`${fixtureOrigin}/source.ts`], + sourcesContent: ["export const retainedSource = true;"], + mappings: "AAAA", + }), + ); + return; + } + if (request.url === "/capture-metadata-ready") { + pageReady = true; + response.writeHead(204).end(); + return; + } + if (request.url !== "/capture-metadata-budget") { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { "content-type": "text/html; charset=utf-8" }); + response.end(page); + }, + ); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + let targetId; + try { + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Browser metadata fixture did not bind a TCP port"); + fixtureOrigin = `http://127.0.0.1:${String(address.port)}`; + const annotations = Array.from( + { length: SCRIPT_COUNT }, + (_, index) => + ``, + ).join(""); + page = `${annotations}`; + const fixtureUrl = `${fixtureOrigin}/capture-metadata-budget`; + const created = await fetch( + `${endpoint}/json/new?${encodeURIComponent(fixtureUrl)}`, + { method: "PUT" }, + ); + if (!created.ok) + throw new Error(`Chrome target creation failed: HTTP ${created.status}`); + const target = await created.json(); + if (typeof target.id !== "string" || target.id.length === 0) + throw new Error("Chrome target creation returned no target ID"); + targetId = target.id; + + const deadline = Date.now() + 15_000; + let served = false; + while (!served && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 20)); + served = pageReady; + } + if (!served) throw new Error("Chrome did not request the metadata fixture"); + + const evidence = await artifactCliEvidence("inspect-web-page", endpoint, [ + targetId, + "--allowed-origins", + fixtureOrigin, + "--observation-ms", + "0", + ]); + const cliFacts = assertMetadataBudget(evidence.normalized_result); + const bundle = await artifactCliEvidence("analyze-web-bundle", endpoint, [ + targetId, + "--allowed-origins", + fixtureOrigin, + "--observation-ms", + "0", + "--fetch-source-maps", + ]); + assertBundleMetadataOmissions( + bundle.normalized_result, + cliFacts.omittedMaps, + ); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"], + env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" }, + stderr: "pipe", + }); + const client = new Client({ name: "browser-metadata-e2e", version: "1" }); + try { + await client.connect(transport); + assertMetadataBudget( + await artifactMcpResult(client, "inspect_web_page", { + cdp_endpoint: endpoint, + allowed_origins: [fixtureOrigin], + target_id: targetId, + observation_ms: 0, + }), + ); + assertBundleMetadataOmissions( + await artifactMcpResult(client, "analyze_web_bundle", { + cdp_endpoint: endpoint, + allowed_origins: [fixtureOrigin], + target_id: targetId, + observation_ms: 0, + fetch_source_maps: true, + }), + cliFacts.omittedMaps, + ); + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } + return { ...cliFacts, public_cli: true, public_stdio_mcp: true }; + } finally { + try { + if (targetId !== undefined) { + const closed = await fetch( + `${endpoint}/json/close/${encodeURIComponent(targetId)}`, + ); + if (!closed.ok) + throw new Error( + `Chrome fixture target cleanup failed: HTTP ${closed.status}`, + ); + } + } finally { + server.closeAllConnections(); + await new Promise((resolve, reject) => + server.close((error) => + error === undefined ? resolve() : reject(error), + ), + ); + } + } +} + +function assertMetadataBudget(result) { + const scripts = result.scripts.items.filter( + (script) => + script.url.includes("capture-metadata-budget") && + script.length >= ANNOTATION_BYTES, + ); + const retainedMaps = scripts.filter( + (script) => script.source_map_url !== null, + ).length; + const omittedMaps = SCRIPT_COUNT - retainedMaps; + const exclusion = result.completeness.excluded.find( + (item) => + item.section === "source_maps" && + item.reason === "resource_budget_exhausted", + ); + if ( + scripts.length !== SCRIPT_COUNT || + retainedMaps <= 0 || + retainedMaps >= SCRIPT_COUNT || + exclusion?.count !== omittedMaps || + !result.completeness.truncated_sections.includes("source_maps") + ) + throw new Error( + `Real Chrome source-map metadata admission was not reported accurately: scripts=${String(scripts.length)}, retained_maps=${String(retainedMaps)}, excluded=${String(exclusion?.count)}`, + ); + return { + scripts: scripts.length, + retainedMaps, + omittedMaps, + declaredBytes: ANNOTATION_BYTES * SCRIPT_COUNT, + }; +} + +function assertBundleMetadataOmissions(result, omittedMaps) { + const unknown = result.unknowns.find( + ({ dimension, reason }) => + dimension === "source_maps" && + reason.includes(String(omittedMaps)) && + reason.includes("source-map declarations"), + ); + if ( + result.observations.source_maps.status !== "included" || + result.observations.source_maps.items.length === 0 || + result.completeness.status !== "partial" || + unknown === undefined + ) + throw new Error( + "Bundle analysis lost capture omissions despite valid retained maps", + ); +} diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index d35756de586..e1102667cba 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -41,6 +41,7 @@ import { verifyBrowserNetworkEvidence } from "./lib/browser-network-e2e.mjs"; import { verifyBrowserScriptExport } from "./lib/browser-script-export-e2e.mjs"; import { verifyBrowserModules } from "./lib/browser-module-e2e.mjs"; import { verifyBrowserDomDestinations } from "./lib/browser-dom-destinations-e2e.mjs"; +import { verifyBrowserCaptureMetadataBudget } from "./lib/browser-capture-metadata-budget-e2e.mjs"; import { artifactCliEvidence, artifactMcpResult } from "./lib/artifact-e2e.mjs"; const REAL_BROWSER_STARTUP_TIMEOUT_MS = 60_000; @@ -332,6 +333,7 @@ try { target_id: target, }); const moduleTrace = await verifyBrowserModules(executable); + const captureMetadata = await verifyBrowserCaptureMetadataBudget(endpoint); report = { browser: observed.value.browser.product, endpoint, @@ -353,6 +355,7 @@ try { networkEvidence, scriptExport, moduleTrace, + captureMetadata, domDestinations, browserScenarioCli: true, browserScenarioAttachCleanup: "disconnected-external", diff --git a/src/browser/CdpCaptureCompleteness.ts b/src/browser/CdpCaptureCompleteness.ts index 48e66e3117c..4c53f835c87 100644 --- a/src/browser/CdpCaptureCompleteness.ts +++ b/src/browser/CdpCaptureCompleteness.ts @@ -65,6 +65,7 @@ export class CdpCaptureCompleteness { reason === "invalid_protocol_value" ) this.#unavailable.add(section); + if (reason === "resource_budget_exhausted") this.#truncated.add(section); const key = `${section}\0${reason}`; const existing = this.#excluded.get(key); const combined = diff --git a/src/browser/CdpCaptureEventHandlers.ts b/src/browser/CdpCaptureEventHandlers.ts index 95cd25a1c38..9da53caffe7 100644 --- a/src/browser/CdpCaptureEventHandlers.ts +++ b/src/browser/CdpCaptureEventHandlers.ts @@ -26,7 +26,11 @@ import { isMainFrameNavigation, } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js"; -import type { CapturedScript, NetworkState } from "./CdpCaptureEventTypes.js"; +import { + CDP_CAPTURE_SCRIPT_METADATA_LIMITS, + type CapturedScript, + type NetworkState, +} from "./CdpCaptureEventTypes.js"; export const handleExecutionContextCreated = ( state: CdpCaptureEventsState, @@ -64,12 +68,18 @@ export const handleScriptParsed = ( state.completeness.exclude("scripts", "invalid_protocol_value"); return; } + const replacedBytes = state.scriptMetadataBytesById.get(scriptId) ?? 0; + state.scriptMetadataBudgetOmissionsById.delete(scriptId); + if (replacedBytes > 0) { + state.scripts.delete(scriptId); + state.scriptMetadataBytesById.delete(scriptId); + state.retainedScriptMetadataBytes -= replacedBytes; + } if (sanitized === undefined) { state.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl)); return; } - const sourceMap = sourceMapForScript(params.sourceMapURL, rawUrl, state); - const script: CapturedScript = { + const baseScript = { scriptId, rawUrl, url: sanitized.url, @@ -84,21 +94,79 @@ export const handleScriptParsed = ( ), isModule: params.isModule === true, language: cdpStringValue(params.scriptLanguage) ?? null, - sourceMapUrl: sourceMap.sanitized, - sourceMapRawUrl: sourceMap.raw, executionContextKey: executionContextKey(params.executionContextId), }; + const baseBytes = scriptMetadataStringBytes(baseScript); + const sourceMapDeclared = + typeof params.sourceMapURL === "string" && params.sourceMapURL.length > 0; + if ( + state.scripts.size >= CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts || + baseBytes > + CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes - + state.retainedScriptMetadataBytes + ) { + state.rejectedScriptMetadataBudgetCount += 1; + if (sourceMapDeclared) state.rejectedSourceMapMetadataBudgetCount += 1; + return; + } + const sourceMap = sourceMapForScript( + params.sourceMapURL, + rawUrl, + state, + CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes - + state.retainedScriptMetadataBytes - + baseBytes, + ); + const { budgetExhausted, ...sourceMapFields } = sourceMap; + const script: CapturedScript = { ...baseScript, ...sourceMapFields }; + if (budgetExhausted) + state.scriptMetadataBudgetOmissionsById.set(scriptId, true); + const retainedBytes = + baseBytes + + stringByteLength(sourceMap.sourceMapUrl) + + stringByteLength(sourceMap.sourceMapRawUrl); + state.retainedScriptMetadataBytes += retainedBytes; + state.scriptMetadataBytesById.set(scriptId, retainedBytes); state.scripts.set(scriptId, script); }; +const scriptMetadataStringBytes = (script: { + readonly scriptId: string; + readonly rawUrl: string; + readonly url: string; + readonly origin: string | null; + readonly hash: string; + readonly language: string | null; + readonly executionContextKey: string | null; +}): number => + stringByteLength(script.scriptId) + + stringByteLength(script.rawUrl) + + stringByteLength(script.url) + + stringByteLength(script.origin) + + stringByteLength(script.hash) + + stringByteLength(script.language) + + stringByteLength(script.executionContextKey); + +const stringByteLength = (value: string | null): number => + value === null ? 0 : Buffer.byteLength(value, "utf8"); + const sourceMapForScript = ( value: unknown, scriptUrl: string, state: CdpCaptureEventsState, -): { readonly sanitized: string | null; readonly raw: string | null } => { + remainingBytes: number, +): { + readonly sourceMapUrl: string | null; + readonly sourceMapRawUrl: string | null; + readonly budgetExhausted: boolean; +} => { const declaredUrl = cdpStringValue(value); if (declaredUrl === undefined || declaredUrl === "") - return { sanitized: null, raw: null }; + return { + sourceMapUrl: null, + sourceMapRawUrl: null, + budgetExhausted: false, + }; let rawUrl: string; try { rawUrl = new URL(declaredUrl, scriptUrl).href; @@ -106,12 +174,36 @@ const sourceMapForScript = ( // Unparseable URLs are recorded as unsupported, not as failures. void cause; state.completeness.exclude("source_maps", "unsupported_url"); - return { sanitized: null, raw: null }; + return { + sourceMapUrl: null, + sourceMapRawUrl: null, + budgetExhausted: false, + }; } const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins); - if (sanitized !== undefined) return { sanitized: sanitized.url, raw: rawUrl }; + if (sanitized !== undefined) { + if ( + stringByteLength(rawUrl) + stringByteLength(sanitized.url) > + remainingBytes + ) { + return { + sourceMapUrl: null, + sourceMapRawUrl: null, + budgetExhausted: true, + }; + } + return { + sourceMapUrl: sanitized.url, + sourceMapRawUrl: rawUrl, + budgetExhausted: false, + }; + } state.completeness.exclude("source_maps", exclusionReasonForUrl(rawUrl)); - return { sanitized: null, raw: null }; + return { + sourceMapUrl: null, + sourceMapRawUrl: null, + budgetExhausted: false, + }; }; export const handleRequestWillBeSent = ( diff --git a/src/browser/CdpCaptureEventState.ts b/src/browser/CdpCaptureEventState.ts index acff1d535c6..6a0ab21a6e8 100644 --- a/src/browser/CdpCaptureEventState.ts +++ b/src/browser/CdpCaptureEventState.ts @@ -9,6 +9,11 @@ export interface CdpCaptureEventsState { readonly input: InspectWebPageInput; readonly allowedOrigins: ReadonlySet; readonly scripts: Map; + readonly scriptMetadataBudgetOmissionsById: Map; + rejectedScriptMetadataBudgetCount: number; + rejectedSourceMapMetadataBudgetCount: number; + readonly scriptMetadataBytesById: Map; + retainedScriptMetadataBytes: number; readonly executionContextFrames: Map; readonly network: Map; readonly networkRequestTimestamps: Map; diff --git a/src/browser/CdpCaptureEventTypes.ts b/src/browser/CdpCaptureEventTypes.ts index 3da324ec27c..d8a83e88a10 100644 --- a/src/browser/CdpCaptureEventTypes.ts +++ b/src/browser/CdpCaptureEventTypes.ts @@ -15,3 +15,9 @@ export interface CapturedScript { } export type NetworkState = WebPageInspection["network"]["requests"][number]; + +/** Bounds retained CDP script identities independently from source text capture. */ +export const CDP_CAPTURE_SCRIPT_METADATA_LIMITS = { + retainedBytes: 8 * 1024 * 1024, + scripts: 50_000, +} as const; diff --git a/src/browser/CdpCaptureEvents.ts b/src/browser/CdpCaptureEvents.ts index bb24433eee2..df39015737d 100644 --- a/src/browser/CdpCaptureEvents.ts +++ b/src/browser/CdpCaptureEvents.ts @@ -15,7 +15,13 @@ export { type CapturedScript } from "./CdpCaptureEventTypes.js"; /** Event accumulator that validates and normalizes payloads at ingestion. */ export class CdpCaptureEvents implements CdpCaptureEventsState { + #scriptMetadataBudgetExclusionsRecorded = false; readonly scripts = new Map(); + readonly scriptMetadataBudgetOmissionsById = new Map(); + rejectedScriptMetadataBudgetCount = 0; + rejectedSourceMapMetadataBudgetCount = 0; + readonly scriptMetadataBytesById = new Map(); + retainedScriptMetadataBytes = 0; readonly executionContextFrames = new Map(); readonly network = new Map(); readonly networkRequestTimestamps = new Map(); @@ -51,6 +57,12 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { this.navigationDuringCapture = false; this.committedDocument = undefined; this.scripts.clear(); + this.scriptMetadataBudgetOmissionsById.clear(); + this.rejectedScriptMetadataBudgetCount = 0; + this.rejectedSourceMapMetadataBudgetCount = 0; + this.scriptMetadataBytesById.clear(); + this.retainedScriptMetadataBytes = 0; + this.#scriptMetadataBudgetExclusionsRecorded = false; this.executionContextFrames.clear(); this.network.clear(); this.networkRequestTimestamps.clear(); @@ -81,6 +93,28 @@ export class CdpCaptureEvents implements CdpCaptureEventsState { this.committedDocument = undefined; } + /** Record final script metadata omissions after all scriptParsed replacements. */ + recordScriptMetadataBudgetExclusions(): void { + if (this.#scriptMetadataBudgetExclusionsRecorded) return; + this.#scriptMetadataBudgetExclusionsRecorded = true; + const omittedScripts = this.rejectedScriptMetadataBudgetCount; + const omittedSourceMaps = + this.rejectedSourceMapMetadataBudgetCount + + this.scriptMetadataBudgetOmissionsById.size; + if (omittedScripts > 0) + this.completeness.exclude( + "scripts", + "resource_budget_exhausted", + omittedScripts, + ); + if (omittedSourceMaps > 0) + this.completeness.exclude( + "source_maps", + "resource_budget_exhausted", + omittedSourceMaps, + ); + } + ingest(event: CdpEvent): void { const params = recordValue(event.params); if (params === undefined) return; diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts index 569a1d57098..d8c2754947f 100644 --- a/src/browser/CdpPageCapture.ts +++ b/src/browser/CdpPageCapture.ts @@ -162,6 +162,7 @@ const captureAuthorizedPage = async ( if (state.events.navigationDuringCapture || completedUrl !== attachedUrl) throw new BrowserObservationError("inspect_web_page", "target_changed"); await report(context.progress, 3, "Normalizing browser evidence"); + state.events.recordScriptMetadataBudgetExclusions(); return { inspection: normalizedInspection(state, { attachedUrl, diff --git a/src/domain/browserCompleteness.ts b/src/domain/browserCompleteness.ts index 2fd8769dfe6..64cca6b1330 100644 --- a/src/domain/browserCompleteness.ts +++ b/src/domain/browserCompleteness.ts @@ -36,6 +36,7 @@ const browserExclusionReasonSchema = z.enum([ "out_of_target_scope", "provider_unavailable", "invalid_protocol_value", + "resource_budget_exhausted", ]); export type BrowserExclusionReason = z.infer< typeof browserExclusionReasonSchema diff --git a/src/domain/webBundleAnalyzerResult.ts b/src/domain/webBundleAnalyzerResult.ts index b4ac67a29de..ceeda798668 100644 --- a/src/domain/webBundleAnalyzerResult.ts +++ b/src/domain/webBundleAnalyzerResult.ts @@ -17,12 +17,19 @@ export const buildWebBundleAnalysis = ( const unavailable = inspection.scripts.items .filter((script) => !script.source.included) .map(({ script_key }) => script_key); + const omittedScripts = captureBudgetOmission(inspection, "scripts"); + const omittedSourceMaps = + sourceMaps.status === "not_requested" + ? undefined + : captureBudgetOmission(inspection, "source_maps"); const sourceMapIncomplete = sourceMaps.status === "partial" || sourceMaps.status === "unavailable"; const partial = accumulator.parseFailures > 0 || unavailable.length > 0 || - sourceMapIncomplete; + sourceMapIncomplete || + omittedScripts !== undefined || + omittedSourceMaps !== undefined; return webBundleAnalysisSchema.parse({ capture: buildCaptureObservation(inspection, sourceScripts), observations: { @@ -41,6 +48,8 @@ export const buildWebBundleAnalysis = ( unavailable, parseFailures: accumulator.parseFailures, sourceMaps, + omittedScripts, + omittedSourceMaps, }), completeness: buildCompleteness({ partial, @@ -83,9 +92,20 @@ interface UnknownsInput { readonly unavailable: string[]; readonly parseFailures: number; readonly sourceMaps: WebBundleAnalysis["observations"]["source_maps"]; + readonly omittedScripts: number | null | undefined; + readonly omittedSourceMaps: number | null | undefined; } const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ + ...(input.omittedScripts === undefined + ? [] + : [ + { + dimension: "script_inventory", + reason: captureBudgetReason("script records", input.omittedScripts), + affected_script_keys: [], + }, + ]), ...(input.unavailable.length === 0 ? [] : [ @@ -106,6 +126,18 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ ), }, ]), + ...(input.omittedSourceMaps === undefined + ? [] + : [ + { + dimension: "source_maps" as const, + reason: captureBudgetReason( + "source-map declarations", + input.omittedSourceMaps, + ), + affected_script_keys: [], + }, + ]), ...(input.sourceMaps.status === "not_requested" || input.sourceMaps.status === "included" ? [] @@ -123,6 +155,25 @@ const buildUnknowns = (input: UnknownsInput): WebBundleAnalysis["unknowns"] => [ ]), ]; +const captureBudgetOmission = ( + inspection: WebPageInspection, + section: "scripts" | "source_maps", +): number | null | undefined => { + const exclusions = inspection.completeness.excluded.filter( + (exclusion) => + exclusion.section === section && + exclusion.reason === "resource_budget_exhausted", + ); + if (exclusions.length === 0) return undefined; + if (exclusions.some(({ count }) => count === null)) return null; + return exclusions.reduce((sum, { count }) => sum + (count ?? 0), 0); +}; + +const captureBudgetReason = (label: string, count: number | null): string => + count === null + ? `Capture omitted an unknown number of ${label} because its resource budget was exhausted.` + : `Capture omitted ${String(count)} ${label} because its resource budget was exhausted.`; + interface CompletenessInput { readonly partial: boolean; readonly parsedScripts: number; diff --git a/tests/boundary/browser/cdpCaptureScriptMetadata.test.ts b/tests/boundary/browser/cdpCaptureScriptMetadata.test.ts new file mode 100644 index 00000000000..d43c9ff4172 --- /dev/null +++ b/tests/boundary/browser/cdpCaptureScriptMetadata.test.ts @@ -0,0 +1,120 @@ +import { describe, expect, it } from "vitest"; + +import { CdpCaptureEvents } from "../../../src/browser/CdpCaptureEvents.js"; +import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; +import { CDP_CAPTURE_SCRIPT_METADATA_LIMITS } from "../../../src/browser/CdpCaptureEventTypes.js"; + +describe("CDP script metadata admission", () => { + it("releases a replaced script ID's prior bytes and resets admission on a new document", () => { + const events = captureEvents(); + const origin = "https://app.example"; + events.beginAuthorizedFrame("frame-1"); + const emit = (scriptId: string, bytes: number): void => { + events.ingest({ + method: "Debugger.scriptParsed", + params: { + scriptId, + url: `${origin}/app.js`, + sourceMapURL: `/maps/${"x".repeat(bytes)}.map`, + }, + }); + }; + + emit("same-id", 3 * 1024 * 1024); + emit("same-id", 2 * 1024 * 1024); + emit("next-id", 1024 * 1024); + + expect(events.scripts.get("same-id")?.sourceMapUrl).toContain( + "x".repeat(2 * 1024 * 1024), + ); + expect(events.scripts.get("next-id")?.sourceMapUrl).toContain( + "x".repeat(1024 * 1024), + ); + expect(events.retainedScriptMetadataBytes).toBeLessThanOrEqual( + CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes, + ); + const normalizedAway = `./`.repeat(1_200_000) + "map.js.map"; + events.ingest({ + method: "Debugger.scriptParsed", + params: { + scriptId: "normalized-url", + url: `${origin}/app.js`, + sourceMapURL: normalizedAway, + }, + }); + expect(events.scripts.get("normalized-url")?.sourceMapUrl).toBe( + `${origin}/map.js.map`, + ); + + events.beginAuthorizedFrame("frame-2"); + emit("new-document", 5 * 1024 * 1024); + emit("new-document", 1024 * 1024); + events.recordScriptMetadataBudgetExclusions(); + expect( + events.completeness + .snapshot() + .excluded.find( + ({ section, reason }) => + section === "source_maps" && reason === "resource_budget_exhausted", + ), + ).toBeUndefined(); + + events.beginAuthorizedFrame("frame-3"); + emit("new-document", 3 * 1024 * 1024); + expect(events.scripts.get("new-document")?.sourceMapUrl).toContain( + "x".repeat(3 * 1024 * 1024), + ); + }); + + it("does not retain IDs for scripts rejected by byte or record limits", () => { + const events = captureEvents(); + const origin = "https://app.example"; + events.beginAuthorizedFrame("frame-1"); + const emit = (scriptId: string): void => { + events.ingest({ + method: "Debugger.scriptParsed", + params: { scriptId, url: `${origin}/app.js` }, + }); + }; + + emit("x".repeat(CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes + 1)); + for ( + let index = 0; + index < CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts; + index += 1 + ) + emit(`script-${String(index)}`); + for (let index = 0; index < 10; index += 1) + emit(`rejected-${String(index)}`); + + expect(events.scripts.size).toBe( + CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts, + ); + expect(events.scriptMetadataBudgetOmissionsById.size).toBe(0); + expect(events.rejectedScriptMetadataBudgetCount).toBe(11); + events.recordScriptMetadataBudgetExclusions(); + expect( + events.completeness + .snapshot() + .excluded.find( + ({ section, reason }) => + section === "scripts" && reason === "resource_budget_exhausted", + ), + ).toEqual({ + section: "scripts", + reason: "resource_budget_exhausted", + count: 11, + }); + }); +}); + +const captureEvents = (): CdpCaptureEvents => + new CdpCaptureEvents( + inspectWebPageInputSchema.parse({ + cdp_endpoint: "http://127.0.0.1:9222", + allowed_origins: ["https://app.example"], + target_id: "page", + observation_ms: 0, + }), + new Set(["https://app.example"]), + ); From f7366a517ff0545f01df6202b1ed3231d9dd3b4a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:17:25 +0800 Subject: [PATCH 055/791] fix(verify): close both sides of the browser proxy --- scripts/fixtures/page-cdp-proxy.mjs | 43 ++++++++++++++++++++++++----- scripts/verify-real-browser.mjs | 3 ++ 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/scripts/fixtures/page-cdp-proxy.mjs b/scripts/fixtures/page-cdp-proxy.mjs index 6a002e9d8bf..faf43eff3ad 100644 --- a/scripts/fixtures/page-cdp-proxy.mjs +++ b/scripts/fixtures/page-cdp-proxy.mjs @@ -11,6 +11,7 @@ import WebSocket, { WebSocketServer } from "ws"; */ export async function startPageCdpProxy(upstreamEndpoint) { const clients = new Set(); + const upstreams = new Set(); const server = createServer(async (request, response) => { try { if (request.url !== "/json/version" && request.url !== "/json/list") { @@ -43,13 +44,18 @@ export async function startPageCdpProxy(upstreamEndpoint) { const upstream = new WebSocket( new URL(request.url, upstreamEndpoint).href.replace(/^http/, "ws"), ); + upstreams.add(upstream); const pending = []; clients.add(client); - const close = () => { + const closeClient = () => { clients.delete(client); if (client.readyState < WebSocket.CLOSING) client.close(); if (upstream.readyState < WebSocket.CLOSING) upstream.close(); }; + const finishUpstream = () => { + upstreams.delete(upstream); + closeClient(); + }; client.on("message", (data, binary) => { if (upstream.readyState === WebSocket.OPEN) upstream.send(data, { binary }); @@ -63,10 +69,10 @@ export async function startPageCdpProxy(upstreamEndpoint) { pending.length = 0; webSockets.emit("connection", client, request); }); - client.on("close", close); - client.on("error", close); - upstream.on("close", close); - upstream.on("error", close); + client.on("close", closeClient); + client.on("error", closeClient); + upstream.on("close", finishUpstream); + upstream.on("error", closeClient); }); }); await new Promise((resolve, reject) => { @@ -81,10 +87,20 @@ export async function startPageCdpProxy(upstreamEndpoint) { endpoint, disconnectClients() { for (const client of clients) client.terminate(); + for (const upstream of upstreams) upstream.terminate(); }, async close() { - for (const client of clients) client.terminate(); - webSockets.close(); + const activeClients = [...clients]; + const activeUpstreams = [...upstreams]; + await Promise.all([ + terminateSockets(activeClients), + terminateSockets(activeUpstreams), + ]); + await new Promise((resolve, reject) => + webSockets.close((error) => + error === undefined ? resolve() : reject(error), + ), + ); await new Promise((resolve, reject) => server.close((error) => error === undefined ? resolve() : reject(error), @@ -94,6 +110,19 @@ export async function startPageCdpProxy(upstreamEndpoint) { }; } +async function terminateSockets(sockets) { + const closing = [...sockets] + .filter((socket) => socket.readyState !== WebSocket.CLOSED) + .map( + (socket) => + new Promise((resolve) => { + socket.once("close", resolve); + socket.terminate(); + }), + ); + await Promise.all(closing); +} + async function pageScopedVersion(upstreamEndpoint, version) { const targets = await upstreamJson(upstreamEndpoint, "/json/list"); const page = targets.find( diff --git a/scripts/verify-real-browser.mjs b/scripts/verify-real-browser.mjs index e1102667cba..5209df8ed2b 100644 --- a/scripts/verify-real-browser.mjs +++ b/scripts/verify-real-browser.mjs @@ -363,8 +363,11 @@ try { verified: true, }; } finally { + process.stderr.write("Browser verifier cleanup: closing page proxy\n"); if (pageProxy !== undefined) await pageProxy.close(); + process.stderr.write("Browser verifier cleanup: stopping owned Chrome\n"); if (browser !== undefined) await stopProcess(browser); + process.stderr.write("Browser verifier cleanup: closing fixture site\n"); await site.close(); await rm(profile, { recursive: true, From e762e32afbba3fd32532fd42a3f4fdcd05252ad3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:29:11 +0800 Subject: [PATCH 056/791] refactor(browser): charge source-map records before retention --- src/browser/WebSourceMapFetcher.ts | 91 +++++++++--------------------- 1 file changed, 28 insertions(+), 63 deletions(-) diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 5464fcc0f87..f5657a77b2e 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -470,28 +470,6 @@ const normalizeSourceMap = ( const sourceJsonBytes = new Map(); const nameJsonBytes = new Map(); const originalSources: ParsedSourceMapItem["original_sources"] = []; - let sourceBytes = 0; - for (const { leaf, resolvedSources } of decoded) { - for (const [index, rawSource] of leaf.map.sources.entries()) { - if ((index & 255) === 0) check(); - const resolved = - resolvedSources[index] ?? rawSource ?? "[unknown-source]"; - const sanitized = cachedSanitized(sourcesByRaw, resolved); - sourceBytes += - cachedJsonBytes(sourceJsonBytes, sanitized) + - sourceEntryOverheadBytes(); - const content = leaf.map.sourcesContent?.[index] ?? null; - if (content !== null) - sourceBytes += - jsonBytes(content) + - artifactMetadataBytes(sourceMediaType(rawSource), content) - - 4; - } - } - consumeExpandedBytes( - context.budget, - sourceBytes + originalSourcesCount(decoded), - ); for (const { leaf, resolvedSources } of decoded) { for (const [index, rawSource] of leaf.map.sources.entries()) { if ((index & 255) === 0) check(); @@ -499,6 +477,17 @@ const normalizeSourceMap = ( resolvedSources[index] ?? rawSource ?? "[unknown-source]"; const sanitized = cachedSanitized(sourcesByRaw, resolved); const content = leaf.map.sourcesContent?.[index] ?? null; + consumeExpandedBytes( + context.budget, + cachedJsonBytes(sourceJsonBytes, sanitized) + + sourceEntryOverheadBytes() + + (originalSources.length === 0 ? 0 : 1) + + (content === null + ? 0 + : jsonBytes(content) + + artifactMetadataBytes(sourceMediaType(rawSource), content) - + 4), + ); originalSources.push({ source: sanitized, artifact: @@ -508,39 +497,6 @@ const normalizeSourceMap = ( }); } } - let mappingBytes = 0; - let mappingCount = 0; - for (const { leaf, rows, resolvedSources } of decoded) - for (const [line, row] of rows.entries()) { - if ((line & 255) === 0) check(); - for (const segment of row) { - if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; - if (segment.length === 1) continue; - const rawSource = leaf.map.sources[segment[1]]; - const source = cachedSanitized( - sourcesByRaw, - resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]", - ); - const name = - segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; - mappingBytes += - cachedJsonBytes(sourceJsonBytes, source) + - (name === null ? 4 : cachedJsonBytes(nameJsonBytes, name)) + - mappingRowBytes( - leaf.offset.line + line + 1, - segment[0] + (line === 0 ? leaf.offset.column : 0), - segment[2] + 1, - segment[3], - ); - mappingCount += 1; - if ( - mappingBytes > - WEB_SOURCE_MAP_LIMITS.outputBytes - context.budget.expandedBytes - ) - throw expandedOutputFailure(); - } - } - consumeExpandedBytes(context.budget, mappingBytes + mappingCount); const mappings: ParsedSourceMapItem["mappings"] = []; for (const { leaf, rows, resolvedSources } of decoded) for (const [line, row] of rows.entries()) { @@ -554,10 +510,24 @@ const normalizeSourceMap = ( const source = cachedSanitized(sourcesByRaw, resolved); const name = segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; + const generatedLine = leaf.offset.line + line + 1; + const generatedColumn = + segment[0] + (line === 0 ? leaf.offset.column : 0); + consumeExpandedBytes( + context.budget, + cachedJsonBytes(sourceJsonBytes, source) + + (name === null ? 4 : cachedJsonBytes(nameJsonBytes, name)) + + mappingRowBytes( + generatedLine, + generatedColumn, + segment[2] + 1, + segment[3], + ) + + (mappings.length === 0 ? 0 : 1), + ); mappings.push({ - generated_line: leaf.offset.line + line + 1, - generated_column: - segment[0] + (line === 0 ? leaf.offset.column : 0), + generated_line: generatedLine, + generated_column: generatedColumn, source, original_line: segment[2] + 1, original_column: segment[3], @@ -858,11 +828,6 @@ const sourceMapSuccessReservation = (request: WebSourceMapRequest): number => { const sourceEntryOverheadBytes = (): number => Buffer.byteLength('{"source":,"artifact":null}'); -const originalSourcesCount = ( - decoded: ReturnType, -): number => - decoded.reduce((count, { leaf }) => count + leaf.map.sources.length, 0); - const cachedJsonBytes = (cache: Map, value: string): number => { const cached = cache.get(value); if (cached !== undefined) return cached; From 480b16b9c151e1d4bfbca8b49a0668d74ceaa400 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:39:02 +0800 Subject: [PATCH 057/791] fix(ghidra): parse long procedure selectors without recursion --- bridge/ghidra/ReaGhidraBridge.java | 41 +++++++++++++++++-- scripts/lib/real-ghidra-boundaries.mjs | 19 +++++++++ .../lib/real-ghidra-namespace-annotations.mjs | 8 +++- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index c6e61648655..31805ad75e4 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1621,8 +1621,7 @@ private List strings() throws Exception { private Function resolveProcedure(String value) throws Exception { // Explicit address syntax wins; an unprefixed identifier is first an // exact symbol name, even when every character is a hexadecimal digit. - boolean explicitAddress = value.matches("(?i)^0x[0-9a-f]+$") || - value.matches("(?i)^(?:[a-z0-9._~-]|%[0-9a-f]{2})+:0x[0-9a-f]+$"); + boolean explicitAddress = isExplicitReaAddress(value); if (!explicitAddress) { List matches = new ArrayList<>(); for (FunctionEntry entry : procedures()) { @@ -2997,6 +2996,42 @@ private Address requireAddress(JsonObject params, String name) { return parseReaAddress(requireString(params, name)); } + private static boolean isExplicitReaAddress(String value) { + if (value.regionMatches(true, 0, "0x", 0, 2) && + isHexadecimal(value, 2, value.length())) + return true; + int separator = value.toLowerCase(Locale.ROOT).lastIndexOf(":0x"); + if (separator <= 0 || !isHexadecimal(value, separator + 3, value.length())) + return false; + // Classify the encoded space iteratively. A repeated regex alternation + // can exhaust Java's stack on a long ordinary symbol name. + for (int index = 0; index < separator; index++) { + char item = value.charAt(index); + if ((item >= 'A' && item <= 'Z') || + (item >= 'a' && item <= 'z') || + (item >= '0' && item <= '9') || + item == '.' || item == '_' || item == '~' || item == '-') + continue; + if (item != '%' || index + 2 >= separator || + !isHexadecimal(value, index + 1, index + 3)) + return false; + index += 2; + } + return true; + } + + private static boolean isHexadecimal(String value, int start, int end) { + if (start >= end) return false; + for (int index = start; index < end; index++) { + char item = value.charAt(index); + if (!((item >= '0' && item <= '9') || + (item >= 'a' && item <= 'f') || + (item >= 'A' && item <= 'F'))) + return false; + } + return true; + } + private Address parseReaAddress(String value) { Address address = tryParseAddress(value); if (address == null) { @@ -3017,7 +3052,7 @@ private Address tryParseAddress(String value) { else if (value.startsWith("0x") || value.startsWith("0X")) { offset = value.substring(2); } - if (!offset.matches("[0-9A-Fa-f]+")) { + if (!isHexadecimal(offset, 0, offset.length())) { return null; } AddressSpace space = spaceName == null diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 833a3422204..17ddff340ff 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -210,6 +210,24 @@ export async function verifyGhidraBoundaries( entrypoint, env, }); + const longSelector = "REA_MISSING_" + "x".repeat(32768); + for (const selected of [ + longSelector, + `${longSelector}:0x1`, + `${"%78".repeat(12000)}:0x1`, + `${longSelector}%GG:0x1`, + ]) { + await invalid( + "procedure_address", + { procedure: selected }, + /Unknown|invalid/u, + ); + assert.equal( + await call("procedure_address", { procedure: address }), + address, + "Invalid selectors must preserve the live provider connection", + ); + } const baseline = await call("inspect_native_instruction", { address }); assert.equal(baseline.status, "decoded"); const bytes = await call("read_bytes", { address, length: baseline.length }); @@ -724,6 +742,7 @@ export async function verifyGhidraBoundaries( equivalent_instruction_address_spellings: true, qualified_annotation_name_roundtrip: true, oversized_result_retention_and_complete_export: true, + long_selector_rejection_and_provider_recovery: true, source_immutable: true, reopen_discards_edits: true, long_tmpdir_private_socket_cleanup: true, diff --git a/scripts/lib/real-ghidra-namespace-annotations.mjs b/scripts/lib/real-ghidra-namespace-annotations.mjs index f37e0441506..4f6d8c12775 100644 --- a/scripts/lib/real-ghidra-namespace-annotations.mjs +++ b/scripts/lib/real-ghidra-namespace-annotations.mjs @@ -82,7 +82,13 @@ export async function verifyGhidraNamespaceAnnotations({ }), address, ); - for (const name of ["other::literal", `${namespace}::child::literal`]) { + for (const name of [ + "other::literal", + `${namespace}::child::literal`, + "0xnothex", + "address%GG:0x1", + "address:0xnothex", + ]) { const literal = await call("annotate_native_function", { procedure: address, name, From b3f1b4e822aec5626d1952a88f50c7520dc265d3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:39:05 +0800 Subject: [PATCH 058/791] fix(mcp): retain oversized errors before transport delivery --- docs/testing.md | 10 +- scripts/lib/real-ghidra-large-results.mjs | 40 ++++ src/server/EvidenceMcpServer.ts | 206 ++++++++++++++++++ src/server/createServer.ts | 8 +- src/server/toolResult.ts | 29 ++- .../mcp/oversizedErrorDeliveryMcp.test.ts | 115 ++++++++++ 6 files changed, 393 insertions(+), 15 deletions(-) create mode 100644 src/server/EvidenceMcpServer.ts create mode 100644 tests/boundary/mcp/oversizedErrorDeliveryMcp.test.ts diff --git a/docs/testing.md b/docs/testing.md index f56710fd41e..ce5e24bc6e7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -366,10 +366,16 @@ behavior, and independent CLI/MCP database ownership. Large-result probes compile initialized host-native data sized from the pinned MCP SDK receive budget. Real byte reads, annotation edits and function dossiers exceed that budget while preserving the connection and active analysis run. -Each delivery error must identify its successfully retained Evidence record; -export must recover every source byte and complete annotation, with CLI parity +Successful-result and oversized-error delivery constraints must identify their +successfully retained Evidence records; +export must recover every source byte, complete annotation and original error +diagnostic, with CLI parity and an unchanged executable. The focused formatter check retains only the separate case where recording was not acknowledged. +Long ordinary procedure names and encoded address-space selectors must produce +normal validation errors without exhausting Java's regex stack or losing the +private bridge connection. Each rejection is followed by a real provider lookup; +address-like literal names still resolve exactly after annotation. Malformed annotation readback, memory completeness, and inventory data remain separate SDK/provider integration cases; success from a real provider cannot establish rejection of a contradictory provider response. diff --git a/scripts/lib/real-ghidra-large-results.mjs b/scripts/lib/real-ghidra-large-results.mjs index 6185904a906..76738567e13 100644 --- a/scripts/lib/real-ghidra-large-results.mjs +++ b/scripts/lib/real-ghidra-large-results.mjs @@ -64,6 +64,24 @@ export async function verifyGhidraLargeResults({ assert.equal(expected[0], 0x5a); assert.equal(expected.at(-1), 0xa5); const before = await call("binary_session"); + // This request fits the SDK receive limit, but repeating the producer's + // diagnostic in text and structured content cannot fit the response. + const missingSelector = + "REA_MISSING_" + + "x".repeat(Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 2) + 64 * 1024); + const lookupError = await reject("procedure_address", { + procedure: missingSelector, + }); + const lookupId = retainedReference(lookupError, "procedure_address"); + assert.equal( + lookupError.details.reported_limits.original_error_code, + "invalid_request", + ); + assert.equal( + await call("procedure_address", { procedure: entry.address }), + entry.address, + ); + const readError = await reject("read_bytes", { address: payload.address, length, @@ -116,6 +134,28 @@ export async function verifyGhidraLargeResults({ assert.equal(record.subject.digest.sha256, sha256); return record; }; + const originalError = bundle.records.find( + ({ evidence_id }) => evidence_id === lookupId, + ); + assert.ok(originalError); + assert.equal(originalError.operation, "mcp_tool_error"); + assert.equal(originalError.provider.id, "rea-mcp"); + assert.equal(originalError.subject, null); + assert.equal(originalError.parameters.tool_name, "procedure_address"); + assert.equal(originalError.normalized_result.error.code, "invalid_request"); + assert.equal(originalError.raw_result.isError, true); + assert.deepEqual( + originalError.raw_result.structuredContent, + originalError.normalized_result, + ); + assert.equal( + originalError.normalized_result.error.details.operation, + "procedure_address", + ); + assert.equal( + originalError.normalized_result.error.details.issues[0].message, + `Unknown Ghidra procedure name or address: ${missingSelector}`, + ); const readRecord = retained(readId); const read = readRecord.normalized_result; assert.equal(read.requested_bytes, length); diff --git a/src/server/EvidenceMcpServer.ts b/src/server/EvidenceMcpServer.ts new file mode 100644 index 00000000000..9e9fbae4937 --- /dev/null +++ b/src/server/EvidenceMcpServer.ts @@ -0,0 +1,206 @@ +import { + McpServer, + isCallToolResult, + isJSONRPCRequest, + isJSONRPCNotification, + isJSONRPCResultResponse, + type Implementation, + type JSONRPCMessage, + type McpServerOptions, + type RequestId, + type Transport, + type TransportSendOptions, +} from "@modelcontextprotocol/server"; + +import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { parseMcpResponseBudget } from "../config/mcpResponseBudget.js"; +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { createEvidence } from "../domain/evidence.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; +import { toErrorToolResult } from "./toolResult.js"; +import { + encodeToolResult, + MCP_RESULT_BUDGET_BYTES, +} from "./toolResultEncoding.js"; + +/** Bind oversized MCP error recovery to this server's Evidence ledger. */ +export class EvidenceMcpServer extends McpServer { + constructor( + info: Implementation, + options: McpServerOptions, + private readonly recordEvidence: + | EvidenceWriter["recordEvidence"] + | undefined, + ) { + super(info, options); + } + + /** Preserve the SDK transport lifecycle while retaining oversized errors. */ + override connect(transport: Transport): Promise { + return super.connect( + new ErrorEvidenceTransport(transport, this.recordEvidence), + ); + } +} + +class ErrorEvidenceTransport implements Transport { + onclose: Transport["onclose"]; + onerror: Transport["onerror"]; + onmessage: Transport["onmessage"]; + readonly hasPerRequestStream?: boolean; + private readonly pendingTools = new Map(); + + constructor( + private readonly transport: Transport, + private readonly recordEvidence: + | EvidenceWriter["recordEvidence"] + | undefined, + ) { + if (transport.hasPerRequestStream !== undefined) + this.hasPerRequestStream = transport.hasPerRequestStream; + transport.onmessage = (message, extra) => { + if ( + isJSONRPCRequest(message) && + message.method === "tools/call" && + typeof message.params?.["name"] === "string" + ) + this.pendingTools.set(message.id, message.params["name"]); + // The SDK suppresses replies after cancellation, so send() cannot release + // this bookkeeping. Keep transport authentication outside retained data. + if ( + isJSONRPCNotification(message) && + message.method === "notifications/cancelled" + ) { + const id = message.params?.["requestId"]; + if (typeof id === "string" || typeof id === "number") + this.pendingTools.delete(id); + } + this.onmessage?.(message, extra); + }; + transport.onerror = (error) => this.onerror?.(error); + transport.onclose = () => { + this.pendingTools.clear(); + this.onclose?.(); + }; + } + + get sessionId(): string | undefined { + return this.transport.sessionId; + } + set sessionId(value: string | undefined) { + this.transport.sessionId = value; + } + + start(): Promise { + return this.transport.start(); + } + + async close(): Promise { + try { + await this.transport.close(); + } finally { + this.pendingTools.clear(); + } + } + + setProtocolVersion(version: string): void { + this.transport.setProtocolVersion?.(version); + } + + setSupportedProtocolVersions(versions: string[]): void { + this.transport.setSupportedProtocolVersions?.(versions); + } + + async send( + message: JSONRPCMessage, + options?: TransportSendOptions, + ): Promise { + const id = + "id" in message && + (typeof message.id === "string" || typeof message.id === "number") + ? message.id + : undefined; + const tool = id === undefined ? undefined : this.pendingTools.get(id); + try { + await this.transport.send(this.recoverError(message, tool), options); + } finally { + if (!isJSONRPCRequest(message) && id !== undefined) + this.pendingTools.delete(id); + } + } + + private recoverError( + message: JSONRPCMessage, + tool: string | undefined, + ): JSONRPCMessage { + if ( + !isJSONRPCResultResponse(message) || + !isCallToolResult(message.result) || + message.result.isError !== true || + message.result.structuredContent === undefined + ) + return message; + const structured = jsonObjectSchema.parse(message.result.structuredContent); + const configured = parseMcpResponseBudget( + process.env.REA_MCP_MAX_RESPONSE_BYTES, + ); + const budget = + configured.ok && configured.value !== undefined + ? configured.value - 1024 + : MCP_RESULT_BUDGET_BYTES; + const encoded = encodeToolResult(structured, budget); + if (encoded.ok) return message; + const operation = tool ?? "mcp_tool_error"; + const originalError = analysisErrorProjectionSchema.safeParse( + structured["error"], + ); + // Retain the already projected diagnostic. Transport authentication and raw + // request arguments are deliberately excluded: the application owns their + // explicit sensitivity policy, and its error projection is authoritative. + const evidence = createEvidence( + undefined, + { id: "rea-mcp", name: "REA MCP delivery", version: "1" }, + { + operation: "mcp_tool_error", + parameters: { tool_name: operation }, + result: structured, + rawResult: jsonObjectSchema.parse(message.result), + }, + ); + const recorded = this.recordEvidence?.(evidence); + const retained = recorded !== undefined && recorded.ok; + const failure = new AnalysisResourceConstraintError( + operation, + "transport", + "The tool failed, but its complete error response exceeds the MCP response budget.", + { + boundary: "mcp-response", + ...(originalError.success + ? { original_error_code: originalError.data.code } + : {}), + result_budget_bytes: budget, + response_bytes_at_least: encoded.bytesAtLeast, + constraint: encoded.constraint, + ...(retained + ? { + evidence_reference: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + } + : { retention: recorded === undefined ? "unavailable" : "failed" }), + ...(recorded !== undefined && !recorded.ok + ? { retention_error: projectAnalysisError(recorded.error) } + : {}), + }, + { + remediationAction: retained + ? "Export the retained original error through export_evidence_bundle to a caller-selected path. The connection remains usable." + : "Configure a larger REA_MCP_MAX_RESPONSE_BYTES and matching client receive buffer, then retry. The original error could not be retained by this server.", + }, + ); + return { ...message, result: toErrorToolResult(failure) }; + } +} diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 73d724ffdbc..c105d7a971a 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -7,7 +7,8 @@ import type { RecordedCrashService } from "../application/binaryDiagnostics/Reco import { registerBinaryDiagnosticsTools } from "./registerBinaryDiagnosticsTools.js"; import { createBinaryLayoutService } from "../composition/binaryDiagnostics.js"; import type { BinaryLayoutService } from "../application/binaryDiagnostics/BinaryLayoutService.js"; -import { McpServer } from "@modelcontextprotocol/server"; +import type { McpServer } from "@modelcontextprotocol/server"; +import { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import { isAbsolute } from "node:path"; import type { AnalysisOperationPort } from "../application/AnalysisProvider.js"; @@ -145,7 +146,7 @@ const installSessionToolAvailability = ( }; const createMcpServer = (session: BinarySessionPort | undefined): McpServer => - new McpServer( + new EvidenceMcpServer( { name: PRODUCT_IDENTITY.mcpServerKey, version: PRODUCT_IDENTITY.packageVersion, @@ -157,6 +158,9 @@ const createMcpServer = (session: BinarySessionPort | undefined): McpServer => ? ACTIVE_TARGET_INSTRUCTIONS : TARGET_FREE_INSTRUCTIONS, }, + session === undefined + ? undefined + : (evidence) => session.recordEvidence(evidence), ); /** diff --git a/src/server/toolResult.ts b/src/server/toolResult.ts index 9313f6dfa50..cb641abcabb 100644 --- a/src/server/toolResult.ts +++ b/src/server/toolResult.ts @@ -27,7 +27,7 @@ export const toCallToolResult = ( ): CallToolResult => result.ok ? successResult(result.value, contract, context) - : errorResult(result.error); + : toErrorToolResult(result.error); /** Deliver Evidence using the producer's explicit recording acknowledgment. */ export const toEvidenceToolResult = ( @@ -36,7 +36,7 @@ export const toEvidenceToolResult = ( recorded: Result | undefined, ): CallToolResult => recorded !== undefined && !recorded.ok - ? errorResult(recorded.error) + ? toErrorToolResult(recorded.error) : toCallToolResult( { ok: true, value: evidence }, contract, @@ -45,16 +45,23 @@ export const toEvidenceToolResult = ( : { retainedEvidenceId: evidence.evidence_id }, ); -const errorResult = (error: AnalysisError): CallToolResult => { +/** Project an error without allocating oversized repeated MCP text. */ +export const toErrorToolResult = (error: AnalysisError): CallToolResult => { const projected = projectAnalysisError(error); const structuredContent = { error: projected }; + const configured = parseMcpResponseBudget( + process.env.REA_MCP_MAX_RESPONSE_BYTES, + ); + const budget = + configured.ok && configured.value !== undefined + ? configured.value - 1024 + : MCP_RESULT_BUDGET_BYTES; + const encoded = encodeToolResult(structuredContent, budget); + // The session-bound transport retains an oversized projected error before + // replacing it with a recoverable delivery constraint. Avoid allocating its + // repeated text representation before that boundary has acknowledged it. return { - content: [ - { - type: "text", - text: JSON.stringify(structuredContent), - }, - ], + content: encoded.ok ? [{ type: "text", text: encoded.text }] : [], structuredContent, isError: true, }; @@ -71,14 +78,14 @@ const successResult = ( const configured = parseMcpResponseBudget( process.env.REA_MCP_MAX_RESPONSE_BYTES, ); - if (!configured.ok) return errorResult(configured.error); + if (!configured.ok) return toErrorToolResult(configured.error); const budget = configured.value === undefined ? MCP_RESULT_BUDGET_BYTES : configured.value - 1024; const encoded = encodeToolResult(candidate, budget); if (!encoded.ok) - return errorResult( + return toErrorToolResult( new AnalysisResourceConstraintError( contract.name, "transport", diff --git a/tests/boundary/mcp/oversizedErrorDeliveryMcp.test.ts b/tests/boundary/mcp/oversizedErrorDeliveryMcp.test.ts new file mode 100644 index 00000000000..c5d6a206c6e --- /dev/null +++ b/tests/boundary/mcp/oversizedErrorDeliveryMcp.test.ts @@ -0,0 +1,115 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { + STDIO_DEFAULT_MAX_BUFFER_SIZE, + type CallToolResult, +} from "@modelcontextprotocol/server"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { AnalysisInputError } from "../../../src/domain/analysisErrorCore.js"; +import type { EvidenceWriter } from "../../../src/application/investigation/InvestigationRecordPort.js"; +import { EvidenceIntegrityError } from "../../../src/domain/evidenceErrors.js"; +import { err } from "../../../src/domain/result.js"; +import { EvidenceMcpServer } from "../../../src/server/EvidenceMcpServer.js"; +import { toErrorToolResult } from "../../../src/server/toolResult.js"; + +const oversizedFailure = () => + toErrorToolResult( + new AnalysisInputError("procedure_address", undefined, [ + { + path: ["procedure"], + reason: "invalid_value", + message: `Unknown Ghidra procedure name or address: REA_MISSING_${"x".repeat(Math.ceil(STDIO_DEFAULT_MAX_BUFFER_SIZE / 2) + 64 * 1024)}`, + }, + ]), + ); + +const exerciseDeliveryFailure = async ( + recordEvidence: EvidenceWriter["recordEvidence"] | undefined, +) => { + const server = new EvidenceMcpServer( + { name: "oversized-error-test", version: "1" }, + { capabilities: {} }, + recordEvidence, + ); + server.registerTool("large_failure", { inputSchema: {} }, async () => + oversizedFailure(), + ); + server.registerTool( + "healthy", + { inputSchema: {} }, + async (): Promise => ({ + content: [{ type: "text", text: "still connected" }], + }), + ); + const client = new Client({ name: "oversized-error-client", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + + const failure = await client.callTool({ + name: "large_failure", + arguments: {}, + }); + expect(failure.isError).toBe(true); + expect(failure.structuredContent).toMatchObject({ + error: { + code: "resource_constraint", + details: { + reported_limits: { + original_error_code: "invalid_request", + }, + }, + }, + }); + const reportedLimits = z + .object({ + error: z.object({ + code: z.literal("resource_constraint"), + details: z.object({ + reported_limits: z.record(z.string(), z.unknown()), + }), + }), + }) + .parse(failure.structuredContent).error.details.reported_limits; + expect(failure.structuredContent).toMatchObject({ + error: { + details: { + reported_limits: { + boundary: "mcp-response", + }, + }, + }, + }); + expect(reportedLimits).not.toHaveProperty("evidence_reference"); + expect(reportedLimits.retention).toBe( + recordEvidence === undefined ? "unavailable" : "failed", + ); + if (recordEvidence !== undefined) + expect(reportedLimits.retention_error).toMatchObject({ + code: "evidence_integrity_mismatch", + }); + expect(Buffer.byteLength(JSON.stringify(failure))).toBeLessThan( + STDIO_DEFAULT_MAX_BUFFER_SIZE, + ); + + const healthy = await client.callTool({ name: "healthy", arguments: {} }); + expect(healthy.content).toContainEqual({ + type: "text", + text: "still connected", + }); + } finally { + await Promise.all([client.close(), server.close()]); + } +}; + +it("keeps the SDK connection usable when oversized-error retention is unavailable or fails", async () => { + await exerciseDeliveryFailure(undefined); + await exerciseDeliveryFailure(() => + err( + new EvidenceIntegrityError("fixture ledger rejected the Evidence record"), + ), + ); +}); From 7a7460f04f3cfd7e365d70ff891bdd0c474901b6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:43:28 +0800 Subject: [PATCH 059/791] docs: add Persian README translation with full language navigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds README_fa.md (Persian/Farsi) contributed via PR #1140, keeps onboarding commands, links, and structure aligned with English, links فارسی from every translated selector, and registers fa in the readme consistency matrix. Co-authored-by: Arad Ashkan <65897590+AradAshkan@users.noreply.github.com> --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 2 +- README_es.md | 2 +- README_fa.md | 266 ++++++++++++++++++++ README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 2 +- README_pt-BR.md | 2 +- README_ru.md | 2 +- README_th.md | 2 +- README_tr.md | 2 +- README_uk.md | 2 +- README_vi.md | 2 +- README_zh-TW.md | 2 +- README_zh.md | 2 +- tests/conformance/readmeConsistency.test.ts | 1 + 17 files changed, 282 insertions(+), 15 deletions(-) create mode 100644 README_fa.md diff --git a/README.md b/README.md index 51d21e4d613..d5b22d365a7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@
-**English** · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +**English** · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Reverse Engineer Anything diff --git a/README_ar.md b/README_ar.md index 8e977c3260a..ed947e0404f 100644 --- a/README_ar.md +++ b/README_ar.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · **العربية** +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · **العربية** · [فارسی](README_fa.md) # REA: هندسة عكسية لأي شيء diff --git a/README_de.md b/README_de.md index a6342dc0f6b..60253aa6494 100644 --- a/README_de.md +++ b/README_de.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · **Deutsch** · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · **Deutsch** · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Alles reverse-engineeren diff --git a/README_es.md b/README_es.md index 4d0a06d4f1c..4173b0cfa84 100644 --- a/README_es.md +++ b/README_es.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · **Español** · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · **Español** · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Ingeniería inversa de cualquier cosa diff --git a/README_fa.md b/README_fa.md new file mode 100644 index 00000000000..6dd8c0008dd --- /dev/null +++ b/README_fa.md @@ -0,0 +1,266 @@ +
+ +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · **فارسی** + +# REA: مهندسی معکوس هر چیزی + +### یک MCP برای مهندسی معکوس فایل‌های باینری، برنامه‌ها و رفتار زمان اجرا. + +**قابلیتی دیده‌اید که دوستش دارید؟ سازوکارش را تا سطح باینری درک کنید.** + +[![npm version](https://img.shields.io/npm/v/rea-agents?style=flat-square&color=cb3837)](https://www.npmjs.com/package/rea-agents) +[![CI](https://img.shields.io/github/actions/workflow/status/morluto/rea/ci.yml?branch=main&style=flat-square&label=CI)](https://github.com/morluto/rea/actions/workflows/ci.yml) +[![MCP tool catalog](https://img.shields.io/badge/MCP-tool_catalog-5c4ee5?style=flat-square)](docs/mcp-contracts.md#generated-catalog) +[![Node.js 22+](https://img.shields.io/badge/Node.js-22.19%2B-339933?style=flat-square&logo=nodedotjs&logoColor=white)](https://nodejs.org/) +[![skills.sh](https://skills.sh/b/morluto/rea?style=flat-square)](https://skills.sh/morluto/rea/reverse-engineer-anything) +[![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) +[![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) + +morluto%2Frea | Trendshift + +**[وب‌سایت](https://rea.tools/) · [راهنماها](https://rea.tools/guides/) · [نمونه‌های عملی](https://rea.tools/showcase/)** + +[شروع سریع](#شروع-سریع) · [REA چگونه کار می‌کند؟](#rea-چگونه-کار-میکند) · [چه چیزهایی را میتوان تحلیل کرد؟](#چه-چیزهایی-را-میتوان-تحلیل-کرد) · [نمونه‌های عملی](#نمونههای-عملی) · [پرسش‌های متداول](#پرسشهای-متداول) · [مستندات](#مستندات) + +npx rea-agents setup + +
+ +اجرای پل تحلیل REA در Hopper هنگام بررسی یک فایل باینری بومی + +
+ +
Discord
به انجمن مهندسی معکوس بپیوندید

دیسکورد · پرسش و پاسخ · نمایش دستاوردها
+ +
+ +
+ +--- + +
+ +قابلیتی در یک برنامه دیده‌اید که می‌خواهید در محصول خودتان داشته باشید؟ از ایجنت خود بخواهید با REA آن را بررسی کند. ایجنت می‌تواند بدون دسترسی به کد منبع برنامه، نحوهٔ کار قابلیت را بررسی و توضیح دهد، شواهد را نشان دهد و نسخه‌ای مشابه برای پروژهٔ شما بسازد. + +REA ایجنت شما را به ابزارهایی برای بررسی باینری‌های بومی، برنامه‌های JavaScript و Electron، اسمبلی‌های .NET و وب‌سایت‌ها متصل می‌کند. همین ابزارها از ترمینال نیز قابل استفاده‌اند. تحلیل به‌صورت محلی انجام می‌شود و نتایج شامل شواهد و محدودیت‌های پشت هر نتیجه‌گیری هستند. + +فرایند راه‌اندازی، REA را در ایجنت شما ثبت می‌کند و دستورالعمل‌های گردش‌کار متناظر را نصب می‌کند. تحلیل بومی می‌تواند از نصب موجود Hopper یا Ghidra استفاده کند؛ همچنین راه‌اندازی می‌تواند با تأیید شما Hopper را به‌صورت اختیاری نصب کند. تحلیل ایستای JavaScript به هیچ‌یک از این موتورهای تحلیل نیاز ندارد. + +> برای دستورالعمل‌های راه‌اندازی، راهنماهای تصویری و مطالعات موردی واقعی، **[به وب‌سایت REA مراجعه کنید](https://rea.tools/)**. + +## شروع سریع + +### ایجنت خود را راه‌اندازی کنید + +پس از نصب Node.js و npm، دستور زیر را اجرا کنید: + +```bash +npx rea-agents setup +``` + +ایجنت‌های موردنظر را انتخاب کنید، تغییرات پیشنهادی را بررسی و تأیید کنید. فرایند راه‌اندازی، سرور MCP مربوط به REA و دستورالعمل‌های گردش‌کار متناظر را اضافه می‌کند و از پیکربندی موجود نسخهٔ پشتیبان می‌گیرد. پس از آن ایجنت خود را مجدداً راه‌اندازی کنید. + +راه‌اندازی از Claude Code، Codex، Cursor، Gemini CLI، Grok Build و [ایجنت‌های دیگر](docs/installation.md#supported-agents) پشتیبانی می‌کند. برای پیکربندی ارائه‌دهنده و ثبت دستی MCP، [نصب و راه‌اندازی](docs/installation.md) را ببینید. + +### از ایجنت خود بپرسید + +```text +بررسی کن جست‌وجو در برنامهٔ Notes چگونه کار می‌کند، شواهدش را نشان بده و قابلیتی مشابه برای پروژهٔ من بساز. +``` + +به‌جای Notes، نام برنامهٔ هدف و قابلیتی را قرار دهید که می‌خواهید درک کنید. + +### استفاده از ترمینال + +برای بررسی پوشهٔ استخراج‌شدهٔ یک برنامهٔ JavaScript/Electron یا فایل ASAR: + +```bash +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +``` + +نتیجه شامل ماژول‌ها، importها، مرزهای Electron و شواهد مربوط به آن‌هاست. مسیر را با مسیر برنامهٔ هدف جایگزین کنید؛ مثلاً در ویندوز `"D:/apps/example"`. + +برای نصب دستور `rea` جهت استفادهٔ مداوم: + +```bash +npm install --global rea-agents +rea --help +``` + +برای تحلیل بومی ابتدا یک ارائه‌دهنده پیکربندی کنید. برای دستورات بومی، انتخاب ارائه‌دهنده، snapshotها و اسکریپت‌نویسی، [راهنمای CLI و شواهد](docs/cli.md) را ببینید. + +### به‌روزرسانی REA + +REA به‌سرعت تغییر می‌کند و نسخه‌های جدید معمولاً شامل رفع اشکال‌های متعددی هستند. نصب خود را به‌روز نگه دارید. + +برای CLI نصب‌شده از طریق npm: + +```bash +rea update +``` + +برای به‌روزرسانی ثبت ایجنت‌ها و skill، دستور setup نمایش‌داده‌شده توسط به‌روزرسانی را اجرا کنید. + +اگر از `npx` استفاده می‌کنید، راه‌اندازی ایجنت را با دستور زیر به‌روز کنید: + +```bash +npx rea-agents@latest setup +``` + +تغییرات راه‌اندازی را بررسی کنید و ایجنت را دوباره اجرا کنید. برای اجرای یک‌بارهٔ دستورات CLI، از `npx rea-agents@latest` و سپس دستور موردنظر استفاده کنید. + +## REA چگونه کار می‌کند؟ + +ایجنت شما از طریق MCP، REA را فراخوانی می‌کند تا هدف را بررسی و کد مرتبط را ردیابی کند. REA یافته‌ها را همراه با شواهد برمی‌گرداند. ایجنت از این نتایج برای طرح پرسش‌های تکمیلی، توضیح رفتار یا نوشتن و آزمایش یک پیاده‌سازی استفاده می‌کند. دستورات CLI نیز از همین گردش‌کارها بهره می‌برند. + +![جریان بررسی REA: ایجنت دربارهٔ هدف محلی سؤال می‌پرسد، REA آن را با ابزارهای تحلیل بررسی و ردیابی می‌کند و ایجنت از کد، ارجاعات و ناشناخته‌های بازگردانده‌شده برای توضیح، پیاده‌سازی و آزمایش استفاده می‌کند.](website/public/assets/figures/rea-investigation-flow.svg) + +[نمایش تصویر در اندازهٔ کامل](website/public/assets/figures/rea-investigation-flow.svg). + + + +## چه چیزهایی را می‌توان تحلیل کرد؟ + +REA به Node.js 22.x (>=22.19)، 24.x (>=24.11) یا 26+، به‌همراه npm نیاز دارد. ابزارهای اضافی و پشتیبانی سیستم میزبان به نوع هدف بستگی دارند: + +| هدف | خروجی REA | پیش‌نیازها و راهنما | +| --------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| باینری‌های بومی | شبه‌کد، اسمبلی، رشته‌ها، نمادها، فراخوانی‌ها و ارجاعات | Hopper، Ghidra یا IDA؛ [تحلیل بومی](https://rea.tools/guides/native/) | +| ساختار ELF آفلاین | بخش‌ها، سگمنت‌ها، نمادها و relocationهای اصلی و گزینه‌های احتمالی کاهش آسیب‌پذیری ایستا | pwntools تأمین‌شده توسط فراخواننده روی Linux x64؛ [تشخیص باینری](docs/binary-diagnostics.md) | +| بایت‌کد EVM | selectorهای dispatch، offsetهای بایتی، آرگومان‌های استنباط‌شده و قابلیت تغییر وضعیت | دادهٔ خام/hex محلی؛ [راهنمای EVM آفلاین](docs/evm-bytecode.md) | +| کرش‌های ثبت‌شدهٔ لینوکس | یادداشت‌های خام، رجیسترها/سیگنال‌های تمام threadهای ثبت‌شده و گزینه‌های احتمالی نگاشت | pwntools تأمین‌شده توسط فراخواننده؛ GDB/pwndbg اختیاری؛ [کرش‌های ثبت‌شده](docs/recorded-crashes.md) | +| JavaScript / Electron | ماژول‌ها، importها، source mapها، مسیرها، IPC و ارتباط افزونه‌های بومی | Node.js و npm؛ [تحلیل برنامه](https://rea.tools/guides/javascript/) | +| وب‌سایت‌ها | ساختار صفحه، اسکریپت‌ها، مشاهدات شبکه و اسکرین‌شات‌های درخواستی | مرورگر مبتنی بر Chrome؛ [تحلیل مرورگر](https://rea.tools/guides/browser/) | +| داده‌های ضبط‌شدهٔ شبکه | درخواست‌ها، پاسخ‌ها، payloadهای آشکارشده و مکان آن‌ها در منبع | HAR؛ ابزار mitmdump در لینوکس برای ضبط‌های بومی mitmproxy؛ [راهنمای ضبط شبکه](docs/web-network-captures.md) | +| اسمبلی‌های .NET | فراداده، دستورالعمل‌های CIL، وابستگی‌های بومی اعلام‌شده و مقایسهٔ buildها | بررسی ایستا؛ [راهنمای کد مدیریت‌شده](docs/managed-code-analysis.md) | +| فایل‌های APK اندروید | تعاریف Manifest، کلاس‌ها، متدهای دیکامپایل‌شده و ارجاعات | JADX بدون رابط گرافیکی و JDK کامل در Linux/macOS؛ [راهنمای اندروید](docs/android-analysis.md) | +| Firmware | ناحیه‌ها، نتایج استخراج و تحویل به ابزارهای تحلیل بومی | Binwalk / Unblob در لینوکس؛ [راهنمای Firmware](docs/firmware-analysis.md) | +| بسته‌ها و منابع | فهرست فایل‌ها، digestها، فایل‌های plist، ساختار bundle اپل و منابع استخراج‌شده | [راهنمای آرتیفکت و JavaScript](docs/javascript-artifact-reconstruction.md)، [برنامه‌های اپل](docs/apple-application-analysis.md) | +| رفتار فرایند | خروجی ترمینال، تعاملات، وضعیت خروج، مشاهدات سیستم فایل و مقایسهٔ اجراها | Linux/macOS با PTY بومی؛ [ضبط فرایند](docs/process-capture.md) | + +بررسی ایستای JavaScript و .NET فایل‌های ارائه‌شده را بدون اجرای برنامه می‌خواند. ضبط زمان اجرا، هدف انتخاب‌شده را با مجوزهای کاربری شما اجرا می‌کند یا با آن تعامل دارد؛ آثار این عملیات در راهنمای مربوط به هر نوع ضبط توضیح داده شده‌اند. + + + +فرمت‌های بومی و پشتیبانی سیستم میزبان بر اساس ارائه‌دهنده متفاوت‌اند. [راه‌اندازی Hopper و Ghidra](docs/installation.md#hopper)، [راهنمای IDA](docs/ida-provider.md) و [پشتیبانی آزمایشی Ghidra در ویندوز](docs/windows-ghidra-p0.md) را ببینید. Ghidra همچنین از [تحلیل DOS شانزده‌بیتی](docs/ghidra-dos.md) پشتیبانی می‌کند. برای انتخاب ارائه‌دهنده، به [راهنمای CLI](docs/cli.md#choose-a-provider) مراجعه کنید. برای قابلیت‌هایی که پس از آخرین انتشار npm اضافه شده‌اند، [وضعیت انتشار](docs/installation.md#released-package-and-main) را بررسی کنید. + +## نمونه‌های عملی + +### DX-Ball: بازسازی محاسبهٔ موقعیت استریوی صدا + +یک فراخوانی صوتی را تا تابع کمکی تبدیل موقعیت به pan دنبال کنید، دستورالعمل‌ها را بررسی کنید و شبه‌کد ناقص را به C تبدیل کنید. بازسازی حاصل، هر ۳٬۲۰۵ مورد آزمایش x86 اصلی را با موفقیت پشت سر می‌گذارد و هر ۶۳ بایت تابع کامپایل‌شده را دقیقاً بازتولید می‌کند. + +[مطالعهٔ موردی](https://rea.tools/showcase/dx-ball/) · [مخزن بازسازی](https://github.com/N0zoM1z0/dx-ball) + +### Notion: ردیابی پل کلیپ‌بورد Electron + +API کلیپ‌بورد در renderer را پیدا کنید، مسیر آن را از preload و IPC تا فرایند اصلی دنبال کنید و قالب غنی کلیپ‌بورد را بررسی کنید. + +[مطالعهٔ موردی](https://rea.tools/showcase/notion/) + +### TH04: بازیابی محاسبهٔ حلقهٔ گلوله در DOS + +دستورالعمل‌های ۱۶بیتی بازی اصلی PC-98 را بررسی کنید، محاسبات زاویهٔ ثابت و هدف‌گیری‌شده را بازیابی کنید و کد C++ بازسازی‌شده را با خروجی کامپایلر تاریخی مقایسه کنید. + +[مطالعهٔ موردی](https://rea.tools/showcase/th04/) · [مخزن بازسازی](https://github.com/N0zoM1z0/th04) + +اگر از REA برای بررسی موضوع جالبی استفاده کرده‌اید، خوشحال می‌شویم آن را به اشتراک بگذارید. یک [issue](https://github.com/morluto/rea/issues) یا [pull request](https://github.com/morluto/rea/pulls) ثبت کنید و هدف، پرسش خود، نقش REA و یافته‌هایتان را توضیح دهید. + +## پرسش‌های متداول + +
کدام ایجنت‌ها می‌توانند از REA استفاده کنند؟ + +هر ایجنتی که از سرورهای MCP محلی پشتیبانی کند. فرایند setup، [ایجنت‌های پشتیبانی‌شده](docs/installation.md#supported-agents) را پیکربندی می‌کند؛ سایر کلاینت‌ها می‌توانند از [ثبت دستی MCP](docs/installation.md#mcp-registry) استفاده کنند. + +
+ +
آیا به Hopper، Ghidra یا IDA نیاز دارم؟ + +تحلیل عمیق بومی به یکی از آن‌ها نیاز دارد. بررسی ایستای JavaScript و .NET بدون موتور تحلیل بومی کار می‌کند. setup می‌تواند با تأیید شما Hopper را نصب کند؛ Ghidra و IDA از نصب‌های موجود شما استفاده می‌کنند. [راه‌اندازی ارائه‌دهنده](docs/installation.md#hopper) را ببینید. + +
+ +
آیا باید ابتدا Hopper را اجرا کنم؟ + +REA زمانی که عملیاتی به Hopper نیاز داشته باشد آن را اجرا می‌کند. در macOS، ممکن است هنگام نخستین اجرا پنجره‌ای برای انتخاب حالت آزمایشی یا فعال‌سازی مجوز نمایش داده شود. [راه‌اندازی Hopper و رفع اشکال](docs/installation.md#launcher-paths-and-troubleshooting) را ببینید. + +
+ +
نصب skill از skills.sh چه کاری انجام می‌دهد؟ + +این skill دستورالعمل‌های بررسی را برای ایجنت فراهم می‌کند. برای ثبت سرور MCP مربوط به REA و نصب دستورالعمل‌های متناظر، `rea setup` را اجرا کنید و سپس ایجنت را مجدداً راه‌اندازی کنید. [نصب مستقل skill](docs/installation.md#skill-only-installation) را ببینید. + +
+ +
REA چه نوع کدی برمی‌گرداند؟ + +تحلیل بومی، شبه‌کد و اسمبلی برمی‌گرداند. تحلیل JavaScript/Electron ماژول‌ها و روابط میان آن‌ها را بازیابی می‌کند. ایجنت شما از این یافته‌ها برای نوشتن و آزمایش یک پیاده‌سازی استفاده می‌کند؛ [نمونه‌های عملی](#نمونههای-عملی) مثال‌های گام‌به‌گام ارائه می‌دهند. + +
+ +
آیا REA برنامهٔ من را آپلود می‌کند؟ + +REA اهداف را به‌صورت محلی تحلیل می‌کند. ایجنت شما نتایج ابزارها را دریافت می‌کند و ارائه‌دهندهٔ مدل آن سیاست مستقل خود را دربارهٔ داده‌ها دارد. + +
+ +
اگر با خطا مواجه شدم چه کنم؟ + +ابتدا به‌روزرسانی کنید؛ ممکن است مشکل در نسخه‌ای جدید رفع شده باشد. + +برای CLI نصب‌شده با npm: + +```bash +rea update +``` + +برای راه‌اندازی ایجنت با `npx`: + +```bash +npx rea-agents@latest setup +``` + +اگر از ایجنت استفاده می‌کنید، [به‌روزرسانی راه‌اندازی](#بهروزرسانی-rea) را تکمیل و ایجنت را مجدداً اجرا کنید. همان کار را دوباره امتحان کنید. اگر مشکل ادامه داشت، یک [issue ثبت کنید](https://github.com/morluto/rea/issues) و نسخهٔ REA، نوع هدف، مراحل بازتولید و خروجی خطا را ذکر کنید. + +
+ +## مستندات + +از [راهنماهای عملی وب‌سایت](https://rea.tools/guides/) شروع کنید. برای گزینه‌های دقیق، پیش‌نیازها و قراردادهای خروجی: + +- [نصب و راه‌اندازی](docs/installation.md): ثبت ایجنت، پیکربندی ارائه‌دهنده، به‌روزرسانی و حذف نصب. +- [آمادگی و رفع اشکال](docs/installation.md#check-readiness-for-your-task): عیب‌یابی یک ایجنت یا موتور تحلیل. +- [CLI و شواهد](docs/cli.md): دستورات، انتخاب ارائه‌دهنده، snapshotها، ورود/خروج داده و وضعیت‌های خروج. +- [قراردادهای MCP](docs/mcp-contracts.md) و [پرامپت‌های ایجنت](docs/mcp-prompts.md): نتایج ابزارها، نشست‌ها و بررسی‌های هدایت‌شده. +- [فهرست ابزارها](docs/mcp-contracts.md#generated-catalog): فهرست تولیدشده هنگام build از ابزارها، ارائه‌دهندگان و دستورات CLI. +- [نقشهٔ راه](docs/roadmap.md): برنامه‌های آینده و پیگیری قابلیت‌ها. + +آسیب‌پذیری‌ها را از طریق [SECURITY.md](SECURITY.md) گزارش کنید. + +## مشارکت + +از کمک شما به REA استقبال می‌کنیم! برای گزارش خطا یا پیشنهاد قابلیت، یک [issue باز کنید](https://github.com/morluto/rea/issues) یا برای بهبود کد و مستندات [pull request بفرستید](https://github.com/morluto/rea/pulls). + +برای راه‌اندازی محیط توسعه و بررسی‌ها، [CONTRIBUTING.md](CONTRIBUTING.md)، برای مراحل اعتبارسنجی [آزمایش‌ها](docs/testing.md) و برای ساختار پروژه [نقشهٔ معماری](docs/architecture.mermaid) را ببینید. + +## لینک‌های پروژه + +[وب‌سایت](https://rea.tools/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [امنیت](SECURITY.md) + +## تاریخچهٔ ستاره‌ها + +🎉 **۲۰٬۰۰۰ ستارهٔ GitHub — سپاسگزاریم!** + +از همهٔ کسانی که از REA استفاده می‌کنند، خطاها را گزارش می‌دهند، buildها را آزمایش می‌کنند و در بهبود پروژه مشارکت دارند سپاسگزاریم. + +تاریخچهٔ ستاره‌های GitHub پروژهٔ REA + +## سلب مسئولیت + +REA ابزارهایی برای پژوهش، تحلیل و بازسازی قانونی در حوزهٔ مهندسی معکوس فراهم می‌کند. مسئولیت دریافت مجوزهای لازم و رعایت قوانین مربوط بر عهدهٔ شماست. این پروژه استفادهٔ غیرقانونی یا بدون مجوز را تأیید نمی‌کند. + +## مجوز + +[MIT](LICENSE) + +
diff --git a/README_ja.md b/README_ja.md index 61efe98257b..73aec64d196 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · **日本語** · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · **日本語** · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA:あらゆるものをリバースエンジニアリング diff --git a/README_ko.md b/README_ko.md index bc41bcc0c0a..22adf4a3c7a 100644 --- a/README_ko.md +++ b/README_ko.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · **한국어** · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · **한국어** · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: 무엇이든 리버스 엔지니어링 diff --git a/README_pl.md b/README_pl.md index 404166822e4..3637e314d78 100644 --- a/README_pl.md +++ b/README_pl.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · **Polski** · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · **Polski** · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Reverse Engineer Anything diff --git a/README_pt-BR.md b/README_pt-BR.md index a83a1b83308..b717439c8b8 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · **Português (Brasil)** · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · **Português (Brasil)** · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Faça engenharia reversa de qualquer coisa diff --git a/README_ru.md b/README_ru.md index 07051c98fd9..979ad9a25f8 100644 --- a/README_ru.md +++ b/README_ru.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · **Русский** · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · **Русский** · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Реверс-инжиниринг чего угодно diff --git a/README_th.md b/README_th.md index e2f4d7454cc..fb864599442 100644 --- a/README_th.md +++ b/README_th.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · **ไทย** · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · **ไทย** · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Reverse Engineer Anything diff --git a/README_tr.md b/README_tr.md index 8966e6a8665..0f078a81a29 100644 --- a/README_tr.md +++ b/README_tr.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **Türkçe** · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · **Türkçe** · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Her şeyi tersine mühendislikle inceleyin diff --git a/README_uk.md b/README_uk.md index 52bdd78a048..5633f429444 100644 --- a/README_uk.md +++ b/README_uk.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Українська** · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · **Українська** · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Реверс-інжиніринг будь-чого diff --git a/README_vi.md b/README_vi.md index 0edf0b48a50..1d85b64108a 100644 --- a/README_vi.md +++ b/README_vi.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · **Tiếng Việt** · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · **Tiếng Việt** · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA: Kỹ thuật dịch ngược cho mọi thứ diff --git a/README_zh-TW.md b/README_zh-TW.md index 9b0915116d2..2849e624b88 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -1,6 +1,6 @@
-[English](README.md) · [简体中文](README_zh.md) · **繁體中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · [简体中文](README_zh.md) · **繁體中文** · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA:逆向分析一切 diff --git a/README_zh.md b/README_zh.md index 0e7624e0d9d..53872616c03 100644 --- a/README_zh.md +++ b/README_zh.md @@ -1,6 +1,6 @@
-[English](README.md) · **简体中文** · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) +[English](README.md) · **简体中文** · [繁體中文](README_zh-TW.md) · [日本語](README_ja.md) · [한국어](README_ko.md) · [Türkçe](README_tr.md) · [Русский](README_ru.md) · [Tiếng Việt](README_vi.md) · [ไทย](README_th.md) · [Deutsch](README_de.md) · [Español](README_es.md) · [Українська](README_uk.md) · [Polski](README_pl.md) · [Português (Brasil)](README_pt-BR.md) · [العربية](README_ar.md) · [فارسی](README_fa.md) # REA:逆向分析一切 diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index db61e5dd765..38ecfb3fbf3 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -21,6 +21,7 @@ const readmeLanguages = [ { path: "README_pl.md", label: "Polski" }, { path: "README_pt-BR.md", label: "Português (Brasil)" }, { path: "README_ar.md", label: "العربية" }, + { path: "README_fa.md", label: "فارسی" }, ] as const; const translatedReadmes = readmeLanguages From 70e8a8f86060496452ccbc1a49b39822efef4113 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:47:48 +0800 Subject: [PATCH 060/791] fix(contracts): preserve local inputs and comparison uncertainty --- src/contracts/managed/managedToolContracts.ts | 9 +- src/contracts/native/nativeToolContracts.ts | 6 +- src/contracts/sessionLifecycleInputs.ts | 13 +- src/contracts/sessionPathInputs.test.ts | 23 ++++ src/contracts/sessionStatusContract.ts | 9 +- src/contracts/sessionToolContracts.ts | 14 +- src/contracts/sessionToolSchemas.ts | 10 +- src/contracts/toolOutputSchemaGroups.ts | 4 +- src/contracts/toolOutputSchemaPrimitives.ts | 17 ++- src/domain/browserScenarioValues.ts | 13 +- src/domain/bundleComparison.ts | 5 +- src/domain/changedBehavior.test.ts | 35 +++-- src/domain/changedBehavior.ts | 16 ++- src/domain/comparisonSourceEvidence.ts | 57 ++++++++ src/domain/firmware/firmwareAnalysis.ts | 13 +- .../javascriptApplicationAnalysis.ts | 3 +- src/domain/javascript/javascriptRecovery.ts | 15 +-- src/domain/localPath.ts | 7 + src/domain/native/nativeCallObservation.ts | 20 +-- src/domain/process/processScenario.test.ts | 117 ---------------- src/domain/process/processScenario.ts | 23 ++-- ...uctionVerification.functionSources.test.ts | 16 +-- src/domain/reconstructionVerification.ts | 64 ++------- .../reconstructionVerificationSchemas.ts | 4 +- src/domain/staticRuntimeCorrelation.ts | 59 ++------ .../capture/ProcessCaptureError.test.ts | 33 ----- src/process/capture/ProcessCaptureError.ts | 4 +- src/server/registerArtifactComparisonTool.ts | 22 ++- src/server/registerFunctionComparisonTool.ts | 12 +- src/server/registerProcessComparisonTool.ts | 11 +- src/server/runDerivedOperation.test.ts | 22 +++ src/server/runDerivedOperation.ts | 23 +++- .../mcp/comparisonUnknownsMcp.test.ts | 126 ++++++++++++++++++ tests/boundary/mcp/nativeDemangleMcp.test.ts | 8 ++ .../process/processCaptureLifecycle.test.ts | 13 ++ 35 files changed, 460 insertions(+), 386 deletions(-) create mode 100644 src/domain/comparisonSourceEvidence.ts delete mode 100644 src/domain/process/processScenario.test.ts delete mode 100644 src/process/capture/ProcessCaptureError.test.ts create mode 100644 tests/boundary/mcp/comparisonUnknownsMcp.test.ts diff --git a/src/contracts/managed/managedToolContracts.ts b/src/contracts/managed/managedToolContracts.ts index 85b5c19506b..f256fc5dff5 100644 --- a/src/contracts/managed/managedToolContracts.ts +++ b/src/contracts/managed/managedToolContracts.ts @@ -4,13 +4,14 @@ import type { ToolContract } from "../toolContracts.js"; import { managedOutputSchemas } from "../toolOutputSchemas.js"; import { toolContractMetadata } from "../toolEffects.js"; import { requireOutputSchema } from "../toolOutputSchemaPrimitives.js"; -import { isAbsoluteLocalPath } from "../../domain/localPath.js"; +import { + isAbsoluteLocalPath, + localPathStringSchema, +} from "../../domain/localPath.js"; /** Exact caller boundary for execution-free PE/CLI triage and identity. */ const managedTargetInputSchema = z.strictObject({ - path: z - .string() - .min(1) + path: localPathStringSchema .refine(isAbsoluteLocalPath, { message: "path must be an absolute local filesystem path (for example /tmp/Example.dll or C:\\Example.dll)", diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index afeb9f652a4..89b7b8c1e7a 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -8,6 +8,7 @@ import { import type { ToolContract } from "../toolContracts.js"; import { nativeOutputSchemas } from "../toolOutputSchemas.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { localPathStringSchema } from "../../domain/localPath.js"; import { toolContractMetadata } from "../toolEffects.js"; import { requireOutputSchema } from "../toolOutputSchemaPrimitives.js"; @@ -69,7 +70,8 @@ export const swiftSymbolsSchema = z z .string() .min(1) - .regex(/^[^\n]*$/u, "Each Swift symbol must be one line."), + .regex(/^[^\n]*$/u, "Each Swift symbol must be one line.") + .regex(/^[^\0]*$/u, "Swift symbols cannot contain NUL."), ) .min(1); @@ -104,7 +106,7 @@ export const NATIVE_TOOL_CONTRACTS = [ "inspect_plist", "Parse Info.plist from the active artifact by default, or pass any local plist path. Returns normalized JSON rather than plutil text.", z.object({ - path: z.string().min(1).optional(), + path: localPathStringSchema.optional(), }), ), native( diff --git a/src/contracts/sessionLifecycleInputs.ts b/src/contracts/sessionLifecycleInputs.ts index 68b61b35fa2..a230c9a4f1b 100644 --- a/src/contracts/sessionLifecycleInputs.ts +++ b/src/contracts/sessionLifecycleInputs.ts @@ -1,12 +1,13 @@ import { executableFormatHintSchema } from "../domain/dosCom.js"; -import { isAbsoluteLocalPath } from "../domain/localPath.js"; +import { + isAbsoluteLocalPath, + localPathStringSchema, +} from "../domain/localPath.js"; import { z } from "zod"; import { analysisProviderSelectorSchema } from "./providerSelection.js"; /** Caller-supplied analysis snapshot path; must name its destination absolutely. */ -const snapshotPathSchema = z - .string() - .min(1) +const snapshotPathSchema = localPathStringSchema .refine(isAbsoluteLocalPath, { message: "snapshot_path must be an absolute local filesystem path (for example /tmp/rea/analysis.json or C:\\rea\\analysis.json)", @@ -16,9 +17,7 @@ const snapshotPathSchema = z ); /** Caller-supplied analysis target path; must name the target absolutely. */ -const binaryTargetPathSchema = z - .string() - .min(1) +const binaryTargetPathSchema = localPathStringSchema .refine(isAbsoluteLocalPath, { message: "path must be an absolute local filesystem path (for example /tmp/fixture.bin or C:\\analysis\\fixture.bin)", diff --git a/src/contracts/sessionPathInputs.test.ts b/src/contracts/sessionPathInputs.test.ts index 6f415582447..f8030aa5d60 100644 --- a/src/contracts/sessionPathInputs.test.ts +++ b/src/contracts/sessionPathInputs.test.ts @@ -147,6 +147,29 @@ it.each(cases)("$name requires an absolute local path", (pathCase) => { expect(pathCase.omitted(), pathCase.name).toBe(true); }); +it("preserves whitespace in selected browser and Electron filesystem names", () => { + const executable = "/opt/browser with trailing space "; + expect( + browserScenarioBrowserSchema.parse({ + mode: "launch", + executable_path: executable, + }), + ).toMatchObject({ executable_path: executable }); + const application = "/tmp/application "; + const root = "/tmp/application root "; + expect( + electronActiveObservationInputSchema.parse({ + executable_path: executable, + application_path: application, + application_root: root, + }), + ).toMatchObject({ + executable_path: executable, + application_path: application, + application_root: root, + }); +}); + it("keeps evidence bundle export overwrite disabled by default", () => { expect( exportEvidenceBundleInputSchema.safeParse({ path: "/tmp/bundle.json" }), diff --git a/src/contracts/sessionStatusContract.ts b/src/contracts/sessionStatusContract.ts index 303034c9c19..7b509b39a9c 100644 --- a/src/contracts/sessionStatusContract.ts +++ b/src/contracts/sessionStatusContract.ts @@ -1,6 +1,9 @@ import { z } from "zod"; -import { isAbsoluteLocalPath } from "../domain/localPath.js"; +import { + isAbsoluteLocalPath, + localPathStringSchema, +} from "../domain/localPath.js"; /** Optional expectations used to compare the live session with its caller. */ export const binarySessionInputSchema = z.strictObject({ @@ -9,9 +12,7 @@ export const binarySessionInputSchema = z.strictObject({ .string() .regex(/^[a-f0-9]{64}$/u) .optional(), - expected_server_path: z - .string() - .min(1) + expected_server_path: localPathStringSchema .refine(isAbsoluteLocalPath, { message: "expected_server_path must be an absolute local filesystem path (for example /opt/rea/dist/main.js)", diff --git a/src/contracts/sessionToolContracts.ts b/src/contracts/sessionToolContracts.ts index 5e9de09cfdd..484e61bd29e 100644 --- a/src/contracts/sessionToolContracts.ts +++ b/src/contracts/sessionToolContracts.ts @@ -1,6 +1,9 @@ import { z } from "zod"; -import { isAbsoluteLocalPath } from "../domain/localPath.js"; +import { + isAbsoluteLocalPath, + localPathStringSchema, +} from "../domain/localPath.js"; import { sessionOutputSchemas, requireOutputSchema, @@ -31,7 +34,10 @@ import { examplesFor } from "./toolContractHelpers.js"; import type { ToolContract } from "./toolContractTypes.js"; import { toolContractMetadata } from "./toolEffects.js"; -const session = ( +const session = < + Name extends keyof typeof sessionOutputSchemas, + Schema extends z.ZodObject, +>( name: Name, description: string, inputSchema: Schema, @@ -48,9 +54,7 @@ const session = ( /** Session-owned Evidence bundle export options. */ export const exportEvidenceBundleInputSchema = z.strictObject({ - path: z - .string() - .min(1) + path: localPathStringSchema .refine(isAbsoluteLocalPath, { message: "path must be an absolute local filesystem path (for example /tmp/rea/evidence.json or C:\\rea\\evidence.json)", diff --git a/src/contracts/sessionToolSchemas.ts b/src/contracts/sessionToolSchemas.ts index 87afe287abe..408c14d1de3 100644 --- a/src/contracts/sessionToolSchemas.ts +++ b/src/contracts/sessionToolSchemas.ts @@ -1,13 +1,15 @@ import { z } from "zod"; -import { isAbsoluteLocalPath } from "../domain/localPath.js"; +import { + isAbsoluteLocalPath, + localPathStringSchema, +} from "../domain/localPath.js"; import { artifactComparisonInputSchema } from "../domain/artifactComparison.js"; import { bundleComparisonInputSchema } from "../domain/bundleComparison.js"; import { callPathInputSchema } from "../domain/callPath.js"; import { changedBehaviorInputSchema } from "../domain/changedBehavior.js"; import { functionComparisonInputSchema } from "../domain/functionComparison.js"; -import { jsonValueSchema } from "../domain/jsonValue.js"; import { processScenarioSchema } from "../domain/process/processCapture.js"; import { processTraceSpecificationSchema } from "../domain/process/processTraceComparison.js"; import { recordUnknownInputSchema } from "../domain/residualUnknown.js"; @@ -38,9 +40,7 @@ export const addressContextInputSchema = z.strictObject({ /** Session-owned Evidence bundle import options. */ export const importEvidenceBundleInputSchema = z.strictObject({ - path: z - .string() - .min(1) + path: localPathStringSchema .refine(isAbsoluteLocalPath, { message: "path must be an absolute local filesystem path (for example /tmp/evidence.json or C:\\rea\\evidence.json)", diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index a5bfaa0796d..0da2b520c8b 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -373,7 +373,7 @@ export const managedWorkflowOutputSchemas: Readonly< }; /** Exact structured-content schemas for target lifecycle operations. */ -export const sessionOutputSchemas: Readonly> = { +export const sessionOutputSchemas = { open_binary: lifecycleResultOf( z.object({ path: z.string(), @@ -469,4 +469,4 @@ export const sessionOutputSchemas: Readonly> = { unknown: residualUnknownSchema, }), ), -}; +} satisfies Readonly>; diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index d5d441fab7c..a4c1c636ebb 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -25,7 +25,8 @@ import { prefixedDigestSchema } from "./../domain/digests.js"; /** Inline result with its complete Evidence record. */ export const inlineEvidenceRecordSchema = evidenceEnvelopeSchema; -export const evidenceResultOf = (schema: z.ZodType) => +/** Wrap a result and its Evidence while preserving the result schema type. */ +export const evidenceResultOf = (schema: Schema) => z.strictObject({ result: schema, evidence_id: prefixedDigestSchema("ev"), @@ -33,14 +34,18 @@ export const evidenceResultOf = (schema: z.ZodType) => }); const resultOf = evidenceResultOf; -export const lifecycleResultOf = (schema: z.ZodType) => +/** Wrap a lifecycle result while preserving its exact schema type. */ +export const lifecycleResultOf = (schema: Schema) => z.object({ result: schema }); /** Resolve a required named output schema or reject contract drift. */ -export const requireOutputSchema = ( - schemas: Readonly>, - name: string, -): z.ZodObject => { +export const requireOutputSchema = < + Schemas extends Readonly>, + Name extends keyof Schemas & string, +>( + schemas: Schemas, + name: Name, +): Schemas[Name] => { const schema = schemas[name]; if (schema === undefined) throw new Error(`Missing output schema for ${name}`); diff --git a/src/domain/browserScenarioValues.ts b/src/domain/browserScenarioValues.ts index 6313c7ca9ea..0344b628cf3 100644 --- a/src/domain/browserScenarioValues.ts +++ b/src/domain/browserScenarioValues.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -import { isAbsoluteLocalPath } from "./localPath.js"; +import { isAbsoluteLocalPath, localPathStringSchema } from "./localPath.js"; import { browserNetworkContentSelectionSchema } from "./browserNetworkEvidence.js"; import { @@ -12,14 +12,13 @@ export const scenarioIdentifierSchema = z .string() .regex(/^[A-Za-z][A-Za-z0-9._-]*$/u); -const browserExecutablePathSchema = z - .string() - .trim() - .min(1) - .refine(isAbsoluteLocalPath, { +const browserExecutablePathSchema = localPathStringSchema.refine( + isAbsoluteLocalPath, + { message: "executable_path must be an absolute local filesystem path (for example /opt/chromium/chrome or C:\\chromium\\chrome.exe)", - }); + }, +); const browserScenarioBaseUrlSchema = z .string() diff --git a/src/domain/bundleComparison.ts b/src/domain/bundleComparison.ts index ec9c731535a..cb01980a47f 100644 --- a/src/domain/bundleComparison.ts +++ b/src/domain/bundleComparison.ts @@ -11,6 +11,7 @@ import type { ResidualUnknown } from "./residualUnknown.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { comparisonStatusSchema } from "./comparisonStatus.js"; +import { localPathStringSchema } from "./localPath.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const classificationSchema = z.enum([ @@ -24,8 +25,8 @@ const classificationSchema = z.enum([ /** Strict input for canonical Evidence bundle comparison. */ export const bundleComparisonInputSchema = z.strictObject({ - left_bundle_path: z.string().min(1), - right_bundle_path: z.string().min(1), + left_bundle_path: localPathStringSchema, + right_bundle_path: localPathStringSchema, record_pairs: z .array( z.object({ diff --git a/src/domain/changedBehavior.test.ts b/src/domain/changedBehavior.test.ts index d47660f0754..cc19c4c50b1 100644 --- a/src/domain/changedBehavior.test.ts +++ b/src/domain/changedBehavior.test.ts @@ -42,7 +42,18 @@ const comparison = ( ? "rea.process-comparison" : "rea.artifact-comparison", operation, - parameters: input.salt === undefined ? {} : { salt: input.salt }, + parameters: { + ...(process + ? { + left_evidence_id: left.evidence_id, + right_evidence_id: right.evidence_id, + } + : { + left_evidence_ids: [left.evidence_id], + right_evidence_ids: [right.evidence_id], + }), + ...(input.salt === undefined ? {} : { salt: input.salt }), + }, result, confidence: "derived", authority: "analyst-inference", @@ -273,13 +284,17 @@ describe("changed behavior aggregation", () => { processResult({ status: "changed", terminal: "changed" }), { salt: comparisonIndex, - evidenceLinks: Array.from( - { length: 200 }, - (_, linkIndex) => - `ev_${(comparisonIndex * 200 + linkIndex + 1) - .toString(16) - .padStart(64, "0")}`, - ), + evidenceLinks: [ + left.evidence_id, + right.evidence_id, + ...Array.from( + { length: 200 }, + (_, linkIndex) => + `ev_${(comparisonIndex * 200 + linkIndex + 1) + .toString(16) + .padStart(64, "0")}`, + ), + ], }, ), ); @@ -287,7 +302,7 @@ describe("changed behavior aggregation", () => { const result = findChangedBehavior(comparisons); expect(result.summary.observed_changes).toBe(comparisons.length); - expect(result.evidence_links).toHaveLength(comparisons.length * 201); - expect(result.findings.items[0]?.evidence_links).toHaveLength(201); + expect(result.evidence_links).toHaveLength(comparisons.length * 201 + 2); + expect(result.findings.items[0]?.evidence_links).toHaveLength(203); }); }); diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index 37cdd97ac73..36f62e94fff 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { artifactComparisonResultSchema } from "./artifactComparison.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js"; import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js"; import { functionComparisonResultSchema } from "./functionComparison.js"; import { @@ -20,7 +21,7 @@ const evidenceIdSchema = prefixedDigestSchema("ev"); /** Input for aggregating existing comparison Evidence. */ export const changedBehaviorInputSchema = z.strictObject({ - comparisons: z.array(evidenceSchema).default([]), + comparisons: z.array(evidenceSchema).min(1), }); const findingSchema = z.object({ @@ -34,7 +35,7 @@ const findingSchema = z.object({ ]), status: comparisonStatusSchema, source_comparison_id: evidenceIdSchema, - evidence_links: z.array(evidenceIdSchema).min(3), + evidence_links: z.array(evidenceIdSchema).min(2), limitations: z.array(z.string()), }); @@ -56,7 +57,7 @@ export const changedBehaviorResultSchema = z findings: z.object({ items: z.array(findingSchema), }), - evidence_links: z.array(evidenceIdSchema).min(3), + evidence_links: z.array(evidenceIdSchema).min(2), limitations: z.array(z.string()), }) .superRefine((result, context) => { @@ -185,11 +186,14 @@ const parseComparisonEvidence = (input: unknown): Evidence => { throw new TypeError( "Comparison Evidence operation, predicate, and provider disagree", ); - if (evidence.evidence_links.length < 2) + if ( + !comparisonSourceEvidenceIds(evidence).every((id) => + evidence.evidence_links.includes(id), + ) + ) throw new TypeError( - "Comparison Evidence must cite both source observations", + "Comparison Evidence must cite both selected source observations", ); - assertUnique(evidence.evidence_links, "source Evidence links"); parseResult(evidence); return evidence; }; diff --git a/src/domain/comparisonSourceEvidence.ts b/src/domain/comparisonSourceEvidence.ts new file mode 100644 index 00000000000..420428b4852 --- /dev/null +++ b/src/domain/comparisonSourceEvidence.ts @@ -0,0 +1,57 @@ +import { z } from "zod"; + +import { uniqueSorted } from "./canonicalOrdering.js"; +import { prefixedDigestSchema } from "./digests.js"; +import type { Evidence } from "./evidence.js"; + +const evidenceIdSchema = prefixedDigestSchema("ev"); +const sideParametersSchema = z.object({ + left_evidence_id: evidenceIdSchema.optional(), + right_evidence_id: evidenceIdSchema.optional(), + left_evidence_ids: z.array(evidenceIdSchema).min(1).optional(), + right_evidence_ids: z.array(evidenceIdSchema).min(1).optional(), +}); + +/** Resolve each selected side and validate aliases and duplicates within that role. */ +export const comparisonSourceEvidenceSides = ( + evidence: Evidence, +): { left: string[]; right: string[] } => { + const parameters = sideParametersSchema.parse(evidence.parameters); + if ( + (parameters.left_evidence_id === undefined) !== + (parameters.right_evidence_id === undefined) || + (parameters.left_evidence_ids === undefined) !== + (parameters.right_evidence_ids === undefined) + ) + throw new TypeError( + "Comparison Evidence source representation omitted one side", + ); + const side = (single: string | undefined, multiple: string[] | undefined) => { + if ( + single !== undefined && + multiple !== undefined && + (multiple.length !== 1 || multiple[0] !== single) + ) + throw new TypeError( + "Comparison Evidence source representations disagree", + ); + const selected = single === undefined ? (multiple ?? []) : [single]; + if (selected.length === 0) + throw new TypeError("Comparison Evidence omitted its source parameters"); + if (new Set(selected).size !== selected.length) + throw new TypeError( + "Comparison Evidence repeats a source within one side", + ); + return selected; + }; + return { + left: side(parameters.left_evidence_id, parameters.left_evidence_ids), + right: side(parameters.right_evidence_id, parameters.right_evidence_ids), + }; +}; + +/** Resolve the source closure without counting observations shared by both roles twice. */ +export const comparisonSourceEvidenceIds = (evidence: Evidence): string[] => { + const sides = comparisonSourceEvidenceSides(evidence); + return uniqueSorted([...sides.left, ...sides.right]); +}; diff --git a/src/domain/firmware/firmwareAnalysis.ts b/src/domain/firmware/firmwareAnalysis.ts index 8a7f1f5de4d..abea685e7e2 100644 --- a/src/domain/firmware/firmwareAnalysis.ts +++ b/src/domain/firmware/firmwareAnalysis.ts @@ -1,14 +1,12 @@ import { z } from "zod"; import { digestSchema } from "../digests.js"; import { jsonValueSchema } from "../jsonValue.js"; -import { isAbsoluteLocalPath } from "../localPath.js"; +import { isAbsoluteLocalPath, localPathStringSchema } from "../localPath.js"; const bytes = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); const range = z.strictObject({ offset: bytes, length: bytes.positive() }); const target = { - path: z - .string() - .min(1) + path: localPathStringSchema .refine(isAbsoluteLocalPath, { message: "path must be an absolute local filesystem path (for example /tmp/firmware.bin or C:\\firmware\\firmware.bin)", @@ -23,9 +21,10 @@ export const firmwareInputSchemas = { inspect_firmware_regions: z.strictObject(target), extract_firmware: z.strictObject({ ...target, - output_directory: z - .string() - .min(1) + output_directory: localPathStringSchema + .refine(isAbsoluteLocalPath, { + message: "output_directory must be an absolute local filesystem path", + }) .describe("Absolute, absent output directory; parent must exist"), range: range .optional() diff --git a/src/domain/javascript/javascriptApplicationAnalysis.ts b/src/domain/javascript/javascriptApplicationAnalysis.ts index b7f867a84a0..313ab529122 100644 --- a/src/domain/javascript/javascriptApplicationAnalysis.ts +++ b/src/domain/javascript/javascriptApplicationAnalysis.ts @@ -12,12 +12,13 @@ import { } from "./javascriptSemanticGraph.js"; import { digestSchema } from "../digests.js"; import { prefixedDigestSchema } from "../digests.js"; +import { localPathStringSchema } from "../localPath.js"; const countSchema = z.number().int().min(0); /** Public target-free request for static JavaScript application analysis. */ export const analyzeJavaScriptApplicationInputSchema = z.strictObject({ - input_path: z.string().min(1), + input_path: localPathStringSchema, format: z.enum(["auto", "asar", "directory"]).default("auto"), }); diff --git a/src/domain/javascript/javascriptRecovery.ts b/src/domain/javascript/javascriptRecovery.ts index 90c1f5df170..f5cf2b80529 100644 --- a/src/domain/javascript/javascriptRecovery.ts +++ b/src/domain/javascript/javascriptRecovery.ts @@ -1,17 +1,16 @@ import { z } from "zod"; import { digestSchema } from "../digests.js"; import { analyzeJavaScriptApplicationInputSchema } from "./javascriptApplicationAnalysis.js"; +import { localPathStringSchema } from "../localPath.js"; /** Recover derived readable modules from one explicitly selected local script. */ export const javascriptRecoveryInputSchema = z.strictObject({ - path: z - .string() - .min(1) - .describe("Absolute path to a UTF-8 JavaScript file; never executed"), - output_directory: z - .string() - .min(1) - .describe("Absolute absent output directory; parent must exist"), + path: localPathStringSchema.describe( + "Absolute path to a UTF-8 JavaScript file; never executed", + ), + output_directory: localPathStringSchema.describe( + "Absolute absent output directory; parent must exist", + ), extraction_mode: z .enum(["structural", "heuristic", "inspection"]) .default("structural") diff --git a/src/domain/localPath.ts b/src/domain/localPath.ts index 9248de4d71b..d1dd98fcee0 100644 --- a/src/domain/localPath.ts +++ b/src/domain/localPath.ts @@ -1,4 +1,11 @@ import { isAbsolute, relative, sep } from "node:path"; +import { z } from "zod"; + +/** Preserve selected filesystem names while rejecting OS-invalid NUL bytes. */ +export const localPathStringSchema = z + .string() + .min(1) + .regex(/^[^\0]*$/u, "Local filesystem paths cannot contain NUL"); /** True when the value is an absolute filesystem path on the host platform. */ export const isAbsoluteLocalPath = (value: string): boolean => diff --git a/src/domain/native/nativeCallObservation.ts b/src/domain/native/nativeCallObservation.ts index b07a5a0f129..b4e5495f8d9 100644 --- a/src/domain/native/nativeCallObservation.ts +++ b/src/domain/native/nativeCallObservation.ts @@ -1,16 +1,21 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; const hexSchema = z.string().regex(/^0x[0-9a-f]+$/u); -/** Objective-C class names and selectors never contain spaces or brackets. */ -const objcName = z +const nativeString = z .string() + .regex( + /^[^\0]*$/u, + "Native launch and breakpoint strings cannot contain NUL", + ); +/** Objective-C class names and selectors never contain spaces or brackets. */ +const objcName = nativeString .min(1) .regex(/^[^\s[\]]+$/u, "Use the bare name without spaces or brackets"); const functionBreakpoint = { kind: z.literal("function"), - name: z - .string() + name: nativeString .min(1) .describe( "Exact symbol name, such as `open`, a C++ or mangled Swift symbol, or `-[NSString length]`.", @@ -28,8 +33,7 @@ const objcMethodBreakpoint = { export const nativeCallBreakpointSchema = z.discriminatedUnion("kind", [ z.strictObject({ ...functionBreakpoint, - module: z - .string() + module: nativeString .min(1) .optional() .describe( @@ -51,7 +55,7 @@ export const nativeCallBreakpointSchema = z.discriminatedUnion("kind", [ */ export const nativeCallObservationInputSchema = z.strictObject({ breakpoints: z.array(nativeCallBreakpointSchema).min(1), - arguments: z.array(z.string()).default([]), + arguments: z.array(nativeString).default([]), environment: z .record( z @@ -62,7 +66,7 @@ export const nativeCallObservationInputSchema = z.strictObject({ ) .default({}) .describe("Overrides on top of the environment REA runs with."), - working_directory: z.string().min(1).optional(), + working_directory: localPathStringSchema.optional(), duration_ms: z .number() .int() diff --git a/src/domain/process/processScenario.test.ts b/src/domain/process/processScenario.test.ts deleted file mode 100644 index 9f669328d31..00000000000 --- a/src/domain/process/processScenario.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { processScenarioSchema } from "./processScenario.js"; - -describe("direct process scenarios", () => { - it("defaults to one command with inherited environment and no filesystem scan", () => { - expect(processScenarioSchema.parse({ executable: "node" })).toMatchObject({ - executable: "node", - arguments: [], - working_directory: ".", - environment: {}, - filesystem_observation_paths: [], - events: [], - }); - }); - - it("accepts caller-selected environment and explicit filesystem observations", () => { - const environment = { "app.setting": "value", "1": "numbered" }; - const scenario = processScenarioSchema.parse({ - executable: "node", - arguments: ["./task.mjs"], - working_directory: "./project", - environment, - filesystem_observation_paths: ["state"], - }); - expect(scenario.environment).toEqual(environment); - expect(scenario.filesystem_observation_paths).toEqual(["state"]); - }); - - it("rejects malformed environment names and adapter-reserved identity", () => { - for (const name of ["", "KEY=VALUE", "KEY\0VALUE", "REA_PROCESS_RUN_ID"]) { - const result = processScenarioSchema.safeParse({ - executable: "node", - environment: { [name]: "value" }, - }); - expect(result.success).toBe(false); - if (name === "" && !result.success) { - const invalidKey = result.error.issues[0]; - expect(invalidKey?.code).toBe("invalid_key"); - if (invalidKey?.code === "invalid_key") { - expect(invalidKey.issues[0]?.message).toBe( - "Environment names must not be empty", - ); - } - } - if (name === "REA_PROCESS_RUN_ID" && !result.success) { - const invalidKey = result.error.issues[0]; - expect(invalidKey?.code).toBe("invalid_key"); - if (invalidKey?.code === "invalid_key") { - expect(invalidKey.issues[0]?.message).toBe( - "REA_PROCESS_RUN_ID is reserved by the process adapter", - ); - } - } - } - expect( - processScenarioSchema.safeParse({ - executable: "node", - environment: { "REA_PROCESS_RUN_ID\n": "caller-value" }, - }).success, - ).toBe(true); - }); - - it("rejects NUL in strings passed to child-process APIs", () => { - const nul = "\0"; - for (const input of [ - { executable: `node${nul}` }, - { executable: "node", arguments: [nul] }, - { executable: "node", working_directory: `.${nul}` }, - { executable: "node", environment: { KEY: nul } }, - { executable: "node", filesystem_observation_paths: [nul] }, - ]) - expect(processScenarioSchema.safeParse(input).success).toBe(false); - expect( - processScenarioSchema.safeParse({ - executable: "node", - events: [{ type: "input", at_ms: 0, data: nul }], - }).success, - ).toBe(true); - }); - - it("keeps bounded capture controls and ordered scheduled interaction", () => { - const scenario = processScenarioSchema.parse({ - executable: "node", - events: [ - { type: "input", at_ms: 10, data: "answer", sensitive: true }, - { type: "resize", at_ms: 20, columns: 100, rows: 30 }, - { type: "signal", at_ms: 30, signal: "SIGTERM" }, - ], - limits: { output_bytes: 2_000_000, filesystem_depth: 0 }, - }); - expect(scenario.events).toHaveLength(3); - expect(scenario.events[0]).toMatchObject({ sensitive: true }); - expect(scenario.limits.output_bytes).toBe(2_000_000); - expect(scenario.limits.filesystem_depth).toBe(0); - }); - - it("rejects events after timeout and unknown orchestration fields", () => { - expect( - processScenarioSchema.safeParse({ - executable: "node", - timeout_ms: 10, - events: [{ type: "input", at_ms: 11, data: "late" }], - }).success, - ).toBe(false); - for (const [field, value] of [ - ["command_shims", []], - ["replay", {}], - ["reactive", {}], - ["checkpoints", []], - ] as const) - expect( - processScenarioSchema.safeParse({ executable: "node", [field]: value }) - .success, - ).toBe(false); - }); -}); diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index 382cf3196d1..f62c947d77c 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -91,7 +91,7 @@ export const processComparisonContract = ( * Defaults are part of the evidence contract and must remain deterministic. */ export const processScenarioSchema = z - .object({ + .strictObject({ executable: childProcessString.min(1), arguments: z.array(childProcessString).default([]), working_directory: childProcessString.default("."), @@ -100,7 +100,7 @@ export const processScenarioSchema = z .array(childProcessString.min(1)) .default([]), terminal: z - .object({ + .strictObject({ columns: z.number().int().min(1).max(65_535).default(80), rows: z.number().int().min(1).max(65_535).default(24), scrollback: z.number().int().min(0).default(1_000), @@ -109,19 +109,19 @@ export const processScenarioSchema = z events: z .array( z.discriminatedUnion("type", [ - z.object({ + z.strictObject({ ...timedEventBase, type: z.literal("input"), data: z.string(), sensitive: z.boolean().default(false), }), - z.object({ + z.strictObject({ ...timedEventBase, type: z.literal("resize"), columns: z.number().int().min(1).max(65_535), rows: z.number().int().min(1).max(65_535), }), - z.object({ + z.strictObject({ ...timedEventBase, type: z.literal("signal"), signal: z.enum(["SIGINT", "SIGTERM", "SIGKILL"]), @@ -133,7 +133,7 @@ export const processScenarioSchema = z idle_timeout_ms: positiveBudget.default(30_000), settle_ms: z.number().int().safe().nonnegative().default(100), limits: z - .object({ + .strictObject({ output_bytes: positiveBudget.default(1_000_000), files: positiveBudget.default(10_000), file_bytes: positiveBudget.default(10_000_000), @@ -149,15 +149,19 @@ export const processScenarioSchema = z filesystem_depth: 16, }), normalization: z - .object({ + .strictObject({ paths: z.boolean().default(false), pids: z.boolean().default(true), ports: z.boolean().default(true), time_bucket_ms: positiveBudget.default(10), patterns: z .array( - z.object({ - pattern: z.string(), + z.strictObject({ + pattern: z + .string() + .describe( + "Literal text to replace in captured output; not a regular expression.", + ), replacement: z.string(), }), ) @@ -171,7 +175,6 @@ export const processScenarioSchema = z patterns: [], }), }) - .strict() .superRefine((scenario, context) => { for (let index = 1; index < scenario.events.length; index += 1) { const event = scenario.events[index]; diff --git a/src/domain/reconstructionVerification.functionSources.test.ts b/src/domain/reconstructionVerification.functionSources.test.ts index 93e55d9e0e9..af3f88594ba 100644 --- a/src/domain/reconstructionVerification.functionSources.test.ts +++ b/src/domain/reconstructionVerification.functionSources.test.ts @@ -322,10 +322,6 @@ describe("function reconstruction malformed source IDs", () => { ); it.each([ - [ - "same singular source on both sides", - { ...singular, right_evidence_id: left.evidence_id }, - ], [ "duplicate left plural source", { ...plural, left_evidence_ids: [left.evidence_id, left.evidence_id] }, @@ -334,17 +330,9 @@ describe("function reconstruction malformed source IDs", () => { "duplicate right plural source", { ...plural, right_evidence_ids: [right.evidence_id, right.evidence_id] }, ], - [ - "same plural source on both sides", - { ...plural, right_evidence_ids: [left.evidence_id] }, - ], - [ - "overlapping plural sides", - { ...plural, right_evidence_ids: [right.evidence_id, left.evidence_id] }, - ], - ])("rejects %s with the existing uniqueness guard", (_name, parameters) => { + ])("rejects %s", (_name, parameters) => { expect(admittedVerification(comparison(parameters))).toThrow( - /unique and two-sided/u, + /repeats a source within one side/u, ); }); }); diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts index 3de1d850964..c010c7d4093 100644 --- a/src/domain/reconstructionVerification.ts +++ b/src/domain/reconstructionVerification.ts @@ -6,6 +6,7 @@ import { z } from "zod"; import { artifactComparisonResultSchema } from "./artifactComparison.js"; import { artifactInventoryResultSchema } from "./artifactGraph.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js"; import { parseEvidence, type Evidence } from "./evidence.js"; import { parseEvidenceBundle } from "./evidenceBundle.js"; import { functionComparisonResultSchema } from "./functionComparison.js"; @@ -36,13 +37,10 @@ import { } from "./reconstructionUnknowns.js"; import type { ResidualUnknown } from "./residualUnknown.js"; -const evidenceIdSchema = prefixedDigestSchema("ev"); - export { reconstructionVerificationInputSchema, reconstructionVerificationResultSchema, } from "./reconstructionVerificationSchemas.js"; -import { prefixedDigestSchema } from "./../domain/digests.js"; export type { ReconstructionVerificationResult } from "./reconstructionVerificationSchemas.js"; const providers = { @@ -130,12 +128,8 @@ const evaluateClaim = ( `Missing comparison Evidence ${claim.comparison_evidence_id}`, ); validateComparisonIdentity(claim, comparison); - const sides = sourceSides(claim, comparison); - const allSourceIds = [...sides.left, ...sides.right]; - if (new Set(allSourceIds).size !== allSourceIds.length) - throw new TypeError( - "Comparison source Evidence must be unique and two-sided", - ); + const sides = comparisonSourceEvidenceSides(comparison); + const allSourceIds = uniqueSorted([...sides.left, ...sides.right]); if (!sameSet(comparison.evidence_links, allSourceIds)) throw new TypeError( "Comparison Evidence closure disagrees with its parameters", @@ -224,52 +218,6 @@ const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => { throw new TypeError("Comparison Evidence identity or authority disagrees"); }; -const sourceSides = ( - claim: Claim, - evidence: Evidence, -): { left: string[]; right: string[] } => { - const parameters = evidence.parameters; - let singularSides: { left: string[]; right: string[] } | undefined; - if ( - claim.kind === "behavioral" || - (claim.kind === "structural-function" && - ("left_evidence_id" in parameters || "right_evidence_id" in parameters)) - ) { - const parsed = z - .object({ - left_evidence_id: evidenceIdSchema, - right_evidence_id: evidenceIdSchema, - }) - .passthrough() - .parse(parameters); - singularSides = { - left: [parsed.left_evidence_id], - right: [parsed.right_evidence_id], - }; - if ( - claim.kind === "behavioral" || - !("left_evidence_ids" in parameters || "right_evidence_ids" in parameters) - ) - return singularSides; - } - const parsed = z - .object({ - left_evidence_ids: z.array(evidenceIdSchema).min(1), - right_evidence_ids: z.array(evidenceIdSchema).min(1), - }) - .passthrough() - .parse(parameters); - if ( - singularSides !== undefined && - (parsed.left_evidence_ids.length !== 1 || - parsed.right_evidence_ids.length !== 1 || - parsed.left_evidence_ids[0] !== singularSides.left[0] || - parsed.right_evidence_ids[0] !== singularSides.right[0]) - ) - throw new TypeError("Function comparison source Evidence aliases disagree"); - return { left: parsed.left_evidence_ids, right: parsed.right_evidence_ids }; -}; - const observedStatus = (claim: Claim, evidence: Evidence): ObservedStatus => { if (claim.kind === "behavioral") { const result = processCaptureComparisonSchema.parse( @@ -335,6 +283,10 @@ const authorityLimitations = ( const requiredAuthority = claim.kind === "behavioral" ? "controlled-replay" : "shipped-artifact"; const limitations: string[] = []; + if (sides.left.some((id) => sides.right.includes(id))) + limitations.push( + "A source observation appears on both comparison sides; independent original and reconstruction observations are unavailable.", + ); if ( sources.some( (source) => @@ -393,5 +345,5 @@ const count = ( status: ClaimResult["status"], ): number => items.filter((item) => item.status === status).length; const sameSet = (left: readonly string[], right: readonly string[]): boolean => - left.length === right.length && + uniqueSorted(left).length === uniqueSorted(right).length && uniqueSorted(left).every((id, index) => id === uniqueSorted(right)[index]); diff --git a/src/domain/reconstructionVerificationSchemas.ts b/src/domain/reconstructionVerificationSchemas.ts index 4ce88ba7a0d..13f5009f5b9 100644 --- a/src/domain/reconstructionVerificationSchemas.ts +++ b/src/domain/reconstructionVerificationSchemas.ts @@ -89,7 +89,7 @@ export const reconstructionClaimResultSchema = z.object({ comparison_evidence_id: evidenceIdSchema, left_evidence_ids: z.array(evidenceIdSchema).min(1), right_evidence_ids: z.array(evidenceIdSchema).min(1), - evidence_links: z.array(evidenceIdSchema).min(3), + evidence_links: z.array(evidenceIdSchema).min(2), unknown_ids: z.array(unknownIdSchema), limitations: z.array(z.string()), }); @@ -118,7 +118,7 @@ export const reconstructionVerificationResultSchema = z unknown_ids: z.array(unknownIdSchema), }), ), - evidence_links: z.array(evidenceIdSchema).min(3), + evidence_links: z.array(evidenceIdSchema).min(2), limitations: z.array(z.string()), }) .superRefine((result, context) => { diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts index d5558ad277c..91c0afa1de8 100644 --- a/src/domain/staticRuntimeCorrelation.ts +++ b/src/domain/staticRuntimeCorrelation.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { artifactComparisonResultSchema } from "./artifactComparison.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +import { comparisonSourceEvidenceIds } from "./comparisonSourceEvidence.js"; import { evidenceSchema, parseEvidence, type Evidence } from "./evidence.js"; import { functionComparisonResultSchema } from "./functionComparison.js"; import { @@ -392,66 +393,22 @@ const assertComparisonIdentity = (evidence: Evidence): void => { evidence.provider.version !== "1") || evidence.confidence !== "derived" || evidence.authority !== "analyst-inference" || - evidence.subject !== null || - evidence.evidence_links.length < 2 || - new Set(evidence.evidence_links).size !== evidence.evidence_links.length + evidence.subject !== null ) throw new TypeError( "Comparison Evidence operation, predicate, provider, or citations disagree", ); - const parameterLinks = comparisonParameterLinks(evidence); - if (!sameSet(parameterLinks, evidence.evidence_links)) + const expectedLinks = comparisonSourceEvidenceIds(evidence); + const actualLinks = uniqueSorted(evidence.evidence_links); + if ( + expectedLinks.length !== actualLinks.length || + expectedLinks.some((id, index) => id !== actualLinks[index]) + ) throw new TypeError( "Comparison Evidence closure disagrees with its source parameters", ); }; -const comparisonParameterLinks = (evidence: Evidence): string[] => { - if (evidence.operation === "compare_process_captures") { - const parameters = z - .object({ - left_evidence_id: evidenceIdSchema, - right_evidence_id: evidenceIdSchema, - }) - .passthrough() - .parse(evidence.parameters); - return [parameters.left_evidence_id, parameters.right_evidence_id]; - } - const parameters = z - .object({ - left_evidence_id: evidenceIdSchema.optional(), - right_evidence_id: evidenceIdSchema.optional(), - left_evidence_ids: z.array(evidenceIdSchema).min(1).optional(), - right_evidence_ids: z.array(evidenceIdSchema).min(1).optional(), - }) - .passthrough() - .parse(evidence.parameters); - const left = - parameters.left_evidence_id === undefined - ? (parameters.left_evidence_ids ?? []) - : [parameters.left_evidence_id]; - const right = - parameters.right_evidence_id === undefined - ? (parameters.right_evidence_ids ?? []) - : [parameters.right_evidence_id]; - if (left.length === 0 || right.length === 0) - throw new TypeError("Comparison Evidence omitted its source parameters"); - return [...left, ...right]; -}; - -const sameSet = ( - left: readonly string[], - right: readonly string[], -): boolean => { - if (left.length !== right.length) return false; - const rightSet = new Set(right); - return ( - rightSet.size === right.length && - new Set(left).size === left.length && - left.every((item) => rightSet.has(item)) - ); -}; - const observedPattern = ( staticStatus: ComparisonStatus, runtimeStatus: ComparisonStatus, diff --git a/src/process/capture/ProcessCaptureError.test.ts b/src/process/capture/ProcessCaptureError.test.ts deleted file mode 100644 index d554d8ee89f..00000000000 --- a/src/process/capture/ProcessCaptureError.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; -import { - ProcessCaptureError, - processCaptureCancelled, -} from "./ProcessCaptureError.js"; - -describe("process capture error projection", () => { - it("reports incomplete cleanup resources without exposing its cause", () => { - const projected = projectAnalysisError( - new ProcessCaptureError("terminal cleanup failed", { - cause: new Error("secret-token"), - reason: "cleanup_incomplete", - cleanupResources: ["process_group"], - }), - ); - - expect(projected).toMatchObject({ - code: "cleanup_incomplete", - details: { cleanup: "incomplete", resources: ["process_group"] }, - }); - expect(JSON.stringify(projected)).not.toContain("secret-token"); - }); - - it("projects caller cancellation as completed cleanup", () => { - expect(projectAnalysisError(processCaptureCancelled())).toMatchObject({ - code: "cancelled", - category: "cancelled", - details: { operation: "process_capture", cleanup: "complete" }, - }); - }); -}); diff --git a/src/process/capture/ProcessCaptureError.ts b/src/process/capture/ProcessCaptureError.ts index 4f242255943..1eea782a5fd 100644 --- a/src/process/capture/ProcessCaptureError.ts +++ b/src/process/capture/ProcessCaptureError.ts @@ -31,14 +31,14 @@ export class ProcessCaptureError extends AnalysisError { constructor(message: string, options?: ProcessCaptureErrorOptions) { super(message, options); - this.userMessage = options?.userMessage; + this.userMessage = options?.userMessage ?? message; this.userCategory = options?.userCategory; this.reason = options?.reason ?? (options?.userCategory === "cancelled" ? "cancelled" : "capture_failed"); this.cleanupIncomplete = this.reason === "cleanup_incomplete"; this.cleanupResources = options?.cleanupResources ?? []; - this.executionFailure = options?.executionFailure; + this.executionFailure = options?.executionFailure ?? message; this.partialObservation = options?.partialObservation; this.cleanupReport = options?.cleanupReport; } diff --git a/src/server/registerArtifactComparisonTool.ts b/src/server/registerArtifactComparisonTool.ts index a832a1a561d..11a64ccf7a9 100644 --- a/src/server/registerArtifactComparisonTool.ts +++ b/src/server/registerArtifactComparisonTool.ts @@ -56,7 +56,12 @@ export const registerArtifactComparisonTool = ( recordDerivedEvidence( session, evidence, - artifactUnknownInput(left, right, comparison.status), + artifactUnknownInput( + left, + right, + comparison.status, + evidence.evidence_id, + ), ), ); }, @@ -67,16 +72,21 @@ const artifactUnknownInput = ( left: { readonly evidence_id: string }, right: { readonly evidence_id: string }, status: ReturnType["status"], + comparisonEvidenceId: string, ): RecordUnknownInput | undefined => { - if (status === "unchanged" || left.evidence_id === right.evidence_id) - return undefined; + if (status === "unchanged") return undefined; + const contradictory = + (status === "changed" || status === "contradiction") && + left.evidence_id !== right.evidence_id; return { - question: `Artifact comparison is ${status}`, + question: `Artifact comparison is ${status} (comparison ${comparisonEvidenceId})`, severity: status === "unknown" || status === "truncated" ? "high" : "medium", domain: "artifact-comparison", - supporting_evidence_ids: [left.evidence_id], - contradicting_evidence_ids: [right.evidence_id], + supporting_evidence_ids: contradictory + ? [left.evidence_id] + : [...new Set([left.evidence_id, right.evidence_id])], + contradicting_evidence_ids: contradictory ? [right.evidence_id] : [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, diff --git a/src/server/registerFunctionComparisonTool.ts b/src/server/registerFunctionComparisonTool.ts index 8fff00abad6..af8a113768d 100644 --- a/src/server/registerFunctionComparisonTool.ts +++ b/src/server/registerFunctionComparisonTool.ts @@ -87,6 +87,7 @@ export const registerFunctionComparisonTool = ( status: comparison.status, leftIds, rightIds, + comparisonEvidenceId: evidence.evidence_id, }), ); return toEvidenceToolResult(evidence, contract, recorded); @@ -98,18 +99,23 @@ const functionUnknownInput = ({ status, leftIds, rightIds, + comparisonEvidenceId, }: { status: ReturnType["status"]; leftIds: readonly string[]; rightIds: readonly string[]; + comparisonEvidenceId: string; }): RecordUnknownInput | undefined => { if (status === "unchanged") return undefined; + const supportingIds = [...new Set([...leftIds, ...rightIds])]; + const contradictingIds = rightIds.filter((id) => !leftIds.includes(id)); + const contradictory = status === "changed"; return { - question: `Function comparison is ${status}`, + question: `Function comparison is ${status} (comparison ${comparisonEvidenceId})`, severity: status === "changed" ? "medium" : "high", domain: "function-comparison", - supporting_evidence_ids: [...leftIds], - contradicting_evidence_ids: [...rightIds], + supporting_evidence_ids: contradictory ? [...leftIds] : supportingIds, + contradicting_evidence_ids: contradictory ? contradictingIds : [], required_authority: "shipped-artifact", required_confidence: "observed", required_environment: null, diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts index 931086bc1c0..a87aa6a0c2e 100644 --- a/src/server/registerProcessComparisonTool.ts +++ b/src/server/registerProcessComparisonTool.ts @@ -182,14 +182,19 @@ const comparisonUnknownInput = ( .filter(([, status]) => status !== "unchanged") .map(([scope]) => scope) .join(", "); + const contradictory = + comparison.status === "changed" && + parsed.left_evidence_id !== parsed.right_evidence_id; // Unknown identity uses the question, not its supporting records. Bind the // question to this comparison so distinct capture pairs and policies coexist. return { - question: `Process captures disagree across: ${differingScopes} (comparison ${parsed.comparison_evidence_id})`, + question: `${contradictory ? "Process captures disagree" : `Process comparison is ${comparison.status}`} across: ${differingScopes} (comparison ${parsed.comparison_evidence_id})`, severity: "high", domain: "process-comparison", - supporting_evidence_ids: [parsed.left_evidence_id], - contradicting_evidence_ids: [parsed.right_evidence_id], + supporting_evidence_ids: contradictory + ? [parsed.left_evidence_id] + : [...new Set([parsed.left_evidence_id, parsed.right_evidence_id])], + contradicting_evidence_ids: contradictory ? [parsed.right_evidence_id] : [], required_authority: "controlled-replay", required_confidence: "observed", required_environment: null, diff --git a/src/server/runDerivedOperation.test.ts b/src/server/runDerivedOperation.test.ts index e1705cdc0d0..f2404279ac2 100644 --- a/src/server/runDerivedOperation.test.ts +++ b/src/server/runDerivedOperation.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { runDerivedOperation } from "./runDerivedOperation.js"; +import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; const context = ( controller: AbortController, @@ -84,7 +85,28 @@ describe("derived MCP operation boundary", () => { error: { _tag: "AnalysisInputError", operation: "compare_artifacts", + issues: [ + { + path: [], + reason: "invalid_value", + message: "malformed comparison evidence", + }, + ], }, }); }); + + it("preserves a tagged integrity failure instead of replacing it with invalid input", async () => { + const failure = new EvidenceIntegrityError( + "Source Evidence does not match its digest", + ); + const result = await runDerivedOperation( + context(new AbortController()), + "compare_artifacts", + () => { + throw failure; + }, + ); + expect(result).toEqual({ ok: false, error: failure }); + }); }); diff --git a/src/server/runDerivedOperation.ts b/src/server/runDerivedOperation.ts index 0a7a55da67a..bc0c7ef9d1d 100644 --- a/src/server/runDerivedOperation.ts +++ b/src/server/runDerivedOperation.ts @@ -1,10 +1,13 @@ import type { ServerContext } from "@modelcontextprotocol/server"; import { setImmediate } from "node:timers/promises"; +import { z } from "zod"; +import { AnalysisError } from "../domain/analysisErrorBase.js"; import { AnalysisCancelledError, AnalysisInputError, } from "../domain/analysisErrorCore.js"; +import { describeValidationFailure } from "../domain/evidenceBundle.js"; import { err, ok, type Result } from "../domain/result.js"; import { mcpProgressReporter, type McpProgressContext } from "./mcpProgress.js"; @@ -18,7 +21,7 @@ export const runDerivedOperation = async ( context: DerivedOperationContext, operation: string, compute: () => Value, -): Promise> => { +): Promise> => { const progress = mcpProgressReporter(context); await progress.report({ phase: "prepare", @@ -34,11 +37,21 @@ export const runDerivedOperation = async ( value = compute(); } catch (cause: unknown) { // Typed causes already name the failed constraint; keep their issues. + if (cause instanceof AnalysisError) return err(cause); return err( - cause instanceof AnalysisCancelledError || - cause instanceof AnalysisInputError - ? cause - : new AnalysisInputError(operation, { cause }), + new AnalysisInputError( + operation, + { cause }, + cause instanceof TypeError || cause instanceof z.ZodError + ? [ + { + path: [], + reason: "invalid_value", + message: describeValidationFailure(cause), + }, + ] + : [], + ), ); } await progress.report({ diff --git a/tests/boundary/mcp/comparisonUnknownsMcp.test.ts b/tests/boundary/mcp/comparisonUnknownsMcp.test.ts new file mode 100644 index 00000000000..ebbfb815e1e --- /dev/null +++ b/tests/boundary/mcp/comparisonUnknownsMcp.test.ts @@ -0,0 +1,126 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { ARTIFACT_COMPARISON_EXAMPLE } from "../../../src/contracts/artifactComparisonExample.js"; +import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../../src/domain/evidence.js"; +import { jsonValueSchema } from "../../../src/domain/jsonValue.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; + +const connect = async () => { + const session = createTestBinarySession(() => { + throw new Error("Evidence comparison must not launch a provider"); + }); + const server = createServer(session, session); + const client = new Client({ name: "comparison-unknowns", version: "1" }); + onTestFinished(async () => { + await Promise.allSettled([client.close(), server.close(), session.close()]); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client, session }; +}; + +const observeAgain = ( + source: Evidence, + observation: number, + result = source.normalized_result, +) => + createEvidence( + source.subject === null + ? undefined + : { + path: `/tmp/fixture-${observation}`, + sha256: source.subject.digest.sha256, + format: source.subject.format, + }, + source.provider, + { + predicateType: source.predicate_type, + operation: source.operation, + parameters: { observation }, + result, + confidence: source.confidence, + authority: source.authority, + }, + ); + +it("retains one open unknown for an incomplete artifact compared with itself", async () => { + const { client, session } = await connect(); + const source = ARTIFACT_COMPARISON_EXAMPLE.left; + const result = jsonValueSchema.parse(source.normalized_result); + if (result === null || Array.isArray(result) || typeof result !== "object") + throw new Error("Expected an inventory object"); + const incomplete = observeAgain(source, 1, { + ...result, + nodes: [], + occurrences: [], + }); + const response = await client.callTool({ + name: "compare_artifacts", + arguments: { left: incomplete, right: incomplete }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toMatchObject({ + result: { status: "truncated" }, + }); + expect(session.listUnknowns({ domain: "artifact-comparison" })).toMatchObject( + [ + { + status: "open", + supporting_evidence_ids: [incomplete.evidence_id], + contradicting_evidence_ids: [], + }, + ], + ); +}); + +it.each([ + { + tool: "compare_artifacts", + domain: "artifact-comparison", + pair: ARTIFACT_COMPARISON_EXAMPLE, + }, + { + tool: "compare_functions", + domain: "function-comparison", + pair: FUNCTION_COMPARISON_EXAMPLE, + }, +])( + "keeps independent $domain observations and repeated comparisons idempotent", + async ({ tool, domain, pair }) => { + const { client, session } = await connect(); + const right = observeAgain(pair.right, 2); + for (const source of [pair.right, right]) { + const response = await client.callTool({ + name: tool, + arguments: { left: pair.left, right: source }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + expect(response.structuredContent).toMatchObject({ + result: { status: "changed" }, + }); + } + expect(session.listUnknowns({ domain })).toHaveLength(2); + const before = session.exportEvidenceBundle(); + const repeated = await client.callTool({ + name: tool, + arguments: { left: pair.left, right }, + }); + expect(repeated.isError, JSON.stringify(repeated)).not.toBe(true); + const compared = parseEvidence( + z.object({ evidence: z.unknown() }).parse(repeated.structuredContent) + .evidence, + ); + expect(session.evidenceById(compared.evidence_id)).toEqual(compared); + expect(session.exportEvidenceBundle()).toEqual(before); + }, +); diff --git a/tests/boundary/mcp/nativeDemangleMcp.test.ts b/tests/boundary/mcp/nativeDemangleMcp.test.ts index 706826ca6e1..6974f5b0200 100644 --- a/tests/boundary/mcp/nativeDemangleMcp.test.ts +++ b/tests/boundary/mcp/nativeDemangleMcp.test.ts @@ -70,6 +70,14 @@ describe.skipIf(process.platform !== "darwin")( expect(JSON.stringify(spanning.content)).toContain( "Each Swift symbol must be one line.", ); + const invalidNativeString = await client.callTool({ + name: "demangle_swift", + arguments: { symbols: ["$s4main\0FooV"] }, + }); + expect(invalidNativeString.isError).toBe(true); + expect(JSON.stringify(invalidNativeString.content)).toContain( + "Swift symbols cannot contain NUL.", + ); }, ); }, diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index dc4219d14ee..520caa14b80 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -9,6 +9,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { itWithCaptureCapability } from "./processCaptureCapability.js"; import { captureProcessScenario } from "../../../src/process/capture/ProcessHarness.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { snapshotRoots } from "../../../src/process/capture/FilesystemSnapshot.js"; import { ProcessCaptureError } from "../../../src/process/capture/ProcessCaptureError.js"; import { @@ -264,6 +265,13 @@ itWithCaptureCapability( expect(result.error.executionFailure).toBe( "fixture final snapshot failure", ); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + details: { cleanup: "incomplete", resources: ["owned_process_group"] }, + }); + expect(projected).not.toHaveProperty("stack"); + expect(projected).not.toHaveProperty("cause"); expect(partial).toBeDefined(); if (partial === undefined || !("observations" in partial)) throw new Error("expected incomplete process observations"); @@ -489,6 +497,11 @@ itWithCaptureCapability( ); if (result.ok) throw new Error("expected initial snapshot cancellation"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cancelled", + category: "cancelled", + details: { operation: "process_capture", cleanup: "complete" }, + }); expect(result.error).toMatchObject({ reason: "cancelled", userCategory: "cancelled", From 6c13fc8e0a318ebe7be6b038089b350d99f8e286 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:48:05 +0800 Subject: [PATCH 061/791] fix(android): report startup and selector failure causes --- bridge/android/ReaJadxBridge.java | 19 +- docs/android-analysis.md | 12 ++ scripts/verify/android/analysis.mjs | 10 ++ scripts/verify/android/cli-cancellation.mjs | 5 +- scripts/verify/android/failure-recovery.mjs | 189 ++++++++++++++++++++ src/android/JadxMcpTransport.ts | 5 + src/android/JadxProtocol.ts | 6 + src/android/JadxProvider.ts | 18 ++ src/android/JadxSession.ts | 28 ++- src/domain/android/androidAnalysis.ts | 8 +- 10 files changed, 288 insertions(+), 12 deletions(-) create mode 100644 scripts/verify/android/failure-recovery.mjs diff --git a/bridge/android/ReaJadxBridge.java b/bridge/android/ReaJadxBridge.java index 51ac0f70325..aa49606ee64 100644 --- a/bridge/android/ReaJadxBridge.java +++ b/bridge/android/ReaJadxBridge.java @@ -25,6 +25,11 @@ class ReaJadxBridge { private record MethodSnapshot(String name, String shortId, Map summary) {} private record ClassSnapshot(JavaClass node, List methods, Map summary) {} + private static final class InputFailure extends IllegalArgumentException { + final String field; + InputFailure(String field, String message) { super(message); this.field = field; } + } + private ReaJadxBridge(SessionHolder holder) { this.holder = holder; } private static CallToolResult text(String value, boolean failed) { @@ -100,8 +105,9 @@ private ClassSnapshot selected(JsonObject args) { } String name = string(args, "class_name"); List candidates = classes.get(name); - if (candidates == null) throw new IllegalArgumentException("Exact class not found: " + name); - if (candidates.size() != 1) throw new IllegalArgumentException("Ambiguous metadata class identity: " + name + if (candidates == null) throw new InputFailure("class_name", "Exact class not found: " + name + + "; use search_android_classes to discover fully qualified names."); + if (candidates.size() != 1) throw new InputFailure("class_name", "Ambiguous metadata class identity: " + name + "; raw identities: " + candidates.stream().map(candidate -> candidate.node().getRawName()).toList()); return candidates.get(0); } @@ -109,7 +115,8 @@ private ClassSnapshot selected(JsonObject args) { private List methods(ClassSnapshot cls, JsonObject args) { String name = string(args, "method_name"); List result = cls.methods().stream().filter(method -> method.name().equals(name)).toList(); - if (result.isEmpty()) throw new IllegalArgumentException("Method not found: " + name); + if (result.isEmpty()) throw new InputFailure("method_name", "Method not found: " + name + + "; use inspect_android_class to discover available methods."); return result; } @@ -141,7 +148,7 @@ private CallToolResult handle(String name, JsonObject args) { if (name.equals("get_method_body")) { List overloads = methods(cls, args); int index = integer(args, "overload_index", 0); - if (index >= overloads.size()) throw new IllegalArgumentException("overload_index out of range: " + index); + if (index >= overloads.size()) throw new InputFailure("overload_index", "overload_index out of range: " + index); JavaMethod method = method(cls, overloads.get(index)); JadxSession.SmartCode code = current.getMethodBodySmart(method, current.getMaxSourceBytes(), true); return json(Map.of("class_name", cls.summary().get("full_name"), @@ -155,7 +162,7 @@ private CallToolResult handle(String name, JsonObject args) { if (name.equals("get_xrefs_to_class")) uses = cls.node().getUseIn(); else { List overloads = methods(cls, args); - if (overloads.size() != 1) throw new IllegalArgumentException("Method references require a unique overload"); + if (overloads.size() != 1) throw new InputFailure("method_name", "Method references require a unique overload"); target += "." + overloads.get(0).name(); uses = method(cls, overloads.get(0)).getUseIn(); } @@ -211,6 +218,8 @@ public static void main(String[] args) throws Exception { tool.getInputSchema().getRequired(), null), description, null, null, null, null, null, null); server.addTool(narrowed, (connection, request, continuation) -> { try { return bridge.handle(name, request.getArguments()); } + catch (InputFailure error) { return text(JSON.toJson(Map.of( + "kind", "invalid-input", "field", error.field, "message", error.getMessage())), true); } catch (IllegalArgumentException error) { return text(error.getMessage(), true); } }); } diff --git a/docs/android-analysis.md b/docs/android-analysis.md index d72ba49776b..4251b763a7b 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -130,6 +130,13 @@ target/configuration changes, failure, timeout, cancellation, MCP disconnect or server shutdown. One-shot CLI commands always join cleanup before returning. REA never writes to the original APK or launches target code. +If Java cannot start the bridge, REA reports the selected executable and retained +startup diagnostics. Select a full JDK through `JAVA_HOME` or `PATH`, and use +`java --list-modules` to confirm `jdk.compiler` is present. A runnable JRE alone +cannot compile the bridge. Missing or ambiguous class and method selections are +input failures; discover names with `search_android_classes` and +`inspect_android_class` before correcting the selected argument. + If process cleanup cannot be confirmed, REA retains the workspace and reports its location. That provider instance blocks subsequent engine launches until the caller resolves the reported resources and starts a fresh instance. @@ -177,6 +184,11 @@ export REA_ANDROID_TEST_APK="$PWD/_reference/apk-integration/ApiDemos-debug.apk" npm run verify:android ``` +The lane also uses the selected full JDK's `jlink` to create a disposable JRE +without the compiler module. It checks real Java startup failures, missing +selectors, recovery after failures, and CLI cancellation with owned cleanup. +The original APK and host Java configuration remain unchanged. + The download script verifies fixed SHA-256 values, reuses matching files and refuses to overwrite a different existing file. APK/JAR files live under ignored `_reference/` and are excluded from package contents. The lane exercises actual diff --git a/scripts/verify/android/analysis.mjs b/scripts/verify/android/analysis.mjs index 90ff16e0c6c..2c42e7696c8 100644 --- a/scripts/verify/android/analysis.mjs +++ b/scripts/verify/android/analysis.mjs @@ -12,6 +12,7 @@ import { JADX_RELEASE } from "../../../dist/android/JadxRelease.js"; import fixture from "../../fixtures/android/apidemos.json" with { type: "json" }; import { verifyAndroidCliCancellation } from "./cli-cancellation.mjs"; import { verifyAndroidMetadata } from "./metadata.mjs"; +import { verifyAndroidFailureRecovery } from "./failure-recovery.mjs"; const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); const apk = process.env.REA_ANDROID_TEST_APK; @@ -206,6 +207,15 @@ await verifyAndroidCliCancellation({ repository, execute, }); +await verifyAndroidFailureRecovery({ + java, + entrypoint, + environment, + path, + repository, + execute, + fixture, +}); assert.equal( await hashAndroidFile(apk), fixture.sha256, diff --git a/scripts/verify/android/cli-cancellation.mjs b/scripts/verify/android/cli-cancellation.mjs index 849b0c59a3b..bc75e8aa141 100644 --- a/scripts/verify/android/cli-cancellation.mjs +++ b/scripts/verify/android/cli-cancellation.mjs @@ -13,6 +13,9 @@ export const verifyAndroidCliCancellation = async ({ repository, execute, }) => { + // Native identity inspection must be ready before observing the short-lived + // engine; compiling its helper after acquisition can miss the whole request. + await readProcessRunId(process.pid); const child = spawn( process.execPath, [entrypoint, "inspect-android-package", path, "--format", "json"], @@ -86,7 +89,7 @@ export const verifyAndroidCliCancellation = async ({ ]).finally(() => clearTimeout(timer)); assert.equal(result.code, 143, `${stdout}\n${stderr}`); assert.equal(result.signal, null); - assert.equal(JSON.parse(stdout).code, "cancelled"); + assert.equal(JSON.parse(stdout).code, "cancelled", `${stdout}\n${stderr}`); assert.throws(() => process.kill(-group, 0), { code: "ESRCH" }); console.log( "PASS real CLI SIGTERM cancellation and owned Java group cleanup", diff --git a/scripts/verify/android/failure-recovery.mjs b/scripts/verify/android/failure-recovery.mjs new file mode 100644 index 00000000000..8402eab597c --- /dev/null +++ b/scripts/verify/android/failure-recovery.mjs @@ -0,0 +1,189 @@ +import assert from "node:assert/strict"; +import { access, mkdtemp, mkdir, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, isAbsolute, join } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; + +/** Real unusable Java runtimes and selectors must lead to applicable recovery. */ +export async function verifyAndroidFailureRecovery({ + java, + entrypoint, + environment, + path, + repository, + execute, + fixture, +}) { + const root = await mkdtemp(join(tmpdir(), "rea-android-recovery-")); + try { + let jlink; + for (const candidate of isAbsolute(java) + ? [join(dirname(java), "jlink")] + : (environment.PATH ?? "") + .split(delimiter) + .map((directory) => join(directory, "jlink"))) { + try { + await access(candidate); + jlink = candidate; + break; + } catch (cause) { + if (cause.code !== "ENOENT" && cause.code !== "ENOTDIR") throw cause; + } + } + if (jlink === undefined) + throw new Error( + "Android failure-recovery verification requires the full JDK's jlink via JAVA_HOME or PATH.", + ); + const jre = join(root, "jre"); + await execute(jlink, ["--add-modules", "java.base", "--output", jre], { + timeout: 60_000, + }); + await execute(join(jre, "bin/java"), ["-version"], { timeout: 10_000 }); + const empty = join(root, "empty-path"); + await mkdir(empty); + if (process.platform === "linux") { + // Keep the real ownership prerequisite available while removing Java. + const { stdout } = await execute("/bin/sh", ["-c", "command -v ps"], { + env: environment, + timeout: 10_000, + }); + const ps = stdout.trim(); + assert.ok( + isAbsolute(ps), + "Linux recovery verification requires procps ps", + ); + await symlink(ps, join(empty, "ps")); + } + const withoutJavaHome = { ...environment }; + delete withoutJavaHome.JAVA_HOME; + const environments = [ + { + label: "JRE without compiler", + env: { ...environment, JAVA_HOME: jre }, + observation: "jdk.compiler", + }, + { + label: "missing java", + env: { ...withoutJavaHome, PATH: empty }, + observation: "ENOENT", + }, + ]; + if (process.platform === "darwin") { + // /usr/bin/java either resolves an installed runtime or reports the actual stub failure. + try { + await execute("/usr/bin/java", ["-version"], { timeout: 10_000 }); + } catch { + environments.push({ + label: "macOS Java stub", + env: { ...withoutJavaHome, PATH: "/usr/bin:/bin" }, + observation: "Unable to locate a Java Runtime", + }); + } + } + for (const scenario of environments) { + let failure; + try { + await execute( + process.execPath, + [entrypoint, "inspect-android-package", path, "--json"], + { + cwd: repository, + env: scenario.env, + timeout: 45_000, + maxBuffer: 8 * 1024 * 1024, + }, + ); + assert.fail(`${scenario.label} unexpectedly succeeded`); + } catch (cause) { + assert.equal(cause.code, 1); + failure = JSON.parse(cause.stdout); + } + assert.equal(failure.code, "capability_unavailable"); + assert.ok( + JSON.stringify(failure).includes(scenario.observation), + JSON.stringify(failure), + ); + assert.match(failure.message, /full JDK/u); + assert.match(failure.remediation.action, /java --list-modules/u); + assert.doesNotMatch(JSON.stringify(failure), /rea doctor/u); + console.log(`PASS CLI applicable recovery: ${scenario.label}`); + } + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + cwd: repository, + env: environment, + stderr: "pipe", + }); + const client = new Client({ + name: "rea-real-android-failures", + version: "1", + }); + try { + await client.connect(transport); + for (const name of [ + "inspect_android_class", + "inspect_android_method", + "trace_android_references", + ]) { + const response = await client.callTool( + { + name, + arguments: { + path, + class_name: "no.such.Class", + ...(name === "inspect_android_method" + ? { method_name: "missing" } + : {}), + }, + }, + { timeout: 150_000 }, + ); + assert.equal(response.isError, true); + assert.equal(response.structuredContent.error.code, "invalid_request"); + assert.deepEqual( + response.structuredContent.error.details.issues[0].path, + ["class_name"], + ); + assert.match( + response.structuredContent.error.details.issues[0].message, + /search_android_classes/u, + ); + assert.doesNotMatch(JSON.stringify(response), /rea doctor/u); + console.log(`PASS MCP selector correction: ${name}`); + } + const method = await client.callTool( + { + name: "inspect_android_method", + arguments: { + path, + class_name: fixture.class_name, + method_name: "noSuchMethod", + }, + }, + { timeout: 150_000 }, + ); + assert.equal(method.structuredContent.error.code, "invalid_request"); + assert.deepEqual(method.structuredContent.error.details.issues[0].path, [ + "method_name", + ]); + await client.ping(); + const control = await client.callTool( + { name: "inspect_android_package", arguments: { path } }, + { timeout: 150_000 }, + ); + assert.notEqual(control.isError, true, JSON.stringify(control)); + assert.equal( + control.structuredContent.result.package_name, + fixture.package, + ); + console.log("PASS MCP valid package after selector failures"); + } finally { + await client.close(); + await transport.close(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/src/android/JadxMcpTransport.ts b/src/android/JadxMcpTransport.ts index a2dc084b8a2..f99aacff9cd 100644 --- a/src/android/JadxMcpTransport.ts +++ b/src/android/JadxMcpTransport.ts @@ -109,6 +109,11 @@ export class JadxMcpTransport implements Transport { return this.#supervisor?.snapshot().stderr.text ?? ""; } + /** Whether the retained startup/operation diagnostics lost bytes. */ + diagnosticsTruncated(): boolean { + return this.#supervisor?.snapshot().diagnosticTruncated ?? false; + } + /** Preserve the failed protocol/resource constraint after the SDK closes requests. */ failureReason(): string | null { return this.#failure; diff --git a/src/android/JadxProtocol.ts b/src/android/JadxProtocol.ts index dd8933d291a..804b7ffa7e6 100644 --- a/src/android/JadxProtocol.ts +++ b/src/android/JadxProtocol.ts @@ -4,6 +4,12 @@ import { jsonValueSchema } from "../domain/jsonValue.js"; import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; const count = z.number().int().nonnegative(); +/** Selector failures authored by REA's bridge, separate from arbitrary engine errors. */ +export const jadxInputFailureSchema = z.strictObject({ + kind: z.literal("invalid-input"), + field: z.enum(["class_name", "method_name", "overload_index"]), + message: z.string().min(1), +}); /** Values observed inside the exact JVM producing the analysis. */ export const jadxRuntimeSchema = z.object({ engine_reported_version: z.string().min(1), diff --git a/src/android/JadxProvider.ts b/src/android/JadxProvider.ts index 9c0be14d93f..cde877313bc 100644 --- a/src/android/JadxProvider.ts +++ b/src/android/JadxProvider.ts @@ -73,6 +73,24 @@ const executionError = (context: { "JADX response violated the pinned upstream protocol", { cause }, ); + if (session !== undefined && !session.initialized()) { + const stderr = session.transport.diagnostics(); + const reason = `Java could not start REA's Android bridge using ${session.transport.options.command}: ${cause instanceof Error ? cause.message : String(cause)}.${stderr.length === 0 ? "" : ` Java diagnostics: ${stderr}`} Select a full JDK including jdk.compiler via JAVA_HOME, or a working java on PATH; verify it with java --list-modules. A JRE cannot compile REA's source bridge. If the JDK works, verify the REA bridge and configured JADX JAR installation.`; + return new AnalysisCapabilityUnavailableError( + "jadx", + request.operation, + reason, + { + cause, + userMessage: reason, + capturedOutput: { + stdout: "", + stderr, + truncated: session.transport.diagnosticsTruncated(), + }, + }, + ); + } return new ProviderAdapterError("jadx", request.operation, { cause, diagnostics: { diff --git a/src/android/JadxSession.ts b/src/android/JadxSession.ts index 87308b1d888..a2436de30ba 100644 --- a/src/android/JadxSession.ts +++ b/src/android/JadxSession.ts @@ -6,6 +6,7 @@ import { import type { AndroidRequest } from "../domain/android/androidAnalysis.js"; import { AnalysisCapabilityUnavailableError, + AnalysisInputError, AnalysisOutputError, } from "../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../domain/binaryTarget.js"; @@ -16,6 +17,7 @@ import { analyzeJadxRequest, type JadxToolPort } from "./JadxAnalysis.js"; import { JadxMcpTransport, type JadxLauncher } from "./JadxMcpTransport.js"; import { jadxLoadSchema, + jadxInputFailureSchema, jadxRuntimeSchema, parseJadxEnvelope, parseJadxJson, @@ -32,6 +34,7 @@ export class JadxSession { readonly transport: JadxMcpTransport; readonly #client = new Client({ name: "rea-android-adapter", version: "1" }); #loaded: ReturnType | undefined; + #connected = false; constructor( options: Omit, @@ -58,8 +61,10 @@ export class JadxSession { signal.throwIfAborted(); this.transport.beginOperation(); const raw: JsonValue[] = []; - if (this.#loaded === undefined) + if (!this.#connected) { await this.#client.connect(this.transport, { timeout: 30_000 }); + this.#connected = true; + } const server = this.#client.getServerVersion(); if ( server?.name !== JADX_BRIDGE_IDENTITY.name || @@ -126,6 +131,11 @@ export class JadxSession { } } + /** Whether the selected Java runtime established the REA bridge protocol. */ + initialized(): boolean { + return this.#connected; + } + /** Join owned cleanup even if SDK connection teardown itself fails. */ async close(): Promise { try { @@ -156,7 +166,20 @@ export class JadxSession { input, response: jsonValueSchema.parse(response), }); - if (envelope.failed) + if (envelope.failed) { + if (envelope.text.startsWith("{")) { + const failure = jadxInputFailureSchema.safeParse( + parseJadxJson(envelope.text, request.operation), + ); + if (failure.success) + throw new AnalysisInputError(request.operation, undefined, [ + { + path: [failure.data.field], + reason: "invalid_value", + message: failure.data.message, + }, + ]); + } throw new ProviderAdapterError("jadx", request.operation, { diagnostics: { upstream_operation: name, @@ -165,6 +188,7 @@ export class JadxSession { target_sha256: target.sha256, }, }); + } return envelope.text; }; return { diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index 7d9f2c9766b..a692001d9e2 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -1,10 +1,10 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; const target = { - path: z - .string() - .min(1) - .describe("Local APK path; the target is never executed"), + path: localPathStringSchema.describe( + "Local APK path; the target is never executed", + ), }; const classTarget = { ...target, From 44fd18305fa5a8893ebb5b952099fdf45dbd6e55 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:48:13 +0800 Subject: [PATCH 062/791] fix(capture): retain Electron observations and HAR resource failures --- docs/electron-observation.md | 36 +- docs/web-network-captures.md | 9 + scripts/electron-active-hook-boundaries.cjs | 6 +- scripts/electron-active-hook.cjs | 64 ++- scripts/verify-real-electron.mjs | 5 +- scripts/verify/electron/tool-contracts.mjs | 364 +++++++++++++++++ .../PlaywrightElectronActiveActions.ts | 61 +-- .../PlaywrightElectronActiveProjection.ts | 10 +- .../PlaywrightElectronActiveProvider.ts | 121 +++++- src/browser/history/CaptureFailures.ts | 55 +++ src/browser/history/CaptureRelease.ts | 6 + src/browser/history/HarCaptureAdapter.ts | 9 +- .../history/HistoricalCaptureDecoder.ts | 9 +- src/cli/electronCommands.ts | 63 +-- .../javascript/electronToolContracts.ts | 2 +- src/domain/analysisErrorBase.ts | 4 +- src/domain/analysisErrorPresentation.ts | 4 +- src/domain/analysisErrorProjection.ts | 6 +- src/domain/browserObservationError.ts | 38 +- src/domain/explicitSensitiveFailure.ts | 23 ++ .../javascript/electronActiveObservation.ts | 23 +- .../tools/historicalCaptureResources.test.ts | 115 ++++++ .../tools/inspectorLifecycle.test.ts | 126 ++++++ .../tools/localToolContracts.test.ts | 379 ++++++++++++++++++ .../browser/electronActiveHook.test.ts | 5 +- .../browser/historicalCaptureDecoder.test.ts | 2 +- tests/fixtures/localToolsMcp.ts | 62 +++ 27 files changed, 1462 insertions(+), 145 deletions(-) create mode 100644 scripts/verify/electron/tool-contracts.mjs create mode 100644 tests/acceptance/tools/historicalCaptureResources.test.ts create mode 100644 tests/acceptance/tools/inspectorLifecycle.test.ts create mode 100644 tests/acceptance/tools/localToolContracts.test.ts create mode 100644 tests/fixtures/localToolsMcp.ts diff --git a/docs/electron-observation.md b/docs/electron-observation.md index 62ade70954c..37c38712c70 100644 --- a/docs/electron-observation.md +++ b/docs/electron-observation.md @@ -86,25 +86,33 @@ rea capture-electron-scenario scenario.json --json The result records action status and targets, correlated app/window/WebContents, preload, session, navigation, shell, permission, popup, download, protocol, -native-addon, process, and IPC timeline events. Event retention has a 16 MiB -accounting budget for full channel strings, argument-shape arrays, and an -estimated object-graph cost. Admission computes an exact JSON byte length -incrementally, then charges twice that size plus 256 bytes per event and 8 bytes -per argument shape. The doubled JSON charge covers the timeline and IPC views; -the additional charge estimates their in-process object and array storage. This -is a resource estimate, not an exact V8 heap measurement. An event that does not -fit is dropped whole. The result reports the estimate, serialized-byte upper -bound, dropped event families, and -observed/dropped totals for IPC and runtime events. Retained sequence numbers -keep their original capture identities. Payload values are never retained. +native-addon, process, and IPC timeline events. IPC channels and argument-shape +metadata retain their observed lengths; payload values are never retained. Renderer crash/restart and deep-link actions are synthetic scenario controls; their attempted and observed outcomes remain in the timeline. The active hook blocks and records external shell/navigation, permission, download, popup, updater, and OS-integration effects. The timeline is explicitly partial when attachment starts after application activity. The -owned experiment has an internal 60-second deadline and a 5-second per-action -timeout so hung work reaches process cleanup; these are lifecycle timeouts, not -caller-selected capture limits. +startup has a 60-second deadline. Actions wait until completion or request +cancellation; a click whose target never appears remains pending until cancelled. +CLI SIGINT/SIGTERM requests cancellation and awaits owned-process cleanup. +Window IDs, URLs, and cached titles come from one main-process Electron snapshot, +so a crashed renderer does not require a successful renderer-side title query. +Window instrumentation uses Electron's window-created event, including runtimes +with a read-only `BrowserWindow` export. Preload configuration comes from the +window's runtime metadata, including Electron's optional internal preload +descriptor when web preferences omit it. This descriptor is verified on +Electron 44.7.0; runtimes that expose neither representation retain an unavailable +path. A configured path or working IPC handler does not prove preload execution. +If cleanup cannot be verified, the error retains collected observations in +`details.partial_observation` and marks their application cleanup `unverified`. +Cleanup uncertainty does not authorize signaling unrelated processes. + +The real verifier exercises compiled CLI and stdio MCP discovery, page/script +inspection, static/runtime reconciliation, active observation, and CLI SIGTERM +cleanup before checking the IPC, renderer crash/reload, utility/child-process, +deep-link, and blocked-effect fixture. A cleanup error keeps the full lane failed +even when the retained capture contains useful observations. Active Electron Evidence can also be supplied to [`reconcile_javascript_runtime`](javascript-runtime-reconciliation.md). That diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index 01c2041c6f7..4e91c2f1257 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -126,6 +126,15 @@ process cleanup. Owned decoder diagnostics retain at most 1 MiB across stdout and stderr; the collector drops overflow as it arrives, reports the limit failure and verifies cleanup. Heap limits alone do not establish aggregate RSS limits. +Observed decoder memory exhaustion returns `resource_constraint`, with the +selected capture, applicable limits and retained process diagnostics. HAR's +old-generation heap is fixed; inherited `NODE_OPTIONS` cannot raise it. +Record selection happens after complete decoding and does not reduce that +workload. Use a smaller capture exported by its producer or another decoder +with sufficient capacity, retaining the original for provenance. Subsets do +not establish complete-capture coverage. Repeating the same workload or +running provider health diagnostics does not resolve the heap constraint. + Stable regular-file reads reject symlinks, replacement and concurrent changes. Private snapshots, declarations and tool configuration are removed before the result returns. Cleanup uncertainty retains the selected capture, resource and diff --git a/scripts/electron-active-hook-boundaries.cjs b/scripts/electron-active-hook-boundaries.cjs index 81d199a188d..4bec74dce09 100644 --- a/scripts/electron-active-hook-boundaries.cjs +++ b/scripts/electron-active-hook-boundaries.cjs @@ -195,9 +195,11 @@ const createElectronActiveBoundaryPatches = ({ }; }; - const patchSessionManager = (sessionModule) => { + const patchSessionManager = (sessionModule, app) => { if (sessionModule === null || sessionModule === undefined) return; - patchSession(sessionModule.defaultSession); + if (app === undefined || app.isReady()) + patchSession(sessionModule.defaultSession); + else app.once("ready", () => patchSession(sessionModule.defaultSession)); for (const method of ["fromPartition", "fromPath"]) { const original = sessionModule[method]; if (typeof original !== "function") continue; diff --git a/scripts/electron-active-hook.cjs b/scripts/electron-active-hook.cjs index 8d7766c9560..022acd30a6a 100644 --- a/scripts/electron-active-hook.cjs +++ b/scripts/electron-active-hook.cjs @@ -543,31 +543,44 @@ const patchWebContents = (webContents, windowId) => { boundaries.patchWindowOpenHandler(webContents, contentsId); }; -const patchBrowserWindow = (electron) => { - const original = electron.BrowserWindow; - if (typeof original !== "function") return; - electron.BrowserWindow = new Proxy(original, { - construct(target, args, newTarget) { - const window = Reflect.construct(target, args, newTarget); - const targetId = identity(window, "window"); - recordRuntime("window-lifecycle", "created", "completed", { - target: targetId, - argument_shape_values: args, - }); - const preload = args[0]?.webPreferences?.preload; +const patchWindows = (app) => { + if (app === null || app === undefined) return; + app.on("browser-window-created", (_event, window) => { + const targetId = identity(window, "window"); + recordRuntime("window-lifecycle", "created", "completed", { + target: targetId, + capture_method: "event-emitter", + }); + try { + // Electron 44 separates the preload descriptor from WebPreferences. + // Read the engine's descriptor, never infer a path from application code. + const preferences = window.webContents.getLastWebPreferences?.(); + const preload = + typeof preferences?.preload === "string" + ? preferences.preload + : window.webContents._getPreloadScript?.()?.filePath; if (typeof preload === "string") recordRuntime("preload", "configured", "completed", { target: targetId, artifact_path: preload, }); - patchEmitter(window, (event) => - recordRuntime("window-lifecycle", event, "observed", { + } catch (cause) { + recordRuntime( + "error", + cause instanceof Error ? cause.message : String(cause), + "failed", + { target: targetId, - }), + error: true, + }, ); - patchWebContents(window.webContents, targetId); - return window; - }, + } + patchEmitter(window, (event) => + recordRuntime("window-lifecycle", event, "observed", { + target: targetId, + }), + ); + patchWebContents(window.webContents, targetId); }); }; @@ -599,10 +612,10 @@ try { patchIpcMain(electron.ipcMain); boundaries.patchIpcRenderer(electron.ipcRenderer); boundaries.patchContextBridge(electron.contextBridge); - patchBrowserWindow(electron); + patchWindows(electron.app); boundaries.patchUtilityProcess(electron.utilityProcess); boundaries.patchShell(electron.shell); - boundaries.patchSessionManager(electron.session); + boundaries.patchSessionManager(electron.session, electron.app); boundaries.patchApplicationEffects(electron.app); boundaries.patchNativeAddonLoading(); boundaries.patchChildProcess(); @@ -620,8 +633,17 @@ try { }); throw reason; }); -} catch { +} catch (cause) { globalThis.__reaElectronActiveHookError = true; + recordRuntime( + "error", + cause instanceof Error ? cause.message : String(cause), + "failed", + { + target: "active-hook-installation", + error: true, + }, + ); } globalThis.__reaElectronActiveSnapshot = () => ({ diff --git a/scripts/verify-real-electron.mjs b/scripts/verify-real-electron.mjs index bf6f2a169e3..177ee8e0813 100644 --- a/scripts/verify-real-electron.mjs +++ b/scripts/verify-real-electron.mjs @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; import { PlaywrightElectronActiveProvider } from "../dist/browser/PlaywrightElectronActiveProvider.js"; import { electronActiveObservationInputSchema } from "../dist/domain/javascript/electronActiveObservation.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; +import { verifyElectronToolContracts } from "./verify/electron/tool-contracts.mjs"; const executable = process.env.REA_ELECTRON_EXECUTABLE; if (executable === undefined || executable.length === 0) @@ -14,6 +15,7 @@ if (executable === undefined || executable.length === 0) ); const repositoryRoot = fileURLToPath(new URL("..", import.meta.url)); +await verifyElectronToolContracts({ executable, repositoryRoot }); const applicationRoot = process.env.REA_ELECTRON_APPLICATION_ROOT ?? join(repositoryRoot, "tests/conformance/readiness/electron"); @@ -60,6 +62,7 @@ const output = { timeline: result.value.timeline, windows: result.value.windows, verified: + result.value.coverage.status === "partial_attach" && result.value.actions.every(({ status }) => status === "completed") && result.value.actions.some( ({ kind, window_index }) => @@ -104,8 +107,8 @@ const output = { kind === "process-lifecycle" && event === "child.spawn", ), }; +process.stdout.write(`${JSON.stringify(output)}\n`); if (!output.verified) throw new Error( "real Electron verifier did not observe the readiness IPC round trip", ); -process.stdout.write(`${JSON.stringify(output)}\n`); diff --git a/scripts/verify/electron/tool-contracts.mjs b/scripts/verify/electron/tool-contracts.mjs new file mode 100644 index 00000000000..7404238dc98 --- /dev/null +++ b/scripts/verify/electron/tool-contracts.mjs @@ -0,0 +1,364 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { access, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import Ajv2020 from "ajv/dist/2020.js"; + +import { + cleanupOwnedProcessGroup, + readProcessRunId, +} from "../../../dist/process/ProcessOwnership.js"; + +const execute = promisify(execFile); + +/** Exercise public Electron tools and cancellation using a real owned application. */ +export async function verifyElectronToolContracts({ + executable, + repositoryRoot, +}) { + const root = await realpath( + await mkdtemp(join(tmpdir(), "rea-electron-e2e-")), + ); + const main = join(root, "main.cjs"); + const ready = join(root, "ready"); + const entrypoint = join(repositoryRoot, "scripts", "rea.mjs"); + const runId = randomUUID(); + await writeFile( + main, + `const {app,BrowserWindow}=require("electron"); +app.whenReady().then(()=>{const window=new BrowserWindow(); +window.loadFile("renderer.html");require("node:fs").writeFileSync(${JSON.stringify(ready)},"ready");});`, + ); + await writeFile( + join(root, "renderer.html"), + '', + ); + await writeFile( + join(root, "renderer.js"), + 'document.querySelector("#run").addEventListener("click",()=>console.log("fixture-click"));', + ); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env: { + PATH: process.env.PATH, + REA_LOG_LEVEL: "silent", + HOPPER_LAUNCHER_PATH: "/rea-unconfigured/hopper", + }, + stderr: "pipe", + }); + const client = new Client({ name: "electron-real-e2e", version: "1" }); + let serverStderr = ""; + transport.stderr?.on("data", (chunk) => { + serverStderr += chunk.toString(); + }); + let application; + let stderr = ""; + try { + await readProcessRunId(process.pid); + application = spawn( + executable, + [ + "--remote-debugging-address=127.0.0.1", + "--remote-debugging-port=0", + `--user-data-dir=${join(root, "profile")}`, + main, + ], + { + cwd: root, + env: { ...process.env, REA_PROCESS_RUN_ID: runId }, + detached: true, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + application.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + const endpoint = await waitForEndpoint(application, () => stderr); + await client.connect(transport); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const validators = new Map( + (await client.listTools()).tools.map((tool) => { + ajv.compile(tool.inputSchema); + return [ + tool.name, + tool.outputSchema ? ajv.compile(tool.outputSchema) : undefined, + ]; + }), + ); + const call = async (name, arguments_) => { + console.log(`PROBE ${name}`); + const result = await client.callTool( + { name, arguments: arguments_ }, + { timeout: 150_000 }, + ); + assert.notEqual(result.isError, true, JSON.stringify(result)); + const validate = validators.get(name); + assert.ok( + validate?.(result.structuredContent), + JSON.stringify(validate?.errors), + ); + return result.structuredContent; + }; + let targets; + for (let attempt = 0; attempt < 30; attempt++) { + targets = await call("list_electron_targets", { cdp_endpoint: endpoint }); + if (targets.result.targets.length > 0) break; + await pause(100); + } + const target = targets.result.targets.find( + (item) => item.file_path === join(root, "renderer.html"), + ); + assert.ok(target, JSON.stringify(targets)); + const page = await call("inspect_electron_page", { + cdp_endpoint: endpoint, + target_id: target.target_id, + include_script_sources: true, + observation_ms: 50, + }); + assert.ok( + page.result.scripts.items.some( + (script) => + script.file_path === join(root, "renderer.js") && + script.source.included, + ), + ); + const staticAnalysis = await call("analyze_javascript_application", { + input_path: root, + }); + const reconciled = await call("reconcile_javascript_runtime", { + static_layers: [ + { + analysis: staticAnalysis.evidence, + runtime_mappings: [{ kind: "file-root", root }], + }, + ], + runtime_observations: [page.evidence], + }); + assert.ok(reconciled.result.runtime_captures.length > 0); + const cli = await execute( + process.execPath, + [ + entrypoint, + "inspect-electron-page", + endpoint, + target.target_id, + "--include-script-sources", + "--json", + ], + { + env: { ...process.env, REA_LOG_LEVEL: "silent" }, + timeout: 60_000, + maxBuffer: 32 * 1024 * 1024, + }, + ); + assert.equal( + JSON.parse(cli.stdout).normalized_result.target.file_path, + target.file_path, + ); + const input = { + executable_path: executable, + application_path: main, + application_root: root, + actions: [{ step_id: "settle", kind: "wait", duration_ms: 50 }], + }; + const active = await client.callTool( + { name: "capture_electron_scenario", arguments: input }, + { timeout: 150_000 }, + ); + const capture = active.isError + ? active.structuredContent.error.details.partial_observation.capture + : active.structuredContent.result; + if (active.isError) { + assert.equal( + active.structuredContent.error.code, + "cleanup_incomplete", + JSON.stringify(active), + ); + assert.equal(capture.application.cleanup, "unverified"); + } else { + assert.ok( + validators.get("capture_electron_scenario")(active.structuredContent), + ); + assert.equal(capture.application.cleanup, "terminated-owned-process"); + } + assert.equal(capture.actions[0].status, "completed"); + assert.equal( + capture.coverage.status, + "partial_attach", + JSON.stringify(capture), + ); + assert.ok( + capture.timeline.events.some( + (event) => + event.kind === "window-lifecycle" && event.event === "created", + ), + ); + await client.ping(); + console.log( + `PASS live Electron CLI/MCP discovery, inspection, reconciliation, and active observations; active cleanup=${capture.application.cleanup}`, + ); + if (process.platform !== "win32") + await verifyCancellation({ input, ready, entrypoint, root }); + } catch (cause) { + throw new Error( + `Electron public-tool verification failed: ${cause.message}\nMCP stderr:\n${serverStderr}`, + { cause }, + ); + } finally { + await client.close(); + await transport.close(); + if (application?.pid) { + const cleanup = await cleanupOwnedProcessGroup({ + runId, + leaderPid: application.pid, + processGroupId: application.pid, + expectedParentPid: process.pid, + expectedCommand: executable, + }); + assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); + } + await rm(root, { recursive: true, force: true }); + } +} + +const pause = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + +async function waitForEndpoint(application, diagnostics) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const match = /DevTools listening on ws:\/\/127\.0\.0\.1:(\d+)\//u.exec( + diagnostics(), + ); + if (match) return `http://127.0.0.1:${match[1]}`; + assert.equal(application.exitCode, null, diagnostics()); + await pause(100); + } + throw new Error( + `Electron did not expose a loopback endpoint: ${diagnostics()}`, + ); +} + +async function verifyCancellation({ input, ready, entrypoint, root }) { + await rm(ready, { force: true }); + const child = spawn( + process.execPath, + [ + entrypoint, + "capture-electron-scenario", + JSON.stringify({ + ...input, + actions: [{ step_id: "wait", kind: "wait", duration_ms: 30_000 }], + }), + "--json", + ], + { + cwd: root, + env: { ...process.env, REA_LOG_LEVEL: "silent" }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolve({ code, signal })); + }); + let ownership; + try { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + const table = ( + await execute("ps", ["-eo", "pid=,ppid=,pgid=,args="], { + timeout: 5_000, + maxBuffer: 2 * 1024 * 1024, + }) + ).stdout; + for (const row of table.split("\n")) { + const entry = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/u.exec(row); + if ( + entry && + Number(entry[2]) === child.pid && + entry[4].includes(input.application_path) + ) { + const runId = await readProcessRunId(Number(entry[1])); + assert.ok(runId, "Electron child lacked its ownership token"); + ownership = { + runId, + leaderPid: Number(entry[1]), + processGroupId: Number(entry[3]), + }; + } + } + if (ownership) { + try { + await access(ready); + break; + } catch { + /* wait for actual fixture readiness */ + } + } + assert.equal(child.exitCode, null, `${stdout}\n${stderr}`); + await pause(100); + } + assert.ok(ownership, "CLI did not acquire an owned Electron application"); + await access(ready); + // The fixture's readiness precedes Playwright's Inspector acquisition; + // interrupt the long action rather than its separate startup boundary. + await pause(1_000); + child.kill("SIGTERM"); + let timer; + const result = await Promise.race([ + exited, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("Electron CLI cancellation timed out")), + 15_000, + ); + }), + ]).finally(() => clearTimeout(timer)); + assert.equal(result.code, 143, `${stdout}\n${stderr}`); + assert.equal(result.signal, null); + const error = JSON.parse(stdout); + assert.ok(["cancelled", "cleanup_incomplete"].includes(error.code), stdout); + assert.equal(error.category, "cancelled", stdout); + assert.doesNotMatch(error.message, /rea doctor/u); + assert.equal( + error.details?.partial_observation?.kind, + "electron-active-observation", + stdout, + ); + assert.ok( + error.details.partial_observation.capture.actions.some( + (action) => action.status === "cancelled", + ), + ); + assert.throws(() => process.kill(-ownership.processGroupId, 0), { + code: "ESRCH", + }); + console.log( + `PASS real Electron CLI SIGTERM, retained cancelled actions, and owned group termination; reported code=${error.code}`, + ); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + await exited; + } + if (ownership) { + const cleanup = await cleanupOwnedProcessGroup(ownership); + assert.equal(cleanup.cleaned, true, JSON.stringify(cleanup)); + } + } +} diff --git a/src/browser/PlaywrightElectronActiveActions.ts b/src/browser/PlaywrightElectronActiveActions.ts index bb1469e4650..46cb72c25f2 100644 --- a/src/browser/PlaywrightElectronActiveActions.ts +++ b/src/browser/PlaywrightElectronActiveActions.ts @@ -103,6 +103,8 @@ const metricSchema = z.strictObject({ const windowMetadataSchema = z.strictObject({ window_id: z.string(), web_contents_id: z.string(), + url: z.string(), + title: z.string(), visible: z.boolean().nullable(), destroyed: z.boolean(), }); @@ -114,11 +116,6 @@ export type ElectronHookSnapshot = z.infer; export type ElectronHookEvent = z.infer; export type ElectronMetrics = z.infer[]; type ElectronWindowMetadata = z.infer; -type ElectronPageSnapshot = { - readonly index: number; - readonly url: string; - readonly title: string; -}; /** Run explicit actions against provider-owned Electron windows. */ export const runElectronActions = async ( @@ -128,12 +125,12 @@ export const runElectronActions = async ( ): Promise => { const actions: ElectronActions = []; for (const action of input.actions) { - if (options.signal?.aborted === true) - throw new BrowserObservationError(OPERATION, "cancelled"); const actionStartedAt = Date.now(); const windowIndex = actionWindowIndex(action); let selectedWindow: Page | undefined; try { + if (options.signal?.aborted === true) + throw new BrowserObservationError(OPERATION, "cancelled"); selectedWindow = windowIndex === null ? undefined @@ -322,25 +319,23 @@ export const readApplicationState = async ( readonly electronVersion: string; readonly hookSnapshot: ElectronHookSnapshot; }> => { - const pages = application.windows(); - const pageSnapshotsPromise: Promise = Promise.all( - pages.map(async (page: Page, index): Promise => ({ - index, - url: page.url(), - title: await page.title(), - })), - ); const windowMetadataPromise: Promise = application .evaluate(({ BrowserWindow }) => BrowserWindow.getAllWindows().map( (window: { readonly id: number; - readonly webContents: { readonly id: number }; + readonly webContents: { + readonly id: number; + getURL(): string; + getTitle(): string; + }; isDestroyed(): boolean; isVisible(): boolean; }) => ({ window_id: `window:${String(window.id)}`, web_contents_id: `webContents:${String(window.webContents.id)}`, + url: window.webContents.getURL(), + title: window.webContents.getTitle(), visible: window.isDestroyed() ? null : window.isVisible(), destroyed: window.isDestroyed(), }), @@ -392,33 +387,15 @@ export const readApplicationState = async ( }; }) .then((value) => hookSnapshotSchema.parse(value)); - const [ - pageSnapshots, - windowMetadata, - processMetrics, - electronVersion, - hookSnapshot, - ] = await Promise.all([ - pageSnapshotsPromise, - windowMetadataPromise, - processMetricsPromise, - electronVersionPromise, - hookSnapshotPromise, - ]); + const [windowMetadata, processMetrics, electronVersion, hookSnapshot] = + await Promise.all([ + windowMetadataPromise, + processMetricsPromise, + electronVersionPromise, + hookSnapshotPromise, + ]); return { - windows: pageSnapshots.map((page) => { - const metadata = windowMetadata[page.index]; - if (metadata === undefined) - throw new BrowserObservationError(OPERATION, "window_metadata_missing"); - return { - window_id: metadata.window_id, - web_contents_id: metadata.web_contents_id, - url: page.url, - title: page.title, - visible: metadata.visible, - destroyed: metadata.destroyed, - }; - }), + windows: windowMetadata, metrics: processMetrics, electronVersion, hookSnapshot, diff --git a/src/browser/PlaywrightElectronActiveProjection.ts b/src/browser/PlaywrightElectronActiveProjection.ts index d105d560635..e7b79d633f3 100644 --- a/src/browser/PlaywrightElectronActiveProjection.ts +++ b/src/browser/PlaywrightElectronActiveProjection.ts @@ -58,7 +58,10 @@ const eventFamily = ( kind: ElectronHookEvent["kind"], processType: string | null, ): string => { - if (kind === "preload" && processType === "main") + if ( + kind === "preload" && + (processType === "main" || processType === "browser") + ) return "preload-configuration"; if (kind.startsWith("ipc-renderer")) return processType === "renderer" ? "renderer-ipc" : "ipc"; @@ -163,6 +166,11 @@ export const projectElectronActiveCapture = ( "IPC direction and sender/receiver identifiers are observed only where Electron exposes them at the hooked boundary.", "The runtime timeline records lifecycle, navigation, shell, permission, popup, download, protocol, preload, native-addon, process, and IPC events; activity before hook installation is unavailable.", "The preload and renderer process contexts are not instrumented by the main-process -r hook; preload configuration and contextBridge API-shape events are main-boundary observations, not proof of renderer-side execution.", + ...(!hookSnapshot.events.some(({ kind }) => kind === "preload") + ? [ + "Preload configuration was not exposed by the observed Electron window metadata. Preload paths and execution remain unavailable.", + ] + : []), "Process metrics are an Electron API snapshot and do not prove hostile-local-user isolation.", ...(hookSnapshot.dropped > 0 ? [ diff --git a/src/browser/PlaywrightElectronActiveProvider.ts b/src/browser/PlaywrightElectronActiveProvider.ts index 590376955be..2e1dd8b8e14 100644 --- a/src/browser/PlaywrightElectronActiveProvider.ts +++ b/src/browser/PlaywrightElectronActiveProvider.ts @@ -71,6 +71,12 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati let application: ElectronApplication | undefined; let ownership: OwnedProcessGroup | undefined; let lineage: ProcessLineageObservation | undefined; + let capture: ElectronActiveObservationResult | undefined; + let paths: ElectronPaths | undefined; + let initialState: + | Awaited> + | undefined; + let actions: ElectronActiveObservationResult["actions"] = []; let outcome: Result; const runId = randomUUID(); try { @@ -78,7 +84,7 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati throw new BrowserObservationError(OPERATION, "cancelled"); await prepareProcessOwnershipInspection(options.signal); const startupDeadline = Date.now() + STARTUP_TIMEOUT_MS; - const paths = await canonicalPaths(input); + paths = await canonicalPaths(input); const captureBaseline = await systemProcessOwnershipHost.captureBaseline?.(options.signal); if (options.signal?.aborted) @@ -109,24 +115,66 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati sweepTokenOwnedProcesses: true, ...(captureBaseline === undefined ? {} : { captureBaseline }), }; - const actions = await runElectronActions(application, input, options); + lineage = await observeOwnedProcessLineage(ownership); + initialState = await readApplicationState(application); + actions = await runElectronActions(application, input, options); const state = await runWithExecutionLimits( readApplicationState(application), - options.signal, + undefined, + options.signal?.aborted ? Date.now() + 5_000 : undefined, ); - outcome = ok(projectElectronActiveCapture(paths, actions, state)); + capture = projectElectronActiveCapture(paths, actions, state); + outcome = options.signal?.aborted + ? err( + new BrowserObservationError(OPERATION, "cancelled", { + partialObservation: { + kind: "electron-active-observation", + capture, + }, + }), + ) + : ok(capture); } catch (cause: unknown) { + if ( + options.signal?.aborted && + paths !== undefined && + initialState !== undefined + ) { + const retainedCapture = projectElectronActiveCapture( + paths, + actions, + initialState, + ); + capture = { + ...retainedCapture, + limitations: [ + ...retainedCapture.limitations, + `Final Electron state was unavailable after cancellation: ${cause instanceof Error ? cause.message : String(cause)}. Windows, metrics, IPC, and timeline are the snapshot collected before actions and may omit their effects.`, + ], + }; + } outcome = err( - options.signal?.aborted === true && - (cause === options.signal.reason || - (cause instanceof Error && cause.name === "AbortError")) - ? new BrowserObservationError(OPERATION, "cancelled", { cause }) + options.signal?.aborted === true + ? new BrowserObservationError(OPERATION, "cancelled", { + cause, + ...(capture === undefined + ? {} + : { + partialObservation: { + kind: "electron-active-observation" as const, + capture, + }, + }), + }) : providerError(cause), ); } if (application !== undefined) { - if (ownership !== undefined) - lineage = await observeOwnedProcessLineage(ownership); + if (ownership !== undefined) { + const currentLineage = await observeOwnedProcessLineage(ownership); + if (currentLineage.status === "verified" || lineage === undefined) + lineage = currentLineage; + } let closeError: unknown; try { await runWithExecutionLimits( @@ -146,13 +194,51 @@ export class PlaywrightElectronActiveProvider implements ElectronActiveObservati : await cleanupElectronProcesses(ownership, lineage); if (!cleanup.cleaned) return err( - providerError( - new BrowserObservationError(OPERATION, "cleanup_failed", { - cause: new Error(cleanup.reason), - }), - ), + new BrowserObservationError(OPERATION, "cleanup_failed", { + cause: outcome.ok ? new Error(cleanup.reason) : outcome.error, + detail: `Electron cleanup could not be verified: ${cleanup.reason}.${capture !== undefined ? " Collected observations are retained in partial_observation." : ""}${outcome.ok ? "" : ` Capture also failed: ${outcome.error.message}.`} Inspect the reported process resources before launching another capture.`, + cleanup: { + reason: cleanup.reason, + resources: + ownership === undefined + ? ["owned_process_group"] + : [ + "owned_process_group", + `run:${ownership.runId}`, + `process-group:${String(ownership.processGroupId)}`, + ], + }, + ...(capture !== undefined + ? { + partialObservation: { + kind: "electron-active-observation" as const, + capture: { + ...capture, + application: { + ...capture.application, + cleanup: "unverified" as const, + }, + }, + }, + } + : {}), + }), + ); + if (closeError !== undefined) + return err( + new BrowserObservationError(OPERATION, "cleanup_failed", { + cause: outcome.ok ? closeError : outcome.error, + detail: `Electron transport teardown failed after process cleanup: ${closeError instanceof Error ? closeError.message : String(closeError)}.${capture !== undefined ? " Collected observations are retained in partial_observation." : ""}${outcome.ok ? "" : ` Capture also failed: ${outcome.error.message}.`}`, + ...(capture !== undefined + ? { + partialObservation: { + kind: "electron-active-observation" as const, + capture, + }, + } + : {}), + }), ); - if (closeError !== undefined) return err(providerError(closeError)); if (outcome.ok && cleanup.unverified !== undefined) outcome = ok({ ...outcome.value, @@ -294,5 +380,8 @@ const providerError = (cause: unknown): AnalysisError => OPERATION, { cause, + diagnostics: { + message: cause instanceof Error ? cause.message : String(cause), + }, }, ); diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index fec40d54972..14b9858e71e 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -8,6 +8,7 @@ import { AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisOutputError, + AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; @@ -17,6 +18,7 @@ import { type InspectWebNetworkCaptureInput, } from "../../domain/webNetworkCapture.js"; import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; +import { HAR_CAPTURE_HEAP_LIMITS } from "./CaptureRelease.js"; const OPERATION = "inspect_web_network_capture"; /** Preserve the origin and typed cause of a historical capture failure. */ @@ -55,6 +57,18 @@ export const historicalCaptureFailure = ( ); if (cause.reason === "output-limit") return new AnalysisOutputError(OPERATION, cause.message); + if ( + phase === "decoder" && + input.format === "har" && + cause.reason === "process" && + cause.snapshot?.signal === "SIGABRT" && + /FATAL ERROR:[^\r\n]*heap out of memory/u.test(cause.snapshot.stderr.text) + ) + return captureMemoryFailure( + input, + "The HAR decoder exhausted its fixed V8 heap", + cause, + ); } if (options?.signal?.aborted) return new AnalysisCancelledError(OPERATION); if ( @@ -144,3 +158,44 @@ export const historicalCaptureFailure = ( }, }); }; + +/** Preserve a decoder's observed memory failure and its actual workload limits. */ +export const captureMemoryFailure = ( + input: InspectWebNetworkCaptureInput, + reason: string, + cause?: unknown, +): AnalysisResourceConstraintError => + new AnalysisResourceConstraintError( + OPERATION, + "memory", + `${reason}: ${input.capture_path}. No complete capture result is available.`, + { + boundary: "historical-capture-decoder", + capture_path: input.capture_path, + format: input.format, + maximum_input_bytes: WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + maximum_reply_bytes: WEB_NETWORK_CAPTURE_LIMITS.outputBytes, + ...(input.format === "har" + ? { + old_generation_heap_mib: HAR_CAPTURE_HEAP_LIMITS.oldGenerationMiB, + semi_space_heap_mib: HAR_CAPTURE_HEAP_LIMITS.semiSpaceMiB, + } + : {}), + ...(cause instanceof OwnedCommandFailure + ? { observed_signal: cause.snapshot?.signal ?? null } + : {}), + }, + { + cause, + remediationAction: `Use a smaller capture exported by its producer, or another decoder with sufficient capacity for the original capture. Preserve the original for provenance; a subset does not establish complete-capture coverage.${input.format === "har" ? ` REA's HAR old-generation heap is fixed at ${HAR_CAPTURE_HEAP_LIMITS.oldGenerationMiB} MiB; inherited NODE_OPTIONS cannot raise it.` : ""}`, + ...(cause instanceof OwnedCommandFailure && cause.snapshot !== null + ? { + capturedOutput: { + stdout: cause.snapshot.stdout.text, + stderr: cause.snapshot.stderr.text, + truncated: cause.snapshot.diagnosticTruncated ?? false, + }, + } + : {}), + }, + ); diff --git a/src/browser/history/CaptureRelease.ts b/src/browser/history/CaptureRelease.ts index 4463befe9e5..e9ac3e36623 100644 --- a/src/browser/history/CaptureRelease.ts +++ b/src/browser/history/CaptureRelease.ts @@ -5,6 +5,12 @@ export const HAR_CAPTURE_PROVIDER_IDENTITY = { version: "har-schema@2.0.0;jsonc-parser@3.3.1;lossless-json@4.3.1", } as const; +/** Fixed V8 heap settings used by the owned HAR decoder process. */ +export const HAR_CAPTURE_HEAP_LIMITS = { + oldGenerationMiB: 192, + semiSpaceMiB: 8, +} as const; + /** Native executable profile verified with unchanged mitmproxy raw-state decoding. */ export const MITMPROXY_CAPTURE_PROVIDER_IDENTITY = { id: "mitmproxy-native-tnetstring", diff --git a/src/browser/history/HarCaptureAdapter.ts b/src/browser/history/HarCaptureAdapter.ts index b66167a5f45..bae33fa7e48 100644 --- a/src/browser/history/HarCaptureAdapter.ts +++ b/src/browser/history/HarCaptureAdapter.ts @@ -1,6 +1,9 @@ import { fileURLToPath } from "node:url"; import type { HistoricalCaptureFormatAdapter } from "./HistoricalCaptureFormatAdapter.js"; -import { HAR_CAPTURE_PROVIDER_IDENTITY } from "./CaptureRelease.js"; +import { + HAR_CAPTURE_PROVIDER_IDENTITY, + HAR_CAPTURE_HEAP_LIMITS, +} from "./CaptureRelease.js"; /** HAR's parser/schema packages run in one bounded Node heap, without network or target execution. */ export class HarCaptureAdapter implements HistoricalCaptureFormatAdapter { @@ -12,8 +15,8 @@ export class HarCaptureAdapter implements HistoricalCaptureFormatAdapter { return { command: process.execPath, arguments: [ - "--max-old-space-size=192", - "--max-semi-space-size=8", + `--max-old-space-size=${HAR_CAPTURE_HEAP_LIMITS.oldGenerationMiB}`, + `--max-semi-space-size=${HAR_CAPTURE_HEAP_LIMITS.semiSpaceMiB}`, "--v8-pool-size=1", "--single-threaded", fileURLToPath(new URL("./HarCaptureProcess.js", import.meta.url)), diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index e180ad6ad4e..bb60625010b 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -1,4 +1,7 @@ -import { historicalCaptureFailure } from "./CaptureFailures.js"; +import { + historicalCaptureFailure, + captureMemoryFailure, +} from "./CaptureFailures.js"; import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import type { HistoricalCaptureFormatAdapter } from "./HistoricalCaptureFormatAdapter.js"; import { randomUUID } from "node:crypto"; @@ -160,7 +163,9 @@ export class HistoricalCaptureDecoder { }, }, ]); - if (reply.reason === "decoder" || reply.reason === "resource-limit") + if (reply.reason === "resource-limit") + throw captureMemoryFailure(input, reply.message); + if (reply.reason === "decoder") throw new ProviderAdapterError(input.format, OPERATION, { diagnostics: { phase: "decoder", diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index c24fd472a2d..89dc40eb688 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -44,7 +44,7 @@ const registerElectronActiveCommand = ( logger: Logger, ): void => { cli.command(CLI_COMMANDS.captureElectronScenario, { - description: "Run one bounded owned Electron scenario", + description: "Run one owned Electron scenario with cancellable actions", args: z.object({ inputJson: z .string() @@ -53,33 +53,40 @@ const registerElectronActiveCommand = ( ), }), run: ({ args }) => - logCliCommand(logger, CLI_COMMANDS.captureElectronScenario, async () => { - const input = await parseCliJsonInput( - args.inputJson, - "capture_electron_scenario", - ); - if (!input.ok) return input.error; - const parsed = electronActiveObservationInputSchema.safeParse( - resolveCliJsonPaths(input.value, [ - ["executable_path"], - ["application_path"], - ["application_root"], - ]), - ); - if (!parsed.success) - return inputError( - "capture_electron_scenario", - parsed.error.issues, - input.value, - ); - const { createElectronScenarioProvider } = - await import("../composition/electronScenario.js"); - const result = await captureElectronScenario( - createElectronScenarioProvider(), - parsed.data, - ); - return result.ok ? result.value : cliError(result.error); - }), + withCommandCancellation((signal) => + logCliCommand( + logger, + CLI_COMMANDS.captureElectronScenario, + async () => { + const input = await parseCliJsonInput( + args.inputJson, + "capture_electron_scenario", + ); + if (!input.ok) return input.error; + const parsed = electronActiveObservationInputSchema.safeParse( + resolveCliJsonPaths(input.value, [ + ["executable_path"], + ["application_path"], + ["application_root"], + ]), + ); + if (!parsed.success) + return inputError( + "capture_electron_scenario", + parsed.error.issues, + input.value, + ); + const { createElectronScenarioProvider } = + await import("../composition/electronScenario.js"); + const result = await captureElectronScenario( + createElectronScenarioProvider(), + parsed.data, + { signal }, + ); + return result.ok ? result.value : cliError(result.error); + }, + ), + ), }); }; diff --git a/src/contracts/javascript/electronToolContracts.ts b/src/contracts/javascript/electronToolContracts.ts index 7c689768990..c0e860f957a 100644 --- a/src/contracts/javascript/electronToolContracts.ts +++ b/src/contracts/javascript/electronToolContracts.ts @@ -112,7 +112,7 @@ export const ELECTRON_TOOL_CONTRACTS = [ name: "capture_electron_scenario", ...toolContractMetadata("capture_electron_scenario"), description: - "Use this for a provider-owned Electron run when passive CDP or Inspector observation cannot exercise application behavior. REA owns startup and teardown, accepts caller-defined click/wait actions plus window-targeted renderer reload/crash and synthetic open-url/second-instance delivery, and returns correlated window/WebContents/process/preload/session/navigation/shell/IPC evidence without retaining payload values. Results identify observed and unavailable event families, coverage status, action targets, and truncation. External shell, navigation, permission, download, popup, updater, and OS-integration effects are blocked and recorded. Use passive Electron tools for observation-only work.", + "Use this for a provider-owned Electron run when passive CDP or Inspector observation cannot exercise application behavior. REA owns startup and teardown, accepts caller-defined click/wait actions plus window-targeted renderer reload/crash and synthetic open-url/second-instance delivery, and returns correlated window/WebContents/process/preload/session/navigation/shell/IPC evidence without retaining payload values. Actions wait for completion or request cancellation; a missing click target can keep the request pending. Results identify observed and unavailable event families, coverage status, action targets, and truncation. Cleanup failures retain collected observations with explicit cleanup uncertainty. External shell, navigation, permission, download, popup, updater, and OS-integration effects are blocked and recorded. Use passive Electron tools for observation-only work.", kind: "electron-provider", inputSchema: electronActiveObservationInputSchema, outputSchema: evidenceResultOf(electronActiveObservationResultSchema), diff --git a/src/domain/analysisErrorBase.ts b/src/domain/analysisErrorBase.ts index 4ba3bd99126..3b90873904d 100644 --- a/src/domain/analysisErrorBase.ts +++ b/src/domain/analysisErrorBase.ts @@ -86,9 +86,11 @@ import type { } from "./process/processCapture.js"; import type { FileOffsetPartialObservation } from "./native/fileOffsetPartialObservation.js"; import type { NativeCallPartialObservation } from "./native/nativeCallPartialObservation.js"; +import type { ElectronActivePartialObservation } from "./javascript/electronActiveObservation.js"; /** Provider-neutral evidence collected before a typed analysis failure. */ export type AnalysisPartialObservation = | PartialProcessCaptureObservation | NativeCallPartialObservation - | FileOffsetPartialObservation; + | FileOffsetPartialObservation + | ElectronActivePartialObservation; diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index 215cf58bda9..cc3dc7e40b4 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -96,7 +96,7 @@ export const analysisErrorCategory = ( if (error._tag === "ProcessCaptureError") return error.userCategory ?? "execution_failure"; if (error instanceof BrowserObservationError) - return browserErrorCategory(error.reason); + return error.userCategory ?? browserErrorCategory(error.reason); if ( error instanceof HopperRemoteError && error.diagnosticType === "invalid_request" @@ -117,6 +117,8 @@ export const analysisErrorCategory = ( const browserErrorCategory = ( reason: BrowserObservationError["reason"], ): AnalysisErrorProjection["category"] => { + if (reason === "cancelled") return "cancelled"; + if (reason === "timeout") return "timeout"; if (reason === "payload_limit") return "truncated"; if ( reason === "target_not_found" || diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 39509368a81..66e5c77e9c3 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -96,7 +96,9 @@ const underlyingErrorCode = ( ? "provider_unavailable" : "capability_unavailable"; if (error instanceof BrowserObservationError) - return browserErrorCode(error.reason); + return error.userCategory === "cancelled" + ? "cancelled" + : browserErrorCode(error.reason); if (error instanceof ArtifactOperationError) return artifactOperationCode(error); if (error instanceof EvidenceFileError) return evidenceFileCode(error.reason); @@ -109,6 +111,8 @@ const underlyingErrorCode = ( const browserErrorCode = ( reason: BrowserObservationError["reason"], ): AnalysisErrorProjection["code"] => { + if (reason === "cancelled") return "cancelled"; + if (reason === "timeout") return "provider_timeout"; if (reason === "payload_limit") return "truncated"; if ( reason === "target_not_found" || diff --git a/src/domain/browserObservationError.ts b/src/domain/browserObservationError.ts index e698a97bab6..439995aeb20 100644 --- a/src/domain/browserObservationError.ts +++ b/src/domain/browserObservationError.ts @@ -2,7 +2,10 @@ import type { BrowserObservationFailureReason, BrowserObservationOperation, } from "./browserObservationErrors.js"; -import { AnalysisError } from "./analysisErrorBase.js"; +import { + AnalysisError, + type AnalysisErrorOptions, +} from "./analysisErrorBase.js"; import { AnalysisCancelledError } from "./analysisErrorCore.js"; /** A bounded passive browser observation failed at its CDP boundary. */ @@ -16,16 +19,39 @@ export class BrowserObservationError extends AnalysisError { constructor( readonly operation: BrowserObservationOperation, readonly reason: BrowserObservationFailureReason, - options?: ErrorOptions & { readonly detail?: string }, + options?: AnalysisErrorOptions & { readonly detail?: string }, ) { super(`Browser observation ${operation} failed: ${reason}`, options); - this.userMessage = options?.detail; - this.cleanupIncomplete = reason === "cleanup_failed"; + this.userMessage = + options?.detail ?? observationFailureMessage(operation, reason); + this.cleanupIncomplete = + this.cleanup !== undefined || reason === "cleanup_failed"; this.cleanupResources = - reason === "cleanup_failed" ? ["browser_transport"] : []; + this.cleanup?.resources ?? + (reason === "cleanup_failed" ? ["browser_transport"] : []); this.userCategory = - reason === "cancelled" || options?.cause instanceof AnalysisCancelledError + reason === "cancelled" || + options?.cause instanceof AnalysisCancelledError || + (options?.cause instanceof AnalysisError && + options.cause.userCategory === "cancelled") ? "cancelled" : undefined; } } + +const observationFailureMessage = ( + operation: BrowserObservationOperation, + reason: BrowserObservationFailureReason, +): string | undefined => { + if (reason === "cancelled") + return `${operation} was cancelled. Collected observations and cleanup status are reported when available.`; + if (reason === "timeout") + return `${operation} exceeded its execution deadline. Review the target's responsiveness and retained observations before retrying.`; + if (reason === "endpoint_unreachable") + return `${operation} could not reach the selected loopback debugging endpoint. Verify that the target process is running with debugging enabled and that the endpoint's port matches it, then retry.`; + if (reason === "target_not_found") + return `${operation} could not find the selected target at the debugging endpoint. Refresh target discovery for that endpoint and retry with a returned target ID.`; + if (reason === "disconnected") + return `${operation} lost its debugging connection. Verify that the target process is still running, refresh target discovery, and retry the observation.`; + return undefined; +}; diff --git a/src/domain/explicitSensitiveFailure.ts b/src/domain/explicitSensitiveFailure.ts index f26324f9b69..dd7a14366ae 100644 --- a/src/domain/explicitSensitiveFailure.ts +++ b/src/domain/explicitSensitiveFailure.ts @@ -5,6 +5,7 @@ import { AnalysisInputError, AnalysisOutputError, AnalysisCapabilityUnavailableError, + AnalysisResourceConstraintError, } from "./analysisErrorCore.js"; import { redactExplicitText } from "./explicitSensitiveValues.js"; import type { JsonValue } from "./jsonValue.js"; @@ -118,6 +119,28 @@ export const redactExplicitFailure = ( : { userMessage: text(error.userMessage) }), }, ); + if (error instanceof AnalysisResourceConstraintError) + return new AnalysisResourceConstraintError( + error.operation, + error.resource, + text(error.reason), + error.reportedLimits === null ? null : diagnostics(error.reportedLimits), + { + cause: error, + ...(error.remediationAction === undefined + ? {} + : { remediationAction: text(error.remediationAction) }), + ...(error.capturedOutput === undefined + ? {} + : { + capturedOutput: { + stdout: text(error.capturedOutput.stdout), + stderr: text(error.capturedOutput.stderr), + truncated: error.capturedOutput.truncated, + }, + }), + }, + ); if (error instanceof ProviderCleanupError) return new ProviderCleanupError( error.providerId, diff --git a/src/domain/javascript/electronActiveObservation.ts b/src/domain/javascript/electronActiveObservation.ts index 7a3fd9af6a3..be509206586 100644 --- a/src/domain/javascript/electronActiveObservation.ts +++ b/src/domain/javascript/electronActiveObservation.ts @@ -1,8 +1,8 @@ import { z } from "zod"; -import { isAbsoluteLocalPath } from "../localPath.js"; +import { isAbsoluteLocalPath, localPathStringSchema } from "../localPath.js"; -const pathInputSchema = z.string().trim().min(1).refine(isAbsoluteLocalPath, { +const pathInputSchema = localPathStringSchema.refine(isAbsoluteLocalPath, { message: "Electron executable, application, and root paths must be absolute local filesystem paths (for example /Applications/Electron.app/Contents/MacOS/Electron)", }); @@ -76,7 +76,11 @@ export const electronActiveObservationInputSchema = z.strictObject({ .describe( "Absolute local filesystem root for application-relative paths; omit to derive it from the application path. Relative paths are rejected.", ), - args: z.array(z.string()).default([]), + args: z + .array( + z.string().regex(/^[^\0]*$/u, "Electron arguments must not contain NUL"), + ) + .default([]), actions: z.array(actionSchema).default([]), }); export type ElectronActiveObservationInput = z.infer< @@ -309,3 +313,16 @@ export const electronActiveObservationResultSchema = z.strictObject({ export type ElectronActiveObservationResult = z.infer< typeof electronActiveObservationResultSchema >; + +/** Observations collected before an Electron capture failed its cleanup boundary. */ +export interface ElectronActivePartialObservation { + readonly kind: "electron-active-observation"; + readonly capture: Omit & { + readonly application: Omit< + ElectronActiveObservationResult["application"], + "cleanup" + > & { + readonly cleanup: "unverified" | "terminated-owned-process"; + }; + }; +} diff --git a/tests/acceptance/tools/historicalCaptureResources.test.ts b/tests/acceptance/tools/historicalCaptureResources.test.ts new file mode 100644 index 00000000000..b0e07f353d2 --- /dev/null +++ b/tests/acceptance/tools/historicalCaptureResources.test.ts @@ -0,0 +1,115 @@ +import { execFile } from "node:child_process"; +import { truncate, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../../src/domain/webNetworkCapture.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const cli = fileURLToPath(new URL("../../../scripts/rea.mjs", import.meta.url)); +const entry = JSON.stringify({ + startedDateTime: "2026-01-01T00:00:00.000Z", + time: 1, + request: { + method: "GET", + url: "https://example.invalid/data", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + queryString: [], + headersSize: -1, + bodySize: 0, + }, + response: { + status: 200, + statusText: "OK", + httpVersion: "HTTP/1.1", + cookies: [], + headers: [], + content: { size: 512, mimeType: "text/plain", text: "x".repeat(512) }, + redirectURL: "", + headersSize: -1, + bodySize: 512, + }, + cache: {}, + timings: { send: 0, wait: 1, receive: 0 }, +}); +const capture = (count: number) => + `{"log":{"version":"1.2","creator":{"name":"synthetic","version":"1"},"entries":[${Array(count).fill(entry).join(",")}]}}`; + +it("reports real HAR heap exhaustion through CLI and MCP and preserves server usability", async () => { + const root = await createTestTempDirectory("rea-har-capacity-"); + const path = join(root, "caller-marked.har"); + const dense = capture(34_098); + expect(Buffer.byteLength(dense)).toBeLessThan( + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + ); + await writeFile(path, dense); + const { call, client } = await connectLocalToolsMcp(); + const response = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + sensitive_values: ["caller-marked"], + }); + expect(response.isError).toBe(true); + expect(response.structuredContent).toMatchObject({ + error: { + code: "resource_constraint", + retryable: false, + message: expect.stringContaining("fixed V8 heap"), + remediation: { + action: expect.stringContaining( + "smaller capture exported by its producer", + ), + }, + details: { + resource: "memory", + reported_limits: { + old_generation_heap_mib: 192, + observed_signal: "SIGABRT", + }, + captured_output: { + stderr: expect.stringContaining("heap out of memory"), + truncated: false, + }, + }, + }, + }); + expect(JSON.stringify(response)).not.toContain("caller-marked"); + expect(JSON.stringify(response)).not.toContain("rea doctor"); + await expect( + execute(process.execPath, [ + cli, + "inspect-web-network-capture", + path, + "har", + "--json", + ]), + ).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining('"code": "resource_constraint"'), + }); + const small = join(root, "small.har"); + await writeFile(small, capture(2)); + await client.ping(); + const control = await call("inspect_web_network_capture", { + capture_path: small, + format: "har", + }); + expect(control.isError, JSON.stringify(control)).not.toBe(true); + expect(control.structuredContent).toMatchObject({ + result: { total_records: 2 }, + }); + await truncate(path, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1); + const oversized = await call("inspect_web_network_capture", { + capture_path: path, + format: "har", + }); + expect(oversized.structuredContent).toMatchObject({ + error: { code: "invalid_request" }, + }); +}, 60_000); diff --git a/tests/acceptance/tools/inspectorLifecycle.test.ts b/tests/acceptance/tools/inspectorLifecycle.test.ts new file mode 100644 index 00000000000..91c0b8f34c9 --- /dev/null +++ b/tests/acceptance/tools/inspectorLifecycle.test.ts @@ -0,0 +1,126 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("observes a real Node Inspector and releases attachments after cancellation and failures", async () => { + const root = await createTestTempDirectory("rea-live-inspector-"); + const fixture = join(root, "probe #百分比 %.cjs"); + await writeFile(fixture, "setInterval(() => {}, 25);\n"); + const child = spawn(process.execPath, ["--inspect=127.0.0.1:0", fixture], { + stdio: ["ignore", "ignore", "pipe"], + }); + const terminate = async () => { + if (child.exitCode === null && child.signalCode === null) { + const exited = once(child, "exit"); + child.kill("SIGTERM"); + await exited; + } + }; + onTestFinished(terminate); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + const address = /ws:\/\/127\.0\.0\.1:\d+\/[^\s]+/u; + await expect.poll(() => stderr, { timeout: 5000 }).toMatch(address); + const socket = address.exec(stderr)?.[0]; + if (socket === undefined) throw new Error("Node omitted its Inspector URL"); + const endpoint = `http://127.0.0.1:${new URL(socket).port}`; + const { client, call } = await connectLocalToolsMcp(); + const discovery = await call("list_javascript_runtime_targets", { + inspector_endpoint: endpoint, + }); + const targets = toolContract( + "list_javascript_runtime_targets", + ).outputSchema.parse(discovery.structuredContent); + const id = targets.result.targets[0]?.target_id; + if (id === undefined) throw new Error("Node Inspector target was not listed"); + const request = { + inspector_endpoint: endpoint, + target_id: id, + observation_ms: 100, + }; + const observed = await call("observe_javascript_runtime", request); + const observation = toolContract( + "observe_javascript_runtime", + ).outputSchema.parse(observed.structuredContent); + expect(observation.result.scripts.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + location: expect.objectContaining({ file_path: fixture }), + }), + ]), + ); + const missing = await call("observe_javascript_runtime", { + ...request, + target_id: "../missing-target", + }); + expect(missing.isError).toBe(true); + expect(missing.structuredContent).toMatchObject({ + error: { + code: "target_unavailable", + message: expect.stringContaining("Refresh target discovery"), + details: { reason: "target_not_found" }, + }, + }); + for (const invalid of [ + { + name: "observe_javascript_runtime", + arguments: { ...request, observation_ms: -1 }, + }, + { + name: "list_javascript_runtime_targets", + arguments: { + inspector_endpoint: endpoint.replace( + "http://", + "http://user:password@", + ), + }, + }, + ]) + expect((await call(invalid.name, invalid.arguments)).isError).toBe(true); + + const attachments = () => (stderr.match(/Debugger attached/gu) ?? []).length; + const detachments = () => (stderr.match(/Debugger ending on/gu) ?? []).length; + const before = attachments(); + const controller = new AbortController(); + const cancelled = expect( + client.callTool( + { + name: "observe_javascript_runtime", + arguments: { ...request, observation_ms: 10_000 }, + }, + { signal: controller.signal }, + ), + ).rejects.toThrow("cancel live observation"); + await expect.poll(attachments, { timeout: 5000 }).toBe(before + 1); + controller.abort(new Error("cancel live observation")); + await cancelled; + await expect.poll(detachments, { timeout: 5000 }).toBe(attachments()); + expect(child.exitCode).toBeNull(); + expect(child.signalCode).toBeNull(); + expect((await call("observe_javascript_runtime", request)).isError).not.toBe( + true, + ); + await client.ping(); + await terminate(); + const unreachable = await call("list_javascript_runtime_targets", { + inspector_endpoint: endpoint, + }); + expect(unreachable.isError).toBe(true); + expect(unreachable.structuredContent).toMatchObject({ + error: { + code: "provider_unavailable", + message: expect.stringContaining( + "target process is running with debugging enabled", + ), + details: { reason: "endpoint_unreachable" }, + }, + }); +}, 30_000); diff --git a/tests/acceptance/tools/localToolContracts.test.ts b/tests/acceptance/tools/localToolContracts.test.ts new file mode 100644 index 00000000000..41e2ecd100f --- /dev/null +++ b/tests/acceptance/tools/localToolContracts.test.ts @@ -0,0 +1,379 @@ +import { execFile } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { itWithCaptureCapability as captureTest } from "../../boundary/process/processCaptureCapability.js"; +import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const cli = fileURLToPath(new URL("../../../scripts/rea.mjs", import.meta.url)); +const nul = "/tmp/selected\0filename"; +const electron = { + executable_path: "/tmp/Electron", + application_path: "/tmp/main.js", +}; +const pathRequests: readonly [string, Record][] = [ + ["open_binary", { path: nul }], + ["open_binary", { path: process.execPath, snapshot_path: nul }], + ["close_binary", { snapshot_path: nul }], + ["binary_session", { expected_server_path: nul }], + ["export_evidence_bundle", { path: nul }], + ["import_evidence_bundle", { path: nul }], + ["inspect_managed_artifact", { path: nul }], + ["inspect_android_package", { path: nul }], + ["search_android_classes", { path: nul, query: "" }], + ["inspect_android_class", { path: nul, class_name: "probe.Class" }], + [ + "inspect_android_method", + { path: nul, class_name: "probe.Class", method_name: "run" }, + ], + ["trace_android_references", { path: nul, class_name: "probe.Class" }], + ["inspect_firmware_regions", { path: nul }], + ["extract_firmware", { path: nul, output_directory: "/tmp/out" }], + ["extract_firmware", { path: "/tmp/firmware.bin", output_directory: nul }], + ["inspect_plist", { path: nul }], + ...[ + { kind: "function", name: "main\0suffix" }, + { kind: "function", name: "main", module: "lib.dylib\0suffix" }, + { kind: "objc-method", selector: "run\0suffix" }, + { kind: "objc-method", class_name: "Probe\0suffix", selector: "run" }, + ].map((breakpoint): [string, Record] => [ + "observe_native_calls", + { breakpoints: [breakpoint] }, + ]), + [ + "observe_native_calls", + { + breakpoints: [{ kind: "function", name: "main" }], + working_directory: nul, + }, + ], + [ + "observe_native_calls", + { breakpoints: [{ kind: "function", name: "main" }], arguments: ["a\0b"] }, + ], + ["analyze_javascript_application", { input_path: nul }], + ["recover_javascript_sources", { path: nul, output_directory: "/tmp/out" }], + [ + "recover_javascript_sources", + { path: "/tmp/main.js", output_directory: nul }, + ], + [ + "compare_bundles", + { left_bundle_path: nul, right_bundle_path: "/tmp/right.json" }, + ], + [ + "compare_bundles", + { left_bundle_path: "/tmp/left.json", right_bundle_path: nul }, + ], + [ + "capture_browser_scenario", + { + browser: { mode: "launch", executable_path: nul }, + start_url: { url: "http://127.0.0.1:1" }, + actions: [ + { step_id: "settle", action: "wait_for_timeout", duration_ms: 0 }, + ], + }, + ], + ...["executable_path", "application_path", "application_root"].map( + (key): [string, Record] => [ + "capture_electron_scenario", + { ...electron, [key]: nul }, + ], + ), + ["capture_electron_scenario", { ...electron, args: ["a\0b"] }], + ["demangle_swift", { symbols: ["abc\0def"] }], +]; + +it("rejects OS-invalid local inputs through the real advertised MCP contracts", async () => { + const { call } = await connectLocalToolsMcp(); + for (const [name, arguments_] of pathRequests) { + const response = await call(name, arguments_); + expect(response.isError, name).toBe(true); + expect(JSON.stringify(response.content), name).toContain("NUL"); + } + const relative = await call("extract_firmware", { + path: "/tmp/firmware.bin", + output_directory: "relative-output", + }); + expect(relative.isError).toBe(true); + expect(JSON.stringify(relative.content)).toContain( + "absolute local filesystem path", + ); +}); + +const ignoredOptions = [ + { terminal: { columns: 80, row: 12 } }, + { events: [{ type: "input", at_ms: 0, data: "x", secret: true }] }, + { events: [{ type: "resize", at_ms: 0, columns: 80, rows: 24, row: 12 }] }, + { events: [{ type: "signal", at_ms: 0, signal: "SIGTERM", pid: 1 }] }, + { normalization: { path: true } }, + { limits: { file_count: 1 } }, + { + normalization: { + patterns: [{ pattern: "x", replacement: "y", regex: true }], + }, + }, +]; + +captureTest( + "rejects malformed process choices through CLI and MCP before launching the command", + async () => { + const root = await createTestTempDirectory("rea-local-contracts-"); + const marker = join(root, "launched"); + const scenario = { + executable: process.execPath, + arguments: [ + "-e", + "require('node:fs').writeFileSync(process.argv[1],'launched')", + marker, + ], + }; + const { call } = await connectLocalToolsMcp(); + const invalid = [ + ...ignoredOptions, + ...["", "KEY=VALUE", "KEY\0VALUE", "REA_PROCESS_RUN_ID"].map((key) => ({ + environment: { [key]: "value" }, + })), + { executable: `node\0` }, + { arguments: ["\0"] }, + { working_directory: "/tmp/dir\0" }, + { environment: { KEY: "\0" } }, + { filesystem_observation_paths: ["\0"] }, + { timeout_ms: 10, events: [{ type: "input", at_ms: 11, data: "late" }] }, + { + events: [ + { type: "resize", at_ms: 2, columns: 80, rows: 24 }, + { type: "input", at_ms: 1, data: "out of order" }, + ], + }, + ...["command_shims", "replay", "reactive", "checkpoints"].map( + (field) => ({ + [field]: [], + }), + ), + ]; + for (const options of invalid) { + const request = { ...scenario, ...options }; + expect( + (await call("capture_process_scenario", request)).isError, + JSON.stringify(options), + ).toBe(true); + const path = join(root, "scenario.json"); + await writeFile(path, JSON.stringify(request)); + await expect( + execute(process.execPath, [cli, "capture-process", path, "--json"]), + ).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining("invalid_request"), + }); + } + await expect(access(marker)).rejects.toMatchObject({ code: "ENOENT" }); + const missing = join(root, "missing-executable"); + const failed = await call("capture_process_scenario", { + executable: missing, + }); + expect(failed.structuredContent).toMatchObject({ + error: { + code: "process_capture_failed", + message: expect.stringContaining(missing), + details: { execution_failure: expect.stringContaining(missing) }, + }, + }); + await writeFile( + join(root, "scenario.json"), + JSON.stringify({ executable: missing }), + ); + await expect( + execute(process.execPath, [ + cli, + "capture-process", + join(root, "scenario.json"), + "--json", + ]), + ).rejects.toMatchObject({ + code: 1, + stdout: expect.stringContaining(missing), + }); + }, + 60_000, +); + +captureTest.each(["unknown", "truncated"] as const)( + "captures real process choices and composes a %s self-comparison", + async (status) => { + const root = await createTestTempDirectory("rea-local-capture-"); + const { call } = await connectLocalToolsMcp(); + const captured = await call("capture_process_scenario", { + executable: process.execPath, + arguments: [ + "-e", + `require('node:fs').writeFileSync('state','selected');console.log(process.env['app.setting'],process.env.MY_PASSWORD,process.env['REA_PROCESS_RUN_ID'+String.fromCharCode(10)]);${status === "truncated" ? "console.log('x'.repeat(5000))" : ""}`, + ], + working_directory: root, + environment: { + "app.setting": "value", + "1": "numbered", + MY_PASSWORD: "ordinary-evidence", + "REA_PROCESS_RUN_ID\n": "caller-value", + }, + filesystem_observation_paths: [root], + limits: status === "truncated" ? { output_bytes: 128 } : {}, + }); + expect(captured.isError, JSON.stringify(captured)).not.toBe(true); + const capture = toolContract("capture_process_scenario").outputSchema.parse( + captured.structuredContent, + ); + const source = capture.evidence; + if (status === "unknown") + expect( + capture.result.frames.map((frame) => frame.data).join(""), + ).toContain("value ordinary-evidence caller-value"); + else expect(capture.result.truncated).toBe(true); + const environment = await call("capture_process_scenario", { + executable: "/usr/bin/printenv", + arguments: ["1"], + environment: { "1": "numbered" }, + }); + const environmentEnvelope = toolContract( + "capture_process_scenario", + ).outputSchema.parse(environment.structuredContent); + const received = environmentEnvelope.result.frames + .map((frame) => frame.data) + .join(""); + expect(received).toBe("numbered\r\n"); + expect(await readFile(join(root, "state"), "utf8")).toBe("selected"); + expect(capture.result.filesystem_effects).toEqual( + expect.arrayContaining([expect.objectContaining({ status: "created" })]), + ); + expect(capture.result.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + const compared = await call("compare_process_captures", { + left: source, + right: source, + }); + expect(compared.isError, JSON.stringify(compared)).not.toBe(true); + const comparison = toolContract( + "compare_process_captures", + ).outputSchema.parse(compared.structuredContent); + expect(comparison.result.status).toBe(status); + const verification = await call("verify_reconstruction", { + specification: { + name: "Self comparison retains uncertainty", + claims: [ + { + claim_id: "control", + title: "Control capture", + comparison_evidence_id: comparison.evidence_id, + kind: "behavioral", + dimension: "overall", + }, + ], + }, + }); + expect(verification.isError, JSON.stringify(verification)).not.toBe(true); + expect(verification.structuredContent).toMatchObject({ + result: { + status: "unknown", + claims: { + items: [ + { + left_evidence_ids: [source.evidence_id], + right_evidence_ids: [source.evidence_id], + evidence_links: expect.arrayContaining([ + source.evidence_id, + comparison.evidence_id, + ]), + }, + ], + }, + }, + }); + const behavior = await call("find_changed_behavior", { + comparisons: [comparison.evidence], + }); + expect(behavior.isError, JSON.stringify(behavior)).not.toBe(true); + expect(behavior.structuredContent).toMatchObject({ + result: { behavior_status: status }, + }); + const unknowns = await call("list_unknowns", { + domain: "process-comparison", + }); + expect(unknowns.structuredContent).toMatchObject({ + result: { + items: [ + { + status: "open", + supporting_evidence_ids: [source.evidence_id], + contradicting_evidence_ids: [], + }, + ], + }, + }); + const bundle = await call("get_evidence_bundle", {}); + expect( + (await call("compare_process_captures", { left: source, right: source })) + .isError, + ).not.toBe(true); + expect((await call("get_evidence_bundle", {})).structuredContent).toEqual( + bundle.structuredContent, + ); + const path = join(root, "capture.json"); + await writeFile(path, JSON.stringify(source)); + const cliComparison = await execute(process.execPath, [ + cli, + "compare-process-captures", + path, + path, + "--json", + ]); + expect(parseEvidence(JSON.parse(cliComparison.stdout))).toEqual( + comparison.evidence, + ); + }, +); + +captureTest( + "accepts NUL as actual terminal input while preserving capture defaults", + async () => { + const { call } = await connectLocalToolsMcp(); + const response = await call("capture_process_scenario", { + executable: process.execPath, + arguments: [ + "-e", + "process.stdin.setRawMode(true);process.stdin.resume();process.stdin.once('data',bytes=>{console.log('stdin-byte:'+bytes[0]);process.exit(0)});console.log('ready')", + ], + events: [{ type: "input", at_ms: 1000, data: "\0" }], + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const capture = toolContract("capture_process_scenario").outputSchema.parse( + response.structuredContent, + ); + expect(capture.result.frames.map((frame) => frame.data).join("")).toContain( + "stdin-byte:0", + ); + expect(capture.result.manifest.scenario).toMatchObject({ + environment: {}, + filesystem_observation_paths: [], + terminal: { columns: 80, rows: 24, scrollback: 1000 }, + }); + expect(capture.result.interaction_events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "input", + data: "\0", + outcome: "dispatched", + }), + ]), + ); + }, +); diff --git a/tests/boundary/browser/electronActiveHook.test.ts b/tests/boundary/browser/electronActiveHook.test.ts index f4cd1e19fc2..ab6596d8214 100644 --- a/tests/boundary/browser/electronActiveHook.test.ts +++ b/tests/boundary/browser/electronActiveHook.test.ts @@ -10,6 +10,7 @@ const temporary: string[] = []; const electronModuleSource = `const { EventEmitter } = require("node:events"); const app = new EventEmitter(); +app.isReady = () => true; app.relaunch = () => undefined; app.setAsDefaultProtocolClient = () => true; app.setLoginItemSettings = () => undefined; @@ -40,6 +41,8 @@ class BrowserWindow extends EventEmitter { this.id = 3; this.webContents = new WebContents(); this.options = options; + this.webContents.getLastWebPreferences = () => options.webPreferences; + app.emit("browser-window-created", {}, this); } } const utilityProcess = { @@ -66,7 +69,7 @@ const contextBridge = { }; module.exports = { app, - BrowserWindow, + get BrowserWindow() { return BrowserWindow; }, contextBridge, ipcMain, ipcRenderer, diff --git a/tests/boundary/browser/historicalCaptureDecoder.test.ts b/tests/boundary/browser/historicalCaptureDecoder.test.ts index 5187d1f6a76..23c4604a31d 100644 --- a/tests/boundary/browser/historicalCaptureDecoder.test.ts +++ b/tests/boundary/browser/historicalCaptureDecoder.test.ts @@ -266,7 +266,7 @@ it("cancels after snapshot acquisition before launch and removes private input d it.each([ { reason: "input-limit", tag: "AnalysisInputError" }, - { reason: "resource-limit", tag: "ProviderAdapterError" }, + { reason: "resource-limit", tag: "AnalysisResourceConstraintError" }, { reason: "decoder", tag: "ProviderAdapterError" }, { reason: "limit", tag: "AnalysisOutputError" }, ])( diff --git a/tests/fixtures/localToolsMcp.ts b/tests/fixtures/localToolsMcp.ts new file mode 100644 index 00000000000..53a84b6c983 --- /dev/null +++ b/tests/fixtures/localToolsMcp.ts @@ -0,0 +1,62 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { fileURLToPath } from "node:url"; +import { expect, onTestFinished } from "vitest"; +import { z } from "zod"; + +/** Start the built production server without configuring deep binary providers. */ +export async function connectLocalToolsMcp() { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ + fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)), + "mcp", + ], + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + REA_LOG_LEVEL: "silent", + HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper", + }, + stderr: "pipe", + }); + const client = new Client({ name: "local-tools-e2e", version: "1" }); + onTestFinished(async () => { + await client.close(); + await transport.close(); + }); + await client.connect(transport); + const tools = (await client.listTools()).tools; + const validators = new Map( + tools.flatMap((tool) => { + new Ajv2020({ strict: false, validateFormats: false }).compile( + tool.inputSchema, + ); + return tool.outputSchema === undefined + ? [] + : [ + [ + tool.name, + new Ajv2020({ strict: false, validateFormats: false }).compile( + z.record(z.string(), z.unknown()).parse(tool.outputSchema), + ), + ] as const, + ]; + }), + ); + const call = async (name: string, arguments_: Record) => { + const response = await client.callTool({ name, arguments: arguments_ }); + const validate = validators.get(name); + if ( + response.isError !== true && + response.structuredContent !== undefined && + validate !== undefined + ) + expect( + validate(response.structuredContent), + JSON.stringify(validate.errors), + ).toBe(true); + return response; + }; + return { client, call }; +} From 2b2751547662f8cb142f96d91eff50388a1ecb31 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 06:48:36 +0800 Subject: [PATCH 063/791] fix(process): stabilize native capture and ownership prerequisites --- bridge/native/rea_lldb_tracer.py | 221 +++++++++++------- bridge/process/ProcessRunTokenReader.swift | 12 +- docs/testing.md | 16 +- scripts/verify-evm-interface.mjs | 18 ++ scripts/verify-native-calls.mjs | 76 +++++- scripts/verify-native-ui.mjs | 70 +++++- scripts/verify/firmware/analysis.mjs | 8 + src/process/ProcessOwnershipObservation.ts | 17 ++ .../darwinProcessRunTokenReader.test.ts | 5 +- tests/conformance/native/ui.swift | 4 +- .../fixtures/processRunTokenParserProbe.swift | 8 - 11 files changed, 341 insertions(+), 114 deletions(-) diff --git a/bridge/native/rea_lldb_tracer.py b/bridge/native/rea_lldb_tracer.py index 45b2893e891..b99123bf4b8 100644 --- a/bridge/native/rea_lldb_tracer.py +++ b/bridge/native/rea_lldb_tracer.py @@ -323,31 +323,46 @@ def _wanted(spec, symbol): return symbol.startswith("-[" if method_type == "instance" else "+[") -def _record(context, thread, index, sequence, location, remaining_bytes): +def _read_register(frame, name): + error = lldb.SBError() + value = frame.FindRegister(name) + if not value.IsValid(): + raise RuntimeError("LLDB breakpoint register %s is unavailable" % name) + number = value.GetValueAsUnsigned(error) + if error.Fail(): + raise RuntimeError( + "LLDB breakpoint register %s could not be read: %s" % (name, error) + ) + return number + + +def _record(context, thread, frame, index, sequence, location, remaining_bytes): target, process, config = context["target"], context["process"], context["config"] - frame = thread.GetFrameAtIndex(0) symbol = location["symbol"] or "" if not _wanted(config["breakpoints"][index], symbol): return None names = ARGUMENT_REGISTERS.get(config["architecture"], []) + objc_method = symbol.startswith("-[") or symbol.startswith("+[") + values = { + name: _read_register(frame, name) + for name in names[: max(config["argument_registers"], 2 if objc_method else 0)] + } registers = [ - {"name": name, "value": _hex(frame.FindRegister(name).GetValueAsUnsigned())} + {"name": name, "value": _hex(values[name])} for name in names[: config["argument_registers"]] ] receiver_class = None selector = None - if symbol.startswith("-[") or symbol.startswith("+["): + if objc_method: error = lldb.SBError() text = process.ReadCStringFromMemory( - frame.FindRegister(names[1]).GetValueAsUnsigned(), + values[names[1]], MAX_SELECTOR_BYTES, error, ) selector = text if error.Success() and text else None if symbol.startswith("-["): - receiver_class = _dynamic_class( - target, frame.FindRegister(names[0]).GetValueAsUnsigned() - ) + receiver_class = _dynamic_class(target, values[names[0]]) estimated_base = ( _estimated_json_size(location) + _estimated_json_size(receiver_class) @@ -370,79 +385,100 @@ def _record(context, thread, index, sequence, location, remaining_bytes): return record -def _handle_stop(context, events, other_stops): - """Record breakpoint hits of one stop; return an outcome when tracing must end.""" - process = context["process"] - for thread in process: - reason = thread.GetStopReason() - if reason == lldb.eStopReasonBreakpoint: - index = context["breakpoint_index"].get(thread.GetStopReasonDataAtIndex(0)) - if index is None: - continue - frame = thread.GetFrameAtIndex(0) - location = _location(frame.GetPCAddress(), context["target"]) - symbol = location["symbol"] or "" - if not _wanted(context["config"]["breakpoints"][index], symbol): - continue - frame_count = max( - 0, - min(thread.GetNumFrames() - 1, context["config"]["backtrace_frames"]), - ) - event_frame_count = 1 + frame_count - if not _can_retain_trace( - context["retained_trace_bytes"], - context["retained_frames"], - 0, - event_frame_count, - ): - return "resource-limit" +# One trace runs in each owned LLDB subprocess. Its callbacks execute at the +# breakpoint before LLDB resumes, unlike snapshots of later public stop events. +_TRACE_CONTEXT = None + + +def _handle_breakpoint(context, frame, index): + """Read one hit while LLDB's breakpoint callback owns the stopped frame.""" + if not frame.IsValid() or frame.GetPC() == lldb.LLDB_INVALID_ADDRESS: + raise RuntimeError("LLDB breakpoint callback did not supply a readable frame") + thread = frame.GetThread() + events = context["events"] + location = _location(frame.GetPCAddress(), context["target"]) + symbol = location["symbol"] or "" + if not _wanted(context["config"]["breakpoints"][index], symbol): + return None + frame_count = max( + 0, + min(thread.GetNumFrames() - 1, context["config"]["backtrace_frames"]), + ) + event_frame_count = 1 + frame_count + if not _can_retain_trace( + context["retained_trace_bytes"], + context["retained_frames"], + 0, + event_frame_count, + ): + return "resource-limit" + try: + remaining_bytes = MAX_TRACE_BYTES - context["retained_trace_bytes"] + record = _record( + context, thread, frame, index, len(events), location, remaining_bytes + ) + except ValueError: + return "resource-limit" + if record is not None: + if not _fits_trace_budget(record, remaining_bytes): + return "resource-limit" + record_bytes = len(json.dumps(record).encode("utf-8")) + if context["retained_trace_bytes"] + record_bytes > MAX_TRACE_BYTES: + return "resource-limit" + for depth in range(1, frame_count + 1): try: - remaining_bytes = MAX_TRACE_BYTES - context["retained_trace_bytes"] - record = _record( - context, thread, index, len(events), location, remaining_bytes + location = _location( + thread.GetFrameAtIndex(depth).GetPCAddress(), context["target"] ) except ValueError: return "resource-limit" - if record is not None: - if not _fits_trace_budget(record, remaining_bytes): - return "resource-limit" - record_bytes = len(json.dumps(record).encode("utf-8")) - if context["retained_trace_bytes"] + record_bytes > MAX_TRACE_BYTES: - return "resource-limit" - for depth in range(1, frame_count + 1): - try: - location = _location( - thread.GetFrameAtIndex(depth).GetPCAddress(), context["target"] - ) - except ValueError: - return "resource-limit" - if ( - _estimated_json_size(location) + record_bytes + 1 - > MAX_TRACE_BYTES - context["retained_trace_bytes"] - ): - return "resource-limit" - location_bytes = len(json.dumps(location).encode("utf-8")) - addition = location_bytes + (1 if record["backtrace"] else 0) - if ( - context["retained_trace_bytes"] - + record_bytes - + addition - > MAX_TRACE_BYTES - ): - return "resource-limit" - record["backtrace"].append(location) - record_bytes += addition - journal = context.get("observation_journal") - if journal is not None and not journal.append( - {"kind": "event", "event": record} - ): - return "resource-limit" - events.append(record) - context["retained_frames"] += event_frame_count - context["retained_trace_bytes"] += record_bytes - if len(events) >= context["config"]["max_events"]: - return "event-limit" - elif reason in (lldb.eStopReasonSignal, lldb.eStopReasonException): + if ( + _estimated_json_size(location) + record_bytes + 1 + > MAX_TRACE_BYTES - context["retained_trace_bytes"] + ): + return "resource-limit" + location_bytes = len(json.dumps(location).encode("utf-8")) + addition = location_bytes + (1 if record["backtrace"] else 0) + if ( + context["retained_trace_bytes"] + + record_bytes + + addition + > MAX_TRACE_BYTES + ): + return "resource-limit" + record["backtrace"].append(location) + record_bytes += addition + journal = context.get("observation_journal") + if journal is not None and not journal.append( + {"kind": "event", "event": record} + ): + return "resource-limit" + events.append(record) + context["retained_frames"] += event_frame_count + context["retained_trace_bytes"] += record_bytes + if len(events) >= context["config"]["max_events"]: + return "event-limit" + return None + + +def breakpoint_hit(frame, location, internal_dict): + """LLDB callback: capture before resuming, and stop on a limit or failure.""" + context = _TRACE_CONTEXT + if context is None or context["outcome"] is not None or context["failure"] is not None: + return True + try: + index = context["breakpoint_index"][location.GetBreakpoint().GetID()] + context["outcome"] = _handle_breakpoint(context, frame, index) + except Exception as error: + context["failure"] = "%s: %s" % (type(error).__name__, error) + return True + return context["outcome"] is not None + + +def _handle_stop(context, other_stops): + """Retain non-breakpoint stop observations; callbacks already capture hits.""" + for thread in context["process"]: + if thread.GetStopReason() in (lldb.eStopReasonSignal, lldb.eStopReasonException): stop_description = thread.GetStopDescription(256) journal = context.get("observation_journal") if journal is not None and not journal.append( @@ -510,7 +546,17 @@ def _wait_for_exit(listener, process, seconds): listener.WaitForEvent(1, event) +def _failed_launch_description(process): + if process.GetState() != lldb.eStateExited or process.GetExitStatus() != -1: + return None + description = process.GetExitDescription() + if isinstance(description, str) and description.lower().startswith(("attach failed", "launch failed")): + return description + return None + + def trace(debugger, config): + global _TRACE_CONTEXT debugger.SetAsync(True) # Stop at the symbol itself, where argument registers still hold the ABI # arguments, even when line tables would let LLDB skip the prologue. @@ -577,6 +623,11 @@ def trace(debugger, config): stdout.close() stderr.close() return {"status": "launch-error", "error": error.GetCString()} + launch_failure = _failed_launch_description(process) + if launch_failure is not None: + stdout.close(expected_exit=True) + stderr.close(expected_exit=True) + return {"status": "launch-error", "error": launch_failure} stdout.mark_launched() stderr.mark_launched() # Publish the PID before any post-launch file or module inspection. The @@ -650,6 +701,11 @@ def trace(debugger, config): "observation_journal": observation_journal, } events = [] + context.update({"events": events, "outcome": None, "failure": None}) + _TRACE_CONTEXT = context + for breakpoints in created: + for breakpoint in breakpoints: + breakpoint.SetScriptCallbackFunction("rea_lldb_tracer.breakpoint_hit") other_stops = [] outcome = None deadline = started + config["duration_ms"] / 1000 @@ -670,7 +726,9 @@ def trace(debugger, config): if state == lldb.eStateExited: outcome = "exited" elif state == lldb.eStateStopped and not lldb.SBProcess.GetRestartedFromEvent(event): - outcome = _handle_stop(context, events, other_stops) + if context["failure"] is not None: + raise RuntimeError(context["failure"]) + outcome = context["outcome"] or _handle_stop(context, other_stops) if outcome is None: process.Continue() pid = process.GetProcessID() @@ -685,6 +743,12 @@ def trace(debugger, config): process_exited = process.GetState() == lldb.eStateExited stdout.close(expected_exit=process_exited) stderr.close(expected_exit=process_exited) + # SBTarget.Launch may succeed before debugserver reports an asynchronous + # attach failure as an exited SBProcess. Its -1 status is not a target + # exit code; preserve the debugger's concrete launch denial. + launch_failure = _failed_launch_description(process) + if launch_failure is not None: + return {"status": "launch-error", "error": launch_failure} launch_identity_stable = ( module_path_before is not None and module_path_after == module_path_before @@ -733,6 +797,7 @@ def trace(debugger, config): } finally: + _TRACE_CONTEXT = None if observation_journal is not None: try: observation_journal.handle.close() diff --git a/bridge/process/ProcessRunTokenReader.swift b/bridge/process/ProcessRunTokenReader.swift index 1438f815773..21a2726fbb2 100644 --- a/bridge/process/ProcessRunTokenReader.swift +++ b/bridge/process/ProcessRunTokenReader.swift @@ -152,7 +152,14 @@ private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation { } bytes = Array(bytes.prefix(size)) do { - return RunTokenObservation(pid: pid, state: "readable", run_id: try token(in: bytes, pointerSize: pointerSize), reason: nil) + let runId = try token(in: bytes, pointerSize: pointerSize) + // A complete empty environment is readable for an ordinary image. Apple + // platform images can expose the same layout after the kernel withholds + // their environment, so they remain explicitly unverified without a token. + if runId == nil && isPlatformBinary(pid: pid) { + return unavailable(pid: pid, reason: "platform_binary_environment_withheld") + } + return RunTokenObservation(pid: pid, state: "readable", run_id: runId, reason: nil) } catch RunTokenReadError.duplicateToken { return unavailable(pid: pid, reason: "duplicate_run_token") } catch RunTokenReadError.invalidTokenEncoding { @@ -259,9 +266,6 @@ func token(in bytes: [UInt8], pointerSize: Int = MemoryLayout. throw RunTokenReadError.ambiguousEnvironmentBoundary } let environmentEntries = trailingEntries[.. result.events.map((event) => ({ + breakpoint_index: event.breakpoint_index, symbol: event.symbol, file_address: event.file_address, selector: event.selector, @@ -159,8 +155,7 @@ const checkBounds = async (binary) => { binary, [ JSON.stringify({ - breakpoints: [{ kind: "function", name: "nanosleep" }], - arguments: ["--wait"], + breakpoints: [{ kind: "function", name: "rea_call_add" }], max_events: 2, duration_ms: 30_000, }), @@ -189,8 +184,9 @@ const checkBounds = async (binary) => { }; const cliFailure = async (binary) => { + let output; try { - await exec( + const completed = await exec( process.execPath, [ fileURLToPath(new URL("rea.mjs", import.meta.url)), @@ -204,10 +200,11 @@ const cliFailure = async (binary) => { timeout: CLI_TIMEOUT_MS, }, ); + output = completed.stdout; } catch (cause) { return JSON.parse(cause.stdout); } - throw new Error("observe-native-calls unexpectedly succeeded"); + throw new Error(`observe-native-calls unexpectedly succeeded: ${output}`); }; /** Hardened runtime blocks the debugger unless get-task-allow is granted. */ @@ -226,6 +223,22 @@ const checkHardenedRuntime = async (directory, binary) => { const denied = await cliFailure(blocked); assert.equal(denied.code, "capability_unavailable"); assert.match(denied.details.reason, /^debugger-attach-denied: /u); + await withArtifactMcp(blocked, async (client) => { + const deniedMcp = await client.callTool( + { + name: "observe_native_calls", + arguments: INPUT, + }, + { timeout: CLI_TIMEOUT_MS }, + ); + assert.equal(deniedMcp.isError, true, JSON.stringify(deniedMcp)); + assert.equal(deniedMcp.structuredContent.error.code, denied.code); + assert.match( + deniedMcp.structuredContent.error.details.reason, + /^debugger-attach-denied: /u, + ); + assert.ok(deniedMcp.structuredContent.error.details.partial_observation); + }); const allowed = join(directory, "debuggable"); const entitlements = join(directory, "get-task-allow.plist"); await writeFile( @@ -267,14 +280,55 @@ try { "observe_native_calls", INPUT, ); + assert.equal( + viaMcp.process.outcome, + "exited", + JSON.stringify(viaMcp.coverage), + ); assert.deepEqual(stable(viaMcp), stable(viaCli)); assert.equal(viaMcp.process.stdout.text, viaCli.process.stdout.text); + + // Two caller selections may resolve to one address. Each must retain its + // own hit; consuming only the first stop-reason ID loses the second. + const overlapping = { + breakpoints: [ + { kind: "function", name: "rea_call_add" }, + { kind: "function", name: "rea_call_add", module: basename(binary) }, + ], + duration_ms: 30_000, + argument_registers: 2, + }; + const paired = await artifactMcpResult( + client, + "observe_native_calls", + overlapping, + ); + assert.equal(paired.process.outcome, "exited"); + assert.equal(paired.events.length, 6); + for (const index of [0, 1]) { + const hits = paired.events.filter( + (event) => event.breakpoint_index === index, + ); + assert.deepEqual( + hits.map((event) => event.registers[0].value), + ["0x0", "0x1", "0x2"], + ); + assert.ok(hits.every((event) => event.symbol === "rea_call_add")); + } + const capped = await artifactMcpResult(client, "observe_native_calls", { + ...overlapping, + max_events: 2, + }); + assert.equal(capped.process.outcome, "event-limit"); + assert.equal(capped.events.length, 2); + assert.equal(capped.process.terminated, true); }); await checkBounds(binary); await checkHardenedRuntime(directory, binary); console.log( JSON.stringify({ ok: true, + developer_mode: developerMode.stdout.trim(), mocked: false, cli: true, stdio_mcp: true, diff --git a/scripts/verify-native-ui.mjs b/scripts/verify-native-ui.mjs index 8e2f3e05e88..174989b142c 100644 --- a/scripts/verify-native-ui.mjs +++ b/scripts/verify-native-ui.mjs @@ -1,4 +1,5 @@ import { execFile, spawn } from "node:child_process"; +import assert from "node:assert/strict"; import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,6 +8,8 @@ import { promisify } from "node:util"; import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; import { observeNativeUi } from "../dist/native/NativeUiObservation.js"; import { NATIVE_UI_HELPER_MAX_BUFFER } from "../dist/native/NativeUiOutputBudget.js"; +import { artifactMcpResult, withArtifactMcp } from "./lib/artifact-e2e.mjs"; +import { observeProcessStartIdentity } from "../dist/process/ProcessOwnershipObservation.js"; if (process.platform !== "darwin") throw new Error( @@ -14,7 +17,9 @@ if (process.platform !== "darwin") ); const root = await mkdtemp(join(tmpdir(), "rea-ui-fixture-")); let child; +let childExit; let fixturePid; +let fixtureIdentity; let report; try { const childRetrievalTest = join(root, "child-retrieval-test"); @@ -89,6 +94,7 @@ try { ["-W", "-n", "-g", "--stdout", coordinates, join(root, "Fixture.app")], { stdio: "ignore" }, ); + childExit = new Promise((resolve) => child.once("exit", resolve)); let scope; const deadline = Date.now() + 20000; while (Date.now() < deadline) { @@ -104,6 +110,12 @@ try { "Source-owned UI fixture failed to expose one window within 20 seconds", ); fixturePid = scope.pid; + fixtureIdentity = await observeProcessStartIdentity(fixturePid); + assert.equal( + fixtureIdentity?.state, + "readable", + "Fixture OS start identity is unavailable", + ); const target = await parseBinaryTarget(executable); if (!target.ok) throw target.error; const observation = await observeNativeUi(target.value, "observe_native_ui", { @@ -180,6 +192,43 @@ try { `Native UI CLI did not preserve the selected scenario: ${JSON.stringify(evidence.error ?? evidence.normalized_result?.steps?.map(({ outcome, reason }) => ({ outcome, reason })))}`, ); scenarioStatus = "selected-AX-button-action-and-CLI-parity-observed"; + await withArtifactMcp(executable, async (client) => { + const observed = await artifactMcpResult(client, "observe_native_ui", { + ...scope, + screenshot: false, + }); + if ( + !observed.initial.nodes.some( + (node) => + node.role === "AXButton" && + JSON.stringify(node.path) === JSON.stringify(button.path), + ) + ) + throw new Error("MCP omitted the selected fixture's actual AX button"); + const captured = await artifactMcpResult( + client, + "capture_native_ui_scenario", + { + ...scope, + screenshot: false, + steps: [ + { kind: "click", path: button.path }, + { kind: "wait", milliseconds: 200 }, + ], + }, + ); + if ( + captured.steps.length !== 2 || + captured.steps.some((step) => step.outcome !== "completed") || + !captured.steps + .at(-1) + ?.after?.nodes.some( + (node) => node.title === "REA fixture incremented", + ) + ) + throw new Error("MCP did not observe the selected fixture's AX action"); + }); + scenarioStatus = "selected-AX-button-action-and-CLI-MCP-parity-observed"; } const mismatch = await observeNativeUi( { ...target.value, sha256: "0".repeat(64) }, @@ -201,17 +250,24 @@ try { target_owned: true, }; } finally { - if (fixturePid !== undefined) { - try { - process.kill(fixturePid, "SIGTERM"); - } catch (error) { - if (error.code !== "ESRCH") throw error; + if (fixturePid !== undefined && fixtureIdentity?.state === "readable") { + const current = await observeProcessStartIdentity(fixturePid); + if (current !== undefined) { + assert.ok( + current.state === "readable" && + current.identity === fixtureIdentity.identity, + "Fixture PID identity changed; refusing to signal another process", + ); + try { + process.kill(fixturePid, "SIGTERM"); + } catch (cause) { + if (cause.code !== "ESRCH") throw cause; + } } } if (child !== undefined && child.exitCode === null) { - const exited = new Promise((resolve) => child.once("exit", resolve)); child.kill("SIGTERM"); - await exited; + await childExit; } await rm(root, { recursive: true, force: true }); } diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index f72a78b4a67..7f234240bce 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -36,6 +36,14 @@ for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { ); await access(path); } +try { + await exec("7z", ["i"], { timeout: 10_000 }); +} catch (cause) { + throw new Error( + "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", + { cause }, + ); +} const fixtureRoot = resolve( process.env.REA_FIRMWARE_FIXTURE_ROOT ?? join(repository, "_reference/firmware-integration/generated"), diff --git a/src/process/ProcessOwnershipObservation.ts b/src/process/ProcessOwnershipObservation.ts index 723c1b27c85..6dc460d9704 100644 --- a/src/process/ProcessOwnershipObservation.ts +++ b/src/process/ProcessOwnershipObservation.ts @@ -1,4 +1,5 @@ import { readFile } from "node:fs/promises"; +import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import { launcherIdentityFailure } from "./ProcessOwnershipIdentity.js"; import { descendantsOf, liveProcesses } from "./ProcessOwnershipProcessTree.js"; @@ -300,6 +301,22 @@ export const createSystemProcessOwnershipHost = ( platform, prepare: async (signal) => { await darwinTokens?.prepare(signal); + if (platform === "linux") { + try { + const processes = await listProcesses(signal); + if (!processes.some(({ pid }) => pid === process.pid)) + throw new Error("ps did not report REA's current process"); + } catch (cause: unknown) { + if (isExpectedAbort(cause, signal)) throw cause; + const reason = `Linux process ownership inspection requires a procps-compatible ps on REA's PATH before launching a child: ${errorMessage(cause)}`; + throw new AnalysisCapabilityUnavailableError( + "process-ownership", + "prepare_owned_process", + reason, + { cause, userMessage: reason }, + ); + } + } }, listProcesses, async environment(pid) { diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 81fbf999723..8e2977f509d 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -548,7 +548,6 @@ it.skipIf(!onDarwin)( clearedAppleVectorTokenFailsClosed: true, callerPfzBeforeTokenRead: true, emptyLaterArgumentRead: true, - emptyEnvironmentFailsClosed: true, emptyArgv0Read: true, emptyEnvironmentRecordsBeforeTokenFailClosed: true, duplicateTokenFailsClosed: true, @@ -677,7 +676,7 @@ it.skipIf(!onDarwin)( ); it.skipIf(!onDarwin)( - "reads owned tokens with empty later arguments across executable path padding", + "reads owned tokens across executable padding and distinguishes a real empty environment", async () => { const root = await mkdtemp(join(tmpdir(), "rea-process-token-padding-")); const host = createSystemProcessOwnershipHost("darwin"); @@ -699,6 +698,7 @@ it.skipIf(!onDarwin)( ), ); } + children.push(await startNodeChild({})); const processes = await host.listProcesses(); const entries = children.map(({ pid }) => { const entry = processes.find((process) => process.pid === pid); @@ -710,6 +710,7 @@ it.skipIf(!onDarwin)( expect(children.map(({ pid }) => observations?.get(pid))).toEqual([ { state: "readable", runId: "synthetic-padding-0" }, { state: "readable", runId: "synthetic-padding-1" }, + { state: "readable" }, ]); } finally { for (const { child } of children) await stopNodeChild(child); diff --git a/tests/conformance/native/ui.swift b/tests/conformance/native/ui.swift index 0ee842b492c..1d399f00de0 100644 --- a/tests/conformance/native/ui.swift +++ b/tests/conformance/native/ui.swift @@ -4,6 +4,7 @@ let app = NSApplication.shared app.setActivationPolicy(.accessory) let window = NSWindow(contentRect: NSRect(x: 120, y: 120, width: 360, height: 180), styleMask: [.titled, .closable], backing: .buffered, defer: false) window.title = "REA source-owned UI verification" +window.animationBehavior = .none final class FixtureController: NSObject { @objc func increment(_ sender: NSButton) { sender.title = "REA fixture incremented" } } @@ -13,7 +14,8 @@ button.title = "Increment REA fixture" button.target = controller button.action = #selector(FixtureController.increment(_:)) window.contentView?.addSubview(button) -window.orderFront(nil) +window.makeKeyAndOrderFront(nil) +app.activate(ignoringOtherApps: true) DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) { let record = "{\"pid\":\(ProcessInfo.processInfo.processIdentifier),\"window_id\":\(window.windowNumber)}\n" FileHandle.standardOutput.write(Data(record.utf8)) diff --git a/tests/fixtures/processRunTokenParserProbe.swift b/tests/fixtures/processRunTokenParserProbe.swift index b7185d4337a..a753511f96a 100644 --- a/tests/fixtures/processRunTokenParserProbe.swift +++ b/tests/fixtures/processRunTokenParserProbe.swift @@ -132,13 +132,6 @@ struct ProcessRunTokenParserProbe { } catch RunTokenReadError.duplicateToken { duplicateTokenFailsClosed = true } - let emptyEnvironmentFailsClosed: Bool - do { - _ = try token(in: fixture(arguments: ["/fixture/executable"], environment: [])) - emptyEnvironmentFailsClosed = false - } catch RunTokenReadError.environmentUnavailable { - emptyEnvironmentFailsClosed = true - } let missingAppleBoundaryFailsClosed: Bool do { _ = try token(in: fixture( @@ -158,7 +151,6 @@ struct ProcessRunTokenParserProbe { "alternatePaddingRead": alternatePadding == "alternate-padding-token", "truncatedArgvFailsClosed": truncatedArgvFailsClosed, "duplicateTokenFailsClosed": duplicateTokenFailsClosed, - "emptyEnvironmentFailsClosed": emptyEnvironmentFailsClosed, "missingAppleBoundaryFailsClosed": missingAppleBoundaryFailsClosed, "emptyArgv0Read": emptyArgv0Read == "empty-argv0-token", "finalAssignmentArgumentRead": finalAssignmentArgumentRead == "final-assignment-token", From 329cf206d1111c051733c7115cabff4b46722deb Mon Sep 17 00:00:00 2001 From: vilmerfrost <101505019+vilmerfrost@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:51:53 +0200 Subject: [PATCH 064/791] fix(hopper): preserve owned basic-block endpoint instructions (#1118) * fix(hopper): preserve owned basic-block endpoint instructions * test(hopper): share probe runner and restore fixture hashability Reuse one probe helper with the facade test's execFile options (encoding, timeout, maxBuffer) instead of repeating the invocation. Give the fixture Block a hash consistent with its equality so the equality-based ownership doubles stay hashable. * fix(hopper): align owned instruction walks with normalized block ranges --------- Co-authored-by: Vilmer Frost Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- bridge/hopper_bridge.py | 44 ++-- .../hopper/hopperInstructions.test.ts | 105 ++++++++++ tests/fixtures/hopperInstructionProbe.py | 191 ++++++++++++++++++ 3 files changed, 328 insertions(+), 12 deletions(-) create mode 100644 tests/boundary/providers/hopper/hopperInstructions.test.ts create mode 100644 tests/fixtures/hopperInstructionProbe.py diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 660ff52ea23..d19fb0ae87d 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -318,34 +318,54 @@ def _containing_procedure(document, address): def _basic_block_end(procedure, block): """Normalize native block endpoints to an exclusive byte address.""" + owner_at = getattr(procedure, "getBasicBlockAtAddress", None) + if not callable(owner_at): + raise CapabilityUnavailableError( + "Instruction enumeration requires Hopper basic-block ownership" + ) end = block.getEndingAddress() # Hopper 6.1 returns the final instruction address, despite the Python # documentation describing an exclusive end. Membership distinguishes # that representation from an actual exclusive endpoint in other builds. - if procedure.getBasicBlockAtAddress(end) == block: + if owner_at(end) == block: instruction = procedure.getSegment().getInstructionAtAddress(end) if instruction is not None and instruction.getInstructionLength() > 0: return end + instruction.getInstructionLength() return end +def _block_instruction_addresses(procedure, block): + """Walk either endpoint convention using Hopper's block ownership API. + + Some Hopper builds return the final instruction start, despite the public + API documenting an exclusive end. Equality compares fresh native wrappers; + probing ownership also keeps an exclusive endpoint in the next block out. + """ + segment = procedure.getSegment() + address = block.getStartingAddress() + end = _basic_block_end(procedure, block) + while address < end: + if procedure.getBasicBlockAtAddress(address) != block: + break + instruction = segment.getInstructionAtAddress(address) + if instruction is None: + break + length = instruction.getInstructionLength() + if length <= 0 or address + length > end: + break + yield address + address += length + + def _instruction_addresses(procedure): result = [] seen = set() - segment = procedure.getSegment() for block in procedure.basicBlockIterator(): - address = block.getStartingAddress() - end = _basic_block_end(procedure, block) - while address < end and address not in seen: + for address in _block_instruction_addresses(procedure, block): + if address in seen: + continue seen.add(address) - instruction = segment.getInstructionAtAddress(address) - if instruction is None: - break result.append(address) - length = instruction.getInstructionLength() - if length <= 0: - break - address += length return result diff --git a/tests/boundary/providers/hopper/hopperInstructions.test.ts b/tests/boundary/providers/hopper/hopperInstructions.test.ts new file mode 100644 index 00000000000..db22c3b9f09 --- /dev/null +++ b/tests/boundary/providers/hopper/hopperInstructions.test.ts @@ -0,0 +1,105 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +const execute = promisify(execFile); +const bridge = fileURLToPath( + new URL("../../../../bridge/hopper_bridge.py", import.meta.url), +); +const probe = fileURLToPath( + new URL("../../../fixtures/hopperInstructionProbe.py", import.meta.url), +); +const resultSchema = z.object({ + addresses: z.array(z.number()), + blocks: z.array(z.object({ start: z.string(), end: z.string() })), + length: z.number(), + assembly: z.array(z.string()), + fast: z.array(z.string()), + dossier: z.array(z.string()), + references: z.array(z.object({ source_address: z.string() })), +}); + +type ProbeResult = z.infer; + +async function runProbe(scenario: string): Promise { + const { stdout } = await execute("python3", [probe, bridge, scenario], { + encoding: "utf8", + timeout: 3_000, + maxBuffer: 1_024 * 1_024, + }); + return resultSchema.parse(JSON.parse(stdout)); +} + +describe("Hopper instruction block boundaries", () => { + it.each(["inclusive", "exclusive", "duplicate"])( + "preserves branches and returns with %s endpoints without duplicates or adjacent instructions", + async (scenario) => { + const result = await runProbe(scenario); + expect(result.addresses).toEqual([0x1000, 0x1004, 0x1008]); + if (scenario !== "duplicate") { + expect(result.blocks).toEqual([ + { start: "0x1000", end: "0x1008" }, + { start: "0x1008", end: "0x100c" }, + ]); + expect(result.length).toBe(12); + } + const lines = ["0x1000: mov", "0x1004: b.ne", "0x1008: ret"]; + expect(result.assembly).toEqual(lines); + expect(result.fast).toEqual(lines); + expect(result.dossier).toEqual(lines); + expect(result.references.map((edge) => edge.source_address)).toEqual([ + "0x1004", + ]); + }, + ); + + it.each([ + ["gap", [0x1000]], + ["zero_length", [0x1000]], + ["negative_length", [0x1000]], + ["cross_boundary", []], + ["foreign_owner", [0x1000, 0x1004]], + ["reversed", []], + ])("stops safely at %s", async (scenario, addresses) => { + const result = await runProbe(scenario); + expect(result.addresses).toEqual(addresses); + expect(result.assembly).toHaveLength(addresses.length); + expect(result.fast).toEqual(result.assembly); + expect(result.dossier).toEqual(result.assembly); + }); + + it("uses instruction lengths without assuming ARM64 fixed-width encoding", async () => { + const result = await runProbe("variable_length"); + expect(result.addresses).toEqual([0x1000, 0x1003, 0x1005]); + expect(result.blocks).toEqual([ + { start: "0x1000", end: "0x1005" }, + { start: "0x1005", end: "0x1006" }, + ]); + expect(result.length).toBe(6); + expect(result.assembly).toEqual([ + "0x1000: mov", + "0x1003: b.ne", + "0x1005: ret", + ]); + expect(result.fast).toEqual(result.assembly); + expect(result.dossier).toEqual(result.assembly); + }); + + it("reports missing ownership support instead of guessing endpoint semantics", async () => { + const { stdout } = await execute( + "python3", + [probe, bridge, "no_ownership"], + { + encoding: "utf8", + timeout: 3_000, + maxBuffer: 1_024 * 1_024, + }, + ); + expect( + z.object({ error: z.string() }).parse(JSON.parse(stdout)).error, + ).toBe("Instruction enumeration requires Hopper basic-block ownership"); + }); +}); diff --git a/tests/fixtures/hopperInstructionProbe.py b/tests/fixtures/hopperInstructionProbe.py new file mode 100644 index 00000000000..8e023a0ce6c --- /dev/null +++ b/tests/fixtures/hopperInstructionProbe.py @@ -0,0 +1,191 @@ +"""Exercise the real bridge instruction walker with Hopper-shaped objects.""" + +import json +from pathlib import Path +import sys + + +class Block: + def __init__(self, key, start, end, members): + self.key, self.start, self.end, self.members = key, start, end, members + + def __eq__(self, other): + return isinstance(other, Block) and self.key == other.key + + def __hash__(self): + return hash(self.key) + + def getStartingAddress(self): + return self.start + + def getEndingAddress(self): + return self.end + + def getSuccessorCount(self): + return 0 + + +class Instruction: + def __init__(self, mnemonic, length=4): + self.mnemonic, self.length = mnemonic, length + + def getInstructionLength(self): + return self.length + + def getInstructionString(self): + return self.mnemonic + + def getArgumentCount(self): + return 0 + + +class Segment: + def __init__(self, instructions): + self.instructions = instructions + + def getInstructionAtAddress(self, address): + return self.instructions.get(address) + + def getNameAtAddress(self, address): + return "fixture" + + def getReferencesFromAddress(self, address): + return [0x9000] if self.instructions[address].mnemonic.startswith("b") else [] + + def getReferencesOfAddress(self, address): + return [] + + def getCommentAtAddress(self, address): + return None + + def getInlineCommentAtAddress(self, address): + return None + + +class Procedure: + def __init__(self, blocks, segment): + self.blocks, self.segment = blocks, segment + + def basicBlockIterator(self): + return iter(self.blocks) + + def getBasicBlockAtAddress(self, address): + for block in self.blocks: + if address in block.members: + # Hopper returns fresh wrappers whose native handles compare equal. + return Block(block.key, block.start, block.end, block.members) + return None + + def getBasicBlockCount(self): + return len(self.blocks) + + def getAllCallees(self): + return [] + + def getAllCallers(self): + return [] + + def getSegment(self): + return self.segment + + def getEntryPoint(self): + return self.blocks[0].start + + def signatureString(self): + return "int fixture()" + + def getLocalVariableList(self): + return [] + + def getAllCallerProcedures(self): + return [] + + def getAllCalleeProcedures(self): + return [] + + def decompile(self): + return "fixture pseudocode" + + +def main(): + path = sys.argv[1] + bridge = {"__file__": path, "__name__": "rea_hopper_bridge"} + exec(compile(Path(path).read_text(encoding="utf-8"), path, "exec"), bridge) + scenario = sys.argv[2] + instructions = {0x1000: Instruction("mov"), 0x1004: Instruction("b.ne"), + 0x1008: Instruction("ret"), 0x100C: Instruction("nop")} + if scenario == "inclusive": + blocks = [Block(0, 0x1000, 0x1004, {0x1000, 0x1004}), + Block(1, 0x1008, 0x1008, {0x1008})] + elif scenario == "exclusive": + blocks = [Block(0, 0x1000, 0x1008, {0x1000, 0x1004}), + Block(1, 0x1008, 0x100C, {0x1008})] + elif scenario == "duplicate": + block = Block(0, 0x1000, 0x1008, {0x1000, 0x1004, 0x1008}) + blocks = [block, block] + elif scenario == "gap": + instructions.pop(0x1004) + blocks = [Block(0, 0x1000, 0x1008, {0x1000, 0x1004, 0x1008})] + elif scenario == "zero_length": + instructions[0x1004] = Instruction("invalid", 0) + blocks = [Block(0, 0x1000, 0x1008, {0x1000, 0x1004, 0x1008})] + elif scenario == "negative_length": + instructions[0x1004] = Instruction("invalid", -4) + blocks = [Block(0, 0x1000, 0x1008, {0x1000, 0x1004, 0x1008})] + elif scenario == "cross_boundary": + instructions[0x1000] = Instruction("invalid", 8) + blocks = [Block(0, 0x1000, 0x1004, {0x1000})] + elif scenario == "foreign_owner": + blocks = [Block(0, 0x1000, 0x100C, {0x1000, 0x1004})] + elif scenario == "reversed": + blocks = [Block(0, 0x1008, 0x1000, {0x1008})] + elif scenario == "variable_length": + instructions = {0x1000: Instruction("mov", 3), + 0x1003: Instruction("b.ne", 2), + 0x1005: Instruction("ret", 1), + 0x1006: Instruction("nop", 1)} + blocks = [Block(0, 0x1000, 0x1003, {0x1000, 0x1003}), + Block(1, 0x1005, 0x1005, {0x1005})] + elif scenario == "no_ownership": + blocks = [Block(0, 0x1000, 0x1004, {0x1000})] + else: + raise ValueError("Unknown scenario") + segment = Segment(instructions) + procedure = Procedure(blocks, segment) + if scenario == "no_ownership": + procedure.getBasicBlockAtAddress = None + try: + bridge["_instruction_addresses"](procedure) + except bridge["CapabilityUnavailableError"] as error: + print(json.dumps({"error": str(error)})) + return + raise AssertionError("Missing ownership API was silently accepted") + class API: + def require_analysis_complete(self, document, method): + pass + + class Document: + def getSegmentsList(self): + return [] + + document = Document() + bridge["_api"] = lambda: API() + bridge["_document"] = lambda name=None: document + bridge["_procedure"] = lambda document, value=None: procedure + bridge["_segment"] = lambda document, address: segment + bridge["_containing_procedure"] = lambda document, address: (None, None) + bridge["_search_inventory"] = lambda document, kind: [] + dossier = bridge["_analyze_function"](document, {}) + print(json.dumps({ + "blocks": dossier["basic_blocks"], + "length": bridge["_dispatch"]("procedure_info", {})["length"], + "addresses": bridge["_instruction_addresses"](procedure), + "assembly": bridge["_assembly"](procedure).splitlines(), + "fast": bridge["_read_function_instructions"](document, {})["instructions"], + "dossier": dossier["assembly"], + "references": bridge["_procedure_references"](document, {})["references"], + })) + + +if __name__ == "__main__": + main() From e8939ca2208bc41454b73b90a32418714e6a2969 Mon Sep 17 00:00:00 2001 From: vilmerfrost <101505019+vilmerfrost@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:52:07 +0200 Subject: [PATCH 065/791] fix(hopper): enforce caller-supplied request deadlines (#1119) * fix(hopper): enforce caller-supplied request deadlines * fix(hopper): correct idle deadline state and name timeout bound Zero-ms deadlines never enter the FIFO, so report providerState not_started instead of busy; the busy wording tells callers to wait for an active request that does not exist. Name the Node timer ceiling once and reuse it in validation and the error message. * fix(hopper): enforce elapsed deadlines before dispatch and replies * fix(hopper): distinguish unsent requests in timeout diagnostics * test(hopper): exercise request deadlines against native Hopper --------- Co-authored-by: Vilmer Frost Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/testing.md | 9 + package.json | 1 + scripts/verify-real-hopper-deadlines.mjs | 126 ++++++++ src/hopper/HopperClient.ts | 5 + src/hopper/HopperRequestQueue.test.ts | 300 ++++++++++++++++++ src/hopper/HopperRequestQueue.ts | 100 +++++- .../hopper/hopperClient.deadlines.test.ts | 151 +++++++++ 7 files changed, 688 insertions(+), 4 deletions(-) create mode 100644 scripts/verify-real-hopper-deadlines.mjs create mode 100644 src/hopper/HopperRequestQueue.test.ts create mode 100644 tests/boundary/providers/hopper/hopperClient.deadlines.test.ts diff --git a/docs/testing.md b/docs/testing.md index d63f49a1238..70fadaac337 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -221,6 +221,15 @@ decompilation, reference and search operations. A literal `--help` trace query p that selector data is preserved independently of global CLI flags. Unmapped annotation destinations and oversized later batch names fail before any earlier edit is applied. + +`verify:hopper:deadlines` checks the native client's optional request deadlines +on macOS with an owned source-built fixture. Zero and elapsed deadlines must +leave native comments unchanged, including when a synchronous progress observer +delays timer dispatch. It also observes a short analysis deadline and verifies +subsequent wire recovery and clean shutdown. Caller timeout does not interrupt +Hopper's synchronous native operation. Socket boundary tests deterministically +cover active and queued expiry, late replies and timer cleanup. + The Linux demo lane remains a separate `verify:hopper:linux` command. `verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the diff --git a/package.json b/package.json index 9276c2e1417..68d6b5bb7fd 100644 --- a/package.json +++ b/package.json @@ -138,6 +138,7 @@ "typecheck": "tsc --noEmit", "verify:test-temp-hygiene": "npm run build:cached && node scripts/verify-test-temp-hygiene.mjs", "verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs", + "verify:hopper:deadlines": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper-deadlines.mjs", "verify:hopper:fat": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs", "verify:hopper:fat64": "npm run build:cached && node scripts/verify-real-hopper-fat.mjs --fat64", "verify:apple-dispatch": "npm run build:cached && node scripts/verify/apple/dispatch.mjs", diff --git a/scripts/verify-real-hopper-deadlines.mjs b/scripts/verify-real-hopper-deadlines.mjs new file mode 100644 index 00000000000..4382b417416 --- /dev/null +++ b/scripts/verify-real-hopper-deadlines.mjs @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { copyFile, mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { firstProcedureAddress } from "../dist/application/RealHopperAssertions.js"; +import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { resolveHopperAnalysisProfile } from "../dist/hopper/HopperAnalysisProfile.js"; +import { HopperApplicationLauncher } from "../dist/hopper/BridgeLauncher.js"; +import { HopperClient } from "../dist/hopper/HopperClient.js"; +import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js"; +import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs"; + +// These options belong to the native client, below the CLI/MCP tool surface. +assert.equal( + process.platform, + "darwin", + "This lane requires native macOS Hopper", +); +const fixtures = await loadRealHopperFixtureTargets( + process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH ?? + "build/conformance/manifest.json", +); +const root = await realpath( + await mkdtemp(join(tmpdir(), "rea-hopper-deadlines-")), +); +const target = join(root, "deadline-fixture"); +let client; +try { + await copyFile(fixtures.primary.path, target); + const parsed = await parseBinaryTarget(target); + if (!parsed.ok) throw parsed.error; + const launcherPath = + process.env.HOPPER_LAUNCHER_PATH ?? + "/Applications/Hopper Disassembler.app/Contents/MacOS/hopper"; + const resolved = await resolveHopperAnalysisProfile(parsed.value, { + launcherPath, + loaderArgsOverride: [], + provider: HOPPER_PROVIDER_IDENTITY, + }); + if (!resolved.ok) throw resolved.error; + const image = resolved.value.profile?.parameters.prepared_image; + client = new HopperClient({ + launcher: new HopperApplicationLauncher({ + launcherPath, + targetPath: target, + targetKind: "executable", + launchMode: "native", + loaderArgs: resolved.value.compatibility.loaderArgs, + ...(image === undefined + ? {} + : { + preparedImage: { image, sourceSha256: parsed.value.sha256 }, + }), + bridgeScriptPath: fileURLToPath( + new URL("../bridge/hopper_bridge.py", import.meta.url), + ), + }), + }); + const call = async (name, args = {}) => { + const result = await client.callTool(name, args); + if (!result.ok) throw result.error; + return result.value; + }; + const procedures = await call("list_procedures"); + const address = firstProcedureAddress(procedures); + const original = await call("comment", { address }); + for (const timeoutMs of [0, 10]) { + const result = await client.callTool( + "set_comment", + { address, comment: "expired mutation must never reach Hopper" }, + { + timeoutMs, + ...(timeoutMs === 0 + ? {} + : { + progress: { + report: () => { + const started = performance.now(); + while (performance.now() - started < 30) { + // Delay the timer callback beyond the admitted deadline. + } + return Promise.resolve(); + }, + }, + }), + }, + ); + assert.equal(result.ok, false); + assert.equal(result.error._tag, "HopperTimeoutError"); + assert.equal(result.error.providerState, "not_started"); + assert.equal(await call("comment", { address }), original); + assert.equal(client.requestActivity(), null); + } + const active = await client.callTool( + "analyze_function", + { procedure: address }, + { timeoutMs: 1 }, + ); + if (!active.ok) assert.equal(active.error._tag, "HopperTimeoutError"); + const activityAfterDeadline = client.requestActivity(); + await call("health"); + assert.equal(client.requestActivity(), null); + const closed = await client.closeWithOutcome(); + if (!closed.ok) throw closed.error; + console.log( + JSON.stringify( + { + expiredNativeMutations: "not_applied", + delayedProgress: "deadline_enforced", + activeAnalysis: active.ok + ? "completed_within_deadline" + : "caller_timed_out", + nativeWorkRetained: activityAfterDeadline !== null, + lateReplyRecovery: "passed", + cleanShutdown: true, + }, + null, + 2, + ), + ); +} finally { + if (client !== undefined) await client.close(); + await rm(root, { recursive: true, force: true }); +} diff --git a/src/hopper/HopperClient.ts b/src/hopper/HopperClient.ts index 20430c16e67..29edb484b15 100644 --- a/src/hopper/HopperClient.ts +++ b/src/hopper/HopperClient.ts @@ -405,6 +405,8 @@ export class HopperClient { arguments_: Readonly> = {}, options: { readonly signal?: AbortSignal; + /** Deadline after startup, including waiting for Hopper's serial bridge. */ + readonly timeoutMs?: number; readonly progress?: ProgressReporter; } = {}, ): Promise> { @@ -638,6 +640,9 @@ export class HopperClient { const startedAt = performance.now(); const result = await this.#requests.run(id, method, params, { ...(options.signal === undefined ? {} : { signal: options.signal }), + ...(options.timeoutMs === undefined + ? {} + : { timeoutMs: options.timeoutMs }), ...(options.progress !== undefined ? { progress: options.progress } : {}), }); this.#retainRequestHealth(result, id, method); diff --git a/src/hopper/HopperRequestQueue.test.ts b/src/hopper/HopperRequestQueue.test.ts new file mode 100644 index 00000000000..183d67bae45 --- /dev/null +++ b/src/hopper/HopperRequestQueue.test.ts @@ -0,0 +1,300 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + HopperCancelledError, + HopperProtocolError, +} from "../domain/hopperErrors.js"; +import { ok } from "../domain/result.js"; +import { HopperRequestQueue } from "./HopperRequestQueue.js"; + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +describe("HopperRequestQueue deadlines", () => { + it("times out active callers while retaining the wire until the late reply", async () => { + vi.useFakeTimers(); + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const result = vi.fn(); + const pending = queue + .run(1, "analyze_function", {}, { timeoutMs: 25 }) + .then(result); + const next = queue.run(2, "echo", {}, {}); + await vi.advanceTimersByTimeAsync(25); + expect(result).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ + _tag: "HopperTimeoutError", + timeoutMs: 25, + operation: "analyze_function", + requestId: 1, + providerState: "busy", + }), + }); + expect(sent).toEqual([1]); + expect(queue.activity()).toMatchObject({ + callerState: "cancelled", + queuedRequests: 1, + }); + expect(vi.getTimerCount()).toBe(0); + expect(queue.accept(1, ok("late"))).toBe(true); + expect(sent).toEqual([1, 2]); + queue.accept(2, ok("next")); + await pending; + await expect(next).resolves.toEqual(ok("next")); + expect(result).toHaveBeenCalledTimes(1); + expect(queue.activity()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("expires a queued request without transmitting it", async () => { + vi.useFakeTimers(); + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const active = queue.run(1, "active", {}, {}); + const settled = vi.fn(); + const queued = queue.run(2, "queued", {}, { timeoutMs: 10 }).then(settled); + await vi.advanceTimersByTimeAsync(10); + expect(settled).toHaveBeenCalledWith( + expect.objectContaining({ + ok: false, + error: expect.objectContaining({ + _tag: "HopperTimeoutError", + requestId: 2, + providerState: "not_started", + }), + }), + ); + expect(queue.hasQueued(2)).toBe(false); + queue.accept(1, ok(null)); + await Promise.all([active, queued]); + expect(sent).toEqual([1]); + expect(vi.getTimerCount()).toBe(0); + }); + + it("cleans deadline and abort observers on success before the deadline", async () => { + vi.useFakeTimers(); + const queue = new HopperRequestQueue(() => {}); + const controller = new AbortController(); + const remove = vi.spyOn(controller.signal, "removeEventListener"); + const settled = vi.fn(); + const pending = queue + .run(1, "echo", {}, { timeoutMs: 25, signal: controller.signal }) + .then(settled); + queue.accept(1, ok(42)); + controller.abort(); + await vi.advanceTimersByTimeAsync(100); + await pending; + expect(settled).toHaveBeenCalledExactlyOnceWith(ok(42)); + expect(remove).toHaveBeenCalledWith("abort", expect.any(Function)); + expect(vi.getTimerCount()).toBe(0); + }); + + it("keeps explicit cancellation distinct from a later timeout", async () => { + vi.useFakeTimers(); + const queue = new HopperRequestQueue(() => {}); + const controller = new AbortController(); + const pending = queue.run( + 1, + "echo", + {}, + { timeoutMs: 25, signal: controller.signal }, + ); + controller.abort(); + await expect(pending).resolves.toEqual({ + ok: false, + error: new HopperCancelledError(), + }); + expect(vi.getTimerCount()).toBe(0); + queue.accept(1, ok(null)); + await vi.advanceTimersByTimeAsync(100); + expect(queue.activity()).toBeNull(); + }); + + it("does not recreate timers when a progress observer cancels synchronously", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const queue = new HopperRequestQueue(send); + const controller = new AbortController(); + const pending = queue.run( + 1, + "echo", + {}, + { + timeoutMs: 100, + signal: controller.signal, + progress: { + report: () => { + controller.abort(); + return Promise.resolve(); + }, + }, + }, + ); + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperCancelledError" }, + }); + expect(vi.getTimerCount()).toBe(0); + expect(send).not.toHaveBeenCalled(); + expect(queue.activity()).toBeNull(); + const next = queue.run(2, "next", {}, {}); + expect(send).toHaveBeenCalledExactlyOnceWith( + { id: 2, method: "next", params: {} }, + expect.any(Function), + ); + queue.accept(2, ok(null)); + await next; + }); + + it("cleans all deadlines on teardown, including after an active timeout", async () => { + vi.useFakeTimers(); + const queue = new HopperRequestQueue(() => {}); + const result = vi.fn(); + const active = queue.run(1, "active", {}, { timeoutMs: 10 }).then(result); + const queued = queue.run(2, "queued", {}, { timeoutMs: 100 }); + await vi.advanceTimersByTimeAsync(10); + const failure = new HopperProtocolError("closed"); + queue.failAll(failure); + await active; + await expect(queued).resolves.toEqual({ ok: false, error: failure }); + expect(result).toHaveBeenCalledTimes(1); + expect(queue.activity()).toBeNull(); + expect(queue.queuedCount()).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("HopperRequestQueue deadline admission and races", () => { + it("removes queued cancelled work and honours cancellation before admission", async () => { + vi.useFakeTimers(); + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const active = queue.run(1, "active", {}, {}); + const controller = new AbortController(); + const queued = queue.run( + 2, + "queued", + {}, + { timeoutMs: 100, signal: controller.signal }, + ); + controller.abort(); + await expect(queued).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperCancelledError" }, + }); + await expect( + queue.run( + 3, + "pre-aborted", + {}, + { timeoutMs: 100, signal: controller.signal }, + ), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperCancelledError" }, + }); + expect(queue.queuedCount()).toBe(0); + queue.accept(1, ok(null)); + await active; + expect(sent).toEqual([1]); + expect(vi.getTimerCount()).toBe(0); + }); + + it.each([NaN, Infinity, -1, 0.5, 2_147_483_648])( + "rejects invalid deadline %s without sending", + async (timeoutMs) => { + vi.useFakeTimers(); + const send = vi.fn(); + const queue = new HopperRequestQueue(send); + await expect( + queue.run(1, "echo", {}, { timeoutMs }), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperProtocolError" }, + }); + expect(send).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }, + ); + + it("expires zero deadlines without sending and settles only once if aborted afterward", async () => { + vi.useFakeTimers(); + const send = vi.fn(); + const queue = new HopperRequestQueue(send); + await expect( + queue.run(1, "echo", {}, { timeoutMs: 0 }), + ).resolves.toMatchObject({ + ok: false, + error: { + _tag: "HopperTimeoutError", + timeoutMs: 0, + providerState: "not_started", + }, + }); + expect(send).not.toHaveBeenCalled(); + const controller = new AbortController(); + const settled = vi.fn(); + const pending = queue + .run(2, "echo", {}, { timeoutMs: 10, signal: controller.signal }) + .then(settled); + await vi.advanceTimersByTimeAsync(10); + controller.abort(); + queue.accept(2, ok("late")); + await pending; + expect(settled).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + ok: false, + error: expect.objectContaining({ _tag: "HopperTimeoutError" }), + }), + ); + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("Hopper deadlines before timer dispatch", () => { + it("does not transmit overdue queued work when the event loop delays its timer", async () => { + vi.useFakeTimers(); + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const active = queue.run(1, "active", {}, {}); + const expired = queue.run(2, "set_comment", {}, { timeoutMs: 10 }); + clock.mockReturnValue(20); + queue.accept(1, ok(null)); + await expect(expired).resolves.toMatchObject({ + ok: false, + error: { + _tag: "HopperTimeoutError", + operation: "set_comment", + providerState: "not_started", + }, + }); + await active; + expect(sent).toEqual([1]); + expect(queue.activity()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("accepts an overdue active reply without turning elapsed timeout into success", async () => { + vi.useFakeTimers(); + const clock = vi.spyOn(performance, "now").mockReturnValue(0); + const sent: number[] = []; + const queue = new HopperRequestQueue(({ id }) => sent.push(id)); + const expired = queue.run(1, "active", {}, { timeoutMs: 10 }); + const next = queue.run(2, "next", {}, {}); + clock.mockReturnValue(20); + expect(queue.accept(1, ok("late"))).toBe(true); + await expect(expired).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperTimeoutError" }, + }); + expect(sent).toEqual([1, 2]); + queue.accept(2, ok(null)); + await next; + expect(queue.activity()).toBeNull(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/hopper/HopperRequestQueue.ts b/src/hopper/HopperRequestQueue.ts index 4cf2d7e0039..6ae162e06f9 100644 --- a/src/hopper/HopperRequestQueue.ts +++ b/src/hopper/HopperRequestQueue.ts @@ -10,14 +10,20 @@ import { HopperProcessError, HopperProtocolError, HopperRemoteError, + HopperTimeoutError, } from "../domain/hopperErrors.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { err, type Result } from "../domain/result.js"; export type HopperRequestResult = Result; +/** Node timer ceiling; larger deadlines would be clamped instead of honored. */ +const MAX_TIMEOUT_MS = 2_147_483_647; + export interface HopperRequestQueueOptions { readonly signal?: AbortSignal; + /** Caller deadline from admission, including time waiting in the FIFO. */ + readonly timeoutMs?: number; readonly progress?: ProgressReporter; } @@ -48,8 +54,12 @@ interface QueuedRequest { readonly onAbort: (() => void) | undefined; callerState: "waiting" | "cancelled"; callerSettled: boolean; + transmitted: boolean; startedAt: number | undefined; heartbeat: NodeJS.Timeout | undefined; + deadline: NodeJS.Timeout | undefined; + readonly deadlineAt: number | undefined; + readonly timeoutMs: number | undefined; } /** FIFO that keeps the wire serialized until Hopper actually replies. */ @@ -59,7 +69,7 @@ export class HopperRequestQueue { constructor(private readonly send: RequestSender) {} - /** Queue one request until Hopper replies or the caller cancels it. */ + /** Settle the caller on reply, cancellation, or deadline; retain active wire work. */ run( id: number, method: string, @@ -68,6 +78,32 @@ export class HopperRequestQueue { ): Promise { if (options.signal?.aborted === true) return Promise.resolve(err(new HopperCancelledError())); + const timeoutMs = options.timeoutMs; + if ( + timeoutMs !== undefined && + (!Number.isInteger(timeoutMs) || + timeoutMs < 0 || + timeoutMs > MAX_TIMEOUT_MS) + ) + return Promise.resolve( + err( + new HopperProtocolError( + `Hopper timeoutMs must be an integer between 0 and ${String(MAX_TIMEOUT_MS)}`, + ), + ), + ); + if (timeoutMs === 0) + return Promise.resolve( + err( + new HopperTimeoutError( + 0, + method, + id, + "not_started", + hopperOperationStage(method), + ), + ), + ); return new Promise((resolve) => { let entry: QueuedRequest; const onAbort = @@ -82,12 +118,22 @@ export class HopperRequestQueue { onAbort, callerState: "waiting", callerSettled: false, + transmitted: false, startedAt: undefined, heartbeat: undefined, + deadline: undefined, + deadlineAt: + timeoutMs === undefined ? undefined : performance.now() + timeoutMs, + timeoutMs, }; this.#queue.push(entry); if (onAbort !== undefined) options.signal?.addEventListener("abort", onAbort, { once: true }); + if (timeoutMs !== undefined) + entry.deadline = setTimeout( + () => this.#cancel(entry, this.#timeoutError(entry, timeoutMs)), + timeoutMs, + ); if (this.#active !== undefined || this.#queue.length > 1) this.#report( entry, @@ -101,6 +147,7 @@ export class HopperRequestQueue { accept(id: number, result: HopperRequestResult): boolean { const entry = this.#active; if (entry === undefined || entry.id !== id) return false; + this.#expireIfElapsed(entry); this.#releaseWire(entry); if (!entry.callerSettled) this.#settleCaller( @@ -187,6 +234,10 @@ export class HopperRequestQueue { if (this.#active !== undefined) return; const entry = this.#queue.shift(); if (entry === undefined) return; + if (this.#expireIfElapsed(entry)) { + this.#drain(); + return; + } if (entry.signal?.aborted === true) { this.#settleCaller(entry, err(new HopperCancelledError()), "cancelled"); this.#drain(); @@ -195,6 +246,17 @@ export class HopperRequestQueue { this.#active = entry; entry.startedAt = performance.now(); this.#report(entry, `${entry.method} started on Hopper's serial bridge`); + this.#expireIfElapsed(entry); + // A progress observer may cancel or tear down synchronously before send. + // No native work exists yet, so it is safe to release this reserved slot. + if (this.#active !== entry || entry.callerSettled) { + if (this.#active === entry) { + this.#active = undefined; + this.#releaseWire(entry); + this.#drain(); + } + return; + } entry.heartbeat = setInterval(() => { if (entry.callerSettled || entry.startedAt === undefined) return; const elapsed = Math.round(performance.now() - entry.startedAt); @@ -204,6 +266,7 @@ export class HopperRequestQueue { ); }, 1_000); try { + entry.transmitted = true; this.send( { id: entry.id, method: entry.method, params: entry.params }, () => @@ -229,16 +292,43 @@ export class HopperRequestQueue { } } - #cancel(entry: QueuedRequest): void { + #timeoutError(entry: QueuedRequest, timeoutMs: number): HopperTimeoutError { + return new HopperTimeoutError( + timeoutMs, + entry.method, + entry.id, + entry.transmitted ? "busy" : "not_started", + hopperOperationStage(entry.method), + ); + } + + #expireIfElapsed(entry: QueuedRequest): boolean { + if ( + entry.deadlineAt !== undefined && + entry.timeoutMs !== undefined && + performance.now() >= entry.deadlineAt + ) + this.#settleCaller( + entry, + err(this.#timeoutError(entry, entry.timeoutMs)), + "cancelled", + ); + return entry.callerSettled; + } + + #cancel( + entry: QueuedRequest, + error: HopperError = new HopperCancelledError(), + ): void { if (entry.callerSettled) return; if (this.#active === entry) { - this.#settleCaller(entry, err(new HopperCancelledError()), "cancelled"); + this.#settleCaller(entry, err(error), "cancelled"); return; } const index = this.#queue.indexOf(entry); if (index < 0) return; this.#queue.splice(index, 1); - this.#settleCaller(entry, err(new HopperCancelledError()), "cancelled"); + this.#settleCaller(entry, err(error), "cancelled"); } #settleCaller( @@ -249,6 +339,8 @@ export class HopperRequestQueue { if (entry.callerSettled) return; entry.callerSettled = true; entry.callerState = callerState; + if (entry.deadline !== undefined) clearTimeout(entry.deadline); + entry.deadline = undefined; if (this.#active === entry && entry.heartbeat !== undefined) { clearInterval(entry.heartbeat); entry.heartbeat = undefined; diff --git a/tests/boundary/providers/hopper/hopperClient.deadlines.test.ts b/tests/boundary/providers/hopper/hopperClient.deadlines.test.ts new file mode 100644 index 00000000000..0202cd077d1 --- /dev/null +++ b/tests/boundary/providers/hopper/hopperClient.deadlines.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi } from "vitest"; +import { access } from "node:fs/promises"; + +import { + HopperFixtureLauncher, + startHopperFixtureClient, +} from "./hopperClient.fixture.js"; + +describe("HopperClient request deadlines", () => { + it("propagates deadlines through the socket client and accepts late replies safely", async () => { + const launcher = new HopperFixtureLauncher(); + const client = await startHopperFixtureClient(launcher); + const settled = vi.fn(); + const pending = client + .callTool("echo", { gate: "deadline" }, { timeoutMs: 100 }) + .then(settled); + const request = await launcher.waitForRequest("echo", "deadline"); + try { + await vi.waitFor( + () => + expect(settled).toHaveBeenCalledWith({ + ok: false, + error: expect.objectContaining({ + _tag: "HopperTimeoutError", + timeoutMs: 100, + operation: "echo", + }), + }), + { timeout: 1000 }, + ); + expect(client.requestActivity()).toMatchObject({ + callerState: "cancelled", + operation: "echo", + }); + } finally { + await launcher.release(request); + await pending; + } + await expect( + client.callTool("echo", { value: "alive" }, { timeoutMs: 1000 }), + ).resolves.toEqual({ + ok: true, + value: { value: "alive" }, + }); + expect(settled).toHaveBeenCalledTimes(1); + expect(client.requestActivity()).toBeNull(); + }); + + it("never transmits a request whose deadline expired in the FIFO", async () => { + const launcher = new HopperFixtureLauncher(); + const client = await startHopperFixtureClient(launcher); + const active = client.callTool("echo", { gate: "hold-active" }); + const request = await launcher.waitForRequest("echo", "hold-active"); + try { + await expect( + client.callTool("echo", { value: "expired" }, { timeoutMs: 25 }), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperTimeoutError" }, + }); + expect( + launcher.requests.some(({ params }) => params?.value === "expired"), + ).toBe(false); + } finally { + await launcher.release(request); + await active; + } + await expect(client.callTool("echo", { value: "next" })).resolves.toEqual({ + ok: true, + value: { value: "next" }, + }); + expect( + launcher.requests.some(({ params }) => params?.value === "expired"), + ).toBe(false); + }); + + it("closes and releases pending callers and private resources after an active timeout", async () => { + const launcher = new HopperFixtureLauncher(); + const client = await startHopperFixtureClient(launcher); + const pending = client.callTool( + "echo", + { gate: "timeout-close" }, + { timeoutMs: 100 }, + ); + await launcher.waitForRequest("echo", "timeout-close"); + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperTimeoutError" }, + }); + const queued = client.callTool( + "echo", + { value: "queued" }, + { timeoutMs: 1000 }, + ); + await vi.waitFor(() => + expect(client.requestActivity()?.queuedRequests).toBe(1), + ); + await client.close(); + await expect(queued).resolves.toMatchObject({ + ok: false, + error: { _tag: "HopperProcessError" }, + }); + expect(client.requestActivity()).toBeNull(); + for (const directory of launcher.directories) + await expect(access(directory)).rejects.toBeDefined(); + expect( + launcher.processes.every( + (child) => child.exitCode !== null || child.signalCode !== null, + ), + ).toBe(true); + }); +}); + +it("does not send a mutation after its synchronous progress observer consumes the deadline", async () => { + const launcher = new HopperFixtureLauncher(); + const client = await startHopperFixtureClient(launcher); + const result = await client.callTool( + "set_comment", + { comment: "must not reach the native boundary" }, + { + timeoutMs: 10, + progress: { + report: () => { + const started = performance.now(); + while (performance.now() - started < 30) { + // Exercise an observer delaying dispatch of the actual deadline timer. + } + return Promise.resolve(); + }, + }, + }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "HopperTimeoutError", + operation: "set_comment", + providerState: "not_started", + }, + }); + await expect( + client.callTool("echo", { value: "recovered" }), + ).resolves.toEqual({ + ok: true, + value: { value: "recovered" }, + }); + expect(launcher.requests.some(({ method }) => method === "set_comment")).toBe( + false, + ); + expect(client.requestActivity()).toBeNull(); +}); From c1d0002470dc2b6bde73e4ffe438e734dbb24d8d Mon Sep 17 00:00:00 2001 From: Hasan Mehdi Rizvi Date: Fri, 9 Oct 2026 04:27:40 +0530 Subject: [PATCH 066/791] test(browser): pin DOM and source-map URL policy classifications (#1120) * test(browser): pin DOM and source-map URL policy classifications Cover the unsupported DOM scope, credentialed destinations, and the source-map fetcher's pre-request filtering of credentials, other origins and non-http(s) URLs, so a later vocabulary unification can show exactly what changes. Refs #829 * test(browser): assert exact sanitized credentialed DOM URL --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../CdpCaptureDocuments.baseUrl.test.ts | 38 +++++++++++++++++++ .../browser/webSourceMapFetcher.test.ts | 31 +++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/src/browser/CdpCaptureDocuments.baseUrl.test.ts b/src/browser/CdpCaptureDocuments.baseUrl.test.ts index c2c9378b894..94b04dc65bf 100644 --- a/src/browser/CdpCaptureDocuments.baseUrl.test.ts +++ b/src/browser/CdpCaptureDocuments.baseUrl.test.ts @@ -100,6 +100,44 @@ describe("DOM metadata document base URLs", () => { }); }); + it.each([ + "javascript:void(0)", + "mailto:someone@example.test", + "data:text/plain,hello", + "ftp://app.example.test/file", + ])("classifies the non-http(s) destination %s as unsupported", (href) => { + const result = capture(snapshot(undefined, "assets/", documentUrl, href)); + expect(result.urls[1]).toMatchObject({ + url: null, + destination_scope: "unsupported", + }); + }); + + it("classifies an unparseable destination as unsupported", () => { + const result = capture( + snapshot(undefined, "assets/", documentUrl, "https://[invalid"), + ); + expect(result.urls[1]).toMatchObject({ + url: null, + destination_scope: "unsupported", + }); + }); + + it("classifies a same-origin destination with credentials as approved", () => { + const result = capture( + snapshot( + undefined, + "assets/", + documentUrl, + "https://user:secret@app.example.test/guide", + ), + ); + expect(result.urls[1]).toMatchObject({ + url: `${origin}/guide`, + destination_scope: "approved", + }); + }); + it("does not authorize a foreign document using its approved base URL", () => { const result = capture( snapshot( diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index 7d251b41ad5..2ce7903a3c0 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -193,6 +193,37 @@ describe("web source-map fetching and validation: fetching maps and following re }); }); + it.each([ + ["embedded credentials", "https://user:secret@app.example.test/a.map"], + ["a different scheme on the same host", "http://app.example.test/a.map"], + ["another origin", "https://private.example.test/a.map"], + ["a non-http(s) scheme", "ftp://app.example.test/a.map"], + ["a file URL", "file:///tmp/a.map"], + ["an unparseable URL", "not a url"], + ])( + "filters a declared fetch URL with %s before any request", + async (_description, fetchUrl) => { + const calls: string[] = []; + const result = await fetchWebSourceMaps( + [{ ...request, fetchUrl }], + input(), + undefined, + { + fetch: (url) => { + calls.push(String(url)); + return Promise.resolve(new Response("{}", { status: 200 })); + }, + }, + ); + + expect(calls).toEqual([]); + expect(result).toMatchObject({ + status: "unavailable", + items: [{ status: "policy_filtered" }], + }); + }, + ); + it("follows an approved redirect chain without an arbitrary hop ceiling", async () => { const calls: string[] = []; const result = await fetchWebSourceMaps([request], input(), undefined, { From 1d7115b7b470719a32e477ca3cbb681f0ad408b8 Mon Sep 17 00:00:00 2001 From: BaaRai <130491490+BaaRai@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:10:38 +0200 Subject: [PATCH 067/791] fix(javascript): omit self-referential import edges with disclosure (#1130) * fix(javascript): omit self-referential import edges with disclosure A specifier that resolves back to the importing module itself produced an imports edge from a module to itself, which the application graph schema forbids; one such specifier failed result validation and lost the whole analysis. Skip those edges at composition time and disclose the omission count in the graph limitations. Fixes #1129 * test(javascript): verify self-import behavior through the compiled CLI --------- Co-authored-by: BaaRai Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- .../JavaScriptArtifactGraphBuilder.ts | 15 +- .../JavaScriptModuleRelationships.ts | 33 ++++- .../javascriptSelfImportRelationships.test.ts | 139 ++++++++++++++++++ 3 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 tests/boundary/cli/javascriptSelfImportRelationships.test.ts diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index c043cfb0011..70712207dca 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -15,6 +15,7 @@ import { import { addJavaScriptStaticFindings } from "./JavaScriptArtifactGraphFindings.js"; import { addJavaScriptModuleRelationships, + type JavaScriptModuleRelationshipOmissions, addJavaScriptSourceModules, } from "./JavaScriptModuleRelationships.js"; import { @@ -88,7 +89,7 @@ const buildJavaScriptArtifactGraphInput = ( const packageRoots = addJavaScriptPackageNodes(context); addJavaScriptSourceModules(context); addJavaScriptBundlerNodes(context); - addJavaScriptModuleRelationships(context); + const relationshipOmissions = addJavaScriptModuleRelationships(context); addJavaScriptStaticFindings(context); addElectronBoundaries(context); addJavaScriptHtmlRoles(context); @@ -103,7 +104,11 @@ const buildJavaScriptArtifactGraphInput = ( nodes: accumulator.nodes(), edges: accumulator.edges(), coverage, - limitations: graphLimitations(context, coverage.status), + limitations: graphLimitations( + context, + coverage.status, + relationshipOmissions, + ), }; }; @@ -171,6 +176,7 @@ const applicationGraphResourceLimit = ( const graphLimitations = ( context: JavaScriptArtifactGraphContext, coverage: "complete" | "partial" | "unknown" | "unavailable", + relationshipOmissions: JavaScriptModuleRelationshipOmissions, ): string[] => { const ipc = collectElectronIpcRecords(context.analysis); const pairings = classifyElectronIpcPairings(ipc); @@ -189,6 +195,11 @@ const graphLimitations = ( ); return [ ...context.analysis.limitations, + ...(relationshipOmissions.selfImports > 0 + ? [ + `${String(relationshipOmissions.selfImports)} import ${relationshipOmissions.selfImports === 1 ? "specifier" : "specifiers"} resolved back to the importing module itself and ${relationshipOmissions.selfImports === 1 ? "was" : "were"} omitted; application graph edges cannot be self-referential.`, + ] + : []), "CommonJS and ESM binding relationships were recovered from inert syntax and resolved only within the inventoried artifact container.", "Webpack/Rspack factories were recovered from AST literals; REA did not invoke push handlers or bundle bootstrap code.", "Static imports, entrypoints, workers, endpoints, and storage relationships do not prove runtime execution.", diff --git a/src/application/javascript/JavaScriptModuleRelationships.ts b/src/application/javascript/JavaScriptModuleRelationships.ts index d3a8c825b9e..ab033e744be 100644 --- a/src/application/javascript/JavaScriptModuleRelationships.ts +++ b/src/application/javascript/JavaScriptModuleRelationships.ts @@ -129,27 +129,40 @@ export const addJavaScriptSourceModules = ( } }; +/** Relationship compositions the application graph cannot represent, for disclosure. */ +export interface JavaScriptModuleRelationshipOmissions { + /** Import specifiers that resolved back to the importing module itself. */ + readonly selfImports: number; +} + +type RelationshipOmissionCounter = { selfImports: number }; + /** Compose bounded CommonJS and ESM binding relationships across artifact files. */ export const addJavaScriptModuleRelationships = ( context: JavaScriptArtifactGraphContext, -): void => { +): JavaScriptModuleRelationshipOmissions => { + const omissions = { selfImports: 0 }; for (const analyzed of context.analysis.files) { const { file, semantic } = analyzed; const source = context.sourceModuleNodes.get(file.path); if (semantic === null || source === undefined) continue; for (const link of semantic.ir.moduleLinks) { const input = { context, file, semantic, source, link }; - if (isExportLink(link)) addExportRelationship(input); + if (isExportLink(link)) addExportRelationship(input, omissions); else if (link.specifier !== null) - addImportRelationship(input, source, { + addImportRelationship(input, source, omissions, { specifier: link.specifier, importedPath: link.importedName === null ? [] : [link.importedName], }); } } + return omissions; }; -const addExportRelationship = (input: RelationshipInput): void => { +const addExportRelationship = ( + input: RelationshipInput, + omissions: RelationshipOmissionCounter, +): void => { const { context, file, semantic, source, link } = input; if (link.exportedName === null) return; const baseCoverage = semanticCoverage(semantic); @@ -213,15 +226,25 @@ const addExportRelationship = (input: RelationshipInput): void => { evidence: relationshipEvidence(input, "expose-module-export", []), }); const origin = moduleOriginForExport(semantic.ir, link); - if (origin !== null) addImportRelationship(input, exported, origin); + if (origin !== null) + addImportRelationship(input, exported, omissions, origin); }; const addImportRelationship = ( input: RelationshipInput, source: ApplicationNode, + omissions: RelationshipOmissionCounter, origin: JavaScriptModuleOrigin, ): void => { const target = resolveModuleTarget(input, origin.specifier); + // A specifier that resolves back to the importing module itself (a literal + // self-import, self-require, or artifact-confined path alias) + // cannot become an edge: the application graph forbids self-referential + // edges, and emitting one fails result validation for the whole analysis. + if (target.node.node_id === source.node_id) { + omissions.selfImports += 1; + return; + } input.context.accumulator.addEdge({ source_node_id: source.node_id, target_node_id: target.node.node_id, diff --git a/tests/boundary/cli/javascriptSelfImportRelationships.test.ts b/tests/boundary/cli/javascriptSelfImportRelationships.test.ts new file mode 100644 index 00000000000..c3e1238bef3 --- /dev/null +++ b/tests/boundary/cli/javascriptSelfImportRelationships.test.ts @@ -0,0 +1,139 @@ +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { + findExportNode, + findGraphEdge, + findSourceModule, + writeFixtureFiles, +} from "../../support/javascriptApplicationFixture.js"; + +cliTest( + "returns literal self-import evidence while preserving sibling imports and self-re-exports", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-self-import-"); + await writeFixtureFiles(root, { + "a.js": + 'import "./a.js";\nexport const value = 1;\nexport { value as forwarded } from "./a.js";\n', + "b.js": 'import { value } from "./a.js";\nconsole.log(value);\n', + }); + const output = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(output.exitCode).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + const { graph } = result; + expect( + graph.edges.filter((edge) => edge.source_node_id === edge.target_node_id), + ).toEqual([]); + expect(graph.limitations).toContainEqual( + expect.stringMatching(/^1 import specifier .*was omitted;/u), + ); + expect(result.limitations).toEqual( + expect.arrayContaining(graph.limitations), + ); + const source = findSourceModule(graph, "a.js"); + expect(source).toBeDefined(); + expect( + findGraphEdge(graph, findSourceModule(graph, "b.js"), source, "imports") + ?.properties, + ).toMatchObject({ + specifier: "./a.js", + resolution_status: "resolved", + }); + expect( + findGraphEdge( + graph, + findExportNode(graph, "a.js", "forwarded"), + source, + "imports", + )?.properties, + ).toMatchObject({ + specifier: "./a.js", + resolution_status: "resolved", + }); + expect( + graph.edges.find( + (edge) => + edge.source_node_id === source?.node_id && + edge.properties.kind === "static-import" && + edge.properties.specifier === "./a.js" && + edge.evidence.location.available && + edge.evidence.location.value.kind === "source-range" && + edge.evidence.location.value.start.line === 1, + )?.evidence.location, + ).toMatchObject({ + available: true, + value: { kind: "source-range", source: "a.js", start: { line: 1 } }, + }); + }, +); + +cliTest( + "discloses CommonJS and TypeScript self-imports while retaining unresolved extension rewrites", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-self-require-"); + await writeFixtureFiles(root, { + "c.cjs": 'const self = require("./c.cjs");\nmodule.exports = self;\n', + "util.ts": + 'import "./util.ts";\nimport "./util.js";\nexport const value = 1;\n', + }); + const output = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(output.exitCode).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + const { graph } = result; + expect( + graph.edges.filter((edge) => edge.source_node_id === edge.target_node_id), + ).toEqual([]); + expect(graph.limitations).toContainEqual( + expect.stringMatching(/^2 import specifiers .*were omitted;/u), + ); + expect(result.limitations).toEqual( + expect.arrayContaining(graph.limitations), + ); + for (const [path, specifier] of [ + ["c.cjs", "./c.cjs"], + ["util.ts", "./util.ts"], + ] as const) { + const source = findSourceModule(graph, path); + expect(source).toBeDefined(); + expect( + graph.edges.find( + (edge) => + edge.source_node_id === source?.node_id && + edge.properties.specifier === specifier && + edge.properties.resolved_path === path, + )?.evidence.location, + ).toMatchObject({ + available: true, + value: { kind: "source-range", source: path, start: { line: 1 } }, + }); + expect( + graph.nodes.some((node) => + node.observations.some( + ({ properties }) => + properties.semantic_role === "module-reference" && + properties.declared_specifier === specifier, + ), + ), + ).toBe(false); + } + expect( + graph.edges.find( + (edge) => + edge.source_node_id === findSourceModule(graph, "util.ts")?.node_id && + edge.properties.specifier === "./util.js" && + edge.properties.module_link_kind === "import", + )?.properties, + ).toMatchObject({ resolved_path: null, resolution_status: "not-found" }); + }, +); From 4034883e9860a97edee03260fab59f9229debfa6 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:13:13 +0800 Subject: [PATCH 068/791] fix(artifacts): preserve keyed archive integer precision Integrate the numeric precision correction from #1135 with complete supplemental metadata handling. Keep decoded numbers with an explicit limitation for mixed XML content or unclassified integer text, rather than assigning another literal's exact decimal. Project nonnumeric UID and CF$UID dictionary values recursively while preserving numeric references, graph identity and original byte digests. Add filesystem, built CLI and MCP regressions for the review findings, adjacent XML representations and numeric reference controls. Retain the original native Foundation qualification fixtures and verifier. Verification: 84 focused tests; npm run check; npm run docs:check; npm run verify:test-discovery; npm run verify:keyed-archive (real Foundation, CLI and stdio MCP). No real Hopper or Ghidra verification was needed or run. Co-authored-by: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Signed-off-by: morluto <76467478+morluto@users.noreply.github.com> --- docs/native-investigation.md | 22 +++ scripts/verify/apple/keyed-archive.mjs | 22 ++- src/artifacts/apple/KeyedArchiveReader.ts | 28 ++- src/artifacts/apple/PlistNumberLiterals.ts | 174 ++++++++++++++++++ src/domain/apple/plistNumbers.ts | 91 +++++++++ src/domain/propertyListKeys.ts | 2 +- .../keyedArchiveIntegerAmbiguity.test.ts | 163 ++++++++++++++++ .../keyedArchiveNativeIntegers.test.ts | 61 ++++++ .../keyedArchiveNumberMetadata.test.ts | 39 ++++ .../keyedArchiveScalarIdentity.test.ts | 115 ++++++++++++ .../mcp/keyedArchiveNumberMetadataMcp.test.ts | 44 +++++ .../native/keyed-archive-integers.swift | 134 ++++++++++++++ tests/fixtures/keyedArchiveIntegers.ts | 143 ++++++++++++++ tests/fixtures/keyedArchiveNumberMetadata.ts | 154 ++++++++++++++++ 14 files changed, 1189 insertions(+), 3 deletions(-) create mode 100644 src/artifacts/apple/PlistNumberLiterals.ts create mode 100644 src/domain/apple/plistNumbers.ts create mode 100644 tests/boundary/filesystem/keyedArchiveIntegerAmbiguity.test.ts create mode 100644 tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts create mode 100644 tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts create mode 100644 tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts create mode 100644 tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts create mode 100644 tests/conformance/native/keyed-archive-integers.swift create mode 100644 tests/fixtures/keyedArchiveIntegers.ts create mode 100644 tests/fixtures/keyedArchiveNumberMetadata.ts diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 5d97e7c6ffc..8ad7aadbdea 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -383,3 +383,25 @@ A keyed-archive hierarchy UID without an object-table entry marks the hierarchy as partial while preserving decoded objects and known links. UID 0 remains archived nil and does not count as a missing reference. Archive and evidence digests identify the original serialized bytes. + +## Keyed archive integer precision + +The archive reader observes XML number element types and binary number markers. +Integers beyond the exact range of a JSON number are reported with the original +decimal text as `{ "$plist_type": "integer", "decimal": "" }` when +that source value can be associated unambiguously. Integral real values remain +numbers. Colliding integer/real values, unclassified numbers, and malformed UID +markers remain decoded with an explicit precision limitation. The limitation +also counts observed unsafe integer literals, including observations outside the +selected object page. Incomplete supplemental metadata produces an explicit +precision note without rejecting an archive accepted by the byte decoder. +Mixed XML number content (comments, CDATA, processing instructions or child +elements) and integer text the supplemental reader cannot classify disable exact +association and retain decoded numbers with that precision note. Nonnumeric +`UID` and `CF$UID` dictionary values remain ordinary data and are projected +recursively; numeric reference markers retain their original representation. + +Value projection preserves serialized node kinds, IDs, references and pagination. +Exact and ambiguous value counts describe emitted root/object value instances; +a root and an object may contain the same value. Archive digests identify the +original bytes. diff --git a/scripts/verify/apple/keyed-archive.mjs b/scripts/verify/apple/keyed-archive.mjs index 4a162e72e87..d8bc284ba5a 100644 --- a/scripts/verify/apple/keyed-archive.mjs +++ b/scripts/verify/apple/keyed-archive.mjs @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, spawnSync } from "node:child_process"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -151,6 +151,26 @@ try { [{ source: null, path: ["UID"], target: 1, status: "resolved" }], ); } + const numericTests = spawnSync( + "npm", + [ + "run", + "test:focused", + "--", + "tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts", + ], + { + cwd: fileURLToPath(new URL("../../../", import.meta.url)), + stdio: "inherit", + timeout: 240_000, + }, + ); + if (numericTests.error) throw numericTests.error; + assert.equal( + numericTests.status, + 0, + "Native Foundation integer cases failed", + ); report = { ok: true, mocked: false, diff --git a/src/artifacts/apple/KeyedArchiveReader.ts b/src/artifacts/apple/KeyedArchiveReader.ts index 54401c4a1ed..d7d4dc99fd1 100644 --- a/src/artifacts/apple/KeyedArchiveReader.ts +++ b/src/artifacts/apple/KeyedArchiveReader.ts @@ -4,6 +4,11 @@ import { z } from "zod"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { projectPlistValue } from "../../domain/apple/plistValue.js"; +import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js"; +import { + binaryPlistNumberLiterals, + xmlPlistNumberLiterals, +} from "./PlistNumberLiterals.js"; import { omittedPrototypeKeysLimitation, parseXmlPropertyList, @@ -32,15 +37,36 @@ export const decodeKeyedArchiveBytes = ( "Selected file is a compiled NIBArchive, not a Foundation plist archive; decode it with decode_interface_builder", ); const binary = bytes.subarray(0, 8).toString("ascii") === "bplist00"; + const xmlText = binary ? undefined : decodeXmlPlistText(bytes); const parsed = binary ? { value: parseBinary(bytes), omittedPrototypeKeys: 0 } - : parseXmlPropertyList(decodeXmlPlistText(bytes)); + : parseXmlPropertyList(xmlText ?? ""); const graph = projectKeyedArchive(normalizePlist(parsed.value), selection); + const numbers = createPlistNumberProjection( + binary + ? binaryPlistNumberLiterals(bytes) + : xmlPlistNumberLiterals(xmlText ?? ""), + ); + // Classify the serialized graph before projecting values to typed decimals. + // Numeric IDs, node kinds, references and pagination retain their original meaning. + const roots = Object.fromEntries( + Object.entries(graph.roots).map(([name, value]) => [ + name, + numbers.project(value), + ]), + ); + const objects = graph.objects.map((object) => ({ + ...object, + value: numbers.project(object.value), + })); return { archive_format: binary ? ("binary-plist" as const) : ("xml-plist" as const), ...graph, + roots, + objects, limitations: [ ...graph.limitations, + ...numbers.limitations(), ...(parsed.omittedPrototypeKeys === 0 ? [] : [omittedPrototypeKeysLimitation(parsed.omittedPrototypeKeys)]), diff --git a/src/artifacts/apple/PlistNumberLiterals.ts b/src/artifacts/apple/PlistNumberLiterals.ts new file mode 100644 index 00000000000..b491a72746c --- /dev/null +++ b/src/artifacts/apple/PlistNumberLiterals.ts @@ -0,0 +1,174 @@ +import type { PlistNumberLiterals } from "../../domain/apple/plistNumbers.js"; +import { decodeXmlText } from "../../domain/propertyListKeys.js"; + +const collector = () => { + const integers = new Map>(); + const reals = new Set(); + let unsafeIntegerLiterals = 0; + let incompleteMetadata = false; + return { + integer: (literal: string) => { + if (!/^[+-]?\d+$/u.test(literal)) { + incompleteMetadata = true; + return; + } + const exact = BigInt(literal); + if ( + exact >= BigInt(Number.MIN_SAFE_INTEGER) && + exact <= BigInt(Number.MAX_SAFE_INTEGER) + ) + return; + unsafeIntegerLiterals += 1; + const rounded = Number(exact); + integers.set( + rounded, + (integers.get(rounded) ?? new Set()).add(exact.toString()), + ); + }, + real: (value: number) => { + reals.add(value); + }, + markIncomplete: () => { + incompleteMetadata = true; + }, + result: (incomplete = false): PlistNumberLiterals => ({ + incomplete: incomplete || incompleteMetadata, + integers, + reals, + unsafeIntegerLiterals, + }), + }; +}; +// Consume quoted tag attributes and opaque comment/CDATA tokens whole. +const XML_TOKEN = + /||<(?:[^>"']|"[^"]*"|'[^']*')*>/gu; +const OPAQUE_ELEMENTS = new Set(["string", "key", "data", "date"]); + +/** Observe plist number element types without interpreting strings or class code. */ +export const xmlPlistNumberLiterals = (xml: string): PlistNumberLiterals => { + const output = collector(); + const elements: string[] = []; + let number: + | { + kind: "integer" | "real"; + text: string; + depth: number; + plainText: boolean; + } + | undefined; + let cursor = 0; + for (const match of xml.matchAll(XML_TOKEN)) { + const index = match.index; + if (number !== undefined) number.text += xml.slice(cursor, index); + const token = match[0]; + cursor = index + token.length; + if (token.startsWith("", + ); + bytes = Buffer.from(xml); + } + const path = join(root, "archive.plist"); + await writeFile(path, bytes); + return { + root, + path, + digest: createHash("sha256").update(bytes).digest("hex"), + }; +}; +const verify = ( + result: ReturnType, + scenario: (typeof scenarios)[number], + digest: string, +) => { + expect(result.archive_sha256).toBe(digest); + expect(result.objects).toHaveLength(2); + expect(result.references).toContainEqual( + expect.objectContaining({ source: null, target: 1, status: "resolved" }), + ); + const value = result.objects.find(({ id }) => id === 1)?.value; + expect(value).toBeDefined(); + expect(value).not.toBeNull(); + expect(Array.isArray(value)).toBe(false); + expect(typeof value).toBe("object"); + const fields = value as Record; + const precisionNote = result.limitations.some( + (note) => + /integer|number/iu.test(note) && + /precision|exact range|rounded/iu.test(note), + ); + if (scenario === "integer-collision") { + // Neither colliding decoded slot may be assigned one arbitrary exact literal. + expect(fields.first).toBe(9007199254740992); + expect(fields.second).toBe(9007199254740992); + expect(precisionNote).toBe(true); + } else if (scenario === "integer-real-collision") { + expect(fields.first).toBe(9007199254740992); + expect(fields.real).toBe(9007199254740992); + expect(precisionNote).toBe(true); + } else if (scenario === "opaque-xml") { + expect(fields.text).toBe("9007199254740993"); + expect(fields.real).toBe(9007199254740996); + expect(fields.first).toBeDefined(); + const exact = + typeof fields.first === "object" && + fields.first !== null && + JSON.stringify(fields.first) === + JSON.stringify({ $plist_type: "integer", decimal: "9007199254740993" }); + expect(exact || precisionNote).toBe(true); + // If a literal count is reported, opaque content must not inflate it. + const countNote = result.limitations.find((note) => + /integer literal\(s\)/u.test(note), + ); + if (countNote !== undefined) + expect(countNote).toMatch(/^1 observed integer literal/u); + } else { + expect(fields.real).toBe(9007199254740992); + } +}; +it.each(scenarios)( + "filesystem retains numeric meaning for %s", + async (scenario) => { + const item = await fixture(scenario); + const result = await inspectBundleKeyedArchive({ + bundlePath: item.root, + targetSha256: item.digest, + parameters: { path: "archive.plist", limit: 100 }, + }); + verify(result, scenario, item.digest); + }, +); +cliTest.for(scenarios)( + "built CLI retains numeric meaning for %s", + async (scenario, { cli }) => { + const item = await fixture(scenario); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", item.path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode).toBe(0); + verify( + keyedArchiveResultSchema.parse( + parseEvidence(output.json).normalized_result, + ), + scenario, + item.digest, + ); + }, +); diff --git a/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts b/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts new file mode 100644 index 00000000000..9ba2583e73b --- /dev/null +++ b/tests/boundary/filesystem/keyedArchiveNativeIntegers.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect } from "vitest"; +import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + keyedArchiveIntegerFormats, + keyedArchiveIntegerCases, + keyedArchiveIntegerFixture, + expectKeyedArchiveIntegerCase, +} from "../../fixtures/keyedArchiveIntegers.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const cases = keyedArchiveIntegerFormats.flatMap((format) => + keyedArchiveIntegerCases.map((selectedCase) => ({ format, selectedCase })), +); +// Selected by the existing required native keyed-archive verifier, not Linux skips. +describe.skipIf(process.platform !== "darwin")( + "Foundation archive integer precision", + () => { + it.each(cases)( + "filesystem preserves $format $selectedCase numeric meaning", + async ({ format, selectedCase }) => { + const fixture = await keyedArchiveIntegerFixture(format); + const result = await inspectBundleKeyedArchive({ + bundlePath: fixture.root, + targetSha256: fixture.digest, + parameters: { path: fixture.selected.filename, limit: 100 }, + }); + expectKeyedArchiveIntegerCase( + result, + selectedCase, + fixture.selected, + fixture.digest, + ); + }, + ); + cliTest.for(cases)( + "built CLI preserves $format $selectedCase numeric meaning", + async ({ format, selectedCase }, { cli }) => { + const fixture = await keyedArchiveIntegerFixture(format); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", fixture.archivePath, "--json"], + environment: { + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "auto", + }, + }); + expect(output.exitCode).toBe(0); + const result = keyedArchiveResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expectKeyedArchiveIntegerCase( + result, + selectedCase, + fixture.selected, + fixture.digest, + ); + }, + ); + }, +); diff --git a/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts new file mode 100644 index 00000000000..14034f8b3ca --- /dev/null +++ b/tests/boundary/filesystem/keyedArchiveNumberMetadata.test.ts @@ -0,0 +1,39 @@ +import { it } from "vitest"; + +import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + archiveNumberMetadataCases, + expectKeyedArchiveNumberMetadata, + keyedArchiveNumberMetadataFixture, +} from "../../fixtures/keyedArchiveNumberMetadata.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +it.each(archiveNumberMetadataCases)( + "filesystem preserves numeric meaning for $name", + async (item) => { + const fixture = await keyedArchiveNumberMetadataFixture(item); + const result = await inspectBundleKeyedArchive({ + bundlePath: fixture.root, + targetSha256: fixture.digest, + parameters: { path: "archive.plist" }, + }); + expectKeyedArchiveNumberMetadata(result, item, fixture.digest); + }, +); + +cliTest.for(archiveNumberMetadataCases)( + "built CLI preserves numeric meaning for $name", + async (item, { cli }) => { + const fixture = await keyedArchiveNumberMetadataFixture(item); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", fixture.path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expectKeyedArchiveNumberMetadata( + parseEvidence(output.json).normalized_result, + item, + fixture.digest, + ); + }, +); diff --git a/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts b/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts new file mode 100644 index 00000000000..47756552ff7 --- /dev/null +++ b/tests/boundary/filesystem/keyedArchiveScalarIdentity.test.ts @@ -0,0 +1,115 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { build } from "plist"; +import { expect, it } from "vitest"; +import { inspectBundleKeyedArchive } from "../../../src/artifacts/apple/KeyedArchiveReader.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const cases = ["unsafe-integer", "integral-real"] as const; +const fixture = async (kind: (typeof cases)[number]) => { + const root = await createTestTempDirectory("rea-integer-scalar-"); + const xml = build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", "__VALUE__"], + $top: { root: { CF$UID: 1 } }, + }).replace( + "__VALUE__", + kind === "unsafe-integer" + ? "9007199254740993" + : "9007199254740992", + ); + const bytes = Buffer.from(xml); + const path = join(root, "archive.plist"); + await writeFile(path, bytes); + return { + root, + path, + digest: createHash("sha256").update(bytes).digest("hex"), + }; +}; +const verify = ( + result: ReturnType, + kind: (typeof cases)[number], + digest: string, +) => { + expect(result.archive_sha256).toBe(digest); + expect(result.roots).toEqual({ root: { CF$UID: 1 } }); + expect( + result.objects.map(({ id, kind: objectKind, status }) => ({ + id, + kind: objectKind, + status, + })), + ).toEqual([ + { id: 0, kind: "nil", status: "nil" }, + { id: 1, kind: "scalar", status: "observed" }, + ]); + expect(result.references).toEqual([ + { + source: null, + path: ["root"], + target: 1, + status: "resolved", + raw: { CF$UID: 1 }, + }, + ]); + expect(result.total_objects).toBe(2); + expect(result.total_references).toBe(1); + expect(result.offset).toBe(0); + expect(result.next_offset).toBeNull(); + expect(result.truncated).toBe(false); + const value = result.objects.find(({ id }) => id === 1)?.value; + expect(value).toBeDefined(); + if (kind === "integral-real") expect(value).toBe(9007199254740992); + else { + const exact = + typeof value === "object" && + value !== null && + JSON.stringify(value) === + JSON.stringify({ $plist_type: "integer", decimal: "9007199254740993" }); + const disclosed = result.limitations.some( + (note) => + /integer|number/iu.test(note) && + /precision|exact range|rounded/iu.test(note), + ); + expect(exact || disclosed).toBe(true); + } +}; +it.each(cases)( + "filesystem preserves primitive scalar graph for %s", + async (kind) => { + const item = await fixture(kind); + verify( + await inspectBundleKeyedArchive({ + bundlePath: item.root, + targetSha256: item.digest, + parameters: { path: "archive.plist", limit: 100 }, + }), + kind, + item.digest, + ); + }, +); +cliTest.for(cases)( + "built CLI preserves primitive scalar graph for %s", + async (kind, { cli }) => { + const item = await fixture(kind); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", item.path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode).toBe(0); + verify( + keyedArchiveResultSchema.parse( + parseEvidence(output.json).normalized_result, + ), + kind, + item.digest, + ); + }, +); diff --git a/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts b/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts new file mode 100644 index 00000000000..88b415de53a --- /dev/null +++ b/tests/boundary/mcp/keyedArchiveNumberMetadataMcp.test.ts @@ -0,0 +1,44 @@ +import { expect } from "vitest"; +import { z } from "zod"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { + archiveNumberMetadataCases, + expectKeyedArchiveNumberMetadata, + keyedArchiveNumberMetadataFixture, +} from "../../fixtures/keyedArchiveNumberMetadata.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +mcpTest.for(archiveNumberMetadataCases)( + "MCP preserves numeric meaning for $name", + async (item, { mcp, onTestFinished }) => { + const fixture = await keyedArchiveNumberMetadataFixture(item); + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession(configured.value); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect(createServer(session, session)); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: fixture.path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const result = await client.callTool({ + name: "inspect_keyed_archive", + arguments: {}, + }); + expect(result.isError, JSON.stringify(result.content)).not.toBe(true); + expectKeyedArchiveNumberMetadata( + z + .object({ result: keyedArchiveResultSchema }) + .parse(result.structuredContent).result, + item, + fixture.digest, + ); + }, +); diff --git a/tests/conformance/native/keyed-archive-integers.swift b/tests/conformance/native/keyed-archive-integers.swift new file mode 100644 index 00000000000..6f98cbb0c8c --- /dev/null +++ b/tests/conformance/native/keyed-archive-integers.swift @@ -0,0 +1,134 @@ +import Foundation +import CryptoKit + +// Foundation writes the archive. XMLParser only observes inert plist element types. +final class PlistElements: NSObject, XMLParserDelegate { + struct Frame { + let kind: String + var values: [Any] = [] + var fields: [String: Any] = [:] + var key: String? + var text = "" + } + var frames: [Frame] = [] + var value: Any? + func parser(_ parser: XMLParser, didStartElement name: String, namespaceURI: String?, qualifiedName: String?, attributes: [String: String]) { + frames.append(Frame(kind: name)) + } + func parser(_ parser: XMLParser, foundCharacters text: String) { + if !frames.isEmpty { frames[frames.count - 1].text += text } + } + func parser(_ parser: XMLParser, didEndElement name: String, namespaceURI: String?, qualifiedName: String?) { + let frame = frames.removeLast() + if name == "key" { + frames[frames.count - 1].key = frame.text + return + } + let item: Any + switch name { + case "dict": item = frame.fields + case "array": item = frame.values + case "integer", "real": + item = ["numberKind": name, "literal": frame.text.trimmingCharacters(in: .whitespacesAndNewlines)] + case "string": item = frame.text + case "true": item = true + case "false": item = false + case "plist": value = frame.values.first; return + default: parser.abortParsing(); return + } + if frames.last?.kind == "dict" { + guard let key = frames.last?.key else { parser.abortParsing(); return } + frames[frames.count - 1].fields[key] = item + frames[frames.count - 1].key = nil + } else if !frames.isEmpty { + frames[frames.count - 1].values.append(item) + } + } +} +func elements(_ bytes: Data) throws -> [String: Any] { + let delegate = PlistElements() + let parser = XMLParser(data: bytes) + parser.shouldResolveExternalEntities = false + parser.delegate = delegate + guard parser.parse(), let value = delegate.value as? [String: Any] else { + throw NSError(domain: "REA.IntegerOracle", code: 1) + } + return value +} +func rows(_ value: Any, path: [Any] = []) -> [[String: Any]] { + if let fields = value as? [String: Any] { + // Reference markers identify objects; their UID is not an archived numeric value. + if fields.count == 1 && fields["CF$UID"] != nil { return [] } + if let kind = fields["numberKind"] as? String, let literal = fields["literal"] as? String { + return [["path": path, "kind": kind, "literal": literal]] + } + return fields.keys.sorted().flatMap { rows(fields[$0]!, path: path + [$0]) } + } + if let values = value as? [Any] { + return values.enumerated().flatMap { rows($0.element, path: path + [$0.offset]) } + } + return [] +} +func literal(_ value: Any?, kind: String) throws -> String { + guard let fields = value as? [String: String], fields["numberKind"] == kind, let text = fields["literal"] else { + throw NSError(domain: "REA.IntegerOracle", code: 2) + } + return text +} +let output = URL(fileURLWithPath: CommandLine.arguments[1]) +let positive: Int64 = 9_007_199_254_740_993 +let negative: Int64 = Int64.min +let unsigned: UInt64 = 9_007_199_254_740_993 +let real: Double = 9_007_199_254_740_992 +var archives: [String: Any] = [:] +for format in ["xml", "binary"] { + let archiver = NSKeyedArchiver(requiringSecureCoding: false) + archiver.outputFormat = format == "xml" ? .xml : .binary + archiver.encode(positive, forKey: "unsafeSigned") + archiver.encode(negative, forKey: "unsafeNegative") + archiver.encode(Int64(9_007_199_254_740_991), forKey: "safeInteger") + archiver.encode(Int64(-42), forKey: "safeNegative") + archiver.encode(real, forKey: "integralReal") + archiver.encode(NSNumber(value: unsigned), forKey: "boxedUnsigned") + archiver.encode(NSNumber(value: real), forKey: "boxedReal") + archiver.finishEncoding() + let bytes = archiver.encodedData + let native = try PropertyListSerialization.propertyList(from: bytes, options: [], format: nil) + let xml = try PropertyListSerialization.data(fromPropertyList: native, format: .xml, options: 0) + let parsed = try elements(xml) + guard let top = parsed["$top"] as? [String: Any], let table = parsed["$objects"] as? [Any] else { + throw NSError(domain: "REA.IntegerOracle", code: 3) + } + guard try literal(top["unsafeSigned"], kind: "integer") == String(positive), + try literal(top["unsafeNegative"], kind: "integer") == String(negative), + try literal(top["safeInteger"], kind: "integer") == "9007199254740991", + try literal(top["safeNegative"], kind: "integer") == "-42", + Double(try literal(top["integralReal"], kind: "real")) == real else { + throw NSError(domain: "REA.IntegerOracle", code: 4) + } + var selectedRows: [[String: Any]] = [] + for key in ["unsafeSigned", "unsafeNegative", "safeInteger", "safeNegative", "integralReal"] { + for row in rows(top[key]!) { + selectedRows.append(row.merging(["container": "roots", "path": [key], "root": key]) { _, rhs in rhs }) + } + } + for key in ["boxedUnsigned", "boxedReal"] { + guard let reference = top[key] as? [String: Any], + let uid = Int(try literal(reference["CF$UID"], kind: "integer")), uid > 0, uid < table.count else { + throw NSError(domain: "REA.IntegerOracle", code: 5) + } + let numericRows = rows(table[uid]) + let expectedKind = key == "boxedUnsigned" ? "integer" : "real" + guard numericRows.contains(where: { row in + guard row["kind"] as? String == expectedKind, let text = row["literal"] as? String else { return false } + return key == "boxedUnsigned" ? text == String(unsigned) : Double(text) == real + }) else { throw NSError(domain: "REA.IntegerOracle", code: 6) } + for row in numericRows { + selectedRows.append(row.merging(["container": "objects", "objectId": uid, "root": key]) { _, rhs in rhs }) + } + } + let filename = "numbers.\(format).plist" + try bytes.write(to: output.appendingPathComponent(filename)) + archives[format] = ["filename": filename, "sha256": SHA256.hash(data: bytes).map { String(format: "%02x", $0) }.joined(), "rows": selectedRows] +} +try JSONSerialization.data(withJSONObject: archives, options: [.sortedKeys]).write(to: output.appendingPathComponent("oracle.json")) diff --git a/tests/fixtures/keyedArchiveIntegers.ts b/tests/fixtures/keyedArchiveIntegers.ts new file mode 100644 index 00000000000..239319ad094 --- /dev/null +++ b/tests/fixtures/keyedArchiveIntegers.ts @@ -0,0 +1,143 @@ +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect } from "vitest"; +import { z } from "zod"; +import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +const rowSchema = z.object({ + container: z.enum(["roots", "objects"]), + root: z.string(), + objectId: z.number().int().nonnegative().optional(), + path: z.array(z.union([z.string(), z.number().int().nonnegative()])), + kind: z.enum(["integer", "real"]), + literal: z.string(), +}); +const archiveSchema = z.object({ + filename: z.string(), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), + rows: z.array(rowSchema), +}); +const oracleSchema = z.object({ xml: archiveSchema, binary: archiveSchema }); +export const keyedArchiveIntegerFormats = ["xml", "binary"] as const; +export const keyedArchiveIntegerCases = ["controls", "unsafe"] as const; +type Format = (typeof keyedArchiveIntegerFormats)[number]; +type Case = (typeof keyedArchiveIntegerCases)[number]; +type Row = z.infer; +export const keyedArchiveIntegerFixture = async (format: Format) => { + const root = await createTestTempDirectory("rea-keyed-integers-"); + await promisify(execFile)( + "/usr/bin/xcrun", + [ + "swift", + "-module-cache-path", + join(root, "modules"), + "tests/conformance/native/keyed-archive-integers.swift", + root, + ], + { maxBuffer: 1024 * 1024 }, + ); + const oracle = oracleSchema.parse( + JSON.parse(await readFile(join(root, "oracle.json"), "utf8")), + ); + for (const entry of [oracle.xml, oracle.binary]) { + expect(entry.rows).toContainEqual( + expect.objectContaining({ + root: "unsafeSigned", + kind: "integer", + literal: "9007199254740993", + }), + ); + expect(entry.rows).toContainEqual( + expect.objectContaining({ + root: "unsafeNegative", + kind: "integer", + literal: "-9223372036854775808", + }), + ); + expect( + entry.rows.some( + (row) => + row.root === "boxedUnsigned" && + row.kind === "integer" && + row.literal === "9007199254740993", + ), + ).toBe(true); + expect( + entry.rows.some( + (row) => + row.root === "boxedReal" && + row.kind === "real" && + Number(row.literal) === 9007199254740992, + ), + ).toBe(true); + } + const selected = oracle[format]; + const archivePath = join(root, selected.filename); + const bytes = await readFile(archivePath); + const digest = createHash("sha256").update(bytes).digest("hex"); + expect(digest).toBe(selected.sha256); + return { root, selected, archivePath, digest }; +}; +const at = (value: unknown, path: Row["path"]): unknown => { + for (const key of path) { + if (value === null || typeof value !== "object") return undefined; + value = (value as Record)[key]; + } + return value; +}; +export const expectKeyedArchiveIntegerCase = ( + result: z.infer, + selectedCase: Case, + selected: z.infer, + digest: string, +) => { + expect(result.archive_sha256).toBe(digest); + const unsafe = (row: Row) => + row.kind === "integer" && + (BigInt(row.literal) > BigInt(Number.MAX_SAFE_INTEGER) || + BigInt(row.literal) < BigInt(Number.MIN_SAFE_INTEGER)); + const rows = selected.rows.filter((row) => + selectedCase === "unsafe" ? unsafe(row) : !unsafe(row), + ); + expect(rows.length).toBeGreaterThan(0); + for (const row of rows) { + const value = at( + row.container === "roots" + ? result.roots + : result.objects.find((object) => object.id === row.objectId)?.value, + row.path, + ); + expect( + value, + `${row.root} numeric slot remains present`, + ).not.toBeUndefined(); + if (unsafe(row)) { + const exact = + typeof value === "object" && + value !== null && + (value as Record).$plist_type === "integer" && + (value as Record).decimal === + BigInt(row.literal).toString(); + // The accepted alternate contract must explicitly disclose numeric precision loss. + const disclosed = result.limitations.some( + (reason) => + /integer|number/iu.test(reason) && + /precision|exact range|rounded/iu.test(reason), + ); + expect( + exact || disclosed, + `${row.root} ${row.literal} must be exact or disclose numeric loss`, + ).toBe(true); + } else { + expect(typeof value, `${row.root} retains its numeric type`).toBe( + "number", + ); + expect(value).toBe(Number(row.literal)); + } + } +}; diff --git a/tests/fixtures/keyedArchiveNumberMetadata.ts b/tests/fixtures/keyedArchiveNumberMetadata.ts new file mode 100644 index 00000000000..f63b86fb0bf --- /dev/null +++ b/tests/fixtures/keyedArchiveNumberMetadata.ts @@ -0,0 +1,154 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build } from "plist"; +import { expect } from "vitest"; + +import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js"; +import type { JsonValue } from "../../src/domain/jsonValue.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +const EXACT_INTEGER = { $plist_type: "integer", decimal: "9007199254740993" }; +const ROUNDED = 9007199254740992; + +/** Accepted decoder representations whose supplemental metadata needs care. */ +export const mixedArchiveNumberElements = [ + "9007199254740992", + "9007199254740992", + "90071992547409923", + "9007199254740992", + "3", + "", + "90071992547409923", + "90071992547409923", + "9007199254740992x", +] as const; + +/** Ordinary dictionaries and malformed numeric references at the same boundary. */ +export const archiveNumberMetadataCases = [ + ...mixedArchiveNumberElements.map((element) => ({ + name: element, + source: { first: "__INTEGER__", other: "__OTHER__" }, + element, + expected: { first: ROUNDED, other: ROUNDED }, + incomplete: true, + malformedReference: false, + })), + ...(["UID", "CF$UID"] as const).flatMap((key) => [ + { + name: `${key} dictionary data`, + source: { [key]: { amount: "__INTEGER__" } }, + element: "", + expected: { [key]: { amount: EXACT_INTEGER } }, + incomplete: false, + malformedReference: false, + }, + { + name: `${key} array data`, + source: { [key]: ["__INTEGER__"] }, + element: "", + expected: { [key]: [EXACT_INTEGER] }, + incomplete: false, + malformedReference: false, + }, + { + name: `${key} malformed numeric reference`, + source: { [key]: "__INTEGER__" }, + element: "", + expected: { [key]: ROUNDED }, + incomplete: false, + malformedReference: true, + }, + ]), + { + name: "entity-encoded integer text", + source: { amount: "__INTEGER__" }, + element: "", + expected: { amount: EXACT_INTEGER }, + incomplete: false, + malformedReference: false, + integerElement: "9007199254740993", + }, +] satisfies readonly { + name: string; + source: JsonValue; + element: string; + expected: JsonValue; + incomplete: boolean; + malformedReference: boolean; + integerElement?: string; +}[]; + +type MetadataCase = (typeof archiveNumberMetadataCases)[number]; + +/** Write one inert archive and retain the original byte digest for assertions. */ +export const keyedArchiveNumberMetadataFixture = async (item: MetadataCase) => { + const root = await createTestTempDirectory("rea-keyed-number-metadata-"); + const xml = build({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: ["$null", item.source], + $top: { root: { CF$UID: 1 } }, + }) + .replace( + "__INTEGER__", + "integerElement" in item + ? item.integerElement + : "9007199254740993", + ) + .replace("__OTHER__", item.element); + const bytes = Buffer.from(xml); + const path = join(root, "archive.plist"); + await writeFile(path, bytes); + return { + root, + path, + digest: createHash("sha256").update(bytes).digest("hex"), + }; +}; + +/** Require exact ordinary values and honest limitations for incomplete metadata. */ +export const expectKeyedArchiveNumberMetadata = ( + value: unknown, + item: MetadataCase, + digest: string, +) => { + const result = keyedArchiveResultSchema.parse(value); + expect(result.archive_sha256).toBe(digest); + expect(result.roots).toEqual({ root: { CF$UID: 1 } }); + expect(result.objects).toHaveLength(2); + expect(result.objects[1]).toMatchObject({ + id: 1, + kind: "dictionary", + value: item.expected, + }); + expect(result.references).toContainEqual({ + source: null, + path: ["root"], + target: 1, + status: "resolved", + raw: { CF$UID: 1 }, + }); + if (item.incomplete) { + expect(result.limitations).toContainEqual( + expect.stringMatching(/metadata.*completely observed.*precision/u), + ); + expect( + result.limitations.some((note) => note.includes("unambiguous integer")), + ).toBe(false); + } else if (item.malformedReference) { + expect(result.references).toContainEqual({ + source: 1, + path: [], + target: null, + status: "malformed", + raw: item.expected, + }); + } else { + expect(result.references).toHaveLength(1); + expect(result.limitations).toContainEqual( + expect.stringMatching(/^1 unambiguous integer value/u), + ); + } +}; From 65a809a0da0555aa8eeafbc2ec2559517471be02 Mon Sep 17 00:00:00 2001 From: Chris Ozsvath <142112319+MegaPhoenix92@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:18:07 -0400 Subject: [PATCH 069/791] fix(process): avoid cold ownership helper deadline exhaustion (#1139) * fix(process): avoid cold ownership helper deadline exhaustion Fixes #1137. * test(process): fix cold ownership test deadlines and cleanup * fix(process): isolate Swift cache overrides and retain diagnostics * test(electron): assert blank path rejection without trim assumptions --------- Co-authored-by: Chris Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- src/javascript/sourceMaps/SourceMapCommand.ts | 21 +++++ src/process/DarwinProcessRunTokenReader.ts | 17 ++-- .../applications/electronBlankPathCli.test.ts | 2 +- tests/acceptance/webSourceLocationCli.test.ts | 13 ++- .../javascript/sourceMapDecoder.test.ts | 28 +++++- .../darwinProcessRunTokenReader.test.ts | 91 ++++++++++++++++++- 6 files changed, 156 insertions(+), 16 deletions(-) diff --git a/src/javascript/sourceMaps/SourceMapCommand.ts b/src/javascript/sourceMaps/SourceMapCommand.ts index eec1f884471..2caebcdb2fe 100644 --- a/src/javascript/sourceMaps/SourceMapCommand.ts +++ b/src/javascript/sourceMaps/SourceMapCommand.ts @@ -26,6 +26,7 @@ import { type OwnedProviderProcessSpawnOptions, type SpawnedOwnedProviderProcess, } from "../../process/ProviderProcess.js"; +import { prepareProcessOwnershipInspection } from "../../process/ProcessOwnershipObservation.js"; const OPERATION = "trace_web_source_location"; const PROVIDER = "source-map-decoder"; @@ -43,6 +44,8 @@ export interface SourceMapDecoderDependencies { readonly launcher?: SourceMapCodecLauncher; readonly createRuntime?: () => Promise; readonly environment?: Readonly; + /** Prepare host ownership inspection outside the codec execution deadline. */ + readonly prepareOwnership?: (signal?: AbortSignal) => Promise; } /** Run a trusted codec command and release its process/root before returning output. */ @@ -52,6 +55,24 @@ export const runSourceMapCommand = async ( options: ExecutionOptions | undefined, dependencies: SourceMapDecoderDependencies, ): Promise> => { + try { + await (dependencies.prepareOwnership ?? prepareProcessOwnershipInspection)( + options?.signal, + ); + } catch (cause: unknown) { + return err( + options?.signal?.aborted === true + ? new AnalysisCancelledError(OPERATION) + : new ProviderAdapterError(PROVIDER, OPERATION, { + cause, + diagnostics: { + map_path: path, + error_message: message(cause), + phase: "process_ownership_preparation", + }, + }), + ); + } const deadline = new ProviderStartupDeadline( WEB_SOURCE_MAP_LIMITS.decodeTimeoutMs, options?.signal, diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index 17668806371..5c176dc0774 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -149,21 +149,18 @@ export const createDarwinProcessRunTokenReader = ( exitCleanupInstalled = true; if (signal.aborted || closed) throw abortReason(signal); const output = join(operationRoot, "reader"); + const compilerEnvironment = { ...process.env }; + // Use Swift's reusable default cache for REA's internal helper compilation. + delete compilerEnvironment.CLANG_MODULE_CACHE_PATH; try { await execFileOutput( options.xcrun ?? "/usr/bin/xcrun", - [ - "swiftc", - "-module-cache-path", - join(operationRoot, "modules"), - ...sourceFiles, - "-o", - output, - ], + ["swiftc", ...sourceFiles, "-o", output], { timeout: 60_000, maxBuffer: 1024 * 1024, signal, + env: compilerEnvironment, }, ); } catch (cause: unknown) { @@ -173,7 +170,9 @@ export const createDarwinProcessRunTokenReader = ( ? String(cause.code) : "unknown"; throw new DarwinProcessOwnershipInspectionError( - `macOS process ownership inspection requires the Apple Swift compiler via xcrun (compiler result: ${code})`, + code === "ENOENT" + ? `macOS process ownership inspection requires the Apple Swift compiler via xcrun (compiler result: ${code})` + : `macOS process ownership helper compilation failed via xcrun (compiler result: ${code}): ${cause instanceof Error ? cause.message : String(cause)}`, { cause }, ); } diff --git a/tests/acceptance/applications/electronBlankPathCli.test.ts b/tests/acceptance/applications/electronBlankPathCli.test.ts index f060aed34d0..a03968ca11f 100644 --- a/tests/acceptance/applications/electronBlankPathCli.test.ts +++ b/tests/acceptance/applications/electronBlankPathCli.test.ts @@ -24,7 +24,7 @@ cliTest.for(["", " \t "])( details: { operation: "capture_electron_scenario", issues: expect.arrayContaining([ - { path: ["application_root"], reason: "out_of_range", minimum: 1 }, + expect.objectContaining({ path: ["application_root"] }), ]), }, }); diff --git a/tests/acceptance/webSourceLocationCli.test.ts b/tests/acceptance/webSourceLocationCli.test.ts index 2e7b3a2fe32..92baa900c33 100644 --- a/tests/acceptance/webSourceLocationCli.test.ts +++ b/tests/acceptance/webSourceLocationCli.test.ts @@ -36,8 +36,9 @@ cliTest( environment: { REA_LOG_LEVEL: "silent", REA_BROWSER_EXECUTABLE: "relative-unconfigured-browser", + CLANG_MODULE_CACHE_PATH: map, }, - timeoutMs: 15000, + timeoutMs: 45000, }); expect(response.exitCode).toBe(0); expect(response.json).toMatchObject({ @@ -56,7 +57,11 @@ cliTest( fileURLToPath(new URL("../../scripts/rea.mjs", import.meta.url)), "mcp", ], - env: { ...getDefaultEnvironment(), REA_LOG_LEVEL: "silent" }, + env: { + ...getDefaultEnvironment(), + REA_LOG_LEVEL: "silent", + CLANG_MODULE_CACHE_PATH: map, + }, stderr: "pipe", }); const client = new Client({ @@ -96,7 +101,7 @@ cliTest( "--json", ], environment: { REA_LOG_LEVEL: "silent" }, - timeoutMs: 15000, + timeoutMs: 45000, }); expect(invalid.exitCode).toBe(1); expect(invalid.json).toMatchObject({ @@ -104,4 +109,6 @@ cliTest( details: { issues: [{ path: ["generated_position"] }] }, }); }, + // Allow cold CLI and MCP ownership preparation across the full parity sequence. + 180_000, ); diff --git a/tests/boundary/javascript/sourceMapDecoder.test.ts b/tests/boundary/javascript/sourceMapDecoder.test.ts index 39748547928..6b3938b3cb8 100644 --- a/tests/boundary/javascript/sourceMapDecoder.test.ts +++ b/tests/boundary/javascript/sourceMapDecoder.test.ts @@ -1,7 +1,7 @@ import { fileURLToPath } from "node:url"; import type { SourceMapDecoderDependencies } from "../../../src/javascript/sourceMaps/SourceMapDecoder.js"; import { access } from "node:fs/promises"; -import { expect, it } from "vitest"; +import { expect, it, vi } from "vitest"; import { SourceMapDecoder } from "../../../src/javascript/sourceMaps/SourceMapDecoder.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; @@ -41,6 +41,32 @@ const input = { position: webSourceLocationArgs.generated_position, }; +it("starts the codec deadline after cold ownership preparation", async () => { + vi.useFakeTimers(); + try { + let prepared = false; + const responsePromise = realDecoder({ + prepareOwnership: async (signal) => { + await vi.advanceTimersByTimeAsync(20_001); + expect(signal?.aborted).not.toBe(true); + prepared = true; + }, + launcher: (options) => { + expect(prepared).toBe(true); + expect(options.signal?.aborted).not.toBe(true); + return Promise.reject(new Error("observed launch after preparation")); + }, + }).trace(input); + const response = await responsePromise; + if (response.ok) throw new Error("expected launch failure"); + expect(response.error).toMatchObject({ + diagnostics: { error_message: "observed launch after preparation" }, + }); + } finally { + vi.useRealTimers(); + } +}); + it("runs the real pinned codec with an independent heap and removes its private request directory", async () => { let path: string | undefined; const environment = { diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 8e2977f509d..e32dcc813b4 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -2,6 +2,7 @@ import { execFile, spawn, type ChildProcess } from "node:child_process"; import { access, chmod, + mkdir, mkdtemp, readFile, rm, @@ -14,6 +15,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { _electron as electron } from "playwright-core"; import { expect, it, vi } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createSystemProcessOwnershipHost, @@ -41,10 +43,10 @@ const state = fs.existsSync(statePath) ? JSON.parse(fs.readFileSync(statePath, "utf8")) : { calls: 0 }; state.calls += 1; -const moduleIndex = process.argv.indexOf("-module-cache-path"); -state.root = path.dirname(process.argv[moduleIndex + 1]); const outputIndex = process.argv.indexOf("-o"); state.output = process.argv[outputIndex + 1]; +state.root = path.dirname(state.output); +state.arguments = process.argv.slice(2); state.releasePath = releasePath; fs.writeFileSync(statePath, JSON.stringify(state)); if (state.calls === 1) { @@ -106,6 +108,10 @@ const waitForCompileState = async (statePath: string) => { root: parsed.root, output: parsed.output, releasePath: parsed.releasePath, + arguments: + "arguments" in parsed && Array.isArray(parsed.arguments) + ? parsed.arguments + : [], }; } catch { await new Promise((resolve) => setTimeout(resolve, 10)); @@ -161,6 +167,87 @@ it("reports an actionable missing Swift compiler without installing it", async ( } }); +it.skipIf(process.platform === "win32")( + "preserves compiler diagnostics when Swift preparation fails", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-compiler-failure-test-", + ); + const executable = join(directory, "failing-xcrun"); + await writeFile( + executable, + '#!/usr/bin/env node\nprocess.stderr.write("module cache is not writable\\n"); process.exitCode = 1;\n', + ); + await chmod(executable, 0o755); + const reader = createDarwinProcessRunTokenReader({ xcrun: executable }); + try { + await expect(reader.prepare()).rejects.toThrow( + /compilation failed.*module cache is not writable/su, + ); + } finally { + await reader.close(); + } + }, +); + +it + .skipIf(!onDarwin || process.getuid?.() === 0) + .each(["file", "read-only directory"])( + "ignores unusable inherited Swift module cache: %s", + async (kind) => { + const directory = await createTestTempDirectory( + "rea-process-token-inherited-cache-test-", + ); + const cacheParent = join(directory, "cache-parent"); + if (kind === "file") await writeFile(cacheParent, "not a directory"); + else { + await mkdir(cacheParent); + await chmod(cacheParent, 0o500); + } + const cachePath = join(cacheParent, "modules"); + vi.stubEnv("CLANG_MODULE_CACHE_PATH", cachePath); + const reader = createDarwinProcessRunTokenReader(); + try { + const executable = await reader.prepare(); + const { stdout } = await execFileOutput( + executable, + ["--identities", String(process.pid)], + { timeout: 15_000 }, + ); + expect(JSON.parse(stdout)).toMatchObject({ + results: [{ pid: process.pid, state: "readable" }], + }); + expect(process.env.CLANG_MODULE_CACHE_PATH).toBe(cachePath); + } finally { + vi.unstubAllEnvs(); + if (kind === "read-only directory") await chmod(cacheParent, 0o700); + await reader.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "uses the compiler-managed module cache instead of rebuilding system modules per REA process", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-module-cache-test-", + ); + const fakeCompiler = await writeBlockingCompiler(directory); + const reader = createDarwinProcessRunTokenReader({ + xcrun: fakeCompiler.executable, + }); + const preparation = reader.prepare(); + try { + const state = await waitForCompileState(fakeCompiler.statePath); + expect(state.arguments).not.toContain("-module-cache-path"); + await writeFile(fakeCompiler.releasePath, "release"); + await preparation; + } finally { + await Promise.all([reader.close(), Promise.allSettled([preparation])]); + } + }, +); + it.skipIf(process.platform === "win32")( "rethrows compile cancellation, removes its temporary root, and permits retry", async () => { From d84bc8f11abd6cbf5612daf5c8f7e1805d23c16e Mon Sep 17 00:00:00 2001 From: Selim <36766571+LiquidSnake0@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:21:36 +0200 Subject: [PATCH 070/791] feat(ghidra): make the startup deadline configurable (#974) * feat(ghidra): make the startup deadline configurable Large binaries (an 8 MB PE with about 20,000 functions) need more than the fixed 330 s for import and full auto-analysis, so every first request timed out. REA_GHIDRA_STARTUP_TIMEOUT_MS overrides the deadline in milliseconds; absent, empty or invalid values keep the 330 s default. * docs(ghidra): document REA_GHIDRA_STARTUP_TIMEOUT_MS * fix(ghidra): parse startup deadlines through runtime configuration --------- Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> --- README.md | 2 + docs/installation.md | 8 ++- docs/mcp-contracts.md | 10 +-- src/config/environment.ts | 2 + src/config/ghidraStartupTimeout.ts | 20 ++++++ src/config/parseConfig.ts | 1 + src/config/types.ts | 1 + src/ghidra/GhidraClient.ts | 5 +- src/ghidra/GhidraDefaults.ts | 2 - src/ghidra/GhidraProvider.behavior.test.ts | 74 ++++++++++++++++++++++ src/ghidra/GhidraProviderClient.ts | 37 +++++++++-- 11 files changed, 145 insertions(+), 17 deletions(-) create mode 100644 src/config/ghidraStartupTimeout.ts delete mode 100644 src/ghidra/GhidraDefaults.ts diff --git a/README.md b/README.md index d5b22d365a7..8a38c7cfbfb 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,8 @@ Native formats and host support vary by provider. See [IDA guide](docs/ida-provider.md), and [experimental Windows Ghidra support](docs/windows-ghidra-p0.md). Ghidra also supports [16-bit DOS analysis](docs/ghidra-dos.md). +For large binaries, raise its startup deadline with +`REA_GHIDRA_STARTUP_TIMEOUT_MS`. For provider selection, see the [CLI guide](docs/cli.md#choose-a-provider). Check [release availability](docs/installation.md#released-package-and-main) for features added since the latest npm release. diff --git a/docs/installation.md b/docs/installation.md index 5dcf4d99a11..6ee14c8cc2a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -495,8 +495,12 @@ private native-owned bearer descriptor and Job Object process ownership. Operations begin only after default auto-analysis completes. `open_binary` selects and validates the target/provider binding; it does not wait for Ghidra import and auto-analysis. The first Ghidra-backed query starts that work lazily. -The provider startup deadline is 330,000 ms for import, analysis, bridge, and -health readiness; a startup failure is returned by the query that triggered it, +The provider startup deadline is 330,000 ms by default for import, analysis, +bridge, and health readiness. Large binaries can need more: set +`REA_GHIDRA_STARTUP_TIMEOUT_MS` to an integer between 1 and 2,147,483,647 +milliseconds in the server environment. An absent, empty, invalid or out-of-range +value keeps the default. REA parses the supplied configuration environment and +passes the deadline to each provider client; a startup failure is returned by the query that triggered it, without exposing partial analysis. This deadline is separate from MCP transport initialization and the client's deadline for that individual tool call. A client can time out earlier even when Ghidra would complete within its startup deadline. diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index fe7a4df8a3b..5a8b92de38c 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -111,11 +111,11 @@ auto-analysis, bridge connection, and health readiness before returning analysis These deadlines have different owners: -| Deadline | Owner and effect | -| -------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| MCP initialize | The client bounds transport/REA connection startup, before any target query. | -| Ghidra startup | REA allows 330,000 ms for engine readiness from the first provider query; startup failure is reported by that query. | -| Individual tool call | The client bounds its wait, including cold engine startup. The pinned client SDK 2.3.1 defaults to 60,000 ms and can cancel earlier than REA's startup deadline. | +| Deadline | Owner and effect | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| MCP initialize | The client bounds transport/REA connection startup, before any target query. | +| Ghidra startup | REA allows 330,000 ms by default; `REA_GHIDRA_STARTUP_TIMEOUT_MS` accepts 1–2,147,483,647 ms. Invalid values keep the default. Startup failure is reported by the first provider query. | +| Individual tool call | The client bounds its wait, including cold engine startup. The pinned client SDK 2.3.1 defaults to 60,000 ms and can cancel earlier than REA's startup deadline. | For an already connected client using the pinned SDK, request options are the **second** argument of `callTool`: diff --git a/src/config/environment.ts b/src/config/environment.ts index aebcd5be5e9..fe7e1245e91 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -4,6 +4,7 @@ import { isAbsolute } from "node:path"; import { ConfigurationError } from "../domain/configurationErrors.js"; import { err, ok, type Result } from "../domain/result.js"; import { analysisProviderSelectorSchema } from "../contracts/providerSelection.js"; +import { ghidraStartupTimeoutSchema } from "./ghidraStartupTimeout.js"; import { mcpResponseBudgetSchema } from "./mcpResponseBudget.js"; const environmentSchema = z.object({ @@ -29,6 +30,7 @@ const environmentSchema = z.object({ .min(1) .refine(isAbsolute, "REA_GHIDRA_NATIVEAOT_JAR must be absolute") .optional(), + REA_GHIDRA_STARTUP_TIMEOUT_MS: ghidraStartupTimeoutSchema, REA_ILSPY_CMD_PATH: z .string() .min(1) diff --git a/src/config/ghidraStartupTimeout.ts b/src/config/ghidraStartupTimeout.ts new file mode 100644 index 00000000000..e476321ea5a --- /dev/null +++ b/src/config/ghidraStartupTimeout.ts @@ -0,0 +1,20 @@ +import { z } from "zod"; + +/** Default absolute deadline for Ghidra import, analysis, bridge, and readiness. */ +export const DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS = 330_000; + +// Node schedules delays above this signed 32-bit boundary as a 1 ms timer. +const MAX_TIMER_DELAY_MS = 2_147_483_647; + +/** Parse the optional setting without reading the ambient process environment. */ +export const ghidraStartupTimeoutSchema = z + .string() + .optional() + .transform((raw) => { + const value = Number(raw); + return Number.isSafeInteger(value) && + value > 0 && + value <= MAX_TIMER_DELAY_MS + ? value + : DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS; + }); diff --git a/src/config/parseConfig.ts b/src/config/parseConfig.ts index 4ab6e02ee5a..36ddb4544ce 100644 --- a/src/config/parseConfig.ts +++ b/src/config/parseConfig.ts @@ -53,6 +53,7 @@ export const parseConfig = ( : { idaMcpConfigPath: env.REA_IDA_MCP_CONFIG }), ghidraInstallDir: env.GHIDRA_INSTALL_DIR, ghidraJavaHome: env.JAVA_HOME, + ghidraStartupTimeoutMs: env.REA_GHIDRA_STARTUP_TIMEOUT_MS, ...(env.REA_GHIDRA_NATIVEAOT_JAR === undefined ? {} : { ghidraNativeAotJar: env.REA_GHIDRA_NATIVEAOT_JAR }), diff --git a/src/config/types.ts b/src/config/types.ts index 2e27260b5ff..149a77f4c4c 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -7,6 +7,7 @@ export interface AppConfig { readonly idaMcpConfigPath?: string; readonly ghidraInstallDir: string | undefined; readonly ghidraJavaHome: string | undefined; + readonly ghidraStartupTimeoutMs: number; readonly ghidraNativeAotJar?: string; readonly ilspyCmdPath: string | undefined; readonly hopperLauncherPath: string; diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index 4930795ad87..a7225df6e17 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -14,7 +14,7 @@ import { type ProviderProcessSnapshot, ProviderProcessSupervisor, } from "../process/ProviderProcess.js"; -import { GHIDRA_STARTUP_TIMEOUT_MS } from "./GhidraDefaults.js"; +import { DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS } from "../config/ghidraStartupTimeout.js"; import type { GhidraClientOptions, GhidraRequestOptions, @@ -124,7 +124,8 @@ export class GhidraClient { constructor(options: GhidraClientOptions) { this.#options = { ...options, - startupTimeoutMs: options.startupTimeoutMs ?? GHIDRA_STARTUP_TIMEOUT_MS, + startupTimeoutMs: + options.startupTimeoutMs ?? DEFAULT_GHIDRA_STARTUP_TIMEOUT_MS, platform: options.platform ?? process.platform, transport: options.transport ?? "unix-socket", }; diff --git a/src/ghidra/GhidraDefaults.ts b/src/ghidra/GhidraDefaults.ts deleted file mode 100644 index d04bc3e9dbb..00000000000 --- a/src/ghidra/GhidraDefaults.ts +++ /dev/null @@ -1,2 +0,0 @@ -/** Absolute import, analysis, bridge, and health startup deadline. */ -export const GHIDRA_STARTUP_TIMEOUT_MS = 330_000; diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index cff1ba04aa4..cc1181cbd0e 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -26,6 +26,7 @@ import { GHIDRA_SESSION_CAPABILITIES } from "./GhidraSessionValues.js"; import { err, ok } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { silentLogger } from "../logger.js"; +import { ProviderStartupDeadline } from "../process/ProviderDeadline.js"; const INSTALL = "/opt/ghidra_12.1.4_PUBLIC"; const installationHost = (): GhidraInstallationHost => ({ @@ -383,6 +384,79 @@ describe("Ghidra platform support", () => { }); }); +describe("Ghidra startup configuration", () => { + it("carries supplied startup settings through configuration, the provider, the timer and timeout diagnostics", async () => { + const target = executableTarget("elf", "x86_64"); + for (const [raw, expected] of [ + [undefined, 330_000], + ["", 330_000], + [" ", 330_000], + ["900000", 900_000], + ["1260000", 1_260_000], + ["2147483647", 2_147_483_647], + ["2147483648", 330_000], + ["9007199254740991", 330_000], + ["0", 330_000], + ["-5", 330_000], + ["abc", 330_000], + ["1.5", 330_000], + ] as const) { + const config = parseConfig({ + GHIDRA_INSTALL_DIR: INSTALL, + ...(raw === undefined ? {} : { REA_GHIDRA_STARTUP_TIMEOUT_MS: raw }), + }); + if (!config.ok) throw config.error; + let timeout: number | undefined; + const ghidra = new GhidraProvider( + config.value, + silentLogger, + installationHost(), + (options) => { + timeout = options.startupTimeoutMs; + return { + start: () => + Promise.resolve( + err( + new GhidraSessionError( + "timeout", + "Startup deadline elapsed", + {}, + ), + ), + ), + callTool: () => Promise.resolve(ok([])), + close: () => Promise.resolve(), + }; + }, + ); + const profile = await ghidra.resolveAnalysisProfile(target); + if (!profile.ok) throw profile.error; + if (profile.value.profile === null) + throw new Error("Expected a bound profile"); + const client = ghidra.createClient(target, profile.value.profile); + try { + const failure = await client.execute("health", {}); + expect(timeout, String(raw)).toBe(expected); + expect(failure).toMatchObject({ + ok: false, + error: { _tag: "AnalysisTimeoutError", timeoutMs: expected }, + }); + if (timeout === undefined) + throw new Error("Provider omitted startup deadline"); + const deadline = new ProviderStartupDeadline(timeout); + try { + expect(await deadline.wait(20)).toBe("elapsed"); + expect(deadline.signal.aborted).toBe(false); + } finally { + deadline.dispose(); + } + } finally { + await client.close(); + } + } + }); +}); + describe("Ghidra client projection", () => { it("commits exact provider, isolation, and resource semantics", async () => { const toolCalls: Array<{ diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index aed08e04b44..e4456e31a69 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -24,7 +24,6 @@ import { err, ok, type Result } from "../domain/result.js"; import type { Logger } from "../logger.js"; import { GhidraClient } from "./GhidraClient.js"; import type { GhidraClientOptions } from "./GhidraClientTypes.js"; -import { GHIDRA_STARTUP_TIMEOUT_MS } from "./GhidraDefaults.js"; import { isGhidraFunctionOperation, parseGhidraFunctionInput, @@ -132,6 +131,7 @@ export const createGhidraProviderClient = (input: { ] : []; const client = clientFactory({ + startupTimeoutMs: config.ghidraStartupTimeoutMs, platform: installation.platform, launcher: new GhidraHeadlessLauncher({ analyzeHeadlessPath: prerequisites.value.analyzeHeadlessPath, @@ -204,7 +204,13 @@ export const createGhidraProviderClient = (input: { if (operation === "health") { const started = await client.start(options?.signal); if (!started.ok) - return err(projectSessionError(operation, started.error)); + return err( + projectSessionError( + operation, + started.error, + config.ghidraStartupTimeoutMs, + ), + ); const failed = await checkExtensions(operation, started.value); if (failed !== undefined) return err(failed); return ok( @@ -221,7 +227,13 @@ export const createGhidraProviderClient = (input: { if (extensions.length > 0) { const started = await client.start(options?.signal); if (!started.ok) - return err(projectSessionError(operation, started.error)); + return err( + projectSessionError( + operation, + started.error, + config.ghidraStartupTimeoutMs, + ), + ); const failed = await checkExtensions(operation, started.value); if (failed !== undefined) return err(failed); } @@ -230,7 +242,14 @@ export const createGhidraProviderClient = (input: { input.value, options?.signal === undefined ? {} : { signal: options.signal }, ); - if (!called.ok) return err(projectSessionError(operation, called.error)); + if (!called.ok) + return err( + projectSessionError( + operation, + called.error, + config.ghidraStartupTimeoutMs, + ), + ); const result = isGhidraFunctionOperation(operation) ? parseGhidraFunctionResult(operation, called.value) : parseGhidraInventoryResult(operation, called.value); @@ -242,7 +261,12 @@ export const createGhidraProviderClient = (input: { operation, result.value, client, - (failure) => projectSessionError(operation, failure), + (failure) => + projectSessionError( + operation, + failure, + config.ghidraStartupTimeoutMs, + ), ); if (!attested.ok) return attested; normalized = attested.value; @@ -320,6 +344,7 @@ const unavailableClient = (failure: AnalysisError): AnalysisClient => ({ const projectSessionError = ( operation: AnalysisOperation, failure: GhidraSessionError, + startupTimeoutMs: number, ): AnalysisError => { if (failure.cause instanceof AnalysisAccessDeniedError) return new AnalysisAccessDeniedError( @@ -348,7 +373,7 @@ const projectSessionError = ( if (failure.kind === "timeout" || failure.kind === "analysis_timeout") return new AnalysisTimeoutError( operation, - failure.timeoutMs ?? GHIDRA_STARTUP_TIMEOUT_MS, + failure.timeoutMs ?? startupTimeoutMs, ); if (failure.kind === "remote" && failure.remoteCode === "decompile_cancelled") return new AnalysisCancelledError(operation); From 1f7900249783956070af49c10f86002a0dfdc42b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:28:47 +0800 Subject: [PATCH 071/791] docs: clarify contribution scope and cosmetic suggestions --- .github/pull_request_template.md | 1 + CONTRIBUTING.md | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bfd3275adee..8ffbeb73bad 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -70,6 +70,7 @@ limits, security/privacy/containment implications, and meaningful proof gaps. -- ## Review checklist +- [ ] This PR addresses a concrete problem or an agreed enhancement. - [ ] The PR has one focused outcome and the title follows `type(scope): outcome`. - [ ] Related issue is linked, or the reason for not linking one is stated above. - [ ] Tests cover changed observable behavior and meaningful failure paths. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee321d58a4e..bd87b8c2f1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,29 @@ REA welcomes focused bug fixes, documentation improvements, tests, and reverse-engineering workflow enhancements. Open an issue before a large contract or architecture change so its scope can be agreed before implementation. +## Choosing a contribution + +Small contributions are welcome when they solve a concrete problem, including +broken commands, misleading documentation, and missing regression coverage. +Explain the problem, keep the diff focused, and share how you checked the change. + +Please avoid standalone cosmetic changes without a clear benefit, such as fixing +a typo in an internal comment, changing capitalization or punctuation in +already-clear prose, or reformatting unchanged code. Include related polish in a +useful fix when practical. For cosmetic-only suggestions, please open an issue +instead of a standalone PR so they can be collected and addressed together. +Corrections to confusing explanations or incorrect instructions are welcome. + +Avoid speculative cleanup without a concrete problem to solve. Keep related +corrections in one PR. + +Before requesting review, read the final diff and run the relevant checks. Be clear +about anything you couldn't test. You should be able to explain the changes you +submit. + +We may decline changes when the benefit doesn't justify the review or maintenance +work. + For capability organization and provider composition, follow the incremental [migration guide](docs/capability-migration.md). Run `npm run verify:test-discovery` after adding or moving tests. From 3e598b2d38a728cd9fd4a359741500a021ba9d3a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:09:41 +0800 Subject: [PATCH 072/791] fix(ghidra): admit Linux arm64 with matching native tools --- AGENTS.md | 2 +- docs/installation.md | 13 ++-- docs/native-investigation.md | 2 +- docs/provider-evaluation.md | 6 +- docs/roadmap.md | 4 +- docs/testing.md | 8 ++- .../lib/real-ghidra-namespace-annotations.mjs | 49 +++++++++++++- scripts/verify-real-ghidra.mjs | 3 +- src/ghidra/GhidraDoctor.test.ts | 2 +- src/ghidra/GhidraInstallation.test.ts | 64 ++++++++++++------- src/ghidra/GhidraInstallation.ts | 52 +++++++-------- 11 files changed, 139 insertions(+), 66 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 0a2bc53b996..e24455c1db7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Product Direction -REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. +REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64/arm64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. Prioritize: diff --git a/docs/installation.md b/docs/installation.md index 6ee14c8cc2a..2985c12f4ee 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -415,14 +415,17 @@ copying license secrets into logs, or killing unrelated Hopper processes. ## Ghidra -REA connects to an existing Ghidra installation on Linux x64, macOS x64/arm64, +REA connects to an existing Ghidra installation on Linux x64/arm64, macOS x64/arm64, or experimental Windows x64 P0. It accepts Ghidra 12.1.x and the 64-bit full JDK declared by that installation's `application.java.min` and `application.java.max`. Current 12.1 releases require JDK 21 or newer and set no maximum. The bridge is verified with Ghidra 12.1.4 -and JDK 21. On macOS, the installation -must include the native decompiler for the host architecture; REA does not -build it or change Gatekeeper quarantine settings. +and JDK 21. Each installation must include the native decompiler for the host +architecture in `Ghidra/Features/Decompiler/os//` or the corresponding +`build/os//` directory. Linux ARM64 uses `linux_arm_64`; official +release archives may require you to build that native component separately. +REA checks the executable prerequisite and does not build or install native +tools, or change Gatekeeper quarantine settings. The adapter exposes 25 read-only operations: thirteen inventory/name/search operations and twelve function-analysis operations. These cover metadata, @@ -525,7 +528,7 @@ continues. These messages redact known bridge authentication tokens and preserve local paths and other analysis context. Run `GHIDRA_INSTALL_DIR=... npm run verify:ghidra` from a source checkout to -compile and analyze debug and stripped host-native fixtures (ELF on Linux x64 +compile and analyze debug and stripped host-native fixtures (ELF on Linux x64/arm64 or Mach-O on macOS), plus a native DWARF 4 type-layout object. This lane needs a host C compiler in addition to Ghidra and its JDK. diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 8ad7aadbdea..96433fc516f 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -339,7 +339,7 @@ the tool fails with a `debugger-attach-denied` reason. ## Provider and verification boundaries Install a Ghidra 12.1.x release and the 64-bit full JDK it declares, then configure REA to -use them. Ghidra analysis supports Linux x64 and macOS x64/arm64; macOS requires +use them. Ghidra analysis supports Linux x64/arm64 and macOS x64/arm64; each installation requires the matching native decompiler. Experimental Windows x64 P0 admits native x86-64 PE applications on local NTFS using bundled Job Object ownership, protected runtime DACLs, and handle-based path admission. See [Windows Ghidra P0](windows-ghidra-p0.md) and diff --git a/docs/provider-evaluation.md b/docs/provider-evaluation.md index baf5740852d..8aa655e52ad 100644 --- a/docs/provider-evaluation.md +++ b/docs/provider-evaluation.md @@ -2,11 +2,11 @@ This guide describes repository main. Check the [released package boundary](installation.md#released-package-and-main) when using npm. -Ghidra read-only analysis is available on Linux x64 and macOS x64/arm64. +Ghidra read-only analysis is available on Linux x64/arm64 and macOS x64/arm64. Install a Ghidra 12.1.x release and the 64-bit full JDK that release declares, then configure REA to use them. Current 12.1 releases require JDK 21 or newer and set no maximum. The bridge is verified with Ghidra 12.1.4 and JDK 21. -macOS installations need the matching native decompiler. +Each installation needs the matching native decompiler. REA imports one target into a temporary project and exposes 25 read-only operations after analysis completes. They cover inventories, search, @@ -121,7 +121,7 @@ capability from a successful import. | Result extent | Function instruction scans and native API boundary observations remain complete. P-code flow is the explicit bounded exception; its count fields distinguish exact retained counts from lower-bound or known-omitted counts. Caller cancellation and provider failures remain distinct. | `npm run verify:ghidra` compiles the versioned C oracle into debug and stripped -host-native targets (x86-64 ELF on Linux x64 or Mach-O on macOS), plus a native +host-native targets (x86-64/AArch64 ELF on Linux or Mach-O on macOS), plus a native DWARF 4 type-layout object. It proves the admitted operations, external functions, resolved thunks, exports, stripped-name behavior, direct and targetless indirect calls, typed references, strings/xrefs, multi-block CFG, bounded p-code def-use/effect links, semantic enhanced workflows, cancellation, startup deadlines, diff --git a/docs/roadmap.md b/docs/roadmap.md index 62323d591cc..b4ac0ba5454 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -69,10 +69,10 @@ verified paths for an existing Ghidra installation. It configures Claude Code, Claude Desktop, Codex, Cursor, Gemini CLI, Windsurf, Devin, OpenCode, Antigravity, GitHub Copilot CLI, Command Code, VS Code, and Grok Build using each client's configuration format. Grok Bot is detected locally, but its connector is added from the Grok Bot chat because the signed-in account store is not a data-directory file. -Ghidra analysis supports Linux x64 and macOS x64/arm64 with Ghidra 12.1.x and the +Ghidra analysis supports Linux x64/arm64 and macOS x64/arm64 with Ghidra 12.1.x and the 64-bit full JDK that installation declares. Current 12.1 releases require JDK 21 or newer and set no maximum; the bridge is verified with Ghidra 12.1.4 and JDK 21. -macOS also requires the matching native decompiler. The +Each installation requires the matching native decompiler. The adapter exposes thirteen inventory operations and twelve function-analysis operations, for 25 read-only operations total. Linux and macOS additionally support atomic function-name and entry-comment edits with refreshed analysis; metadata diff --git a/docs/testing.md b/docs/testing.md index 70fadaac337..f684673e8a6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -307,12 +307,18 @@ prove behavior on the verified Ghidra 12.1.4 and JDK 21 build. | Ghidra lane | Supported runner/target | Additional local tools | | ------------------------------------------ | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------- | -| `npm run verify:ghidra` | Linux x64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 | +| `npm run verify:ghidra` | Linux x64/arm64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:switch` | Linux x64 ELF; GCC/Clang optimized and stripped switch fixtures | GCC, Clang, GNU nm/objdump/strip, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:aarch64-jump-table` | Any supported Ghidra host; AArch64 ELF; byte/halfword tables; host ARM64 Mach-O | Clang with AArch64 target support, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:cross-format` | Any supported Ghidra host; also analyzes AArch64 ELF, x86-64 PE, and x86-64 Mach-O | `clang`, LLD, and `lld-link` in addition to host-lane prerequisites | | `npm run verify:ghidra:windows` | Controlled Windows x64 with native x86-64 PE; `-- --x86` selects native x86 PE | Ghidra 12.1.4, full JDK 21, and the matching native artifact | +The host lane also verifies namespaced C++ ABI symbols and annotation identity. +That facet requires Ghidra's matching `demangler_gnu_v2_41` native component in +`GPL/DemanglerGnu/os//` or `build/os//`, in addition to the +native decompiler. The namespace verifier reports missing or non-executable +components before testing the fixture. REA does not build or install them. + Windows native conformance runs with `npm run verify:windows-native` and does not require Ghidra or Java. An optional independently compiled Windows fixture adds in-place reparse, breakaway, environment, and token observations. diff --git a/scripts/lib/real-ghidra-namespace-annotations.mjs b/scripts/lib/real-ghidra-namespace-annotations.mjs index 4f6d8c12775..5fd734dd753 100644 --- a/scripts/lib/real-ghidra-namespace-annotations.mjs +++ b/scripts/lib/real-ghidra-namespace-annotations.mjs @@ -1,11 +1,13 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { constants } from "node:fs"; +import { access, mkdtemp, readFile, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { inspectGhidraInstallation } from "../../dist/ghidra/GhidraInstallation.js"; /** Exercise qualified annotation names against real demangled C++ ABI symbols. */ export async function verifyGhidraNamespaceAnnotations({ @@ -15,6 +17,7 @@ export async function verifyGhidraNamespaceAnnotations({ entrypoint, env, }) { + await requireNamespaceDemangler(env); const exec = promisify(execFile); const compiler = process.env.REA_CC ?? "cc"; try { @@ -155,3 +158,45 @@ export async function verifyGhidraNamespaceAnnotations({ await rm(workspace, { recursive: true, force: true }); } } + +async function requireNamespaceDemangler(env) { + const inspection = inspectGhidraInstallation({ + installDir: env.GHIDRA_INSTALL_DIR, + ...(env.JAVA_HOME === undefined ? {} : { javaHome: env.JAVA_HOME }), + }); + if ( + inspection.status !== "available" || + inspection.nativeDecompilerPath === null + ) + throw new Error( + `Ghidra namespace verification requires an available installation: ${JSON.stringify(inspection)}`, + ); + const nativeDirectory = basename(dirname(inspection.nativeDecompilerPath)); + const executable = + process.platform === "win32" + ? "demangler_gnu_v2_41.exe" + : "demangler_gnu_v2_41"; + const failures = []; + for (const location of ["os", "build/os"]) { + const path = join( + inspection.installDir, + "GPL", + "DemanglerGnu", + location, + nativeDirectory, + executable, + ); + try { + await access(path, constants.X_OK); + if ((await stat(path)).isFile()) return; + failures.push(`${path}: not a regular file`); + } catch (cause) { + failures.push( + `${path}: ${cause instanceof Error ? cause.message : String(cause)}`, + ); + } + } + throw new Error( + `Ghidra namespace annotation verification requires the matching GNU demangler native component. Provide ${executable} for ${nativeDirectory} in GPL/DemanglerGnu/os or its build/os equivalent; REA does not build it. Checked: ${failures.join("; ")}`, + ); +} diff --git a/scripts/verify-real-ghidra.mjs b/scripts/verify-real-ghidra.mjs index d960b3d1b5d..da4f15b8ac9 100644 --- a/scripts/verify-real-ghidra.mjs +++ b/scripts/verify-real-ghidra.mjs @@ -299,11 +299,12 @@ function nativeFixtureTarget(platform, architecture) { "darwin-arm64": { format: "mach-o", architecture: "arm64" }, "darwin-x64": { format: "mach-o", architecture: "x86_64" }, "linux-x64": { format: "elf", architecture: "x86_64" }, + "linux-arm64": { format: "elf", architecture: "arm64" }, }; const target = supportedTargets[`${platform}-${architecture}`]; if (target === undefined) throw new Error( - `The Ghidra verifier does not support a host-native fixture on ${platform}/${architecture}; use a supported Linux x64 or macOS x64/arm64 host.`, + `The Ghidra verifier does not support a host-native fixture on ${platform}/${architecture}; use a supported Linux or macOS x64/arm64 host.`, ); return target; } diff --git a/src/ghidra/GhidraDoctor.test.ts b/src/ghidra/GhidraDoctor.test.ts index b7c55e8acfa..e25405dacc5 100644 --- a/src/ghidra/GhidraDoctor.test.ts +++ b/src/ghidra/GhidraDoctor.test.ts @@ -138,7 +138,7 @@ describe("Ghidra doctor integration", () => { ], [ "unsupported architecture", - inspection({ installDir: INSTALL, architecture: "arm64" }), + inspection({ installDir: INSTALL, architecture: "arm" }), "ghidra-architecture", ], ] as const)("distinguishes %s", async (_label, ghidra, checkName) => { diff --git a/src/ghidra/GhidraInstallation.test.ts b/src/ghidra/GhidraInstallation.test.ts index 9f5e869c0ed..18f45e26be4 100644 --- a/src/ghidra/GhidraInstallation.test.ts +++ b/src/ghidra/GhidraInstallation.test.ts @@ -12,10 +12,12 @@ import { const INSTALL = "/opt/ghidra"; const PROPERTIES = `${INSTALL}/Ghidra/application.properties`; const HEADLESS = `${INSTALL}/support/analyzeHeadless`; +const LINUX_DECOMPILER = `${INSTALL}/Ghidra/Features/Decompiler/os/linux_x86_64/decompile`; const MAC_DECOMPILER = `${INSTALL}/Ghidra/Features/Decompiler/os/mac_arm_64/decompile`; const WINDOWS_INSTALL = "C:\\tools\\ghidra_12.1.4_PUBLIC"; const WINDOWS_PROPERTIES = `${WINDOWS_INSTALL}\\Ghidra\\application.properties`; const WINDOWS_HEADLESS = `${WINDOWS_INSTALL}\\support\\analyzeHeadless.bat`; +const WINDOWS_DECOMPILER = `${WINDOWS_INSTALL}\\Ghidra\\Features\\Decompiler\\os\\win_x86_64\\decompile.exe`; const JAVA: GhidraJavaObservation = { version: "21.0.11", major: SUPPORTED_GHIDRA_JAVA_MAJOR, @@ -31,7 +33,8 @@ const host = ( path === PROPERTIES ? `application.version=${SUPPORTED_GHIDRA_VERSION}\n` : undefined, - executable: (path) => path === HEADLESS || path === MAC_DECOMPILER, + executable: (path) => + path === HEADLESS || path === MAC_DECOMPILER || path === LINUX_DECOMPILER, probeJava: () => JAVA, ...overrides, }); @@ -108,28 +111,29 @@ describe("Ghidra installation inspection", () => { }, ); - it.each(["x64", "arm64"] as const)( - "accepts macOS %s with matching native tools", - (architecture) => { - const decompiler = `${INSTALL}/Ghidra/Features/Decompiler/os/${architecture === "arm64" ? "mac_arm_64" : "mac_x86_64"}/decompile`; - const macHost: GhidraInstallationHost = { - ...host({ - platform: "darwin", + it.each([ + ["darwin", "x64", "mac_x86_64"], + ["darwin", "arm64", "mac_arm_64"], + ["linux", "arm64", "linux_arm_64"], + ] as const)( + "accepts %s/%s with the matching distribution or locally built native tool", + (platform, architecture, nativeDirectory) => { + for (const location of ["os", "build/os"]) { + const decompiler = `${INSTALL}/Ghidra/Features/Decompiler/${location}/${nativeDirectory}/decompile`; + expect( + inspectGhidraInstallation( + { installDir: INSTALL, platform, architecture }, + host({ + executable: (path) => path === HEADLESS || path === decompiler, + }), + ), + ).toMatchObject({ + status: "available", + platform, architecture, - executable: (path) => path === HEADLESS || path === decompiler, - }), - }; - expect( - inspectGhidraInstallation( - { installDir: INSTALL, platform: "darwin", architecture }, - macHost, - ), - ).toMatchObject({ - status: "available", - platform: "darwin", - architecture, - nativeDecompilerPath: decompiler, - }); + nativeDecompilerPath: decompiler, + }); + } }, ); @@ -141,7 +145,8 @@ describe("Ghidra installation inspection", () => { path === WINDOWS_PROPERTIES ? `application.version=${SUPPORTED_GHIDRA_VERSION}\n` : undefined, - executable: (path) => path === WINDOWS_HEADLESS, + executable: (path) => + path === WINDOWS_HEADLESS || path === WINDOWS_DECOMPILER, probeJava: (command, environment) => { expect(command).toBe("C:\\Java\\jdk-21\\bin\\java.exe"); expect(environment.PATH).toMatch(/^C:\\Java\\jdk-21\\bin;/u); @@ -198,7 +203,7 @@ describe("Ghidra installation rejection diagnostics", () => { options: { installDir: INSTALL, platform: "linux" as const, - architecture: "arm64" as const, + architecture: "arm" as const, }, override: {}, failed: "architecture", @@ -237,6 +242,17 @@ describe("Ghidra installation rejection diagnostics", () => { failed: "headless", code: "executable_missing", }, + { + name: "only a foreign-architecture decompiler on Linux arm64", + options: { + installDir: INSTALL, + platform: "linux" as const, + architecture: "arm64" as const, + }, + override: {}, + failed: "native_decompiler", + code: "executable_missing", + }, { name: "missing Java", options: { diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index 9a2eb858d70..a521fc171bf 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -22,6 +22,19 @@ import { export { SUPPORTED_GHIDRA_JAVA_MAJOR, SUPPORTED_GHIDRA_VERSION }; +const NATIVE_PLATFORMS: Readonly< + Partial< + Record< + NodeJS.Platform, + Readonly>> + > + > +> = { + linux: { x64: "linux_x86_64", arm64: "linux_arm_64" }, + darwin: { x64: "mac_x86_64", arm64: "mac_arm_64" }, + win32: { x64: "win_x86_64" }, +}; + /** Caller-owned paths and host coordinates used for one installation probe. */ export interface GhidraInstallationOptions { readonly installDir?: string; @@ -228,14 +241,11 @@ const installationCoordinates = ( "support", platform === "win32" ? "analyzeHeadless.bat" : "analyzeHeadless", ); - const decompilerPlatform = - platform === "darwin" - ? architecture === "arm64" - ? "mac_arm_64" - : "mac_x86_64" - : null; + const decompilerPlatform = NATIVE_PLATFORMS[platform]?.[architecture]; + const decompilerExecutable = + platform === "win32" ? "decompile.exe" : "decompile"; const nativeDecompilerPath = - installDir === null || decompilerPlatform === null + installDir === null || decompilerPlatform === undefined ? null : ([ path.join( @@ -245,7 +255,7 @@ const installationCoordinates = ( "Decompiler", "os", decompilerPlatform, - "decompile", + decompilerExecutable, ), path.join( installDir, @@ -255,7 +265,7 @@ const installationCoordinates = ( "build", "os", decompilerPlatform, - "decompile", + decompilerExecutable, ), ].find((candidate) => host.executable(candidate)) ?? null); const properties = @@ -305,23 +315,18 @@ const installationChecks = ({ }), installationCheck({ name: "platform", - passed: - platform === "linux" || platform === "win32" || platform === "darwin", + passed: NATIVE_PLATFORMS[platform] !== undefined, code: "unsupported_host", detail: platform, remediation: - "Use Linux or Windows x64, or macOS x64/arm64 with matching native Ghidra tools.", + "Use Linux or macOS x64/arm64, or Windows x64 with matching native Ghidra tools.", }), installationCheck({ name: "architecture", - passed: - ((platform === "linux" || platform === "win32") && - architecture === "x64") || - (platform === "darwin" && - (architecture === "x64" || architecture === "arm64")), + passed: NATIVE_PLATFORMS[platform]?.[architecture] !== undefined, code: "unsupported_host", detail: architecture, - remediation: "Use x64 on Linux or Windows, or x64/arm64 on macOS.", + remediation: "Use x64/arm64 on Linux or macOS, or x64 on Windows.", }), installationCheck({ name: "installation", @@ -346,15 +351,12 @@ const installationChecks = ({ }), installationCheck({ name: "native_decompiler", - passed: platform !== "darwin" || coordinates.nativeDecompilerPath !== null, + passed: coordinates.nativeDecompilerPath !== null, code: "executable_missing", detail: - platform !== "darwin" - ? "not required on this platform" - : (coordinates.nativeDecompilerPath ?? - "matching macOS native decompiler was not found"), - remediation: - "REA does not build native tools. Build Ghidra's native components for this macOS architecture or provide an installation containing Ghidra/Features/Decompiler/os//decompile.", + coordinates.nativeDecompilerPath ?? + `Matching native decompiler for ${platform}/${architecture} was not found`, + remediation: `Provide an executable Ghidra/Features/Decompiler/os/${NATIVE_PLATFORMS[platform]?.[architecture] ?? ""}/${platform === "win32" ? "decompile.exe" : "decompile"}, or its build/os equivalent. REA does not build or install native tools.`, }), javaCheck( coordinates.java, From 1d9b96a9c497694592137cf95bea2af0a190abe7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:01:38 +0800 Subject: [PATCH 073/791] fix(cli): preserve web cancellation exit status --- src/cli/webRuntimeCommands.ts | 28 +++++----- tests/acceptance/webRuntimeCli.test.ts | 76 ++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/cli/webRuntimeCommands.ts b/src/cli/webRuntimeCommands.ts index 4843c0e23cf..9f52397fc60 100644 --- a/src/cli/webRuntimeCommands.ts +++ b/src/cli/webRuntimeCommands.ts @@ -37,9 +37,9 @@ export const registerWebRuntimeCommands = ( observationMs: observationDuration(10_000, 1), }), run: ({ args, options }) => - logCliCommand(logger, "observe-web-execution", async () => { - const result = await withCommandCancellation((signal) => - createWebRuntimeService().observe( + withCommandCancellation((signal) => + logCliCommand(logger, "observe-web-execution", async () => { + const result = await createWebRuntimeService().observe( { cdp_endpoint: args.endpoint, target_id: args.targetId, @@ -47,10 +47,10 @@ export const registerWebRuntimeCommands = ( observation_ms: options.observationMs, }, { progress: runtimeProgress, signal }, - ), - ); - return result.ok ? result.value : browserCliError(result.error); - }), + ); + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); cli.command(CLI_COMMANDS.inspectWebEventListeners, { description: @@ -66,9 +66,9 @@ export const registerWebRuntimeCommands = ( }), options: z.object({ ...browserScopeOptions }), run: ({ args, options }) => - logCliCommand(logger, "inspect-web-event-listeners", async () => { - const result = await withCommandCancellation((signal) => - createWebRuntimeService().inspect( + withCommandCancellation((signal) => + logCliCommand(logger, "inspect-web-event-listeners", async () => { + const result = await createWebRuntimeService().inspect( { cdp_endpoint: args.endpoint, target_id: args.targetId, @@ -76,9 +76,9 @@ export const registerWebRuntimeCommands = ( allowed_origins: options.allowedOrigins, }, { signal }, - ), - ); - return result.ok ? result.value : browserCliError(result.error); - }), + ); + return result.ok ? result.value : browserCliError(result.error); + }), + ), }); }; diff --git a/tests/acceptance/webRuntimeCli.test.ts b/tests/acceptance/webRuntimeCli.test.ts index 9b308407430..53205e23e5d 100644 --- a/tests/acceptance/webRuntimeCli.test.ts +++ b/tests/acceptance/webRuntimeCli.test.ts @@ -1,5 +1,7 @@ import { execFile, spawn } from "node:child_process"; import { once } from "node:events"; +import { createServer } from "node:http"; +import type { Socket } from "node:net"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { expect, it, onTestFinished } from "vitest"; @@ -104,3 +106,77 @@ it("translates SIGINT after real CLI arming into awaited instrumentation cleanup clearTimeout(timer); } }); + +it.each([ + ["observe-web-execution", "SIGINT", 130], + ["observe-web-execution", "SIGTERM", 143], + ["inspect-web-event-listeners", "SIGINT", 130], + ["inspect-web-event-listeners", "SIGTERM", 143], +] as const)( + "%s preserves %s status when cancellation interrupts CDP discovery", + async (command, signal, expectedExitCode) => { + const sockets = new Set(); + const server = createServer(() => undefined); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (address === null || typeof address === "string") + throw new Error("Expected a TCP loopback listener"); + const endpoint = `http://127.0.0.1:${String(address.port)}`; + let contactedResolve!: () => void; + const contacted = new Promise((resolve) => { + contactedResolve = resolve; + }); + server.on("request", contactedResolve); + + const args = + command === "observe-web-execution" + ? [command, endpoint, "page-fixture", "--observation-ms", "30000", "--json"] + : [command, endpoint, "page-fixture", "body", "--json"]; + const child = spawn(process.execPath, [entrypoint, ...args], { + env: environment(), + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + child.stdout.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + stdout += chunk; + }); + const timeout = setTimeout(() => child.kill("SIGKILL"), 10_000); + try { + let contactTimeout: NodeJS.Timeout | undefined; + try { + await Promise.race([ + contacted, + new Promise((_resolve, reject) => { + contactTimeout = setTimeout( + () => reject(new Error("CLI did not contact the CDP fixture")), + 5_000, + ); + }), + ]); + } finally { + if (contactTimeout !== undefined) clearTimeout(contactTimeout); + } + const closed = once(child, "close"); + child.kill(signal); + const [exitCode, terminationSignal] = await closed; + expect(terminationSignal).toBeNull(); + expect(exitCode).toBe(expectedExitCode); + expect(JSON.parse(stdout)).toMatchObject({ + code: "cancelled", + category: "cancelled", + }); + } finally { + clearTimeout(timeout); + if (child.exitCode === null && child.signalCode === null) + child.kill("SIGKILL"); + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } + }, +); From a69b8337ab38707aaad2ac2fedb652f05afeb71c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:01:40 +0800 Subject: [PATCH 074/791] fix(cli): support MCP doctor help --- src/mcpDoctor.ts | 13 ++++++ tests/boundary/cli/mcpDoctorHelp.test.ts | 50 ++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 tests/boundary/cli/mcpDoctorHelp.test.ts diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index b50c450a4aa..f6c8510bfd6 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -11,6 +11,17 @@ import { MCP_STARTUP_POLICY } from "./mcpStartupPolicy.js"; const OUTPUT_FORMATS = ["toon", "json", "yaml", "md", "jsonl"] as const; type OutputFormat = (typeof OUTPUT_FORMATS)[number]; +const MCP_DOCTOR_HELP = `Usage: rea mcp doctor [options] + +Validate MCP server startup, tool and prompt inventories, identity, and request flow. + +Options: + --json Output JSON + --format Output format: ${OUTPUT_FORMATS.join(", ")} + --format= Set the output format + --full-output Include complete diagnostic details + -h, --help Show this help +`; interface McpDoctorOptions { readonly command: string; @@ -172,6 +183,8 @@ export const runProductionMcpDoctorCli = async ( arguments_: readonly string[], input: { readonly dispatcherPath: string; readonly packageRoot: string }, ): Promise<{ readonly output: string; readonly exitCode: number }> => { + if (arguments_.includes("--help") || arguments_.includes("-h")) + return { output: MCP_DOCTOR_HELP, exitCode: 0 }; const parsed = parseOutputArguments(arguments_); if (!parsed.ok) return { diff --git a/tests/boundary/cli/mcpDoctorHelp.test.ts b/tests/boundary/cli/mcpDoctorHelp.test.ts new file mode 100644 index 00000000000..8f91efeca7b --- /dev/null +++ b/tests/boundary/cli/mcpDoctorHelp.test.ts @@ -0,0 +1,50 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +const execute = promisify(execFile); + +const runCli = async (arguments_: readonly string[]) => { + try { + const result = await execute(process.execPath, ["scripts/rea.mjs", ...arguments_], { + cwd: process.cwd(), + env: { ...process.env, REA_LOG_LEVEL: "silent" }, + timeout: 15_000, + }); + return { exitCode: 0, ...result }; + } catch (cause: unknown) { + if ( + typeof cause === "object" && + cause !== null && + "code" in cause && + typeof cause.code === "number" && + "stdout" in cause && + typeof cause.stdout === "string" && + "stderr" in cause && + typeof cause.stderr === "string" + ) + return { + exitCode: cause.code, + stdout: cause.stdout, + stderr: cause.stderr, + }; + throw cause; + } +}; + +it("shows MCP doctor help without starting a server and keeps option errors actionable", async () => { + const parentHelp = await runCli(["mcp", "--help"]); + expect(parentHelp.exitCode).toBe(0); + expect(parentHelp.stdout).toContain("doctor"); + + const help = await runCli(["mcp", "doctor", "--help"]); + expect(help.exitCode).toBe(0); + expect(help.stdout).toContain("Usage: rea mcp doctor [options]"); + expect(help.stdout).toContain("--format "); + expect(help.stdout).toContain("--full-output"); + + const invalid = await runCli(["mcp", "doctor", "--not-an-option"]); + expect(invalid.exitCode).toBe(1); + expect(invalid.stdout).toContain("Unknown mcp doctor option: --not-an-option"); +}); From 55c9ce26d1b52f359a21512103088048f9aa7492 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:01:36 +0800 Subject: [PATCH 075/791] test(cli): verify self-import parity and cleanup --- .../applicationWorkflowCli.test.ts | 172 +++++++++++++++++- tests/acceptance/webRuntimeCli.test.ts | 49 +++-- 2 files changed, 202 insertions(+), 19 deletions(-) diff --git a/tests/acceptance/applications/applicationWorkflowCli.test.ts b/tests/acceptance/applications/applicationWorkflowCli.test.ts index 56bee2d95f6..265a8473798 100644 --- a/tests/acceptance/applications/applicationWorkflowCli.test.ts +++ b/tests/acceptance/applications/applicationWorkflowCli.test.ts @@ -1,8 +1,10 @@ import { execFile } from "node:child_process"; import { copyFile, mkdir, rm, writeFile } from "node:fs/promises"; -import { join, relative } from "node:path"; +import { join, relative, resolve } from "node:path"; import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { afterEach, describe, expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -13,7 +15,11 @@ import { SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, } from "../../../src/contracts/javascript/javascriptApplicationWorkflowExamples.js"; import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js"; -import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { + javascriptApplicationAnalysisResultSchema, + type JavaScriptApplicationAnalysisResult, +} from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { z } from "zod"; const execute = promisify(execFile); const temporary: string[] = []; @@ -26,6 +32,144 @@ afterEach(async () => { ); }); +const writeSelfImportFixture = async (root: string): Promise => { + await Promise.all([ + writeFile( + join(root, "esm.js"), + 'import "./esm.js"; export const value = 1;\n', + ), + writeFile( + join(root, "common.cjs"), + 'const self = require("./common.cjs"); module.exports = self;\n', + ), + writeFile( + join(root, "util.ts"), + 'import "./util.ts"; export const value = 1;\n', + ), + ]); +}; + +const expectSelfImportEvidence = ( + graph: JavaScriptApplicationAnalysisResult["graph"], +): void => { + const rawSelfImports = graph.edges.filter( + ({ relation, properties, evidence }) => + relation === "imports" && + (properties.kind === "static-import" || properties.kind === "require") && + evidence.location.available && + evidence.location.value.kind === "source-range" && + ((properties.specifier === "./esm.js" && + properties.kind === "static-import" && + evidence.location.value.source === "esm.js") || + (properties.specifier === "./common.cjs" && + properties.kind === "require" && + evidence.location.value.source === "common.cjs") || + (properties.specifier === "./util.ts" && + properties.kind === "static-import" && + evidence.location.value.source === "util.ts")), + ); + expect(rawSelfImports).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + kind: "static-import", + specifier: "./esm.js", + }), + evidence: expect.objectContaining({ + authority: "static-relationship-inference", + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + kind: "source-range", + source: "esm.js", + start: expect.objectContaining({ line: 1, column: 0 }), + }), + }), + }), + }), + expect.objectContaining({ + properties: expect.objectContaining({ + kind: "require", + specifier: "./common.cjs", + }), + evidence: expect.objectContaining({ + authority: "static-relationship-inference", + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + kind: "source-range", + source: "common.cjs", + start: expect.objectContaining({ line: 1, column: 13 }), + }), + }), + }), + }), + expect.objectContaining({ + properties: expect.objectContaining({ + kind: "static-import", + specifier: "./util.ts", + }), + evidence: expect.objectContaining({ + authority: "static-relationship-inference", + location: expect.objectContaining({ + available: true, + value: expect.objectContaining({ + kind: "source-range", + source: "util.ts", + start: expect.objectContaining({ line: 1, column: 0 }), + }), + }), + }), + }), + ]), + ); + expect(rawSelfImports).toHaveLength(3); +}; + +const analyzeThroughStdioMcp = async ( + inputPath: string, +): Promise => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { + PATH: process.env.PATH ?? "", + REA_LOG_LEVEL: "silent", + }, + stderr: "pipe", + }); + const client = new Client({ + name: "javascript-self-import-parity", + version: "1", + }); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: inputPath }, + }); + expect(response.isError).not.toBe(true); + expect(response.content).toEqual( + expect.arrayContaining([expect.objectContaining({ type: "text" })]), + ); + const result = z + .object({ result: javascriptApplicationAnalysisResultSchema }) + .parse(response.structuredContent).result; + expect(result).toMatchObject({ + input_path: inputPath, + format: "directory", + }); + return result; + } finally { + try { + await client.close(); + } finally { + await transport.close(); + } + } +}; + describe("JavaScript application path CLI", () => { it("accepts a relative local application path and preserves canonical Evidence identity", async () => { const root = await createTestTempDirectory("rea-relative-application-cli-"); @@ -52,6 +196,30 @@ describe("JavaScript application path CLI", () => { subject: { local_path: absolute.value.subject?.local_path }, }); }, 20_000); + + it("returns the self-import graph consistently through CLI and stdio MCP", async () => { + const root = await createTestTempDirectory( + "rea-self-import-application-cli-", + ); + temporary.push(root); + await writeSelfImportFixture(root); + + const analyzed = await runCli([ + "analyze-javascript-application", + root, + "--json", + ]); + const result = z + .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) + .parse(analyzed); + const graph = result.normalized_result.graph; + expectSelfImportEvidence(graph); + const mcpResult = await analyzeThroughStdioMcp(root); + expect(mcpResult.graph).toEqual(graph); + expect(mcpResult.semantic_graph).toEqual( + result.normalized_result.semantic_graph, + ); + }, 20_000); }); describe("application workflow CLI parity", () => { diff --git a/tests/acceptance/webRuntimeCli.test.ts b/tests/acceptance/webRuntimeCli.test.ts index 53205e23e5d..084687dc2e9 100644 --- a/tests/acceptance/webRuntimeCli.test.ts +++ b/tests/acceptance/webRuntimeCli.test.ts @@ -107,12 +107,19 @@ it("translates SIGINT after real CLI arming into awaited instrumentation cleanup } }); -it.each([ - ["observe-web-execution", "SIGINT", 130], - ["observe-web-execution", "SIGTERM", 143], - ["inspect-web-event-listeners", "SIGINT", 130], - ["inspect-web-event-listeners", "SIGTERM", 143], -] as const)( +const cancellationSignals = + process.platform === "win32" + ? (["SIGINT"] as const) + : (["SIGINT", "SIGTERM"] as const); +const webRuntimeCancellationCases = ( + ["observe-web-execution", "inspect-web-event-listeners"] as const +).flatMap((command) => + cancellationSignals.map( + (signal) => [command, signal, signal === "SIGINT" ? 130 : 143] as const, + ), +); + +it.each(webRuntimeCancellationCases)( "%s preserves %s status when cancellation interrupts CDP discovery", async (command, signal, expectedExitCode) => { const sockets = new Set(); @@ -127,20 +134,24 @@ it.each([ if (address === null || typeof address === "string") throw new Error("Expected a TCP loopback listener"); const endpoint = `http://127.0.0.1:${String(address.port)}`; - let contactedResolve!: () => void; - const contacted = new Promise((resolve) => { - contactedResolve = resolve; - }); - server.on("request", contactedResolve); + const contacted = once(server, "request"); const args = command === "observe-web-execution" - ? [command, endpoint, "page-fixture", "--observation-ms", "30000", "--json"] + ? [ + command, + endpoint, + "page-fixture", + "--observation-ms", + "30000", + "--json", + ] : [command, endpoint, "page-fixture", "body", "--json"]; const child = spawn(process.execPath, [entrypoint, ...args], { env: environment(), stdio: ["ignore", "pipe", "pipe"], }); + const closed = once(child, "close"); let stdout = ""; child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk: string) => { @@ -162,7 +173,6 @@ it.each([ } finally { if (contactTimeout !== undefined) clearTimeout(contactTimeout); } - const closed = once(child, "close"); child.kill(signal); const [exitCode, terminationSignal] = await closed; expect(terminationSignal).toBeNull(); @@ -170,13 +180,18 @@ it.each([ expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled", category: "cancelled", + details: { cleanup: "complete" }, }); } finally { clearTimeout(timeout); - if (child.exitCode === null && child.signalCode === null) - child.kill("SIGKILL"); - for (const socket of sockets) socket.destroy(); - await new Promise((resolve) => server.close(() => resolve())); + try { + if (child.exitCode === null && child.signalCode === null) + child.kill("SIGKILL"); + await closed; + } finally { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + } } }, ); From c715af80587f808de816d3add6a6f989437ab66a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:04:11 +0800 Subject: [PATCH 076/791] fix(android): report actual provider readiness --- scripts/verify/android/failure-recovery.mjs | 95 +++++ src/android/JadxConfiguration.ts | 337 ++++++++++++++++-- src/android/JadxProvider.ts | 11 +- src/application/CapabilityInventory.test.ts | 6 +- src/application/CapabilityInventory.ts | 13 +- .../android/AndroidAnalysisPort.ts | 3 + src/process/ExecFileOutput.ts | 32 ++ src/server/createServer.ts | 8 +- src/server/registerSessionStatusTool.ts | 34 +- src/server/registerSessionTools.ts | 28 +- src/server/sessionAvailabilityPolicy.ts | 3 - .../android/jadxConfiguration.test.ts | 246 ++++++++++++- tests/boundary/mcp/sessionMcp.test.ts | 45 +++ tests/fixtures/android/jadx.ts | 12 +- 14 files changed, 807 insertions(+), 66 deletions(-) diff --git a/scripts/verify/android/failure-recovery.mjs b/scripts/verify/android/failure-recovery.mjs index 8402eab597c..9b1c10be22d 100644 --- a/scripts/verify/android/failure-recovery.mjs +++ b/scripts/verify/android/failure-recovery.mjs @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { delimiter, dirname, isAbsolute, join } from "node:path"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { TOOL_CONTRACTS } from "../../../dist/contracts/toolContracts.js"; /** Real unusable Java runtimes and selectors must lead to applicable recovery. */ export async function verifyAndroidFailureRecovery({ @@ -17,6 +18,15 @@ export async function verifyAndroidFailureRecovery({ }) { const root = await mkdtemp(join(tmpdir(), "rea-android-recovery-")); try { + const withoutJar = { ...environment }; + delete withoutJar.REA_JADX_MCP_JAR; + await verifyAndroidReadiness({ + entrypoint, + environment: withoutJar, + repository, + available: false, + remediation: /REA_JADX_MCP_JAR/u, + }); let jlink; for (const candidate of isAbsolute(java) ? [join(dirname(java), "jlink")] @@ -107,8 +117,24 @@ export async function verifyAndroidFailureRecovery({ assert.match(failure.message, /full JDK/u); assert.match(failure.remediation.action, /java --list-modules/u); assert.doesNotMatch(JSON.stringify(failure), /rea doctor/u); + await verifyAndroidReadiness({ + entrypoint, + environment: scenario.env, + repository, + available: false, + remediation: new RegExp( + scenario.observation.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"), + "u", + ), + }); console.log(`PASS CLI applicable recovery: ${scenario.label}`); } + await verifyAndroidReadiness({ + entrypoint, + environment, + repository, + available: true, + }); const transport = new StdioClientTransport({ command: process.execPath, args: [entrypoint, "mcp"], @@ -187,3 +213,72 @@ export async function verifyAndroidFailureRecovery({ await rm(root, { recursive: true, force: true }); } } + +const verifyAndroidReadiness = async ({ + entrypoint, + environment, + repository, + available, + remediation, +}) => { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + cwd: repository, + env: environment, + stderr: "pipe", + }); + const client = new Client({ name: "rea-android-readiness", version: "1" }); + try { + await client.connect(transport); + const catalog = await client.listTools(); + const catalogNames = catalog.tools.map(({ name }) => name).sort(); + const status = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + assert.notEqual(status.isError, true, JSON.stringify(status)); + const availability = status.structuredContent?.result?.tool_availability; + assert.ok(Array.isArray(availability)); + assert.deepEqual( + availability.map(({ name }) => name).sort(), + TOOL_CONTRACTS.map(({ name }) => name).sort(), + "Readiness must retain one availability record for the complete canonical catalog", + ); + assert.deepEqual( + catalogNames, + TOOL_CONTRACTS.map(({ name }) => name).sort(), + "Readiness must not truncate the SDK tool catalog", + ); + const byName = new Map(availability.map((item) => [item.name, item])); + for (const name of [ + "inspect_android_package", + "search_android_classes", + "inspect_android_class", + "inspect_android_method", + "trace_android_references", + ]) { + const item = byName.get(name); + assert.ok(item, `Missing Android availability entry ${name}`); + assert.equal(item.available, available, `${name} readiness mismatch`); + if (available) { + assert.equal(item.reason, "available"); + assert.equal(item.remediation, null); + } else { + assert.equal(item.reason, "provider_missing"); + assert.match(item.remediation, remediation); + } + } + const graph = byName.get("project_android_application_graph"); + assert.ok(graph, "Missing execution-free Android graph projection"); + assert.equal(graph.available, true); + assert.equal(graph.reason, "available"); + assert.equal(graph.remediation, null); + console.log( + `PASS stdio Android readiness: ${available ? "ready" : "unavailable"}`, + ); + } finally { + await client.close(); + await transport.close(); + } +}; diff --git a/src/android/JadxConfiguration.ts b/src/android/JadxConfiguration.ts index 81cb02d634d..500571546cd 100644 --- a/src/android/JadxConfiguration.ts +++ b/src/android/JadxConfiguration.ts @@ -1,10 +1,19 @@ -import { access, realpath, stat } from "node:fs/promises"; import { constants } from "node:fs"; +import { access, realpath, stat } from "node:fs/promises"; import { isAbsolute, join } from "node:path"; import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; import type { AndroidOperation } from "../domain/android/androidAnalysis.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import type { ProviderAvailability } from "../application/AnalysisProvider.js"; +import { + execFileOutput, + execFileOutputFailure, +} from "../process/ExecFileOutput.js"; import { JADX_JAR_CONFIGURATION_REMEDIATION } from "./JadxRelease.js"; +const JAVA_MODULE_LISTING_TIMEOUT_MS = 10_000; +const JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES = 1024 * 1024; + /** Caller-supplied tools, resolved only when an Android operation is selected. */ export interface JadxConfiguration { readonly jar: string; @@ -12,51 +21,184 @@ export interface JadxConfiguration { readonly jvmArguments: readonly string[]; } -/** Admit explicit local tooling without installing or changing host configuration. */ +type ConfigurationFailureCode = + | "not_configured" + | "executable_missing" + | "runtime_missing" + | "unsupported_host" + | "unsupported_version" + | "version_unresolved" + | "open_options_invalid"; + +class JadxConfigurationFailure extends Error { + constructor( + readonly code: ConfigurationFailureCode, + message: string, + readonly diagnostics: Readonly>, + options?: ErrorOptions, + ) { + super(message, options); + } +} + +/** Inspect the same JAR and Java prerequisites that gate actual execution. */ +export const inspectJadxAvailability = async ( + environment: Readonly>, + signal?: AbortSignal, +): Promise => { + try { + const configuration = await readJadxConfiguration(environment, signal); + return { + status: "available", + code: null, + reason: null, + diagnostics: { + configured: true, + java_executable: configuration.java, + jdk_compiler: true, + jar_path: configuration.jar, + }, + }; + } catch (cause) { + if (signal?.aborted === true) throw cause; + if (cause instanceof JadxConfigurationFailure) + return { + status: "unavailable", + code: cause.code, + reason: cause.message, + diagnostics: cause.diagnostics, + }; + throw cause; + } +}; + +/** Resolve caller-selected tools and preserve the same prerequisite diagnostics. */ export const resolveJadxConfiguration = async ( environment: Readonly>, operation: AndroidOperation, + signal?: AbortSignal, ): Promise => { - const unavailable = (reason: string) => - new AnalysisCapabilityUnavailableError("jadx", operation, reason); - if (process.platform !== "linux" && process.platform !== "darwin") - throw unavailable( - `Android JADX subprocess ownership is unsupported on ${process.platform}; use Linux or macOS. The metadata bridge has real-provider verification on macOS arm64.`, + try { + return await readJadxConfiguration(environment, signal); + } catch (cause) { + if (signal?.aborted === true) throw cause; + if (!(cause instanceof JadxConfigurationFailure)) throw cause; + const output = cause.diagnostics; + throw new AnalysisCapabilityUnavailableError( + "jadx", + operation, + cause.message, + { + cause, + userMessage: cause.message, + ...(typeof output.stdout === "string" && + typeof output.stderr === "string" + ? { + capturedOutput: { + stdout: output.stdout, + stderr: output.stderr, + truncated: output.output_truncated === true, + }, + } + : {}), + }, ); + } +}; + +const readJadxConfiguration = async ( + environment: Readonly>, + signal?: AbortSignal, +): Promise => { + requireSupportedHost(); + const jar = await resolveJar(environment); + const java = resolveJava(environment); + await requireJavaHomeExecutable(environment.JAVA_HOME, java); + const jvmArguments = resolveJvmArguments(environment); + await inspectJavaRuntime(java, environment, signal); + return { jar, java, jvmArguments }; +}; + +const requireSupportedHost = (): void => { + if (process.platform === "linux" || process.platform === "darwin") return; + throw configurationFailure( + "unsupported_host", + `Android JADX subprocess ownership is unsupported on ${process.platform}; use Linux or macOS. The metadata bridge has real-provider verification on macOS arm64.`, + { platform: process.platform }, + ); +}; + +const resolveJar = async ( + environment: Readonly>, +): Promise => { const jar = environment.REA_JADX_MCP_JAR; - if (environment.JAVA_HOME !== undefined && !isAbsolute(environment.JAVA_HOME)) - throw unavailable( - "JAVA_HOME must select an existing JDK by absolute path; omit it to use java on PATH.", - ); if (jar === undefined || !isAbsolute(jar)) - throw unavailable(JADX_JAR_CONFIGURATION_REMEDIATION); - const java = - environment.JAVA_HOME === undefined - ? "java" - : join(environment.JAVA_HOME, "bin", "java"); + throw configurationFailure( + "not_configured", + JADX_JAR_CONFIGURATION_REMEDIATION, + { jar_configured: false }, + ); try { await access(jar, constants.R_OK); if (!(await stat(jar)).isFile()) - throw unavailable(`REA_JADX_MCP_JAR is not a regular file: ${jar}`); + throw configurationFailure( + "executable_missing", + `REA_JADX_MCP_JAR is not a regular file: ${jar}`, + { jar_path: jar }, + ); + return await realpath(jar); } catch (cause) { - if (cause instanceof AnalysisCapabilityUnavailableError) throw cause; - throw unavailable( + if (cause instanceof JadxConfigurationFailure) throw cause; + throw configurationFailure( + "executable_missing", `Cannot read REA_JADX_MCP_JAR ${jar}: ${cause instanceof Error ? cause.message : String(cause)}`, + { jar_path: jar }, + cause, ); } - if (environment.JAVA_HOME !== undefined) { - try { - await access(java, constants.R_OK | constants.X_OK); - if (!(await stat(java)).isFile()) - throw unavailable(`JAVA_HOME java is not a regular file: ${java}`); - } catch (cause) { - if (cause instanceof AnalysisCapabilityUnavailableError) throw cause; - throw unavailable( - `Cannot execute JAVA_HOME java ${java}: ${cause instanceof Error ? cause.message : String(cause)}`, - ); - } +}; + +const resolveJava = ( + environment: Readonly>, +): string => { + if (environment.JAVA_HOME === undefined) return "java"; + if (isAbsolute(environment.JAVA_HOME)) + return join(environment.JAVA_HOME, "bin", "java"); + throw configurationFailure( + "open_options_invalid", + "JAVA_HOME must select an existing JDK by absolute path; omit it to use java on PATH.", + { java_home: environment.JAVA_HOME }, + ); +}; + +const requireJavaHomeExecutable = async ( + javaHome: string | undefined, + java: string, +): Promise => { + if (javaHome === undefined) return; + try { + await access(java, constants.R_OK | constants.X_OK); + if ((await stat(java)).isFile()) return; + throw configurationFailure( + "runtime_missing", + `JAVA_HOME java is not a regular file: ${java}`, + { java_executable: java }, + ); + } catch (cause) { + if (cause instanceof JadxConfigurationFailure) throw cause; + throw configurationFailure( + "runtime_missing", + `Cannot execute JAVA_HOME java ${java}: ${cause instanceof Error ? cause.message : String(cause)}`, + { java_executable: java }, + cause, + ); } - const jvmArguments: string[] = []; +}; + +const resolveJvmArguments = ( + environment: Readonly>, +): readonly string[] => { + const result: string[] = []; for (const [name, prefix, suffix] of [ ["REA_JADX_HEAP_MIB", "-Xmx", "m"], ["REA_JADX_ACTIVE_PROCESSOR_COUNT", "-XX:ActiveProcessorCount=", ""], @@ -64,17 +206,142 @@ export const resolveJadxConfiguration = async ( const value = environment[name]; if (value === undefined) continue; if (!/^[1-9]\d*$/u.test(value) || !Number.isSafeInteger(Number(value))) - throw unavailable( + throw configurationFailure( + "open_options_invalid", `${name} must be a positive safe integer; received ${value}.`, + { setting: name, value }, ); if ( name === "REA_JADX_ACTIVE_PROCESSOR_COUNT" && Number(value) > 2_147_483_647 ) - throw unavailable( + throw configurationFailure( + "open_options_invalid", `${name} exceeds the JVM's signed 32-bit processor-count range.`, + { setting: name, value }, ); - jvmArguments.push(`${prefix}${value}${suffix}`); + result.push(`${prefix}${value}${suffix}`); } - return { jar: await realpath(jar), java, jvmArguments }; + return result; }; + +const inspectJavaRuntime = async ( + java: string, + environment: Readonly>, + signal?: AbortSignal, +): Promise => { + let stdout: string; + let stderr: string; + try { + ({ stdout, stderr } = await execFileOutput(java, ["--list-modules"], { + env: { ...environment }, + timeout: JAVA_MODULE_LISTING_TIMEOUT_MS, + maxBuffer: JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES, + ...(signal === undefined ? {} : { signal }), + })); + } catch (cause) { + if (signal?.aborted === true) throw cause; + const output = execFileOutputFailure(cause); + if (output === undefined) throw cause; + const diagnostics = { + java_executable: java, + phase: "module-inspection", + exit_code: typeof output.code === "number" ? output.code : null, + process_error_code: + typeof output.code === "string" ? output.code : null, + signal: output.signal, + killed: output.killed, + stdout: output.stdout, + stderr: output.stderr, + output_truncated: output.outputTruncated, + } satisfies Readonly>; + if (output.outputTruncated) + throw configurationFailure( + "version_unresolved", + `The selected Java launcher ${java} exceeded the ${JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES / (1024 * 1024)} MiB output budget while listing modules. REA retained partial output but could not establish compiler or version availability. Inspect the selected launcher for verbose or unbounded output; ` + + "java --list-modules should return a concise module listing.", + { + ...diagnostics, + failure: "output_limit", + output_limit_bytes: JAVA_MODULE_LISTING_MAX_OUTPUT_BYTES, + }, + cause, + ); + if (output.killed) + throw configurationFailure( + "version_unresolved", + `The selected Java launcher ${java} was terminated or did not finish the module-listing probe within ${JAVA_MODULE_LISTING_TIMEOUT_MS / 1000} seconds. Inspect the launcher and its startup responsiveness, then verify it returns compiler and version modules with java --list-modules.`, + { + ...diagnostics, + failure: "probe_terminated", + deadline_ms: JAVA_MODULE_LISTING_TIMEOUT_MS, + }, + cause, + ); + throw configurationFailure( + "runtime_missing", + `Cannot inspect selected Java runtime ${java}: ${output.stderr.trim() || errorText(cause)}. Select a full JDK including jdk.compiler via JAVA_HOME or PATH; verify it with java --list-modules.`, + diagnostics, + cause, + ); + } + const compiler = /^jdk\.compiler(?:@|$)/mu.test(stdout); + if (!compiler) + throw configurationFailure( + "runtime_missing", + `Selected Java runtime ${java} does not provide jdk.compiler. Select a full JDK including jdk.compiler via JAVA_HOME or PATH; verify it with java --list-modules. A JRE cannot compile REA's source bridge.`, + { + java_executable: java, + phase: "module-inspection", + jdk_compiler: false, + stdout, + stderr, + output_truncated: false, + }, + ); + const major = /^java\.base@(\d+)(?:[.+_-]|$)/mu.exec(stdout)?.[1]; + if (major === undefined) + throw configurationFailure( + "version_unresolved", + `The selected Java launcher ${java} did not report a parseable java.base version in its module listing. Inspect the launcher output and verify it with java --list-modules.`, + { + java_executable: java, + phase: "module-inspection", + jdk_compiler: true, + java_major: null, + stdout, + stderr, + output_truncated: false, + }, + ); + if (Number(major) < 17) + throw configurationFailure( + "unsupported_version", + `Selected Java runtime ${java} must be JDK 17 or newer and include jdk.compiler; verify it with java --list-modules.`, + { + java_executable: java, + phase: "module-inspection", + jdk_compiler: true, + java_major: major ?? null, + stdout, + stderr, + output_truncated: false, + }, + ); +}; + +const configurationFailure = ( + code: ConfigurationFailureCode, + message: string, + diagnostics: Readonly>, + cause?: unknown, +): JadxConfigurationFailure => + new JadxConfigurationFailure( + code, + message, + diagnostics, + cause === undefined ? undefined : { cause }, + ); + +const errorText = (cause: unknown): string => + cause instanceof Error ? cause.message : String(cause); diff --git a/src/android/JadxProvider.ts b/src/android/JadxProvider.ts index cde877313bc..1fca1aa936a 100644 --- a/src/android/JadxProvider.ts +++ b/src/android/JadxProvider.ts @@ -23,7 +23,10 @@ import { hashAndroidFile, snapshotAndroidTarget, } from "./AndroidTargetSnapshot.js"; -import { resolveJadxConfiguration } from "./JadxConfiguration.js"; +import { + inspectJadxAvailability, + resolveJadxConfiguration, +} from "./JadxConfiguration.js"; import type { JadxLauncher } from "./JadxMcpTransport.js"; import { JadxSession } from "./JadxSession.js"; @@ -175,6 +178,11 @@ export class JadxProvider implements AndroidAnalysisPort { readonly launcher?: JadxLauncher, ) {} + /** Report actual local prerequisites without starting JADX or opening a target. */ + inspectAvailability(signal?: AbortSignal) { + return inspectJadxAvailability(this.environment, signal); + } + /** Serialize expensive work and bind both raw and normalized results to the APK. */ async execute( target: BinaryTarget, @@ -267,6 +275,7 @@ export class JadxProvider implements AndroidAnalysisPort { const configuration = await resolveJadxConfiguration( this.environment, request.operation, + signal, ); signal.throwIfAborted(); const jarHash = await hashAndroidFile(configuration.jar); diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 8ddf4f7244e..58d7d4e2cb2 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -40,6 +40,10 @@ const enabledPolicy: Parameters[1] = { v8InspectorObservationEnabled: true, }; +const availableAndroidAvailability: NonNullable< + Parameters[1]["androidAnalysisAvailability"] +> = { status: "available", code: null, reason: null, diagnostics: {} }; + const status = ( options: { readonly open?: boolean; @@ -110,7 +114,7 @@ describe("capability inventory: provider status", () => { expect( entry("inspect_android_package", status(), { processCaptureEnabled: true, - androidAnalysisEnabled: true, + androidAnalysisAvailability: availableAndroidAvailability, }), ).toMatchObject({ available: true, diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index b600de075b8..4f85c9aeb39 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -14,6 +14,7 @@ import { } from "../contracts/toolOutputSchemaPrimitives.js"; import type { ToolKind } from "../contracts/toolContractTypes.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { ProviderAvailability } from "./AnalysisProvider.js"; import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; import { clientRequirementsFor, @@ -38,7 +39,7 @@ export type AvailabilityPolicy = { readonly recordedCrashEnabled?: boolean; readonly firmwareInspectionEnabled?: boolean; readonly firmwareExtractionEnabled?: boolean; - readonly androidAnalysisEnabled?: boolean; + readonly androidAnalysisAvailability?: ProviderAvailability; readonly javascriptRecoveryEnabled?: boolean; readonly webModuleResolutionEnabled?: boolean; readonly browserObservationEnabled?: boolean; @@ -321,12 +322,16 @@ const workflowAvailabilityFor = ({ }; } if (kind === "android-provider") - return policy.androidAnalysisEnabled === true + return policy.androidAnalysisAvailability?.status === "available" ? { reason: "available", remediation: null } : { - reason: "provider_missing", + reason: + policy.androidAnalysisAvailability?.code === "unsupported_host" + ? "unsupported_host" + : "provider_missing", remediation: - "Set REA_JADX_MCP_JAR to a caller-supplied jadx-headless-mcp 0.7.1 JAR and provide Java on Linux or macOS. Only Linux has real-provider verification.", + policy.androidAnalysisAvailability?.reason ?? + "Set REA_JADX_MCP_JAR to a caller-supplied jadx-headless-mcp 0.7.1 JAR and provide a full JDK on Linux or macOS.", }; const browser = browserProviderAvailability(name, kind, policy); if (browser !== null) return browser; diff --git a/src/application/android/AndroidAnalysisPort.ts b/src/application/android/AndroidAnalysisPort.ts index 66ce27abfe4..d58461849b6 100644 --- a/src/application/android/AndroidAnalysisPort.ts +++ b/src/application/android/AndroidAnalysisPort.ts @@ -6,9 +6,12 @@ import type { AndroidRequest } from "../../domain/android/androidAnalysis.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { Result } from "../../domain/result.js"; +import type { ProviderAvailability } from "../AnalysisProvider.js"; /** Static Android inspection boundary; implementation details belong to the provider. */ export interface AndroidAnalysisPort { + /** Inspect the selected JAR and Java runtime without opening an APK. */ + inspectAvailability(signal?: AbortSignal): Promise; /** Cancel active work and join cleanup of any retained provider resources. */ close(): Promise; execute( diff --git a/src/process/ExecFileOutput.ts b/src/process/ExecFileOutput.ts index 232a93a7c12..4db994101a2 100644 --- a/src/process/ExecFileOutput.ts +++ b/src/process/ExecFileOutput.ts @@ -8,6 +8,38 @@ type ExecFileOutputOptions = Omit< "encoding" | "maxBuffer" > & { readonly maxBuffer?: number }; +/** Captured subprocess output read from an execFileOutput rejection. */ +export interface ExecFileOutputFailure { + readonly stdout: string; + readonly stderr: string; + readonly code: string | number | null; + readonly signal: string | null; + readonly killed: boolean; + readonly outputTruncated: boolean; +} + +/** Read captured output metadata from a failed execFileOutput invocation. */ +export const execFileOutputFailure = ( + cause: unknown, +): ExecFileOutputFailure | undefined => { + if (!(cause instanceof Error)) return undefined; + const stdout = Reflect.get(cause, "stdout"); + const stderr = Reflect.get(cause, "stderr"); + const code = Reflect.get(cause, "code"); + const signal = Reflect.get(cause, "signal"); + const killed = Reflect.get(cause, "killed"); + if (typeof stdout !== "string" || typeof stderr !== "string") + return undefined; + return { + stdout, + stderr, + code: typeof code === "string" || typeof code === "number" ? code : null, + signal: typeof signal === "string" ? signal : null, + killed: killed === true, + outputTruncated: code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", + }; +}; + /** Run a shell-free command while capturing its complete UTF-8 output. */ export const execFileOutput = ( command: string, diff --git a/src/server/createServer.ts b/src/server/createServer.ts index c105d7a971a..86783185538 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -122,10 +122,6 @@ const installSessionToolAvailability = ( options.firmwareAnalysis !== undefined || (process.platform === "linux" && isAbsolute(process.env.REA_UNBLOB_COMMAND ?? "")), - androidAnalysisEnabled: - options.androidAnalysis !== undefined || - ((process.platform === "linux" || process.platform === "darwin") && - isAbsolute(process.env.REA_JADX_MCP_JAR ?? "")), browserObservationEnabled: options.browserObservation !== undefined, javascriptRecoveryEnabled: options.javascriptRecovery !== undefined || @@ -176,6 +172,7 @@ export const createServer = ( const startedAt = new Date().toISOString(); const logger = options.logger ?? silentLogger; const server = createMcpServer(session); + const android = options.androidAnalysis ?? createAndroidAnalysisProvider(); const availability = installSessionToolAvailability(server, session, options); const toolLogger = logger.child({ layer: "server" }); const { @@ -196,7 +193,6 @@ export const createServer = ( recordEvidenceWithUnknown, }; registerBinaryAnalysisTools(toolContext); - const android = options.androidAnalysis ?? createAndroidAnalysisProvider(); const previousOnclose = server.server.onclose; server.server.onclose = () => { previousOnclose?.(); @@ -285,6 +281,8 @@ export const createServer = ( ...(availability === undefined ? {} : { availabilityPolicy: availability.policy }), + androidAnalysisAvailability: (signal) => + android.inspectAvailability(signal), startedAt, }); } diff --git a/src/server/registerSessionStatusTool.ts b/src/server/registerSessionStatusTool.ts index a7ce3cf5504..e1843675149 100644 --- a/src/server/registerSessionStatusTool.ts +++ b/src/server/registerSessionStatusTool.ts @@ -1,6 +1,8 @@ import type { McpServer } from "@modelcontextprotocol/server"; import type { BinarySessionPort } from "../application/binary/BinarySession.js"; +import type { ProviderAvailability } from "../application/AnalysisProvider.js"; +import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { buildCapabilityInventory } from "../application/CapabilityInventory.js"; import { toolContract } from "../contracts/toolContracts.js"; import type { ClientFeatureAvailability } from "../contracts/toolOutputSchemaPrimitives.js"; @@ -10,6 +12,7 @@ import { mcpClientMetadata } from "./mcpClientMetadata.js"; import type { SessionAvailability } from "./sessionAvailabilityPolicy.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { toCallToolResult } from "./toolResult.js"; +import { err } from "../domain/result.js"; type ToolAvailability = ReturnType[number]; @@ -20,24 +23,49 @@ export interface SessionStatusToolOptions { readonly contract: ReturnType>; readonly startedAt: string; readonly availabilityPolicy: () => SessionAvailability; + readonly androidAnalysisAvailability: ( + signal: AbortSignal, + ) => Promise; } /** Register the read-only provider and target status operation. */ export const registerSessionStatusTool = ( options: SessionStatusToolOptions, ): void => { - const { server, session, contract, startedAt, availabilityPolicy } = options; + const { + server, + session, + contract, + startedAt, + availabilityPolicy, + androidAnalysisAvailability, + } = options; server.registerTool( contract.name, toolRegistrationOptions(contract), - (input, context) => { + async (input, context) => { const { client, clientFeatures, protocolVersion } = mcpClientMetadata(context); + let androidAvailability: ProviderAvailability; + try { + androidAvailability = await androidAnalysisAvailability( + context.mcpReq.signal, + ); + } catch (cause) { + if (!context.mcpReq.signal.aborted) throw cause; + return toCallToolResult( + err(new AnalysisCancelledError("binary_session")), + contract, + ); + } const status = session.status(); const statusObject = jsonObjectSchema.parse(status); const toolAvailability = buildCapabilityInventory( status, - availabilityPolicy(), + { + ...availabilityPolicy(), + androidAnalysisAvailability: androidAvailability, + }, clientFeatures, ); const serverIdentity = createServerIdentity({ diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index fab581dab6f..c93536d28e0 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -17,6 +17,7 @@ import type { Evidence } from "../domain/evidence.js"; import type { ProcessCapture } from "../domain/process/processCapture.js"; import { ok, type Result } from "../domain/result.js"; import type { Logger } from "../logger.js"; +import type { ProviderAvailability } from "../application/AnalysisProvider.js"; import { mcpProgressReporter } from "./mcpProgress.js"; import { registerArtifactComparisonTool } from "./registerArtifactComparisonTool.js"; import { registerBundleComparisonTool } from "./registerBundleComparisonTool.js"; @@ -136,13 +137,22 @@ export interface LifecycleToolRegistration { readonly statusContract: ReturnType>; readonly startedAt: string; readonly availabilityPolicy: () => SessionAvailability; + readonly androidAnalysisAvailability: ( + signal: AbortSignal, + ) => Promise; } const registerLifecycleTools = ( registration: LifecycleToolRegistration, ): void => { - const { server, session, statusContract, startedAt, availabilityPolicy } = - registration; + const { + server, + session, + statusContract, + startedAt, + availabilityPolicy, + androidAnalysisAvailability, + } = registration; registerOpenLifecycleTool(registration); registerCloseLifecycleTool(registration); registerSessionStatusTool({ @@ -151,6 +161,7 @@ const registerLifecycleTools = ( contract: statusContract, startedAt, availabilityPolicy, + androidAnalysisAvailability, }); }; @@ -206,6 +217,9 @@ const registerOpenLifecycleTool = ({ export interface SessionToolOptions { readonly startedAt?: string; readonly availabilityPolicy?: () => SessionAvailability; + readonly androidAnalysisAvailability?: ( + signal: AbortSignal, + ) => Promise; } const registerContextTools = ( @@ -265,6 +279,8 @@ export const registerSessionTools = ( options.availabilityPolicy, {}, ), + androidAnalysisAvailability: + options.androidAnalysisAvailability ?? missingAndroidAvailability, }); registerEvidenceTools({ server, @@ -287,3 +303,11 @@ export const registerSessionTools = ( registerUnknownTools({ server, session }); registerContextTools(server, session); }; + +const missingAndroidAvailability = async (): Promise => ({ + status: "unavailable", + code: "not_configured", + reason: + "Set REA_JADX_MCP_JAR to a caller-supplied jadx-headless-mcp 0.7.1 JAR and provide a full JDK on Linux or macOS.", + diagnostics: { configured: false }, +}); diff --git a/src/server/sessionAvailabilityPolicy.ts b/src/server/sessionAvailabilityPolicy.ts index 829102d6489..66259f4a8a7 100644 --- a/src/server/sessionAvailabilityPolicy.ts +++ b/src/server/sessionAvailabilityPolicy.ts @@ -16,7 +16,6 @@ export interface SessionAvailabilityDefaults { | "electronObservationEnabled" | "electronAutomationEnabled" | "v8InspectorObservationEnabled" - | "androidAnalysisEnabled" | "javascriptRecoveryEnabled" | "webModuleResolutionEnabled" | "binaryLayoutEnabled" @@ -45,8 +44,6 @@ export const sessionAvailabilityPolicy = ( defaults.optionalFeatures?.firmwareInspectionEnabled ?? false, firmwareExtractionEnabled: defaults.optionalFeatures?.firmwareExtractionEnabled ?? false, - androidAnalysisEnabled: - defaults.optionalFeatures?.androidAnalysisEnabled ?? false, javascriptRecoveryEnabled: defaults.optionalFeatures?.javascriptRecoveryEnabled ?? false, webModuleResolutionEnabled: diff --git a/tests/boundary/android/jadxConfiguration.test.ts b/tests/boundary/android/jadxConfiguration.test.ts index 127919fa2dc..e6194ca2491 100644 --- a/tests/boundary/android/jadxConfiguration.test.ts +++ b/tests/boundary/android/jadxConfiguration.test.ts @@ -1,9 +1,13 @@ -import { chmod, mkdir, realpath, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readFile, realpath, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect, it as test } from "vitest"; -import { resolveJadxConfiguration } from "../../../src/android/JadxConfiguration.js"; +import { + inspectJadxAvailability, + resolveJadxConfiguration, +} from "../../../src/android/JadxConfiguration.js"; +import { AnalysisCapabilityUnavailableError } from "../../../src/domain/analysisErrorCore.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; const it = test.skipIf(process.platform === "win32"); @@ -25,31 +29,33 @@ it("rejects an explicit JAVA_HOME without an executable java binary", async () = }); }); -it("accepts a verified explicit java binary and canonicalizes the JAR", async () => { +it("accepts a full JDK and canonicalizes the selected JAR", async () => { const root = await createTestTempDirectory("rea-jadx-config-"); const jar = join(root, "jadx-headless-mcp.jar"); - const javaHome = join(root, "jdk"); - const java = join(javaHome, "bin", "java"); - await mkdir(join(javaHome, "bin"), { recursive: true }); + const javaHome = await createFakeJdk(root); await writeFile(jar, "fixture jar"); - await writeFile(java, "#!/bin/sh\n"); - await chmod(java, 0o700); await expect( resolveJadxConfiguration( { REA_JADX_MCP_JAR: jar, JAVA_HOME: javaHome }, "inspect_android_package", ), - ).resolves.toEqual({ jar: await realpath(jar), java, jvmArguments: [] }); + ).resolves.toEqual({ + jar: await realpath(jar), + java: join(javaHome, "bin", "java"), + jvmArguments: [], + }); }); it("configures heap and visible processors independently without forcing either", async () => { const root = await createTestTempDirectory("rea-jadx-config-"); const jar = join(root, "engine.jar"); + const javaHome = await createFakeJdk(root); await writeFile(jar, "fixture jar"); const configuration = await resolveJadxConfiguration( { REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, REA_JADX_HEAP_MIB: "8192", REA_JADX_ACTIVE_PROCESSOR_COUNT: "4", }, @@ -61,7 +67,11 @@ it("configures heap and visible processors independently without forcing either" ]); await expect( resolveJadxConfiguration( - { REA_JADX_MCP_JAR: jar, REA_JADX_ACTIVE_PROCESSOR_COUNT: "2147483648" }, + { + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + REA_JADX_ACTIVE_PROCESSOR_COUNT: "2147483648", + }, "inspect_android_package", ), ).rejects.toMatchObject({ @@ -72,7 +82,11 @@ it("configures heap and visible processors independently without forcing either" for (const value of ["", "0", "-1", "1.5", "8g", " 4", "9007199254740992"]) await expect( resolveJadxConfiguration( - { REA_JADX_MCP_JAR: jar, [name]: value }, + { + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + [name]: value, + }, "inspect_android_package", ), ).rejects.toMatchObject({ @@ -80,3 +94,213 @@ it("configures heap and visible processors independently without forcing either" reason: expect.stringContaining(name), }); }); + +it("distinguishes a runnable JRE from a missing Java executable", async () => { + const root = await createTestTempDirectory("rea-jadx-runtime-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeFile(jar, "fixture jar"); + const jreHome = await createFakeJdk(root, false); + await expect( + resolveJadxConfiguration( + { REA_JADX_MCP_JAR: jar, JAVA_HOME: jreHome }, + "inspect_android_package", + ), + ).rejects.toMatchObject({ + _tag: "AnalysisCapabilityUnavailableError", + reason: expect.stringContaining("does not provide jdk.compiler"), + userMessage: expect.stringContaining("A JRE cannot compile"), + capturedOutput: { + stdout: "java.base@21\n", + stderr: "", + truncated: false, + }, + }); + + const emptyPath = join(root, "empty-path"); + await mkdir(emptyPath); + const availability = await inspectJadxAvailability({ + REA_JADX_MCP_JAR: jar, + PATH: emptyPath, + }); + expect(availability).toMatchObject({ + status: "unavailable", + code: "runtime_missing", + reason: expect.stringContaining("spawn java ENOENT"), + diagnostics: { + java_executable: "java", + phase: "module-inspection", + exit_code: null, + process_error_code: "ENOENT", + }, + }); +}); + +it("preserves exact Java probe output and marks max-buffer capture as truncated", async () => { + const root = await createTestTempDirectory("rea-jadx-output-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + await writeFile(jar, "fixture jar"); + await writeFile( + java, + `#!/bin/sh\nexec ${process.execPath} -e 'process.stdout.write("x".repeat(2 * 1024 * 1024))'\n`, + ); + await chmod(java, 0o700); + + const failure = await resolveJadxConfiguration( + { REA_JADX_MCP_JAR: jar, JAVA_HOME: javaHome }, + "inspect_android_package", + ).then( + () => undefined, + (cause: unknown) => cause, + ); + expect(failure).toBeInstanceOf(AnalysisCapabilityUnavailableError); + if (!(failure instanceof AnalysisCapabilityUnavailableError)) return; + expect(failure.reason).toContain("exceeded the 1 MiB output budget"); + expect(failure.reason).not.toContain("Select a full JDK"); + expect(failure.capturedOutput).toMatchObject({ + stdout: expect.stringMatching(/^x/u), + stderr: "", + truncated: true, + }); + expect(failure.capturedOutput?.stdout.length).toBeLessThan(2 * 1024 * 1024); + await expect( + inspectJadxAvailability({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }), + ).resolves.toMatchObject({ + status: "unavailable", + code: "version_unresolved", + diagnostics: { + exit_code: null, + process_error_code: "ERR_CHILD_PROCESS_STDIO_MAXBUFFER", + failure: "output_limit", + output_truncated: true, + }, + }); +}); + +it("distinguishes an unreported Java version from an unsupported old version", async () => { + const root = await createTestTempDirectory("rea-jadx-version-listing-"); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeFile(jar, "fixture jar"); + + const unresolvedHome = await createFakeJdk(root); + await writeFile( + join(unresolvedHome, "bin", "java"), + "#!/bin/sh\nprintf 'jdk.compiler@21\\n'\n", + ); + await chmod(join(unresolvedHome, "bin", "java"), 0o700); + await expect( + inspectJadxAvailability({ REA_JADX_MCP_JAR: jar, JAVA_HOME: unresolvedHome }), + ).resolves.toMatchObject({ + status: "unavailable", + code: "version_unresolved", + reason: expect.stringContaining("did not report a parseable java.base version"), + diagnostics: { stdout: "jdk.compiler@21\n", java_major: null }, + }); + + const oldHome = await createFakeJdk(root); + await writeFile( + join(oldHome, "bin", "java"), + "#!/bin/sh\nprintf 'java.base@16\\njdk.compiler@16\\n'\n", + ); + await chmod(join(oldHome, "bin", "java"), 0o700); + await expect( + resolveJadxConfiguration( + { REA_JADX_MCP_JAR: jar, JAVA_HOME: oldHome }, + "inspect_android_package", + ), + ).rejects.toMatchObject({ + _tag: "AnalysisCapabilityUnavailableError", + reason: expect.stringContaining("must be JDK 17 or newer"), + }); +}); + +it("cancels an in-flight Java readiness probe", async () => { + const root = await createTestTempDirectory("rea-jadx-cancel-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + const marker = join(root, "java-probe-started"); + await mkdir(bin, { recursive: true }); + await writeFile(jar, "fixture jar"); + await writeFile( + java, + `#!/bin/sh\nexec ${process.execPath} -e 'require("node:fs").writeFileSync(process.argv[1], "ready"); setTimeout(() => {}, 30_000)' '${marker}'\n`, + ); + await chmod(java, 0o700); + const controller = new AbortController(); + const pending = inspectJadxAvailability( + { REA_JADX_MCP_JAR: jar, JAVA_HOME: javaHome }, + controller.signal, + ); + try { + await expect + .poll(() => + readFile(marker, "utf8").then( + () => true, + () => false, + ), + ) + .toBe(true); + controller.abort(); + await expect(pending).rejects.toMatchObject({ name: "AbortError" }); + } finally { + controller.abort(); + await pending.catch(() => undefined); + } +}); + +it("reports a killed Java probe as unresolved instead of missing runtime", async () => { + const root = await createTestTempDirectory("rea-jadx-timeout-"); + const jar = join(root, "jadx-headless-mcp.jar"); + const javaHome = join(root, "fake-jdk"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + const marker = join(root, "java-probe-started"); + await mkdir(bin, { recursive: true }); + await writeFile(jar, "fixture jar"); + await writeFile( + java, + `#!/bin/sh\nexec ${process.execPath} -e 'require("node:fs").writeFileSync(process.argv[1], "ready"); setTimeout(() => {}, 30_000)' '${marker}'\n`, + ); + await chmod(java, 0o700); + + const availability = await inspectJadxAvailability({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: javaHome, + }); + expect(availability).toMatchObject({ + status: "unavailable", + code: "version_unresolved", + reason: expect.stringContaining("within 10 seconds"), + diagnostics: { + java_executable: java, + failure: "probe_terminated", + killed: true, + deadline_ms: 10_000, + }, + }); + expect(availability.reason).not.toContain("Select a full JDK"); +}); + +const createFakeJdk = async ( + root: string, + compiler = true, +): Promise => { + const javaHome = join(root, compiler ? "fake-jdk" : "fake-jre"); + const bin = join(javaHome, "bin"); + const java = join(bin, "java"); + await mkdir(bin, { recursive: true }); + await writeFile( + java, + `#!/bin/sh\nprintf 'java.base@21${compiler ? "\\njdk.compiler@21" : ""}\\n'\n`, + ); + await chmod(java, 0o700); + return javaHome; +}; diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 1d40bec8439..90fadfcf6c4 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -15,7 +15,9 @@ import type { AnalysisClient, AnalysisProvider, CapabilityDescriptor, + ProviderAvailability, } from "../../../src/application/AnalysisProvider.js"; +import type { AndroidAnalysisPort } from "../../../src/application/android/AndroidAnalysisPort.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createServer } from "../../../src/server/createServer.js"; import { silentLogger } from "../../../src/logger.js"; @@ -149,6 +151,49 @@ describe("MCP snapshot lifecycle ordering", () => { }); describe("target-free MCP lifecycle", () => { + it("takes the session snapshot after Android readiness inspection completes", async () => { + directory = await createTestTempDirectory("rea-mcp-status-readiness-"); + const targetPath = join(directory, "target.hop"); + await writeFile(targetPath, "fixture target"); + const entered = createDeferred(); + const availability = createDeferred(); + const closed: string[] = []; + const session = createTestBinarySession(provider(closed)); + const androidAnalysis: AndroidAnalysisPort = { + inspectAvailability: () => { + entered.resolve(); + return availability.promise; + }, + close: () => Promise.resolve(), + execute: () => Promise.resolve(ok(null)), + }; + const server = createServer(session, session, { + androidAnalysis, + logger: silentLogger, + }); + const mcp = new Client({ name: "status-readiness", version: "1.0.0" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(mcp, server); + await server.connect(serverTransport); + await mcp.connect(clientTransport); + + const status = mcp.callTool({ name: "binary_session", arguments: {} }); + await entered.promise; + expect((await session.open(targetPath)).ok).toBe(true); + availability.resolve({ + status: "available", + code: null, + reason: null, + diagnostics: { configured: true }, + }); + + expect(structured(await status).result).toMatchObject({ + open: true, + path: targetPath, + }); + }); + it("reopens replaced content at one canonical path through MCP", async () => { directory = await createTestTempDirectory("rea-mcp-replaced-target-"); const targetPath = join(directory, "mutable.hop"); diff --git a/tests/fixtures/android/jadx.ts b/tests/fixtures/android/jadx.ts index a9633e1de22..a537ed4769e 100644 --- a/tests/fixtures/android/jadx.ts +++ b/tests/fixtures/android/jadx.ts @@ -1,4 +1,4 @@ -import { access, rm, writeFile } from "node:fs/promises"; +import { access, chmod, mkdir, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, onTestFinished } from "vitest"; @@ -20,6 +20,15 @@ export const createJadxProtocolFixture = async (mode = "normal") => { const root = await createTestTempDirectory("rea-android-boundary-"); const apk = join(root, "fixture.apk"); const jar = join(root, "fixture.jar"); + const javaHome = join(root, "jdk"); + const javaBin = join(javaHome, "bin"); + const java = join(javaBin, "java"); + await mkdir(javaBin, { recursive: true }); + await writeFile( + java, + `#!/usr/bin/env node\nif (process.argv[2] === "--list-modules") console.log("java.base@21.0.0\\njdk.compiler@21.0.0");\n`, + ); + await chmod(java, 0o755); await writeOrderedZip(apk, ["AndroidManifest.xml", "classes.dex"]); await writeFile(jar, "synthetic engine bytes; not a Java archive"); const launches: { @@ -32,6 +41,7 @@ export const createJadxProtocolFixture = async (mode = "normal") => { }[] = []; const environment: NodeJS.ProcessEnv = { ...process.env, + JAVA_HOME: javaHome, REA_JADX_MCP_JAR: jar, }; const provider = createAndroidAnalysisProvider( From 80c3d36dfa2597f34fdeb69e0ba75277ee50c85c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:27:50 +0800 Subject: [PATCH 077/791] fix(ghidra): preserve POSIX runtime paths with spaces --- scripts/lib/native-value-e2e.mjs | 2 +- src/ghidra/GhidraAnalysisProfile.ts | 6 +-- src/ghidra/GhidraJavaLaunch.ts | 3 +- src/ghidra/GhidraLauncher.ts | 39 +++++++++---------- src/ghidra/GhidraLauncherEnvironment.test.ts | 5 +-- .../providers/ghidra/ghidraLauncher.test.ts | 26 ++++++------- 6 files changed, 38 insertions(+), 43 deletions(-) diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index 10e2269088a..1249ed32731 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -80,7 +80,7 @@ export async function verifyNativeValueE2e( const workspace = await mkdtemp(join(tmpdir(), "rea-native-values-e2e-")); const runtime = join( workspace, - `runtime${process.platform === "darwin" ? " with spaces" : ""}-${"x".repeat(100)}`, + `runtime${process.platform === "win32" ? "" : " with spaces"}-${"x".repeat(100)}`, ); await mkdir(runtime); const env = { ...environment(), TMPDIR: runtime }; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 1c2ec4d7f35..418136f7488 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -45,9 +45,9 @@ export const resolveGhidraAnalysisProfile = ( function_references: "complete-body-and-entry-reference-manager-v2", location_resolution: "explicit-address-exact-name-first-v2", process_launch: - installation.platform === "darwin" - ? "inspected-jvm-launch-support-v1" - : "official-headless-script-v1", + installation.platform === "win32" + ? "official-headless-script-v1" + : "inspected-jvm-launch-support-v1", ...(dos ? { load_image_evidence: dosCom diff --git a/src/ghidra/GhidraJavaLaunch.ts b/src/ghidra/GhidraJavaLaunch.ts index 6f50ddf899a..869574b8ac6 100644 --- a/src/ghidra/GhidraJavaLaunch.ts +++ b/src/ghidra/GhidraJavaLaunch.ts @@ -3,6 +3,7 @@ import { execFileOutput } from "../process/ExecFileOutput.js"; /** Launch coordinates obtained from Ghidra's own release configuration helper. */ export const ghidraJavaLaunch = async (options: { + readonly platform: NodeJS.Platform; readonly analyzeHeadlessPath: string; readonly javaHome: string; readonly homeRoot: string; @@ -71,7 +72,7 @@ export const ghidraJavaLaunch = async (options: { environment, arguments: [ ...vmArguments, - "-Xdock:name=Ghidra-Headless", + ...(options.platform === "darwin" ? ["-Xdock:name=Ghidra-Headless"] : []), `-Xmx${environment.GHIDRA_HEADLESS_MAXMEM || environment.GHIDRA_MAXMEM || "2G"}`, "-XX:ParallelGCThreads=2", "-XX:CICompilerCount=2", diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index f030c855913..de833d0078b 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -139,13 +139,14 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { platform, scriptCommand.command, ); - if (platform === "darwin" && this.options.javaHome === undefined) + if (platform !== "win32" && this.options.javaHome === undefined) throw new GhidraLaunchError( - "macOS Ghidra requires its inspected JDK home", + "POSIX Ghidra requires its inspected JDK home", ); const command = - platform === "darwin" && this.options.javaHome !== undefined + platform !== "win32" && this.options.javaHome !== undefined ? await ghidraJavaLaunch({ + platform, analyzeHeadlessPath: this.options.analyzeHeadlessPath, javaHome: this.options.javaHome, homeRoot: paths.homeRoot, @@ -161,9 +162,9 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { command: command.command, arguments: command.arguments, runId: session.runId, - // macOS launches the inspected JVM directly: platform shell wrappers - // with withheld environments cannot establish run-token ownership. - expectedCommand: platform === "darwin" ? command.command : null, + // POSIX launches the inspected JVM directly; script wrappers cannot + // preserve JVM property paths containing spaces as single arguments. + expectedCommand: platform !== "win32" ? command.command : null, windowsVerbatimArguments: platform === "win32", platform, env: command.environment, @@ -441,7 +442,7 @@ const ghidraLaunchEnvironment = ( }; }; -/** Encode provider-owned JVM paths for the parser that actually consumes them. */ +/** Encode provider-owned JVM paths for the platform launcher that consumes them. */ export const ghidraHeadlessJavaOptions = ( homeRoot: string, tempRoot: string, @@ -451,20 +452,16 @@ export const ghidraHeadlessJavaOptions = ( "JDK_JAVA_OPTIONS" | "GHIDRA_HEADLESS_JAVA_OPTIONS" > => { const options = [`-Duser.home=${homeRoot}`, `-Djava.io.tmpdir=${tempRoot}`]; - // analyzeHeadless.bat quotes VMARG_LIST, then launch.bat expands it again. - // Inner quotes fail in cmd.exe; unquoted paths with spaces split at the JVM. - // JDK_JAVA_OPTIONS is parsed by Java itself, outside that batch expansion. - return platform === "win32" - ? { - JDK_JAVA_OPTIONS: [...options, "-XX:-UsePerfData"] - .map(quoteWindowsBatchToken) - .join(" "), - GHIDRA_HEADLESS_JAVA_OPTIONS: "", - } - : { - JDK_JAVA_OPTIONS: "", - GHIDRA_HEADLESS_JAVA_OPTIONS: options.join(" "), - }; + if (platform === "win32") + return { + JDK_JAVA_OPTIONS: [...options, "-XX:-UsePerfData"] + .map(quoteWindowsBatchToken) + .join(" "), + GHIDRA_HEADLESS_JAVA_OPTIONS: "", + }; + // POSIX launches the JVM directly, so these properties travel as separate + // argv items instead of the shell-split analyzeHeadless option list. + return { JDK_JAVA_OPTIONS: "", GHIDRA_HEADLESS_JAVA_OPTIONS: "" }; }; const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true; diff --git a/src/ghidra/GhidraLauncherEnvironment.test.ts b/src/ghidra/GhidraLauncherEnvironment.test.ts index a66037515da..6319d44ae1b 100644 --- a/src/ghidra/GhidraLauncherEnvironment.test.ts +++ b/src/ghidra/GhidraLauncherEnvironment.test.ts @@ -18,14 +18,13 @@ describe("Ghidra headless JVM environment", () => { }); it.each(["linux", "darwin"] as const)( - "keeps the %s headless-script option list and clears inherited JDK options", + "passes %s isolated JVM paths through direct argv instead of script options", (platform) => { expect( ghidraHeadlessJavaOptions("/tmp/rea/home", "/tmp/rea/tmp", platform), ).toEqual({ JDK_JAVA_OPTIONS: "", - GHIDRA_HEADLESS_JAVA_OPTIONS: - "-Duser.home=/tmp/rea/home -Djava.io.tmpdir=/tmp/rea/tmp", + GHIDRA_HEADLESS_JAVA_OPTIONS: "", }); }, ); diff --git a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts index 01b84aee6af..66672324a77 100644 --- a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts +++ b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts @@ -73,10 +73,8 @@ const expectIsolatedEnvironment = ( JAVA_HOME: javaHome, REA_PROCESS_RUN_ID: "d6fcbb66-e829-4ff6-a535-0035aec63139", }); - expect(environment.PATH).toMatch( - process.platform === "win32" - ? /^C:\\Java\\jdk-21\\bin;/u - : /^\/opt\/jdk-21\/bin:/u, + expect(environment.PATH).toBe( + `${join(javaHome, "bin")}${process.platform === "win32" ? ";" : ":"}${process.env.PATH ?? ""}`, ); }; @@ -200,7 +198,9 @@ describe("Ghidra headless launcher", () => { ).toThrow(/metacharacters/u); }); - it("keeps authority in a private descriptor and isolates Ghidra state", async () => { + it.skipIf(process.platform !== "win32")( + "keeps authority in a private descriptor and isolates Ghidra state through the batch launcher", + async () => { vi.stubEnv("GHIDRA_JAVA_OPTIONS", "-javaagent:/unapproved/agent.jar"); vi.stubEnv("JAVA_TOOL_OPTIONS", "-Duser.home=/unapproved/home"); vi.stubEnv("JDK_JAVA_OPTIONS", "-XX:MaxRAMPercentage=99"); @@ -210,13 +210,12 @@ describe("Ghidra headless launcher", () => { runtimes.push(runtime); const runtimeRoot = runtime.path; const token = "secret-token-that-must-not-leak"; - const javaHome = - process.platform === "win32" ? "C:\\Java\\jdk-21" : "/opt/jdk-21"; + const javaHome = "C:\\Java\\jdk-21"; const launcher = new GhidraHeadlessLauncher({ analyzeHeadlessPath: fixturePath, - // This fixture exercises the official script route. Native macOS JVM - // launching is covered by the real Ghidra acceptance lane. - platform: process.platform === "win32" ? "win32" : "linux", + // POSIX follows the real inspected-JVM route; Windows uses the official + // batch script because its launcher contract differs. + platform: "win32", javaHome, bridgeScriptPath: "/package/bridge/ReaGhidraBridge.java", }); @@ -247,9 +246,7 @@ describe("Ghidra headless launcher", () => { }); expectIsolatedEnvironment(capture.environment, runtimeRoot, javaHome); expect(capture.environment.GHIDRA_HEADLESS_JAVA_OPTIONS).toBe( - process.platform === "win32" - ? "" - : `-Duser.home=${join(runtimeRoot, "home")} -Djava.io.tmpdir=${join(runtimeRoot, "tmp")}`, + "", ); if (process.platform !== "win32") expect( @@ -259,5 +256,6 @@ describe("Ghidra headless launcher", () => { const cleaned = await launched.value.cleanup?.(); expect(cleaned).toMatchObject({ cleaned: true }); await expect(access(join(runtimeRoot, "project"))).resolves.toBeUndefined(); - }); + }, + ); }); From 67756dacc0d435a61d2d0c403412151769da028f Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:45:49 +0800 Subject: [PATCH 078/791] fix(ci): align diagnostic regressions and format merged changes --- docs/testing.md | 2 +- src/android/JadxConfiguration.ts | 3 +- tests/acceptance/nonBinaryComposition.test.ts | 64 +++++----- .../android/jadxConfiguration.test.ts | 9 +- tests/boundary/cli/mcpDoctorHelp.test.ts | 18 ++- .../providers/ghidra/ghidraLauncher.test.ts | 110 +++++++++--------- .../contracts/analysisErrorProjection.test.ts | 12 +- 7 files changed, 123 insertions(+), 95 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f684673e8a6..7adcf3123c8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -307,7 +307,7 @@ prove behavior on the verified Ghidra 12.1.4 and JDK 21 build. | Ghidra lane | Supported runner/target | Additional local tools | | ------------------------------------------ | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------- | -| `npm run verify:ghidra` | Linux x64/arm64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 | +| `npm run verify:ghidra` | Linux x64/arm64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:switch` | Linux x64 ELF; GCC/Clang optimized and stripped switch fixtures | GCC, Clang, GNU nm/objdump/strip, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:aarch64-jump-table` | Any supported Ghidra host; AArch64 ELF; byte/halfword tables; host ARM64 Mach-O | Clang with AArch64 target support, Ghidra 12.1.4, and full JDK 21 | | `npm run verify:ghidra:cross-format` | Any supported Ghidra host; also analyzes AArch64 ELF, x86-64 PE, and x86-64 Mach-O | `clang`, LLD, and `lld-link` in addition to host-lane prerequisites | diff --git a/src/android/JadxConfiguration.ts b/src/android/JadxConfiguration.ts index 500571546cd..f423bf2280a 100644 --- a/src/android/JadxConfiguration.ts +++ b/src/android/JadxConfiguration.ts @@ -247,8 +247,7 @@ const inspectJavaRuntime = async ( java_executable: java, phase: "module-inspection", exit_code: typeof output.code === "number" ? output.code : null, - process_error_code: - typeof output.code === "string" ? output.code : null, + process_error_code: typeof output.code === "string" ? output.code : null, signal: output.signal, killed: output.killed, stdout: output.stdout, diff --git a/tests/acceptance/nonBinaryComposition.test.ts b/tests/acceptance/nonBinaryComposition.test.ts index f7c28fa4109..0cb337de7f6 100644 --- a/tests/acceptance/nonBinaryComposition.test.ts +++ b/tests/acceptance/nonBinaryComposition.test.ts @@ -34,33 +34,43 @@ const connect = async (environment: NodeJS.ProcessEnv) => { return client; }; -cliTest.skipIf(process.platform === "win32")( - "preserves the selected APK configuration through the compiled CLI and MCP factories", - async ({ cli }) => { - const root = await createTestTempDirectory("rea-android-composition-"); - const apk = join(root, "fixture.apk"); - await writeOrderedZip(apk, ["AndroidManifest.xml", "classes.dex"]); - const environment = { - REA_LOG_LEVEL: "silent", - JAVA_HOME: "relative-fixture-jdk", - }; - const cliResult = await cli.run({ - arguments: ["inspect-android-package", apk, "--json"], - environment, - timeoutMs: 10_000, - }); - expect(cliResult.exitCode).toBe(1); - expect(cliResult.json).toMatchObject({ code: "capability_unavailable" }); - const client = await connect(environment); - const response = await client.callTool({ - name: "inspect_android_package", - arguments: { path: apk }, - }); - expect(response.isError).toBe(true); - expect(response.structuredContent).toEqual({ error: cliResult.json }); - expect(JSON.stringify(response.structuredContent)).toContain("JAVA_HOME"); - }, -); +for (const { jarConfigured, setting } of [ + { jarConfigured: false, setting: "REA_JADX_MCP_JAR" }, + { jarConfigured: true, setting: "JAVA_HOME" }, +]) + cliTest.skipIf(process.platform === "win32")( + `preserves the APK ${setting} failure through the compiled CLI and MCP factories`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-android-composition-"); + const apk = join(root, "fixture.apk"); + await writeOrderedZip(apk, ["AndroidManifest.xml", "classes.dex"]); + const jar = join(root, "jadx-headless-mcp.jar"); + await writeFile( + jar, + "fixture jar; validation must fail before execution", + ); + const environment = { + REA_LOG_LEVEL: "silent", + JAVA_HOME: "relative-fixture-jdk", + ...(jarConfigured ? { REA_JADX_MCP_JAR: jar } : {}), + }; + const cliResult = await cli.run({ + arguments: ["inspect-android-package", apk, "--json"], + environment, + timeoutMs: 10_000, + }); + expect(cliResult.exitCode).toBe(1); + expect(cliResult.json).toMatchObject({ code: "capability_unavailable" }); + const client = await connect(environment); + const response = await client.callTool({ + name: "inspect_android_package", + arguments: { path: apk }, + }); + expect(response.isError).toBe(true); + expect(response.structuredContent).toEqual({ error: cliResult.json }); + expect(JSON.stringify(response.structuredContent)).toContain(setting); + }, + ); cliTest.skipIf(process.platform !== "linux")( "preserves the missing firmware command reason through both caller paths", diff --git a/tests/boundary/android/jadxConfiguration.test.ts b/tests/boundary/android/jadxConfiguration.test.ts index e6194ca2491..0f6a8654a78 100644 --- a/tests/boundary/android/jadxConfiguration.test.ts +++ b/tests/boundary/android/jadxConfiguration.test.ts @@ -195,11 +195,16 @@ it("distinguishes an unreported Java version from an unsupported old version", a ); await chmod(join(unresolvedHome, "bin", "java"), 0o700); await expect( - inspectJadxAvailability({ REA_JADX_MCP_JAR: jar, JAVA_HOME: unresolvedHome }), + inspectJadxAvailability({ + REA_JADX_MCP_JAR: jar, + JAVA_HOME: unresolvedHome, + }), ).resolves.toMatchObject({ status: "unavailable", code: "version_unresolved", - reason: expect.stringContaining("did not report a parseable java.base version"), + reason: expect.stringContaining( + "did not report a parseable java.base version", + ), diagnostics: { stdout: "jdk.compiler@21\n", java_major: null }, }); diff --git a/tests/boundary/cli/mcpDoctorHelp.test.ts b/tests/boundary/cli/mcpDoctorHelp.test.ts index 8f91efeca7b..e619d819d2e 100644 --- a/tests/boundary/cli/mcpDoctorHelp.test.ts +++ b/tests/boundary/cli/mcpDoctorHelp.test.ts @@ -7,11 +7,15 @@ const execute = promisify(execFile); const runCli = async (arguments_: readonly string[]) => { try { - const result = await execute(process.execPath, ["scripts/rea.mjs", ...arguments_], { - cwd: process.cwd(), - env: { ...process.env, REA_LOG_LEVEL: "silent" }, - timeout: 15_000, - }); + const result = await execute( + process.execPath, + ["scripts/rea.mjs", ...arguments_], + { + cwd: process.cwd(), + env: { ...process.env, REA_LOG_LEVEL: "silent" }, + timeout: 15_000, + }, + ); return { exitCode: 0, ...result }; } catch (cause: unknown) { if ( @@ -46,5 +50,7 @@ it("shows MCP doctor help without starting a server and keeps option errors acti const invalid = await runCli(["mcp", "doctor", "--not-an-option"]); expect(invalid.exitCode).toBe(1); - expect(invalid.stdout).toContain("Unknown mcp doctor option: --not-an-option"); + expect(invalid.stdout).toContain( + "Unknown mcp doctor option: --not-an-option", + ); }); diff --git a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts index 66672324a77..1e009c46042 100644 --- a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts +++ b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts @@ -201,61 +201,63 @@ describe("Ghidra headless launcher", () => { it.skipIf(process.platform !== "win32")( "keeps authority in a private descriptor and isolates Ghidra state through the batch launcher", async () => { - vi.stubEnv("GHIDRA_JAVA_OPTIONS", "-javaagent:/unapproved/agent.jar"); - vi.stubEnv("JAVA_TOOL_OPTIONS", "-Duser.home=/unapproved/home"); - vi.stubEnv("JDK_JAVA_OPTIONS", "-XX:MaxRAMPercentage=99"); - vi.stubEnv("_JAVA_OPTIONS", "-Xmx99G"); - const parent = await createTestTempDirectory("rea-launcher-test-"); - const runtime = await createGhidraTestRuntime(parent); - runtimes.push(runtime); - const runtimeRoot = runtime.path; - const token = "secret-token-that-must-not-leak"; - const javaHome = "C:\\Java\\jdk-21"; - const launcher = new GhidraHeadlessLauncher({ - analyzeHeadlessPath: fixturePath, - // POSIX follows the real inspected-JVM route; Windows uses the official - // batch script because its launcher contract differs. - platform: "win32", - javaHome, - bridgeScriptPath: "/package/bridge/ReaGhidraBridge.java", - }); - const launched = await launcher.launch({ - runtimeRoot, - transport: "unix-socket", - endpointPath: join(runtimeRoot, "bridge.sock"), - token, - runId: "d6fcbb66-e829-4ff6-a535-0035aec63139", - targetPath: "/tmp/fixture", - targetSha256: "b".repeat(64), - providerVersion: "12.1.4", - profileDigest: "a".repeat(64), - }); - if (!launched.ok) throw launched.error; - const capturePath = join(runtimeRoot, "launch-capture.json"); - await vi.waitFor(() => access(`${capturePath}.ready`), { timeout: 10_000 }); - const capture = launchCaptureSchema.parse( - JSON.parse(await readFile(capturePath, "utf8")), - ); - const encodedArguments = JSON.stringify(capture.arguments); - const encodedEnvironment = JSON.stringify(capture.environment); - expect(encodedArguments).not.toContain(token); - expect(encodedEnvironment).not.toContain(token); - expect(capture).toMatchObject({ - ...(process.platform === "win32" ? {} : { descriptor_mode: 0o600 }), - descriptor_has_token: true, - }); - expectIsolatedEnvironment(capture.environment, runtimeRoot, javaHome); - expect(capture.environment.GHIDRA_HEADLESS_JAVA_OPTIONS).toBe( - "", - ); - if (process.platform !== "win32") - expect( - (await stat(join(runtimeRoot, "ownership.json"))).mode & 0o777, - ).toBe(0o600); + vi.stubEnv("GHIDRA_JAVA_OPTIONS", "-javaagent:/unapproved/agent.jar"); + vi.stubEnv("JAVA_TOOL_OPTIONS", "-Duser.home=/unapproved/home"); + vi.stubEnv("JDK_JAVA_OPTIONS", "-XX:MaxRAMPercentage=99"); + vi.stubEnv("_JAVA_OPTIONS", "-Xmx99G"); + const parent = await createTestTempDirectory("rea-launcher-test-"); + const runtime = await createGhidraTestRuntime(parent); + runtimes.push(runtime); + const runtimeRoot = runtime.path; + const token = "secret-token-that-must-not-leak"; + const javaHome = "C:\\Java\\jdk-21"; + const launcher = new GhidraHeadlessLauncher({ + analyzeHeadlessPath: fixturePath, + // POSIX follows the real inspected-JVM route; Windows uses the official + // batch script because its launcher contract differs. + platform: "win32", + javaHome, + bridgeScriptPath: "/package/bridge/ReaGhidraBridge.java", + }); + const launched = await launcher.launch({ + runtimeRoot, + transport: "unix-socket", + endpointPath: join(runtimeRoot, "bridge.sock"), + token, + runId: "d6fcbb66-e829-4ff6-a535-0035aec63139", + targetPath: "/tmp/fixture", + targetSha256: "b".repeat(64), + providerVersion: "12.1.4", + profileDigest: "a".repeat(64), + }); + if (!launched.ok) throw launched.error; + const capturePath = join(runtimeRoot, "launch-capture.json"); + await vi.waitFor(() => access(`${capturePath}.ready`), { + timeout: 10_000, + }); + const capture = launchCaptureSchema.parse( + JSON.parse(await readFile(capturePath, "utf8")), + ); + const encodedArguments = JSON.stringify(capture.arguments); + const encodedEnvironment = JSON.stringify(capture.environment); + expect(encodedArguments).not.toContain(token); + expect(encodedEnvironment).not.toContain(token); + expect(capture).toMatchObject({ + ...(process.platform === "win32" ? {} : { descriptor_mode: 0o600 }), + descriptor_has_token: true, + }); + expectIsolatedEnvironment(capture.environment, runtimeRoot, javaHome); + expect(capture.environment.GHIDRA_HEADLESS_JAVA_OPTIONS).toBe(""); + if (process.platform !== "win32") + expect( + (await stat(join(runtimeRoot, "ownership.json"))).mode & 0o777, + ).toBe(0o600); - const cleaned = await launched.value.cleanup?.(); - expect(cleaned).toMatchObject({ cleaned: true }); - await expect(access(join(runtimeRoot, "project"))).resolves.toBeUndefined(); + const cleaned = await launched.value.cleanup?.(); + expect(cleaned).toMatchObject({ cleaned: true }); + await expect( + access(join(runtimeRoot, "project")), + ).resolves.toBeUndefined(); }, ); }); diff --git a/tests/conformance/contracts/analysisErrorProjection.test.ts b/tests/conformance/contracts/analysisErrorProjection.test.ts index 8eb58e5a31c..e98804bc0c5 100644 --- a/tests/conformance/contracts/analysisErrorProjection.test.ts +++ b/tests/conformance/contracts/analysisErrorProjection.test.ts @@ -10,7 +10,7 @@ import { projectAnalysisError } from "../../../src/domain/analysisErrorProjectio import { type AnalysisError } from "../../../src/domain/analysisErrorBase.js"; describe("analysis error projection contract", () => { - it("accepts every closed error-reason variant without exposing diagnostics", () => { + it("accepts every closed error-reason variant", () => { const variants: AnalysisError[] = [ ...( [ @@ -40,7 +40,9 @@ describe("analysis error projection contract", () => { ).map((reason) => new UnknownRegistryError(reason)), ...(["capture_failed", "cleanup_incomplete", "cancelled"] as const).map( (reason) => - new ProcessCaptureError("SECRET capture diagnostic", { reason }), + new ProcessCaptureError("API_TOKEN=selected capture diagnostic", { + reason, + }), ), ...( [ @@ -62,7 +64,11 @@ describe("analysis error projection contract", () => { JSON.stringify({ variant, projected }), ).toMatchObject({ success: true }); expect(projected.code).toMatch(/^[a-z][a-z0-9_]*$/u); - expect(JSON.stringify(projected)).not.toContain("SECRET"); + if (variant instanceof ProcessCaptureError) + expect(projected).toMatchObject({ + message: variant.message, + details: { execution_failure: variant.message }, + }); } }); }); From 96af2fa7444a3777d51ff46202ef563146929dbd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 07:47:33 +0800 Subject: [PATCH 079/791] fix(browser): preserve captures when response bodies are unavailable --- scripts/fixtures/browser-verifier-site.mjs | 12 ++ scripts/lib/browser-verifier-assertions.mjs | 13 ++ src/browser/CdpPageCapture.ts | 35 +++- .../cdpBrowserProvider.network.test.ts | 164 ++++++++++++++++++ 4 files changed, 218 insertions(+), 6 deletions(-) diff --git a/scripts/fixtures/browser-verifier-site.mjs b/scripts/fixtures/browser-verifier-site.mjs index 841e7f90092..d5ccb7923dc 100644 --- a/scripts/fixtures/browser-verifier-site.mjs +++ b/scripts/fixtures/browser-verifier-site.mjs @@ -7,6 +7,13 @@ export async function startBrowserVerifierSite() { let port = 0; let sessionGeneration = 0; const server = createServer((request, response) => { + if (request.url === "/slow-json") { + response.setHeader("content-type", "application/json"); + response.write('{"pending":'); + const timer = setTimeout(() => response.end("true}"), 350); + response.once("close", () => clearTimeout(timer)); + return; + } if (request.url === "/screenshot-noise") { response.setHeader("content-type", "text/html"); response.end(` + + + + +
+
+ +

Find out where
exported notes go.

+

+ Use REA to inspect a small app, follow its Export notes button and + check one detail yourself. +

+

+ A supplied example · Node.js and npm · Your coding agent +

+
+
+ +
+
+

1. Connect your agent to REA

+

+ Use a coding agent such as Claude Code, Codex or Cursor. You’ll + need Node.js and npm; the + setup guide lists supported versions. +

+
+
+ Your coding agent + +
+

+ Install REA and connect it to this coding agent using npx + rea-agents@latest setup. Show me the setup plan for approval, + then verify the installation. +

+
+

Or run this command in your terminal:

+
+ npx rea-agents@latest setup + +
+

+ Review the plan, approve the setup, then restart your agent. + Already connected? Continue below. +

+
+
+

2. Ask where the CSV is written

+

+ The example is a six-file Electron app with an + Export notes button. Your agent can inspect its + code without installing or launching Electron. +

+
+
+ Your coding agent + +
+

+ Download https://rea.tools/examples/notes-example.zip into a + new example folder and unpack it. Use REA to find what happens + when I click Export notes. Show where notes.csv is written, + which fields it contains, and the relevant files and + lines. +

+
+

+ Your agent downloads the files and gives the unpacked folder to + REA for inspection. +

+

+ Download example ZIP +

+

+ If your agent can’t download files, unzip this archive yourself. + Give it the path to the notes-example folder and the + same export question. +

+
+
+

3. Follow the answer back to the code

+

Look for a short explanation with file and line references:

+
+

+ The button sends a notes:export message. The main + process formats the notes as CSV and writes + notes.csv in the user’s Downloads folder. The + columns are id and title. +

+
+
+

From the button to the file writer

+
    +
  1. + Page · renderer.jsExport notesCalls notes.exportCsv(). +
  2. +
  3. + Bridge · preload.jsnotes:exportSends a named message to the main process. +
  4. +
  5. + Handler · main.jsWrite notes.csvCalls the formatter in csv.js and writes the + file. +
  6. +
+
+ REA’s static analysis pairs the message with its handler. The + destination is read from code. +
+
+
+
+ main.js · lines 8–12Source +
+
ipcMain.handle("notes:export", () => {
+  const destination = path.join(app.getPath("downloads"), "notes.csv");
+  writeFileSync(destination, toCsv(notes), "utf8");
+  return destination;
+});
+
+

+ The file path, formatter and write call are together in this + handler. Ask your agent to show these lines if its answer leaves + them out. +

+

+ See the full code trace and recorded REA result. +

+
+
+

4. Check a prediction

+

+ A comma separates CSV columns. What happens when a note title + itself contains a comma or quotation marks? +

+
+
+ Your coding agent + +
+

+ Predict the CSV for a note with id 1 and title Hello, "REA". + Then run the example’s toCsv formatter with Node.js to check + it. Show the prediction and actual output. +

+
+
+
+ Expected formatter output +
+
id,title
+1,"Hello, ""REA"""
+
+

+ The title stays in one quoted field; each quotation mark is + doubled. This checks the formatter directly with Node.js. +

+
+ Run the check yourself +

+ Open a terminal in the unpacked + notes-example folder: +

+
+
+ Node.js · CSV formatter +
+
node <<'JS'
+const { toCsv } = require('./csv.js');
+process.stdout.write(toCsv([{ id: 1, title: 'Hello, "REA"' }]));
+JS
+
+

+ This command uses a shell with here-document support, such as + Bash or Zsh. Your agent can run the same JavaScript in your + platform’s shell. +

+
+
+
+

5. Ask another question

+

Try a follow-up in the same conversation:

+
+

+ What happens if the file can’t be written? Follow the error back + to the page. +

+
+

+ Or give your agent an app of your own and ask about one button, + file or behavior. +

+ +
+
+
+
+ +

+ Top + + diff --git a/website/public/index.html b/website/public/index.html index e315e63ff28..c04b4c6b25e 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -50,55 +50,249 @@
-
-

Software analysis · CLI & MCP

+
+

Reverse engineering, with your coding agent

Find out how
software works.

- Use REA with your coding agent to inspect binaries, trace application - code and observe runtime behavior. + REA gives your agent the tools to inspect a program and explain what + it does.

-
-
-
- Your coding agent +
+ +
+
+
+

What is reverse engineering?

+

+ Finding out how software works by examining the program. +

+
+
+

+ DX-Ball: how does the game place a sound on the left or right? +

+
+
+
+ Original instructions · selected excerpt +
+
MOV EAX, dword ptr [EBP + 0x8]
+MOV dword ptr [EBP + -0xc], EAX
+FILD dword ptr [EBP + -0xc]
+FMUL double ptr [0x00420068]
+FSUB double ptr [0x00420070]
+FMUL double ptr [0x004210a0]
+CALL 0x0041678c
+
+
+
Readable summary · C
+
// x is the horizontal position
+double pan = (double)x;
+pan *= 1.5625;
+pan -= 500.0;
+pan *= pan_scale;
+return (int)pan;
+
+
+
+
+ Left side + + x = 0 → pan = −500 + Sound to the left +
+
+ Center + + x = 320 → pan = 0 + Centered sound +
+
+ Right side + + x = 640 → pan = +500 + Sound to the right +
+
+
+ Examples of the calculation with the recorded initial scale of 1. + The code is a readable summary of the inspected function. +
+
+

+ To explain this, an analyst identifies the input, follows the caller + and reads the values at those addresses. +

+
+
+ +
+
+
+
+

Do I need to read assembly to get started?

+

No.

+

Ask your agent about the behavior you want to understand.

+
+
+
+
+ Example analysis · DX-Ball + +
+

Target · DXBALL.EXE

+

+ Use REA to inspect this DX-Ball executable. How does it + calculate left/right sound panning? Show the relevant + code. +

+
+

+ Agent · explanation from the findings +

+

+ This function converts a brick’s horizontal position into a + left/right sound value. +

+

+ At the initial scale of 1, the screen center produces zero. + Positions to the left produce negative values; positions to + the right produce positive values. +

+
+
+

+ Example prompt and explanation based on the recorded + investigation. Supply the executable to your agent. +

+
+
+
+

What REA found for the agent

+
    +
  1. + 01 · Inspect the function + An integer input + The instructions read [EBP + 0x8]. The initial + pseudocode had missed this input. +
  2. +
  3. + 02 · Follow the caller + A brick’s screen position + The caller passes 20 + 30 × tile_x, giving the + input its meaning. +
  4. +
  5. + 03 · Read the stored data + The calculation’s constants + Byte reads supply 1.5625, 500.0 and + the initial scale 1.0. +
  6. +
+
+ The agent uses these findings to explain the behavior and write + readable C. +
+
+

+ The reconstruction project checked its maintained C function against + 3,205 original-x86 input cases and all 63 compiled bytes. + See the code and checks. +

+
+
+ +
+
+
+

Your first investigation

+

Try REA on a small app.

+

+ Find out where an app exports its notes, then check how it formats + a title containing commas and quotes. +

+

We provide the example files, a prompt and a result to check.

+ +
+
+

Set up REA with your agent

+
+
+ Your coding agent + +
+

+ Install REA and connect it to this coding agent using npx + rea-agents@latest setup. Show me the setup plan for approval, + then verify the installation. +

+
+

Or run this command in your terminal:

+
+ npx rea-agents@latest setup
-

- Install REA and connect it to this coding agent using npx - rea-agents@latest setup. Show me the setup plan for approval, - then verify the installation. +

+ Setup shows a plan for approval. Restart your agent after setup.

-

Or run this command in your terminal:

-
- npx rea-agents@latest setup - -
@@ -139,34 +333,6 @@

Browser & runtime activity

-
-
-
-
- Ask a question about a local app, binary or browser. Your agent uses REA through CLI or MCP to inspect and trace with analysis tools. REA returns code, references and unknowns. Your agent uses the evidence to explain, implement and test, and can ask follow-up questions. -
-
- REA connects your agent to analysis tools. The findings guide the - next question, explanation or implementation. - Open figure -
-
-
-
-
@@ -177,12 +343,6 @@

Rebuilding DX-Ball

in C, using REA to inspect the executable and independent tests to check the recovered behavior.

-

- To recover a sound-pan calculation, the agent used REA to inspect - the instructions, follow a caller and read the constants. Those - findings led to a C function checked against the original behavior - and all 63 compiled bytes. -

Follow the investigation Date: Thu, 8 Oct 2026 19:01:52 -0700 Subject: [PATCH 096/791] fix(artifacts): recheck failed DMG detaches before reporting cleanup failure (#1155) * fix(artifacts): recheck failed DMG detaches before reporting cleanup failure NativeDmgArtifactReader.close() detached each device once and, when a detach failed, asked `hdiutil info` once whether the device was still attached. On APFS images, detaching the synthesized container ejects the image disk too. `hdiutil info` can still list that ejected device briefly, and a just-read volume can report busy. Either way a real cleanup succeeded but close() reported "DMG detach or mount-root cleanup failed" and kept the mount root. macOS CI hit this intermittently in verify:macos-bundle: `hdiutil detach /dev/disk23` failed with "No such file or directory" (already ejected) and the device was still counted as attached. The #1103 review also raised the missing recheck. After the first pass, recheck the remaining devices up to three times with 250/500/1000 ms settle delays. Drop any that hdiutil no longer lists, retry the detach of any still listed, and fail only for devices that stay attached. The host seam gains an optional delay so tests run without real waits. Co-Authored-By: Claude Opus 5.5 * test(artifacts): share DMG detach fixtures to stay under the duplication gate The new busy-device and recheck cases copied the fake hdiutil host and the APFS close sequence from neighbouring tests, which pushed jscpd to its 2.5% threshold in Static checks. Share one single-device host between the busy and retained-attachment cases and one APFS close helper between the ejected-device cases. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- src/artifacts/NativeDmgArtifactReader.ts | 59 ++++-- .../native/nativeDmgArtifactReader.test.ts | 186 ++++++++++++------ 2 files changed, 175 insertions(+), 70 deletions(-) diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts index cc88a95deb9..f3fcce3936f 100644 --- a/src/artifacts/NativeDmgArtifactReader.ts +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -15,6 +15,12 @@ import { import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; const DETACH_TIMEOUT_MS = 120_000; +/** + * Waits before rechecking devices whose detach failed. A related device's + * later detach can eject one, `hdiutil info` can still list an ejected device + * briefly, and a volume can report busy just after it was read. + */ +const DETACH_SETTLE_DELAYS_MS = [250, 500, 1000] as const; const attachOutputSchema = z.object({ "system-entities": z.array( z.object({ @@ -43,9 +49,13 @@ export interface NativeDmgHost { readonly exitCode: number; readonly cause?: unknown; }>; + /** Wait between detach rechecks; tests may resolve immediately. */ + delay?(milliseconds: number): Promise; } const systemHost: NativeDmgHost = { + delay: (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)), async run(arguments_, signal, options) { try { const { stdout, stderr } = await execFileOutput( @@ -128,23 +138,33 @@ export class NativeDmgArtifactReader implements ArtifactReader { async close(): Promise { let detachFailure: unknown; - const remainingDevices: string[] = []; + let remainingDevices: string[] = []; for (const device of [...this.#devices].reverse()) { - try { - await runChecked(this.host, ["detach", device], undefined, { - timeoutMs: DETACH_TIMEOUT_MS, - }); - this.#provenance.push(command(["detach", device], ["mount"])); - } catch (cause: unknown) { - // Detaching a synthesized APFS container also ejects the image that - // backs it, so a later whole disk of the same image may already be - // gone. Only a device that is still attached is a cleanup failure. - if (await this.#isAttached(device)) { - remainingDevices.push(device); - detachFailure ??= cause; + const failure = await this.#detach(device); + if (failure !== undefined) { + remainingDevices.push(device); + detachFailure ??= failure; + } + } + // Detaching a synthesized APFS container also ejects the image that backs + // it, so a device whose own detach failed may be gone by the end of the + // pass, and hdiutil can list an ejected device or report busy briefly. + // Only a device that stays attached through every recheck is a failure. + for (const delayMs of DETACH_SETTLE_DELAYS_MS) { + if (remainingDevices.length === 0) break; + await (this.host.delay ?? systemHost.delay)?.(delayMs); + const stillAttached: string[] = []; + for (const device of remainingDevices) { + if (!(await this.#isAttached(device))) continue; + const failure = await this.#detach(device); + if (failure !== undefined) { + stillAttached.push(device); + detachFailure = failure; } } + remainingDevices = stillAttached; } + if (remainingDevices.length === 0) detachFailure = undefined; this.#devices = remainingDevices.reverse(); if (this.#devices.length === 0 && this.#mountRoot !== undefined) await rm(this.#mountRoot, { recursive: true, force: true }).then( @@ -164,6 +184,19 @@ export class NativeDmgArtifactReader implements ArtifactReader { ); } + /** Detach one device; return the failure only while it remains attached. */ + async #detach(device: string): Promise { + try { + await runChecked(this.host, ["detach", device], undefined, { + timeoutMs: DETACH_TIMEOUT_MS, + }); + this.#provenance.push(command(["detach", device], ["mount"])); + return undefined; + } catch (cause: unknown) { + return (await this.#isAttached(device)) ? cause : undefined; + } + } + /** * Report whether hdiutil still lists a device; unknown state counts as * attached. Like detach, this cleanup query runs after the inventory diff --git a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts index 8b0df273056..a4196a44727 100644 --- a/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts +++ b/tests/boundary/providers/native/nativeDmgArtifactReader.test.ts @@ -83,6 +83,7 @@ describe("native DMG artifact reader", () => { await rm(mountRoot, { recursive: true, force: true }); }); const host: NativeDmgHost = { + delay: () => Promise.resolve(), run(arguments_) { if (arguments_[0] === "detach") return Promise.reject(new Error("detach failed")); @@ -396,14 +397,19 @@ it("owns the canonical mount root and detaches each observed whole image once", // hdiutil reports the image disk and synthesized APFS container separately; // detaching the container can already eject the image disk. -const apfsHost = (stillListed: readonly string[]) => { +const apfsHost = ( + stillListed: readonly string[], + listedInfoReads = Infinity, +) => { const calls: string[][] = []; + let infoReads = 0; let mountRoot: string | undefined; onTestFinished(async () => { if (mountRoot !== undefined) await rm(mountRoot, { recursive: true, force: true }); }); const host: NativeDmgHost = { + delay: () => Promise.resolve(), async run(arguments_) { calls.push([...arguments_]); if (arguments_[0] === "detach" && arguments_[1] === "/dev/disk4") @@ -412,7 +418,7 @@ const apfsHost = (stillListed: readonly string[]) => { return { stdout: build({ images: - stillListed.length === 0 + stillListed.length === 0 || (infoReads += 1) > listedInfoReads ? [] : [ { @@ -445,18 +451,91 @@ const apfsHost = (stillListed: readonly string[]) => { return { calls, host }; }; +// One mounted device whose detach reports busy while `busy(attempt)` holds. +const singleDeviceHost = (busy: (attempt: number) => boolean) => { + const state = { + attached: true, + attempts: 0, + mountRoot: undefined as string | undefined, + waits: [] as number[], + }; + onTestFinished(async () => { + if (state.mountRoot !== undefined) + await rm(state.mountRoot, { recursive: true, force: true }); + }); + const host: NativeDmgHost = { + delay: (milliseconds) => { + state.waits.push(milliseconds); + return Promise.resolve(); + }, + async run(args) { + if (args[0] === "attach") { + state.mountRoot = args[args.indexOf("-mountroot") + 1]; + if (state.mountRoot === undefined) + throw new Error("missing mount root"); + const mountPoint = join(state.mountRoot, "Fixture"); + await mkdir(mountPoint); + await writeFile(join(mountPoint, "hello.txt"), "owned observation"); + return { + stdout: build({ + "system-entities": [ + { "dev-entry": "/dev/disk7", "mount-point": mountPoint }, + ], + }), + exitCode: 0, + }; + } + if (args[0] === "info") + return { + stdout: build({ + images: state.attached + ? [{ "system-entities": [{ "dev-entry": "/dev/disk7" }] }] + : [], + }), + exitCode: 0, + }; + if (args[0] === "detach") { + state.attempts += 1; + if (busy(state.attempts)) + return { + stdout: "", + stderr: "hdiutil: detach failed - Resource busy", + exitCode: 16, + }; + state.attached = false; + } + return { stdout: "", exitCode: 0 }; + }, + }; + return { host, state }; +}; + +/** Close an APFS fixture and return the cleanup-phase hdiutil calls. */ +const closeApfsFixture = async ( + ...listing: Parameters +): Promise<{ + readonly reader: NativeDmgArtifactReader; + readonly cleanup: readonly string[][]; +}> => { + const { calls, host } = apfsHost(...listing); + const reader = await NativeDmgArtifactReader.create( + "/tmp/image.dmg", + undefined, + host, + ); + await reader.close(); + return { + reader, + cleanup: calls.filter( + (call) => call[0] !== "verify" && call[0] !== "attach", + ), + }; +}; + describe("APFS disk image detach", () => { it("accepts an image disk that its container detach already ejected", async () => { - const { calls, host } = apfsHost([]); - const reader = await NativeDmgArtifactReader.create( - "/tmp/image.dmg", - undefined, - host, - ); - await reader.close(); - expect( - calls.filter((call) => call[0] !== "verify" && call[0] !== "attach"), - ).toEqual([ + const { reader, cleanup } = await closeApfsFixture([]); + expect(cleanup).toEqual([ ["detach", "/dev/disk5"], ["detach", "/dev/disk4"], ["info", "-plist"], @@ -482,51 +561,8 @@ describe("APFS disk image detach", () => { }); it("retains a failed attachment and its mount root until a later close detaches it", async () => { - let mountRoot: string | undefined; let busy = true; - let attached = true; - onTestFinished(async () => { - if (mountRoot !== undefined) - await rm(mountRoot, { recursive: true, force: true }); - }); - const host: NativeDmgHost = { - async run(args) { - if (args[0] === "attach") { - mountRoot = args[args.indexOf("-mountroot") + 1]; - if (mountRoot === undefined) throw new Error("missing mount root"); - const mountPoint = join(mountRoot, "Fixture"); - await mkdir(mountPoint); - await writeFile(join(mountPoint, "hello.txt"), "owned observation"); - return { - stdout: build({ - "system-entities": [ - { "dev-entry": "/dev/disk7", "mount-point": mountPoint }, - ], - }), - exitCode: 0, - }; - } - if (args[0] === "info") - return { - stdout: build({ - images: attached - ? [{ "system-entities": [{ "dev-entry": "/dev/disk7" }] }] - : [], - }), - exitCode: 0, - }; - if (args[0] === "detach") { - if (busy) - return { - stdout: "", - stderr: "hdiutil: detach failed - Resource busy", - exitCode: 16, - }; - attached = false; - } - return { stdout: "", exitCode: 0 }; - }, - }; + const { host, state } = singleDeviceHost(() => busy); const reader = await NativeDmgArtifactReader.create( "/tmp/image.dmg", undefined, @@ -535,6 +571,7 @@ describe("APFS disk image detach", () => { await expect(reader.close()).rejects.toMatchObject({ reason: "unavailable", }); + const { mountRoot } = state; if (mountRoot === undefined) throw new Error("missing owned mount root"); expect .soft( @@ -545,7 +582,7 @@ describe("APFS disk image detach", () => { .toBe("owned observation"); busy = false; await reader.close(); - expect.soft(attached).toBe(false); + expect.soft(state.attached).toBe(false); await expect( readFile(join(mountRoot, "Fixture", "hello.txt")), ).rejects.toMatchObject({ @@ -553,3 +590,38 @@ describe("APFS disk image detach", () => { }); }); }); + +describe("DMG detach recheck", () => { + it("accepts an ejected device that hdiutil listed once more after its failed detach", async () => { + const { cleanup } = await closeApfsFixture( + ["/dev/disk4", "/dev/disk4s1"], + 1, + ); + expect(cleanup).toEqual([ + ["detach", "/dev/disk5"], + ["detach", "/dev/disk4"], + ["info", "-plist"], + ["info", "-plist"], + ]); + }); + + it("detaches a device that was busy on its first attempt within one close", async () => { + const { host, state } = singleDeviceHost((attempt) => attempt === 1); + const reader = await NativeDmgArtifactReader.create( + "/tmp/image.dmg", + undefined, + host, + ); + await reader.close(); + expect(state).toMatchObject({ + attempts: 2, + attached: false, + waits: [250], + }); + if (state.mountRoot === undefined) + throw new Error("missing owned mount root"); + await expect( + readFile(join(state.mountRoot, "Fixture", "hello.txt")), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); From b620e2a2bdcf3683b494d137bc409813735d5572 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:01:59 -0700 Subject: [PATCH 097/791] fix(javascript): omit self-referential static references with disclosure (#1157) * fix(javascript): omit self-referential static references with disclosure A bundled module that requires its own key (`r(50)` inside webpack module 50) resolved, through the static-reference composer, to its own module node. The resulting self-referential imports edge failed result validation and lost the entire analysis ("Result schema rejected 1 issue ... (custom)"). This is the same invariant as #1146; #1152 fixed the bundler async-chunk composer, and this is the remaining adjacent composer. Skip a reference whose resolved target is its own source, count it, and disclose the omission the way #1152 discloses async-chunk references. Sibling requires are kept. Co-Authored-By: Claude Opus 5.5 * refactor(javascript): share the self-reference omission disclosure The self-import (#1130), async-chunk (#1152) and static-reference disclosures repeated one sentence template in three composers, and the new test repeated the analyze-and-check sequence of its neighbour. That put jscpd at its 2.5% threshold. Share the template through selfReferenceOmissions with identical wording, and share one test helper for the graph-wide self-reference invariants. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../JavaScriptArtifactGraphBuilder.ts | 18 +++-- .../JavaScriptArtifactGraphBundlers.ts | 11 +-- .../JavaScriptArtifactGraphContext.ts | 15 ++++ .../JavaScriptArtifactGraphFindings.ts | 19 ++++- .../javascriptBundlerSelfReferences.test.ts | 77 ++++++++++++++----- 5 files changed, 107 insertions(+), 33 deletions(-) diff --git a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts index 78cb0f4b635..9951ee6cec4 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBuilder.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBuilder.ts @@ -7,7 +7,10 @@ import { import type { JavaScriptModuleArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; import type { JavaScriptArtifactFileSet } from "../../domain/javascript/javascriptArtifactFiles.js"; import { JavaScriptArtifactGraphAccumulator } from "./JavaScriptArtifactGraphAccumulator.js"; -import type { JavaScriptArtifactGraphContext } from "./JavaScriptArtifactGraphContext.js"; +import { + selfReferenceOmissions, + type JavaScriptArtifactGraphContext, +} from "./JavaScriptArtifactGraphContext.js"; import { addJavaScriptHtmlRoles, addJavaScriptSourceMapOriginals, @@ -90,7 +93,7 @@ const buildJavaScriptArtifactGraphInput = ( addJavaScriptSourceModules(context); const bundlerLimitations = addJavaScriptBundlerNodes(context); const relationshipOmissions = addJavaScriptModuleRelationships(context); - addJavaScriptStaticFindings(context); + const findingLimitations = addJavaScriptStaticFindings(context); addElectronBoundaries(context); addJavaScriptHtmlRoles(context); addJavaScriptSourceMapOriginals(context); @@ -106,6 +109,7 @@ const buildJavaScriptArtifactGraphInput = ( coverage, limitations: [ ...bundlerLimitations, + ...findingLimitations, ...graphLimitations(context, coverage.status, relationshipOmissions), ], }; @@ -194,11 +198,11 @@ const graphLimitations = ( ); return [ ...context.analysis.limitations, - ...(relationshipOmissions.selfImports > 0 - ? [ - `${String(relationshipOmissions.selfImports)} import ${relationshipOmissions.selfImports === 1 ? "specifier" : "specifiers"} resolved back to the importing module itself and ${relationshipOmissions.selfImports === 1 ? "was" : "were"} omitted; application graph edges cannot be self-referential.`, - ] - : []), + ...selfReferenceOmissions( + relationshipOmissions.selfImports, + ["import specifier", "import specifiers"], + "the importing module", + ), "CommonJS and ESM binding relationships were recovered from inert syntax and resolved only within the inventoried artifact container.", "Webpack/Rspack factories were recovered from AST literals; REA did not invoke push handlers or bundle bootstrap code.", "Static imports, entrypoints, workers, endpoints, and storage relationships do not prove runtime execution.", diff --git a/src/application/javascript/JavaScriptArtifactGraphBundlers.ts b/src/application/javascript/JavaScriptArtifactGraphBundlers.ts index 21ea83e1572..b7d7cfa4bff 100644 --- a/src/application/javascript/JavaScriptArtifactGraphBundlers.ts +++ b/src/application/javascript/JavaScriptArtifactGraphBundlers.ts @@ -7,6 +7,7 @@ import { chunkLookupKey, javascriptAnalysisCoverage, moduleLookupKey, + selfReferenceOmissions, type JavaScriptArtifactGraphContext, type JavaScriptArtifactGraphCoverage, } from "./JavaScriptArtifactGraphContext.js"; @@ -53,11 +54,11 @@ export const addJavaScriptBundlerNodes = ( for (const record of records) selfAsyncChunks += addBundlerRuntimeEdges(context, record); } - return selfAsyncChunks === 0 - ? [] - : [ - `${String(selfAsyncChunks)} bundler async-chunk ${selfAsyncChunks === 1 ? "reference" : "references"} resolved back to the requesting chunk itself and ${selfAsyncChunks === 1 ? "was" : "were"} omitted; application graph edges cannot be self-referential.`, - ]; + return selfReferenceOmissions( + selfAsyncChunks, + ["bundler async-chunk reference", "bundler async-chunk references"], + "the requesting chunk", + ); }; interface BundlerProjectionInput { diff --git a/src/application/javascript/JavaScriptArtifactGraphContext.ts b/src/application/javascript/JavaScriptArtifactGraphContext.ts index 0ef275eeb90..74e0e1a7971 100644 --- a/src/application/javascript/JavaScriptArtifactGraphContext.ts +++ b/src/application/javascript/JavaScriptArtifactGraphContext.ts @@ -306,6 +306,21 @@ export const linkElectronRoleToAsset = ( }); }; +/** + * Disclose relationships omitted because they resolved to their own source; + * the application graph forbids self-referential edges. + */ +export const selfReferenceOmissions = ( + count: number, + noun: readonly [singular: string, plural: string], + owner: string, +): string[] => + count === 0 + ? [] + : [ + `${String(count)} ${count === 1 ? noun[0] : noun[1]} resolved back to ${owner} itself and ${count === 1 ? "was" : "were"} omitted; application graph edges cannot be self-referential.`, + ]; + /** Deterministic key for one recovered module inside a bundle asset. */ export const moduleLookupKey = (path: string, moduleKey: string): string => `${path}\0${moduleKey}`; diff --git a/src/application/javascript/JavaScriptArtifactGraphFindings.ts b/src/application/javascript/JavaScriptArtifactGraphFindings.ts index 9158ff906f0..50ab6854c80 100644 --- a/src/application/javascript/JavaScriptArtifactGraphFindings.ts +++ b/src/application/javascript/JavaScriptArtifactGraphFindings.ts @@ -10,6 +10,7 @@ import { linkElectronRoleToAsset, moduleLookupKey, resolveArtifactPath, + selfReferenceOmissions, sourceNodeFor, type JavaScriptArtifactGraphContext, type JavaScriptArtifactGraphCoverage, @@ -42,14 +43,15 @@ interface ResolvedReference { /** Project imports, workers, roles, endpoints, storage, and source-map links. */ export const addJavaScriptStaticFindings = ( context: JavaScriptArtifactGraphContext, -): void => { +): readonly string[] => { + const omissions = { selfReferences: 0 }; for (const analyzed of context.analysis.files) { const { file, javascript } = analyzed; const asset = context.assetNodes.get(file.path); if (javascript === null || asset === undefined) continue; const coverage = javascriptAnalysisCoverage(javascript); for (const value of javascript.references) - addReference(context, { file, asset, value, coverage }); + addReference(context, { file, asset, value, coverage }, omissions); for (const value of javascript.endpoints) addEndpoint(context, { file, asset, value, coverage }); for (const value of javascript.storage) @@ -59,11 +61,17 @@ export const addJavaScriptStaticFindings = ( for (const value of javascript.source_map_urls) addSourceMapEdge(context, { file, asset, value, coverage }); } + return selfReferenceOmissions( + omissions.selfReferences, + ["static reference", "static references"], + "the referencing module", + ); }; const addReference = ( context: JavaScriptArtifactGraphContext, input: FindingInput, + omissions: { selfReferences: number }, ): void => { const source = sourceNodeFor(context, input.file.path, input.value.module_key) ?? @@ -119,6 +127,13 @@ const addReference = ( }); return; } + // A reference that resolves to its own source, such as a bundled module + // requiring its own key, cannot become an edge: the application graph + // forbids self-referential edges and one would fail the whole analysis. + if (resolved?.node.node_id === source.node_id) { + omissions.selfReferences += 1; + return; + } const target = resolved?.node ?? unresolvedReferenceNode(context, input.file, input.value, input.coverage); diff --git a/tests/boundary/cli/javascriptBundlerSelfReferences.test.ts b/tests/boundary/cli/javascriptBundlerSelfReferences.test.ts index 26555c417ab..4c8676bc9e3 100644 --- a/tests/boundary/cli/javascriptBundlerSelfReferences.test.ts +++ b/tests/boundary/cli/javascriptBundlerSelfReferences.test.ts @@ -3,13 +3,39 @@ import { expect } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; import { findChunkNode, findGraphEdge, writeFixtureFiles, } from "../../support/javascriptApplicationFixture.js"; +/** Analyze one fixture and check the graph-wide self-reference invariants. */ +const analyzeBundle = async ( + cli: TestCli, + prefix: string, + files: Readonly>, +) => { + const root = await createTestTempDirectory(prefix); + await writeFixtureFiles(root, files); + const output = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(output.exitCode).toBe(0); + const result = javascriptApplicationAnalysisResultSchema.parse( + parseEvidence(output.json).normalized_result, + ); + expect( + result.graph.edges.filter( + (edge) => edge.source_node_id === edge.target_node_id, + ), + ).toEqual([]); + expect(result.limitations).toEqual( + expect.arrayContaining(result.graph.limitations), + ); + return result.graph; +}; + for (const scenario of [ { runtime: "webpackChunkdemo", keys: [1], omitted: 1 }, { runtime: "rspackChunkdemo", keys: [1, 2, 3], omitted: 3 }, @@ -17,8 +43,7 @@ for (const scenario of [ cliTest( `preserves ${scenario.runtime} evidence when a chunk requests its own IDs`, async ({ cli }) => { - const root = await createTestTempDirectory("rea-bundler-self-"); - await writeFixtureFiles(root, { + const graph = await analyzeBundle(cli, "rea-bundler-self-", { "runtime.js": ` (self.${scenario.runtime} = self.${scenario.runtime} || []).push([ ${JSON.stringify(scenario.keys)}, @@ -32,19 +57,6 @@ for (const scenario of [ ]); `, }); - const output = await cli.run({ - arguments: ["analyze-javascript-application", root, "--json"], - }); - expect(output.exitCode).toBe(0); - const result = javascriptApplicationAnalysisResultSchema.parse( - parseEvidence(output.json).normalized_result, - ); - const { graph } = result; - expect( - graph.edges.filter( - (edge) => edge.source_node_id === edge.target_node_id, - ), - ).toEqual([]); expect(graph.limitations).toContainEqual( expect.stringMatching( new RegExp( @@ -53,9 +65,6 @@ for (const scenario of [ ), ), ); - expect(result.limitations).toEqual( - expect.arrayContaining(graph.limitations), - ); const source = findChunkNode(graph, scenario.runtime, "1"); expect(source?.observations[0]?.properties.async_chunk_keys).toEqual( expect.arrayContaining([...scenario.keys.map(String), "4", "99"]), @@ -89,3 +98,33 @@ for (const scenario of [ }, ); } + +cliTest( + "preserves bundle evidence when a module requires its own key", + async ({ cli }) => { + const graph = await analyzeBundle(cli, "rea-bundler-self-require-", { + "runtime.js": ` + (self.webpackChunkdemo = self.webpackChunkdemo || []).push([ + [5], {50(e,t,r){r(50); r(51)}, 51(e,t,r){r.r(t)}} + ]); + `, + }); + expect(graph.limitations).toContainEqual( + expect.stringMatching( + /^1 static reference resolved back to the referencing module itself and was omitted;/u, + ), + ); + expect( + graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.kind === "require" && + ["50", "51"].includes(String(properties.specifier)), + ), + ).toEqual([ + expect.objectContaining({ + properties: expect.objectContaining({ specifier: "51" }), + }), + ]); + }, +); From 24a30fd91c144b7f2533e5f5de8c91268773519c Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:12:58 +0000 Subject: [PATCH 098/791] fix(apple): report unusable dylib-resolution roots as invalid input trace_dylib_resolution blamed the artifact for the caller's root selection. A missing root, a directory, or a symlink leading outside the bundle got "Artifact contains an unsafe or conflicting internal path ... correct the artifact". A non-Mach-O file such as Contents/Info.plist got "Artifact is invalid or has changed. Get a fresh copy". Any schema failure, including a bad architecture, was reported as "roots must be normalized paths". Report a caller-selected root that cannot be traced as an invalid_request issue on roots[i] with the failed constraint, and project schema failures through their real issues. The normalized-path rule now carries a readable message. Roots REA derives itself (bundle executables or the active target) keep reporting artifact failures. This matches what #1070 did for inspect_keyed_archive. Co-Authored-By: Claude Opus 5.5 --- src/artifacts/apple/DylibResolutionReader.ts | 42 +++++++++----- src/domain/apple/dylibResolution.ts | 8 ++- tests/boundary/cli/dylibResolutionCli.test.ts | 58 +++++++++++++++---- tests/boundary/mcp/dylibResolutionMcp.test.ts | 52 ++++++++++++++++- 4 files changed, 132 insertions(+), 28 deletions(-) diff --git a/src/artifacts/apple/DylibResolutionReader.ts b/src/artifacts/apple/DylibResolutionReader.ts index 812e1471500..7b07528693d 100644 --- a/src/artifacts/apple/DylibResolutionReader.ts +++ b/src/artifacts/apple/DylibResolutionReader.ts @@ -3,6 +3,7 @@ import { constants } from "node:fs"; import { lstat, open, readlink, realpath } from "node:fs/promises"; import { join, relative, sep } from "node:path"; +import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; import { resolveTreePath } from "../../domain/apple/dyldPaths.js"; import { dylibResolutionInputSchema, @@ -13,10 +14,12 @@ import { type DylibTreeView, type MachoImageFacts, } from "../../domain/apple/dylibResolution.js"; +import { projectInputIssues } from "../../domain/inputIssueProjection.js"; import { ArtifactReaderFailure } from "../ArtifactReader.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { hasMachoMagic, readMachoImage } from "./MachoLoadCommandReader.js"; +const OPERATION = "trace_dylib_resolution"; const HASH_CHUNK_BYTES = 1024 * 1024; /** Device, inode, size and change times of the file whose bytes were parsed. */ @@ -86,9 +89,10 @@ export const traceDylibResolution = async (options: { }): Promise => { const parsed = dylibResolutionInputSchema.safeParse(options.parameters); if (!parsed.success) - throw new ArtifactReaderFailure( - "path", - "roots must be normalized paths relative to the analyzed root", + throw new AnalysisInputError( + OPERATION, + { cause: parsed.error }, + projectInputIssues(parsed.error.issues, options.parameters), ); // Canonicalization is inside the translated region: a revoked permission // keeps its reason and path. @@ -105,7 +109,11 @@ export const traceDylibResolution = async (options: { : options.enumerateRoots ? await executableRoots(root, view, options.signal) : { roots: [target], unclassified: [] }; - const roots = await requireMachoRoots(view, requested); + const roots = await requireMachoRoots( + view, + requested, + parsed.data.roots !== undefined, + ); const trace = await traceDylibLoading(view, { roots, unclassified, @@ -203,29 +211,35 @@ const compare = (left: string, right: string): number => /** * Resolve each root segment by segment, as dependency candidates are, so a * symlink in an intermediate directory cannot lead outside the analyzed root. + * A root the caller selected that fails is an input error on that root; a + * root REA derived itself keeps reporting an artifact failure. */ const requireMachoRoots = async ( view: FilesystemTreeView, roots: readonly string[], + callerSelected: boolean, ): Promise => { const resolved: string[] = []; - for (const path of roots) { + for (const [index, path] of roots.entries()) { + const reject = ( + reason: "path" | "format", + message: string, + ): AnalysisInputError | ArtifactReaderFailure => + callerSelected + ? new AnalysisInputError(OPERATION, undefined, [ + { path: ["roots", index], reason: "invalid_value", message }, + ]) + : new ArtifactReaderFailure(reason, message); const lookup = await resolveTreePath(view, path); if (lookup.kind === "escapes") - throw new ArtifactReaderFailure( - "path", - `Root ${path} resolves outside the analyzed root`, - ); + throw reject("path", `Root ${path} resolves outside the analyzed root`); if (lookup.kind !== "file") - throw new ArtifactReaderFailure( + throw reject( "path", `Root ${path} is not a regular file in the analyzed root`, ); if ((await view.image(lookup.path)).status === "not-mach-o") - throw new ArtifactReaderFailure( - "format", - `Root ${path} is not a Mach-O image`, - ); + throw reject("format", `Root ${path} is not a Mach-O image`); if (!resolved.includes(lookup.path)) resolved.push(lookup.path); } return resolved; diff --git a/src/domain/apple/dylibResolution.ts b/src/domain/apple/dylibResolution.ts index 7f268bad7ca..9fd98a309ad 100644 --- a/src/domain/apple/dylibResolution.ts +++ b/src/domain/apple/dylibResolution.ts @@ -22,7 +22,13 @@ import { /** Normalized path below the analyzed root: no `.`/`..` or empty segments. */ const ROOT_RELATIVE_PATH = /^(?!\/)(?!(?:.*\/)?\.{1,2}(?:\/|$))(?!.*\/\/)(?!.*\/$)[^\\\0]+$/u; -const rootRelativePathSchema = z.string().min(1).regex(ROOT_RELATIVE_PATH); +const rootRelativePathSchema = z + .string() + .min(1) + .regex( + ROOT_RELATIVE_PATH, + "Use a normalized path relative to the analyzed root, without a leading or trailing '/', '//', or '.' and '..' segments", + ); /** Caller selection of process roots and one CPU slice. */ export const dylibResolutionInputSchema = z.strictObject({ diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index aad4e9f0937..7a6b12a4d6a 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -15,7 +15,7 @@ import { import { dylibResolutionResultSchema } from "../../../src/domain/apple/dylibResolution.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; -import { cliTest } from "../../support/cli/cliFixture.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; const ENVIRONMENT = { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }; @@ -29,6 +29,41 @@ const writeFiles = async ( } }; +/** A caller-selected root that cannot be traced is an input issue on that root. */ +const expectRootInputError = async ( + cli: TestCli, + { + app, + root, + reason, + message, + }: { + readonly app: string; + readonly root: string; + readonly reason: string; + readonly message: string; + }, +): Promise => { + const rejected = await cli.run({ + arguments: ["trace-dylib-resolution", app, "--root", root, "--json"], + environment: ENVIRONMENT, + }); + expect(rejected.exitCode).toBe(1); + expect(rejected.json).toMatchObject({ + error: "Analysis failed", + code: "invalid_request", + details: { + issues: [ + { + path: ["roots", 0], + reason, + message: expect.stringContaining(message), + }, + ], + }, + }); +}; + const fixtureApp = async (): Promise => { const app = join( await createTestTempDirectory("rea-dylib-cli-"), @@ -141,17 +176,16 @@ describe("trace-dylib-resolution CLI", () => { const outside = await createTestTempDirectory("rea-dylib-outside-"); await writeFiles(outside, { Tool: machoImage({}) }); await symlink(outside, join(app, "Contents/Linked")); - for (const root of [ - "../outside", - "Contents/Info.plist", - "Contents/Linked/Tool", - ]) { - const rejected = await cli.run({ - arguments: ["trace-dylib-resolution", app, "--root", root, "--json"], - environment: ENVIRONMENT, - }); - expect(rejected.exitCode).toBe(1); - expect(rejected.json).toMatchObject({ error: "Analysis failed" }); + for (const [root, reason, message] of [ + ["../outside", "invalid_format", "Use a normalized path"], + ["Contents/Info.plist", "invalid_value", "is not a Mach-O image"], + [ + "Contents/Linked/Tool", + "invalid_value", + "resolves outside the analyzed root", + ], + ] as const) { + await expectRootInputError(cli, { app, root, reason, message }); } }, ); diff --git a/tests/boundary/mcp/dylibResolutionMcp.test.ts b/tests/boundary/mcp/dylibResolutionMcp.test.ts index 06e48e08ff7..e4cbfc4a26e 100644 --- a/tests/boundary/mcp/dylibResolutionMcp.test.ts +++ b/tests/boundary/mcp/dylibResolutionMcp.test.ts @@ -1,4 +1,4 @@ -import { chmod, mkdir, writeFile } from "node:fs/promises"; +import { chmod, mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; @@ -375,3 +375,53 @@ it.each([ }); }, ); + +it("reports caller-selected roots that cannot be traced as invalid input", async () => { + const app = join( + await createTestTempDirectory("rea-dylib-roots-"), + "Fixture.app", + ); + await mkdir(join(app, "Contents/MacOS"), { recursive: true }); + await writeFile( + join(app, "Contents/Info.plist"), + "CFBundleExecutableApp", + ); + await writeFile( + join(app, "Contents/MacOS/App"), + machoImage({ commands: [buildVersionCommand(1)] }), + ); + await symlink("/bin/ls", join(app, "Contents/MacOS/outside")); + await withClient(async (client) => { + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: app }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + for (const [root, message] of [ + ["Contents/MacOS/Missing", "is not a regular file"], + ["Contents/MacOS", "is not a regular file"], + ["Contents/Info.plist", "is not a Mach-O image"], + ["Contents/MacOS/outside", "resolves outside the analyzed root"], + ] as const) { + const called = await client.callTool({ + name: "trace_dylib_resolution", + arguments: { roots: ["Contents/MacOS/App", root] }, + }); + expect(called.isError, root).toBe(true); + expect(called.structuredContent, root).toMatchObject({ + error: { + code: "invalid_request", + details: { + issues: [ + { + path: ["roots", 1], + reason: "invalid_value", + message: expect.stringContaining(`Root ${root} ${message}`), + }, + ], + }, + }, + }); + } + }); +}); From 33d5ae0b52b8684634ecf14d4e9469968c69d462 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:14:25 +0800 Subject: [PATCH 099/791] fix(hopper): reserve Linux demo application before launch (#1158) * test(hopper): give locals oracle explicit stack storage * fix(hopper): reserve Linux demo application before launch * fix(hopper): serialize stale lease recovery --- docs/installation.md | 13 ++ docs/testing.md | 7 +- scripts/lib/real-hopper-boundaries.mjs | 12 ++ scripts/lib/real-hopper-session-isolation.mjs | 53 +++++ src/hopper/BridgeLauncher.ts | 50 +++-- src/hopper/HopperTargetLease.test.ts | 150 ++++++++++++- src/hopper/HopperTargetLease.ts | 197 +++++++++++++----- .../bridgeTargetLease.integration.test.ts | 149 ++++++++++--- tests/conformance/c/fixture.c | 4 +- 9 files changed, 531 insertions(+), 104 deletions(-) create mode 100644 scripts/lib/real-hopper-session-isolation.mjs diff --git a/docs/installation.md b/docs/installation.md index b8bff07d3d6..17c9be0927c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -310,6 +310,19 @@ host mount and the rest of `/tmp` remain unchanged; this fallback never invokes `sudo`. `rea doctor --provider hopper --json` reports the selected strategy and both host and effective mount facts. +The supported Linux demo forwards additional launches into its existing +application, even across private displays. REA therefore reserves one Linux +Hopper application per user across cooperating REA processes. A competing +CLI or MCP session receives the owning session ID before Hopper is launched, +for both the same target and a different target. Switch targets through that +MCP session, or close it before starting another session. Closing the owner +releases the reservation. This does not attach to or claim ownership of +manually opened Hopper applications. + +Stale lease recovery is serialized. If an interrupted recovery leaves its +reservation directory, the diagnostic identifies that exact path. Stop REA +sessions before removing that recovery directory and retrying. + ### Launcher paths and troubleshooting On macOS, REA uses diff --git a/docs/testing.md b/docs/testing.md index 7adcf3123c8..4f4633bcf7e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -230,7 +230,12 @@ subsequent wire recovery and clean shutdown. Caller timeout does not interrupt Hopper's synchronous native operation. Socket boundary tests deterministically cover active and queued expiry, late replies and timer cleanup. -The Linux demo lane remains a separate `verify:hopper:linux` command. +The Linux demo lane remains a separate `verify:hopper:linux` command. Its +lifecycle checks attempt competing CLI launches for the active target and a +different target, requiring rejection before either can change the owning +MCP session's documents or procedures. Lease boundary tests also preserve +unresponsive or malformed live endpoints while allowing confirmed stale +sockets to recover. `verify:hopper:fat` is a separate macOS lane requiring installed Hopper and the existing Xcode clang/lipo toolchain. It compiles arm64/x86-64 thin executables and diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index b094cf41233..6e6a6dfe90b 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -8,6 +8,7 @@ import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyHopperCliSelectors } from "./real-hopper-cli-selectors.mjs"; +import { verifyLinuxHopperLaunchIsolation } from "./real-hopper-session-isolation.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; import { @@ -327,6 +328,16 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { procedure: entry.value, }); const firstDocument = await call("current_document"); + const linuxApplicationIsolation = + process.platform === "linux" + ? await verifyLinuxHopperLaunchIsolation({ + call, + runCli, + dispatcher: new URL("../rea.mjs", import.meta.url).pathname, + primary, + secondary, + }) + : null; await call("open_binary", { path: secondary }); const secondDocument = await call("current_document"); assert.notEqual(firstDocument, secondDocument); @@ -464,6 +475,7 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) { cliNamedSelectorParity, callerCancellationRecovered: true, closedDocumentAbsent: true, + linuxApplicationIsolation, }; } finally { await call("close_binary"); diff --git a/scripts/lib/real-hopper-session-isolation.mjs b/scripts/lib/real-hopper-session-isolation.mjs new file mode 100644 index 00000000000..9630b29f51c --- /dev/null +++ b/scripts/lib/real-hopper-session-isolation.mjs @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; + +/** Prove competing Linux launches cannot forward documents into the active MCP owner. */ +export async function verifyLinuxHopperLaunchIsolation({ + call, + runCli, + dispatcher, + primary, + secondary, +}) { + const documents = await call("list_documents"); + const current = await call("current_document"); + const procedures = await call("list_procedures"); + for (const target of [primary, secondary]) { + let failure; + try { + await runCli([ + dispatcher, + "inspect", + target, + "--provider", + "hopper", + "--format", + "json", + ]); + } catch (error) { + assert.equal(error.code, 1); + failure = JSON.parse(error.stdout); + } + assert.ok( + failure, + "A competing CLI launch was allowed to reach Linux Hopper", + ); + assert.equal(failure.code, "provider_unavailable"); + assert.match( + failure.message, + /Linux Hopper is already active in REA session/u, + ); + assert.ok(failure.message.includes(target)); + assert.deepEqual( + await call("list_documents"), + documents, + "A rejected CLI launch changed the owning session's documents", + ); + assert.equal(await call("current_document"), current); + assert.deepEqual(await call("list_procedures"), procedures); + } + return { + sameTargetRejected: true, + differentTargetRejected: true, + ownerDocumentsPreserved: true, + }; +} diff --git a/src/hopper/BridgeLauncher.ts b/src/hopper/BridgeLauncher.ts index 436d5f0b54d..9952152f60a 100644 --- a/src/hopper/BridgeLauncher.ts +++ b/src/hopper/BridgeLauncher.ts @@ -28,7 +28,10 @@ import { type HopperMachOImage, } from "./HopperMachOImage.js"; -import { acquireHopperTargetLease } from "./HopperTargetLease.js"; +import { + acquireHopperTargetLease, + acquireLinuxHopperApplicationLease, +} from "./HopperTargetLease.js"; import type { HopperTargetLease } from "./HopperTargetLease.js"; const execFileAsync = promisify(execFile); @@ -96,6 +99,7 @@ export type HopperApplicationLauncherOptions = export interface HopperApplicationLauncherDependencies { readonly platform?: NodeJS.Platform; readonly acquireTargetLease?: typeof acquireHopperTargetLease; + readonly acquireApplicationLease?: typeof acquireLinuxHopperApplicationLease; readonly selectPrivateDisplay?: (options: { readonly helperPath: string; readonly signal?: AbortSignal; @@ -131,7 +135,7 @@ export class HopperApplicationLauncher implements BridgeLauncher { HopperStartError | HopperCancelledError | HopperProcessError > > { - const leaseResult = await this.#acquireTargetLease(session.runId); + const leaseResult = await this.#acquireLaunchLease(session.runId); if (!leaseResult.ok) return leaseResult; const lease = leaseResult.value; let leaseTransferred = false; @@ -267,32 +271,44 @@ export class HopperApplicationLauncher implements BridgeLauncher { } } - async #acquireTargetLease( + async #acquireLaunchLease( runId: string, ): Promise> { - if ((this.dependencies.platform ?? process.platform) !== "darwin") - return ok(undefined); + const platform = this.dependencies.platform ?? process.platform; + const applicationLease = + platform === "linux" && usesLinuxDemo(this.options); + if (!applicationLease && platform !== "darwin") return ok(undefined); try { - const acquisition = await ( - this.dependencies.acquireTargetLease ?? acquireHopperTargetLease - )({ - targetPath: this.options.targetPath, - targetKind: this.options.targetKind, - loaderArgs: this.options.loaderArgs, - runId, - }); + const acquisition = applicationLease + ? await ( + this.dependencies.acquireApplicationLease ?? + acquireLinuxHopperApplicationLease + )({ runId }) + : await ( + this.dependencies.acquireTargetLease ?? acquireHopperTargetLease + )({ + targetPath: this.options.targetPath, + targetKind: this.options.targetKind, + loaderArgs: this.options.loaderArgs, + runId, + }); if (!acquisition.acquired) return err( new HopperStartError({ ownerRunId: acquisition.owner.runId, - userMessage: - `This Hopper target is already open in REA session ${acquisition.owner.runId}. ` + - "Use that REA session or close it before opening this target again.", + userMessage: applicationLease + ? `Linux Hopper is already active in REA session ${acquisition.owner.runId}; it cannot open ${this.options.targetPath} in another session. This build forwards new launches to the existing application. Use the owning REA session to switch targets, or close it before retrying.` + : `This Hopper target is already open in REA session ${acquisition.owner.runId}. ` + + "Use that REA session or close it before opening this target again.", }), ); return ok(acquisition.lease); } catch (cause: unknown) { - return err(new HopperStartError({ cause })); + return err( + cause instanceof HopperStartError + ? cause + : new HopperStartError({ cause }), + ); } } diff --git a/src/hopper/HopperTargetLease.test.ts b/src/hopper/HopperTargetLease.test.ts index fd9795957e3..7c7bd03ae0e 100644 --- a/src/hopper/HopperTargetLease.test.ts +++ b/src/hopper/HopperTargetLease.test.ts @@ -1,9 +1,16 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { once } from "node:events"; +import { lstat, mkdir, mkdtemp, readdir, rm, rmdir } from "node:fs/promises"; +import { createServer, type Socket } from "node:net"; import { join } from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; -import { acquireHopperTargetLease } from "./HopperTargetLease.js"; +import { + acquireHopperTargetLease, + acquireLinuxHopperApplicationLease, +} from "./HopperTargetLease.js"; const directories: string[] = []; @@ -66,6 +73,145 @@ describe("Hopper target leases", () => { }); }); +describe("Linux Hopper application leases", () => { + it("reserves the shared application until its owner releases it", async () => { + const directory = await temporaryDirectory(); + const first = await acquireLinuxHopperApplicationLease({ + directory, + runId: "first", + }); + expect(first.acquired).toBe(true); + if (!first.acquired) return; + try { + expect( + await acquireLinuxHopperApplicationLease({ + directory, + runId: "second", + }), + ).toMatchObject({ + acquired: false, + owner: { runId: "first", processId: process.pid }, + }); + } finally { + await first.lease.release(); + } + const reopened = await acquireLinuxHopperApplicationLease({ + directory, + runId: "third", + }); + expect(reopened.acquired).toBe(true); + if (reopened.acquired) await reopened.lease.release(); + }); + + it.each(["silent", "malformed", "invalid-pid", "closed"] as const)( + "keeps a live endpoint when its owner response is %s", + async (behavior) => { + const directory = await temporaryDirectory(); + const socketPath = await releasedApplicationSocket(directory); + let responsive = false; + const sockets = new Set(); + const owner = { runId: "existing-owner", processId: process.pid }; + const server = createServer((socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("error", () => undefined); + if (responsive) socket.end(`${JSON.stringify(owner)}\n`); + else if (behavior === "malformed") socket.end("not JSON\n"); + else if (behavior === "invalid-pid") + socket.end('{"runId":"existing-owner","processId":-1}\n'); + else if (behavior === "closed") socket.end(); + }); + server.listen(socketPath); + await once(server, "listening"); + try { + await expect( + acquireLinuxHopperApplicationLease({ directory, runId: "new-owner" }), + ).rejects.toMatchObject({ + _tag: "HopperStartError", + userMessage: expect.stringContaining("left unchanged"), + }); + expect((await lstat(socketPath)).isSocket()).toBe(true); + responsive = true; + expect( + await acquireLinuxHopperApplicationLease({ + directory, + runId: "retry", + }), + ).toEqual({ + acquired: false, + owner, + }); + } finally { + for (const socket of sockets) socket.destroy(); + server.close(); + await once(server, "close"); + } + }, + ); + + it("recovers a socket left by a process that exited without releasing it", async () => { + const directory = await temporaryDirectory(); + const socketPath = await releasedApplicationSocket(directory); + await leaveStaleSocket(socketPath); + expect((await lstat(socketPath)).isSocket()).toBe(true); + const recovered = await acquireLinuxHopperApplicationLease({ + directory, + runId: "recovered", + }); + expect(recovered.acquired).toBe(true); + if (recovered.acquired) await recovered.lease.release(); + }); +}); + +it("leaves a stale socket unchanged while another process reserves its recovery", async () => { + const directory = await temporaryDirectory(); + const socketPath = await releasedApplicationSocket(directory); + await leaveStaleSocket(socketPath); + const recoveryPath = `${socketPath}.recovery`; + await mkdir(recoveryPath, { mode: 0o700 }); + const stale = await lstat(socketPath); + await expect( + acquireLinuxHopperApplicationLease({ directory, runId: "competing" }), + ).rejects.toMatchObject({ + _tag: "HopperStartError", + userMessage: expect.stringContaining(recoveryPath), + }); + expect((await lstat(socketPath)).ino).toBe(stale.ino); + await rmdir(recoveryPath); + const recovered = await acquireLinuxHopperApplicationLease({ + directory, + runId: "after-recovery", + }); + expect(recovered.acquired).toBe(true); + if (recovered.acquired) await recovered.lease.release(); + expect(await readdir(directory)).toEqual([]); +}); + +const leaveStaleSocket = async (socketPath: string): Promise => { + await promisify(execFile)(process.execPath, [ + "--input-type=module", + "-e", + 'import {createServer} from "node:net"; createServer().listen(process.argv[1],()=>process.exit(0));', + socketPath, + ]); +}; + +const releasedApplicationSocket = async ( + directory: string, +): Promise => { + const initial = await acquireLinuxHopperApplicationLease({ + directory, + runId: "initial", + }); + if (!initial.acquired) + throw new Error("Initial application lease was unavailable"); + const [name] = await readdir(directory); + await initial.lease.release(); + if (name === undefined) + throw new Error("Application lease did not create a socket"); + return join(directory, name); +}; + const leaseInput = { targetPath: "/tmp/sample-binary", targetKind: "executable" as const, diff --git a/src/hopper/HopperTargetLease.ts b/src/hopper/HopperTargetLease.ts index 3d5d808e69e..ad291ef22ee 100644 --- a/src/hopper/HopperTargetLease.ts +++ b/src/hopper/HopperTargetLease.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { chmod, lstat, mkdir, realpath, rm } from "node:fs/promises"; +import { chmod, lstat, mkdir, realpath, rm, rmdir } from "node:fs/promises"; import { createConnection, createServer, @@ -23,7 +23,7 @@ export type HopperTargetLeaseAcquisition = | { readonly acquired: true; readonly lease: HopperTargetLease } | { readonly acquired: false; readonly owner: HopperTargetLeaseOwner }; -/** Shared private parent of per-target lease sockets; it outlives individual sessions. */ +/** Shared private parent of Hopper lease sockets; it outlives individual sessions. */ export const HOPPER_TARGET_LEASE_DIRECTORY = join( "/tmp", `rea-hopper-${process.getuid?.() ?? 0}`, @@ -37,8 +37,6 @@ export const acquireHopperTargetLease = async (input: { readonly runId: string; readonly directory?: string; }): Promise => { - const directory = input.directory ?? HOPPER_TARGET_LEASE_DIRECTORY; - await ensureLeaseDirectory(directory); // The canonical path only derives the lease key; it never authorizes access. // Common expected `realpath` failures include a missing target or path // component (ENOENT, ENOTDIR), an unreadable component (EACCES), or a @@ -49,48 +47,115 @@ export const acquireHopperTargetLease = async (input: { const targetPath = await realpath(input.targetPath).catch(() => resolve(input.targetPath), ); + return acquireLease({ + ...input, + identity: { + targetPath, + targetKind: input.targetKind, + loaderArgs: input.loaderArgs, + }, + }); +}; + +/** Reserve the Linux demo's singleton application before it can forward a target to another owner. */ +export const acquireLinuxHopperApplicationLease = (input: { + readonly runId: string; + readonly directory?: string; +}): Promise => + acquireLease({ ...input, identity: { application: "linux-hopper-demo" } }); + +const acquireLease = async (input: { + readonly identity: Readonly>; + readonly runId: string; + readonly directory?: string; +}): Promise => { + const directory = input.directory ?? HOPPER_TARGET_LEASE_DIRECTORY; + await ensureLeaseDirectory(directory); const key = createHash("sha256") - .update( - JSON.stringify({ - targetPath, - targetKind: input.targetKind, - loaderArgs: input.loaderArgs, - }), - ) + .update(JSON.stringify(input.identity)) .digest("hex") .slice(0, 32); const socketPath = join(directory, `${key}.sock`); const owner = { runId: input.runId, processId: process.pid }; + const acquisition = await tryAcquireLease(socketPath, owner); + if (typeof acquisition !== "string") return acquisition; - for (let attempt = 0; attempt < 2; attempt += 1) { - const server = createServer((socket) => sendOwner(socket, owner)); - const bound = await bind(server, socketPath); - if (bound) { - await chmod(socketPath, 0o600); - let released = false; - return { - acquired: true, - lease: { - release: async () => { - if (released) return; - released = true; - await closeServer(server); - await rm(socketPath, { force: true }); - }, - }, - }; + // Serialize stale cleanup through the replacement bind: otherwise two + // recoverers can unlink the socket that one of them has just acquired. + const recoveryPath = `${socketPath}.recovery`; + await reserveRecovery(recoveryPath); + let recovered: HopperTargetLeaseAcquisition | "missing" | "stale" | undefined; + try { + recovered = await tryAcquireLease(socketPath, owner); + if (recovered === "stale") await rm(socketPath, { force: true }); + if (typeof recovered === "string") + recovered = await tryAcquireLease(socketPath, owner); + if (typeof recovered !== "string") return recovered; + throw new HopperStartError({ + userMessage: + "REA could not reserve Hopper for this request. Close competing REA sessions and retry.", + }); + } finally { + try { + await rmdir(recoveryPath); + } catch (cause: unknown) { + if (typeof recovered !== "string" && recovered?.acquired) + await recovered.lease.release(); + throw new HopperStartError({ + cause, + userMessage: `REA could not remove its Hopper recovery directory at ${JSON.stringify(recoveryPath)}. Check this directory's write access before retrying.`, + }); } + } +}; - const existingOwner = await readOwner(socketPath); - if (existingOwner !== undefined) - return { acquired: false, owner: existingOwner }; - await rm(socketPath, { force: true }); +const reserveRecovery = async (recoveryPath: string): Promise => { + try { + await mkdir(recoveryPath, { mode: 0o700 }); + } catch (cause: unknown) { + const busy = + typeof cause === "object" && + cause !== null && + Reflect.get(cause, "code") === "EEXIST"; + throw new HopperStartError({ + cause, + userMessage: busy + ? `REA could not reserve Hopper lease recovery at ${JSON.stringify(recoveryPath)}; the existing socket was left unchanged. Retry after concurrent startup finishes. If recovery was interrupted, stop REA sessions and remove this recovery directory before retrying.` + : `REA could not create its Hopper recovery directory at ${JSON.stringify(recoveryPath)}; the existing socket was left unchanged. Check the parent directory's write access before retrying.`, + }); } +}; - throw new HopperStartError({ - userMessage: - "REA could not reserve this Hopper target. Close competing REA sessions and retry.", - }); +const tryAcquireLease = async ( + socketPath: string, + owner: HopperTargetLeaseOwner, +): Promise => { + const server = createServer((socket) => sendOwner(socket, owner)); + if (!(await bind(server, socketPath))) { + const existingOwner = await readOwner(socketPath); + return typeof existingOwner === "string" + ? existingOwner + : { acquired: false, owner: existingOwner }; + } + try { + await chmod(socketPath, 0o600); + } catch (cause: unknown) { + await closeServer(server); + throw cause; + } + let released = false; + return { + acquired: true, + lease: { + release: async () => { + if (released) return; + released = true; + // Node removes a bound Unix socket on close. A second unlink here + // could remove a new owner's socket bound after that close. + await closeServer(server); + }, + }, + }; }; const ensureLeaseDirectory = async (directory: string): Promise => { @@ -139,15 +204,29 @@ const sendOwner = (socket: Socket, owner: HopperTargetLeaseOwner): void => { const readOwner = ( socketPath: string, -): Promise => - new Promise((resolveOwner) => { +): Promise => + new Promise((resolveOwner, rejectOwner) => { const socket = createConnection(socketPath); let response = ""; - const timer = setTimeout(() => { + let settled = false; + const fail = (reason: string, cause?: unknown): void => { + if (settled) return; + settled = true; + clearTimeout(timer); socket.destroy(); - resolveOwner(undefined); - }, 250); - const finish = (owner: HopperTargetLeaseOwner | undefined): void => { + rejectOwner( + new HopperStartError({ + cause, + userMessage: `REA could not verify the existing Hopper lease (${reason}); it was left unchanged. Retry when the owning REA session is responsive.`, + }), + ); + }; + const timer = setTimeout(() => fail("owner response timed out"), 250); + const finish = ( + owner: HopperTargetLeaseOwner | "missing" | "stale", + ): void => { + if (settled) return; + settled = true; clearTimeout(timer); socket.destroy(); resolveOwner(owner); @@ -158,24 +237,34 @@ const readOwner = ( if (newline < 0) return; const parsed = safeParseJson(response.slice(0, newline)); if (!parsed.ok) { - finish(undefined); + fail("malformed owner response"); return; } const value: unknown = parsed.value; + const runId = + typeof value === "object" && value !== null + ? Reflect.get(value, "runId") + : undefined; + const processId = + typeof value === "object" && value !== null + ? Reflect.get(value, "processId") + : undefined; if ( - typeof value === "object" && - value !== null && - typeof Reflect.get(value, "runId") === "string" && - typeof Reflect.get(value, "processId") === "number" + typeof runId === "string" && + runId.length > 0 && + typeof processId === "number" && + Number.isSafeInteger(processId) && + processId > 0 ) - finish({ - runId: Reflect.get(value, "runId") as string, - processId: Reflect.get(value, "processId") as number, - }); - else finish(undefined); + finish({ runId, processId }); + else fail("invalid owner identity"); + }); + socket.on("error", (cause: NodeJS.ErrnoException) => { + if (cause.code === "ENOENT") finish("missing"); + else if (cause.code === "ECONNREFUSED") finish("stale"); + else fail(cause.code ?? "owner connection failed", cause); }); - socket.on("error", () => finish(undefined)); - socket.on("end", () => finish(undefined)); + socket.on("end", () => fail("owner closed without a complete response")); }); const closeServer = (server: Server): Promise => diff --git a/tests/boundary/providers/hopper/bridgeTargetLease.integration.test.ts b/tests/boundary/providers/hopper/bridgeTargetLease.integration.test.ts index 019947aa5ac..c82243e8e1c 100644 --- a/tests/boundary/providers/hopper/bridgeTargetLease.integration.test.ts +++ b/tests/boundary/providers/hopper/bridgeTargetLease.integration.test.ts @@ -1,8 +1,87 @@ import { describe, expect, it } from "vitest"; import { HopperApplicationLauncher } from "../../../../src/hopper/BridgeLauncher.js"; +import { HopperStartError } from "../../../../src/domain/hopperErrors.js"; -describe("Hopper target leases in the launcher", () => { +describe("Linux Hopper application leases in the launcher", () => { + it.each(["/first-target", "/different-target"])( + "rejects Linux demo launch for %s before it can forward a document to another owner", + async (targetPath) => { + let probedDisplay = false; + const launcher = new HopperApplicationLauncher( + { + launcherPath: "/unused/Hopper", + targetPath, + targetKind: "executable", + loaderArgs: [], + bridgeScriptPath: "/rea/hopper_bridge.py", + launchMode: "verified_linux_demo", + demoHelperPath: "/rea/hopper-demo-x11.py", + }, + { + platform: "linux", + acquireApplicationLease: async () => ({ + acquired: false, + owner: { runId: "owning-session", processId: 17 }, + }), + selectPrivateDisplay: async () => { + probedDisplay = true; + throw new Error("Launch must stop before probing the display"); + }, + }, + ); + expect( + await launcher.launch({ + directory: "/missing/rea-launch", + socketPath: "/missing/rea-launch/bridge.sock", + token: "token", + runId: "new-session", + }), + ).toMatchObject({ + ok: false, + error: { + _tag: "HopperStartError", + ownerRunId: "owning-session", + userMessage: expect.stringContaining(targetPath), + }, + }); + expect(probedDisplay).toBe(false); + }, + ); + + it("preserves the reason that an existing lease could not be verified", async () => { + const failure = new HopperStartError({ + userMessage: "Existing owner timed out; the lease was left unchanged", + }); + const launcher = new HopperApplicationLauncher( + { + launcherPath: "/unused/Hopper", + targetPath: "/target", + targetKind: "executable", + loaderArgs: [], + bridgeScriptPath: "/rea/hopper_bridge.py", + launchMode: "verified_linux_demo", + demoHelperPath: "/rea/helper.py", + }, + { + platform: "linux", + acquireApplicationLease: async () => { + throw failure; + }, + }, + ); + expect( + await launcher.launch({ + directory: "/missing/rea-launch", + socketPath: "/missing/rea-launch/bridge.sock", + token: "token", + runId: "new-session", + }), + ).toEqual({ ok: false, error: failure }); + }); +}); + +describe("Hopper launch leases", () => { it("reports an active target owner without launching another document", async () => { let leaseChecks = 0; const launcher = new HopperApplicationLauncher( @@ -45,34 +124,46 @@ describe("Hopper target leases in the launcher", () => { }); }); - it("releases a target lease when bridge setup fails", async () => { - let releases = 0; - const launcher = new HopperApplicationLauncher( - { - launcherPath: "/unused/hopper", - targetPath: "/target", - targetKind: "executable", - loaderArgs: [], - bridgeScriptPath: "/rea/hopper_bridge.py", - launchMode: "native", - }, - { - platform: "darwin", - acquireTargetLease: async () => ({ - acquired: true, - lease: { release: async () => void (releases += 1) }, - }), - }, - ); + it.each(["darwin", "linux"] as const)( + "releases the %s launch lease when bridge setup fails", + async (platform) => { + let releases = 0; + const launcher = new HopperApplicationLauncher( + { + launcherPath: "/unused/hopper", + targetPath: "/target", + targetKind: "executable", + loaderArgs: [], + bridgeScriptPath: "/rea/hopper_bridge.py", + ...(platform === "linux" + ? { + launchMode: "verified_linux_demo" as const, + demoHelperPath: "/unused/helper.py", + } + : { launchMode: "native" as const }), + }, + { + platform, + acquireTargetLease: async () => ({ + acquired: true, + lease: { release: async () => void (releases += 1) }, + }), + acquireApplicationLease: async () => ({ + acquired: true, + lease: { release: async () => void (releases += 1) }, + }), + }, + ); - const result = await launcher.launch({ - directory: "/missing/rea-launch-test", - socketPath: "/missing/rea-launch-test/bridge.sock", - token: "token", - runId: "failed-session", - }); + const result = await launcher.launch({ + directory: "/missing/rea-launch-test", + socketPath: "/missing/rea-launch-test/bridge.sock", + token: "token", + runId: "failed-session", + }); - expect(result.ok).toBe(false); - expect(releases).toBe(1); - }); + expect(result.ok).toBe(false); + expect(releases).toBe(1); + }, + ); }); diff --git a/tests/conformance/c/fixture.c b/tests/conformance/c/fixture.c index 66d48543661..6708ac9dd0e 100644 --- a/tests/conformance/c/fixture.c +++ b/tests/conformance/c/fixture.c @@ -25,7 +25,9 @@ __attribute__((noinline, used)) int rea_branch(int value) { __attribute__((noinline, used)) int rea_entry(void) { puts("REA_C_ENTRY"); - return rea_branch(5); + /* Give the native locals oracle explicit stack storage. */ + volatile int input = 5; + return rea_branch(input); } int main(void) { return rea_entry() == 12 ? 0 : 1; } From 637bcb5e4182b6c60e922b18688642d8e1008e90 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:16:57 +0000 Subject: [PATCH 100/791] fix(artifacts): report wrong target kinds as unsupported targets decode_interface_builder, inspect_asset_catalog, inspect_keyed_archive and trace_dylib_resolution refused a target of the wrong kind (for example a plain Mach-O for the asset catalog, or a plist for dyld tracing) as an `unavailable` artifact failure: "Artifact processing is unavailable for the current target or host. Check artifact support and required tools." Nothing was missing on the host; the operation does not support that kind of target. Raise AnalysisUnsupportedTargetError for these checks and let translateFailure keep it. The result is now `unsupported_target`, naming the operation, the target path and the requirement, with advice to pick a supported target or another operation. Co-Authored-By: Claude Opus 5.5 --- src/artifacts/ArtifactProvider.ts | 38 ++++++++++++++----- tests/boundary/mcp/dylibResolutionMcp.test.ts | 30 ++++++++++++++- 2 files changed, 57 insertions(+), 11 deletions(-) diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 941b75d0f0f..a711f266bf8 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -24,6 +24,7 @@ import type { BinaryTarget } from "../domain/binaryTarget.js"; import { AnalysisCapabilityUnavailableError, AnalysisInputError, + AnalysisUnsupportedTargetError, } from "../domain/analysisErrorCore.js"; import { ArtifactOperationError } from "../domain/artifactOperationError.js"; import { type AnalysisError } from "../domain/analysisErrorBase.js"; @@ -89,8 +90,8 @@ class ArtifactClient implements AnalysisClient { this.target.sourcePath === undefined || !this.target.sourcePath.toLowerCase().endsWith(".app") ) - throw new ArtifactReaderFailure( - "unavailable", + throw this.unsupportedTarget( + operation, "decode_interface_builder requires an active .app bundle target", ); const limits = interfaceBuilderLimitsSchema.parse(parameters); @@ -135,8 +136,8 @@ class ArtifactClient implements AnalysisClient { bundlePath === undefined || (!standalone && !bundlePath.toLowerCase().endsWith(".app")) ) - throw new ArtifactReaderFailure( - "unavailable", + throw this.unsupportedTarget( + operation, "inspect_keyed_archive requires an active plist or .app bundle", ); const result = await inspectBundleKeyedArchive({ @@ -214,6 +215,18 @@ class ArtifactClient implements AnalysisClient { return Promise.resolve(); } + /** The active target's kind is outside this operation's supported targets. */ + private unsupportedTarget( + operation: string, + reason: string, + ): AnalysisUnsupportedTargetError { + return new AnalysisUnsupportedTargetError( + operation, + this.target.sourcePath ?? this.target.path, + reason, + ); + } + private async inspectAssetCatalog( parameters: Readonly>, options?: ExecutionOptions, @@ -224,8 +237,8 @@ class ArtifactClient implements AnalysisClient { bundlePath === undefined || !bundlePath.toLowerCase().endsWith(".app") ) - throw new ArtifactReaderFailure( - "unavailable", + throw this.unsupportedTarget( + "inspect_asset_catalog", "inspect_asset_catalog requires an active .app bundle target", ); const result = await analyzeAppleAssetCatalogs({ @@ -250,8 +263,8 @@ class ArtifactClient implements AnalysisClient { options?: ExecutionOptions, ) { if (this.target.kind !== "executable" || this.target.format !== "mach-o") - throw new ArtifactReaderFailure( - "unavailable", + throw this.unsupportedTarget( + "trace_dylib_resolution", "trace_dylib_resolution requires an active Mach-O or .app bundle target", ); // Only a target opened from an app bundle directory carries its Info.plist; @@ -360,8 +373,13 @@ const translateFailure = ( cause.details, cause.message, ); - // Caller-selection failures are already typed; keep their correction details. - if (cause instanceof AnalysisInputError) return cause; + // Caller-selection and unsupported-target failures are already typed; keep + // their correction details instead of reducing them to an I/O failure. + if ( + cause instanceof AnalysisInputError || + cause instanceof AnalysisUnsupportedTargetError + ) + return cause; return new ArtifactOperationError(operation, "io"); }; diff --git a/tests/boundary/mcp/dylibResolutionMcp.test.ts b/tests/boundary/mcp/dylibResolutionMcp.test.ts index 06e48e08ff7..02d75c45d22 100644 --- a/tests/boundary/mcp/dylibResolutionMcp.test.ts +++ b/tests/boundary/mcp/dylibResolutionMcp.test.ts @@ -126,7 +126,35 @@ it("rejects targets that are not Mach-O", async () => { arguments: {}, }); expect(called.isError).toBe(true); - expect(JSON.stringify(called.structuredContent)).toContain("unavailable"); + expect(called.structuredContent).toMatchObject({ + error: { code: "unsupported_target", category: "unsupported_target" }, + }); + }); +}); + +it("reports target kinds that artifact operations cannot inspect as unsupported", async () => { + const directory = await createTestTempDirectory("rea-artifact-target-"); + const plist = join(directory, "Info.plist"); + await writeFile(plist, ""); + const tool = join(directory, "tool"); + await writeFile(tool, machoImage({ commands: [buildVersionCommand(1)] })); + await withClient(async (client) => { + for (const [path, operation, requirement] of [ + [plist, "decode_interface_builder", "requires an active .app bundle"], + [plist, "inspect_asset_catalog", "requires an active .app bundle"], + [tool, "inspect_keyed_archive", "requires an active plist or .app"], + ] as const) { + const open = { name: "open_binary", arguments: { path } }; + expect((await client.callTool(open)).isError, path).not.toBe(true); + const called = await client.callTool({ name: operation, arguments: {} }); + expect(called.structuredContent, operation).toMatchObject({ + error: { + code: "unsupported_target", + message: expect.stringContaining(`${operation} at ${path}`), + details: { reason: expect.stringContaining(requirement) }, + }, + }); + } }); }); From 44d5b4b25e0831769c1bd4e5426ef498d0bd3c74 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:21:21 +0000 Subject: [PATCH 101/791] test(artifacts): assert asset-catalog target refusal only on macOS Outside Darwin the session reports inspect_asset_catalog as unavailable on the host before the provider checks the target kind, so the Linux test shards would see capability_unavailable rather than unsupported_target. Co-Authored-By: Claude Opus 5.5 --- tests/boundary/mcp/dylibResolutionMcp.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/boundary/mcp/dylibResolutionMcp.test.ts b/tests/boundary/mcp/dylibResolutionMcp.test.ts index 02d75c45d22..e07f9dde0d2 100644 --- a/tests/boundary/mcp/dylibResolutionMcp.test.ts +++ b/tests/boundary/mcp/dylibResolutionMcp.test.ts @@ -141,7 +141,13 @@ it("reports target kinds that artifact operations cannot inspect as unsupported" await withClient(async (client) => { for (const [path, operation, requirement] of [ [plist, "decode_interface_builder", "requires an active .app bundle"], - [plist, "inspect_asset_catalog", "requires an active .app bundle"], + // Elsewhere the session reports asset catalogs as unavailable on the + // host before the provider checks the target kind. + ...(process.platform === "darwin" + ? ([ + [plist, "inspect_asset_catalog", "requires an active .app bundle"], + ] as const) + : []), [tool, "inspect_keyed_archive", "requires an active plist or .app"], ] as const) { const open = { name: "open_binary", arguments: { path } }; From 8e74c9c62fb44556b90f20bb6c12771ac05a18f2 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:21:22 +0000 Subject: [PATCH 102/791] fix(artifacts): refuse unsupported extraction formats before scanning extract_artifact on a format without an extraction reader, such as a DMG, first ran the full inventory scan (mounting the image) and then failed with an `unavailable` artifact failure: "Check artifact support and required tools". The target format, not the host, is unsupported. Check the format before any inventory or output work and raise AnalysisUnsupportedTargetError, which the provider now keeps, as it does for the other target-kind checks in this PR. Co-Authored-By: Claude Opus 5.5 --- .../extraction/ArtifactExtraction.ts | 49 +++++++++++++------ .../artifactProvider.extraction.test.ts | 21 ++++++++ 2 files changed, 56 insertions(+), 14 deletions(-) diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 57e8a0066b3..a309d35f1d7 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -22,6 +22,7 @@ import { type ArtifactNode, type ArtifactOccurrence, } from "../../domain/artifactGraph.js"; +import { AnalysisUnsupportedTargetError } from "../../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; import { scanArtifactInventory } from "../inventory/ArtifactInventory.js"; @@ -38,6 +39,7 @@ export const extractArtifact = async ( signal?: AbortSignal, ): Promise => { const sourcePath = await realpath(input.inputPath); + await requireExtractableFormat(sourcePath, input.inputFormat); const snapshot = await scanArtifactInventory(sourcePath, { signal, }); @@ -240,26 +242,45 @@ const collectNodes = ( if (selected.has(item.artifact_id)) output.set(item.artifact_id, item); }; +const ZIP_FORMATS = ["ipa", "apk", "msix", "appx", "zip"] as const; + +const isZipFormat = ( + format: BinaryTarget["format"], +): format is (typeof ZIP_FORMATS)[number] => + (ZIP_FORMATS as readonly string[]).includes(format); + +/** + * Refuse a format without an extraction reader before inventory work starts. + * The target kind, not the host, is unsupported by extraction. + */ +const requireExtractableFormat = async ( + path: string, + format: BinaryTarget["format"], +): Promise => { + const directory = (await lstat(path)).isDirectory(); + if ( + !directory && + format !== "asar" && + format !== "mach-o" && + !isZipFormat(format) + ) + throw new AnalysisUnsupportedTargetError( + "extract_artifact", + path, + `Artifact format has no extraction reader: ${format}`, + ); + return directory; +}; + const createReader = async ( path: string, format: BinaryTarget["format"], ): Promise => { - if ((await lstat(path)).isDirectory()) + if (await requireExtractableFormat(path, format)) return new DirectoryArtifactReader(path); if (format === "asar") return new AsarArtifactReader(path); - if ( - format === "ipa" || - format === "apk" || - format === "msix" || - format === "appx" || - format === "zip" - ) - return new ZipArtifactReader(path, format); - if (format === "mach-o") return new MachOSliceArtifactReader(path); - throw new ArtifactReaderFailure( - "unavailable", - `Artifact format has no extraction reader: ${format}`, - ); + if (isZipFormat(format)) return new ZipArtifactReader(path, format); + return new MachOSliceArtifactReader(path); }; const preflight = (entry: ArtifactEntry): void => { diff --git a/tests/boundary/providers/native/artifactProvider.extraction.test.ts b/tests/boundary/providers/native/artifactProvider.extraction.test.ts index 534ff59d124..509a3479870 100644 --- a/tests/boundary/providers/native/artifactProvider.extraction.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extraction.test.ts @@ -133,6 +133,27 @@ describe("artifact extraction", () => { "source", ]); }); + it("refuses a format without an extraction reader before inventory or output", async () => { + const root = await createTestTempDirectory("rea-extract-dmg-"); + const image = join(root, "Image.dmg"); + await writeFile(image, "not mounted"); + const output = join(root, "output"); + const result = await new ArtifactProvider() + .createClient(target(image, "dmg")) + .execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisUnsupportedTargetError", + operation: "extract_artifact", + reason: "Artifact format has no extraction reader: dmg", + }, + }); + await expect(access(output)).rejects.toThrow(); + }); }); const inventory = async (targetValue: BinaryTarget) => { const result = await new ArtifactProvider() From de43da71ae982dd0f3d7b5ae49c914729db3a6ec Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:23:44 +0000 Subject: [PATCH 103/791] fix(artifacts): report encrypted archive entries as unsupported extraction extract_artifact on a ZIP containing a password-protected entry failed the whole extraction with "Artifact is invalid or has changed. Get a fresh copy" and named only an opaque occurrence ID. The archive was intact; REA does not decrypt entries, so complete extraction is unsupported for this target. Raise AnalysisUnsupportedTargetError naming the encrypted entry's path. Other unextractable occurrences keep their format failure but now name the logical path alongside the occurrence ID. Co-Authored-By: Claude Opus 5.5 --- src/artifacts/extraction/ArtifactExtraction.ts | 11 +++++++++-- .../filesystem/artifactZipIntegrity.test.ts | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index a309d35f1d7..4cd2268319a 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -67,15 +67,22 @@ export const extractArtifact = async ( nodes, }; const selected = selectedOccurrences.map((occurrence) => { + // REA never decrypts archive entries, so an encrypted entry makes the + // complete extraction unsupported rather than the archive invalid. + if (occurrence.encrypted) + throw new AnalysisUnsupportedTargetError( + "extract_artifact", + sourcePath, + `Archive entry ${occurrence.logical_path} is encrypted; REA does not decrypt archive entries, so the archive cannot be extracted completely`, + ); if ( (occurrence.entry_kind !== "file" && occurrence.entry_kind !== "slice") || occurrence.artifact_id === null || - occurrence.encrypted || occurrence.logical_path === "." ) throw new ArtifactReaderFailure( "format", - `Selected occurrence is not an extractable regular child file: ${occurrence.occurrence_id}`, + `Selected occurrence is not an extractable regular child file: ${occurrence.logical_path} (${occurrence.occurrence_id})`, ); const node = inventory.nodes.get(occurrence.artifact_id); if (node === undefined) diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 61b6b47c482..1796f1bc774 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -89,6 +89,22 @@ it("preserves encrypted-member and corrupt-CRC observations", async () => { hash_status: "unavailable", }), ); + // REA does not decrypt entries, so complete extraction is unsupported. + const extractionRoot = join(root, "encrypted-output"); + expect( + await new ArtifactProvider() + .createClient(encryptedTarget.value) + .execute("extract_artifact", { output_root: extractionRoot }), + ).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisUnsupportedTargetError", + reason: expect.stringContaining("Archive entry secret.txt is encrypted"), + }, + }); + await expect(readFile(extractionRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); const corruptPath = join(root, "corrupt-crc.zip"); const corruptWriter = new ZipWriter(new Uint8ArrayWriter()); From eacf8461dad6bab11a3d5438e6db001a10f33a9b Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:33:16 +0000 Subject: [PATCH 104/791] fix(artifacts): keep cancellation and the caller path in extraction refusals Two follow-ups to #1161 from its post-merge review: - The new format preflight ran before scanArtifactInventory, which was the first cancellation check. An already-cancelled extraction of a DMG or PKG therefore reported unsupported_target instead of cancellation. Check the signal before the preflight. - The unsupported-format and encrypted-entry refusals reported the realpath. When the caller selected the target through a symlink, junction, or other alias, the diagnostic named a different path. Report the caller's input path; I/O still uses the canonical path. Co-Authored-By: Claude Opus 5.5 --- .../extraction/ArtifactExtraction.ts | 22 +++++++----- .../artifactProvider.extraction.test.ts | 35 ++++++++++++++----- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 4cd2268319a..3d2ae4eee90 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -11,6 +11,7 @@ import { type ArtifactEntry, type ArtifactReader, } from "../ArtifactReader.js"; +import { abortIfNeeded } from "../ArtifactHash.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; import { SafeOutputTree } from "../SafeOutputTree.js"; import { ZipArtifactReader } from "../ZipArtifactReader.js"; @@ -38,8 +39,10 @@ export const extractArtifact = async ( input: ArtifactExtractionInput, signal?: AbortSignal, ): Promise => { + // Cancellation wins over every later refusal, including unsupported formats. + abortIfNeeded(signal); const sourcePath = await realpath(input.inputPath); - await requireExtractableFormat(sourcePath, input.inputFormat); + await requireExtractableFormat(sourcePath, input); const snapshot = await scanArtifactInventory(sourcePath, { signal, }); @@ -72,7 +75,7 @@ export const extractArtifact = async ( if (occurrence.encrypted) throw new AnalysisUnsupportedTargetError( "extract_artifact", - sourcePath, + input.inputPath, `Archive entry ${occurrence.logical_path} is encrypted; REA does not decrypt archive entries, so the archive cannot be extracted completely`, ); if ( @@ -130,7 +133,7 @@ const materializeSelection = async ({ const byPath = new Map( selected.map((item) => [item.occurrence.logical_path, item]), ); - const reader = await createReader(sourcePath, input.inputFormat); + const reader = await createReader(sourcePath, input); const output = await SafeOutputTree.create(input.outputRoot); let readerClosed = false; const extracted: ExtractedOccurrence[] = []; @@ -258,12 +261,14 @@ const isZipFormat = ( /** * Refuse a format without an extraction reader before inventory work starts. - * The target kind, not the host, is unsupported by extraction. + * The target kind, not the host, is unsupported by extraction. The error keeps + * the caller's spelling of the path; I/O uses the canonical path. */ const requireExtractableFormat = async ( path: string, - format: BinaryTarget["format"], + input: Pick, ): Promise => { + const format = input.inputFormat; const directory = (await lstat(path)).isDirectory(); if ( !directory && @@ -273,7 +278,7 @@ const requireExtractableFormat = async ( ) throw new AnalysisUnsupportedTargetError( "extract_artifact", - path, + input.inputPath, `Artifact format has no extraction reader: ${format}`, ); return directory; @@ -281,9 +286,10 @@ const requireExtractableFormat = async ( const createReader = async ( path: string, - format: BinaryTarget["format"], + input: Pick, ): Promise => { - if (await requireExtractableFormat(path, format)) + const format = input.inputFormat; + if (await requireExtractableFormat(path, input)) return new DirectoryArtifactReader(path); if (format === "asar") return new AsarArtifactReader(path); if (isZipFormat(format)) return new ZipArtifactReader(path, format); diff --git a/tests/boundary/providers/native/artifactProvider.extraction.test.ts b/tests/boundary/providers/native/artifactProvider.extraction.test.ts index 509a3479870..038b863a79e 100644 --- a/tests/boundary/providers/native/artifactProvider.extraction.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extraction.test.ts @@ -1,4 +1,11 @@ -import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { + access, + mkdir, + readFile, + readdir, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -137,21 +144,33 @@ describe("artifact extraction", () => { const root = await createTestTempDirectory("rea-extract-dmg-"); const image = join(root, "Image.dmg"); await writeFile(image, "not mounted"); + // A caller-specific alias must be the path the refusal reports. + const alias = join(root, "Alias.dmg"); + await symlink(image, alias); const output = join(root, "output"); - const result = await new ArtifactProvider() - .createClient(target(image, "dmg")) - .execute( - "extract_artifact", - artifactExtractionExecutionSchema.parse({ output_root: output }), - ); - expect(result).toMatchObject({ + const extract = (signal?: AbortSignal) => + new ArtifactProvider() + .createClient(target(alias, "dmg")) + .execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + signal === undefined ? undefined : { signal }, + ); + expect(await extract()).toMatchObject({ ok: false, error: { _tag: "AnalysisUnsupportedTargetError", operation: "extract_artifact", + path: alias, reason: "Artifact format has no extraction reader: dmg", }, }); + const controller = new AbortController(); + controller.abort(); + expect(await extract(controller.signal)).toMatchObject({ + ok: false, + error: { _tag: "ArtifactOperationError", reason: "cancelled" }, + }); await expect(access(output)).rejects.toThrow(); }); }); From 0b32577a90aef97e36adf2a7af4f3eccd5d82357 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:37:03 +0000 Subject: [PATCH 105/791] fix(artifacts): match lipo arm64e variant names to their FAT slices Newer lipo names arm64e ABI variants `arm64e.`: `arm64e.v1` for pointer-auth ABI version 1 on freshly built universal binaries, and `arm64e.x1` on recent macOS system binaries such as /bin/ls. The structural FAT parser names the same slice `arm64e`, so the exact name comparison failed and inspect_artifact reported a valid, unmodified universal binary as an integrity mismatch ("Artifact is invalid or has changed"). Accept `arm64e.` for an `arm64e` FAT slice. Offset, size, CPU type, subtype (including the pointer-auth version bits when lipo prints them) and alignment are still compared field by field. The slice keeps lipo's name in its logical path. Fixes #1168 Co-Authored-By: Claude Opus 5.5 --- src/artifacts/MachOSliceArtifactReader.ts | 11 ++- .../artifactProvider.machOSlices.test.ts | 69 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index a242feaf402..fcda0b3d817 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -255,7 +255,7 @@ const lipoMatchesSlice = ( reported.cpu_subtype_code !== null; return ( (hasPhysicalIdentity || structurallyUnique) && - reported.name === observed.architecture && + lipoNameMatches(reported.name, observed.architecture) && (reported.file_offset === null || reported.file_offset === observed.slice_offset) && (reported.size === null || reported.size === observed.slice_size) && @@ -269,6 +269,15 @@ const lipoMatchesSlice = ( ); }; +/** + * Newer lipo names an arm64e ABI variant `arm64e.` (for example + * `arm64e.v1` for pointer-auth ABI version 1). The FAT table names that slice + * `arm64e`; its physical identity is still compared field by field. + */ +const lipoNameMatches = (reported: string, observed: string): boolean => + reported === observed || + (observed === "arm64e" && /^arm64e\.[A-Za-z0-9_]+$/u.test(reported)); + const parseSliceKeyInteger = (value: string | undefined): number | null => { if (value === undefined || !/^\d+$/u.test(value)) return null; const parsed = Number(value); diff --git a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts index 2da3bb2690b..289301a7407 100644 --- a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts +++ b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts @@ -186,3 +186,72 @@ const lipoRunner = (stdout: string): NativeCommandRunner => ({ }), ), }); + +// Newer lipo names arm64e ABI variants `arm64e.`; the FAT table names +// the same slice `arm64e`, and its physical identity must still match. +describe("artifact Mach-O arm64e variant slices", () => { + const ARM64E_V1 = { type: 0x0100000c, subtype: 0x81000002 } as const; + const variantSlices = async ( + cpu: { readonly type: number; readonly subtype: number }, + lipo: (size: number) => string, + ) => { + const root = await createTestTempDirectory("rea-slices-arm64e-"); + const binary = join(root, "fat"); + const x86 = machoImage({ cpu: CPU.x86_64 }); + const arm = machoImage({ cpu }); + await writeFile( + binary, + fatImage([ + { cpu: CPU.x86_64, bytes: x86 }, + { cpu, bytes: arm }, + ]), + ); + const reader = new MachOSliceArtifactReader( + binary, + lipoRunner( + `architecture x86_64\n cputype CPU_TYPE_X86_64\n cpusubtype CPU_SUBTYPE_X86_64_ALL\n offset 4096\n size ${String(x86.length)}\n align 2^12 (4096)\n${lipo(arm.length)}`, + ), + ); + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries.map(({ path, byteOffset }) => [path, byteOffset]); + }; + + it.each([ + [ + "arm64e.v1", + " cputype CPU_TYPE_ARM64\n cpusubtype CPU_SUBTYPE_ARM64E\n capabilities PTR_AUTH_VERSION USERSPACE 1", + ], + [ + "arm64e.x1", + " cputype CPU_TYPE_ARM64\n cpusubtype unknown arm64 cpusubtype", + ], + ])("accepts lipo's %s name for the FAT arm64e slice", async (name, cpu) => { + expect( + await variantSlices( + ARM64E_V1, + (size) => + `architecture ${name}\n${cpu}\n offset 8192\n size ${String(size)}\n align 2^12 (4096)\n`, + ), + ).toEqual([ + ["slices/x86_64", 4096], + [`slices/${name}`, 8192], + ]); + }); + + it.each([ + ["a plain arm64 slice", CPU.arm64, "8192"], + ["a different offset", ARM64E_V1, "8193"], + ])( + "still rejects an arm64e variant name for %s", + async (_label, cpu, offset) => { + await expect( + variantSlices( + cpu, + (size) => + `architecture arm64e.v1\n cputype CPU_TYPE_ARM64\n cpusubtype unknown arm64 cpusubtype\n offset ${offset}\n size ${String(size)}\n align 2^12 (4096)\n`, + ), + ).rejects.toMatchObject({ reason: "integrity" }); + }, + ); +}); From cf77d073fc06ab95ce4181c30d68b1346e990b64 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:02:40 +0000 Subject: [PATCH 106/791] fix(session): require an open target for session-bound native and artifact tools binary_session reported native and artifact operations available whenever their provider declared them, although these operations run on the active target: in a closed session every call failed with target_unavailable, and Mach-O-only operations stayed available with a ZIP or ELF target open. Report target_required for every session-bound contract kind in a closed session, and target_unsupported for Mach-O-only operations unless the open target is a Mach-O executable. A host limitation is reported before target selection, since opening a target cannot fix it; composed operations keep deciding from their requirements. Fixes #1166 Co-Authored-By: Claude Opus 5.5 --- src/application/CapabilityInventory.test.ts | 125 ++++++++++++++++++++ src/application/CapabilityInventory.ts | 61 ++++++++-- tests/boundary/mcp/serverIdentity.test.ts | 117 +++++++++++++++++- 3 files changed, 288 insertions(+), 15 deletions(-) diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 58d7d4e2cb2..f235c740ce4 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -48,6 +48,7 @@ const status = ( options: { readonly open?: boolean; readonly kind?: "executable" | "database" | "archive" | "artifact"; + readonly format?: string; readonly capabilities?: readonly { readonly operation: string; readonly available: boolean; @@ -58,6 +59,7 @@ const status = ( ) => ({ open: options.open ?? false, ...(options.kind === undefined ? {} : { kind: options.kind }), + ...(options.format === undefined ? {} : { format: options.format }), capabilities: (options.capabilities ?? []).map((capability) => ({ ...capability, availability_code: capability.available @@ -237,6 +239,129 @@ describe("capability inventory: provider status", () => { }); }); +describe("capability inventory: active-target operations", () => { + const ACTIVE_TARGET_TOOLS = [ + "inspect_macho", + "inspect_signature", + "list_architectures", + "demangle_swift", + "inspect_artifact", + "extract_artifact", + ] as const; + const declared = ( + names: readonly string[], + availability: { + readonly available: boolean; + readonly reason: string | null; + } = { + available: true, + reason: null, + }, + ) => names.map((operation) => ({ operation, ...availability })); + const reasons = ( + sessionStatus: ReturnType, + names: readonly string[], + policy = enabledPolicy, + ) => + Object.fromEntries( + names.map((name) => [name, entry(name, sessionStatus, policy).reason]), + ); + + it("requires an open target even when the provider declares the operation", () => { + const closed = status({ capabilities: declared(ACTIVE_TARGET_TOOLS) }); + for (const name of ACTIVE_TARGET_TOOLS) + expect(entry(name, closed)).toMatchObject({ + available: false, + reason: "target_required", + remediation: expect.stringContaining("open_binary"), + }); + }); + + it("keeps explicit-path operations available without a target", () => { + const explicitPath = [ + "inspect_managed_artifact", + "inspect_binary_layout", + "inspect_recorded_crash", + "inspect_evm_interface", + ]; + expect( + reasons(status(), explicitPath, { + ...enabledPolicy, + binaryLayoutEnabled: true, + recordedCrashEnabled: true, + evmInterfaceEnabled: true, + }), + ).toEqual( + Object.fromEntries(explicitPath.map((name) => [name, "available"])), + ); + }); + + it("reserves Mach-O operations for an open Mach-O executable", () => { + const capabilities = declared(ACTIVE_TARGET_TOOLS); + const machO = status({ + open: true, + kind: "executable", + format: "mach-o", + capabilities, + }); + const zip = status({ + open: true, + kind: "archive", + format: "zip", + capabilities, + }); + const elf = status({ + open: true, + kind: "executable", + format: "elf", + capabilities, + }); + expect(reasons(machO, ACTIVE_TARGET_TOOLS)).toEqual( + Object.fromEntries( + ACTIVE_TARGET_TOOLS.map((name) => [name, "available"]), + ), + ); + for (const target of [zip, elf]) + expect(reasons(target, ACTIVE_TARGET_TOOLS)).toEqual({ + inspect_macho: "target_unsupported", + inspect_signature: "available", + list_architectures: "target_unsupported", + demangle_swift: "available", + inspect_artifact: "available", + extract_artifact: "available", + }); + expect(entry("inspect_macho", zip).remediation).toContain("Mach-O"); + }); + + it("reports an unsupported host before asking for a target", () => { + const capabilities = declared(["inspect_macho"], { + available: false, + reason: "Native macOS utilities require macOS.", + }).map((capability) => ({ + ...capability, + availability_code: "unsupported_host" as const, + })); + for (const sessionStatus of [ + status({ capabilities }), + status({ open: true, kind: "archive", format: "zip", capabilities }), + ]) + expect(entry("inspect_macho", sessionStatus)).toMatchObject({ + reason: "unsupported_host", + remediation: "Native macOS utilities require macOS.", + }); + const composed = capabilities.map((capability) => ({ + ...capability, + operation: "inspect_native_dispatch_metadata", + })); + expect( + entry( + "inspect_native_dispatch_metadata", + status({ capabilities: composed }), + ).reason, + ).toBe("target_required"); + }); +}); + describe("capability inventory: caller guidance", () => { it("gives every unavailable public tool an actionable remediation", () => { const inventory = buildCapabilityInventory(status(), { diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 4f85c9aeb39..56765f005de 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -78,6 +78,7 @@ type AvailabilityContext = { | "archive" | "artifact" | undefined; + readonly targetFormat: string | undefined; readonly descriptors: ReadonlyMap; readonly policy: AvailabilityPolicy; }; @@ -112,6 +113,23 @@ const ENHANCED_REQUIREMENTS: Readonly> = { ], }; +/** Contract kinds whose operations run on the active session target. */ +const ACTIVE_TARGET_KINDS: ReadonlySet = new Set([ + "official-proxy", + "enhanced", + "native-provider", + "artifact-provider", +]); + +/** Native contracts defined only for an active Mach-O executable. */ +const MACHO_TARGET_OPERATIONS: ReadonlySet = new Set([ + "inspect_macho", + "list_architectures", + "observe_native_calls", + "observe_native_ui", + "capture_native_ui_scenario", +]); + const NAVIGATION_CONTEXT_MODES = [ { name: "current_selection", @@ -143,6 +161,7 @@ export const buildCapabilityInventory = ( kind: contract.kind, targetOpen: status.open, targetKind: status.kind, + targetFormat: status.format, descriptors, policy, }); @@ -211,6 +230,8 @@ const availabilityFor = (context: AvailabilityContext): Availability => { }; const workflowAvailability = workflowAvailabilityFor(context); if (workflowAvailability !== null) return workflowAvailability; + const hostDecision = hostAvailability(context); + if (hostDecision !== null) return hostDecision; const targetDecision = targetAvailability(context); if (targetDecision !== null) return targetDecision; return providerAvailability(context); @@ -401,12 +422,32 @@ const browserProviderAvailability = ( }; }; +/** + * Opening another target cannot make an operation run on an unsupported host. + * Composed operations are decided by their requirements, which another + * provider may satisfy. + */ +const hostAvailability = ({ + descriptors, + kind, + name, +}: AvailabilityContext): Availability | null => { + if (kind === "enhanced") return null; + const descriptor = descriptors.get(name); + return descriptor?.available === false && + descriptor.availability_code === "unsupported_host" + ? { reason: "unsupported_host", remediation: descriptor.reason } + : null; +}; + const targetAvailability = ({ + name, kind, targetKind, + targetFormat, targetOpen, }: AvailabilityContext): Availability | null => { - if (!targetOpen && (kind === "official-proxy" || kind === "enhanced")) + if (!targetOpen && ACTIVE_TARGET_KINDS.has(kind)) return { reason: "target_required", remediation: "Call open_binary with a supported local target.", @@ -422,6 +463,16 @@ const targetAvailability = ({ remediation: "Inventory or extract a native executable, then call open_binary on that executable.", }; + if ( + targetOpen && + MACHO_TARGET_OPERATIONS.has(name) && + (targetKind !== "executable" || targetFormat !== "mach-o") + ) + return { + reason: "target_unsupported", + remediation: + "Call open_binary with a Mach-O executable or an app bundle whose main executable is Mach-O.", + }; return null; }; @@ -445,14 +496,6 @@ const providerAvailability = ({ remediation: "Install or configure a provider that declares this operation.", }; - if ( - !descriptor.available && - descriptor.availability_code === "unsupported_host" - ) - return { - reason: "unsupported_host", - remediation: descriptor.reason, - }; return descriptor.available ? { reason: "available", remediation: null } : { diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 2305a346e2b..12c1e67a519 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -1,9 +1,13 @@ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { describe, expect, it } from "vitest"; +import { z } from "zod"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { machoImage } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { CATALOG_IDENTITY, CLI_COMMAND_NAMES, @@ -21,11 +25,17 @@ import type { CapabilityDescriptor, } from "../../../src/application/AnalysisProvider.js"; -const availabilityProvider = (): AnalysisProvider => { +const availabilityProvider = ( + operations: readonly CapabilityDescriptor["operation"][] = [ + "current_address", + ], +): AnalysisProvider => { const identity = { id: "fixture", name: "Fixture", version: "1" }; - const capability: CapabilityDescriptor = { + const capability = ( + operation: CapabilityDescriptor["operation"], + ): CapabilityDescriptor => ({ provider: identity, - operation: "current_address", + operation, available: true, reason: null, effects: { @@ -38,10 +48,10 @@ const availabilityProvider = (): AnalysisProvider => { requiresRoot: false, }, limitations: [], - }; + }); return { identity: () => identity, - capabilities: () => [capability], + capabilities: () => operations.map(capability), createClient: () => ({ health: () => Promise.resolve(), execute: () => Promise.resolve(observed(null)), @@ -248,6 +258,101 @@ describe("live server identity over MCP", () => { }, 10_000); }); +describe("active-target availability over MCP", () => { + it("advertises active-target tools only when a matching target is open", async () => { + const session = createTestBinarySession( + availabilityProvider(["inspect_macho", "inspect_artifact"]), + ); + const server = createServer(session, session); + const client = new Client({ name: "availability-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + const directory = await createTestTempDirectory("rea-target-availability-"); + const executable = join(directory, "tool"); + const plist = join(directory, "Info.plist"); + await writeFile(executable, machoImage({})); + await writeFile(plist, PLIST_XML); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const listed = await toolNames(client); + expect(await activeTargetReasons(client)).toEqual({ + inspect_macho: "target_required", + inspect_artifact: "target_required", + inspect_managed_artifact: "available", + }); + const call = await client.callTool({ + name: "inspect_macho", + arguments: {}, + }); + expect(call.structuredContent).toMatchObject({ + error: { code: "target_unavailable" }, + }); + await openTarget(client, plist); + expect(await activeTargetReasons(client)).toEqual({ + inspect_macho: "target_unsupported", + inspect_artifact: "available", + inspect_managed_artifact: "available", + }); + await client.callTool({ name: "close_binary", arguments: {} }); + await openTarget(client, executable); + expect(await activeTargetReasons(client)).toEqual({ + inspect_macho: "available", + inspect_artifact: "available", + inspect_managed_artifact: "available", + }); + await client.callTool({ name: "close_binary", arguments: {} }); + expect((await activeTargetReasons(client)).inspect_macho).toBe( + "target_required", + ); + expect(await toolNames(client)).toEqual(listed); + } finally { + await Promise.allSettled([ + client.close(), + server.close(), + session.close(), + ]); + } + }, 10_000); +}); + +const PLIST_XML = + 'CFBundleNameFixture'; + +const toolNames = async (client: Client): Promise => + (await client.listTools()).tools.map(({ name }) => name); + +const openTarget = async (client: Client, path: string): Promise => { + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError ?? false).toBe(false); +}; + +const activeTargetReasons = async ( + client: Client, +): Promise> => { + const status = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + const availability = z + .object({ + result: z.object({ + tool_availability: z.array( + z.object({ name: z.string(), reason: z.string() }).loose(), + ), + }), + }) + .parse(status.structuredContent).result.tool_availability; + return Object.fromEntries( + ["inspect_macho", "inspect_artifact", "inspect_managed_artifact"].map( + (name) => [name, availability.find((tool) => tool.name === name)?.reason], + ), + ); +}; + const assertLiveIdentity = async (client: Client): Promise => { const identity = await client.callTool({ name: "binary_session", From 11b9f0346cf5413045b41ff552a024e11ba46ccc Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 11:06:05 +0800 Subject: [PATCH 107/791] feat(website): explain REA with Calculator and a playable dinosaur lab --- website/README.md | 31 +- website/evidence/calculator-and-dino.md | 132 ++++ website/evidence/dx-ball-sound-pan.md | 6 - website/figures.md | 15 + website/public/assets/dino-demo.js | 235 +++++++ website/public/assets/dino-speed.js | 21 + website/public/assets/home.js | 56 ++ website/public/assets/licenses/dinosaur.txt | 34 + website/public/assets/styles.css | 332 +++++++++ website/public/examples/dino-lab/index.html | 332 +++++++++ website/public/first-investigation/index.html | 2 +- website/public/index.html | 661 +++++++++++++----- website/style-guide.md | 28 +- 13 files changed, 1691 insertions(+), 194 deletions(-) create mode 100644 website/evidence/calculator-and-dino.md create mode 100644 website/public/assets/dino-demo.js create mode 100644 website/public/assets/dino-speed.js create mode 100644 website/public/assets/home.js create mode 100644 website/public/assets/licenses/dinosaur.txt create mode 100644 website/public/examples/dino-lab/index.html diff --git a/website/README.md b/website/README.md index 896dac21390..cbe1512324a 100644 --- a/website/README.md +++ b/website/README.md @@ -6,7 +6,8 @@ An English static website with explanatory figures, worked guides and DX-Ball, Notion, TH04 and CTF investigations. The public files are in `website/public/`. The site uses HTML, CSS and a small -script for copying code and following the assembly-to-C comparison. Python +scripts for copying code, following code comparisons and playing the dinosaur +speed reconstruction. Python packages the downloadable example; there is no frontend bundler or npm dependency. [style-guide.md](style-guide.md) explains the writing, page structure, figures, @@ -25,7 +26,8 @@ Open . Refresh the browser after editing a file. ## Pages -- `public/index.html`: DX-Ball introduction, agent example, setup and case-study overviews. +- `public/index.html`: compact setup, manual/agent comparison, Calculator percentage example, playable dinosaur reconstruction and case-study overviews. +- `public/examples/dino-lab/index.html`: adjustable-speed mini-game, recovered rule, original-game check and browser analysis steps. - `public/first-investigation/index.html`: a guided Notes export investigation, from setup to a checked CSV prediction. - `public/showcase/index.html`: the case-study index. - `public/showcase/dx-ball/index.html`: sound-pan investigation and project status. @@ -95,11 +97,26 @@ Evidence. [evidence/dx-ball-sound-pan.md](evidence/dx-ball-sound-pan.md) records their provenance and the scope of the validation claims. The original executable and complete private Evidence records are not website assets. -The homepage's code comparison, position examples and evidence flow are -selectable HTML. The pan positions 0, 320 and 640 illustrate the inspected -calculation at its recorded initial scale of 1; the C panel is a readable -summary. The linked case study retains the maintained source and its checks. -The agent response is an example explanation based on those findings. +The homepage starts with a short installation prompt and the ordinary setup +command. It compares manual investigation with asking an agent, using the same +Calculator percentage question. The native excerpt comes from an installed +model DLL; Microsoft’s public implementation supplies names and a crosscheck. +The Calculator controls illustrate that rule with generic inputs. + +The dinosaur example connects an inspected running script to a new playable +mini-game. `public/assets/dino-speed.js` owns the recovered speed rule and replay; +`dino-demo.js` imports it for automatic acceleration. The slider chooses a fixed +speed instead. Drawing, jumping and collision code are newly authored for this +teaching game. It starts on input, pauses when hidden or outside the viewport, +and retains a static SVG when JavaScript is disabled. The lab links the speed +module and complete game source, and replays the rule against recorded +original-game values. + +[evidence/calculator-and-dino.md](evidence/calculator-and-dino.md) records both +targets, selected findings, attribution and verification scope. Update the +speed rule, game, check table and evidence together. The original script excerpt +and its BSD license are credited on the pages. Prompt responses are example +explanations based on findings. The general investigation and DX-Ball overview diagrams are maintained as SVG source. Initial layout references were created with the built-in image generation tool; [figures.md](figures.md) retains diff --git a/website/evidence/calculator-and-dino.md b/website/evidence/calculator-and-dino.md new file mode 100644 index 00000000000..67d8160147a --- /dev/null +++ b/website/evidence/calculator-and-dino.md @@ -0,0 +1,132 @@ +# Calculator and dinosaur beginner examples + +Evidence collected on 9 October 2026 with the published REA 4.1.0 package. +The homepage offers compact setup first, then a manual/agent comparison and +two familiar behavior questions. Prompts and replies are examples based on the +findings below. Selected code and generic inputs are public; raw captures stay +outside the website. + +## Windows Calculator + +### Inspected artifacts + +The installed Windows Calculator package is version `11.2508.4.0`, x64. + +| Artifact | SHA-256 | +| --------------------------- | ------------------------------------------------------------------ | +| Windows `calc.exe` launcher | `9c2c8a8588fe6db09c09337e78437cb056cd557db1bcf5240112cbfb7b600efb` | +| Package `CalcViewModel.dll` | `07a1953bbf828a3495662c4b4bde1c80c9075061e0e389eabac5f8c544710df5` | + +REA's function dossier for launcher `0x1400019f4` includes the call to +`ShellExecuteW` with `ms-calculator:`. Evidence: +`ev_e97a6a1a61bfe63476a13e6285d71dd838101066f7863c9af822861050203ea5`. + +The calculation is in the installed app's model DLL. REA inspected function +`0x1801245d0`, whose body occupies two ranges, totaling 10,656 bytes: +`0x1801245d0–0x180126f4c` and `0x180126f55–0x180126f77`. +Evidence: +`ev_830a710c7fb55a08ac896fb5783c14339480a57a96d6d45c57c024b6a43239d0`. + +REA supplied pseudocode, original instructions, calls and references. Its +`case 0x76` checks the operator field against `0x5c` and `0x5b` and constructs +the constant 100 in both paths. One path divides the current value by 100; +the other combines the current value with the previous value divided by 100. +The published assembly is selected exactly from the returned instruction list. + +The binary is stripped. A local instruction scan located candidates using +operator IDs and the constant 100; REA then produced the complete function +dossier. Function selection was analyst-assisted. The illustrative agent +prompt is not a recording of an autonomous end-to-end discovery. + +### Source interpretation and check + +Microsoft's MIT-licensed source at commit +`d125246a4e19842ce1332e6c7839cf0e110027d8` supplies the operator meanings and +an independently readable implementation: + +- [Operator IDs](https://github.com/microsoft/calculator/blob/d125246a4e19842ce1332e6c7839cf0e110027d8/src/CalcManager/Header%20Files/CCommand.h): `IDC_DIV = 91`, `IDC_MUL = 92`, `IDC_PERCENT = 118`. +- [Percentage branch](https://github.com/microsoft/calculator/blob/d125246a4e19842ce1332e6c7839cf0e110027d8/src/CalcManager/CEngine/scifunc.cpp#L98-L111): multiply/divide uses `rat / 100`; other operators use `rat * (m_lastVal / 100)`. +- [Standard-mode tests](https://github.com/microsoft/calculator/blob/d125246a4e19842ce1332e6c7839cf0e110027d8/src/CalculatorUnitTests/CalculatorManagerTest.cpp#L350-L370): `6 × 6 % = 0.36` and `50 + 20 % = 60`. + +These source branches agree with the operator checks and two paths observed +in the installed DLL. The public source commit is a crosscheck; it is not an +identified build commit for that installed package. No PDB symbols were loaded. + +The homepage's named C-like rule is an explanatory summary. Its `200 + 10%` +and `200 × 10%` displays are derived from that rule. Browser checks exercise +both percentage illustrations, their results, operator selection and reset. +The Microsoft tests were inspected, not executed. A probe-owned Calculator +window could be inspected, but it was no longer present at the input-check +step; no completed Windows GUI input result is claimed. + +## Dinosaur speed + +### Running target and source + +REA inspected the HTTP page + in an owned Chrome 150.0.7871.124 +session through a loopback debugging connection. This is wayou's +Chromium-derived browser edition. REA's HTTP(S) page inspection does not +select `chrome://dino/`; the website identifies the HTTP edition as its target. + +The returned `index.js` source contains 90,241 UTF-8 bytes, SHA-256 +`e7a50d337bdbe4299068de034e4564cfe5fd45ca9257ded37b6ada9330cedf0f`. +Evidence: +`ev_7a09de570cf6b64907504046485cab69ad444828964106eabc05958799749905`. + +This digest also matches `index.js` in repository commit +`5455bfa408ec6b707c7300ff194b7390733a766d`. Source locations: + +- [Settings](https://github.com/wayou/t-rex-runner/blob/5455bfa408ec6b707c7300ff194b7390733a766d/index.js#L106-L135): `ACCELERATION: 0.001`, `MAX_SPEED: 13`, `SPEED: 6`. +- [Update branch](https://github.com/wayou/t-rex-runner/blob/5455bfa408ec6b707c7300ff194b7390733a766d/index.js#L565-L571): increment speed on a collision-free update while speed is below the threshold. + +The source begins with the Chromium Authors' 2014 copyright notice. Selected +excerpts are credited on the page. The repository's BSD license and the source +copyright are reproduced in `public/assets/licenses/dinosaur.txt`. + +### Controlled original-game check + +After the passive REA inspection, a separate Playwright controller called the +original page's `Runner.update()` repeatedly. It disabled future update +scheduling and obstacle generation, set the starting speed to 6, and kept the +game in its playing state. The control is credited separately from REA's +read-only inspection. + +| Update count | Observed speed | Display rounded to 3 decimals | +| ------------ | ------------------ | ----------------------------- | +| 0 | 6 | 6.000 | +| 1,000 | 7.000000000000334 | 7.000 | +| 4,000 | 9.99999999999956 | 10.000 | +| 7,000 | 12.999999999997897 | 13.000 | +| 10,000 | 13.000999999997896 | 13.001 | + +The branch compares before adding. Floating-point increments can cross the +threshold by one increment; the reconstruction preserves this behavior rather +than introducing a clamp. + +### New game and reader experiment + +`public/assets/dino-speed.js` contains the reconstructed rule and an update-count +replay. Its rounded results match the original-game check above. The website's +mini-game imports this rule. It advances its own simulation at 60 updates per +second, avoiding a different frame rate changing its demonstration pace. + +The canvas drawing, dinosaur shape, jump physics, rectangle collision detection, +spacing and controls are newly authored teaching code. The reconstruction claim +is scoped to the speed rule. Changing the slider selects a fixed speed between +2 and 18; selecting automatic acceleration restores the starting speed of 6. + +The new game starts only after user input. It supports Play/Pause, Jump, Restart, +canvas Space/Up keys and touch. It pauses when hidden or scrolled out of view. +Without JavaScript, a static SVG, source excerpts and recorded results remain +available. The downloadable speed module and complete game source are linked +from the lab. + +## Review checks + +Website checks cover root and `/rea/` hosting, 320/390/850/1440px layouts, +expanded evidence, copying, no-JavaScript content, reduced motion, the two +percentage rules and the game controls. Game checks include speed selection, +automatic acceleration, a collision, restart, jump height on the rendered +canvas, pause and the original-rule replay. Existing Notes and DX-Ball pages +remain part of layout checks. diff --git a/website/evidence/dx-ball-sound-pan.md b/website/evidence/dx-ball-sound-pan.md index ec02d2705e7..60b7e0e61f9 100644 --- a/website/evidence/dx-ball-sound-pan.md +++ b/website/evidence/dx-ball-sound-pan.md @@ -96,12 +96,6 @@ signature, declaration and return. ## Validation scope -The homepage reuses these observations in a selected seven-instruction excerpt -and a shortened C summary. Its agent answer is an example explanation, not a -recorded transcript. Its position figure uses illustrative inputs 0, 320 and -640 with the recorded initial scale of 1, producing −500, 0 and 500. The checks -below apply to the maintained project function, not the shortened summary. - The reconstruction project's recorded checks establish: - 3,205 direct original-x86 differential cases: integer positions 0–640 at diff --git a/website/figures.md b/website/figures.md index ff8f7bba323..c94db54f952 100644 --- a/website/figures.md +++ b/website/figures.md @@ -42,6 +42,21 @@ branches when changing these figures. Match progress claims to the pinned reconstruction checkpoint. Validate the SVG as XML and inspect its browser rendering at desktop and mobile widths. +## Beginner homepage and dinosaur reconstruction + +The Calculator role comparison and its percentage controls are semantic HTML. +The assembly comes from a real installed model DLL; the named calculation is +a readable summary crosschecked against Microsoft’s public source. See +[evidence/calculator-and-dino.md](evidence/calculator-and-dino.md). + +The dinosaur sequence uses an HTML flow: inspect the loaded script, rebuild the +speed rule, then change and play. Its canvas mini-game and static SVG fallback +are authored code, without generated images or copied game sprites. The game +imports the recovered rule from `public/assets/dino-speed.js`; other mechanics +are new teaching code. Check that the fallback is visible with JavaScript +disabled and hidden when the playable canvas loads. Keep the speed settings, +source excerpt, reconstruction and original-result table consistent. + ## Original image generation prompts ## Prompt 1 diff --git a/website/public/assets/dino-demo.js b/website/public/assets/dino-demo.js new file mode 100644 index 00000000000..9988e9f5292 --- /dev/null +++ b/website/public/assets/dino-demo.js @@ -0,0 +1,235 @@ +import { advanceSpeed, simulateSpeed, SPEED_RULE } from "./dino-speed.js"; + +// A new teaching game around the recovered speed rule. Drawing, jump physics +// and rectangle collision checks are authored for this demo. +function initializeDinoDemo(demo) { + const canvas = demo.querySelector("canvas"); + const context = canvas.getContext("2d"); + if (context === null) return; + const start = demo.querySelector("[data-dino-start]"); + const slider = demo.querySelector("[data-dino-slider]"); + const auto = demo.querySelector("[data-dino-auto]"); + const output = demo.querySelector("[data-dino-speed]"); + const status = demo.querySelector("[data-dino-status]"); + const ground = 148; + const dinosaurX = 52; + const dinosaur = new Path2D( + "M16 18V2H48V24H35V32H25V43H16V53H8V32H0V23H6V29H16Z M12 47V59H5V54H8V47Z M24 42V54H31V59H20V47Z M34 29H44V35H39V32H34Z", + ); + let speed = SPEED_RULE.start; + let height = 0; + let velocity = 0; + let distance = 0; + let obstacles = []; + let running = false; + let crashed = false; + let frame = 0; + let lastTime = 0; + let elapsed = 0; + + const updateSpeedDisplay = () => { + output.value = speed.toFixed(1); + slider.value = speed.toFixed(1); + }; + + const draw = () => { + context.clearRect(0, 0, canvas.width, canvas.height); + context.strokeStyle = "#b7c0c9"; + context.beginPath(); + context.moveTo(0, ground); + context.lineTo(canvas.width, ground); + for (let x = -(distance % 70); x < canvas.width; x += 70) { + context.moveTo(x, ground + 7); + context.lineTo(x + 12, ground + 7); + } + context.stroke(); + context.fillStyle = "#5b6570"; + for (const obstacle of obstacles) { + context.fillRect( + obstacle.x + 6, + ground - obstacle.height, + 8, + obstacle.height, + ); + context.fillRect(obstacle.x, ground - obstacle.height + 12, 6, 7); + context.fillRect(obstacle.x, ground - obstacle.height + 3, 5, 15); + context.fillRect(obstacle.x + 14, ground - obstacle.height + 19, 7, 6); + context.fillRect(obstacle.x + 17, ground - obstacle.height + 8, 4, 17); + } + context.save(); + context.translate(dinosaurX, ground - 59 - height); + context.fillStyle = crashed ? "#8c4b40" : "#294f82"; + context.fill(dinosaur); + context.fillStyle = "#ffffff"; + context.fillRect(39, 7, 4, 4); + context.restore(); + context.fillStyle = "#5b6570"; + context.font = "13px monospace"; + context.textAlign = "right"; + context.fillText( + `DISTANCE ${String(Math.floor(distance / 100)).padStart(4, "0")}`, + 580, + 28, + ); + if (!running) { + context.textAlign = "center"; + context.font = "15px system-ui"; + context.fillText( + crashed ? "Hit a cactus. Try another speed." : "Space or Jump ↑", + 330, + 85, + ); + } + }; + + const stop = (message) => { + running = false; + cancelAnimationFrame(frame); + start.textContent = crashed ? "Play again" : "Play"; + if (message) status.textContent = message; + draw(); + }; + + const reset = () => { + stop(); + crashed = false; + height = 0; + velocity = 0; + distance = 0; + obstacles = [{ x: 550, height: 38 }]; + if (auto.checked) speed = SPEED_RULE.start; + elapsed = 0; + start.textContent = "Play"; + status.textContent = "Press Play, then jump over the cactus."; + updateSpeedDisplay(); + draw(); + }; + + const update = () => { + if (auto.checked) speed = advanceSpeed(speed); + distance += speed; + if (height > 0 || velocity > 0) { + height = Math.max(0, height + velocity); + velocity -= 0.55; + if (height === 0) velocity = 0; + } + obstacles.forEach((obstacle) => { + obstacle.x -= speed; + }); + obstacles = obstacles.filter((obstacle) => obstacle.x > -25); + const last = obstacles[obstacles.length - 1]; + const gap = 400 + speed * 18; + if (last === undefined || last.x < 600 - gap) { + obstacles.push({ x: 600, height: 38 }); + } + const left = dinosaurX + 8; + const right = dinosaurX + 39; + const bottom = ground - height; + if ( + obstacles.some( + (obstacle) => + right > obstacle.x && + left < obstacle.x + 21 && + bottom > ground - obstacle.height + 4, + ) + ) { + crashed = true; + stop("Hit a cactus. Press Play again, or change the speed and restart."); + } + }; + + const animate = (time) => { + if (!running) return; + elapsed += Math.min(time - lastTime, 100); + lastTime = time; + while (elapsed >= 1000 / 60 && running) { + elapsed -= 1000 / 60; + update(); + } + updateSpeedDisplay(); + draw(); + if (running) frame = requestAnimationFrame(animate); + }; + + const play = () => { + if (crashed) reset(); + if (running) return; + running = true; + lastTime = performance.now(); + elapsed = 0; + start.textContent = "Pause"; + status.textContent = "Running. Space or Jump clears a cactus."; + frame = requestAnimationFrame(animate); + }; + + const jump = () => { + if (crashed) reset(); + play(); + if (height === 0) velocity = 10.5; + }; + + start.addEventListener("click", () => { + if (running) stop("Paused. Press Play to continue."); + else play(); + }); + demo.querySelector("[data-dino-jump]").addEventListener("click", jump); + demo.querySelector("[data-dino-reset]").addEventListener("click", reset); + canvas.addEventListener("keydown", (event) => { + if (event.code === "Space" || event.code === "ArrowUp") { + event.preventDefault(); + if (!event.repeat) jump(); + } + }); + canvas.addEventListener("pointerdown", () => { + canvas.focus(); + jump(); + }); + slider.addEventListener("input", () => { + auto.checked = false; + speed = Number(slider.value); + updateSpeedDisplay(); + status.textContent = running + ? `Fixed speed: ${speed.toFixed(1)}. Space or Jump clears a cactus.` + : `Fixed speed: ${speed.toFixed(1)}. Press Play to try it.`; + draw(); + }); + auto.addEventListener("change", () => { + if (auto.checked) speed = SPEED_RULE.start; + updateSpeedDisplay(); + status.textContent = auto.checked + ? "Recovered rule: start at 6, then increase toward 13." + : `Fixed speed: ${speed.toFixed(1)}.`; + }); + document.addEventListener("visibilitychange", () => { + if (document.hidden && running) stop("Paused. Press Play to continue."); + }); + if ("IntersectionObserver" in window) { + new IntersectionObserver((entries) => { + if (!entries[0].isIntersecting && running) + stop("Paused. Press Play to continue."); + }).observe(demo); + } + demo.querySelector(".dino-fallback").setAttribute("hidden", ""); + demo.querySelector(".dino-controls").hidden = false; + canvas.hidden = false; + reset(); +} + +document.querySelectorAll("[data-dino-demo]").forEach(initializeDinoDemo); + +const speedCheck = document.querySelector("[data-speed-check]"); +if (speedCheck !== null) speedCheck.hidden = false; +speedCheck?.addEventListener("click", () => { + let matches = true; + document.querySelectorAll("[data-speed-updates]").forEach((cell) => { + const speed = simulateSpeed( + Number(cell.getAttribute("data-speed-updates")), + ).toFixed(3); + cell.textContent = speed; + matches = + matches && speed === cell.parentElement.children[1].textContent.trim(); + }); + document.querySelector("[data-speed-check-status]").textContent = matches + ? "Replayed the recovered rule. All four values match the controlled original-game check when rounded to three decimals." + : "The replay differs from the recorded original-game values. Compare the speed rule and settings above."; +}); diff --git a/website/public/assets/dino-speed.js b/website/public/assets/dino-speed.js new file mode 100644 index 00000000000..df90fcfa4d0 --- /dev/null +++ b/website/public/assets/dino-speed.js @@ -0,0 +1,21 @@ +// Speed rule recovered from wayou's Chromium-derived browser edition. +// Source and controlled checks: website/evidence/calculator-and-dino.md. +export const SPEED_RULE = Object.freeze({ + start: 6, + acceleration: 0.001, + threshold: 13, +}); + +/** Advance one collision-free update using the inspected speed rule. */ +export function advanceSpeed(speed) { + return speed < SPEED_RULE.threshold ? speed + SPEED_RULE.acceleration : speed; +} + +/** Replay the recovered rule for a chosen number of updates. */ +export function simulateSpeed(updates) { + let speed = SPEED_RULE.start; + for (let update = 0; update < updates; update += 1) { + speed = advanceSpeed(speed); + } + return speed; +} diff --git a/website/public/assets/home.js b/website/public/assets/home.js new file mode 100644 index 00000000000..61f22231bde --- /dev/null +++ b/website/public/assets/home.js @@ -0,0 +1,56 @@ +function initializePercentDemo() { + const demo = document.querySelector("[data-percent-demo]"); + if (demo === null) return; + const display = demo.querySelector("[data-percent-display]"); + const expression = demo.querySelector("[data-percent-expression]"); + const status = demo.querySelector("[data-percent-status]"); + const equals = demo.querySelector('[data-percent-key="equals"]'); + const operations = demo.querySelectorAll("[data-percent-operation]"); + let operation = "add"; + let stage = "input"; + + const render = () => { + const adding = operation === "add"; + const percent = adding ? 20 : 0.1; + const result = adding ? 220 : 20; + const operator = adding ? "+" : "×"; + expression.textContent = + stage === "result" ? `200 ${operator} ${percent} =` : `200 ${operator}`; + display.textContent = String( + stage === "input" ? 10 : stage === "percent" ? percent : result, + ); + equals.disabled = stage !== "percent"; + status.textContent = + stage === "input" + ? "Press % to see what happens to 10." + : stage === "percent" + ? adding + ? "10% of 200 is 20. Press = to add it." + : "For multiplication, 10 becomes 0.1. Press = to multiply." + : `200 ${operator} ${percent} = ${result}.`; + }; + + operations.forEach((button) => { + button.addEventListener("click", () => { + operation = button.getAttribute("data-percent-operation"); + stage = "input"; + operations.forEach((candidate) => { + candidate.setAttribute("aria-pressed", String(candidate === button)); + }); + render(); + }); + }); + demo.querySelectorAll("[data-percent-key]").forEach((button) => { + button.addEventListener("click", () => { + const key = button.getAttribute("data-percent-key"); + stage = + key === "reset" ? "input" : key === "percent" ? "percent" : "result"; + render(); + }); + }); + demo.querySelector(".percent-operation").hidden = false; + demo.querySelector(".percent-controls").hidden = false; + render(); +} + +initializePercentDemo(); diff --git a/website/public/assets/licenses/dinosaur.txt b/website/public/assets/licenses/dinosaur.txt new file mode 100644 index 00000000000..88ec041e776 --- /dev/null +++ b/website/public/assets/licenses/dinosaur.txt @@ -0,0 +1,34 @@ +Selected script excerpts from wayou/t-rex-runner +Copyright (c) 2014 The Chromium Authors. All rights reserved. + +Repository license at commit 5455bfa408ec6b707c7300ff194b7390733a766d: + +BSD 3-Clause License + +Copyright (c) 2022, 牛さん +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index 9a49919cc0d..54bf52f6753 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -1218,6 +1218,338 @@ pre code { border: 0; } +.home-start-grid { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 3.5rem; + align-items: center; +} +.home-start-grid h1 { + font-size: clamp(2.8rem, 5vw, 4.2rem); +} +.home-start-grid .home-setup h2 { + font-size: 1.3rem; + letter-spacing: -0.02em; + margin-bottom: 0.7rem; +} +.home-start-grid .home-setup > p { + font-size: 0.9rem; + color: var(--muted); +} +.home-question { + margin-block: 0 1.8rem; + color: var(--muted); +} +.home-question strong { + color: var(--ink); + font-weight: 500; +} +.home-role-comparison { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); + gap: 0 2.5rem; + margin: 0; +} +.comparison-label { + padding-bottom: 0.8rem; + border-bottom: 1px solid var(--line); + font-size: 0.875rem; + font-weight: 600; +} +.agent-investigation > .comparison-label { + color: var(--accent); +} +.manual-steps { + display: grid; + gap: 1rem; + padding: 0; + list-style: none; +} +.manual-steps li { + display: flex; + gap: 1rem; + margin: 0; +} +.manual-steps li > span { + padding-top: 0.2rem; + color: var(--muted); + font: + 0.75rem "SFMono-Regular", + Consolas, + "Liberation Mono", + monospace; +} +.manual-steps strong, +.manual-steps small { + display: block; +} +.manual-steps strong { + font-size: 0.95rem; + font-weight: 500; +} +.manual-steps small { + margin-top: 0.15rem; + color: var(--muted); + font-size: 0.8rem; +} +.manual-code { + margin-top: 1.3rem; + padding: 1rem 1.25rem; + border-left: 2px solid var(--line); + background: var(--soft); + color: var(--muted); + font-size: 0.8rem; + line-height: 1.65; +} +.agent-investigation-note { + color: var(--muted); + font-size: 0.875rem; +} +.home-role-comparison figcaption { + grid-column: 1 / -1; + margin-top: 1.2rem; + font-size: 0.8rem; + color: var(--muted); +} +.home-example-grid { + display: grid; + grid-template-columns: minmax(0, 0.85fr) minmax(0, 1.15fr); + align-items: start; + gap: 3rem; +} +.percent-figure, +.dino-demo { + min-width: 0; + margin: 0; + padding: 1.5rem; + border: 1px solid var(--line); + border-radius: 5px; +} +.percent-operation, +.percent-controls, +.dino-buttons { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; +} +.percent-operation button, +.percent-controls button, +.dino-buttons button { + min-height: 44px; + border: 1px solid var(--line); + border-radius: 4px; + padding: 0.5rem 0.8rem; + font: inherit; + font-size: 0.85rem; + color: var(--ink); + background: var(--page); + cursor: pointer; +} +.percent-operation button[aria-pressed="true"] { + color: var(--accent); + border-color: var(--accent); + background: #edf2f8; +} +.percent-display { + padding: 1.2rem; + margin-block: 1rem; + border-radius: 4px; + background: var(--soft); + text-align: right; + font-variant-numeric: tabular-nums; +} +.percent-expression { + display: block; + min-height: 1.5rem; + color: var(--muted); + font-size: 0.9rem; +} +.percent-display strong { + font-size: 3rem; + font-weight: 500; + letter-spacing: -0.04em; +} +.percent-controls .percent-key { + min-width: 68px; + font-size: 1.25rem; + color: var(--accent); + border-color: #bbcce0; +} +.percent-controls button:disabled { + opacity: 0.45; + cursor: default; +} +.percent-controls .percent-reset { + margin-left: auto; + border-color: transparent; + color: var(--muted); +} +.percent-status { + min-height: 3rem; + margin-top: 1rem; + font-size: 0.9rem; +} +.percent-figure figcaption, +.dino-demo figcaption { + margin-top: 0.8rem; + color: var(--muted); + font-size: 0.8rem; +} +.percent-explanation h3 { + font-size: 1.2rem; +} +.percent-explanation > p { + color: var(--muted); +} +.percent-rules { + margin-bottom: 0; +} +.percent-rules > div { + padding-block: 0.55rem; +} +.percent-rules dt { + font-size: 0.8rem; + color: var(--muted); + margin-bottom: 0.25rem; +} +.percent-rules dd { + margin: 0; + font-size: 0.9rem; +} +.home-evidence-details { + margin-top: 2rem; + padding-block: 1rem; + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); +} +.home-evidence-details > summary { + cursor: pointer; + font-size: 0.875rem; + font-weight: 500; +} +.home-evidence-details[open] > summary { + margin-bottom: 1.5rem; +} +.calculator-trace { + margin-block: 1.5rem; +} +.dino-example-grid { + grid-template-columns: minmax(0, 1fr) minmax(0, 1.15fr); + gap: 2.5rem; +} +.dino-stage { + overflow: hidden; + margin-block: 0.8rem; + background: var(--soft); + border-radius: 4px; + color: var(--muted); +} +.dino-stage canvas, +.dino-fallback { + display: block; + width: 100%; + height: auto; +} +.dino-stage canvas { + touch-action: manipulation; +} +.dino-buttons .button { + background: var(--ink); + border-color: var(--ink); + color: white; +} +.dino-controls { + display: grid; + gap: 1.2rem; +} +.dino-speed-label { + display: grid; + grid-template-columns: 1fr auto; + gap: 0.5rem; + font-size: 0.875rem; +} +.dino-speed-label output { + font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace; + font-variant-numeric: tabular-nums; + color: var(--accent); +} +.dino-speed-label input { + grid-column: 1 / -1; + width: 100%; + margin: 0; + min-height: 30px; + accent-color: var(--accent); +} +.dino-auto-label { + display: flex; + align-items: center; + gap: 0.6rem; + min-height: 36px; + font-size: 0.8rem; + color: var(--muted); +} +.dino-auto-label input { + width: 17px; + height: 17px; + flex-shrink: 0; + accent-color: var(--accent); +} +.dino-status { + min-height: 2.6rem; + margin: 1rem 0 0; + font-size: 0.8rem; + color: var(--muted); +} +.dino-rule { + color: var(--muted); + font-size: 0.95rem; +} +.dino-rule strong { + color: var(--ink); +} +.dino-loop { + margin-top: 2rem; +} +.dino-lab-demo { + margin-block: 2rem; +} +.dino-lab-demo .dino-stage { + margin-block: 1.5rem; +} +.speed-check-table td:last-child, +.speed-check-table th:last-child { + text-align: right; +} +[hidden] { + display: none !important; +} +@media (max-width: 850px) { + .home-start-grid, + .home-role-comparison, + .home-example-grid { + grid-template-columns: minmax(0, 1fr); + gap: 2rem; + } + .home-start-grid .home-setup { + max-width: 620px; + } + .home-start-grid h1 { + font-size: clamp(2.7rem, 9vw, 4.65rem); + } + .home-role-comparison figcaption { + margin-top: 0; + } +} +@media (max-width: 640px) { + .home-hero { + padding-block: 2.8rem; + } + .percent-figure, + .dino-demo { + padding: 1.1rem; + } +} + @media (max-width: 850px) { .home-agent-grid, .home-try-grid { diff --git a/website/public/examples/dino-lab/index.html b/website/public/examples/dino-lab/index.html new file mode 100644 index 00000000000..a0578a48772 --- /dev/null +++ b/website/public/examples/dino-lab/index.html @@ -0,0 +1,332 @@ + + + + + + Dinosaur speed lab — REA + + + + + + + + + +

+
+
+ +

Inspect → rebuild → change

+

Rebuild the dinosaur’s speed rule.

+

+ REA found how the running game increases its speed. This small + reconstruction lets you keep that rule or choose a new speed and play. +

+
+
+ +
+
+

Play and change the speed

+

+ Press Play, then jump over the cactus. Move the speed slider to + try a slower or faster game; it switches to a fixed speed. Select + the recovered rule to start at 6 and gradually accelerate again. +

+
+

Play the reconstruction

+
+ + + + 6 → 10 → 13 + + + +
+ +

+ Original rule: start at 6, then increase toward 13. +

+
+ Move the slider to set your own speed. Space or Jump clears a + cactus. +
+ +
+
+
+

What REA found

+

+ We inspected + wayou’s Chromium-derived browser edition + with REA. It returned the loaded index.js, including + this branch in the game’s update function: +

+
+
+ REA · selected script source +
+
if (this.currentSpeed < this.config.MAX_SPEED) {
+  this.currentSpeed += this.config.ACCELERATION;
+}
+
+

+ The settings are SPEED: 6, + ACCELERATION: 0.001 and MAX_SPEED: 13. + The game runs this branch on an update without a collision. +

+

Use the same rule in the new game

+
+
+ Reconstruction · dino-speed.js +
+
function advanceSpeed(speed) {
+  return speed < 13 ? speed + 0.001 : speed;
+}
+
+

+ The mini-game uses this function for automatic acceleration. + Moving the slider supplies a fixed speed instead. Drawing, jump + physics and rectangle collision checks are new code for this + teaching game; its simulation advances at 60 updates per second. +

+

+ Download the speed rule + · Read the game code +

+
+
+

Check the reconstruction

+

+ We called the original game’s update function with obstacles and + automatic scheduling disabled. Compare those recorded speeds with + a replay of the recovered rule. +

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
UpdatesOriginal gameReconstructed rule
06.0006.000
1,0007.0007.000
4,00010.00010.000
10,00013.00113.001
+
+ +

+ Values rounded to three decimals. The branch checks before adding, + so the final increment can take speed just above 13. +

+
+
+

Try the analysis yourself

+
    +
  1. + Set up REA with your coding + agent. +
  2. +
  3. + Follow the + browser connection steps, using https://wayou.github.io/t-rex-runner/ as + the page URL. +
  4. +
  5. + Give your agent the page URL and local debugging endpoint, then + copy this prompt. +
  6. +
+
+
+ Your coding agent +
+

+ Use REA to inspect this dinosaur game. Why does it get + faster? Show the speed rule, then build a small version with + an adjustable speed. +

+
+

+ Ask a follow-up: + “Where is jump height calculated? Add a jump-height control to + the new version.” +

+

+ Source: + inspected speed code, Chromium Authors. + Copyright and BSD license. REA 4.1.0 inspected the running HTTP edition in Chrome 150. +

+
+
+
+
+ +

+ Top + + diff --git a/website/public/first-investigation/index.html b/website/public/first-investigation/index.html index ed687aa70d6..3c4828cd222 100644 --- a/website/public/first-investigation/index.html +++ b/website/public/first-investigation/index.html @@ -336,7 +336,7 @@

5. Ask another question

How DX-Ball calculates sound panningContinue from the homepage into the original instructions and + >Read the original instructions and the reconstruction’s checks. diff --git a/website/public/index.html b/website/public/index.html index c04b4c6b25e..747e1715b5e 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -12,6 +12,8 @@ + + @@ -51,15 +53,63 @@
-

Reverse engineering, with your coding agent

-

Find out how
software works.

-

- REA gives your agent the tools to inspect a program and explain what - it does. -

-
- See how it works - Try an example +
+
+

Reverse engineering, with your coding agent

+

Find out how
software works.

+

+ REA gives your agent the tools to inspect a program and explain + what it does. +

+ +
+
+

Set up REA

+

Copy this into your coding agent:

+
+
+ Your coding agent + +
+

+ Install REA and connect it to this coding agent using npx + rea-agents@latest setup. Show me the setup plan for approval, + then verify the installation. +

+
+

Or run this in your terminal:

+
+ npx rea-agents@latest setup + +
+

+ Approve the setup plan, then restart your agent. +

+
@@ -71,228 +121,497 @@

What is reverse engineering?

Finding out how software works by examining the program.

-
-

- DX-Ball: how does the game place a sound on the left or right? +

+ For example: + why does Calculator turn 200 + 10% into + 220? +

+
+
+

Before REA · You investigate

+
    +
  1. + 01 +
    + Find the code behind the button.Which executable or library handles %? +
    +
  2. +
  3. + 02 +
    + Read the instructions.What do these branches and calls mean? +
    +
  4. +
  5. + 03 +
    + Recover the calculation.Follow the inputs, operators and constants. +
    +
  6. +
  7. + 04 +
    + Check your explanation.Does it agree with the program? +
    +
  8. +
+
+ CMP EAX, 0x5c
JZ 0x180124aba
CMP EAX, 0x5b
JZ + 0x180124aba
MOV EDX, 0x64
+
+
+
+

With REA · You ask your agent

+
+
+ Example prompt · Calculator + +
+

+ Use REA to inspect Windows Calculator. Why does 200 + 10% + give 220? Show the code behind the % button. +

+
+

+ Example explanation from the findings +

+

+ With addition, % calculates a percentage of the first + number. +

+

+ 10% of 200 is 20.
Then 200 + 20 = 220.
+

+
+
+

+ Your agent uses REA to locate and read the code, then connects + the findings to your question. +

+
+
+ You describe the behavior. The agent carries out the investigation + with REA. +
+
+
+
+ +
+
+
+

Example 01 · Windows Calculator

+

What does the % button actually do?

+

+ The answer depends on the operator. Compare addition and + multiplication.

-
+
+
+
+

Try the calculation

+ +
+ 200 + + 20 +
+ +

+ 10% of 200 = 20. Then 200 + 20 = 220. +

+
+ Interactive illustration of the inspected percentage rule. +
+
+
+

REA finds the branch. The agent explains it.

+

+ The installed app’s calculation code checks for multiplication + or division before computing the percentage. +

- Original instructions · selected excerpt + Readable summary of the % handler
-
MOV EAX, dword ptr [EBP + 0x8]
-MOV dword ptr [EBP + -0xc], EAX
-FILD dword ptr [EBP + -0xc]
-FMUL double ptr [0x00420068]
-FSUB double ptr [0x00420070]
-FMUL double ptr [0x004210a0]
-CALL 0x0041678c
+
if (operation == multiply || operation == divide) {
+  percent = current / 100;
+} else {
+  percent = current * previous / 100;
+}
-
-
Readable summary · C
-
// x is the horizontal position
-double pan = (double)x;
-pan *= 1.5625;
-pan -= 500.0;
-pan *= pan_scale;
-return (int)pan;
-
-
-
-
- Left side - -
- Center - +
+
+ See the real code and how the answer was checked +
    +
  1. + 01 · Inspect calc.exeA launcherREA finds ShellExecuteW opening + ms-calculator:. +
  2. +
  3. + 02 · Inspect the appCalcViewModel.dllIts % handler checks two operator IDs and uses the constant + 100. +
  4. +
  5. + 03 · Interpret and checkTwo percentage rulesMicrosoft’s public implementation and unit tests explain the + same distinction. +
  6. +
+
+
+
+ REA · selected original instructions
- x = 320 → pan = 0 - Centered sound +
0x180124945: MOV EAX, dword ptr [R13 + 0x18]
+0x180124949: CMP EAX, 0x5c
+0x18012494c: JZ 0x180124aba
+0x180124952: CMP EAX, 0x5b
+0x180124955: JZ 0x180124aba
+0x18012495b: MOV EDX, 0x64
-
- Right side - -
- Examples of the calculation with the recorded initial scale of 1. - The code is a readable summary of the inspected function. -
- -

- To explain this, an analyst identifies the input, follows the caller - and reads the values at those addresses. -

+

+ Inspected with REA 4.1.0: Windows Calculator 11.2508.4.0, x64. The + readable summary uses names from Microsoft’s public source to + explain the recovered branch. +

+

+ Percentage implementation + · + Microsoft’s test cases + · Native setup guide +

+
-
+
-
-
-

Do I need to read assembly to get started?

-

No.

-

Ask your agent about the behavior you want to understand.

-
+
+

Example 02 · Chrome’s dinosaur game

+

Find the speed rule. Rebuild it. Change it.

+

+ Why does the dinosaur game get faster? Use REA to read the running + page’s script, then turn the answer into a game you can adjust. +

+
+
Example analysis · DX-BallExample prompt · Dinosaur game
-

Target · DXBALL.EXE

Use REA to inspect this DX-Ball executable. How does it - calculate left/right sound panning? Show the relevant - code.Use REA to inspect this dinosaur game. Why does it get + faster? Show the speed rule, then build a small version with + an adjustable speed.

-
-

- Agent · explanation from the findings -

-

- This function converts a brick’s horizontal position into a - left/right sound value. -

-

- At the initial scale of 1, the screen center produces zero. - Positions to the left produce negative values; positions to - the right produce positive values. -

-

- Example prompt and explanation based on the recorded - investigation. Supply the executable to your agent. + Inspected target: + wayou’s Chromium-derived browser edition.

+
+
+ REA · loaded index.js · selected lines +
+
if (this.currentSpeed < this.config.MAX_SPEED) {
+  this.currentSpeed += this.config.ACCELERATION;
+}
+
+

+ Start at 6. Add 0.001 each + update without a collision, while speed is below + 13. +

+ Open the speed lab →
+
+

Play the reconstruction

+
+ + + + 6 → 10 → 13 + + + +
+ +

+ Original rule: start at 6, then increase toward 13. +

+
+ Move the slider to set your own speed. Space or Jump clears a + cactus. +
+ +
-
-

What REA found for the agent

+
+

From the running game to a new version

  1. - 01 · Inspect the function - An integer input - The instructions read [EBP + 0x8]. The initial - pseudocode had missed this input.01 · InspectREA returns the loaded scriptThe agent reads the update function and its speed + settings.
  2. - 02 · Follow the caller - A brick’s screen position - The caller passes 20 + 30 × tile_x, giving the - input its meaning.02 · RebuildUse the recovered ruleStart at 6 and add 0.001 while speed is below 13.
  3. - 03 · Read the stored data - The calculation’s constants - Byte reads supply 1.5625, 500.0 and - the initial scale 1.0.03 · Change and playMake speed adjustableKeep the original acceleration, or choose your own + pace.
-
- The agent uses these findings to explain the behavior and write - readable C. -
-

- The reconstruction project checked its maintained C function against - 3,205 original-x86 input cases and all 63 compiled bytes. - See the code and checks. -

+
+ + See the script, checks and how to try the analysis + +

+ REA inspected the HTTP browser edition through a local debugging + connection and returned the loaded index.js, + including its source and digest. +

+
+
+ Selected settings from the inspected script +
+
ACCELERATION: 0.001,
+MAX_SPEED: 13,
+SPEED: 6
+
+

+ We called the original game’s update function in a controlled + browser check, with obstacles and automatic scheduling disabled. + After 4,000 updates, speed was 10.0; after 10,000, it was 13.0, + rounded to one decimal. +

+

+ The new mini-game keeps that speed rule. Its drawing, jumping and + collision code are a small teaching implementation. + Open the lab to see the new code + and run the same speed check. +

+

+ To inspect the target yourself, follow the + browser connection steps + using + the dinosaur page. Give your agent that page URL and your local debugging + endpoint, then copy the prompt above. +

+

+ Inspected speed code + · + Chromium Authors · BSD license +

+
-
+

Your first investigation

Try REA on a small app.

- Find out where an app exports its notes, then check how it formats - a title containing commas and quotes. -

-

We provide the example files, a prompt and a result to check.

- -
-
-

Set up REA with your agent

-
-
- Your coding agent - -
-

- Install REA and connect it to this coding agent using npx - rea-agents@latest setup. Show me the setup plan for approval, - then verify the installation. -

-
-

Or run this command in your terminal:

-
- npx rea-agents@latest setup - -
-

- Setup shows a plan for approval. Restart your agent after setup. + Download our Notes example, trace its CSV export, and check one + changed input.

+ Try the guided example
diff --git a/website/style-guide.md b/website/style-guide.md index 28a007fb55a..c322c1acd21 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -64,8 +64,17 @@ code excerpt with readable logic and a figure that explains the result without requiring the reader to understand either language. Show a simple agent prompt and the specific facts REA supplies to answer it. -Offer a supplied target for the next step. Keep the installation prompt and -normal setup command visible. A first exercise should take the reader through +Put compact setup at the beginning of the homepage: a copyable installation +prompt and the ordinary setup command. Follow it with the beginner examples. +Show how the reader's task changes using the same question on both sides of a +manual/agent comparison. Let returned evidence make REA's contribution visible. + +When a small reconstruction makes the finding tangible, let readers change an +input or setting and try the result. Connect inspection, recovered logic, +reconstruction and the new experiment. Keep the recovered rule distinct from +new teaching mechanics, and check it against the inspected program. + +Offer a supplied target for the next step. A first exercise should take the reader through setup, a question, an answer with source references, one prediction they can check, and a follow-up. Introduce specialized prerequisites when the chosen target requires them. @@ -114,13 +123,14 @@ purposes, and immutable commit links for checkpoint-dependent facts. Each figure should answer a question that a reader can name. Choose the visual form that fits it: -| Relationship | Existing example | Useful form | -| ----------------------------------------- | ---------------------------------------- | -------------------------------------------------- | -| Analysis and verification steps | DX-Ball sound pan | Flow with separate verification branches | -| Code across process boundaries | Electron CSV export and Notion clipboard | HTML flow with APIs and channel names | -| A mathematical transformation | TH04's fixed and aimed rings | SVG drawn from the angle formula | -| Original instructions and readable source | DX-Ball and TH04 | Selectable HTML code with matching step highlights | -| A request caused by an action | Notes browser export | Flow, observed result table and script excerpt | +| Relationship | Existing example | Useful form | +| ----------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | +| Analysis and verification steps | DX-Ball sound pan | Flow with separate verification branches | +| Code across process boundaries | Electron CSV export and Notion clipboard | HTML flow with APIs and channel names | +| A mathematical transformation | TH04's fixed and aimed rings | SVG drawn from the angle formula | +| Original instructions and readable source | DX-Ball and TH04 | Selectable HTML code with matching step highlights | +| A request caused by an action | Notes browser export | Flow, observed result table and script excerpt | +| Recovered behavior and a new experiment | Dinosaur speed reconstruction | Inspected code, playable demo and original-result check | Use SVG or semantic HTML for diagrams with exact text, numbers and arrows. Generated illustrations can help explore a layout, but the published labels, From 5cc841d5c154f692bdc257e106cc4b212dc19dc0 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:07:40 +0000 Subject: [PATCH 108/791] fix(setup): stop uninstall before removals when a client config is unsafe The installation guide says malformed client configuration stops `rea uninstall`, but uninstall reported that client as failed and still removed other clients' registrations, the bundled skill, and with --purge-data REA's cache and state. Inspect every client configuration first, using the conditions setup already treats as blocking (unsafe path, unreadable file, malformed document), and stop before any removal when one fails. Purge paths that are symbolic links are still retained and reported per item, as the guide now states. Fixes #1164 Co-Authored-By: Claude Opus 5.5 --- docs/installation.md | 6 +- src/application/Uninstall.ts | 119 ++++++++++++++---- src/cliLogging.test.ts | 1 + .../filesystem/clientLifecycle.test.ts | 47 +++++++ 4 files changed, 144 insertions(+), 29 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 17c9be0927c..1bc9fa28624 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -573,8 +573,10 @@ rea uninstall --purge-data Uninstall preserves Hopper, Node.js, Evidence files, captures, unrelated skills and other MCP servers. Purging removes only REA's cache and state under -`~/.rea`; malformed client configuration or purge-path symlinks stop the -operation. See the [CLI guide](cli.md#output-and-exit-status) for exit statuses. +`~/.rea`. A client configuration that is malformed, unreadable, or at an unsafe +path stops the operation before anything is removed; a purge path that is a +symbolic link is retained and reported rather than followed. See the +[CLI guide](cli.md#output-and-exit-status) for exit statuses. ## MCP Registry diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index d0203ae2f50..0f926d44039 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -81,21 +81,39 @@ export const isUninstallFailure = (result: UninstallResult): boolean => { /** Filesystem boundary used by the contained uninstall workflow. */ export interface UninstallHost { clients(): Promise; + /** Report a configuration that cannot be read safely, before any removal. */ + inspectClient(client: SetupClient): Promise; removeClient(client: SetupClient): Promise; removeSkill(): Promise; purgeData(): Promise; } -/** Remove only REA-owned registrations and managed files, optionally including local state. */ +/** + * Remove only REA-owned registrations and managed files, optionally including + * local state. A client configuration that cannot be read safely stops the + * operation before anything is removed. + */ export const runUninstall = async ( purgeData: boolean, host: UninstallHost = systemUninstallHost(), ): Promise => { - const items: UninstallItem[] = []; - for (const client of await host.clients()) - items.push(await host.removeClient(client)); - items.push(await host.removeSkill()); - if (purgeData) items.push(...(await host.purgeData())); + const clients = await host.clients(); + const blocked: UninstallItem[] = []; + for (const client of clients) { + const blocking = await host.inspectClient(client); + if (blocking !== undefined) blocked.push(blocking); + } + const items: UninstallItem[] = + blocked.length > 0 + ? [ + ...blocked, + item( + "uninstall", + "skipped", + "No REA registration, skill, or data was removed. Repair each failed configuration, then rerun uninstall.", + ), + ] + : await removeAll(clients, purgeData, host); items.push({ name: "analysis_engine", status: "retained", @@ -109,12 +127,30 @@ export const runUninstall = async ( }; }; +const removeAll = async ( + clients: readonly SetupClient[], + purgeData: boolean, + host: UninstallHost, +): Promise => { + const items: UninstallItem[] = []; + for (const client of clients) items.push(await host.removeClient(client)); + items.push(await host.removeSkill()); + if (purgeData) items.push(...(await host.purgeData())); + return items; +}; + /** Create uninstall effects contained to detected client configs and REA-owned paths. */ export const systemUninstallHost = ( home = homedir(), fileSystem: UninstallFileSystem = systemFileSystem, ): UninstallHost => ({ clients: () => Promise.resolve(supportedClients(home)), + inspectClient: async (client) => { + const read = await readClientConfiguration(client, fileSystem); + return read.kind === "item" && read.item.status === "failed" + ? read.item + : undefined; + }, removeClient: (client) => removeClient(client, fileSystem), removeSkill: () => removeManagedSkills(home, fileSystem), purgeData: async () => [ @@ -123,43 +159,72 @@ export const systemUninstallHost = ( ], }); -const removeClient = async ( +type ClientConfigurationRead = + | { readonly kind: "item"; readonly item: UninstallItem } + | { + readonly kind: "configuration"; + readonly transactionPath: string; + readonly original: string; + readonly parsed: ClientConfigurationDocument; + }; + +const readClientConfiguration = async ( client: SetupClient, fileSystem: UninstallFileSystem, -): Promise => { +): Promise => { if (client.format === "unsupported") - return item( - client.name, - "skipped", - manualRegistrationRemediation(client.name) ?? - "This client has no documented local MCP configuration boundary.", + return itemRead( + item( + client.name, + "skipped", + manualRegistrationRemediation(client.name) ?? + "This client has no documented local MCP configuration boundary.", + ), ); const resolved = await resolveUninstallConfigPath(client, fileSystem); - if (typeof resolved !== "string") return resolved; + if (typeof resolved !== "string") return itemRead(resolved); const transactionPath = resolved; let original: string; try { original = await fileSystem.readText(transactionPath); } catch (cause: unknown) { - return isMissing(cause) - ? item(client.name, "skipped", "Configuration does not exist.") - : item( - client.name, - "failed", - "Configuration could not be read. Check file permissions, then rerun uninstall.", - ); + return itemRead( + isMissing(cause) + ? item(client.name, "skipped", "Configuration does not exist.") + : item( + client.name, + "failed", + "Configuration could not be read. Check file permissions, then rerun uninstall.", + ), + ); } - let parsed: ClientConfigurationDocument; try { - parsed = parseClientConfiguration(original, client.format); + const parsed = parseClientConfiguration(original, client.format); + return { kind: "configuration", transactionPath, original, parsed }; } catch (cause: unknown) { void cause; - return item( - client.name, - "failed", - `Configuration is not valid ${client.format === "toml" || client.format === "grok" ? "TOML" : client.format === "opencode" ? "JSONC" : "JSON"} and was not changed. Repair it, then rerun uninstall.`, + return itemRead( + item( + client.name, + "failed", + `Configuration is not valid ${client.format === "toml" || client.format === "grok" ? "TOML" : client.format === "opencode" ? "JSONC" : "JSON"} and was not changed. Repair it, then rerun uninstall.`, + ), ); } +}; + +const itemRead = (result: UninstallItem): ClientConfigurationRead => ({ + kind: "item", + item: result, +}); + +const removeClient = async ( + client: SetupClient, + fileSystem: UninstallFileSystem, +): Promise => { + const read = await readClientConfiguration(client, fileSystem); + if (read.kind === "item") return read.item; + const { transactionPath, original, parsed } = read; const name = PRODUCT_IDENTITY.mcpServerKey; // Beside a native OpenCode V2 table, OpenCode also loads a V1 `mcp.rea`. const legacyPath = legacyClientServerPath(parsed, name); diff --git a/src/cliLogging.test.ts b/src/cliLogging.test.ts index e275519e6d0..dc05441094c 100644 --- a/src/cliLogging.test.ts +++ b/src/cliLogging.test.ts @@ -145,6 +145,7 @@ describe("CLI command outcome classification", () => { it("returns nonzero when the real uninstall outcome contains a failed action", async () => { const host: UninstallHost = { clients: () => Promise.resolve([]), + inspectClient: () => Promise.resolve(undefined), removeClient: () => Promise.resolve({ name: "client", status: "failed", detail: "write" }), removeSkill: () => diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 8ea5a99454b..2c66bd42d6f 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -1418,6 +1418,53 @@ describe("client configuration filesystem removal", () => { ); expect(await readFile(config, "utf8")).toBe("not-json"); }); + + it("stops before any removal when one client configuration is malformed", async () => { + const home = await createTestTempDirectory("rea-uninstall-stop-"); + roots.push(home); + const codex = join(home, ".codex/config.toml"); + const cursor = join(home, ".cursor/mcp.json"); + const managed = [ + join(home, ".agents/skills/reverse-engineer-anything/marker.txt"), + join(home, ".rea/cache/marker.txt"), + join(home, ".rea/state/marker.txt"), + ]; + const cursorConfig = JSON.stringify({ + mcpServers: { + rea: { command: "npx", args: ["-y", "rea-agents@6.1.0", "mcp"] }, + other: { command: "echo", args: ["hello"] }, + }, + }); + for (const path of [codex, cursor, ...managed]) + await mkdir(dirname(path), { recursive: true }); + await writeFile(codex, "[mcp_servers.rea]\ncommand = [broken\n"); + await writeFile(cursor, cursorConfig); + for (const path of managed) await writeFile(path, "marker"); + + const stopped = await runUninstall(true, systemUninstallHost(home)); + + expect(stopped.status).toBe("failed"); + expect(stopped.items.map(({ name, status }) => [name, status])).toEqual([ + ["codex", "failed"], + ["uninstall", "skipped"], + ["analysis_engine", "retained"], + ]); + expect(await readFile(codex, "utf8")).toBe( + "[mcp_servers.rea]\ncommand = [broken\n", + ); + expect(await readFile(cursor, "utf8")).toBe(cursorConfig); + for (const path of managed) + expect(await readFile(path, "utf8")).toBe("marker"); + + await writeFile(codex, ""); + const repaired = await runUninstall(true, systemUninstallHost(home)); + expect(repaired.status).toBe("complete"); + expect(JSON.parse(await readFile(cursor, "utf8"))).toEqual({ + mcpServers: { other: { command: "echo", args: ["hello"] } }, + }); + for (const path of managed) + await expect(readFile(path, "utf8")).rejects.toThrow(); + }); }); describe("client configuration filesystem failure reporting", () => { From b4927a0a4fa7e8935b101d214b396bfa78208281 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:11:11 -0700 Subject: [PATCH 109/791] fix(cli): advertise defaulted inputs as optional in --schema and help (#1175) Incur derives `--schema`, the `--llms` manifests, and help synopses from Zod's output view, where a defaulted field always has a value, so every defaulted argument and option was listed as required even though the CLI accepts its omission. Wrap each field that fills an omitted value as optional after registration, keeping its default, constraints, metadata, and parsed value. Fixes #1169 Co-authored-by: Claude Opus 5.5 --- src/cli.ts | 2 + src/cli/omittableDefaults.ts | 66 ++++++++++ .../cli/cliInputSchemaRequired.test.ts | 114 ++++++++++++++++++ 3 files changed, 182 insertions(+) create mode 100644 src/cli/omittableDefaults.ts create mode 100644 tests/boundary/cli/cliInputSchemaRequired.test.ts diff --git a/src/cli.ts b/src/cli.ts index 6f6b84c3f26..8b3d3000850 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,6 +26,7 @@ import { registerBrowserScenarioCommands } from "./cliBrowserScenarioCommands.js import { registerElectronCommands } from "./cli/electronCommands.js"; import { registerJavaScriptRuntimeObservationCommands } from "./cliJavaScriptRuntimeCommands.js"; import { registerApplicationCommands } from "./cli/applicationCommands.js"; +import { presentOmittableDefaults } from "./cli/omittableDefaults.js"; import type { CliInstance } from "./cli/types.js"; import type { CliResultOutput } from "./cli/streamedJsonOutput.js"; @@ -78,5 +79,6 @@ export const createCli = ( registerElectronCommands(cli, logger, resultOutput); registerJavaScriptRuntimeObservationCommands(cli, logger); registerApplicationCommands(cli, logger); + presentOmittableDefaults(cli); return cli; }; diff --git a/src/cli/omittableDefaults.ts b/src/cli/omittableDefaults.ts new file mode 100644 index 00000000000..eaf11c503f1 --- /dev/null +++ b/src/cli/omittableDefaults.ts @@ -0,0 +1,66 @@ +import { Cli, z } from "incur"; + +import type { CliInstance } from "./types.js"; + +type CommandMap = NonNullable>; +type CommandInputs = Partial>; + +const INPUT_GROUPS = ["args", "env", "options"] as const; + +// Incur types its router key without vars, env, or globals; accept any CLI. +const registeredCommands: WeakMap = Cli.toCommands; + +/** + * Advertise parser-filled defaults as omittable CLI inputs. + * + * Incur derives `--schema`, the `--llms` manifests, and help synopses from + * Zod's output view, where a defaulted field always has a value and is + * therefore listed as required. Callers may omit such a field, so each one is + * wrapped as optional while its default, constraints, metadata, and parsed + * value stay unchanged. + */ +export const presentOmittableDefaults = (cli: CliInstance): void => { + const commands = registeredCommands.get(cli); + if (commands !== undefined) presentCommands(commands); +}; + +const presentCommands = (commands: CommandMap): void => { + for (const entry of commands.values()) { + if ("_group" in entry) { + presentCommands(entry.commands); + if (entry.root !== undefined) presentInputs(entry.root); + } else if (!("_alias" in entry) && !("_fetch" in entry)) + presentInputs(entry); + } +}; + +const presentInputs = (command: CommandInputs): void => { + for (const group of INPUT_GROUPS) { + const schema = command[group]; + if (schema instanceof z.ZodObject) + command[group] = withOmittableDefaults(schema); + } +}; + +const withOmittableDefaults = (schema: z.ZodObject): z.ZodObject => { + const omittable = Object.entries(schema.shape).filter(([, field]) => + fillsOmission(field), + ); + if (omittable.length === 0) return schema; + return schema.safeExtend( + Object.fromEntries( + omittable.map(([key, field]) => [key, optionalWithMetadata(field)]), + ), + ); +}; + +/** Zod accepts an omitted value (`optin`) but always produces one (`optout`). */ +const fillsOmission = (field: z.ZodType): boolean => + field._zod.optin === "optional" && field._zod.optout === undefined; + +const optionalWithMetadata = (field: z.ZodType): z.ZodType => { + const metadata = field.meta(); + return metadata === undefined + ? field.optional() + : field.optional().meta(metadata); +}; diff --git a/tests/boundary/cli/cliInputSchemaRequired.test.ts b/tests/boundary/cli/cliInputSchemaRequired.test.ts new file mode 100644 index 00000000000..3bc95236f37 --- /dev/null +++ b/tests/boundary/cli/cliInputSchemaRequired.test.ts @@ -0,0 +1,114 @@ +import { Ajv2020 } from "ajv/dist/2020.js"; +import { Cli, z } from "incur"; +import { describe, expect, it } from "vitest"; + +import { createCli } from "../../../src/cli.js"; +import { createCliInventory } from "../../../scripts/lib/product-catalog.mjs"; + +type CliInstance = ReturnType; +type CommandMap = NonNullable>; + +const INPUT_GROUPS = ["args", "options"] as const; +const registeredCommands: WeakMap = Cli.toCommands; + +const advertisedGroupSchema = z + .object({ + properties: z.record(z.string(), z.unknown()).optional(), + required: z.array(z.string()).optional(), + }) + .loose(); +const advertisedInputSchema = z.object({ + args: advertisedGroupSchema.optional(), + options: advertisedGroupSchema.optional(), +}); + +const serve = async ( + cli: CliInstance, + argv: readonly string[], +): Promise<{ readonly stdout: string; readonly exitCode: number }> => { + let stdout = ""; + let exitCode = 0; + await cli.serve([...argv], { + env: {}, + exit: (code) => { + exitCode = code; + }, + stdout: (text) => { + stdout += text; + }, + }); + return { stdout, exitCode }; +}; + +const advertisedSchema = async (cli: CliInstance, command: string) => { + const result = await serve(cli, [command, "--schema", "--json"]); + expect(result.exitCode, command).toBe(0); + return advertisedInputSchema.parse(JSON.parse(result.stdout)); +}; + +const parserSchema = ( + cli: CliInstance, + command: string, + group: (typeof INPUT_GROUPS)[number], +): z.ZodObject | undefined => { + const entry = registeredCommands.get(cli)?.get(command); + if (entry === undefined || !(group in entry)) return undefined; + const schema: unknown = Reflect.get(entry, group); + return schema instanceof z.ZodObject ? schema : undefined; +}; + +const omissionRejected = (schema: z.ZodObject | undefined): string[] => + Object.entries(schema?.shape ?? {}) + .filter(([, field]) => !field.safeParse(undefined).success) + .map(([key]) => key) + .sort(); + +describe("CLI advertised input schemas", () => { + it("require exactly the inputs whose omission the parser rejects", async () => { + const cli = createCli({}); + const mismatches: string[] = []; + for (const command of createCliInventory(cli).primary) { + const advertised = await advertisedSchema(cli, command); + for (const group of INPUT_GROUPS) { + const required = (advertised[group]?.required ?? []).toSorted(); + const rejected = omissionRejected(parserSchema(cli, command, group)); + if (required.join() !== rejected.join()) + mismatches.push( + `${command} ${group}: advertised [${required.join()}], parser rejects [${rejected.join()}]`, + ); + } + } + expect(mismatches).toEqual([]); + }); + + it("accepts the one-argument keyed-archive call and keeps its constraints", async () => { + const cli = createCli({}); + const advertised = await advertisedSchema(cli, "inspect-keyed-archive"); + const ajv = new Ajv2020({ strict: false }); + const args = ajv.compile(advertised.args ?? {}); + const options = ajv.compile(advertised.options ?? {}); + + expect(args({ path: "./keyed.plist" })).toBe(true); + expect(args({})).toBe(false); + expect(options({})).toBe(true); + expect(options({ limit: 0 })).toBe(false); + expect(advertised.args?.properties?.["archive"]).toMatchObject({ + default: ".", + }); + expect(advertised.options?.properties?.["limit"]).toMatchObject({ + default: 20_000, + minimum: 1, + }); + expect( + parserSchema(cli, "inspect-keyed-archive", "args")?.parse({ path: "x" }), + ).toEqual({ path: "x", archive: "." }); + + const help = await serve(cli, ["inspect-keyed-archive", "--help"]); + expect(help.stdout).toContain( + "Usage: rea inspect-keyed-archive [archive] [options]", + ); + expect(help.stdout).toContain( + "archive Relative archive path when the target is a bundle", + ); + }); +}); From e9c48a61e4c2c05c5b77ef4a815c5b2a65a9337e Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:15:18 +0000 Subject: [PATCH 110/791] fix(setup): keep Codex TOML comments when adding or removing REA Setup and uninstall rewrote the whole Codex config.toml with a fresh TOML serializer, which dropped every comment and respelled unrelated values, although the plan promises to preserve unrelated configuration. Grok Build already edited only REA's server tables. Share that table editor with Codex, without Grok's root disabled_mcp_servers handling, keep the original final line break, and fall back to the full rewrite only when the edited text would not parse to the requested document. Fixes #1163 Co-Authored-By: Claude Opus 5.5 --- .../ClientConfigurationDocument.test.ts | 55 ++++++++ .../ClientConfigurationDocument.ts | 120 +++++++++++++----- .../filesystem/clientLifecycle.test.ts | 29 +++++ 3 files changed, 175 insertions(+), 29 deletions(-) diff --git a/src/application/ClientConfigurationDocument.test.ts b/src/application/ClientConfigurationDocument.test.ts index f706cea6f42..83906e7ee4a 100644 --- a/src/application/ClientConfigurationDocument.test.ts +++ b/src/application/ClientConfigurationDocument.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it } from "vitest"; +import { parse as parseToml } from "smol-toml"; + import { clientConfigurationServersKey, parseClientConfiguration, + serializeClientConfiguration, } from "./ClientConfigurationDocument.js"; describe("JSON client configuration BOM handling", () => { @@ -40,3 +43,55 @@ describe("TOML client configuration BOM handling", () => { }); }); }); + +describe("Codex TOML serialization", () => { + const unrelated = [ + "# Keep this explanation.", + 'notify = ["a", "b"]', + "literal = 'C:\\demo\\path'", + "", + ].join("\n"); + const rea = { command: "npx", args: ["-y", "rea-agents@6.1.0", "mcp"] }; + const registered = `${unrelated}\n[mcp_servers.rea]\ncommand = "npx"\nargs = ["-y", "rea-agents@6.1.0", "mcp"]\n`; + const documentOf = (text: string): Record => + parseClientConfiguration(text, "toml").document; + const serialize = (document: Record, original: string) => + serializeClientConfiguration(document, "toml", original, [ + ["mcp_servers", "rea"], + ]); + + it("adds REA's table without rewriting comments or unrelated values", () => { + const document = { ...documentOf(unrelated), mcp_servers: { rea } }; + const written = serialize(document, unrelated); + + expect(written.startsWith(unrelated)).toBe(true); + expect(parseToml(written)).toEqual(document); + }); + + it("removes REA's table and leaves the rest of the file byte for byte", () => { + const document = { ...documentOf(registered), mcp_servers: {} }; + + expect(serialize(document, registered)).toBe(unrelated); + }); + + it("leaves a root disabled_mcp_servers list, which Codex does not read", () => { + const original = `# Codex ignores this list.\ndisabled_mcp_servers = ["rea"]\n`; + const document = { ...documentOf(original), mcp_servers: { rea } }; + const written = serialize(document, original); + + expect(written.startsWith(original)).toBe(true); + expect(parseToml(written)).toEqual(document); + }); + + it("rewrites the document when an edit would not hold the requested values", () => { + const document = { + ...documentOf(unrelated), + notify: ["changed"], + mcp_servers: { rea }, + }; + const written = serialize(document, unrelated); + + expect(written).not.toContain("# Keep this explanation."); + expect(parseToml(written)).toEqual(document); + }); +}); diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index 2b8a83a2258..991dcd0b4ea 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -399,7 +399,7 @@ const looksLikeTomlHeader = (line: string): boolean => line.trimStart().startsWith("["); /** `mcp_servers.` and tables nested under it, compared by key segment. */ -const isGrokServerTable = ( +const isTomlServerTable = ( path: readonly string[], serverKey: string, ): boolean => path[0] === "mcp_servers" && path[1] === serverKey; @@ -612,7 +612,7 @@ const hasSeparatorComma = (trivia: string): boolean => * Replace or remove `serverKey` inside one `{ ... }` table. A closed inline * table cannot later grow a `[mcp_servers.rea]` header. */ -const editGrokInlineTable = ( +const editInlineServerTable = ( table: string, serverKey: string, entry: unknown, @@ -831,13 +831,16 @@ const disabledServerListLines = ( }; /** - * Replace REA's Grok server tables and assignments. Comments that introduce a - * following table stay. Text inside strings is not treated as a header. + * Replace REA's TOML server tables and assignments. Comments that introduce a + * following table stay. Text inside strings is not treated as a header. Only + * Grok Build reads the root `disabled_mcp_servers` list that registration + * also edits. */ -const upsertGrokServerSection = ( +const upsertTomlServerSection = ( originalText: string, serverKey: string, entry: unknown, + editDisabledList: boolean, ): string => { // smol-toml accepts a leading BOM. Leave it in place or the first header is missed. const bom = originalText.startsWith("\uFEFF"); @@ -887,7 +890,7 @@ const upsertGrokServerSection = ( const header = parseTomlTableHeader(line); if (header !== undefined) { tablePath = header.path; - if (isGrokServerTable(header.path, serverKey)) { + if (isTomlServerTable(header.path, serverKey)) { discardPending(); skipping = true; lineIndex += 1; @@ -914,22 +917,24 @@ const upsertGrokServerSection = ( mode = "none"; continue; } - const disabledLines = disabledServerListLines( - disabledServerListEdit({ - text, - lineStart, - valueOffset, - valueEnd, - statementEnd, - tablePath, - segments: assignment.segments, - serverKey, - entry, - }), - lines, - lineIndex, - endLine, - ); + const disabledLines = editDisabledList + ? disabledServerListLines( + disabledServerListEdit({ + text, + lineStart, + valueOffset, + valueEnd, + statementEnd, + tablePath, + segments: assignment.segments, + serverKey, + entry, + }), + lines, + lineIndex, + endLine, + ) + : undefined; if (disabledLines !== undefined) { flushPending(); kept.push(...disabledLines); @@ -943,10 +948,10 @@ const upsertGrokServerSection = ( absolute.length === 1 && absolute[0] === "mcp_servers" && text[open] === "{"; - if (closesServerTable || isGrokServerTable(absolute, serverKey)) { + if (closesServerTable || isTomlServerTable(absolute, serverKey)) { if (closesServerTable) { flushPending(); - const edited = editGrokInlineTable( + const edited = editInlineServerTable( text.slice(open, valueEnd), serverKey, entry, @@ -1010,7 +1015,7 @@ const upsertGrokServerSection = ( return bom ? `\uFEFF${body}` : body; }; -const grokServerEntry = ( +const tomlServerEntry = ( document: Record, serverKey: string, ): unknown => { @@ -1033,13 +1038,69 @@ const serializeGrokConfiguration = ( document: Record, originalText: string | undefined, ): string => { - const entry = grokServerEntry(document, PRODUCT_IDENTITY.mcpServerKey); + const entry = tomlServerEntry(document, PRODUCT_IDENTITY.mcpServerKey); if (originalText === undefined) return stringifyToml(document); - return upsertGrokServerSection( + return upsertTomlServerSection( originalText, PRODUCT_IDENTITY.mcpServerKey, entry, + true, + ); +}; + +/** Removing the last server leaves an empty table, which equals an absent one. */ +const withoutEmptyServerTable = ( + document: Record, +): Record => { + const servers = objectSchema.safeParse(document.mcp_servers); + if (!servers.success || Object.keys(servers.data).length > 0) return document; + const { mcp_servers: empty, ...rest } = document; + void empty; + return rest; +}; + +/** + * Keep an existing Codex document intact aside from the REA server tables. + * An edit that would change any other parsed value falls back to a full + * rewrite, which keeps every value but not comments or spelling. + */ +const serializeCodexConfiguration = ( + document: Record, + originalText: string | undefined, +): string => { + if ( + originalText === undefined || + isEmptyClientConfigurationText(originalText) + ) + return stringifyToml(document); + const upserted = upsertTomlServerSection( + originalText, + PRODUCT_IDENTITY.mcpServerKey, + tomlServerEntry(document, PRODUCT_IDENTITY.mcpServerKey), + false, ); + // Removing a final table also drops the file's last line break. + const edited = + originalText.endsWith("\n") && !upserted.endsWith("\n") + ? `${upserted}${originalText.endsWith("\r\n") ? "\r\n" : "\n"}` + : upserted; + return tomlTextHolds(edited, document) ? edited : stringifyToml(document); +}; + +const tomlTextHolds = ( + text: string, + document: Record, +): boolean => { + try { + return clientConfigurationValuesEqual( + withoutEmptyServerTable(objectSchema.parse(parseToml(text))), + withoutEmptyServerTable(document), + ); + } catch (cause: unknown) { + // An unparsable edit is rejected in favor of the full rewrite. + void cause; + return false; + } }; /** Comments in the trivia between two properties. `undefined` when there are none. */ @@ -1146,7 +1207,7 @@ const deleteJsonPropertyPreservingComments = ( return `${text.slice(0, previous.offset + previous.length)}${suffix}${remainder}`; }; -/** Serialize a validated client document, retaining JSONC comments. */ +/** Serialize a validated client document, retaining JSONC and TOML comments. */ export const serializeClientConfiguration = ( document: Record, format: ClientConfigurationFormat | undefined, @@ -1155,7 +1216,8 @@ export const serializeClientConfiguration = ( ): string => { if (format === undefined || format === "unsupported") throw new TypeError("client does not have a supported MCP config format"); - if (format === "toml") return stringifyToml(document); + if (format === "toml") + return serializeCodexConfiguration(document, originalText); if (format === "grok") return serializeGrokConfiguration(document, originalText); // An empty original holds no comments or settings to preserve. Writing a diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 8ea5a99454b..7c69000fd7d 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -1310,6 +1310,35 @@ describe("Grok Bot registration", () => { }); }); +describe("Codex configuration comments", () => { + it("keeps Codex comments and value spelling through setup and uninstall", async () => { + const home = await createTestTempDirectory("rea-toml-comments-"); + roots.push(home); + const configPath = join(home, ".codex/config.toml"); + const original = + '# Keep this explanation.\nnotify = ["a", "b"]\nliteral = \'C:\\demo\\path\'\n'; + await mkdir(dirname(configPath), { recursive: true }); + await writeFile(configPath, original); + + expect( + await configureClientConfiguration( + { name: "codex", configPath, format: "toml" }, + {}, + ["/isolated prefix/bin/rea", "mcp"], + ), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + expect(configured.startsWith(original)).toBe(true); + expect(configured).toContain("[mcp_servers.rea]"); + + const removed = await runUninstall(false, systemUninstallHost(home)); + expect(removed.items).toContainEqual( + expect.objectContaining({ name: "codex", status: "removed" }), + ); + expect(await readFile(configPath, "utf8")).toBe(original); + }); +}); + describe("client configuration filesystem removal", () => { it("uninstalls only owned entries and refuses purge symlinks", async () => { const home = await createTestTempDirectory("rea-uninstall-"); From fd975c75fa353840e6e887232ca68496cf8705ea Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:17:11 +0000 Subject: [PATCH 111/791] fix(setup): stop remaining uninstall removals after a client fails A configuration that passes the preflight can still fail when uninstall rereads it to edit, for example when it changes concurrently. Rereading keeps that edit from overwriting the newer file; stopping at the failure keeps later clients, the skill, and REA data from being removed. Co-Authored-By: Claude Opus 5.5 --- docs/installation.md | 3 +- src/application/Uninstall.ts | 21 ++++++- .../filesystem/clientLifecycle.test.ts | 59 +++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 1bc9fa28624..2028ea38dfa 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -574,7 +574,8 @@ rea uninstall --purge-data Uninstall preserves Hopper, Node.js, Evidence files, captures, unrelated skills and other MCP servers. Purging removes only REA's cache and state under `~/.rea`. A client configuration that is malformed, unreadable, or at an unsafe -path stops the operation before anything is removed; a purge path that is a +path stops the operation before anything is removed, and a client that fails +while being updated stops the remaining removals. A purge path that is a symbolic link is retained and reported rather than followed. See the [CLI guide](cli.md#output-and-exit-status) for exit statuses. diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index 0f926d44039..789bdc34e8c 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -91,7 +91,8 @@ export interface UninstallHost { /** * Remove only REA-owned registrations and managed files, optionally including * local state. A client configuration that cannot be read safely stops the - * operation before anything is removed. + * operation before anything is removed; one that fails during removal stops + * the remaining removals. */ export const runUninstall = async ( purgeData: boolean, @@ -127,13 +128,29 @@ export const runUninstall = async ( }; }; +/** + * Each client is reread when it is edited, so a concurrent change is never + * overwritten. A client that fails then still stops the remaining removals. + */ const removeAll = async ( clients: readonly SetupClient[], purgeData: boolean, host: UninstallHost, ): Promise => { const items: UninstallItem[] = []; - for (const client of clients) items.push(await host.removeClient(client)); + for (const client of clients) { + const removed = await host.removeClient(client); + items.push(removed); + if (removed.status === "failed") + return [ + ...items, + item( + "uninstall", + "skipped", + "Uninstall stopped at this failure; the items above report what changed. Repair the failed configuration, then rerun uninstall.", + ), + ]; + } items.push(await host.removeSkill()); if (purgeData) items.push(...(await host.purgeData())); return items; diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 2c66bd42d6f..bf277a24ff4 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -1467,6 +1467,65 @@ describe("client configuration filesystem removal", () => { }); }); +describe("client configuration changed during uninstall", () => { + it("stops the remaining removals when a configuration fails after preflight", async () => { + const home = await createTestTempDirectory("rea-uninstall-race-"); + roots.push(home); + const codex = join(home, ".codex/config.toml"); + const cursor = join(home, ".cursor/mcp.json"); + const skill = join( + home, + ".agents/skills/reverse-engineer-anything/SKILL.md", + ); + const cursorConfig = JSON.stringify({ + mcpServers: { rea: { command: "rea", args: ["mcp"] } }, + }); + for (const path of [codex, cursor, skill]) + await mkdir(dirname(path), { recursive: true }); + await writeFile( + codex, + '[mcp_servers.rea]\ncommand = "rea"\nargs = ["mcp"]\n', + ); + await writeFile(cursor, cursorConfig); + await writeFile(skill, "managed"); + let codexReads = 0; + const fileSystem: UninstallFileSystem = { + ...testFileSystem, + // The preflight read succeeds; the removal reread finds a broken file. + readText: (path) => { + if (path !== codex) return readFile(path, "utf8"); + codexReads += 1; + return codexReads === 1 + ? readFile(path, "utf8") + : Promise.resolve("[mcp_servers.rea]\ncommand = [broken\n"); + }, + }; + + const result = await runUninstall( + false, + systemUninstallHost(home, fileSystem), + ); + + expect(result.status).toBe("failed"); + // Clients ordered before Codex have no configuration in this home. + expect( + result.items + .filter(({ status }) => status !== "skipped") + .map(({ name, status }) => [name, status]), + ).toEqual([ + ["codex", "failed"], + ["analysis_engine", "retained"], + ]); + expect(result.items.map(({ name }) => name)).not.toContain("cursor"); + expect(result.items.at(-2)).toMatchObject({ + name: "uninstall", + status: "skipped", + }); + expect(await readFile(cursor, "utf8")).toBe(cursorConfig); + expect(await readFile(skill, "utf8")).toBe("managed"); + }); +}); + describe("client configuration filesystem failure reporting", () => { it.each([ [ From b8312fa8d184ba9f422d1f5592fb41e49811544a Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:20:43 +0000 Subject: [PATCH 112/791] fix(server): report 2025-era client identity and features in binary_session binary_session read the client's identity, protocol revision, and capabilities only from the 2026-era per-request envelope. A 2025-era connection sends none, so every such client was reported as `client: null` with no negotiated protocol and no features, although it declared them in `initialize`. When a request carries no envelope, read the values the SDK keeps from the connection's initialize handshake. An envelope still takes precedence. Fixes #1167 Co-Authored-By: Claude Opus 5.5 --- src/server/mcpClientMetadata.ts | 34 +++++-- src/server/registerSessionStatusTool.ts | 6 +- tests/boundary/mcp/clientCapabilities.test.ts | 88 +++++++++++++++++++ tests/boundary/mcp/serverIdentity.test.ts | 5 +- 4 files changed, 122 insertions(+), 11 deletions(-) diff --git a/src/server/mcpClientMetadata.ts b/src/server/mcpClientMetadata.ts index 94690eae983..e14cfdb0c93 100644 --- a/src/server/mcpClientMetadata.ts +++ b/src/server/mcpClientMetadata.ts @@ -2,25 +2,45 @@ import { CLIENT_CAPABILITIES_META_KEY, CLIENT_INFO_META_KEY, PROTOCOL_VERSION_META_KEY, + type McpServer, type ServerContext, } from "@modelcontextprotocol/server"; import type { ClientFeatureAvailability } from "../contracts/toolOutputSchemaPrimitives.js"; import type { ConnectedClientIdentity } from "../serverIdentity.js"; -/** Client metadata bound to the current SDK request rather than server-global state. */ -export const mcpClientMetadata = (context: { - readonly mcpReq: Pick; -}) => { +/** Initialize-scoped client metadata the SDK keeps for a 2025-era connection. */ +export type McpConnectionClientMetadata = Pick< + McpServer["server"], + "getClientCapabilities" | "getClientVersion" | "getNegotiatedProtocolVersion" +>; + +/** + * Client metadata for the current request. A 2026-era request carries it in + * its per-request envelope; a 2025-era connection sends no envelope, so its + * `initialize` handshake supplies the values. + */ +export const mcpClientMetadata = ( + context: { readonly mcpReq: Pick }, + connection: McpConnectionClientMetadata, +) => { const envelope = context.mcpReq.envelope; + // The SDK deprecates these accessors only in favor of the envelope, which + // 2025-era requests do not carry; they still return initialize values. + const value = (key: string, initialized: () => unknown): unknown => + envelope === undefined ? initialized() : mcpEnvelopeValue(envelope, key); const client = implementation( - mcpEnvelopeValue(envelope, CLIENT_INFO_META_KEY), + value(CLIENT_INFO_META_KEY, () => connection.getClientVersion()), + ); + const protocolVersion = value(PROTOCOL_VERSION_META_KEY, () => + connection.getNegotiatedProtocolVersion(), ); - const protocolVersion = mcpEnvelopeValue(envelope, PROTOCOL_VERSION_META_KEY); return { ...(client === undefined ? {} : { client }), clientFeatures: capabilityFeatures( - mcpEnvelopeValue(envelope, CLIENT_CAPABILITIES_META_KEY), + value(CLIENT_CAPABILITIES_META_KEY, () => + connection.getClientCapabilities(), + ), ), ...(typeof protocolVersion === "string" ? { protocolVersion } : {}), }; diff --git a/src/server/registerSessionStatusTool.ts b/src/server/registerSessionStatusTool.ts index e1843675149..c2202673066 100644 --- a/src/server/registerSessionStatusTool.ts +++ b/src/server/registerSessionStatusTool.ts @@ -44,8 +44,10 @@ export const registerSessionStatusTool = ( contract.name, toolRegistrationOptions(contract), async (input, context) => { - const { client, clientFeatures, protocolVersion } = - mcpClientMetadata(context); + const { client, clientFeatures, protocolVersion } = mcpClientMetadata( + context, + server.server, + ); let androidAvailability: ProviderAvailability; try { androidAvailability = await androidAnalysisAvailability( diff --git a/tests/boundary/mcp/clientCapabilities.test.ts b/tests/boundary/mcp/clientCapabilities.test.ts index f39d87e0fbd..323f79980df 100644 --- a/tests/boundary/mcp/clientCapabilities.test.ts +++ b/tests/boundary/mcp/clientCapabilities.test.ts @@ -54,3 +54,91 @@ describe("per-request client capability reporting", () => { }, ); }); + +describe("initialize-scoped client metadata", () => { + const statusFor = async ( + capabilities: Record, + meta?: Record, + ) => { + const session = createTestBinarySession(createCacheProvider([])); + const server = createServer(session, session); + const client = new Client( + { name: "initialize-test", version: "2" }, + { capabilities }, + ); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + return ( + await client.callTool({ + name: "binary_session", + arguments: {}, + ...(meta === undefined ? {} : { _meta: meta }), + }) + ).structuredContent; + } finally { + await Promise.allSettled([ + client.close(), + server.close(), + session.close(), + ]); + } + }; + const allFeatures = { + elicitation: { form: {}, url: {} }, + roots: {}, + sampling: {}, + }; + + it("reports a 2025-era client from its initialize handshake", async () => { + expect(await statusFor(allFeatures)).toMatchObject({ + result: { + server_identity: { + client: { name: "initialize-test", version: "2" }, + negotiated_protocol_version: expect.any(String), + }, + client_features: { + elicitation_form: true, + elicitation_url: true, + roots: true, + sampling: true, + }, + }, + }); + expect(await statusFor({})).toMatchObject({ + result: { + client_features: { + elicitation_form: false, + elicitation_url: false, + roots: false, + sampling: false, + }, + }, + }); + }); + + it("prefers a per-request envelope over initialize values", async () => { + expect( + await statusFor(allFeatures, { + [PROTOCOL_VERSION_META_KEY]: "2026-07-28", + [CLIENT_INFO_META_KEY]: { name: "envelope-test", version: "3" }, + [CLIENT_CAPABILITIES_META_KEY]: {}, + }), + ).toMatchObject({ + result: { + server_identity: { + client: { name: "envelope-test", version: "3" }, + negotiated_protocol_version: "2026-07-28", + }, + client_features: { + elicitation_form: false, + elicitation_url: false, + roots: false, + sampling: false, + }, + }, + }); + }); +}); diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 2305a346e2b..8f51487fed1 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -262,7 +262,8 @@ const assertLiveIdentity = async (client: Client): Promise => { server: SDK_IDENTITY.server, client_test: PACKAGE_METADATA.clientSdkVersion, }, - client: null, + client: { name: "identity-test", version: "9" }, + negotiated_protocol_version: expect.any(String), alignment: { state: "unknown" }, }, }, @@ -316,7 +317,7 @@ const assertSessionIdentity = async (client: Client): Promise => { }), ]), client_features: { - elicitation_form: false, + elicitation_form: true, elicitation_url: false, roots: false, sampling: false, From 19c52f5e3f42fb754fc154bbd16552c5111df0c5 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 11:21:18 +0800 Subject: [PATCH 113/791] feat(website): lead with the playable dinosaur example --- website/README.md | 12 +- website/evidence/calculator-and-dino.md | 5 +- website/figures.md | 6 +- website/public/index.html | 497 +++++++++++++----------- website/style-guide.md | 2 + 5 files changed, 279 insertions(+), 243 deletions(-) diff --git a/website/README.md b/website/README.md index cbe1512324a..ae758a8959a 100644 --- a/website/README.md +++ b/website/README.md @@ -26,7 +26,7 @@ Open . Refresh the browser after editing a file. ## Pages -- `public/index.html`: compact setup, manual/agent comparison, Calculator percentage example, playable dinosaur reconstruction and case-study overviews. +- `public/index.html`: compact setup, playable dinosaur reconstruction, manual/agent comparison, Calculator percentage example and case-study overviews. - `public/examples/dino-lab/index.html`: adjustable-speed mini-game, recovered rule, original-game check and browser analysis steps. - `public/first-investigation/index.html`: a guided Notes export investigation, from setup to a checked CSV prediction. - `public/showcase/index.html`: the case-study index. @@ -98,10 +98,12 @@ their provenance and the scope of the validation claims. The original executable and complete private Evidence records are not website assets. The homepage starts with a short installation prompt and the ordinary setup -command. It compares manual investigation with asking an agent, using the same -Calculator percentage question. The native excerpt comes from an installed -model DLL; Microsoft’s public implementation supplies names and a crosscheck. -The Calculator controls illustrate that rule with generic inputs. +command, followed immediately by the playable dinosaur example. Its game +precedes the prompt and code on small screens. The manual/agent comparison +then explains the investigation using the same dinosaur speed question. +Calculator follows as a native example: its excerpt comes from an installed +model DLL, and Microsoft’s public implementation supplies names and a +crosscheck. Its controls illustrate the percentage rule with generic inputs. The dinosaur example connects an inspected running script to a new playable mini-game. `public/assets/dino-speed.js` owns the recovered speed rule and replay; diff --git a/website/evidence/calculator-and-dino.md b/website/evidence/calculator-and-dino.md index 67d8160147a..e89138d54ef 100644 --- a/website/evidence/calculator-and-dino.md +++ b/website/evidence/calculator-and-dino.md @@ -1,8 +1,9 @@ # Calculator and dinosaur beginner examples Evidence collected on 9 October 2026 with the published REA 4.1.0 package. -The homepage offers compact setup first, then a manual/agent comparison and -two familiar behavior questions. Prompts and replies are examples based on the +The homepage offers compact setup first, then a playable dinosaur reconstruction, +a manual/agent comparison of that investigation, and the Calculator example. +Prompts and replies are examples based on the findings below. Selected code and generic inputs are public; raw captures stay outside the website. diff --git a/website/figures.md b/website/figures.md index c94db54f952..a87cf7b19f1 100644 --- a/website/figures.md +++ b/website/figures.md @@ -44,9 +44,9 @@ rendering at desktop and mobile widths. ## Beginner homepage and dinosaur reconstruction -The Calculator role comparison and its percentage controls are semantic HTML. -The assembly comes from a real installed model DLL; the named calculation is -a readable summary crosschecked against Microsoft’s public source. See +The dinosaur role comparison and Calculator percentage controls are semantic +HTML. Calculator’s assembly comes from a real installed model DLL; the named +calculation is a readable summary crosschecked against Microsoft’s public source. See [evidence/calculator-and-dino.md](evidence/calculator-and-dino.md). The dinosaur sequence uses an HTML flow: inspect the loaded script, rebuild the diff --git a/website/public/index.html b/website/public/index.html index 747e1715b5e..636ade994e4 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -62,7 +62,7 @@

Find out how
software works.

what it does.

@@ -113,6 +113,215 @@

Set up REA

+
+
+
+

Example 01 · Chrome’s dinosaur game

+

Change the dinosaur’s speed.

+

+ Inspect the speed rule with REA, rebuild a small version, then + change the speed and play. +

+
+
+
+

Play the reconstruction

+
+ + + + 6 → 10 → 13 + + + +
+ +

+ Original rule: start at 6, then increase toward 13. +

+
+ Move the slider to set your own speed. Space or Jump clears a + cactus. +
+ +
+
+
+
+ Example prompt · Dinosaur game + +
+

+ Use REA to inspect this dinosaur game. Why does it get + faster? Show the speed rule, then build a small version with + an adjustable speed. +

+
+

+ Inspected target: + wayou’s Chromium-derived browser edition. +

+
+
+ REA · loaded index.js · selected lines +
+
if (this.currentSpeed < this.config.MAX_SPEED) {
+  this.currentSpeed += this.config.ACCELERATION;
+}
+
+

+ Start at 6. Add 0.001 each + update without a collision, while speed is below + 13. +

+ Open the speed lab → +
+
+
+

From the running game to a new version

+
    +
  1. + 01 · InspectREA returns the loaded scriptThe agent reads the update function and its speed + settings. +
  2. +
  3. + 02 · RebuildUse the recovered ruleStart at 6 and add 0.001 while speed is below 13. +
  4. +
  5. + 03 · Change and playMake speed adjustableKeep the original acceleration, or choose your own + pace. +
  6. +
+
+
+ + See the script, checks and how to try the analysis + +

+ REA inspected the HTTP browser edition through a local debugging + connection and returned the loaded index.js, + including its source and digest. +

+
+
+ Selected settings from the inspected script +
+
ACCELERATION: 0.001,
+MAX_SPEED: 13,
+SPEED: 6
+
+

+ We called the original game’s update function in a controlled + browser check, with obstacles and automatic scheduling disabled. + After 4,000 updates, speed was 10.0; after 10,000, it was 13.0, + rounded to one decimal. +

+

+ The new mini-game keeps that speed rule. Its drawing, jumping and + collision code are a small teaching implementation. + Open the lab to see the new code + and run the same speed check. +

+

+ To inspect the target yourself, follow the + browser connection steps + using + the dinosaur page. Give your agent that page URL and your local debugging + endpoint, then copy the prompt above. +

+

+ Inspected speed code + · + Chromium Authors · BSD license +

+
+
+
+
@@ -122,11 +331,8 @@

What is reverse engineering?

- For example: - why does Calculator turn 200 + 10% into - 220? + In the dinosaur example: + which code makes the game get faster?

@@ -135,36 +341,43 @@

What is reverse engineering?

  • 01
    - Find the code behind the button.Which executable or library handles %? + Find the script behind the game.Which loaded file controls the dinosaur?
  • 02
    - Read the instructions.What do these branches and calls mean? + Find the update function.Where does the game change its speed?
  • 03
    - Recover the calculation.Follow the inputs, operators and constants. + Recover the speed rule.Follow the starting speed, increment and + threshold.
  • 04
    - Check your explanation.Does it agree with the program? + Check the result.Replay the rule and compare it with the game.
  • +

    Find this branch inside a 90 KB script:

    CMP EAX, 0x5c
    JZ 0x180124aba
    CMP EAX, 0x5b
    JZ - 0x180124aba
    MOV EDX, 0x64
    if (this.currentSpeed < this.config.MAX_SPEED) {
    +   this.currentSpeed += this.config.ACCELERATION;
    + }
    @@ -175,13 +388,13 @@

    What is reverse engineering?

    Example prompt · CalculatorExample prompt · Dinosaur game @@ -189,8 +402,8 @@

    What is reverse engineering?

    Use REA to inspect Windows Calculator. Why does 200 + 10% - give 220? Show the code behind the % button.Use REA to inspect this dinosaur game. Why does it get + faster? Show the speed rule.

    @@ -198,11 +411,14 @@

    What is reverse engineering?

    Example explanation from the findings

    - With addition, % calculates a percentage of the first - number. + It starts at 6. Each update without a collision adds 0.001 + while speed is below 13.

    - 10% of 200 is 20.
    Then 200 + 20 = 220.
    + After 4,000 updates, speed is 10.
    + You can replay the rule to check it.

    @@ -222,7 +438,7 @@

    What is reverse engineering?

    -

    Example 01 · Windows Calculator

    +

    Example 02 · Windows Calculator

    What does the % button actually do?

    The answer depends on the operator. Compare addition and @@ -369,6 +585,30 @@

    REA finds the branch. The agent explains it.

    0x64 = 100
    +
    +
    + Example prompt · Calculator + +
    +

    + Use REA to inspect Windows Calculator. Why does 200 + 10% + give 220? Show the code behind the % button. +

    +

    Inspected with REA 4.1.0: Windows Calculator 11.2508.4.0, x64. The readable summary uses names from Microsoft’s public source to @@ -390,215 +630,6 @@

    REA finds the branch. The agent explains it.

    -
    -
    -
    -

    Example 02 · Chrome’s dinosaur game

    -

    Find the speed rule. Rebuild it. Change it.

    -

    - Why does the dinosaur game get faster? Use REA to read the running - page’s script, then turn the answer into a game you can adjust. -

    -
    -
    -
    -
    -
    - Example prompt · Dinosaur game - -
    -

    - Use REA to inspect this dinosaur game. Why does it get - faster? Show the speed rule, then build a small version with - an adjustable speed. -

    -
    -

    - Inspected target: - wayou’s Chromium-derived browser edition. -

    -
    -
    - REA · loaded index.js · selected lines -
    -
    if (this.currentSpeed < this.config.MAX_SPEED) {
    -  this.currentSpeed += this.config.ACCELERATION;
    -}
    -
    -

    - Start at 6. Add 0.001 each - update without a collision, while speed is below - 13. -

    - Open the speed lab → -
    -
    -

    Play the reconstruction

    -
    - - - - 6 → 10 → 13 - - - -
    - -

    - Original rule: start at 6, then increase toward 13. -

    -
    - Move the slider to set your own speed. Space or Jump clears a - cactus. -
    - -
    -
    -
    -

    From the running game to a new version

    -
      -
    1. - 01 · InspectREA returns the loaded scriptThe agent reads the update function and its speed - settings. -
    2. -
    3. - 02 · RebuildUse the recovered ruleStart at 6 and add 0.001 while speed is below 13. -
    4. -
    5. - 03 · Change and playMake speed adjustableKeep the original acceleration, or choose your own - pace. -
    6. -
    -
    -
    - - See the script, checks and how to try the analysis - -

    - REA inspected the HTTP browser edition through a local debugging - connection and returned the loaded index.js, - including its source and digest. -

    -
    -
    - Selected settings from the inspected script -
    -
    ACCELERATION: 0.001,
    -MAX_SPEED: 13,
    -SPEED: 6
    -
    -

    - We called the original game’s update function in a controlled - browser check, with obstacles and automatic scheduling disabled. - After 4,000 updates, speed was 10.0; after 10,000, it was 13.0, - rounded to one decimal. -

    -

    - The new mini-game keeps that speed rule. Its drawing, jumping and - collision code are a small teaching implementation. - Open the lab to see the new code - and run the same speed check. -

    -

    - To inspect the target yourself, follow the - browser connection steps - using - the dinosaur page. Give your agent that page URL and your local debugging - endpoint, then copy the prompt above. -

    -

    - Inspected speed code - · - Chromium Authors · BSD license -

    -
    -
    -
    -
    diff --git a/website/style-guide.md b/website/style-guide.md index c322c1acd21..d8afc1c0038 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -66,6 +66,8 @@ and the specific facts REA supplies to answer it. Put compact setup at the beginning of the homepage: a copyable installation prompt and the ordinary setup command. Follow it with the beginner examples. +Put the most immediate result first. An interactive example can precede the +process explanation; on mobile, show the demo before its supporting code. Show how the reader's task changes using the same question on both sides of a manual/agent comparison. Let returned evidence make REA's contribution visible. From a503376b2f9d9ba14d1602e58e0e93059e9d9181 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:28:25 +0000 Subject: [PATCH 114/791] fix(doctor): find the Homebrew Hopper cask without writing Homebrew caches `rea doctor`, documented as strictly read-only, ran `brew --prefix --cask hopper-disassembler` when Hopper was not in a standard location. Current Homebrew rejects that option combination, but any brew command naming a cask first fetches the cask API, so each doctor run wrote about 600 files (36 MB) into the user's Homebrew cache. Ask brew only for its Caskroom, which brew answers without loading the cask API, and read the application directory Homebrew recorded in the Hopper cask's install receipt. Fixes #1165 Co-Authored-By: Claude Opus 5.5 --- src/application/Doctor.systemHost.test.ts | 65 +++++++++++++++++++++++ src/application/Doctor.ts | 18 +++++-- src/application/homebrew.ts | 56 +++++++++++++++++++ 3 files changed, 135 insertions(+), 4 deletions(-) diff --git a/src/application/Doctor.systemHost.test.ts b/src/application/Doctor.systemHost.test.ts index c26cf15bd33..7a160eae73d 100644 --- a/src/application/Doctor.systemHost.test.ts +++ b/src/application/Doctor.systemHost.test.ts @@ -1,5 +1,9 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { systemDoctorHost } from "./Doctor.js"; it("threads the selected host and environment into system diagnostics", async () => { @@ -41,3 +45,64 @@ it("threads the selected host and environment into system diagnostics", async () calls.every(({ environment: observed }) => observed === environment), ).toBe(true); }); + +it.each([ + [{ default: { appdir: "/Applications" } }, "/Applications"], + [ + { + default: { appdir: "/Applications" }, + env: { appdir: "/Env Apps" }, + explicit: { appdir: "~/Tools" }, + }, + "HOME/Tools", + ], + [ + { default: { appdir: "/Applications" }, env: { appdir: "/Env Apps" } }, + "/Env Apps", + ], +])( + "locates the Hopper cask from its Caskroom receipt without a cask command (%j)", + async (config, expectedAppdir) => { + const root = await createTestTempDirectory("rea-doctor-caskroom-"); + const caskroom = join(root, "Caskroom"); + const metadata = join(caskroom, "hopper-disassembler", ".metadata"); + await mkdir(metadata, { recursive: true }); + await writeFile(join(metadata, "config.json"), JSON.stringify(config)); + const calls: (readonly string[])[] = []; + const host = systemDoctorHost({ + platform: "darwin", + environment: { HOME: join(root, "home") }, + execFileOutput: async (_command, arguments_) => { + calls.push(arguments_); + return { stdout: `${caskroom}\n`, stderr: "" }; + }, + }); + + expect(await host.brewHopperPath()).toBe( + join( + expectedAppdir.replace("HOME", join(root, "home")), + "Hopper Disassembler.app/Contents/MacOS/hopper", + ), + ); + // A cask argument makes Homebrew fetch and cache its API data. + expect(calls).toEqual([["--caskroom"]]); + }, +); + +it("reports no Homebrew Hopper when the cask has no receipt", async () => { + const caskroom = await createTestTempDirectory("rea-doctor-caskroom-"); + const calls: (readonly string[])[] = []; + const host = systemDoctorHost({ + platform: "darwin", + environment: { HOME: caskroom }, + execFileOutput: async (_command, arguments_) => { + calls.push(arguments_); + return { stdout: `${caskroom}\n`, stderr: "" }; + }, + }); + + expect(await host.brewHopperPath()).toBeUndefined(); + expect( + calls.every((arguments_) => arguments_.join(" ") === "--caskroom"), + ).toBe(true); +}); diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 9ff752f3a14..7b523b42890 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -7,7 +7,7 @@ import { analysisErrorRemediationAction } from "../domain/analysisErrorPresentat import { parseBinaryTarget } from "./BinaryTargetResolver.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; import type { JsonValue } from "../domain/jsonValue.js"; -import { probeHomebrew } from "./homebrew.js"; +import { installedCaskAppdir, probeHomebrew } from "./homebrew.js"; import { linuxHopperBinarySupported, linuxSharedLibrariesAvailable, @@ -380,20 +380,30 @@ export const systemDoctorHost = ( supportedLinuxHopper: linuxHopperBinarySupported, async brewHopperPath() { return probeHomebrew(async (command) => { + let caskroom: string; try { - const prefix = ( + // Without a cask argument, `--caskroom` does not load Homebrew's + // cask API, so doctor writes no Homebrew cache files. + caskroom = ( await hostExecFileOutput( command, - ["--prefix", "--cask", "hopper-disassembler"], + ["--caskroom"], commandEnvironment, ) ).stdout.trim(); - return `${prefix}/Hopper Disassembler.app/Contents/MacOS/hopper`; } catch (cause: unknown) { // best-effort cleanup: optional Homebrew probing; absence means uninstalled. void cause; return undefined; } + const appdir = await installedCaskAppdir( + caskroom, + "hopper-disassembler", + homeDirectory, + ); + return appdir === undefined + ? undefined + : join(appdir, "Hopper Disassembler.app/Contents/MacOS/hopper"); }); }, manualHopperPaths: () => manualHopperPaths(homeDirectory), diff --git a/src/application/homebrew.ts b/src/application/homebrew.ts index 2b2070001b0..ae87108394d 100644 --- a/src/application/homebrew.ts +++ b/src/application/homebrew.ts @@ -1,3 +1,8 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { z } from "zod"; + /** Standard executable names and locations used to discover Homebrew. */ const HOMEBREW_COMMANDS = [ "brew", @@ -19,3 +24,54 @@ export const probeHomebrew = async ( } return undefined; }; + +const caskDirectoriesSchema = z + .object({ appdir: z.string().min(1).optional() }) + .loose(); +const caskConfigSchema = z + .object({ + default: caskDirectoriesSchema.optional(), + env: caskDirectoriesSchema.optional(), + explicit: caskDirectoriesSchema.optional(), + }) + .loose(); + +/** + * Read the application directory Homebrew recorded when it installed a cask, + * in Homebrew's precedence: explicit option, then environment, then default. + * Reading the Caskroom avoids `brew` cask commands, which refresh Homebrew's + * API and bootsnap caches. + */ +export const installedCaskAppdir = async ( + caskroom: string, + token: string, + home: string, +): Promise => { + let text: string; + try { + text = await readFile( + join(caskroom, token, ".metadata", "config.json"), + "utf8", + ); + } catch (cause: unknown) { + // An absent or unreadable receipt means the cask is not installed here. + void cause; + return undefined; + } + let parsed: unknown; + try { + parsed = JSON.parse(text); + } catch (cause: unknown) { + // A malformed receipt does not identify an application directory. + void cause; + return undefined; + } + const config = caskConfigSchema.safeParse(parsed); + if (!config.success) return undefined; + const appdir = + config.data.explicit?.appdir ?? + config.data.env?.appdir ?? + config.data.default?.appdir; + if (appdir === undefined) return undefined; + return appdir.startsWith("~/") ? join(home, appdir.slice(2)) : appdir; +}; From e1afbc1914cc7ff90265313e942f34ea4a58a4fc Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 11:38:07 +0800 Subject: [PATCH 115/791] perf(mcp): reuse input schemas without losing caller guidance (#1162) --- docs/mcp-contracts.md | 8 +- src/contracts/inputSchemaPresentation.ts | 163 ++++++++++++ src/contracts/toolInputReferences.test.ts | 247 ++++++++++++++++++ src/contracts/toolSchemaMetadata.ts | 111 ++------ tests/boundary/mcp/toolSchemaValidity.test.ts | 25 +- tests/fixtures/localJsonSchemaReferences.ts | 203 +++++++++----- 6 files changed, 599 insertions(+), 158 deletions(-) create mode 100644 src/contracts/inputSchemaPresentation.ts create mode 100644 src/contracts/toolInputReferences.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 8a5cc6cf9fe..90ffbc97de2 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -31,9 +31,11 @@ check the complete catalog after SDK conversion and its generated counterpart. This is REA's local compatibility profile; individual model APIs can impose additional limits. -Output schemas share repeated definitions through local references while -preserving their complete fields and validation rules. Canonical Zod validation -and the input compatibility profile continue to apply. +Input and output schemas share repeated definitions through schema-local +references while preserving their complete fields and validation rules. Input +properties retain their descriptions and literal examples. Root input unions +keep their inline object presentation to avoid redundant branch nesting. +Canonical Zod validation and the input compatibility profile continue to apply. For passive `compare_web_captures` inputs, pass each complete `inspect_web_page` result in `before.inspection` or `after.inspection`, with an diff --git a/src/contracts/inputSchemaPresentation.ts b/src/contracts/inputSchemaPresentation.ts new file mode 100644 index 00000000000..54006872896 --- /dev/null +++ b/src/contracts/inputSchemaPresentation.ts @@ -0,0 +1,163 @@ +const schemaMaps = new Set([ + "$defs", + "definitions", + "properties", + "patternProperties", + "dependentSchemas", + "dependencies", +]); +const schemaArrays = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]); +const schemaChildren = new Set([ + "items", + "additionalItems", + "contains", + "additionalProperties", + "unevaluatedProperties", + "unevaluatedItems", + "propertyNames", + "not", + "if", + "then", + "else", +]); + +const isObject = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const resolveReference = ( + root: Readonly>, + reference: string, +): unknown => { + if (!reference.startsWith("#/")) return undefined; + let value: unknown = root; + for (const token of reference.slice(2).split("/")) { + const key = token.replaceAll("~1", "/").replaceAll("~0", "~"); + if (!isObject(value) || !Object.hasOwn(value, key)) return undefined; + value = value[key]; + } + return value; +}; + +const referencedDescription = ( + root: Readonly>, + schema: Readonly>, +): string | undefined => { + let value: unknown = schema; + const active = new Set(); + while (isObject(value)) { + if (typeof value.description === "string") return value.description; + if (typeof value.$ref !== "string" || active.has(value.$ref)) + return undefined; + active.add(value.$ref); + value = resolveReference(root, value.$ref); + } + return undefined; +}; + +const describeSchema = ( + schema: unknown, + root: Readonly>, + describeProperty: (property: string) => string, +): unknown => { + if (!isObject(schema)) return schema; + return Object.fromEntries( + Object.entries(schema).map(([key, value]) => { + if (schemaMaps.has(key) && isObject(value)) + return [ + key, + Object.fromEntries( + Object.entries(value).map(([name, child]) => { + const described = describeSchema(child, root, describeProperty); + return [ + name, + key === "properties" && isObject(described) + ? { + ...described, + description: + referencedDescription(root, described) ?? + describeProperty(name), + } + : described, + ]; + }), + ), + ]; + if (schemaArrays.has(key) && Array.isArray(value)) + return [ + key, + value.map((child) => describeSchema(child, root, describeProperty)), + ]; + if (schemaChildren.has(key)) + return [ + key, + Array.isArray(value) + ? value.map((child) => + describeSchema(child, root, describeProperty), + ) + : describeSchema(value, root, describeProperty), + ]; + // Annotation values and literal const/enum data are not schemas. + return [key, value]; + }), + ); +}; + +const flattenRootUnion = ( + value: Record, + describeProperty: (property: string) => string, +): Record => { + if (!Array.isArray(value.anyOf)) return value; + const branches = value.anyOf; + const objects = branches.filter(isObject); + if (objects.length === 0 || objects.length !== branches.length) return value; + const properties = new Map(); + for (const branch of objects) { + if (!isObject(branch.properties)) return value; + for (const [name, schema] of Object.entries(branch.properties)) { + const previous = properties.get(name); + if (!properties.has(name)) properties.set(name, schema); + else if (JSON.stringify(previous) !== JSON.stringify(schema)) + properties.set(name, { + anyOf: [previous, schema], + description: describeProperty(name), + }); + } + } + // Invocation still uses the exact canonical union, including branch rules. + const { anyOf: _anyOf, required: _required, ...root } = value; + const requiredByBranch = objects.map((branch) => + Array.isArray(branch.required) + ? branch.required.filter( + (item): item is string => typeof item === "string", + ) + : [], + ); + const required = requiredByBranch.reduce((common, current) => + common.filter((name) => current.includes(name)), + ); + const minProperties = Math.max( + typeof root.minProperties === "number" ? root.minProperties : 0, + Math.min(...requiredByBranch.map((names) => names.length)), + ); + return { + ...root, + type: "object", + properties: Object.fromEntries(properties), + ...(minProperties > 0 ? { minProperties } : {}), + ...(required.length > 0 ? { required } : {}), + ...(objects.every((branch) => branch.additionalProperties === false) + ? { additionalProperties: false } + : {}), + }; +}; + +/** Preserve property guidance and literal data while advertising object roots. */ +export const presentInputJsonSchema = ( + root: Record, + describeProperty: (property: string) => string, +): Record => { + const described = describeSchema(root, root, describeProperty); + return isObject(described) + ? flattenRootUnion(described, describeProperty) + : root; +}; diff --git a/src/contracts/toolInputReferences.test.ts b/src/contracts/toolInputReferences.test.ts new file mode 100644 index 00000000000..172daf1936a --- /dev/null +++ b/src/contracts/toolInputReferences.test.ts @@ -0,0 +1,247 @@ +import { Ajv } from "ajv"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { inlineLocalJsonSchemaReferences } from "../../tests/fixtures/localJsonSchemaReferences.js"; +import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; +import { TOOL_CONTRACTS, toolContract } from "./toolContracts.js"; +import { toolInputSchemaWithMetadata } from "./toolSchemaMetadata.js"; + +describe("advertised input references", () => { + it.each(["draft-2020-12", "draft-07"] as const)( + "preserves production native-selector constraints in %s", + (target) => { + const contract = toolContract("observe_native_calls"); + const advertised = toolInputSchemaWithMetadata(contract)[ + "~standard" + ].jsonSchema.input({ target }); + const ajv = + target === "draft-2020-12" + ? new Ajv2020({ strict: false }) + : new Ajv({ strict: false }); + const validate = ajv.compile(advertised); + for (const selector of [ + "setFoo:", + "dataTaskWithRequest:completionHandler:", + "", + "bad selector:", + "bad[selector]:", + "set\0Foo:", + "setFoo:\n", + ]) + for (const extra of [{}, { unexpected: true }]) { + const sample = { + breakpoints: [ + { + kind: "objc-method", + class_name: "NSURLSession", + selector, + ...extra, + }, + ], + }; + expect(validate(sample)).toBe( + contract.inputSchema.safeParse(sample).success, + ); + } + }, + ); + + it.each(["draft-2020-12", "draft-07"] as const)( + "preserves complete input meaning and examples through %s references", + (target) => { + for (const contract of TOOL_CONTRACTS) { + const projection = + toolInputSchemaWithMetadata(contract)["~standard"].jsonSchema.input; + const referenced = projection({ target }); + const inline = projection({ + target, + libraryOptions: { reused: "inline" }, + }); + expect(referenced.type, contract.name).toBe("object"); + expect( + inlineLocalJsonSchemaReferences(referenced), + contract.name, + ).toEqual(inlineLocalJsonSchemaReferences(inline)); + expect(referenced.examples, contract.name).toEqual( + contract.examples.map(({ input }) => input), + ); + expect(projection({ target })).toBe(referenced); + } + }, + ); + + it.each(["draft-2020-12", "draft-07"] as const)( + "retains specialized descriptions, literal metadata and strict validation in %s", + (target) => { + const literal = { + properties: { target: { $ref: "literal caller data" } }, + examples: [{ properties: { path: {} } }], + }; + const range = z + .object({ + offset: z.number().int().nonnegative().max(65535), + length: z.number().int().min(1), + }) + .strict() + .describe("Exact byte range in the selected artifact."); + const canonical = z + .object({ + before: range, + after: range, + metadata: z + .record(z.string(), z.unknown()) + .meta({ default: literal, examples: [literal] }), + constructor: z.string(), + }) + .strict(); + const advertised = toolInputSchemaWithMetadata({ + ...toolContract("open_binary"), + inputSchema: canonical, + })["~standard"].jsonSchema.input({ target }); + expect(inlineLocalJsonSchemaReferences(advertised)).toMatchObject({ + properties: { + before: { description: "Exact byte range in the selected artifact." }, + after: { description: "Exact byte range in the selected artifact." }, + metadata: { default: literal, examples: [literal] }, + constructor: { description: "Value for constructor." }, + }, + }); + const ajv = + target === "draft-2020-12" + ? new Ajv2020({ strict: false }) + : new Ajv({ strict: false }); + const validate = ajv.compile(advertised); + const valid = { + before: { offset: 0, length: 1 }, + after: { offset: 65535, length: 2 }, + metadata: literal, + constructor: "constructor data", + }; + const samples = [ + valid, + ...[ + { offset: -1 }, + { offset: 65536 }, + { offset: 0.5 }, + { length: 0 }, + { unexpected: true }, + ].map((change) => ({ + ...valid, + before: { ...valid.before, ...change }, + })), + { ...valid, after: null }, + { ...valid, unexpected: true }, + ]; + expect(canonical.safeParse(valid).success).toBe(true); + for (const sample of samples) + expect(validate(sample)).toBe(canonical.safeParse(sample).success); + }, + ); +}); + +describe("root input presentation", () => { + it("normalizes repeated string intersections without widening validation", () => { + const root = { + type: "string", + minLength: 1, + description: "Exact selector.", + allOf: [ + { type: "string", pattern: "^[^\\s]+$" }, + { allOf: [{ type: "string", pattern: "^[^\\s]+$" }], minLength: 2 }, + ], + }; + const normalized = inlineLocalJsonSchemaReferences(root); + expect(normalized).toEqual({ + type: "string", + minLength: 2, + pattern: "^[^\\s]+$", + description: "Exact selector.", + }); + const ajv = new Ajv2020({ strict: false }); + const original = ajv.compile(root); + const projected = ajv.compile( + z.record(z.string(), z.unknown()).parse(normalized), + ); + for (const sample of ["", "a", "aa", "a a", "a\n", null, {}]) + expect(projected(sample)).toBe(original(sample)); + }); + + it("preserves prototype-named properties in object-root presentation", () => { + const fields = Object.fromEntries([ + ["__proto__", { type: "string" }], + ["constructor", { type: "number" }], + ]); + const branch = { + type: "object", + properties: fields, + required: ["__proto__"], + additionalProperties: false, + }; + const presented = presentInputJsonSchema( + { anyOf: [branch, branch] }, + (property) => `Parameter ${property}`, + ); + expect(presented).toMatchObject({ + type: "object", + required: ["__proto__"], + minProperties: 1, + additionalProperties: false, + properties: fields, + }); + expect(presented.anyOf).toBeUndefined(); + expect(Object.getPrototypeOf(presented.properties)).toBe(Object.prototype); + expect(Object.hasOwn(presented.properties ?? {}, "__proto__")).toBe(true); + }); + + it("retains reference siblings and does not manufacture root branch constraints", () => { + const branch = { $ref: "#/$defs/object", additionalProperties: false }; + const root = { + $defs: { + object: { type: "object", properties: { value: { type: "string" } } }, + }, + anyOf: [branch], + }; + const presented = presentInputJsonSchema(root, (property) => property); + expect(presented.anyOf).toEqual([branch]); + const ajv = new Ajv2020({ strict: false }); + for (const sample of [{}, { value: "value" }, null]) + expect(ajv.compile(presented)(sample)).toBe(ajv.compile(root)(sample)); + }); + + it("preserves specialized guidance across reference chains without following literal references", () => { + const literal = { properties: { literal: {} }, $ref: "#/$defs/missing" }; + const presented = presentInputJsonSchema( + { + $defs: { + "a~b": { + type: "string", + description: "Literal URL selected by the caller.", + }, + alias: { $ref: "#/$defs/a~0b" }, + }, + type: "object", + properties: { + url: { $ref: "#/$defs/alias" }, + other: { $ref: "#/$defs/alias", description: "Explicit override." }, + }, + default: literal, + const: literal, + examples: [literal], + enum: [literal], + }, + (property) => `Parameter ${property}`, + ); + expect(presented).toMatchObject({ + properties: { + url: { description: "Literal URL selected by the caller." }, + other: { description: "Explicit override." }, + }, + default: literal, + const: literal, + examples: [literal], + enum: [literal], + }); + }); +}); diff --git a/src/contracts/toolSchemaMetadata.ts b/src/contracts/toolSchemaMetadata.ts index 3a845a9ba6f..8f5d9b48331 100644 --- a/src/contracts/toolSchemaMetadata.ts +++ b/src/contracts/toolSchemaMetadata.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import type { ToolContract } from "./toolContractTypes.js"; +import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; const PROPERTY_DESCRIPTIONS: Readonly> = { addresses: "Ordered provider-normalized procedure addresses to analyze.", @@ -56,11 +57,25 @@ export const toolInputSchemaWithMetadata = ( withAdvertisedJsonSchema( contract.inputSchema, "input", - (project) => (options) => - flattenRootUnion({ - ...describeProperties(project(options)), + (project) => (options) => { + const shared = project({ + ...options, + libraryOptions: { reused: "ref", ...options.libraryOptions }, + }); + // Root-union flattening compares branch properties before widening them. + // Keep its inline representation: distinct references can name equal + // schemas and otherwise introduce redundant, deeper anyOf properties. + const compatible = Array.isArray(shared.anyOf) + ? project({ + ...options, + libraryOptions: { ...options.libraryOptions, reused: "inline" }, + }) + : shared; + return { + ...presentInputJsonSchema(compatible, fallbackPropertyDescription), examples: contract.examples.map(({ input }) => input), - }), + }; + }, ); /** Share repeated output definitions without changing canonical validation. */ @@ -122,89 +137,10 @@ const memoizeByTarget = ( }; }; -/** Keep MCP tool roots object-shaped for clients that reject root unions. */ -const flattenRootUnion = ( - value: Record, -): Record => { - if (!Array.isArray(value.anyOf)) return value; - const branches = value.anyOf.filter(isObject); - if (branches.length === 0 || branches.length !== value.anyOf.length) - return value; - - const properties: Record = {}; - for (const branch of branches) { - if (!isObject(branch.properties)) return value; - for (const [name, schema] of Object.entries(branch.properties)) { - const previous = properties[name]; - if (previous === undefined) properties[name] = schema; - else if (JSON.stringify(previous) !== JSON.stringify(schema)) - properties[name] = { - anyOf: [previous, schema], - description: fallbackPropertyDescription(name), - }; - } - } - - // The canonical Zod parser still enforces branch validation at invocation. - const { anyOf: _anyOf, required: _required, ...root } = value; - const requiredByBranch = branches.map((branch) => - Array.isArray(branch.required) - ? branch.required.filter( - (item): item is string => typeof item === "string", - ) - : [], - ); - const required = requiredByBranch.reduce((common, current) => - common.filter((name) => current.includes(name)), - ); - const minProperties = Math.max( - typeof root.minProperties === "number" ? root.minProperties : 0, - Math.min(...requiredByBranch.map((names) => names.length)), - ); - return { - ...root, - type: "object", - properties, - ...(minProperties > 0 ? { minProperties } : {}), - ...(required.length > 0 ? { required } : {}), - ...(branches.every((branch) => branch.additionalProperties === false) - ? { additionalProperties: false } - : {}), - }; -}; - -const describeProperties = ( - value: Readonly>, -): Record => - Object.fromEntries( - Object.entries(value).map(([key, child]) => { - if (key !== "properties" || !isObject(child)) - return [key, describeValue(child)]; - return [ - key, - Object.fromEntries( - Object.entries(child).map(([property, propertySchema]) => [ - property, - isObject(propertySchema) && - typeof propertySchema.description !== "string" - ? { - ...describeProperties(propertySchema), - description: fallbackPropertyDescription(property), - } - : describeValue(propertySchema), - ]), - ), - ]; - }), - ); - -const describeValue = (value: unknown): unknown => { - if (Array.isArray(value)) return value.map(describeValue); - return isObject(value) ? describeProperties(value) : value; -}; - const fallbackPropertyDescription = (property: string): string => { - const explicit = PROPERTY_DESCRIPTIONS[property]; + const explicit = Object.hasOwn(PROPERTY_DESCRIPTIONS, property) + ? PROPERTY_DESCRIPTIONS[property] + : undefined; if (explicit !== undefined) return explicit; const words = property.replaceAll("_", " "); if (property.startsWith("max_")) @@ -231,6 +167,3 @@ const fallbackPropertyDescription = (property: string): string => { return `Whether ${words}.`; return `Value for ${words}.`; }; - -const isObject = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); diff --git a/tests/boundary/mcp/toolSchemaValidity.test.ts b/tests/boundary/mcp/toolSchemaValidity.test.ts index d2ac26ddfee..00497150676 100644 --- a/tests/boundary/mcp/toolSchemaValidity.test.ts +++ b/tests/boundary/mcp/toolSchemaValidity.test.ts @@ -103,25 +103,44 @@ function expectKnownAuthorityHints(tools: readonly ToolSchemas[]): void { function expectRecursivePropertyDescriptions( schema: unknown, path: string, + root: unknown = schema, + active: ReadonlySet = new Set(), ): void { if (!isRecord(schema)) return; + if (typeof schema.$ref === "string" && !active.has(schema.$ref)) + expectRecursivePropertyDescriptions( + resolveReference(root, schema.$ref), + `${path}.${schema.$ref}`, + root, + new Set([...active, schema.$ref]), + ); if (isRecord(schema.properties)) { for (const [property, child] of Object.entries(schema.properties)) { expect(child, `${path}.${property}`).toMatchObject({ description: expect.any(String), }); - expectRecursivePropertyDescriptions(child, `${path}.${property}`); + expectRecursivePropertyDescriptions( + child, + `${path}.${property}`, + root, + active, + ); } } for (const key of ["items", "additionalProperties"]) if (schema[key] !== undefined) - expectRecursivePropertyDescriptions(schema[key], path); + expectRecursivePropertyDescriptions(schema[key], path, root, active); for (const key of ["allOf", "anyOf", "oneOf", "prefixItems"]) { const children = schema[key]; if (Array.isArray(children)) children.forEach((child: unknown, index: number) => - expectRecursivePropertyDescriptions(child, `${path}.${key}[${index}]`), + expectRecursivePropertyDescriptions( + child, + `${path}.${key}[${index}]`, + root, + active, + ), ); } } diff --git a/tests/fixtures/localJsonSchemaReferences.ts b/tests/fixtures/localJsonSchemaReferences.ts index aa8efdf131f..c118623f705 100644 --- a/tests/fixtures/localJsonSchemaReferences.ts +++ b/tests/fixtures/localJsonSchemaReferences.ts @@ -19,6 +19,16 @@ const schemaChildren = new Set([ ]); const lowerBounds = new Set(["minimum", "exclusiveMinimum"]); const upperBounds = new Set(["maximum", "exclusiveMaximum"]); +const annotations = new Set([ + "title", + "description", + "$comment", + "default", + "examples", + "deprecated", + "readOnly", + "writeOnly", +]); const scopedKeywords = new Set([ "properties", "patternProperties", @@ -34,6 +44,102 @@ const scopedKeywords = new Set([ const record = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); +const compatibleSibling = ( + key: string, + previous: unknown, + value: unknown, +): boolean => + previous === undefined || + JSON.stringify(previous) === JSON.stringify(value) || + annotations.has(key) || + (typeof previous === "number" && + typeof value === "number" && + (lowerBounds.has(key) || upperBounds.has(key))); + +const mergeSiblings = ( + schema: Record, + siblings: Record, +): Record => { + const merged = { ...schema, ...siblings }; + for (const [key, value] of Object.entries(siblings)) { + const previous = schema[key]; + if (!compatibleSibling(key, previous, value)) + throw new Error(`Conflicting schema reference sibling: ${key}`); + if (typeof previous !== "number" || typeof value !== "number") continue; + if (lowerBounds.has(key)) merged[key] = Math.max(previous, value); + else if (upperBounds.has(key)) merged[key] = Math.min(previous, value); + } + return merged; +}; + +const normalizeStringIntersection = ( + schema: Record, +): Record => { + if ( + (schema.type !== undefined && schema.type !== "string") || + !Array.isArray(schema.allOf) + ) + return schema; + let requiresString = schema.type === "string"; + const validBounds = (value: Record): boolean => + ["minLength", "maxLength"].every( + (key) => + !Object.hasOwn(value, key) || + (typeof value[key] === "number" && + Number.isInteger(value[key]) && + value[key] >= 0), + ) && + (!Object.hasOwn(value, "pattern") || typeof value.pattern === "string"); + if (!validBounds(schema)) return schema; + const patterns = new Set(); + let minimum = typeof schema.minLength === "number" ? schema.minLength : 0; + let maximum = + typeof schema.maxLength === "number" ? schema.maxLength : Infinity; + if (typeof schema.pattern === "string") patterns.add(schema.pattern); + const collect = (child: unknown): boolean => { + if (!record(child) || (child.type !== undefined && child.type !== "string")) + return false; + if ( + !validBounds(child) || + (Object.hasOwn(child, "allOf") && !Array.isArray(child.allOf)) + ) + return false; + if (child.type === "string") requiresString = true; + if ( + Object.keys(child).some( + (key) => + !["type", "pattern", "minLength", "maxLength", "allOf"].includes(key), + ) + ) + return false; + if (typeof child.pattern === "string") patterns.add(child.pattern); + if (typeof child.minLength === "number") + minimum = Math.max(minimum, child.minLength); + if (typeof child.maxLength === "number") + maximum = Math.min(maximum, child.maxLength); + return !Array.isArray(child.allOf) || child.allOf.every(collect); + }; + if (!schema.allOf.every(collect) || !requiresString) return schema; + const { + allOf: _allOf, + pattern: _pattern, + minLength: _min, + maxLength: _max, + ...root + } = schema; + const unique = [...patterns]; + return { + ...root, + type: "string", + ...(minimum > 0 ? { minLength: minimum } : {}), + ...(maximum < Infinity ? { maxLength: maximum } : {}), + ...(unique.length === 1 ? { pattern: unique[0] } : {}), + ...(unique.length > 1 + ? { allOf: unique.map((pattern) => ({ type: "string", pattern })) } + : {}), + }; +}; + /** Inline finite local references for representation-independent schema assertions. */ export const inlineLocalJsonSchemaReferences = ( root: Readonly>, @@ -68,27 +174,10 @@ export const inlineLocalJsonSchemaReferences = ( visit(siblings, active), ], }; - const merged = { ...referenced, ...siblings }; - for (const [key, entry] of Object.entries(siblings)) { - const previous = referenced[key]; - if ( - previous === undefined || - JSON.stringify(previous) === JSON.stringify(entry) - ) - continue; - if (typeof previous === "number" && typeof entry === "number") { - if (lowerBounds.has(key)) { - merged[key] = Math.max(previous, entry); - continue; - } - if (upperBounds.has(key)) { - merged[key] = Math.min(previous, entry); - continue; - } - } - throw new Error(`Conflicting schema reference sibling: ${key}`); - } - return visit(merged, new Set([...active, reference])); + return visit( + mergeSiblings(referenced, siblings), + new Set([...active, reference]), + ); } if (Array.isArray(value.allOf) && value.allOf.length === 1) { const child = visit(value.allOf[0], active); @@ -96,25 +185,11 @@ export const inlineLocalJsonSchemaReferences = ( if ( record(child) && !Object.keys(siblings).some((key) => scopedKeywords.has(key)) && - Object.entries(siblings).every( - ([key, entry]) => - child[key] === undefined || - JSON.stringify(child[key]) === JSON.stringify(entry) || - (typeof child[key] === "number" && - typeof entry === "number" && - (lowerBounds.has(key) || upperBounds.has(key))), + Object.entries(siblings).every(([key, entry]) => + compatibleSibling(key, child[key], entry), ) ) { - const merged = { ...child, ...siblings }; - for (const [key, entry] of Object.entries(siblings)) { - const previous = child[key]; - if (typeof previous !== "number" || typeof entry !== "number") - continue; - if (lowerBounds.has(key)) merged[key] = Math.max(previous, entry); - else if (upperBounds.has(key)) - merged[key] = Math.min(previous, entry); - } - return visit(merged, active); + return visit(mergeSiblings(child, siblings), active); } } const normalized = { ...value }; @@ -139,31 +214,33 @@ export const inlineLocalJsonSchemaReferences = ( normalized.exclusiveMaximum <= normalized.maximum ) delete normalized.maximum; - return Object.fromEntries( - Object.entries(normalized) - .filter(([key]) => key !== "$defs" && key !== "definitions") - .map(([key, child]) => { - if (schemaMaps.has(key) && record(child)) - return [ - key, - Object.fromEntries( - Object.entries(child).map(([name, schema]) => [ - name, - visit(schema, active), - ]), - ), - ]; - if (schemaArrays.has(key) && Array.isArray(child)) - return [key, child.map((schema) => visit(schema, active))]; - if (schemaChildren.has(key)) - return [ - key, - Array.isArray(child) - ? child.map((schema) => visit(schema, active)) - : visit(child, active), - ]; - return [key, child]; - }), + return normalizeStringIntersection( + Object.fromEntries( + Object.entries(normalized) + .filter(([key]) => key !== "$defs" && key !== "definitions") + .map(([key, child]) => { + if (schemaMaps.has(key) && record(child)) + return [ + key, + Object.fromEntries( + Object.entries(child).map(([name, schema]) => [ + name, + visit(schema, active), + ]), + ), + ]; + if (schemaArrays.has(key) && Array.isArray(child)) + return [key, child.map((schema) => visit(schema, active))]; + if (schemaChildren.has(key)) + return [ + key, + Array.isArray(child) + ? child.map((schema) => visit(schema, active)) + : visit(child, active), + ]; + return [key, child]; + }), + ), ); }; return visit(root, new Set()); From c7172047951e7dbb9df58b57ff1b43ac9bd47909 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:43:43 +0000 Subject: [PATCH 116/791] fix(process): stop cancelled Swift helper builds before removing their files Cancelling the macOS process-ownership helper build left swiftc object files in TMPDIR. Node's execFile sends SIGTERM on abort and rejects before the child exits, and swift-driver runs each frontend job in its own process group: on SIGTERM the driver exits while a job keeps writing, so REA removed its root while objects were still being written outside and inside it. Add an opt-in `stopSignal` to execFileOutput that stops the process with the caller's signal on abort or timeout and settles only after it exits. Swift helper builds stop swiftc with SIGINT, on which swift-driver stops its jobs and deletes their objects, and point TMPDIR inside the root cleanup removes. Fixes #1173 Co-Authored-By: Claude Opus 5.5 --- src/native/NativeUiHelperRuntime.ts | 8 +- src/process/DarwinProcessRunTokenReader.ts | 19 +++- src/process/ExecFileOutput.test.ts | 62 +++++++++++++ src/process/ExecFileOutput.ts | 63 +++++++++++-- .../darwinProcessRunTokenReader.test.ts | 89 +++++++++++++++++++ 5 files changed, 231 insertions(+), 10 deletions(-) diff --git a/src/native/NativeUiHelperRuntime.ts b/src/native/NativeUiHelperRuntime.ts index 95e48889515..f9cf877de6c 100644 --- a/src/native/NativeUiHelperRuntime.ts +++ b/src/native/NativeUiHelperRuntime.ts @@ -1,7 +1,8 @@ -import { mkdtemp, rm, symlink } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, symlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; +import { SWIFTC_INTERRUPT } from "../process/DarwinProcessRunTokenReader.js"; import { execFileOutput } from "../process/ExecFileOutput.js"; import { safeParseJson } from "../domain/safeJson.js"; import type { NativeUiHelper } from "./NativeUiObservation.js"; @@ -22,6 +23,9 @@ export const createNativeUiHelperRuntime = () => { ), main, ); + // Keep swiftc's intermediate objects inside the removable root. + const compilerTemporary = join(root, "tmp"); + await mkdir(compilerTemporary); await execFileOutput( "/usr/bin/xcrun", [ @@ -41,6 +45,8 @@ export const createNativeUiHelperRuntime = () => { { timeout: 60_000, maxBuffer: 1024 * 1024, + stopSignal: SWIFTC_INTERRUPT, + env: { ...process.env, TMPDIR: compilerTemporary }, ...(signal === undefined ? {} : { signal }), }, ); diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index 5c176dc0774..85b889bba93 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -65,6 +65,13 @@ const sourceFiles = [ ), ]; +/** + * swift-driver runs each frontend job in its own process group. On SIGINT it + * stops those jobs and deletes their objects before exiting; on SIGTERM it + * exits while a job can still write objects under TMPDIR. + */ +export const SWIFTC_INTERRUPT = "SIGINT"; + /** The native Darwin process-ownership reader could not be prepared. */ export class DarwinProcessOwnershipInspectionError extends Error { constructor(message: string, options?: ErrorOptions) { @@ -149,7 +156,14 @@ export const createDarwinProcessRunTokenReader = ( exitCleanupInstalled = true; if (signal.aborted || closed) throw abortReason(signal); const output = join(operationRoot, "reader"); - const compilerEnvironment = { ...process.env }; + // swiftc keeps intermediate objects under TMPDIR, so keep them inside + // the root that cleanup removes. + const compilerTemporary = join(operationRoot, "tmp"); + await mkdir(compilerTemporary); + const compilerEnvironment: NodeJS.ProcessEnv = { + ...process.env, + TMPDIR: compilerTemporary, + }; // Use Swift's reusable default cache for REA's internal helper compilation. delete compilerEnvironment.CLANG_MODULE_CACHE_PATH; try { @@ -160,6 +174,7 @@ export const createDarwinProcessRunTokenReader = ( timeout: 60_000, maxBuffer: 1024 * 1024, signal, + stopSignal: SWIFTC_INTERRUPT, env: compilerEnvironment, }, ); diff --git a/src/process/ExecFileOutput.test.ts b/src/process/ExecFileOutput.test.ts index bb963987bf9..9f5cd6f2538 100644 --- a/src/process/ExecFileOutput.test.ts +++ b/src/process/ExecFileOutput.test.ts @@ -1,5 +1,10 @@ +import { access, readFile } from "node:fs/promises"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; + import { execFileOutput } from "./ExecFileOutput.js"; describe("execFileOutput", () => { @@ -32,4 +37,61 @@ describe("execFileOutput", () => { ]), ).rejects.toMatchObject({ code: 7, stdout: "out", stderr: "err" }); }); + + it.skipIf(process.platform === "win32")( + "stops an aborted process with the caller's signal and settles after it exits", + async () => { + const directory = await createTestTempDirectory("rea-exec-stop-"); + const ready = join(directory, "ready"); + const stopped = join(directory, "stopped"); + // The child records the signal it received only after a cleanup delay. + const script = [ + 'const fs = require("node:fs");', + 'for (const name of ["SIGINT", "SIGTERM"])', + ` process.on(name, () => setTimeout(() => { fs.writeFileSync(${JSON.stringify(stopped)}, name); process.exit(0); }, 200));`, + `fs.writeFileSync(${JSON.stringify(ready)}, "");`, + "setInterval(() => {}, 1000);", + ].join("\n"); + const controller = new AbortController(); + const execution = execFileOutput(process.execPath, ["-e", script], { + signal: controller.signal, + stopSignal: "SIGINT", + }).then( + () => undefined, + async (cause: unknown) => ({ + cause, + stoppedBeforeSettling: await readFile(stopped, "utf8").catch( + () => null, + ), + }), + ); + for (let attempt = 0; attempt < 500; attempt += 1) { + if ( + await access(ready).then( + () => true, + () => false, + ) + ) + break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + controller.abort(); + const outcome = await execution; + + expect(outcome?.cause).toMatchObject({ name: "AbortError" }); + expect(outcome?.stoppedBeforeSettling).toBe("SIGINT"); + }, + ); + + it("rejects an already aborted stoppable command without starting it", async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + execFileOutput(process.execPath, ["-e", ""], { + signal: controller.signal, + stopSignal: "SIGINT", + }), + ).rejects.toMatchObject({ name: "AbortError" }); + }); }); diff --git a/src/process/ExecFileOutput.ts b/src/process/ExecFileOutput.ts index 4db994101a2..765a8b9cafe 100644 --- a/src/process/ExecFileOutput.ts +++ b/src/process/ExecFileOutput.ts @@ -6,7 +6,29 @@ import { type ExecFileOutputOptions = Omit< ExecFileOptionsWithStringEncoding, "encoding" | "maxBuffer" -> & { readonly maxBuffer?: number }; +> & { + readonly maxBuffer?: number; + /** + * Signal that stops the process on abort or timeout. The promise then + * settles only after the process exits, so the caller can remove files it + * writes; a process still running after a grace period is killed. Without + * it, Node's execFile sends SIGTERM on abort and rejects before the + * process exits. + */ + readonly stopSignal?: NodeJS.Signals; +}; + +/** How long a stopped process may take to exit before it is killed. */ +const STOP_GRACE_MS = 5_000; + +const abortError = (signal: AbortSignal): Error => { + const error = new Error("The operation was aborted", { + cause: signal.reason, + }); + error.name = "AbortError"; + Reflect.set(error, "code", "ABORT_ERR"); + return error; +}; /** Captured subprocess output read from an execFileOutput rejection. */ export interface ExecFileOutputFailure { @@ -47,22 +69,49 @@ export const execFileOutput = ( options: ExecFileOutputOptions = {}, ): Promise<{ readonly stdout: string; readonly stderr: string }> => new Promise((resolve, reject) => { - execFile( + const { stopSignal, signal, ...execOptions } = options; + const stopped = + stopSignal === undefined || signal === undefined + ? undefined + : { signal, stopSignal }; + if (stopped?.signal.aborted === true) { + reject(abortError(stopped.signal)); + return; + } + let aborted = false; + let grace: NodeJS.Timeout | undefined; + const child = execFile( command, [...arguments_], { - ...options, + ...execOptions, + ...(stopped === undefined + ? signal === undefined + ? {} + : { signal } + : { killSignal: stopped.stopSignal }), encoding: "utf8", maxBuffer: options.maxBuffer ?? Number.POSITIVE_INFINITY, }, (error, stdout, stderr) => { - if (error !== null) { - Reflect.set(error, "stdout", stdout); - Reflect.set(error, "stderr", stderr); - reject(error); + stopped?.signal.removeEventListener("abort", onAbort); + clearTimeout(grace); + const failure = + aborted && stopped !== undefined ? abortError(stopped.signal) : error; + if (failure !== null) { + Reflect.set(failure, "stdout", stdout); + Reflect.set(failure, "stderr", stderr); + reject(failure); return; } resolve({ stdout, stderr }); }, ); + const onAbort = (): void => { + if (stopped === undefined) return; + aborted = true; + child.kill(stopped.stopSignal); + grace = setTimeout(() => child.kill("SIGKILL"), STOP_GRACE_MS); + }; + stopped?.signal.addEventListener("abort", onAbort, { once: true }); }); diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index e32dcc813b4..a0dfb396767 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -15,6 +15,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { _electron as electron } from "playwright-core"; import { expect, it, vi } from "vitest"; +import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { @@ -294,6 +295,94 @@ it.skipIf(process.platform === "win32")( }, ); +const temporaryStateSchema = z.object({ + root: z.string(), + temporary: z.string(), +}); + +/** + * A compiler that behaves like swift-driver: a job in its own process group + * writes an object under TMPDIR, SIGINT stops the job and deletes the object, + * and any other signal leaves the job writing after the driver exits. + */ +const writeTemporaryObjectCompiler = async (directory: string) => { + const executable = join(directory, "fake-temporary-xcrun"); + const statePath = join(directory, "temporary-state.json"); + const source = `#!/usr/bin/env node +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { spawn } = require("node:child_process"); +const temporary = fs.mkdtempSync(path.join(os.tmpdir(), "TemporaryDirectory.")); +const object = path.join(temporary, "ProcessRunTokenReader-1.o"); +const job = spawn(process.execPath, ["-e", + "setTimeout(() => { const fs = require('node:fs'); fs.mkdirSync(" + JSON.stringify(temporary) + + ", { recursive: true }); fs.writeFileSync(" + JSON.stringify(object) + ", 'object'); }, 300)"], + { detached: true, stdio: "ignore" }); +job.unref(); +process.on("SIGINT", () => { + job.kill("SIGKILL"); + fs.rmSync(object, { force: true }); + process.exit(130); +}); +const output = process.argv[process.argv.indexOf("-o") + 1]; +fs.writeFileSync(${JSON.stringify(statePath)}, JSON.stringify({ root: path.dirname(output), temporary })); +setInterval(() => {}, 1000); +`; + await writeFile(executable, source); + await chmod(executable, 0o755); + return { executable, statePath }; +}; + +it.skipIf(process.platform === "win32")( + "removes cancelled compiler temporaries with the helper root", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-temporary-test-", + ); + const compiler = await writeTemporaryObjectCompiler(directory); + const reader = createDarwinProcessRunTokenReader({ + xcrun: compiler.executable, + }); + const controller = new AbortController(); + const preparation = reader.prepare(controller.signal).then( + () => undefined, + (cause: unknown) => cause, + ); + let temporary: string | undefined; + try { + const deadline = Date.now() + 5_000; + let state: z.infer | undefined; + while (state === undefined && Date.now() < deadline) { + const parsed = temporaryStateSchema.safeParse( + await readFile(compiler.statePath, "utf8").then( + (text): unknown => JSON.parse(text), + () => undefined, + ), + ); + if (parsed.success) state = parsed.data; + else await new Promise((resolve) => setTimeout(resolve, 10)); + } + if (state === undefined) throw new Error("fake compiler did not start"); + temporary = state.temporary; + expect(state.temporary.startsWith(`${state.root}/`)).toBe(true); + + controller.abort(); + expect(await preparation).toMatchObject({ name: "AbortError" }); + await reader.close(); + // A job that outlived the compiler would recreate the root by now. + await new Promise((resolve) => setTimeout(resolve, 600)); + await expect(access(state.root)).rejects.toThrow(); + } finally { + if (!controller.signal.aborted) controller.abort(); + await preparation; + await reader.close(); + if (temporary !== undefined) + await rm(temporary, { recursive: true, force: true }); + } + }, +); + it.skipIf(process.platform === "win32").each(["first", "second"] as const)( "keeps shared compilation alive when the %s waiter cancels", async (cancelledWaiter) => { From e750c262fbf37100369ab25bf8659daa1a2c4cc7 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 11:46:06 +0800 Subject: [PATCH 117/791] feat(website): clarify reverse engineering and showcase goals --- website/README.md | 17 +- website/evidence/calculator-and-dino.md | 9 +- website/figures.md | 6 + website/public/assets/home.js | 8 +- website/public/assets/styles.css | 63 +++ website/public/examples/dino-lab/index.html | 8 +- website/public/index.html | 400 +++++++++++--------- website/style-guide.md | 11 +- 8 files changed, 323 insertions(+), 199 deletions(-) diff --git a/website/README.md b/website/README.md index ae758a8959a..f0da06ca4ee 100644 --- a/website/README.md +++ b/website/README.md @@ -26,7 +26,7 @@ Open . Refresh the browser after editing a file. ## Pages -- `public/index.html`: compact setup, playable dinosaur reconstruction, manual/agent comparison, Calculator percentage example and case-study overviews. +- `public/index.html`: compact setup, reverse-engineering definition and purpose, manual/agent comparison, dinosaur and Calculator examples, and case-study overviews. - `public/examples/dino-lab/index.html`: adjustable-speed mini-game, recovered rule, original-game check and browser analysis steps. - `public/first-investigation/index.html`: a guided Notes export investigation, from setup to a checked CSV prediction. - `public/showcase/index.html`: the case-study index. @@ -98,12 +98,15 @@ their provenance and the scope of the validation claims. The original executable and complete private Evidence records are not website assets. The homepage starts with a short installation prompt and the ordinary setup -command, followed immediately by the playable dinosaur example. Its game -precedes the prompt and code on small screens. The manual/agent comparison -then explains the investigation using the same dinosaur speed question. -Calculator follows as a native example: its excerpt comes from an installed -model DLL, and Microsoft’s public implementation supplies names and a -crosscheck. Its controls illustrate the percentage rule with generic inputs. +command, then explains what reverse engineering is and why someone would use +it. The manual/agent comparison introduces REA’s role before the examples. +Dinosaur is the first example; Calculator follows as the native example. +Both name the goal and the reason to inspect the program before showing the +demo. A labeled finding explains what REA returned, and the flow distinguishes +REA’s inspection from the agent’s interpretation and implementation. +Its game precedes the prompt and code on small screens. Calculator’s excerpt +comes from an installed model DLL; Microsoft’s public implementation supplies +names and a crosscheck. Its controls apply the percentage rule to generic inputs. The dinosaur example connects an inspected running script to a new playable mini-game. `public/assets/dino-speed.js` owns the recovered speed rule and replay; diff --git a/website/evidence/calculator-and-dino.md b/website/evidence/calculator-and-dino.md index e89138d54ef..b46d2113eed 100644 --- a/website/evidence/calculator-and-dino.md +++ b/website/evidence/calculator-and-dino.md @@ -1,8 +1,10 @@ # Calculator and dinosaur beginner examples Evidence collected on 9 October 2026 with the published REA 4.1.0 package. -The homepage offers compact setup first, then a playable dinosaur reconstruction, -a manual/agent comparison of that investigation, and the Calculator example. +The homepage offers compact setup first, then a reverse-engineering definition, +purpose and manual/agent comparison. Dinosaur is the first worked example; +Calculator is the second. Each states its goal, why the program is inspected, +what REA returned and how the agent uses it. Prompts and replies are examples based on the findings below. Selected code and generic inputs are public; raw captures stay outside the website. @@ -54,7 +56,8 @@ in the installed DLL. The public source commit is a crosscheck; it is not an identified build commit for that installed package. No PDB symbols were loaded. The homepage's named C-like rule is an explanatory summary. Its `200 + 10%` -and `200 × 10%` displays are derived from that rule. Browser checks exercise +and `200 × 10%` controls compute those cases from that rule using the fixed +inputs 200 and 10. Browser checks exercise both percentage illustrations, their results, operator selection and reset. The Microsoft tests were inspected, not executed. A probe-owned Calculator window could be inspected, but it was no longer present at the input-check diff --git a/website/figures.md b/website/figures.md index a87cf7b19f1..e02f894ef7d 100644 --- a/website/figures.md +++ b/website/figures.md @@ -49,6 +49,12 @@ HTML. Calculator’s assembly comes from a real installed model DLL; the named calculation is a readable summary crosschecked against Microsoft’s public source. See [evidence/calculator-and-dino.md](evidence/calculator-and-dino.md). +The homepage explains reverse engineering before the worked examples. Each +example opens with its goal and reason for inspecting the program. A selected +highlight marks the intended result; the “REA returned” block identifies the +evidence. The flows label REA’s inspection and the agent’s interpretation or +reconstruction explicitly. Preserve this context when moving a demo. + The dinosaur sequence uses an HTML flow: inspect the loaded script, rebuild the speed rule, then change and play. Its canvas mini-game and static SVG fallback are authored code, without generated images or copied game sprites. The game diff --git a/website/public/assets/home.js b/website/public/assets/home.js index 61f22231bde..448052cc795 100644 --- a/website/public/assets/home.js +++ b/website/public/assets/home.js @@ -11,13 +11,15 @@ function initializePercentDemo() { const render = () => { const adding = operation === "add"; - const percent = adding ? 20 : 0.1; - const result = adding ? 220 : 20; + const previous = 200; + const current = 10; + const percent = adding ? (current * previous) / 100 : current / 100; + const result = adding ? previous + percent : previous * percent; const operator = adding ? "+" : "×"; expression.textContent = stage === "result" ? `200 ${operator} ${percent} =` : `200 ${operator}`; display.textContent = String( - stage === "input" ? 10 : stage === "percent" ? percent : result, + stage === "input" ? current : stage === "percent" ? percent : result, ); equals.disabled = stage !== "percent"; status.textContent = diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index 54bf52f6753..55b04ee2209 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -1244,6 +1244,57 @@ pre code { color: var(--ink); font-weight: 500; } +.home-purpose { + max-width: 850px; + margin-bottom: 2rem; + color: var(--muted); +} +.prose-highlight { + padding: 0.04em 0.16em; + border-radius: 2px; + background: #e4ecf5; + color: var(--accent); + font-weight: 600; + box-decoration-break: clone; + -webkit-box-decoration-break: clone; +} +.case-brief { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 1.3fr); + gap: 2.5rem; + margin: 0 0 2rem; + padding-block: 1.2rem; + border-top: 1px solid var(--line); + border-bottom: 1px solid var(--line); +} +.case-brief dt, +.finding-label { + margin-bottom: 0.45rem; + font-size: 0.8rem; + font-weight: 600; + color: var(--accent); +} +.case-brief dd { + margin: 0; + color: var(--muted); + font-size: 0.95rem; +} +.case-brief dd strong { + color: var(--ink); +} +.rea-finding { + margin-bottom: 1.4rem; + padding-left: 1rem; + border-left: 2px solid var(--accent); +} +.rea-finding > :last-child { + margin: 0; + font-size: 0.95rem; + color: var(--muted); +} +.rea-finding strong { + color: var(--ink); +} .home-role-comparison { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); @@ -1416,6 +1467,14 @@ pre code { margin: 0; font-size: 0.9rem; } +.percent-rules dd strong, +.percent-rules dd code { + display: block; +} +.percent-rules dd strong { + margin-bottom: 0.25rem; + font-weight: 500; +} .home-evidence-details { margin-top: 2rem; padding-block: 1rem; @@ -1541,6 +1600,10 @@ pre code { } } @media (max-width: 640px) { + .case-brief { + grid-template-columns: minmax(0, 1fr); + gap: 1.2rem; + } .home-hero { padding-block: 2.8rem; } diff --git a/website/public/examples/dino-lab/index.html b/website/public/examples/dino-lab/index.html index a0578a48772..38038c91dfd 100644 --- a/website/public/examples/dino-lab/index.html +++ b/website/public/examples/dino-lab/index.html @@ -57,8 +57,10 @@

    Inspect → rebuild → change

    Rebuild the dinosaur’s speed rule.

    - REA found how the running game increases its speed. This small - reconstruction lets you keep that rule or choose a new speed and play. + To recreate the game’s acceleration, first find the rule that controls + its speed. REA returned the running script; + we used its rule to build this mini-game with + adjustable speed.

    @@ -149,7 +151,7 @@

    Play and change the speed

    -

    What REA found

    +

    Read the script REA returned

    We inspected Find out how
    software works. what it does.

    @@ -113,16 +113,125 @@

    Set up REA

    +
    +
    +
    +

    What is reverse engineering?

    +

    + Finding out how software works by + examining the program itself. +

    +
    +

    + The goal: understand a feature well enough to + explain it, + change it or + rebuild it. +

    +
    +
    +

    Before REA · You investigate

    +
      +
    1. + 01 +
      + Find the relevant code.Locate the loaded script or the binary function. +
      +
    2. +
    3. + 02 +
      + Work out its rules.Follow the inputs, calls and settings. +
      +
    4. +
    5. + 03 +
      + Use and check the result.Explain the behavior, rebuild it or make a change. +
      +
    6. +
    +
    +
    +

    With REA · You ask your agent

    +
    +
    + Example question · Dinosaur game + +
    +

    + Use REA to inspect this dinosaur game. Why does it get + faster? Show the speed rule. +

    +
    +

    + Example explanation from the findings +

    +

    + The game adds a little speed on each update. Its script sets + the starting speed, the increment and the stopping + point. +

    +
    +
    +
    +
    + REA supplies code and observations from the program. + Your agent uses that evidence to explain, rebuild or test a + feature. +
    +
    +
    +
    +

    Example 01 · Chrome’s dinosaur game

    -

    Change the dinosaur’s speed.

    -

    - Inspect the speed rule with REA, rebuild a small version, then - change the speed and play. -

    +

    Why does the dinosaur get faster?

    +
    +
    +
    Goal
    +
    + Rebuild the game with + adjustable speed. +
    +
    +
    +
    Why reverse engineer it?
    +
    + To reproduce its acceleration, we need + the rule in the running game: how fast it + starts, how much it increases and when it stops. +
    +
    +

    Play the reconstruction

    @@ -193,6 +302,14 @@

    Change the dinosaur’s speed.

    >
    +
    +

    REA returned

    +

    + The script actually running in the page. The + agent reads its update function and speed settings, then uses + that rule in a new game. +

    +
    Change the dinosaur’s speed.
    -

    From the running game to a new version

    +

    What each part does

    1. - 01 · InspectREA returns the loaded script01 · REARead the running game’s scriptThe agent reads the update function and its speed - settings.Return its actual code and settings to the agent.
    2. - 02 · RebuildUse the recovered rule02 · Your agentRebuild the rule, add a controlStart at 6 and add 0.001 while speed is below 13.Use the recovered acceleration in a small game with a speed + slider.
    3. - 03 · Change and playMake speed adjustable03 · YouChange the speed and playKeep the original acceleration, or choose your own + >Try the original acceleration, then choose your own pace.
    4. @@ -322,129 +439,31 @@

      Change the dinosaur’s speed.

    -
    -
    -
    -

    What is reverse engineering?

    -

    - Finding out how software works by examining the program. -

    -
    -

    - In the dinosaur example: - which code makes the game get faster? -

    -
    -
    -

    Before REA · You investigate

    -
      -
    1. - 01 -
      - Find the script behind the game.Which loaded file controls the dinosaur? -
      -
    2. -
    3. - 02 -
      - Find the update function.Where does the game change its speed? -
      -
    4. -
    5. - 03 -
      - Recover the speed rule.Follow the starting speed, increment and - threshold. -
      -
    6. -
    7. - 04 -
      - Check the result.Replay the rule and compare it with the game. -
      -
    8. -
    -
    -

    Find this branch inside a 90 KB script:

    - if (this.currentSpeed < this.config.MAX_SPEED) {
    -   this.currentSpeed += this.config.ACCELERATION;
    - }
    -
    -
    -
    -

    With REA · You ask your agent

    -
    -
    - Example prompt · Dinosaur game - -
    -

    - Use REA to inspect this dinosaur game. Why does it get - faster? Show the speed rule. -

    -
    -

    - Example explanation from the findings -

    -

    - It starts at 6. Each update without a collision adds 0.001 - while speed is below 13. -

    -

    - After 4,000 updates, speed is 10.
    - You can replay the rule to check it.
    -

    -
    -
    -

    - Your agent uses REA to locate and read the code, then connects - the findings to your question. -

    -
    -
    - You describe the behavior. The agent carries out the investigation - with REA. -
    -
    -
    -
    -

    Example 02 · Windows Calculator

    -

    What does the % button actually do?

    -

    - The answer depends on the operator. Compare addition and - multiplication. -

    +

    Why does 200 + 10% give 220?

    +
    +
    +
    Goal
    +
    + Explain the result and + recreate Calculator’s % rule. +
    +
    +
    +
    Why reverse engineer it?
    +
    + The same button uses + different rules after + and ×. We inspect the + installed app to find which number the percentage is applied to. +
    +
    +

    Try the calculation

    @@ -507,11 +526,45 @@

    What does the % button actually do?

    -

    REA finds the branch. The agent explains it.

    +
    +

    REA returned

    +

    + The % handler’s original instructions and decompiled + code. + The agent reads the two branches and explains which rule each + operator uses. +

    +

    - The installed app’s calculation code checks for multiplication - or division before computing the percentage. + For this example, previous = 200 and + current = 10.

    +
    +
    + Example prompt · Calculator + +
    +

    + Use REA to inspect Windows Calculator. Why does 200 + 10% + give 220? Show the code behind the % button. +

    +
    Readable summary of the % handler @@ -524,44 +577,54 @@

    REA finds the branch. The agent explains it.

    -
    Addition
    -
    200 + 10% → 200 + 20 → 220
    +
    With +
    +
    + Take 10% of the first number.10% of 200 = 20 → 200 + 20 = 220 +
    -
    Multiplication
    -
    200 × 10% → 200 × 0.1 → 20
    +
    With ×
    +
    + Turn 10% into 0.1.200 × 0.1 = 20 +
    -
    - See the real code and how the answer was checked -
      +
      +

      From the installed app to an explanation

      +
      1. - 01 · Inspect calc.exeA launcher01 · REA reads calc.exeIt launches the Calculator appREA finds ShellExecuteW opening - ms-calculator:.The code opens ms-calculator:. The agent then + selects the installed app’s library.
      2. - 02 · Inspect the appCalcViewModel.dll02 · REA reads the libraryReturn the % handler’s codeIts % handler checks two operator IDs and uses the constant + >The function checks the operator and uses the constant 100.
      3. - 03 · Interpret and checkTwo percentage rules03 · Your agent interprets itExplain and reproduce the rulesMicrosoft’s public implementation and unit tests explain the - same distinction.Crosscheck the branch with Microsoft’s source, then try + and + × above.
      +
      +
      + See the real code and how the answer was checked +
      @@ -585,30 +648,7 @@

      REA finds the branch. The agent explains it.

      0x64 = 100
      -
      -
      - Example prompt · Calculator - -
      -

      - Use REA to inspect Windows Calculator. Why does 200 + 10% - give 220? Show the code behind the % button. -

      -
      +

      Inspected with REA 4.1.0: Windows Calculator 11.2508.4.0, x64. The readable summary uses names from Microsoft’s public source to diff --git a/website/style-guide.md b/website/style-guide.md index d8afc1c0038..e8c5085d444 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -65,9 +65,14 @@ requiring the reader to understand either language. Show a simple agent prompt and the specific facts REA supplies to answer it. Put compact setup at the beginning of the homepage: a copyable installation -prompt and the ordinary setup command. Follow it with the beginner examples. -Put the most immediate result first. An interactive example can precede the -process explanation; on mobile, show the demo before its supporting code. +prompt and the ordinary setup command. Then explain what reverse engineering +means and what it lets someone do, before the worked examples. +Begin each example with its goal and the reason to inspect that program. +Name the actual evidence REA returned and what the agent did with it. +Use short labels, selective bold text and the shared blue highlight to make +those facts visible when scanning. Keep source, interpretation and new code +attributed to their respective roles. On mobile, the demo can precede its +supporting code once its purpose is clear. Show how the reader's task changes using the same question on both sides of a manual/agent comparison. Let returned evidence make REA's contribution visible. From 2a620c06cdcaa32844c4a80aeb3df1793f1ce09e Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Thu, 8 Oct 2026 20:49:32 -0700 Subject: [PATCH 118/791] fix(javascript): resolve package exports targets only to the exact file (#1181) A selected package exports target names one file, and Node tries no extension or directory index for it. REA still resolved it like a relative specifier, so `exports: "./file"` with only `file.js` present, or `"./folder"` with only `folder/index.js`, reported a resolved import that Node rejects under both import and require. Resolve an exports target only to its exact file and otherwise report not-found with a limitation naming the target. Legacy main entries keep their extension and index fallbacks. Fixes #1174 Co-authored-by: Claude Opus 5.5 --- .../JavaScriptArtifactPathResolution.test.ts | 62 +++++++++++++++++++ .../JavaScriptArtifactPathResolution.ts | 15 ++++- .../javascriptPackageExportTargets.test.ts | 13 ++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts index f798ebfc3e6..1b5bd33c4cf 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts @@ -391,6 +391,68 @@ describe("generic package entry heuristics", () => { ); }); +describe("package exports exact targets", () => { + it.each([ + ["./file", "app/node_modules/fixture/file.js"], + ["./folder", "app/node_modules/fixture/folder/index.js"], + ])( + "resolves exports target %s only to that exact file", + (target, nearMiss) => { + const files = fileMap([ + file("app/consumer.js", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ exports: target }), + ), + file(nearMiss, "root"), + ]); + for (const moduleKind of ["import", "require"] as const) { + const resolution = resolve({ + declaredPath: "fixture", + sourcePath: "app/consumer.js", + context: "module-specifier", + moduleKind, + files, + }); + expect(resolution).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + }); + expect(resolution.limitations.join(" ")).toContain( + "exports targets are exact files", + ); + } + }, + ); + + it("keeps extension and index fallbacks for a legacy main entry", () => { + for (const [main, target] of [ + ["./file", "app/node_modules/fixture/file.js"], + ["./folder", "app/node_modules/fixture/folder/index.js"], + ] as const) { + const files = fileMap([ + file("app/consumer.js", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + JSON.stringify({ main }), + ), + file(target, "root"), + ]); + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/consumer.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ resolution_status: "resolved", resolved_path: target }); + } + }); +}); + describe("directory package entrypoint precedence", () => { it.each([ ["require", '{"main":"actual.cjs"}', "actual.cjs"], diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index dca7ac8e7e4..900f1ba49a0 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -105,8 +105,11 @@ const resolvePackagePath = ( ], }, ); - const resolved = - mode === "files-and-index" + // An exports target names one exact file; Node tries no extension or index. + const resolved = exportsTargetUrl + ? (resolveFileCandidates(input, [confined]) ?? + exactExportsTargetNotFound(confined)) + : mode === "files-and-index" ? (resolveFileCandidates(input, [ ...fileCandidates(confined), ...indexCandidates(confined), @@ -619,6 +622,14 @@ const notFoundCandidate = (): CandidateResolution => ({ ], }); +const exactExportsTargetNotFound = (path: string): CandidateResolution => ({ + resolvedPath: null, + status: "not-found", + limitations: [ + `The selected package exports target ${path} does not exist in the inventoried artifact container; exports targets are exact files, so no extension or directory index was tried.`, + ], +}); + const fileUrlPath = (value: string): string | undefined => { if (!value.startsWith("file://")) return undefined; try { diff --git a/tests/boundary/filesystem/javascriptPackageExportTargets.test.ts b/tests/boundary/filesystem/javascriptPackageExportTargets.test.ts index dafd2cba2bf..4d22c8a7a4e 100644 --- a/tests/boundary/filesystem/javascriptPackageExportTargets.test.ts +++ b/tests/boundary/filesystem/javascriptPackageExportTargets.test.ts @@ -51,6 +51,18 @@ const cases: readonly { nodeError: "ERR_INVALID_PACKAGE_TARGET", rejectedTarget: "./../outside.cjs", }, + { + name: "extensionless target is not completed with an extension", + exports: "./actual", + expected: null, + nodeError: "MODULE_NOT_FOUND", + }, + { + // Node reports ERR_UNSUPPORTED_DIR_IMPORT for import, MODULE_NOT_FOUND for require. + name: "directory target does not load its index", + exports: "./folder", + expected: null, + }, { name: "top-level array skips an invalid target", exports: ["./../outside.cjs", "./actual.cjs"], @@ -166,6 +178,7 @@ it.each( "node_modules/fixture/node_modules/actual.cjs", "node_modules/fixture/node_modules.cjs", "node_modules/fixture/literal%2e.cjs", + "node_modules/fixture/folder/index.js", ].map((path) => [path, `module.exports = ${JSON.stringify(path)};`]), ), }; From 0b04d384cca4f22f271d60bcfe67f79cab89c8de Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:51:37 +0000 Subject: [PATCH 119/791] fix(native): refuse truncated executable headers at target admission Target admission accepted a thin Mach-O after reading only its magic and CPU type, so a 12-byte prefix opened as a valid arm64 executable and the first provider query failed as a generic execution failure. ELF headers were likewise accepted after 20 bytes, and a FAT file whose selected slice lies beyond its end was admitted. Require the complete fixed header (Mach-O 28 or 32 bytes, ELF 52 or 64), Mach-O load commands that fit in the file, and a selected FAT slice within the file, checking commitments against the file size rather than the probed prefix. Each refusal names the failed constraint. Fixes #1189 Co-Authored-By: Claude Opus 5.5 --- src/application/BinaryTargetResolver.ts | 7 ++- src/domain/binaryTarget.fixture.ts | 6 +- src/domain/binaryTarget.test.ts | 49 +++++++++++++++ src/domain/binaryTarget.ts | 63 +++++++++++++++---- .../boundary/filesystem/binaryTarget.test.ts | 26 ++++++++ .../artifactProvider.directoryErrors.test.ts | 6 +- 6 files changed, 137 insertions(+), 20 deletions(-) diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index 938633e9b2e..24fb5358f7a 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -237,6 +237,9 @@ const readExecutableMetadata = async ( const prefix = Buffer.alloc(4096); const prefixRead = await handle.read(prefix, 0, prefix.length, 0); const bytes = prefix.subarray(0, prefixRead.bytesRead); + // Header commitments such as Mach-O load commands and FAT slices are + // checked against the whole file, not the probed prefix. + const fileSize = (await handle.stat()).size; if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) { const offset = mzWindowsHeaderOffset(bytes); if (offset !== null) { @@ -251,7 +254,6 @@ const readExecutableMetadata = async ( ? bytes.readUInt16LE(24) + bytes.readUInt16LE(6) * 4 : 0; const headerBytes = bytes.length >= 28 ? bytes.readUInt16LE(8) * 16 : 0; - const fileSize = (await handle.stat()).size; if ( tableEnd > bytes.length && tableEnd <= headerBytes && @@ -279,11 +281,12 @@ const readExecutableMetadata = async ( return parseExecutableHeader( header.subarray(0, headerRead.bytesRead), hostArchitecture, + fileSize, ); } } } - return parseExecutableHeader(bytes, hostArchitecture); + return parseExecutableHeader(bytes, hostArchitecture, fileSize); }; const readPeMetadata = async ( diff --git a/src/domain/binaryTarget.fixture.ts b/src/domain/binaryTarget.fixture.ts index 69587f09249..bbc1dc5ddf1 100644 --- a/src/domain/binaryTarget.fixture.ts +++ b/src/domain/binaryTarget.fixture.ts @@ -4,7 +4,7 @@ export const elf = ( endian: number, machine: number, ): Buffer => { - const bytes = Buffer.alloc(20); + const bytes = Buffer.alloc(class_ === 1 ? 52 : 64); bytes.set([0x7f, 0x45, 0x4c, 0x46, class_, endian]); if (endian === 1) bytes.writeUInt16LE(machine, 18); else bytes.writeUInt16BE(machine, 18); @@ -13,7 +13,9 @@ export const elf = ( /** Build a minimal thin Mach-O header for binary-target parser tests. */ export const thinMach = (magic: number, cpu: number): Buffer => { - const bytes = Buffer.alloc(8); + const bytes = Buffer.alloc( + magic === 0xfeedfacf || magic === 0xcffaedfe ? 32 : 28, + ); bytes.writeUInt32BE(magic, 0); if (magic === 0xcefaedfe || magic === 0xcffaedfe) bytes.writeUInt32LE(cpu, 4); else bytes.writeUInt32BE(cpu, 4); diff --git a/src/domain/binaryTarget.test.ts b/src/domain/binaryTarget.test.ts index d8f35e713f4..053232ac0ba 100644 --- a/src/domain/binaryTarget.test.ts +++ b/src/domain/binaryTarget.test.ts @@ -90,3 +90,52 @@ describe("executable header parsing", () => { expect(parseExecutableHeader(truncatedDirectories, "x64").ok).toBe(false); }); }); + +describe("executable header completeness", () => { + const reason = (bytes: Buffer, fileSize = bytes.length) => { + const result = parseExecutableHeader(bytes, "arm64", fileSize); + return result.ok ? null : result.error; + }; + + it.each([ + [ + thinMach(0xcffaedfe, 0x0100000c).subarray(0, 12), + "a 64-bit header needs 32 bytes", + ], + [ + thinMach(0xcefaedfe, 0x01000007).subarray(0, 27), + "a 32-bit header needs 28 bytes", + ], + [elf(1, 1, 62).subarray(0, 51), "an ELF32 header needs 52 bytes"], + [elf(2, 1, 183).subarray(0, 63), "an ELF64 header needs 64 bytes"], + ])( + "rejects a header shorter than its fixed size (%#)", + (bytes, constraint) => { + expect(reason(bytes)).toContain(constraint); + }, + ); + + it("rejects Mach-O load commands that extend past the file", () => { + const header = thinMach(0xcffaedfe, 0x0100000c); + header.writeUInt32LE(100, 20); + + expect(reason(header)).toBe( + "truncated Mach-O load commands: the header declares 100 bytes after its 32-byte header; the file has 32", + ); + // The resolver probes a prefix; commitments are checked against the file. + expect(reason(header, 132)).toBeNull(); + }); + + it("rejects a FAT file whose host slice extends past the file", () => { + const bytes = fat([0x01000007, 0x0100000c]); + bytes.writeUInt32BE(4096, 8 + 20 + 8); + bytes.writeUInt32BE(64, 8 + 20 + 12); + + expect(reason(bytes, 4100)).toBe( + "truncated FAT slice: the arm64 slice ends at byte 4160; the file has 4100", + ); + expect(reason(bytes, 4160)).toBeNull(); + // Only the slice REA would analyze must be present. + expect(parseExecutableHeader(bytes, "x64", 4100).ok).toBe(true); + }); +}); diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index d5056c27325..9c9a01a5ca0 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -35,28 +35,39 @@ export const parseExecutableHeader = ( bytes.length >= 4 && bytes.subarray(0, 4).equals(Buffer.from([0x7f, 0x45, 0x4c, 0x46])) ) - return parseElf(bytes); + return parseElf(bytes, fileSize); if (bytes.length >= 2 && bytes[0] === 0x4d && bytes[1] === 0x5a) return parseMz(bytes, fileSize); if (bytes.length < 8) return err("truncated or unsupported binary header"); const magic = bytes.readUInt32BE(0); if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) - return parseThinMachO(bytes, magic); + return parseThinMachO(bytes, magic, fileSize); if ([0xcafebabe, 0xbebafeca, 0xcafebabf, 0xbfbafeca].includes(magic)) - return parseFatMachO(bytes, magic, hostArchitecture); + return parseFatMachO(bytes, magic, hostArchitecture, fileSize); return err("unsupported binary format"); }; const parseThinMachO = ( bytes: Buffer, magic: number, + fileSize: number, ): Result => { - if (bytes.length < 8) return err("truncated Mach-O header"); + const is64 = magic === 0xfeedfacf || magic === 0xcffaedfe; + const headerSize = is64 ? 32 : 28; + if (bytes.length < headerSize) + return err( + `truncated Mach-O header: a ${is64 ? 64 : 32}-bit header needs ${headerSize} bytes; the file has ${fileSize}`, + ); const little = magic === 0xcefaedfe || magic === 0xcffaedfe; - const architecture = machArchitecture( - little ? bytes.readUInt32LE(4) : bytes.readUInt32BE(4), - ); + const read = (offset: number): number => + little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset); + const architecture = machArchitecture(read(4)); if (architecture === undefined) return err("unsupported Mach-O architecture"); + const commandBytes = read(20); + if (headerSize + commandBytes > fileSize) + return err( + `truncated Mach-O load commands: the header declares ${commandBytes} bytes after its ${headerSize}-byte header; the file has ${fileSize}`, + ); return ok({ format: "mach-o", architecture, @@ -68,6 +79,7 @@ const parseFatMachO = ( bytes: Buffer, magic: number, host: NodeJS.Architecture, + fileSize: number, ): Result => { const little = magic === 0xbebafeca || magic === 0xbfbafeca; const is64 = magic === 0xcafebabf || magic === 0xbfbafeca; @@ -79,10 +91,24 @@ const parseFatMachO = ( if (count === 0 || count > 128 || bytes.length < 8 + count * entrySize) return err("truncated or invalid FAT architecture table"); const architectures: BinaryArchitecture[] = []; + const sliceEnds = new Map(); for (let index = 0; index < count; index += 1) { - const architecture = machArchitecture(read(8 + index * entrySize)); - if (architecture !== undefined && !architectures.includes(architecture)) - architectures.push(architecture); + const entry = 8 + index * entrySize; + const architecture = machArchitecture(read(entry)); + if (architecture === undefined || architectures.includes(architecture)) + continue; + architectures.push(architecture); + const offset = is64 + ? little + ? bytes.readBigUInt64LE(entry + 8) + : bytes.readBigUInt64BE(entry + 8) + : BigInt(read(entry + 8)); + const size = is64 + ? little + ? bytes.readBigUInt64LE(entry + 16) + : bytes.readBigUInt64BE(entry + 16) + : BigInt(read(entry + 12)); + sliceEnds.set(architecture, offset + size); } const preferred = host === "arm64" @@ -96,6 +122,11 @@ const parseFatMachO = ( : undefined; if (preferred === undefined || !architectures.includes(preferred)) return err(`FAT binary has no host-compatible ${host} architecture`); + const sliceEnd = sliceEnds.get(preferred) ?? 0n; + if (sliceEnd > BigInt(fileSize)) + return err( + `truncated FAT slice: the ${preferred} slice ends at byte ${sliceEnd}; the file has ${fileSize}`, + ); return ok({ format: "mach-o", architecture: preferred, @@ -103,11 +134,19 @@ const parseFatMachO = ( }); }; -const parseElf = (bytes: Buffer): Result => { - if (bytes.length < 20) return err("truncated ELF header"); +const parseElf = ( + bytes: Buffer, + fileSize: number, +): Result => { + if (bytes.length < 6) return err("truncated ELF header"); if (bytes[4] !== 1 && bytes[4] !== 2) return err("unsupported ELF class"); const little = bytes[5] === 1; if (!little && bytes[5] !== 2) return err("unsupported ELF endianness"); + const headerSize = bytes[4] === 1 ? 52 : 64; + if (bytes.length < headerSize) + return err( + `truncated ELF header: an ELF${bytes[4] === 1 ? 32 : 64} header needs ${headerSize} bytes; the file has ${fileSize}`, + ); const machine = little ? bytes.readUInt16LE(18) : bytes.readUInt16BE(18); const architecture = elfArchitecture(machine); if (architecture === undefined) return err("unsupported ELF architecture"); diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index e9da6d2343a..d0b7a395e26 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -325,6 +325,32 @@ describe("binary target I/O: explicit kinds and executable headers", () => { architecture: "x86_64", }); }); + + it("refuses a truncated Mach-O header before any provider sees it", async () => { + const directory = await createTestTempDirectory("rea-target-"); + const path = join(directory, "truncated"); + await writeFile(path, thinMach(0xcffaedfe, 0x0100000c).subarray(0, 12)); + const result = await parseBinaryTarget(path, directory, "arm64"); + expect(result.ok).toBe(false); + expect(result.ok ? undefined : result.error).toMatchObject({ + path, + reason: + "truncated Mach-O header: a 64-bit header needs 32 bytes; the file has 12", + }); + }); + + it("checks Mach-O load commands against the whole file beyond the probe", async () => { + const directory = await createTestTempDirectory("rea-target-"); + const path = join(directory, "large-commands"); + const header = thinMach(0xcffaedfe, 0x0100000c); + header.writeUInt32LE(8192, 20); + await writeFile(path, Buffer.concat([header, Buffer.alloc(8192)])); + const result = await parseBinaryTarget(path, directory, "arm64"); + expect(result.ok && result.value).toMatchObject({ + format: "mach-o", + architecture: "arm64", + }); + }); }); describe("app executable filename fidelity", () => { diff --git a/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts b/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts index 8a05befd92c..e50a6b5d39b 100644 --- a/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts +++ b/tests/boundary/providers/native/artifactProvider.directoryErrors.test.ts @@ -6,6 +6,7 @@ import { describe, expect, it } from "vitest"; import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; +import { thinMach } from "../../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; describe("artifact directory diagnostics", () => { @@ -22,10 +23,7 @@ describe("artifact directory diagnostics", () => { join(contentsPath, "Info.plist"), "CFBundleExecutableFixture", ); - const machHeader = Buffer.alloc(8); - machHeader.writeUInt32BE(0xfeedfacf, 0); - machHeader.writeUInt32BE(0x0100000c, 4); - await writeFile(executablePath, machHeader); + await writeFile(executablePath, thinMach(0xfeedfacf, 0x0100000c)); await writeFile(blockedPath, "member contents"); await chmod(blockedPath, 0); try { From d0ff57a45c8da8938e7ae16d00bfd5ec63310af3 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:56:38 +0000 Subject: [PATCH 120/791] fix(domain): report target read-permission denials as access_denied Opening a target the current user cannot read returned `target_unavailable`, the same code as a missing target, although the reason named EACCES. Clients that branch on the error code could not choose read-access recovery, contrary to the boundary contract that separates host permission denial from unavailable targets. A BinaryTargetError whose cause is an OS EACCES or EPERM now records that system code and projects as `access_denied` with `system_code`, the filesystem-read boundary, and read-access remediation, while keeping its specific reason and path. This covers target files, app bundle paths and Info.plist reads, and selected plist reads, since every site passes the OS error as the cause. Fixes #1188 Co-Authored-By: Claude Opus 5.5 --- src/domain/analysisErrorPresentation.ts | 5 ++- src/domain/analysisErrorProjection.test.ts | 36 +++++++++++++++++++ src/domain/analysisErrorProjection.ts | 10 +++++- src/domain/configurationErrors.ts | 9 +++++ .../nativePlistSelectionFailureCli.test.ts | 3 +- .../boundary/filesystem/binaryTarget.test.ts | 24 +++++++++++++ 6 files changed, 84 insertions(+), 3 deletions(-) diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index cc3dc7e40b4..db00c41ccf2 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -74,7 +74,10 @@ export const analysisErrorRemediationAction = ( error.constraint === "directory_requires_file" ) return "For a JavaScript/Electron application directory, call analyze_javascript_application with input_path or run `rea analyze `. For binary analysis, select its executable file."; - if (error instanceof AnalysisAccessDeniedError) + if ( + error instanceof AnalysisAccessDeniedError || + (error instanceof BinaryTargetError && error.systemCode !== undefined) + ) return "Check the current process's read access to the selected path. Retry with a readable local file."; if (error instanceof AnalysisArtifactChangedError) return "Wait until the selected file is stable. For an active binary session, reopen the target with open_binary before retrying so REA acquires its current identity; for a CLI command or target-free tool, rerun the operation."; diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index 8177b08abc0..96525d518b6 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -439,6 +439,42 @@ describe("analysis error projection: caller contract", () => { expect(JSON.stringify(projected)).not.toContain("secret-token"); }); + it.each(["EACCES", "EPERM"] as const)( + "reports a target the host refused to read (%s) as access_denied", + (systemCode) => { + const denied = Object.assign(new Error("denied"), { code: systemCode }); + const reason = `permission denied while reading target: ${systemCode}`; + expect( + projectAnalysisError( + new BinaryTargetError("/local/targets/app", reason, { + cause: denied, + }), + ), + ).toMatchObject({ + code: "access_denied", + category: "unavailable", + retryable: false, + remediation: { + action: + "Check the current process's read access to the selected path. Retry with a readable local file.", + }, + details: { + path: "/local/targets/app", + reason, + system_code: systemCode, + boundary: "filesystem-read", + }, + }); + expect( + projectAnalysisError( + new BinaryTargetError("/local/targets/app", "missing", { + cause: Object.assign(new Error("missing"), { code: "ENOENT" }), + }), + ), + ).toMatchObject({ code: "target_unavailable" }); + }, + ); + it("uses explicit capability recovery while retaining the constraint", () => { const projected = projectAnalysisError( new AnalysisCapabilityUnavailableError( diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 66e5c77e9c3..e0419c84af6 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -101,6 +101,10 @@ const underlyingErrorCode = ( : browserErrorCode(error.reason); if (error instanceof ArtifactOperationError) return artifactOperationCode(error); + if (error instanceof BinaryTargetError) + return error.systemCode === undefined + ? "target_unavailable" + : "access_denied"; if (error instanceof EvidenceFileError) return evidenceFileCode(error.reason); if (error instanceof UnknownRegistryError) return unknownRegistryCode(error.reason); @@ -162,6 +166,7 @@ const processCaptureCode = ( type SpecializedErrorTag = | "ArtifactOperationError" + | "BinaryTargetError" | "BrowserObservationError" | "EvidenceFileError" | "ProcessCaptureError" @@ -186,7 +191,6 @@ const STATIC_ERROR_CODES = { HopperStartError: "provider_unavailable", ConfigurationError: "configuration_invalid", NoBinaryOpenError: "target_unavailable", - BinaryTargetError: "target_unavailable", EvidenceIntegrityError: "evidence_integrity_mismatch", ProviderAdapterError: "execution_failure", HopperRemoteError: "execution_failure", @@ -202,6 +206,7 @@ const staticErrorCode = ( ): AnalysisErrorProjection["code"] => { switch (tag) { case "ArtifactOperationError": + case "BinaryTargetError": case "BrowserObservationError": case "EvidenceFileError": case "ProcessCaptureError": @@ -489,6 +494,9 @@ const lifecycleErrorDetails = ( ...(error.constraint === undefined ? {} : { constraint: error.constraint }), + ...(error.systemCode === undefined + ? {} + : { system_code: error.systemCode, boundary: "filesystem-read" }), }; return undefined; }; diff --git a/src/domain/configurationErrors.ts b/src/domain/configurationErrors.ts index c7c07f75e6b..d1c89d03c6b 100644 --- a/src/domain/configurationErrors.ts +++ b/src/domain/configurationErrors.ts @@ -27,6 +27,8 @@ export interface BinaryTargetErrorOptions extends AnalysisErrorOptions { export class BinaryTargetError extends AnalysisError { readonly _tag = "BinaryTargetError"; readonly constraint: BinaryTargetErrorOptions["constraint"]; + /** The OS read-permission code of the cause, when the host denied access. */ + readonly systemCode: "EACCES" | "EPERM" | undefined; constructor( readonly path: string, readonly reason: string, @@ -34,5 +36,12 @@ export class BinaryTargetError extends AnalysisError { ) { super(`Cannot open artifact: ${reason}`, options); this.constraint = options?.constraint; + this.systemCode = readPermissionCode(options?.cause); } } + +const readPermissionCode = (cause: unknown): "EACCES" | "EPERM" | undefined => { + const code: unknown = + cause instanceof Error ? Reflect.get(cause, "code") : undefined; + return code === "EACCES" || code === "EPERM" ? code : undefined; +}; diff --git a/tests/boundary/cli/nativePlistSelectionFailureCli.test.ts b/tests/boundary/cli/nativePlistSelectionFailureCli.test.ts index fc5a4d0d2b0..df3733d5158 100644 --- a/tests/boundary/cli/nativePlistSelectionFailureCli.test.ts +++ b/tests/boundary/cli/nativePlistSelectionFailureCli.test.ts @@ -90,10 +90,11 @@ describe.skipIf(process.platform !== "darwin")( }); expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ - code: "target_unavailable", + code: "access_denied", details: { path: locked, reason: "permission denied while reading plist (EACCES)", + system_code: "EACCES", }, }); }, diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index e9da6d2343a..54a70e00aea 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -22,6 +22,7 @@ import { pe, thinMach, } from "../../../src/domain/binaryTarget.fixture.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("binary target I/O: app plist permissions and decoding", () => { @@ -48,6 +49,7 @@ describe("binary target I/O: app plist permissions and decoding", () => { path: join(app, "Contents", "Info.plist"), reason: expect.stringContaining(message), cause: denied, + systemCode: code, }, }); }); @@ -232,6 +234,28 @@ describe("binary target I/O: filesystem and app bundle target resolution", () => expect((await parseBinaryTarget("missing", directory)).ok).toBe(false); }); + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports an unreadable target as a host read denial", + async () => { + const directory = await createTestTempDirectory("rea-target-denied-"); + const path = join(directory, "unreadable"); + await writeFile(path, thinMach(0xcffaedfe, 0x0100000c)); + await chmod(path, 0); + try { + const result = await parseBinaryTarget(path, directory, "arm64"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ path, systemCode: "EACCES" }); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "access_denied", + details: { path, system_code: "EACCES" }, + }); + } finally { + await chmod(path, 0o600); + } + }, + ); + it("rejects non-regular targets before reading them", async () => { const directory = await createTestTempDirectory("rea-target-"); expect((await parseBinaryTarget(directory)).ok).toBe(false); From facaae865af243656ab8f557aeb42f5909748f2b Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 03:59:19 +0000 Subject: [PATCH 121/791] test(native): build the .app fixture executable with a complete Mach-O header Co-Authored-By: Claude Opus 5.5 --- tests/composition/analysis-sessions/appleApplication.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/composition/analysis-sessions/appleApplication.test.ts b/tests/composition/analysis-sessions/appleApplication.test.ts index 5506ff09d07..6c5cd1fbba2 100644 --- a/tests/composition/analysis-sessions/appleApplication.test.ts +++ b/tests/composition/analysis-sessions/appleApplication.test.ts @@ -18,6 +18,7 @@ import { import { projectAppleApplicationEvidence } from "../../../src/application/apple/AppleApplicationService.js"; import { runProviderAnalysis } from "../../../src/composition/directAnalysis.js"; +import { thinMach } from "../../../src/domain/binaryTarget.fixture.js"; import { appleApplicationProjectionResultSchema } from "../../../src/domain/apple/appleApplication.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; @@ -188,7 +189,8 @@ describe("macOS application projection", () => { it("projects a macOS .app directory inventory with nested bundle roles", async () => { const root = await createTestTempDirectory("rea-apple-macos-"); const app = join(root, "Fixture.app"); - const machO = Uint8Array.from([0xcf, 0xfa, 0xed, 0xfe, 0x0c, 0, 0, 1]); + // A complete thin arm64 header, as target admission requires. + const machO = thinMach(0xcffaedfe, 0x0100000c); const files: [string, string | Uint8Array][] = [ [ "Contents/Info.plist", From 291bc6fb70f47d28d23440f03a8a7fa49eb9a391 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 12:02:58 +0800 Subject: [PATCH 122/791] feat(website): show manual analysis beside a one-prompt workflow --- website/README.md | 9 + website/evidence/calculator-and-dino.md | 9 + website/figures.md | 12 +- .../assets/figures/analyst-thinking.svg | 15 + website/public/assets/home.js | 8 + website/public/assets/main.js | 35 ++- website/public/assets/styles.css | 258 ++++++++++++++++-- website/public/index.html | 228 +++++++++++++--- website/style-guide.md | 5 + 9 files changed, 491 insertions(+), 88 deletions(-) create mode 100644 website/public/assets/figures/analyst-thinking.svg diff --git a/website/README.md b/website/README.md index f0da06ca4ee..0f29bc32bb1 100644 --- a/website/README.md +++ b/website/README.md @@ -100,6 +100,15 @@ and complete private Evidence records are not website assets. The homepage starts with a short installation prompt and the ordinary setup command, then explains what reverse engineering is and why someone would use it. The manual/agent comparison introduces REA’s role before the examples. +The comparison uses the same Calculator question on both sides: three +selectable manual steps pair original instructions with an analyst illustration +and thought bubbles; the agent side begins with “One prompt.” and a copyable +question. Thoughts illustrate the reasoning behind the findings. The shared +code-step script selects the initial note on load, so all three notes remain +readable when JavaScript is disabled. +The original-instruction panel starts expanded on desktop. On narrow screens, +the homepage script collapses it; a native disclosure keeps the code available +while the thought bubble and agent prompt remain close together. Dinosaur is the first example; Calculator follows as the native example. Both name the goal and the reason to inspect the program before showing the demo. A labeled finding explains what REA returned, and the flow distinguishes diff --git a/website/evidence/calculator-and-dino.md b/website/evidence/calculator-and-dino.md index b46d2113eed..a7e9c093e18 100644 --- a/website/evidence/calculator-and-dino.md +++ b/website/evidence/calculator-and-dino.md @@ -36,6 +36,15 @@ the constant 100 in both paths. One path divides the current value by 100; the other combines the current value with the previous value divided by 100. The published assembly is selected exactly from the returned instruction list. +The opening manual/agent comparison selects instructions at +`0x180124945–0x180124992` and `0x180124a14–0x180124a34`, with explicit omissions. +Its three highlights identify the operator comparisons, construction of 100 +and arithmetic helper calls, and the final multiplication path. The analyst +thought bubbles illustrate the work of decoding those facts; they are not a +recorded analyst transcript or a measured comparison of investigation times. +The same percentage question appears in the agent prompt. Helper meanings and +operator names are interpretations checked against the public source below. + The binary is stripped. A local instruction scan located candidates using operator IDs and the constant 100; REA then produced the complete function dossier. Function selection was analyst-assisted. The illustrative agent diff --git a/website/figures.md b/website/figures.md index e02f894ef7d..c0ab93d609c 100644 --- a/website/figures.md +++ b/website/figures.md @@ -13,6 +13,7 @@ The published diagrams are SVG source assets: - `public/assets/figures/rea-investigation-flow.svg` — 1774 × 887 viewBox. - `public/assets/figures/dx-ball-sound-pan-investigation.svg` — 1774 × 887 viewBox. - `public/assets/figures/th04-bullet-ring.svg` — 1000 × 520 viewBox. +- `public/assets/figures/analyst-thinking.svg` — 160 × 120 viewBox. Two original layout references were generated with the built-in image generation tool and reviewed visually. Their prompts are retained below. The final SVGs @@ -44,11 +45,20 @@ rendering at desktop and mobile widths. ## Beginner homepage and dinosaur reconstruction -The dinosaur role comparison and Calculator percentage controls are semantic +The manual/agent comparison and Calculator percentage controls are semantic HTML. Calculator’s assembly comes from a real installed model DLL; the named calculation is a readable summary crosschecked against Microsoft’s public source. See [evidence/calculator-and-dino.md](evidence/calculator-and-dino.md). +The introduction pairs selected Calculator instructions with three selectable +analyst thoughts: decode operator branches, trace arithmetic calls, then name +the percentage rule. The small analyst illustration is authored SVG; the +thought bubbles, findings and labels are selectable HTML. The thoughts +illustrate the analysis rather than reproduce a recorded conversation. The +agent side asks the same question and separates REA’s evidence from the +agent’s explanation. Keep those roles and the instruction highlights aligned +when changing a step. + The homepage explains reverse engineering before the worked examples. Each example opens with its goal and reason for inspecting the program. A selected highlight marks the intended result; the “REA returned” block identifies the diff --git a/website/public/assets/figures/analyst-thinking.svg b/website/public/assets/figures/analyst-thinking.svg new file mode 100644 index 00000000000..53a92d5d933 --- /dev/null +++ b/website/public/assets/figures/analyst-thinking.svg @@ -0,0 +1,15 @@ + + An analyst thinking at a computer + + + + + + + + + + + + + diff --git a/website/public/assets/home.js b/website/public/assets/home.js index 448052cc795..27cd1b32f22 100644 --- a/website/public/assets/home.js +++ b/website/public/assets/home.js @@ -55,4 +55,12 @@ function initializePercentDemo() { render(); } +function initializeExpertAssembly() { + const excerpt = document.querySelector("[data-expert-assembly]"); + if (excerpt && window.matchMedia("(max-width: 640px)").matches) { + excerpt.open = false; + } +} + +initializeExpertAssembly(); initializePercentDemo(); diff --git a/website/public/assets/main.js b/website/public/assets/main.js index 98751f28081..b927dff64f8 100644 --- a/website/public/assets/main.js +++ b/website/public/assets/main.js @@ -39,23 +39,28 @@ initializeCopyButtons(); function initializeStepComparisons() { document.querySelectorAll("[data-step-comparison]").forEach((comparison) => { const buttons = comparison.querySelectorAll("[data-select-step]"); - buttons.forEach((button) => { - button.addEventListener("click", () => { - const selected = button.getAttribute("data-select-step"); - buttons.forEach((candidate) => { - candidate.setAttribute("aria-pressed", String(candidate === button)); - }); - comparison.querySelectorAll("[data-step]").forEach((fragment) => { - fragment.classList.toggle( - "is-active", - fragment.getAttribute("data-step") === selected, - ); - }); - comparison.querySelectorAll("[data-step-note]").forEach((note) => { - note.hidden = note.getAttribute("data-step-note") !== selected; - }); + const selectStep = (button) => { + const selected = button.getAttribute("data-select-step"); + buttons.forEach((candidate) => { + candidate.setAttribute("aria-pressed", String(candidate === button)); + }); + comparison.querySelectorAll("[data-step]").forEach((fragment) => { + fragment.classList.toggle( + "is-active", + fragment.getAttribute("data-step") === selected, + ); + }); + comparison.querySelectorAll("[data-step-note]").forEach((note) => { + note.hidden = note.getAttribute("data-step-note") !== selected; }); + }; + buttons.forEach((button) => { + button.addEventListener("click", () => selectStep(button)); }); + const initial = Array.from(buttons).find( + (button) => button.getAttribute("aria-pressed") === "true", + ); + if (initial) selectStep(initial); }); } diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index 55b04ee2209..3bcc84d9e05 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -1297,10 +1297,18 @@ pre code { } .home-role-comparison { display: grid; - grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); - gap: 0 2.5rem; + grid-template-columns: minmax(0, 1.65fr) minmax(0, 1fr); + gap: 0 2rem; margin: 0; } +.comparison-question { + margin-block: 1.8rem 1.2rem; + font-size: 1.05rem; + color: var(--muted); +} +.comparison-question strong { + color: var(--ink); +} .comparison-label { padding-bottom: 0.8rem; border-bottom: 1px solid var(--line); @@ -1310,47 +1318,204 @@ pre code { .agent-investigation > .comparison-label { color: var(--accent); } -.manual-steps { +.expert-step-selector { + grid-template-columns: repeat(3, minmax(0, 1fr)); + margin-block: 1.2rem 1rem; +} +.expert-step-selector button { + min-height: 44px; + padding: 0.6rem 0.4rem; + font-size: 0.75rem; +} +.expert-step-selector button span { + font-size: 0.7rem; +} +.expert-analysis-grid { display: grid; - gap: 1rem; - padding: 0; + grid-template-columns: minmax(0, 1fr) minmax(0, 0.52fr); + align-items: start; + gap: 0.9rem; +} +.expert-assembly { + min-width: 0; + margin: 0; +} +.expert-assembly .code-header { + padding: 0.7rem 0.8rem; + font-size: 0.7rem; + cursor: pointer; list-style: none; } -.manual-steps li { - display: flex; - gap: 1rem; +.expert-assembly summary::-webkit-details-marker { + display: none; +} +.expert-assembly summary:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 3px; +} +.assembly-disclosure::after { + content: "Show +"; + color: var(--accent); + white-space: nowrap; +} +.expert-assembly[open] .assembly-disclosure::after { + content: "Hide −"; +} +.expert-assembly pre { + margin: 0; + padding: 0.8rem; + font-size: 0.7rem; + line-height: 1.6; +} +.assembly-omission { + color: var(--muted); +} +.expert-reasoning { + min-width: 0; + padding-top: 0.1rem; +} +.expert-thought { + position: relative; + padding: 1rem 0.9rem; + border: 1px solid var(--line); + border-radius: 16px; + background: var(--page); +} +.expert-thought + .expert-thought { + margin-top: 1rem; +} +.expert-thought::before, +.expert-thought::after { + content: ""; + position: absolute; + border: 1px solid var(--line); + border-radius: 50%; + background: var(--page); +} +.expert-thought::before { + bottom: -18px; + right: 38px; + width: 12px; + height: 12px; +} +.expert-thought::after { + bottom: -30px; + right: 47px; + width: 6px; + height: 6px; +} +.expert-thought > p { margin: 0; + font-size: 0.85rem; + line-height: 1.65; +} +.expert-thought > .thought-label { + margin-bottom: 0.6rem; + color: var(--accent); + font-size: 0.7rem; + font-weight: 600; } -.manual-steps li > span { - padding-top: 0.2rem; +.expert-thought > .expert-task { + margin-top: 0.8rem; color: var(--muted); - font: - 0.75rem "SFMono-Regular", - Consolas, - "Liberation Mono", - monospace; -} -.manual-steps strong, -.manual-steps small { + font-size: 0.75rem; +} +.expert-facts { + display: grid; + gap: 0.45rem; + margin: 0.9rem 0 0; + padding-top: 0.8rem; + border-top: 1px solid var(--line); + font-size: 0.7rem; +} +.expert-facts div { + display: flex; + justify-content: space-between; + gap: 0.5rem; +} +.expert-facts dt { + font-family: "SFMono-Regular", Consolas, "Liberation Mono", monospace; +} +.expert-facts dd { + margin: 0; + color: var(--accent); +} +.expert-rule { + display: grid; + gap: 0.3rem; + margin-top: 0.9rem; + padding-top: 0.8rem; + border-top: 1px solid var(--line); + font-size: 0.75rem; +} +.expert-rule strong { + color: var(--accent); + font-size: 1.4rem; +} +.expert-illustration { display: block; + width: 120px; + height: 90px; + margin: 2rem 0 0 auto; } -.manual-steps strong { - font-size: 0.95rem; +.expert-caption { + margin: 0.5rem 0 0; + color: var(--muted); + font-size: 0.65rem; + text-align: right; +} +.manual-work-summary { + margin: 1rem 0 0; + color: var(--muted); + font-size: 0.85rem; +} +.manual-work-summary strong { font-weight: 500; } -.manual-steps small { - margin-top: 0.15rem; +.agent-one-prompt { + margin: 1.2rem 0 0.3rem; + color: var(--accent); + font-size: clamp(1.8rem, 3vw, 2.5rem); + font-weight: 600; + letter-spacing: -0.045em; +} +.agent-intro { + margin: 0 0 1.2rem; color: var(--muted); - font-size: 0.8rem; + font-size: 0.9rem; } -.manual-code { - margin-top: 1.3rem; - padding: 1rem 1.25rem; - border-left: 2px solid var(--line); - background: var(--soft); +.agent-percent-rule { + display: grid; + justify-items: start; + gap: 0.2rem; + margin-top: 0.9rem; + color: var(--accent); +} +.agent-percent-rule code { + color: var(--ink); +} +.agent-percent-rule > span { + margin-left: 0.4rem; color: var(--muted); - font-size: 0.8rem; - line-height: 1.65; +} +.agent-percent-rule strong { + font-size: 1.2rem; +} +.agent-evidence-roles { + display: grid; + gap: 1rem; + margin: 1.2rem 0 0; +} +.agent-evidence-roles dt { + margin-bottom: 0.25rem; + color: var(--accent); + font-size: 0.75rem; + font-weight: 600; +} +.agent-evidence-roles dd { + margin: 0; + color: var(--muted); + font-size: 0.85rem; } .agent-investigation-note { color: var(--muted); @@ -1600,6 +1765,39 @@ pre code { } } @media (max-width: 640px) { + .expert-analysis-grid { + grid-template-columns: minmax(0, 1fr); + } + .expert-reasoning { + display: grid; + grid-template-columns: minmax(0, 1fr) 88px; + column-gap: 0.75rem; + align-items: end; + } + .expert-thought::before { + right: -18px; + bottom: 42px; + } + .expert-thought::after { + right: -30px; + bottom: 36px; + } + .expert-illustration { + width: 88px; + height: 66px; + margin: 0; + } + .expert-caption { + grid-column: 1 / -1; + text-align: left; + } + .expert-step-selector button { + font-size: 0.7rem; + } + .expert-step-selector button span { + display: block; + margin: 0 0 0.2rem; + } .case-brief { grid-template-columns: minmax(0, 1fr); gap: 1.2rem; diff --git a/website/public/index.html b/website/public/index.html index 1722767dde4..26f8d2b0a8e 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -128,51 +128,181 @@

      What is reverse engineering?

      change it or rebuild it.

      +

      + One question: + why does Calculator give 220 for 200 + 10%? +

      -
      +

      Before REA · You investigate

      -
        -
      1. - 01 -
        - Find the relevant code.Locate the loaded script or the binary function. -
        -
      2. -
      3. - 02 -
        - Work out its rules.Follow the inputs, calls and settings. -
        -
      4. -
      5. - 03 -
        - Use and check the result.Explain the behavior, rebuild it or make a change. +
        + + + +
        +
        +
        + + Original x64 · selected instructions + + +
        0x180124945: MOV EAX, dword ptr [R13 + 0x18]
        +0x180124949: CMP EAX, 0x5c
        +0x18012494c: JZ 0x180124aba
        +0x180124952: CMP EAX, 0x5b
        +0x180124955: JZ 0x180124aba
        +0x18012495b: MOV EDX, 0x64
        +0x180124960: LEA RCX, [RSP + 0x160]
        +0x180124968: CALL 0x180122650
        +0x18012496e: LEA RDX, [R13 + 0x160]
        +0x180124975: LEA RCX, [RSP + 0x78]
        +0x18012497a: CALL 0x180109720
        +0x18012497f: MOV RSI, RAX
        +0x180124982: MOV qword ptr [RSP + 0x28], RAX
        +0x180124987: LEA RDX, [RSP + 0x160]
        +0x18012498f: MOV RCX, RAX
        +0x180124992: CALL 0x180123190
        +; … value-copy instructions omitted …
        +0x180124a14: MOV RDX, RDI
        +0x180124a17: LEA RCX, [RSP + 0xf8]
        +0x180124a1f: CALL 0x180109720
        +0x180124a24: LEA R8, [RSP + 0x30]
        +0x180124a29: MOV RDX, RAX
        +0x180124a2c: LEA RCX, [RSP + 0xb8]
        +0x180124a34: CALL 0x180123c10
        +
        +
      6. -
      + +

      Illustrative expert reasoning

      + +
      +

      + You decode the instructions, follow the calls and reconstruct + the calculation. +

      With REA · You ask your agent

      +

      One prompt.

      +

      + Ask your agent in plain English. +

      Example question · Dinosaur gameExample prompt · Calculator @@ -180,30 +310,44 @@

      What is reverse engineering?

      Use REA to inspect this dinosaur game. Why does it get - faster? Show the speed rule.Use REA to inspect Windows Calculator. Why does 200 + 10% + give 220?

      - Example explanation from the findings + Example answer · based on the findings

      - The game adds a little speed on each update. Its script sets - the starting speed, the increment and the stopping - point. + After +, the % button takes + a percentage of the first number.

      +
      + 10% of 200 = 20200 + 20 = 220 +
      +
      +
      +
      REA supplies
      +
      + The handler’s instructions, decompiled code and calls. +
      +
      +
      +
      Your agent explains
      +
      + Which branch applies, what it calculates and how to rebuild + it. +
      +
      +
      - REA supplies code and observations from the program. - Your agent uses that evidence to explain, rebuild or test a - feature. + Selected assembly from the installed Calculator DLL. The thoughts + and reply illustrate the findings. + See the calculation and sources ↓
      diff --git a/website/style-guide.md b/website/style-guide.md index e8c5085d444..e17475d5778 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -75,6 +75,11 @@ attributed to their respective roles. On mobile, the demo can precede its supporting code once its purpose is clear. Show how the reader's task changes using the same question on both sides of a manual/agent comparison. Let returned evidence make REA's contribution visible. +When showing manual analysis, use authentic instructions and specific analyst +questions: what a command ID means, which call performs arithmetic, or which +operand enters a calculation. Numbered selections can connect the code to +illustrative thought bubbles. Keep the agent side brief: one visible prompt, +a concrete answer, and the evidence used to reach it. When a small reconstruction makes the finding tangible, let readers change an input or setting and try the result. Connect inspection, recovered logic, From 42c276af1ec02473a7b6700a31292aae9e53d53e Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:03:10 +0000 Subject: [PATCH 123/791] fix(ghidra): report NativeAOT host and target refusals as unsupported, not failures With REA_GHIDRA_NATIVEAOT_JAR set on a host other than Linux, Ghidra analysis was refused before Ghidra started, as intended, but the refusal was a ProviderAdapterError: `execution_failure` with advice to retry and run doctor, neither of which can change the host. Adapters now say whether a refusal concerns the host or the target. A host refusal is `capability_unavailable` whose message and remediation are the adapter's reason (omit the JAR for ordinary analysis); a target refusal is `unsupported_target` with the target path. Missing or malformed JARs on a supported host remain provider failures. Fixes #1185 Co-Authored-By: Claude Opus 5.5 --- .../extensions/GhidraExtensions.test.ts | 69 +++++++++++++------ src/ghidra/extensions/GhidraExtensions.ts | 48 ++++++++++--- .../extensions/nativeaot/NativeAotAdapter.ts | 12 +++- 3 files changed, 94 insertions(+), 35 deletions(-) diff --git a/src/ghidra/extensions/GhidraExtensions.test.ts b/src/ghidra/extensions/GhidraExtensions.test.ts index b35e63f9cc4..0b4b8eb7bd4 100644 --- a/src/ghidra/extensions/GhidraExtensions.test.ts +++ b/src/ghidra/extensions/GhidraExtensions.test.ts @@ -7,6 +7,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { parseConfig } from "../../config.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; import { ghidraExtensionSchema, @@ -70,6 +71,50 @@ const result = (sha256: string): GhidraExtensionResult => ({ }, }); +describe("optional Ghidra extension refusals", () => { + it("rejects unsupported hosts and target architectures before opening files", async () => { + const host = await resolveGhidraExtensions( + config("/absent.jar"), + target, + "darwin", + ); + expect(host.ok).toBe(false); + if (!host.ok) + // A host restriction is not an execution failure to retry or repair. + expect(projectAnalysisError(host.error)).toMatchObject({ + code: "capability_unavailable", + retryable: false, + message: expect.stringContaining("verified on Linux only"), + remediation: { + action: expect.stringContaining("omit REA_GHIDRA_NATIVEAOT_JAR"), + }, + details: { + provider_id: "ghidra", + operation: "resolve_analysis_profile", + }, + }); + const architecture = await resolveGhidraExtensions( + config("/absent.jar"), + { + ...target, + architecture: "arm64", + availableArchitectures: ["arm64"], + }, + "linux", + ); + expect(architecture.ok).toBe(false); + if (!architecture.ok) + expect(projectAnalysisError(architecture.error)).toMatchObject({ + code: "unsupported_target", + details: { + operation: "resolve_analysis_profile", + path: target.path, + reason: expect.stringContaining("x86-64 ELF or native PE"), + }, + }); + }); +}); + describe("optional Ghidra extension boundary", () => { it("keeps ordinary profiles free of unconfigured extensions", async () => { expect(await resolveGhidraExtensions(config(), target, "linux")).toEqual({ @@ -136,25 +181,6 @@ describe("optional Ghidra extension boundary", () => { ).ok, ).toBe(false); }); - it("rejects unsupported hosts and target architectures before opening files", async () => { - expect( - (await resolveGhidraExtensions(config("/absent.jar"), target, "win32")) - .ok, - ).toBe(false); - expect( - ( - await resolveGhidraExtensions( - config("/absent.jar"), - { - ...target, - architecture: "arm64", - availableArchitectures: ["arm64"], - }, - "linux", - ) - ).ok, - ).toBe(false); - }); it.skipIf(process.platform === "win32")( "rejects a FIFO without requiring another process to open its writer", async () => { @@ -287,9 +313,8 @@ describe("Ghidra extension profile and producer validation", () => { expect(resolved.ok).toBe(false); if (!resolved.ok) expect(resolved.error).toMatchObject({ - diagnostics: { - reason: expect.stringContaining("inspect_managed_artifact"), - }, + _tag: "AnalysisUnsupportedTargetError", + reason: expect.stringContaining("inspect_managed_artifact"), }); }); it("does not substitute another artifact under an unchanged configured path", async () => { diff --git a/src/ghidra/extensions/GhidraExtensions.ts b/src/ghidra/extensions/GhidraExtensions.ts index 5ffff9b6cae..fb878b5220e 100644 --- a/src/ghidra/extensions/GhidraExtensions.ts +++ b/src/ghidra/extensions/GhidraExtensions.ts @@ -5,7 +5,11 @@ import { isAbsolute, join } from "node:path"; import { z } from "zod"; import type { AppConfig } from "../../config.js"; import type { BinaryTarget } from "../../domain/binaryTarget.js"; -import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisUnsupportedTargetError, +} from "../../domain/analysisErrorCore.js"; import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import type { AnalysisError } from "../../domain/analysisErrorBase.js"; import { jsonObjectSchema } from "../../domain/jsonValue.js"; @@ -42,6 +46,12 @@ export const ghidraExtensionResultSchema = z.strictObject({ }); export type GhidraExtensionResult = z.infer; +/** Why an adapter cannot run here: on this host at all, or for this target. */ +export interface GhidraExtensionRefusal { + readonly constraint: "host" | "target"; + readonly reason: string; +} + /** Provider-local registration contract for independently replaceable analysis adapters. */ export interface GhidraExtensionAdapter { readonly id: string; @@ -50,7 +60,7 @@ export interface GhidraExtensionAdapter { unsupportedReason( target: BinaryTarget, platform: NodeJS.Platform, - ): string | null; + ): GhidraExtensionRefusal | null; validate(result: GhidraExtensionResult): string | null; readonly limitations: readonly string[]; } @@ -109,13 +119,8 @@ export const resolveGhidraExtensions = async ( if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); const configured = adapter.configuredPath(config); if (configured === undefined) continue; - const reason = adapter.unsupportedReason(target, platform); - if (reason !== null) - return err( - new ProviderAdapterError("ghidra", "resolve_analysis_profile", { - diagnostics: { extension: adapter.id, reason, path: configured }, - }), - ); + const refusal = adapter.unsupportedReason(target, platform); + if (refusal !== null) return err(extensionRefusalError(refusal, target)); try { const path = await realpath(configured); const bytes = await readJar(path); @@ -168,12 +173,33 @@ export const validateGhidraExtensionProfile = ( adapter.configuredPath(config) !== extension.configured_path ) return `Ghidra extension profile lacks matching explicit local configuration: ${extension.id}`; - const reason = adapter.unsupportedReason(target, platform); - if (reason !== null) return reason; + const refusal = adapter.unsupportedReason(target, platform); + if (refusal !== null) return refusal.reason; } return null; }; +/** + * A deliberate host or target refusal is not an execution failure: retrying + * cannot change it, and its reason already names the remedy. + */ +const extensionRefusalError = ( + refusal: GhidraExtensionRefusal, + target: BinaryTarget, +): AnalysisError => + refusal.constraint === "host" + ? new AnalysisCapabilityUnavailableError( + "ghidra", + "resolve_analysis_profile", + refusal.reason, + { userMessage: refusal.reason }, + ) + : new AnalysisUnsupportedTargetError( + "resolve_analysis_profile", + target.path, + refusal.reason, + ); + /** Snapshot committed extension bytes into the private runtime before Java loads them. */ export const snapshotGhidraExtensions = async ( extensions: readonly GhidraExtension[], diff --git a/src/ghidra/extensions/nativeaot/NativeAotAdapter.ts b/src/ghidra/extensions/nativeaot/NativeAotAdapter.ts index d7b30663cdc..4893dfa27a3 100644 --- a/src/ghidra/extensions/nativeaot/NativeAotAdapter.ts +++ b/src/ghidra/extensions/nativeaot/NativeAotAdapter.ts @@ -8,11 +8,19 @@ export const nativeAotAdapter: GhidraExtensionAdapter = { configuredPath: (config) => config.ghidraNativeAotJar, unsupportedReason: (target, platform) => platform !== "linux" - ? "NativeAOT recovery is verified on Linux only; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis." + ? { + constraint: "host", + reason: + "NativeAOT recovery is verified on Linux only; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis.", + } : !["elf", "pe"].includes(target.format) || target.architecture !== "x86_64" || target.managed === true - ? "NativeAOT recovery requires an x86-64 ELF or native PE target on Linux; PE/CLI and ReadyToRun assemblies use inspect_managed_artifact; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis." + ? { + constraint: "target", + reason: + "NativeAOT recovery requires an x86-64 ELF or native PE target on Linux; PE/CLI and ReadyToRun assemblies use inspect_managed_artifact; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis.", + } : null, validate: validateNativeAotReport, limitations: [ From 9f5dc52be8420fb5e874595d63312bf80d07a3a2 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:04:42 +0000 Subject: [PATCH 124/791] fix(hopper): keep the OS error when the FAT64 source cannot be reopened resolveHopperMachOImage built its BinaryTargetError without the open failure as its cause, so a target that became unreadable after admission still projected as `target_unavailable` instead of `access_denied`. Co-Authored-By: Claude Opus 5.5 --- src/hopper/HopperMachOImage.test.ts | 37 +++++++++++++++++++++++++++++ src/hopper/HopperMachOImage.ts | 12 ++++++++-- 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 src/hopper/HopperMachOImage.test.ts diff --git a/src/hopper/HopperMachOImage.test.ts b/src/hopper/HopperMachOImage.test.ts new file mode 100644 index 00000000000..cb96a57a17a --- /dev/null +++ b/src/hopper/HopperMachOImage.test.ts @@ -0,0 +1,37 @@ +import { chmod, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { resolveHopperMachOImage } from "./HopperMachOImage.js"; + +it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports a FAT64 source that became unreadable as a host read denial", + async () => { + const directory = await createTestTempDirectory("rea-hopper-fat64-"); + const path = join(directory, "universal"); + await writeFile(path, Buffer.alloc(64)); + await chmod(path, 0); + try { + const result = await resolveHopperMachOImage({ + path, + sha256: "0".repeat(64), + kind: "executable", + format: "mach-o", + architecture: "arm64", + availableArchitectures: ["arm64"], + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error).toMatchObject({ systemCode: "EACCES" }); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "access_denied", + details: { path, system_code: "EACCES" }, + }); + } finally { + await chmod(path, 0o600); + } + }, +); diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts index 0b6ac600e56..1ba69861f46 100644 --- a/src/hopper/HopperMachOImage.ts +++ b/src/hopper/HopperMachOImage.ts @@ -35,15 +35,23 @@ export const resolveHopperMachOImage = async ( ): Promise< Result > => { - const invalid = (reason: string) => - err(new BinaryTargetError(target.path, reason)); + const invalid = (reason: string, cause?: unknown) => + err( + new BinaryTargetError( + target.path, + reason, + cause === undefined ? undefined : { cause }, + ), + ); if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); let file: FileHandle; try { file = await open(target.path, "r"); } catch (cause: unknown) { + // Keep the OS error so a read-permission denial reports access_denied. return invalid( `Cannot read FAT64 source for Hopper image preparation: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, ); } try { From 0a0244db4212bc1f59dec4b925fa90b3b473136a Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:08:43 +0000 Subject: [PATCH 125/791] fix(config): name the rejected setting and its constraint in configuration errors Every configuration failure projected the same message, "REA configuration is invalid. Run `rea doctor` and fix the reported setting.", with no details. The environment parser discarded the issue that names the variable (for example `REA_GHIDRA_NATIVEAOT_JAR must be absolute`), and doctor can report healthy for that environment, so neither the CLI, the MCP startup message, nor doctor told callers which setting to correct. ConfigurationError now carries the rejected settings and their constraints (never their values). The projection names them in its message, lists them in `details.settings`, and asks the caller to correct those settings where REA is started. Errors without a named setting keep the previous message. Fixes #1184 Co-Authored-By: Claude Opus 5.5 --- src/config.test.ts | 43 +++++++++++++++++++++++++ src/config/environment.ts | 6 ++++ src/config/mcpResponseBudget.ts | 13 ++++++-- src/config/parsers.ts | 30 +++++++++++++++-- src/domain/analysisErrorPresentation.ts | 12 ++++++- src/domain/analysisErrorProjection.ts | 12 ++++++- src/domain/configurationErrors.ts | 18 +++++++++++ src/ida/IdaConfiguration.ts | 20 +++++++++--- src/ida/IdaProvider.ts | 5 +-- 9 files changed, 145 insertions(+), 14 deletions(-) diff --git a/src/config.test.ts b/src/config.test.ts index dd228e4a61d..6643fbdd102 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { parseConfig } from "./config.js"; +import { projectAnalysisError } from "./domain/analysisErrorProjection.js"; describe("runtime configuration", () => { it("allows target-free startup and applies runtime defaults", () => { @@ -155,3 +156,45 @@ describe("runtime collection configuration", () => { }, ); }); + +describe("configuration failure diagnostics", () => { + it.each([ + ["GHIDRA_INSTALL_DIR", "./fixture"], + ["JAVA_HOME", "./jdk"], + ["REA_GHIDRA_NATIVEAOT_JAR", "./missing.jar"], + ])( + "names the rejected %s and its absolute-path constraint", + (setting, value) => { + const result = parseConfig({ [setting]: value }); + expect(result.ok).toBe(false); + if (result.ok) return; + const constraint = `${setting} must be absolute`; + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "configuration_invalid", + message: `REA configuration is invalid: ${constraint}.`, + remediation: { action: expect.stringContaining(`Correct ${setting} `) }, + details: { settings: [{ setting, constraint }] }, + }); + // The rejected value itself is not echoed. + expect(JSON.stringify(projectAnalysisError(result.error))).not.toContain( + value, + ); + }, + ); + + it("names settings rejected by their own parsers", () => { + const result = parseConfig({ HOPPER_LOADER_ARGS_JSON: "not-json" }); + expect(result.ok).toBe(false); + if (!result.ok) + expect(projectAnalysisError(result.error)).toMatchObject({ + details: { + settings: [ + { + setting: "HOPPER_LOADER_ARGS_JSON", + constraint: "HOPPER_LOADER_ARGS_JSON must be valid JSON", + }, + ], + }, + }); + }); +}); diff --git a/src/config/environment.ts b/src/config/environment.ts index fe7e1245e91..f743de57612 100644 --- a/src/config/environment.ts +++ b/src/config/environment.ts @@ -53,9 +53,15 @@ export const parseEnvironment = ( ): Result => { const parsed = environmentSchema.safeParse(environment); if (!parsed.success) { + // Zod messages name the constraint, never the rejected value. + const settings = parsed.error.issues.map((issue) => ({ + setting: String(issue.path[0] ?? "environment"), + constraint: issue.message, + })); return err( new ConfigurationError("Invalid REA environment configuration", { cause: parsed.error, + settings, }), ); } diff --git a/src/config/mcpResponseBudget.ts b/src/config/mcpResponseBudget.ts index 01d69db971b..27126bf0c42 100644 --- a/src/config/mcpResponseBudget.ts +++ b/src/config/mcpResponseBudget.ts @@ -15,6 +15,8 @@ export const mcpResponseBudgetSchema = z .transform(Number) .pipe(z.number().int().safe().min(MINIMUM_RESPONSE_BYTES)); +const budgetConstraint = `REA_MCP_MAX_RESPONSE_BYTES must be a decimal safe integer at least ${String(MINIMUM_RESPONSE_BYTES)} bytes, the default stdio receive-buffer size.`; + /** Parse only the MCP-specific budget without loading unrelated provider settings. */ export const parseMcpResponseBudget = ( value: string | undefined, @@ -24,8 +26,13 @@ export const parseMcpResponseBudget = ( return parsed.success ? ok(parsed.data) : err( - new ConfigurationError( - `REA_MCP_MAX_RESPONSE_BYTES must be a decimal safe integer at least ${String(MINIMUM_RESPONSE_BYTES)} bytes, the default stdio receive-buffer size.`, - ), + new ConfigurationError(budgetConstraint, { + settings: [ + { + setting: "REA_MCP_MAX_RESPONSE_BYTES", + constraint: budgetConstraint, + }, + ], + }), ); }; diff --git a/src/config/parsers.ts b/src/config/parsers.ts index 41637d513cc..344b31eeae5 100644 --- a/src/config/parsers.ts +++ b/src/config/parsers.ts @@ -14,10 +14,21 @@ export const parseStringArray = ( : err( new ConfigurationError(`${name} must encode an array of strings`, { cause: parsed.error, + settings: [ + { + setting: name, + constraint: `${name} must encode an array of strings`, + }, + ], }), ); } catch (cause: unknown) { - return err(new ConfigurationError(`${name} must be valid JSON`, { cause })); + return err( + new ConfigurationError(`${name} must be valid JSON`, { + cause, + settings: [{ setting: name, constraint: `${name} must be valid JSON` }], + }), + ); } }; @@ -32,6 +43,9 @@ export const parseLoaderArgs = ( return err( new ConfigurationError("HOPPER_LOADER_ARGS_JSON must be valid JSON", { cause, + settings: [ + loaderArgsIssue("HOPPER_LOADER_ARGS_JSON must be valid JSON"), + ], }), ); } @@ -41,7 +55,19 @@ export const parseLoaderArgs = ( : err( new ConfigurationError( "HOPPER_LOADER_ARGS_JSON must encode an array of strings", - { cause: parsed.error }, + { + cause: parsed.error, + settings: [ + loaderArgsIssue( + "HOPPER_LOADER_ARGS_JSON must encode an array of strings", + ), + ], + }, ), ); }; + +const loaderArgsIssue = (constraint: string) => ({ + setting: "HOPPER_LOADER_ARGS_JSON", + constraint, +}); diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index cc3dc7e40b4..7676b2b393d 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -78,6 +78,8 @@ export const analysisErrorRemediationAction = ( return "Check the current process's read access to the selected path. Retry with a readable local file."; if (error instanceof AnalysisArtifactChangedError) return "Wait until the selected file is stable. For an active binary session, reopen the target with open_binary before retrying so REA acquires its current identity; for a CLI command or target-free tool, rerun the operation."; + if (error instanceof ConfigurationError && error.settings.length > 0) + return `Correct ${[...new Set(error.settings.map(({ setting }) => setting))].join(", ")} in the environment that starts REA (your shell or the MCP client's registration), then rerun.`; if (error instanceof AnalysisInputError) return "Correct the listed arguments and retry."; if (error instanceof UnknownRegistryError && error.reason === "not-found") @@ -189,7 +191,15 @@ export const analysisErrorUserMessage = (error: AnalysisError): string => { if (error instanceof UnknownRegistryError) return "Evidence state changed before the update completed. Refresh the current state and try again."; if (error instanceof ConfigurationError) - return "REA configuration is invalid. Run `rea doctor` and fix the reported setting."; + return error.settings.length === 0 + ? "REA configuration is invalid. Run `rea doctor` and fix the reported setting." + : `REA configuration is invalid: ${error.settings + .map(({ setting, constraint }) => + constraint.includes(setting) + ? constraint.replace(/\.$/u, "") + : `${setting}: ${constraint}`, + ) + .join("; ")}.`; if (error instanceof NoBinaryOpenError) return error.message; if (error instanceof BinaryTargetError) return error.constraint === "directory_requires_file" diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 66e5c77e9c3..7fd0241cae6 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -18,7 +18,10 @@ import { AnalysisResourceConstraintError, } from "./analysisErrorCore.js"; import { ArtifactOperationError } from "./artifactOperationError.js"; -import { BinaryTargetError } from "./configurationErrors.js"; +import { + BinaryTargetError, + ConfigurationError, +} from "./configurationErrors.js"; import { BrowserObservationError } from "./browserObservationError.js"; import { EvidenceFileError, @@ -482,6 +485,13 @@ const lifecycleErrorDetails = ( }; return Object.keys(details).length === 0 ? undefined : details; } + if (error instanceof ConfigurationError && error.settings.length > 0) + return { + settings: error.settings.map(({ setting, constraint }) => ({ + setting, + constraint, + })), + }; if (error instanceof BinaryTargetError) return { path: error.path, diff --git a/src/domain/configurationErrors.ts b/src/domain/configurationErrors.ts index c7c07f75e6b..5820db7ffff 100644 --- a/src/domain/configurationErrors.ts +++ b/src/domain/configurationErrors.ts @@ -3,9 +3,27 @@ import { type AnalysisErrorOptions, } from "./analysisErrorBase.js"; +/** One rejected configuration setting and the constraint it failed. */ +export interface ConfigurationSettingIssue { + readonly setting: string; + readonly constraint: string; +} + +/** Rejected settings retained for diagnostics; their values are not. */ +export interface ConfigurationErrorOptions extends AnalysisErrorOptions { + readonly settings?: readonly ConfigurationSettingIssue[]; +} + /** Runtime configuration could not be parsed safely. */ export class ConfigurationError extends AnalysisError { readonly _tag = "ConfigurationError"; + readonly settings: readonly ConfigurationSettingIssue[]; + constructor(message: string, options?: ConfigurationErrorOptions) { + super(message, options); + this.settings = (options?.settings ?? []).map( + ({ setting, constraint }) => ({ setting, constraint }), + ); + } } /** No app or binary session exists for an analysis request. */ diff --git a/src/ida/IdaConfiguration.ts b/src/ida/IdaConfiguration.ts index 051ee8d4d2f..26084b73466 100644 --- a/src/ida/IdaConfiguration.ts +++ b/src/ida/IdaConfiguration.ts @@ -50,7 +50,7 @@ export const readIdaConfiguration = ( try { if (statSync(path).size > 65_536) return err( - new ConfigurationError( + idaConfigurationError( "REA_IDA_MCP_CONFIG exceeds the 64 KiB registration size.", ), ); @@ -64,17 +64,27 @@ export const readIdaConfiguration = ( return parsed.success ? ok(parsed.data) : err( - new ConfigurationError( + idaConfigurationError( "REA_IDA_MCP_CONFIG must contain an IDA MCP command/args or local url registration; see docs/ida-provider.md.", - { cause: parsed.error }, + parsed.error, ), ); } catch (cause: unknown) { return err( - new ConfigurationError( + idaConfigurationError( `Cannot read IDA MCP registration ${path}; provide a readable JSON configuration.`, - { cause }, + cause, ), ); } }; + +/** A REA_IDA_MCP_CONFIG rejection that names the setting it concerns. */ +export const idaConfigurationError = ( + constraint: string, + cause?: unknown, +): ConfigurationError => + new ConfigurationError(constraint, { + ...(cause === undefined ? {} : { cause }), + settings: [{ setting: "REA_IDA_MCP_CONFIG", constraint }], + }); diff --git a/src/ida/IdaProvider.ts b/src/ida/IdaProvider.ts index 6f195cb61f2..ff8a46fbba0 100644 --- a/src/ida/IdaProvider.ts +++ b/src/ida/IdaProvider.ts @@ -11,13 +11,14 @@ import type { ProviderTargetSupport, } from "../application/AnalysisProvider.js"; import type { AppConfig } from "../config.js"; -import { ConfigurationError } from "../domain/configurationErrors.js"; +import type { ConfigurationError } from "../domain/configurationErrors.js"; import { createHash } from "node:crypto"; import canonicalize from "canonicalize"; import type { Result } from "../domain/result.js"; import { readIdaConfiguration, type IdaConfiguration, + idaConfigurationError, } from "./IdaConfiguration.js"; import { createIdaMcpConnection, @@ -165,7 +166,7 @@ export class IdaProvider implements AnalysisProviderCandidate { const readIdaConfigurationNotConfigured = () => err( - new ConfigurationError( + idaConfigurationError( "Set REA_IDA_MCP_CONFIG to an upstream IDA MCP JSON registration; see docs/ida-provider.md.", ), ); From f198993f418e6aaa9755e7e6413abcb27c6cd684 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:09:50 +0000 Subject: [PATCH 126/791] fix(ghidra): keep the NativeAOT target refusal's own recovery advice An unsupported-target error always projected the generic "select a target" remediation, which hid the adapter's available workflows (omit REA_GHIDRA_NATIVEAOT_JAR for ordinary analysis, or use inspect_managed_artifact). AnalysisUnsupportedTargetError accepts a refusal-specific remediation, as AnalysisResourceConstraintError does, and the NativeAOT refusal supplies its reason. Co-Authored-By: Claude Opus 5.5 --- src/domain/analysisErrorCore.ts | 5 ++++- src/domain/analysisErrorPresentation.ts | 5 ++++- src/ghidra/extensions/GhidraExtensions.test.ts | 11 ++++++++--- src/ghidra/extensions/GhidraExtensions.ts | 2 ++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/domain/analysisErrorCore.ts b/src/domain/analysisErrorCore.ts index e0ba8982057..c13ba891d6b 100644 --- a/src/domain/analysisErrorCore.ts +++ b/src/domain/analysisErrorCore.ts @@ -7,13 +7,16 @@ import { /** A readable supplied artifact lies outside this operation's supported target formats. */ export class AnalysisUnsupportedTargetError extends AnalysisError { readonly _tag = "AnalysisUnsupportedTargetError"; + /** Refusal-specific recovery, when the producer knows an available workflow. */ + readonly remediationAction: string | undefined; constructor( readonly operation: string, readonly path: string, readonly reason: string, - options?: AnalysisErrorOptions, + options?: AnalysisErrorOptions & { readonly remediationAction?: string }, ) { super(`Unsupported target for ${operation} at ${path}: ${reason}`, options); + this.remediationAction = options?.remediationAction; } } diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index cc3dc7e40b4..ca635935915 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -39,7 +39,10 @@ export const analysisErrorRemediationAction = ( if (error instanceof AnalysisSnapshotMismatchError) return "Run analysis without this snapshot, then save a fresh snapshot using the intended artifact, provider, and analysis profile."; if (error instanceof AnalysisUnsupportedTargetError) - return "Select a target supported by this operation or choose an operation supporting the reported target format."; + return ( + error.remediationAction ?? + "Select a target supported by this operation or choose an operation supporting the reported target format." + ); if (error instanceof AnalysisResourceConstraintError) return ( error.remediationAction ?? diff --git a/src/ghidra/extensions/GhidraExtensions.test.ts b/src/ghidra/extensions/GhidraExtensions.test.ts index 0b4b8eb7bd4..98ea117b8ea 100644 --- a/src/ghidra/extensions/GhidraExtensions.test.ts +++ b/src/ghidra/extensions/GhidraExtensions.test.ts @@ -106,6 +106,9 @@ describe("optional Ghidra extension refusals", () => { if (!architecture.ok) expect(projectAnalysisError(architecture.error)).toMatchObject({ code: "unsupported_target", + remediation: { + action: expect.stringContaining("omit REA_GHIDRA_NATIVEAOT_JAR"), + }, details: { operation: "resolve_analysis_profile", path: target.path, @@ -312,9 +315,11 @@ describe("Ghidra extension profile and producer validation", () => { ); expect(resolved.ok).toBe(false); if (!resolved.ok) - expect(resolved.error).toMatchObject({ - _tag: "AnalysisUnsupportedTargetError", - reason: expect.stringContaining("inspect_managed_artifact"), + expect(projectAnalysisError(resolved.error)).toMatchObject({ + code: "unsupported_target", + remediation: { + action: expect.stringContaining("inspect_managed_artifact"), + }, }); }); it("does not substitute another artifact under an unchanged configured path", async () => { diff --git a/src/ghidra/extensions/GhidraExtensions.ts b/src/ghidra/extensions/GhidraExtensions.ts index fb878b5220e..25007bfa5fe 100644 --- a/src/ghidra/extensions/GhidraExtensions.ts +++ b/src/ghidra/extensions/GhidraExtensions.ts @@ -198,6 +198,8 @@ const extensionRefusalError = ( "resolve_analysis_profile", target.path, refusal.reason, + // The reason names the workflows that remain available. + { remediationAction: refusal.reason }, ); /** Snapshot committed extension bytes into the private runtime before Java loads them. */ From 53568568bf32b36274ba3a532f1e2cfb2daed271 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:13:40 +0000 Subject: [PATCH 127/791] fix(ghidra): reject relative installation and Java paths in doctor and setup A relative GHIDRA_INSTALL_DIR (or JAVA_HOME) that resolved from the current directory passed doctor's Ghidra inspection, so doctor reported a healthy provider and setup wrote the relative path into the MCP registration, but REA's configuration parser rejects relative paths, so `providers`, `capabilities` and the registered MCP server all failed. The installation inspection's configuration check now applies the same absolute-path rule, judged by the inspected platform's path syntax, and names the relative setting. Doctor reports the provider unavailable with that reason, and setup, which registers only an available inspection's environment, no longer writes the path. Fixes #1183 Co-Authored-By: Claude Opus 5.5 --- src/ghidra/GhidraInstallation.test.ts | 45 +++++++++++++++++++++++++++ src/ghidra/GhidraInstallation.ts | 42 ++++++++++++++++++++----- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/ghidra/GhidraInstallation.test.ts b/src/ghidra/GhidraInstallation.test.ts index 18f45e26be4..3c6d46354ca 100644 --- a/src/ghidra/GhidraInstallation.test.ts +++ b/src/ghidra/GhidraInstallation.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { projectGhidraDoctorInspection } from "./GhidraDoctor.js"; import { ghidraJavaEnvironment, inspectGhidraInstallation, @@ -378,3 +379,47 @@ describe("Ghidra release line and JDK bounds", () => { ).toMatchObject({ status: "available", javaVersion: "17.0.20.1" }); }); }); + +describe("Ghidra configuration paths", () => { + it.each([ + [{ installDir: "./ghidra" }, "GHIDRA_INSTALL_DIR must be absolute"], + [{ installDir: INSTALL, javaHome: "./jdk" }, "JAVA_HOME must be absolute"], + [ + { installDir: "ghidra", javaHome: "jdk" }, + "GHIDRA_INSTALL_DIR and JAVA_HOME must be absolute", + ], + ])( + "rejects relative paths that REA's configuration parser rejects (%o)", + (paths, detail) => { + const result = inspectGhidraInstallation( + { ...paths, platform: "linux", architecture: "x64" }, + host(), + ); + expect(result).toMatchObject({ + status: "unavailable", + rejection: { + code: "not_configured", + reason: expect.stringContaining("reject relative paths"), + }, + }); + expect(result.checks).toContainEqual( + expect.objectContaining({ name: "configuration", detail }), + ); + // Setup registers only an available inspection's environment. + expect(projectGhidraDoctorInspection(result)).toMatchObject({ + available: false, + registrationEnvironment: {}, + }); + }, + ); + + it("judges absolute paths by the inspected platform", () => { + const result = inspectGhidraInstallation( + { installDir: "C:\\ghidra", platform: "win32", architecture: "x64" }, + host(), + ); + expect(result.checks).toContainEqual( + expect.objectContaining({ name: "configuration", status: "passed" }), + ); + }); +}); diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index a521fc171bf..d0906fb5276 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -306,13 +306,7 @@ const installationChecks = ({ javaHome, host, }: GhidraInstallationCheckContext): readonly GhidraInstallationCheck[] => [ - installationCheck({ - name: "configuration", - passed: coordinates.installDir !== null, - code: "not_configured", - detail: coordinates.installDir ?? "GHIDRA_INSTALL_DIR is not set", - remediation: `Set GHIDRA_INSTALL_DIR to an extracted Ghidra ${ghidraReleaseLine()} release directory.`, - }), + configurationCheck(coordinates.installDir, javaHome, platform), installationCheck({ name: "platform", passed: NATIVE_PLATFORMS[platform] !== undefined, @@ -366,6 +360,40 @@ const installationChecks = ({ ), ]; +/** + * REA's configuration parser, and so the MCP server and analysis, accept only + * absolute paths; a relative one that happens to resolve here must not pass. + */ +const configurationCheck = ( + installDir: string | null, + javaHome: string | undefined, + platform: NodeJS.Platform, +): GhidraInstallationCheck => { + const { isAbsolute } = platform === "win32" ? win32 : posix; + const relative = [ + ...(installDir !== null && !isAbsolute(installDir) + ? ["GHIDRA_INSTALL_DIR"] + : []), + ...(javaHome !== undefined && !isAbsolute(javaHome) ? ["JAVA_HOME"] : []), + ]; + if (installDir === null || relative.length === 0) + return installationCheck({ + name: "configuration", + passed: installDir !== null, + code: "not_configured", + detail: installDir ?? "GHIDRA_INSTALL_DIR is not set", + remediation: `Set GHIDRA_INSTALL_DIR to an extracted Ghidra ${ghidraReleaseLine()} release directory.`, + }); + const settings = relative.join(" and "); + return installationCheck({ + name: "configuration", + passed: false, + code: "not_configured", + detail: `${settings} must be absolute`, + remediation: `Set ${settings} to ${relative.length === 1 ? "an absolute path" : "absolute paths"}. REA's analysis and MCP server reject relative paths even when they resolve from the current directory.`, + }); +}; + /** Project an installation probe into caller-visible, secret-free diagnostics. */ export const ghidraInstallationDiagnostics = ( inspection: GhidraInstallationInspection, From 6fbdbd682038d012b04f0fe92c470cf1789fb101 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:19:07 +0800 Subject: [PATCH 128/791] fix(ghidra): resolve secondary symbols at function entries (#1195) * fix(ghidra): resolve secondary symbols at function entries * test(ghidra): derive interior bytes from decoded instructions --- bridge/ghidra/ReaGhidraBridge.java | 15 ++- docs/native-investigation.md | 3 + scripts/lib/real-ghidra-boundaries.mjs | 29 ++++- scripts/lib/real-ghidra-entry-aliases.mjs | 126 ++++++++++++++++++++++ src/ghidra/GhidraAnalysisProfile.ts | 2 +- tests/conformance/ghidra/README.md | 3 + tests/conformance/ghidra/entry-aliases.c | 19 ++++ 7 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 scripts/lib/real-ghidra-entry-aliases.mjs create mode 100644 tests/conformance/ghidra/entry-aliases.c diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 31805ad75e4..b64955a2e17 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -1629,9 +1629,20 @@ private Function resolveProcedure(String value) throws Exception { Function function = entry.function; Symbol symbol = function.getSymbol(); String qualified = symbol == null ? function.getName() : symbol.getName(true); - if (function.getName().equals(value) || qualified.equals(value)) { - matches.add(function); + boolean matchesName = function.getName().equals(value) || qualified.equals(value); + if (!matchesName) { + // Imported labels (for example Mach-O _main) can share a + // function entry without becoming its primary symbol. Only + // exact entries qualify; an interior label is not a selector. + for (Symbol entrySymbol : currentProgram.getSymbolTable().getSymbols(function.getEntryPoint())) { + monitor.checkCancelled(); + if (entrySymbol.getName().equals(value) || entrySymbol.getName(true).equals(value)) { + matchesName = true; + break; + } + } } + if (matchesName) matches.add(function); } if (matches.size() > 1) throw new RequestFailure("ambiguous", "Ghidra procedure name is ambiguous: " + value + diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 96433fc516f..902831169f3 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -128,6 +128,9 @@ an explicit `0x` or address-space prefix selects an address; otherwise an exact database symbol name takes precedence over a bare hexadecimal address. Thus a function renamed to `dead` remains selectable by name. Use the names returned by the inventory, including their namespaces and any platform symbol prefix. +Ghidra also accepts secondary symbols at a function entry, such as an imported +`_main` label. A label inside a function is not a procedure-name selector; use +its address when selecting the containing function. Overloads can share a fully qualified name; ambiguity errors return every matching entry address so the caller can select the intended function directly. diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 17ddff340ff..47e90e60e52 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -13,6 +13,7 @@ import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; import { verifyGhidraTargetAdmission } from "./real-ghidra-target-admission.mjs"; import { verifyGhidraLargeResults } from "./real-ghidra-large-results.mjs"; +import { verifyGhidraEntryAliases } from "./real-ghidra-entry-aliases.mjs"; import { verifyGhidraNamespaceAnnotations } from "./real-ghidra-namespace-annotations.mjs"; /** Probe real Ghidra location, annotation and error contracts through public adapters. */ @@ -278,7 +279,25 @@ export async function verifyGhidraBoundaries( await cli("inspect-native-instruction", address.toUpperCase()), baseline, ); - const interior = `0x${(BigInt(address) + 1n).toString(16)}`; + // x86 function entries can begin with a one-byte PUSH. Locate an observed + // multi-byte instruction instead of assuming entry + 1 is an interior byte. + let multiByteInstruction; + for (const range of original.procedure.body.ranges) { + let cursor = BigInt(range.start); + while (cursor <= BigInt(range.end)) { + const instruction = await call("inspect_native_instruction", { + address: `0x${cursor.toString(16)}`, + }); + if (instruction.status === "decoded" && instruction.length > 1) { + multiByteInstruction = instruction; + break; + } + cursor += BigInt(instruction.length ?? 1); + } + if (multiByteInstruction !== undefined) break; + } + assert.ok(multiByteInstruction, "Fixture lacks a multi-byte instruction"); + const interior = `0x${(BigInt(multiByteInstruction.address) + 1n).toString(16)}`; assert.equal( (await call("inspect_native_instruction", { address: interior })).status, "not-instruction-boundary", @@ -693,6 +712,13 @@ export async function verifyGhidraBoundaries( entrypoint, env, }); + await verifyGhidraEntryAliases({ + call, + reject: invalid, + target, + entrypoint, + env, + }); await verifyGhidraLargeResults({ call, reject: invalid, @@ -741,6 +767,7 @@ export async function verifyGhidraBoundaries( imported_source_identity_retained: true, equivalent_instruction_address_spellings: true, qualified_annotation_name_roundtrip: true, + imported_entry_alias_selection: true, oversized_result_retention_and_complete_export: true, long_selector_rejection_and_provider_recovery: true, source_immutable: true, diff --git a/scripts/lib/real-ghidra-entry-aliases.mjs b/scripts/lib/real-ghidra-entry-aliases.mjs new file mode 100644 index 00000000000..3230ed6d1c2 --- /dev/null +++ b/scripts/lib/real-ghidra-entry-aliases.mjs @@ -0,0 +1,126 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +/** Verify imported entry aliases through CLI/MCP without admitting interior labels. */ +export async function verifyGhidraEntryAliases({ + call, + reject, + target, + entrypoint, + env, +}) { + const exec = promisify(execFile); + const compiler = process.env.REA_CC ?? "cc"; + const workspace = await mkdtemp(join(tmpdir(), "rea-ghidra-entry-aliases-")); + const source = fileURLToPath( + new URL("../../tests/conformance/ghidra/entry-aliases.c", import.meta.url), + ); + const path = join(workspace, "entry-aliases"); + const prefix = process.platform === "darwin" ? "_" : ""; + try { + await exec(compiler, ["-O0", "-g", "-fno-inline", source, "-o", path], { + env, + timeout: 30000, + }); + const bytes = await readFile(path); + await call("close_binary"); + await call("open_binary", { path, provider_id: "ghidra" }); + const names = await call("list_names"); + const alias = names.find( + ({ value }) => value === `${prefix}rea_entry_alias`, + ); + assert.ok(alias, "Fixture entry alias was not imported"); + assert.equal( + alias.symbol.primary, + false, + "The regression must exercise a secondary entry symbol", + ); + assert.equal(alias.symbol.source, "imported"); + const canonical = (await call("list_procedures")).find( + ({ address }) => address === alias.address, + ); + assert.ok(canonical, "Fixture alias is not at a function entry"); + for (const procedure of [ + alias.value, + "dead", + canonical.value, + alias.address, + ]) { + assert.equal( + await call("procedure_address", { procedure }), + alias.address, + ); + assert.equal( + (await call("analyze_function", { procedure })).procedure.address, + alias.address, + ); + assert.equal( + await call("procedure_pseudo_code", { procedure }), + await call("procedure_pseudo_code", { procedure: alias.address }), + ); + } + // Leaf and qualified forms of the primary name must not duplicate a match. + assert.equal( + (await call("procedure_info", { procedure: alias.value })).entrypoint, + alias.address, + ); + const interior = names.find( + ({ value }) => value === `${prefix}rea_interior`, + ); + assert.ok(interior); + assert.notEqual(interior.address, alias.address); + const failure = await reject("procedure_address", { + procedure: interior.value, + }); + assert.equal(failure.code, "invalid_request"); + assert.match( + JSON.stringify(failure.details.issues), + /Unknown Ghidra procedure name or address/u, + ); + assert.equal( + await call("procedure_address", { procedure: interior.address }), + alias.address, + "Explicit interior addresses keep their existing containment semantics", + ); + const edited = await call("annotate_native_function", { + procedure: alias.value, + comment: "Selected by imported entry alias", + }); + assert.equal(edited.annotations.address, alias.address); + assert.equal( + edited.annotations.comment, + "Selected by imported entry alias", + ); + assert.equal( + await call("procedure_address", { procedure: alias.value }), + alias.address, + ); + await call("close_binary"); + const { stdout } = await exec( + process.execPath, + [ + entrypoint, + "function", + path, + alias.value, + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + assert.equal( + JSON.parse(stdout).normalized_result.procedure.address, + alias.address, + ); + assert.deepEqual(await readFile(path), bytes); + await call("open_binary", { path: target.path, provider_id: "ghidra" }); + } finally { + await rm(workspace, { recursive: true, force: true }); + } +} diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 418136f7488..57fc2f1f2b0 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -43,7 +43,7 @@ export const resolveGhidraAnalysisProfile = ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", - location_resolution: "explicit-address-exact-name-first-v2", + location_resolution: "explicit-address-exact-entry-symbol-first-v3", process_launch: installation.platform === "win32" ? "official-headless-script-v1" diff --git a/tests/conformance/ghidra/README.md b/tests/conformance/ghidra/README.md index b99acb64431..8f9148d4b93 100644 --- a/tests/conformance/ghidra/README.md +++ b/tests/conformance/ghidra/README.md @@ -7,6 +7,9 @@ binaries and Ghidra projects are never committed. It covers imports, an external `puts` function, linker thunks, source symbols, direct calls, a targetless callback call, two referenced strings, and a multi-block branch. +- `entry-aliases.c` preserves secondary imported entry labels, a bare-hex + alias, and an interior label. The native CLI/MCP lane verifies entry selection, + annotation, interior-label rejection, and unchanged source bytes. - `cross-format.c` is freestanding so the verifier can produce AArch64 ELF, x86-64 PE, and x86-64 Mach-O targets from the same semantics. It preserves an exported entry, direct and indirect calls, a volatile string reference, and a diff --git a/tests/conformance/ghidra/entry-aliases.c b/tests/conformance/ghidra/entry-aliases.c new file mode 100644 index 00000000000..3f99820445a --- /dev/null +++ b/tests/conformance/ghidra/entry-aliases.c @@ -0,0 +1,19 @@ +// Keep secondary imported entry labels and a non-entry control in one function. +#if defined(__APPLE__) +#define REA_SYMBOL(name) "_" name +#else +#define REA_SYMBOL(name) name +#endif + +__attribute__((noinline, used)) int rea_alias_target(void) { + __asm__ volatile(".globl " REA_SYMBOL("rea_interior") "\n" + REA_SYMBOL("rea_interior") ":"); + return 17; +} + +__asm__(".globl " REA_SYMBOL("rea_entry_alias") "\n" + ".set " REA_SYMBOL("rea_entry_alias") ", " REA_SYMBOL("rea_alias_target") "\n" + // A bare hexadecimal identifier must still resolve as an entry label. + ".globl dead\n.set dead, " REA_SYMBOL("rea_alias_target") "\n"); + +int main(void) { return rea_alias_target(); } From b79d379c25bcc8ed61571c41a94b912b5704a067 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:19:10 +0000 Subject: [PATCH 129/791] fix(ghidra): keep and name a runtime directory that close could not remove When removing Ghidra's private runtime directory failed, close_binary reported only a generic `provider-client` resource, and a later close, after the permission was restored, returned success while the directory and its contents remained. The client dropped its reference to the root before removing it, and PrivateRuntimeRoot cached the rejected removal. Keep each runtime or socket root until its removal succeeds, report a failure as a ProviderCleanupError naming the leftover paths, let PrivateRuntimeRoot retry a failed removal, and pass such typed cleanup failures through closeAnalysisClient instead of replacing them. Fixes #1190 Co-Authored-By: Claude Opus 5.5 --- .../binary/AnalysisClientCleanup.test.ts | 45 ++++++++++++++++ .../binary/AnalysisClientCleanup.ts | 5 +- src/ghidra/GhidraClient.ts | 53 +++++++++++++++---- src/process/PrivateRuntimeRoot.ts | 13 +++-- .../providers/ghidra/ghidraClient.test.ts | 31 ++++++++++- 5 files changed, 132 insertions(+), 15 deletions(-) create mode 100644 src/application/binary/AnalysisClientCleanup.test.ts diff --git a/src/application/binary/AnalysisClientCleanup.test.ts b/src/application/binary/AnalysisClientCleanup.test.ts new file mode 100644 index 00000000000..23e69b832f6 --- /dev/null +++ b/src/application/binary/AnalysisClientCleanup.test.ts @@ -0,0 +1,45 @@ +import { expect, it } from "vitest"; + +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { err } from "../../domain/result.js"; +import type { AnalysisClient } from "../AnalysisProvider.js"; +import { closeAnalysisClient } from "./AnalysisClientCleanup.js"; + +const clientClosingWith = (cause: unknown): AnalysisClient => ({ + execute: () => Promise.resolve(err(new ProviderCleanupError("x", [], {}))), + close: () => Promise.reject(cause), +}); + +it("keeps the leftover resources a typed cleanup failure names", async () => { + const leftover = "/tmp/rea-ghidra-fixture"; + const failure = new ProviderCleanupError("ghidra", [leftover], { + reason: "runtime directory retained", + }); + + const closed = await closeAnalysisClient( + clientClosingWith(failure), + "ghidra", + ); + + expect(closed.ok).toBe(false); + if (!closed.ok) + expect(projectAnalysisError(closed.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { resources: [leftover] }, + }); +}); + +it("still reports an untyped close rejection as the provider client", async () => { + const closed = await closeAnalysisClient( + clientClosingWith(new Error("unexpected")), + "ghidra", + ); + + expect(closed.ok).toBe(false); + if (!closed.ok) + expect(projectAnalysisError(closed.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { resources: ["provider-client"] }, + }); +}); diff --git a/src/application/binary/AnalysisClientCleanup.ts b/src/application/binary/AnalysisClientCleanup.ts index 3cd13b21563..a21a33b4be9 100644 --- a/src/application/binary/AnalysisClientCleanup.ts +++ b/src/application/binary/AnalysisClientCleanup.ts @@ -1,6 +1,6 @@ import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; -import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { AnalysisClient, ExecutionOptions } from "../AnalysisProvider.js"; /** Close one provider client and normalize unexpected adapter rejection. */ @@ -17,6 +17,9 @@ export const closeAnalysisClient = async ( await client.close(); return ok(null); } catch (cause: unknown) { + // A typed cleanup failure already names its leftover resources. + if (cause instanceof AnalysisError && cause.cleanupIncomplete) + return err(cause); return err( new ProviderCleanupError( providerId, diff --git a/src/ghidra/GhidraClient.ts b/src/ghidra/GhidraClient.ts index a7225df6e17..96f6919d3c7 100644 --- a/src/ghidra/GhidraClient.ts +++ b/src/ghidra/GhidraClient.ts @@ -3,6 +3,7 @@ import type { Socket } from "node:net"; import { tmpdir } from "node:os"; import type { JsonValue } from "../domain/jsonValue.js"; +import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { err, ok, type Result } from "../domain/result.js"; import { silentLogger, type Logger } from "../logger.js"; import { PendingOperations } from "../process/PendingOperations.js"; @@ -534,17 +535,8 @@ export class GhidraClient { this.#lastDiagnostics = this.#diagnostics(); this.#process = undefined; this.#launch = undefined; - const runtimeRoot = this.#runtimeRoot; - const socketRoot = this.#socketRoot; - this.#runtimeRoot = undefined; - this.#socketRoot = undefined; try { - const removed = await Promise.allSettled([ - runtimeRoot?.close(), - socketRoot?.close(), - ]); - const failed = removed.find((result) => result.status === "rejected"); - if (failed?.status === "rejected") throw failed.reason; + await this.#removeRuntimeRoots(); } catch (cause: unknown) { if (forceStop) forcedStopFailure ??= this.#failure( @@ -565,6 +557,47 @@ export class GhidraClient { } } + /** + * Remove the runtime and socket roots, keeping each one that could not be + * removed so a later close retries it rather than reporting success. + */ + async #removeRuntimeRoots(): Promise { + const roots = [ + { + root: this.#runtimeRoot, + release: () => (this.#runtimeRoot = undefined), + }, + { root: this.#socketRoot, release: () => (this.#socketRoot = undefined) }, + ]; + const leftovers: { readonly path: string; readonly reason: string }[] = []; + let firstCause: unknown; + for (const { root, release } of roots) { + if (root === undefined) continue; + try { + await root.close(); + release(); + } catch (cause: unknown) { + firstCause ??= cause; + leftovers.push({ + path: root.path, + reason: cause instanceof Error ? cause.message : String(cause), + }); + } + } + if (leftovers.length > 0) + throw new ProviderCleanupError( + "ghidra", + leftovers.map(({ path }) => path), + { + reason: + "Ghidra's private runtime directory could not be removed; it is retained for a later close.", + leftover_paths: leftovers.map(({ path }) => path), + failures: leftovers.map(({ path, reason }) => `${path}: ${reason}`), + }, + { cause: firstCause }, + ); + } + #redactAuthentication(value: string): string { for (const token of this.#authenticationTokens) value = value.replaceAll(token, "[REDACTED]"); diff --git a/src/process/PrivateRuntimeRoot.ts b/src/process/PrivateRuntimeRoot.ts index 40d4414b7ec..8fe63ce727b 100644 --- a/src/process/PrivateRuntimeRoot.ts +++ b/src/process/PrivateRuntimeRoot.ts @@ -86,12 +86,19 @@ export class PrivateRuntimeRoot { } } - /** Remove the runtime root; concurrent and repeated callers share cleanup. */ + /** + * Remove the runtime root; concurrent callers share one attempt. A failed + * attempt is not cached, so a later close retries the removal. + */ close(): Promise { - this.#closePromise ??= + this.#closePromise ??= ( this.windowsRuntime === undefined ? rm(this.path, { recursive: true, force: true }) - : Promise.resolve().then(() => this.windowsRuntime?.close()); + : Promise.resolve().then(() => this.windowsRuntime?.close()) + ).catch((cause: unknown) => { + this.#closePromise = undefined; + throw cause; + }); return this.#closePromise; } } diff --git a/tests/boundary/providers/ghidra/ghidraClient.test.ts b/tests/boundary/providers/ghidra/ghidraClient.test.ts index a4dd3185704..b19f8502518 100644 --- a/tests/boundary/providers/ghidra/ghidraClient.test.ts +++ b/tests/boundary/providers/ghidra/ghidraClient.test.ts @@ -1,7 +1,7 @@ import { spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { readFileSync } from "node:fs"; -import { access, mkdir } from "node:fs/promises"; +import { access, chmod, mkdir } from "node:fs/promises"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; @@ -386,6 +386,35 @@ describe("GhidraClient established requests", () => { }); }); +describe("GhidraClient runtime removal failures", () => { + it.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports a runtime root it could not remove and removes it on a later close", + async () => { + const launcher = new FixtureLauncher(); + const client = clientFor(launcher); + await expect(client.start()).resolves.toMatchObject({ ok: true }); + const runtimeRoot = launcher.runtimeRoots[0] ?? ""; + // Without write permission its entries cannot be removed. + await chmod(runtimeRoot, 0o500); + try { + await expect(client.close()).rejects.toMatchObject({ + cleanupIncomplete: true, + cleanupResources: [runtimeRoot], + diagnostics: { leftover_paths: [runtimeRoot] }, + }); + await expect(access(runtimeRoot)).resolves.toBeUndefined(); + } finally { + await chmod(runtimeRoot, 0o700); + } + + await expect(client.close()).resolves.toBeUndefined(); + await expect(access(runtimeRoot)).rejects.toMatchObject({ + code: "ENOENT", + }); + }, + ); +}); + describe("GhidraClient cleanup and diagnostics", () => { it("makes double close idempotent and releases its process runtime", async () => { const launcher = new FixtureLauncher(); From 761881cec33b18d7228bae5e2093e5d44b96ceb1 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 12:19:38 +0800 Subject: [PATCH 130/791] ci(website): publish Cloudflare and Pages from one artifact --- .github/workflows/website-pages.yml | 90 ++++++++++++++++++++++++++++- .gitignore | 1 + website/AGENTS.md | 5 +- website/README.md | 44 +++++++++----- website/style-guide.md | 8 ++- 5 files changed, 127 insertions(+), 21 deletions(-) diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml index 52908cd8b54..881a66dc858 100644 --- a/.github/workflows/website-pages.yml +++ b/.github/workflows/website-pages.yml @@ -19,12 +19,33 @@ jobs: contents: read pages: read steps: + - name: Require Cloudflare publishing credentials + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if [[ -z "$CLOUDFLARE_API_TOKEN" || -z "$CLOUDFLARE_ACCOUNT_ID" ]]; then + echo "::error::Set CLOUDFLARE_API_TOKEN and CLOUDFLARE_ACCOUNT_ID in repository Actions secrets before publishing." + exit 1 + fi - name: Check out the repository uses: actions/checkout@v4 - name: Prepare the example download run: python3 scripts/prepare-website.py - name: Check website links and publishing configuration run: python3 scripts/verify-website.py + - name: Record the published commit + run: | + python3 - <<'PY' + import json + import os + from pathlib import Path + + Path("website/public/deployment.json").write_text( + json.dumps({"commit": os.environ["GITHUB_SHA"]}) + "\n", + encoding="utf-8", + ) + PY - name: Configure Pages uses: actions/configure-pages@v5 - name: Upload the website @@ -35,14 +56,81 @@ jobs: deploy: needs: prepare runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 15 permissions: + contents: read + actions: read pages: write id-token: write environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Check out publishing configuration + uses: actions/checkout@v4 + - name: Set up Node.js for Wrangler + uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + - name: Download the verified website + uses: actions/download-artifact@v4 + with: + name: github-pages + path: ${{ runner.temp }}/website-archive + - name: Extract the shared publishing artifact + run: | + mkdir -p "$RUNNER_TEMP/rea-website-public" + tar -xf "$RUNNER_TEMP/website-archive/artifact.tar" -C "$RUNNER_TEMP/rea-website-public" + - name: Publish the website to Cloudflare + uses: cloudflare/wrangler-action@v4 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + wranglerVersion: "4.149.0" + packageManager: npm + command: deploy --cwd website --env production --assets "${{ runner.temp }}/rea-website-public" - name: Publish to GitHub Pages id: deployment uses: actions/deploy-pages@v4 + - name: Verify both published revisions + env: + PAGES_URL: ${{ steps.deployment.outputs.page_url }} + run: | + python3 - <<'PY' + import json + import os + import sys + import time + from urllib.error import URLError + from urllib.parse import urljoin + from urllib.request import Request, urlopen + + expected = os.environ["GITHUB_SHA"] + pages_url = os.environ["PAGES_URL"] + if not pages_url: + sys.exit("GitHub Pages returned no published URL.") + pending = {"Cloudflare": "https://rea.tools/", "GitHub Pages": pages_url} + for attempt in range(24): + for name, base in list(pending.items()): + url = urljoin(base, "deployment.json") + "?commit=" + expected + try: + request = Request(url, headers={"Cache-Control": "no-cache"}) + with urlopen(request, timeout=10) as response: + actual = json.load(response).get("commit") + if actual == expected: + print(f"{name} serves {expected}: {base}") + del pending[name] + else: + print(f"Waiting for {name}: observed revision {actual}") + except (URLError, TimeoutError, ValueError) as error: + print(f"Waiting for {name}: {error}") + if not pending: + break + if attempt < 23: + time.sleep(5) + if pending: + sys.exit("Published revision did not match on: " + ", ".join(pending)) + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write(f"Published commit `{expected}` to [rea.tools](https://rea.tools/) " + f"and [GitHub Pages]({pages_url}).\n") + PY diff --git a/.gitignore b/.gitignore index f8266c11d23..daca8832e42 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ Thumbs.db # Generated website download /website/public/examples/notes-example.zip +/website/public/deployment.json # Local Cloudflare development state /website/.wrangler/ diff --git a/website/AGENTS.md b/website/AGENTS.md index a4bf9b8dc55..47bee247a5b 100644 --- a/website/AGENTS.md +++ b/website/AGENTS.md @@ -19,5 +19,6 @@ for asset notes and the source of case-study claims. - Check both local root and `/rea/` paths, mobile layouts, expanded details, copying and downloads. Run `scripts/prepare-website.py` followed by `scripts/verify-website.py`. -- Keep `website-pages.yml` as the sole, manual Pages publisher. Follow the - publication procedure in the website README. +- Keep `website-pages.yml` as the sole, manual production publisher for + Cloudflare and Pages. Both hosts must receive the same verified artifact and + pass the published-commit check. Follow the website README. diff --git a/website/README.md b/website/README.md index 0f29bc32bb1..92b0be51d5b 100644 --- a/website/README.md +++ b/website/README.md @@ -185,7 +185,7 @@ Wrangler version without adding it to REA's package dependencies. Cloudflare documents static-asset requests as free and unlimited, with no additional storage cost. See [billing and limitations](https://developers.cloudflare.com/workers/static-assets/billing-and-limitations/). -### Preview and deploy +### Local and hosted previews Run these commands from the repository root. `--cwd website` makes the build paths consistent with the website configuration. Preview locally first: @@ -230,13 +230,7 @@ See Cloudflare's [nameserver setup procedure](https://developers.cloudflare.com/ for the complete migration steps. A CNAME to the preview's `workers.dev` hostname at the current DNS provider does not configure a Workers Custom Domain. -Publish to the production Worker: - -```sh -npx wrangler@4.149.0 deploy --cwd website --env production -``` - -This creates or updates `rea-website`, attaches `rea.tools`, and lets Cloudflare +The production environment creates or updates `rea-website`, attaches `rea.tools`, and lets Cloudflare manage its DNS record and HTTPS certificate. Deployment can succeed while the zone is pending; verify public HTTPS after activation and certificate issuance. An existing CNAME at `rea.tools` @@ -250,15 +244,34 @@ Verify , , resolve correctly. The root README and browser guide use `https://rea.tools/` as the public website URL. -Deployments are manual. The existing GitHub Pages publisher remains available -as a separate host; it does not publish to Cloudflare. +Production publication uses the synchronized GitHub Actions workflow below. +Local Wrangler commands above publish only the separate preview Worker. -## GitHub Pages +## Synchronized production publishing -`.github/workflows/website-pages.yml` prepares and deploys only `website/public`. -It is the sole Pages publisher, manually triggered and restricted to `main`; +`.github/workflows/website-pages.yml` prepares `website/public` once and deploys +the same artifact to Cloudflare (`rea.tools`) and GitHub Pages. It is the sole +production publisher, manually triggered and restricted to `main`; ordinary pushes and pull requests do not publish the site. +Configure these repository **Actions secrets** before publishing: + +- `CLOUDFLARE_API_TOKEN`: a Cloudflare token using the **Edit Cloudflare Workers** + template, scoped to the website's account and zone. +- `CLOUDFLARE_ACCOUNT_ID`: that Cloudflare account's ID. + +See Cloudflare's [GitHub Actions authentication instructions](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/). +The workflow checks that both secrets are present before preparing a release. +Cloudflare deploys first; GitHub Pages deploys after that step succeeds. + +Preparation adds `deployment.json` with the workflow's commit SHA. Cloudflare +receives the extracted Pages artifact through Wrangler's explicit assets path; +the local custom-build output is not the uploaded directory. The workflow then +checks both public version markers and reports success only when they match +that commit. A failed publication remains failed; fix its cause and rerun the +workflow to complete both deployments. The two hosts can update at different +times while a run is in progress. + Website checks run only for pull requests that change `website/`, the verification script or workflow definitions. They check local links, HTML fragments, SVG XML and the single Pages publisher without installing npm dependencies. The same @@ -277,8 +290,9 @@ the old site. After the site is approved and merged, select **GitHub Actions** under the repository's **Settings → Pages → Build and deployment**. Then run **Publish REA -website** from the Actions tab on `main`. The workflow uses the `github-pages` -environment and the official Pages actions. +website** from the Actions tab on `main`. This publishes both production hosts. +The workflow uses the `github-pages` environment, the official Pages actions and +Cloudflare's Wrangler action. The equivalent CLI command is: diff --git a/website/style-guide.md b/website/style-guide.md index e17475d5778..5a05299c60c 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -233,6 +233,8 @@ New analysis or behavior claims need the real evidence that supports them; a moc or a page-loading check cannot establish those claims. Follow the PR review and CI process, then publish using the existing manual -`website-pages.yml` workflow on `main`. Ordinary commits should not publish the -site. After deployment, check the live page and confirm it matches the reviewed -version. The legacy VitePress build remains separate from the Pages publisher. +`website-pages.yml` workflow on `main`. It publishes the same verified artifact +to Cloudflare and GitHub Pages and checks both published commit markers. +Ordinary commits should not publish the site. After deployment, check both live +pages and confirm they match the reviewed version. The legacy VitePress build +remains separate from the Pages publisher. From 911152a04d3aeb0d361141a8d54959009b269e49 Mon Sep 17 00:00:00 2001 From: Sravan Avvaru <81159574+Sravan1011@users.noreply.github.com> Date: Fri, 9 Oct 2026 10:15:23 +0530 Subject: [PATCH 131/791] fix(cli): complete cleanup after relayed interrupts (#1202) * fix: Handle relayed SIGINT gracefully in withProcessCancellation Changes `process.once` to `process.on` for SIGINT in `DirectAnalysis.ts`. When REA is launched via npx, a single process-group SIGINT triggers two SIGINT events (one direct, one relayed). Previously, the first event would consume the handler and the second would abruptly terminate the Node process, abandoning Ghidra resource cleanup. This change ensures subsequent SIGINTs idempotently request cancellation without crashing the process, matching the behaviour of `withCommandCancellation`. * fix(cli): keep termination guards through provider cleanup * test(cli): exercise repeated analysis signals on macOS --------- Co-authored-by: N0zoM1z0 <1705004398@qq.com> --- .github/workflows/ci.yml | 2 +- docs/mcp-contracts.md | 8 +- src/application/DirectAnalysis.ts | 6 +- .../process/directAnalysisSignals.test.ts | 78 ++++++++++++++++++ .../directAnalysisFactories.test.ts | 6 +- tests/fixtures/directAnalysisInterrupt.mjs | 81 +++++++++++++++++++ 6 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 tests/boundary/process/directAnalysisSignals.test.ts create mode 100644 tests/fixtures/directAnalysisInterrupt.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 099208716a0..14f8f42c55b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -334,7 +334,7 @@ jobs: - name: Process filesystem snapshot descriptor regressions run: npm run test:focused -- src/process/capture/FilesystemSnapshot.test.ts - name: Native Darwin ownership and process cleanup regressions - run: npm run test:focused -- tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/processCaptureLifecycle.test.ts tests/boundary/process/processCli.test.ts tests/boundary/mcp/sessionMcp.test.ts + run: npm run test:focused -- tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/directAnalysisSignals.test.ts tests/boundary/process/processCaptureLifecycle.test.ts tests/boundary/process/processCli.test.ts tests/boundary/mcp/sessionMcp.test.ts reconstruction-readiness: name: Reconstruction readiness diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 90ffbc97de2..1eb03986e5c 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -103,9 +103,11 @@ still run synchronously, so control messages can wait for those phases to release the event loop. A rejected client promise alone does not establish that the server has stopped its work. -CLI calls work without a progress token and translate SIGINT into the same -AbortSignal used by providers. Existing controlled-process cleanup and provider -shutdown rules still apply; REA never kills a process it cannot prove it owns. +Direct CLI analysis calls work without a progress token and translate SIGINT or +SIGTERM into the same AbortSignal used by providers. Signal guards remain active +through provider cleanup, including repeated delivery by package runners. +Existing controlled-process cleanup and provider shutdown rules still apply; +REA never kills a process it cannot prove it owns. ## Ghidra first-query deadlines and recovery diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index db54c06b5f4..f9f15244ffb 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -388,11 +388,15 @@ const withProcessCancellation = async ( if (suppliedSignal !== undefined) return operation(suppliedSignal); const controller = new AbortController(); const cancel = (): void => controller.abort(); - process.once("SIGINT", cancel); + // Package runners can forward a terminal interrupt after REA already received + // it directly. Keep both guards installed until provider cleanup has settled. + process.on("SIGINT", cancel); + process.on("SIGTERM", cancel); try { return await operation(controller.signal); } finally { process.off("SIGINT", cancel); + process.off("SIGTERM", cancel); } }; diff --git a/tests/boundary/process/directAnalysisSignals.test.ts b/tests/boundary/process/directAnalysisSignals.test.ts new file mode 100644 index 00000000000..e72ce596ea6 --- /dev/null +++ b/tests/boundary/process/directAnalysisSignals.test.ts @@ -0,0 +1,78 @@ +import { spawn } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { expect, it, onTestFinished } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it.skipIf(process.platform === "win32").each([ + ["direct", "SIGINT"], + ["native", "SIGINT"], + ["managed", "SIGINT"], + ["direct", "SIGTERM"], + ["native", "SIGTERM"], + ["managed", "SIGTERM"], +] as const)( + "finishes %s cleanup when %s is delivered again", + async (mode, signal) => { + const root = await createTestTempDirectory("rea-direct-analysis-signals-"); + const child = spawn( + process.execPath, + ["tests/fixtures/directAnalysisInterrupt.mjs", root, mode], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + let stdout = "", + stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + const closed = new Promise<{ + code: number | null; + signal: NodeJS.Signals | null; + }>((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, exitSignal) => + resolve({ code, signal: exitSignal }), + ); + }); + onTestFinished(async () => { + await writeFile(join(root, "release-cleanup"), "release"); + if (child.exitCode === null && child.signalCode === null) + child.kill("SIGKILL"); + await closed; + }); + const waitFor = async (name: string) => { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (child.exitCode !== null || child.signalCode !== null) + throw new Error(`Child exited before ${name}: ${stderr}`); + try { + await access(join(root, name)); + return; + } catch (cause: unknown) { + if ( + !(cause instanceof Error) || + !("code" in cause) || + cause.code !== "ENOENT" + ) + throw cause; + } + await delay(20); + } + throw new Error(`Child did not reach ${name}: ${stderr}`); + }; + await waitFor("executing"); + child.kill(signal); + await waitFor("cleanup-started"); + child.kill(signal); + await writeFile(join(root, "release-cleanup"), "release"); + expect(await closed, stderr).toEqual({ code: 0, signal: null }); + expect(await readFile(join(root, "cleanup-complete"), "utf8")).toBe( + "complete", + ); + expect(JSON.parse(stdout)).toMatchObject({ code: "cancelled" }); + }, +); diff --git a/tests/composition/directAnalysisFactories.test.ts b/tests/composition/directAnalysisFactories.test.ts index 2de45c68e09..5dfffb82292 100644 --- a/tests/composition/directAnalysisFactories.test.ts +++ b/tests/composition/directAnalysisFactories.test.ts @@ -68,12 +68,14 @@ describe("one-shot analysis factory boundary", () => { const [path] = await createBinarySessionTargets(); const failure = new Error("fixture execution failed"); const factories = recordingFactories(failure); - const listeners = process.listenerCount("SIGINT"); + const interruptListeners = process.listenerCount("SIGINT"); + const terminateListeners = process.listenerCount("SIGTERM"); await expect( runDirectAnalysis(factories.dependencies, path, "read_bytes", {}), ).rejects.toBe(failure); expect(factories.clientsClosed()).toBe(1); - expect(process.listenerCount("SIGINT")).toBe(listeners); + expect(process.listenerCount("SIGINT")).toBe(interruptListeners); + expect(process.listenerCount("SIGTERM")).toBe(terminateListeners); expect(factories.sessions[0]?.activeTarget()).toBeUndefined(); }); diff --git a/tests/fixtures/directAnalysisInterrupt.mjs b/tests/fixtures/directAnalysisInterrupt.mjs new file mode 100644 index 00000000000..76ecde8ea2f --- /dev/null +++ b/tests/fixtures/directAnalysisInterrupt.mjs @@ -0,0 +1,81 @@ +import { access, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { + runDirectAnalysis, + runProviderAnalysis, +} from "../../dist/application/DirectAnalysis.js"; +import { BinarySession } from "../../dist/application/binary/BinarySession.js"; +import { SessionProviderRouter } from "../../dist/application/binary/SessionProviderRouter.js"; +import { createAnalysisExecution } from "../../dist/application/AnalysisProvider.js"; +import { AnalysisCancelledError } from "../../dist/domain/analysisErrorCore.js"; +import { err, ok } from "../../dist/domain/result.js"; + +const [root, mode] = process.argv.slice(2); +if (!root || !["direct", "native", "managed"].includes(mode)) + throw new Error( + "Expected a fixture root and a direct, native, or managed mode", + ); +const target = join(root, "target.hop"); +await writeFile(target, "fixture"); +const client = { + async execute(operation, _arguments, options) { + if (operation === "health") + return ok( + createAnalysisExecution(null, { + id: "fixture", + name: "Fixture", + version: "1", + }), + ); + await writeFile(join(root, "executing"), "ready"); + await new Promise((resolve) => { + if (options.signal.aborted) resolve(); + else options.signal.addEventListener("abort", resolve, { once: true }); + }); + return err(new AnalysisCancelledError(operation)); + }, + async close() { + await writeFile(join(root, "cleanup-started"), "ready"); + while (true) { + try { + await access(join(root, "release-cleanup")); + break; + } catch (cause) { + if (cause.code !== "ENOENT") throw cause; + } + await delay(5); + } + await writeFile(join(root, "cleanup-complete"), "complete"); + }, +}; +const createSession = () => + new BinarySession(SessionProviderRouter.single(() => client)); +const dependencies = { + createBinarySession: createSession, + createManagedBinarySession: createSession, +}; +const interruptListeners = process.listenerCount("SIGINT"); +const terminateListeners = process.listenerCount("SIGTERM"); +// The production provider owns sockets/process handles; keep this fixture alive +// while its replacement waits only on a signal. +const keepAlive = setInterval(() => {}, 1000); +const result = + mode === "direct" + ? await runDirectAnalysis(dependencies, target, "read_bytes", { + address: "0x0", + length: 1, + }) + : await runProviderAnalysis( + dependencies, + target, + mode === "managed" ? "inspect_managed_artifact" : "procedure_address", + mode === "managed" ? {} : { procedure: "fixture" }, + ); +clearInterval(keepAlive); +if ( + process.listenerCount("SIGINT") !== interruptListeners || + process.listenerCount("SIGTERM") !== terminateListeners +) + throw new Error("Analysis leaked process cancellation listeners"); +process.stdout.write(JSON.stringify(result)); From db675f1d52acda62b6f57b5a8773532f49b259e7 Mon Sep 17 00:00:00 2001 From: Matt Van Horn Date: Thu, 8 Oct 2026 21:45:48 -0700 Subject: [PATCH 132/791] feat(mcp): select views of retained analysis results (#1180) Add explicit summary, facet, item and paged views of retained analysis Evidence without repeating analysis. Preserve captured observations, graph validation, artifact identity, coverage and complete default results. Fixes #1050 Verified with focused source, CLI, pinned-SDK schema and MCP tests, serial static checks, and one complete native Windows Notion ASAR workflow. Real Hopper/Ghidra verification was not performed for this provider-independent projection change. Co-authored-by: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> --- docs/binary-diagnostics.md | 7 + docs/javascript-application-workflows.md | 15 +- docs/javascript-artifact-reconstruction.md | 13 +- docs/mcp-contracts.md | 17 +- docs/mcp-prompts.md | 4 +- scripts/lib/catalog-builders.mjs | 5 + scripts/lib/catalog-core.mjs | 1 + skill-src/reverse-engineer-anything/SKILL.md | 15 +- .../references/evidence-workflows.md | 5 +- .../references/javascript-applications.md | 14 +- .../references/native-and-artifacts.md | 9 + src/application/CapabilityInventory.test.ts | 4 + src/application/InvestigationProviders.ts | 7 + .../analysisView/AnalysisViewService.test.ts | 176 ++++++ .../analysisView/AnalysisViewService.ts | 132 +++++ src/cli.ts | 2 + src/cli/analysisViewCommands.ts | 53 ++ src/cliCommandNames.ts | 1 + src/contracts/analysisViewToolContracts.ts | 63 +++ src/contracts/contractSnapshot.test.ts | 2 + src/contracts/promptContracts.ts | 4 +- src/contracts/toolContracts.test.ts | 2 + src/contracts/toolContracts.ts | 2 + src/contracts/toolEffects.ts | 1 + src/domain/analysisView/analysisView.test.ts | 526 ++++++++++++++++++ src/domain/analysisView/analysisView.ts | 341 ++++++++++++ .../analysisView/analysisViewCoverage.ts | 29 + src/domain/analysisView/binaryLayoutView.ts | 239 ++++++++ .../analysisView/javascriptApplicationView.ts | 318 +++++++++++ src/generatedPackageMetadata.ts | 2 +- src/server/createServer.ts | 2 + src/server/registerAnalysisViewTool.ts | 32 ++ tests/boundary/cli/analysisViewCli.test.ts | 82 +++ tests/boundary/cli/jsonInputStatus.test.ts | 1 + tests/boundary/cli/productCatalog.test.ts | 6 + tests/boundary/mcp/analysisViewMcp.test.ts | 270 +++++++++ .../boundary/mcp/binaryDiagnosticsMcp.test.ts | 20 + .../retainedApplicationEvidenceMcp.test.ts | 24 + tests/fixtures/analysisView.ts | 195 +++++++ 39 files changed, 2610 insertions(+), 31 deletions(-) create mode 100644 src/application/analysisView/AnalysisViewService.test.ts create mode 100644 src/application/analysisView/AnalysisViewService.ts create mode 100644 src/cli/analysisViewCommands.ts create mode 100644 src/contracts/analysisViewToolContracts.ts create mode 100644 src/domain/analysisView/analysisView.test.ts create mode 100644 src/domain/analysisView/analysisView.ts create mode 100644 src/domain/analysisView/analysisViewCoverage.ts create mode 100644 src/domain/analysisView/binaryLayoutView.ts create mode 100644 src/domain/analysisView/javascriptApplicationView.ts create mode 100644 src/server/registerAnalysisViewTool.ts create mode 100644 tests/boundary/cli/analysisViewCli.test.ts create mode 100644 tests/boundary/mcp/analysisViewMcp.test.ts create mode 100644 tests/fixtures/analysisView.ts diff --git a/docs/binary-diagnostics.md b/docs/binary-diagnostics.md index e337e0fa1d1..d1addae1349 100644 --- a/docs/binary-diagnostics.md +++ b/docs/binary-diagnostics.md @@ -84,6 +84,13 @@ uses isolated mode and an owned cache. Exact upstream profiles are recorded in stack can accompany an unknown (`null`) NX indicator. ET_DYN can be a shared library, and an absent canary symbol does not prove every function unprotected. +When complete layout Evidence is already retained, `inspect_analysis_view` / +`inspect-analysis-view` projects a summary, the mitigations or linkage facet, +one section or symbol, or a stable page without re-running the decoder. CLI +one-shot invocations pass portable inline Evidence JSON; an MCP session uses +the retained `evidence_id`. Page `limit` is required and bounded by the +measured MCP stdio budget documented on the tool contract. + Complete results have a 32 MiB input, 64 MiB reply, 1 MiB combined diagnostics and 30-second owned command deadline. The reply budget applies to the decoder record; the CLI/MCP Evidence envelope adds copies and encoding overhead. The diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 3821ccbee7f..6b7655e0f86 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -2,12 +2,15 @@ REA derives a complete reachable feature trace from one authenticated JavaScript Application Graph and compares two authenticated graph versions. The MCP tools -are `trace_application_feature`, `trace_javascript_semantics`, -`compare_application_versions`, `compare_source_to_bundle`, and -`compare_javascript_export_shapes`; their CLI equivalents use the same names -with hyphens. - -These workflows consume Evidence produced by +are `inspect_analysis_view`, `trace_application_feature`, +`trace_javascript_semantics`, `compare_application_versions`, +`compare_source_to_bundle`, and `compare_javascript_export_shapes`; their CLI +equivalents use the same names with hyphens. `inspect_analysis_view` projects a +summary, module page, or one module from already completed application +Evidence; it does not walk relationships. + +`inspect_analysis_view` also projects retained `inspect_binary_layout` +Evidence. The remaining workflows consume Evidence produced by `analyze_javascript_application` or `reconcile_javascript_runtime`. They do not read an artifact, execute application code, attach to a process, or open a native-analysis provider. Static artifact observations, passive runtime diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index 51b4f1a9850..cd3c824c532 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -108,14 +108,15 @@ memory needed to construct the analysis graph itself. MCP prepares the complete repeated response incrementally against the pinned SDK's 10 MiB stdio receive-buffer budget. Oversized results return an actionable transport constraint and the exact same-session Evidence reference. Use -`trace_application_feature` to inspect a selected module's relationships, or -`export_evidence_bundle` to write the complete canonical bundle without a -document-sized allocation. Same-session analysis reads reuse authenticated -immutable snapshots; foreign inline Evidence is still parsed and authenticated. +`inspect_analysis_view` with the retained Evidence ID for a summary, one +module, or a stable page of module identities. Use `trace_application_feature` +once a module, route, or string seed is known, or `export_evidence_bundle` to +write the complete canonical bundle without a document-sized allocation. +Same-session analysis reads reuse authenticated immutable snapshots; foreign +inline Evidence is still parsed and authenticated. See [MCP tool results](mcp-contracts.md#tool-results) for larger client buffers and `REA_MCP_MAX_RESPONSE_BYTES`. Follow-up results remain complete and can also -exceed the transport budget. Compact result views are tracked separately in -[#1050](https://github.com/morluto/rea/issues/1050). +exceed the transport budget. ## What is reconstructed diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 1eb03986e5c..994f301a715 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -194,8 +194,9 @@ room for the JSON-RPC envelope. If a result cannot fit, REA returns `resource_constraint` with `details.resource: "transport"` before constructing a document-sized string. Analysis Evidence remains complete in the current session. Its exact reference is reported in -`details.reported_limits.evidence_reference`; use it with a focused application -workflow, or call `export_evidence_bundle` with a destination path. Complete +`details.reported_limits.evidence_reference`; call `inspect_analysis_view` with +that reference for a summary, one section/module, or a stable page, or call +`export_evidence_bundle` with a destination path. Complete bundle exports stream canonical JSON into an atomically published file. A broad follow-up or `get_evidence_bundle` can also exceed the response budget; exporting preserves the complete session without sending it through a single MCP frame. @@ -212,6 +213,18 @@ Ordinary responses keep their existing complete result contract. ## Retained application Evidence inputs +`inspect_analysis_view` projects a caller-selected view of already completed +`inspect_binary_layout` or `analyze_javascript_application` Evidence. Source is +an exact same-session retained reference or portable inline Evidence. Views are +a summary, a layout mitigations or linkage facet, one section/symbol/module, or +a stable page with a caller-selected positive `limit`. Module pages include +JavaScript assets, bundled modules, and source modules. Select an exact +`node_id` when a path is ambiguous or unavailable. Module items retain their +recorded property values and source locations; summaries include parent +application and semantic coverage. The result carries a digest of the selected +facts. Actual serialized size determines MCP transport admission: reduce the +page size or export retained Evidence if the view is too large. + `trace_application_feature`, `trace_javascript_semantics`, `compare_application_versions`, `compare_source_to_bundle`, and `compare_javascript_export_shapes` accept complete inline application Evidence diff --git a/docs/mcp-prompts.md b/docs/mcp-prompts.md index 160f9752297..3c9167265a3 100644 --- a/docs/mcp-prompts.md +++ b/docs/mcp-prompts.md @@ -27,7 +27,9 @@ prompt argument or tool explicitly accepts one. For a JavaScript/Electron application directory, `investigate_feature` points directly to `analyze_javascript_application` with `input_path`; it does not -require `open_binary`. The CLI equivalent is `rea analyze `. +require `open_binary`. When that Evidence, or `inspect_binary_layout` Evidence, +is already retained, it suggests `inspect_analysis_view` before tracing a +feature seed. The CLI equivalent is `rea analyze `. `open_binary` and the doctor target check admit files and macOS app bundles. A directory outside that opening route is reported with an available analysis action; this does not establish support for every directory format. diff --git a/scripts/lib/catalog-builders.mjs b/scripts/lib/catalog-builders.mjs index 8049261cd6e..f497d43a27a 100644 --- a/scripts/lib/catalog-builders.mjs +++ b/scripts/lib/catalog-builders.mjs @@ -75,6 +75,7 @@ export const toolFamilyCatalog = (sources) => { surface: "application-workflow", contracts: [ ...sources.applicationContracts.APPLICATION_TOOL_CONTRACTS, + ...sources.analysisViewContracts.ANALYSIS_VIEW_TOOL_CONTRACTS, ...sources.webScriptContracts.WEB_SCRIPT_TOOL_CONTRACTS, ...sources.javascriptRecoveryContracts .JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, @@ -246,6 +247,10 @@ export const providerCatalog = (sources) => { name !== "project_apple_application_graph", ), }, + { + identity: sources.artifactProviders.ANALYSIS_VIEW_PROVIDER, + contracts: sources.analysisViewContracts.ANALYSIS_VIEW_TOOL_CONTRACTS, + }, { identity: sources.artifactProviders.ANDROID_APPLICATION_PROVIDER, contracts: sources.applicationContracts.APPLICATION_TOOL_CONTRACTS.filter( diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index 9dcfdbb187a..0bb239e00a8 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -64,6 +64,7 @@ const SOURCE_PATHS = { javascriptRuntimeObservationContracts: "dist/contracts/javascript/javascriptRuntimeObservationToolContracts.js", applicationContracts: "dist/contracts/applicationToolContracts.js", + analysisViewContracts: "dist/contracts/analysisViewToolContracts.js", webScriptContracts: "dist/contracts/webScriptToolContracts.js", javascriptRecoveryContracts: "dist/contracts/javascript/javascriptRecoveryToolContracts.js", diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/skill-src/reverse-engineer-anything/SKILL.md index 404ad45ea8a..c2a16cb4fac 100644 --- a/skill-src/reverse-engineer-anything/SKILL.md +++ b/skill-src/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "33" + version: "34" --- # REA @@ -146,12 +146,13 @@ Start with the default result and use its inline Evidence and graph context. Do not repeat an identical tool call. Make a focused follow-up only when the returned result leaves a specific question unanswered. -If MCP reports `resource_constraint` with `details.resource: "transport"`, use -its reported same-session Evidence reference with a focused application tool, -or call `export_evidence_bundle` to write the complete session to a selected -path. The analysis remains complete in the ledger; avoid repeating it merely -to request the same oversized response. Complete CLI JSON output also streams -without a single MCP frame. +If MCP reports `resource_constraint` with `details.resource: "transport"`, call +`inspect_analysis_view` with its reported same-session Evidence reference for a +summary, one section/module, or a stable page. Use `trace_application_feature` +once a seed is known, or call `export_evidence_bundle` to write the complete +session to a selected path. The analysis remains complete in the ledger; avoid +repeating it merely to request the same oversized response. Complete CLI JSON +output also streams without a single MCP frame. Every conclusion must distinguish observations, inferences, and unknowns. Cite Evidence IDs, preserve limitations and incomplete coverage, and never imply diff --git a/skill-src/reverse-engineer-anything/references/evidence-workflows.md b/skill-src/reverse-engineer-anything/references/evidence-workflows.md index b78dbf40a91..4519b9b39b5 100644 --- a/skill-src/reverse-engineer-anything/references/evidence-workflows.md +++ b/skill-src/reverse-engineer-anything/references/evidence-workflows.md @@ -5,7 +5,10 @@ Evidence ID when another tool or your explanation needs a stable reference, and preserve authority, limitations, coverage, and residual unknowns. If a result is incomplete or paginated, continue only when the remaining data matters to the task; do not fetch a bundle or resource merely to read a result already -returned inline. +returned inline. When complete `inspect_binary_layout` or +`analyze_javascript_application` Evidence is already retained, call +`inspect_analysis_view` for a summary, one section/symbol/module, or a stable +page instead of repeating the producer. Use `record_unknown` to track unresolved questions in the user's investigation and name the authority or environment still required. Supply supporting and diff --git a/skill-src/reverse-engineer-anything/references/javascript-applications.md b/skill-src/reverse-engineer-anything/references/javascript-applications.md index 03868c10c20..6d62eaf9550 100644 --- a/skill-src/reverse-engineer-anything/references/javascript-applications.md +++ b/skill-src/reverse-engineer-anything/references/javascript-applications.md @@ -8,10 +8,12 @@ only when a specific question remains unanswered. An oversized response reports `resource_constraint` and `details.resource: "transport"`. Reuse -`details.reported_limits.evidence_reference` with `trace_application_feature` -for the module or feature under investigation. `export_evidence_bundle` writes -the complete canonical session to a caller-selected file. Both workflows keep -the original Evidence and coverage; a broad follow-up can also exceed framing. +`details.reported_limits.evidence_reference` with `inspect_analysis_view` for a +summary, one module, or a stable page of module identities. Use +`trace_application_feature` once a module, route, or string seed is known. +`export_evidence_bundle` writes the complete canonical session to a +caller-selected file. These workflows keep the original Evidence and coverage; +a broad follow-up can also exceed framing. BrowserWindow preferences, preload and contextBridge surfaces, IPC registrations, utility processes, and native binding requests are static syntax @@ -50,7 +52,9 @@ through the available browser, Electron, or process workflows. When advertised by the connected server, application trace and compare tools accept complete inline Evidence or `{"kind":"retained-evidence","evidence_id":"ev_<64 lowercase hex characters>"}` -for their application input (`application`, or `left`/`right`). This notation is +for their application input (`application`, or `left`/`right`). `inspect_analysis_view` +uses the same retained-reference form in `source`, or portable inline Evidence, +to project a summary, module page, or one module with its recorded observations. This notation is a template: replace it with the actual returned ID. Versions before 4.1.0 accept only full inline Evidence. Use the exact ID returned by the producer on the same MCP connection. Resolution does not run diff --git a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md b/skill-src/reverse-engineer-anything/references/native-and-artifacts.md index bf42555e860..08e2ac8745d 100644 --- a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md +++ b/skill-src/reverse-engineer-anything/references/native-and-artifacts.md @@ -16,6 +16,15 @@ Requested unavailable context fails with setup guidance. See [recorded crashes](https://github.com/morluto/rea/blob/main/docs/recorded-crashes.md) for exact upstream profiles, bounds and verification coverage. +## Offline ELF layout + +Use `inspect_binary_layout` with an explicit local ELF path for file-backed +sections, symbols, relocations, linkage names, and static mitigation inferences +without opening a disassembler target. When that complete record is already +retained, call `inspect_analysis_view` for a summary, mitigations or linkage +facet, one section or symbol, or a stable page. Do not re-run the decoder to +read a single object. + ## Native targets After `open_binary`, use focused search, procedure, or function tools directly. diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index f235c740ce4..e2d6156aa88 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -529,5 +529,9 @@ describe("optional JavaScript recovery availability", () => { expect(entry("analyze_javascript_application", status())).toMatchObject({ available: true, }); + expect(entry("inspect_analysis_view", status())).toMatchObject({ + available: true, + surface: "application", + }); }); }); diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index e99a1773f06..20354c27bdd 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -77,6 +77,13 @@ export const JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER = { version: "1", } as const; +/** Provider identity for selected views of retained analysis Evidence. */ +export const ANALYSIS_VIEW_PROVIDER = { + id: "rea-analysis-view", + name: "REA analysis view projection", + version: "1", +} as const; + /** Provider identity for graph-native feature tracing and version comparison. */ export const JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER = { id: "rea-javascript-application-workflows", diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts new file mode 100644 index 00000000000..29289efae83 --- /dev/null +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -0,0 +1,176 @@ +import { expect, it } from "vitest"; + +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; +import { createEvidence, parseEvidence } from "../../domain/evidence.js"; +import { + analysisViewJavaScriptAnalysisWithSource, + analysisViewJavaScriptEvidence, + analysisViewLayoutEvidence, +} from "../../../tests/fixtures/analysisView.js"; +import { inspectAnalysisView } from "./AnalysisViewService.js"; + +it("projects inline layout Evidence and retains a derived view record", () => { + const parent = analysisViewLayoutEvidence(); + const result = inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view: { + kind: "item", + collection: "sections", + selector: { index: 2 }, + }, + }); + if (!result.ok) throw result.error; + const evidence = parseEvidence(result.value); + expect(evidence.operation).toBe("inspect_analysis_view"); + expect(evidence.predicate_type).toBe("rea.analysis-view"); + expect(evidence.confidence).toBe("derived"); + expect(evidence.evidence_links).toEqual([parent.evidence_id]); + expect(evidence.normalized_result).toMatchObject({ + kind: "item", + parent_evidence_id: parent.evidence_id, + item: { name: { display: ".data" } }, + }); +}); + +it("resolves same-session retained JavaScript Evidence without re-running analysis", () => { + const parent = analysisViewJavaScriptEvidence( + analysisViewJavaScriptAnalysisWithSource(), + ); + const lookup = (evidenceId: string) => + evidenceId === parent.evidence_id ? parent : undefined; + const summary = inspectAnalysisView( + { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { kind: "summary" }, + }, + lookup, + ); + if (!summary.ok) throw summary.error; + expect(summary.value.normalized_result).toMatchObject({ + kind: "summary", + parent_evidence_id: parent.evidence_id, + parent_operation: "analyze_javascript_application", + }); + expect(summary.value.normalized_result).not.toHaveProperty( + "summary.semantic_graph", + ); + const item = inspectAnalysisView( + { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { + kind: "item", + collection: "modules", + selector: { path: "renderer.js" }, + }, + }, + lookup, + ); + if (!item.ok) throw item.error; + expect(JSON.stringify(item.value.normalized_result)).toContain( + "export const secret = 1;", + ); +}); + +it("rejects stale retained IDs and unsupported parent operations", () => { + const missing = inspectAnalysisView( + { + source: { + kind: "retained-evidence", + evidence_id: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.evidence_id, + }, + view: { kind: "summary" }, + }, + () => undefined, + ); + expect(missing).toMatchObject({ + ok: false, + error: { + _tag: "EvidenceIntegrityError", + evidenceId: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.evidence_id, + }, + }); + const unsupported = createEvidence( + undefined, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "inspect_recorded_crash", + parameters: {}, + result: { path: "/artifacts/crash.core" }, + }, + ); + const rejected = inspectAnalysisView({ + source: { kind: "inline", evidence: unsupported }, + view: { kind: "summary" }, + }); + expect(rejected).toMatchObject({ + ok: false, + error: { _tag: "AnalysisUnsupportedTargetError" }, + }); + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: analysisViewLayoutEvidence() }, + view: { kind: "summary", approval: true }, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); +}); + +it("preserves an absent parent subject instead of guessing its format", () => { + const analysis = analysisViewJavaScriptAnalysisWithSource(); + const original = analysisViewJavaScriptEvidence(analysis); + const parent = createEvidence(undefined, original.provider, { + operation: original.operation, + parameters: {}, + result: original.normalized_result, + limitations: original.limitations, + }); + const result = inspectAnalysisView( + { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "summary" }, + }, + () => parent, + ); + if (!result.ok) throw result.error; + expect(result.value.subject).toBeNull(); + expect(result.value.locations).toEqual([ + { kind: "artifact-path", path: analysis.input_path }, + ]); +}); + +it("rejects contradictory subject and analysis artifact digests", () => { + const analysis = analysisViewJavaScriptAnalysisWithSource(); + const original = analysisViewJavaScriptEvidence(analysis); + const parent = createEvidence( + { + path: analysis.input_path, + format: analysis.format, + sha256: "d".repeat(64), + }, + original.provider, + { + operation: original.operation, + parameters: {}, + result: original.normalized_result, + }, + ); + const result = inspectAnalysisView( + { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "summary" }, + }, + () => parent, + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); +}); diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts new file mode 100644 index 00000000000..0d10d5fa8da --- /dev/null +++ b/src/application/analysisView/AnalysisViewService.ts @@ -0,0 +1,132 @@ +import { z } from "zod"; + +import { ANALYSIS_VIEW_PROVIDER } from "../InvestigationProviders.js"; +import type { EvidenceLookup } from "../EvidenceInputResolver.js"; +import { resolveEvidenceInput } from "../EvidenceInputResolver.js"; +import type { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js"; +import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; +import { + createEvidence, + parseEvidence, + type Evidence, +} from "../../domain/evidence.js"; +import { + INSPECT_ANALYSIS_VIEW_OPERATION, + inspectAnalysisViewInputSchema, + projectAnalysisView, + type AnalysisViewResult, + type InspectAnalysisViewInput, +} from "../../domain/analysisView/analysisView.js"; +import { jsonValueSchema } from "../../domain/jsonValue.js"; +import { err, ok, type Result } from "../../domain/result.js"; + +/** Authenticate a selected-view request and project already completed Evidence. */ +export const inspectAnalysisView = ( + rawInput: unknown, + lookup?: EvidenceLookup, +): Result => { + const parsed = inspectAnalysisViewInputSchema.safeParse(rawInput); + if (!parsed.success) + return err( + analysisInputErrorFromIssues( + INSPECT_ANALYSIS_VIEW_OPERATION, + parsed.error.issues, + rawInput, + { cause: parsed.error }, + ), + ); + return inspectAnalysisViewValidated(parsed.data, lookup); +}; + +/** Project a view from input already parsed by a trusted adapter. */ +export const inspectAnalysisViewValidated = ( + input: InspectAnalysisViewInput, + lookup?: EvidenceLookup, +): Result => { + const resolved = resolveAnalysisViewSource(input.source, lookup); + if (!resolved.ok) return resolved; + const parent = resolved.value; + const projected = projectAnalysisView( + { + evidenceId: parent.evidence_id, + operation: parent.operation, + normalizedResult: parent.normalized_result, + limitations: parent.limitations, + }, + input.view, + ); + if (!projected.ok) return projected; + if ( + parent.subject !== null && + parent.subject.digest.sha256 !== projected.value.artifact.sha256 + ) + return err( + new EvidenceIntegrityError( + `Parent subject SHA-256 ${parent.subject.digest.sha256} differs from analysis artifact SHA-256 ${projected.value.artifact.sha256}.`, + ), + ); + return ok(createAnalysisViewEvidence(input, parent, projected.value)); +}; + +const resolveAnalysisViewSource = ( + source: InspectAnalysisViewInput["source"], + lookup: EvidenceLookup | undefined, +): Result => { + if (source.kind === "retained-evidence") + return resolveEvidenceInput(source, lookup); + try { + return ok(parseEvidence(source.evidence)); + } catch (cause: unknown) { + if (cause instanceof z.ZodError) + return err( + analysisInputErrorFromIssues( + INSPECT_ANALYSIS_VIEW_OPERATION, + cause.issues, + source.evidence, + { cause }, + ), + ); + if (cause instanceof TypeError) + return err( + new EvidenceIntegrityError( + cause.message, + cause.message.length > 0 ? { userMessage: cause.message } : undefined, + ), + ); + throw cause; + } +}; + +const createAnalysisViewEvidence = ( + input: InspectAnalysisViewInput, + parent: Evidence, + result: AnalysisViewResult, +): Evidence => { + const target = + parent.subject === null + ? undefined + : { + path: parent.subject.local_path, + sha256: parent.subject.digest.sha256, + format: parent.subject.format, + ...(parent.subject.architecture === null + ? {} + : { architecture: parent.subject.architecture }), + }; + return createEvidence(target, ANALYSIS_VIEW_PROVIDER, { + predicateType: "rea.analysis-view", + operation: INSPECT_ANALYSIS_VIEW_OPERATION, + parameters: { + parent_evidence_id: parent.evidence_id, + view: jsonValueSchema.parse(input.view), + }, + result: jsonValueSchema.parse(result), + rawResult: null, + confidence: "derived", + authority: parent.authority, + limitations: result.limitations, + locations: [{ kind: "artifact-path", path: result.artifact.path }], + evidenceLinks: [parent.evidence_id], + }); +}; diff --git a/src/cli.ts b/src/cli.ts index 8b3d3000850..eb7d4e83d1f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1,4 +1,5 @@ import { registerEvmCommands } from "./cli/evmCommands.js"; +import { registerAnalysisViewCommands } from "./cli/analysisViewCommands.js"; import { registerBinaryDiagnosticsCommands } from "./cli/binaryDiagnosticsCommands.js"; import { Cli } from "incur"; @@ -65,6 +66,7 @@ export const createCli = ( registerAndroidCommands(cli, logger, environment); registerFirmwareCommands(cli, logger, environment); registerBinaryDiagnosticsCommands(cli, logger, environment); + registerAnalysisViewCommands(cli, logger); registerEvidenceCommands(cli, logger); registerProcessCommands(cli, logger, environment); registerBrowserCommands(cli, logger); diff --git a/src/cli/analysisViewCommands.ts b/src/cli/analysisViewCommands.ts new file mode 100644 index 00000000000..480ef67239d --- /dev/null +++ b/src/cli/analysisViewCommands.ts @@ -0,0 +1,53 @@ +import { z } from "incur"; + +import { inspectAnalysisViewValidated } from "../application/analysisView/AnalysisViewService.js"; +import { CLI_COMMANDS } from "../cliCommandNames.js"; +import { parseCliJsonInput } from "../cliJsonInput.js"; +import { logCliCommand } from "../cliLogging.js"; +import { inspectAnalysisViewInputSchema } from "../domain/analysisView/analysisView.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import { jsonValueSchema } from "../domain/jsonValue.js"; +import type { Logger } from "../logger.js"; +import type { CliInstance } from "./types.js"; + +/** Project selected views of completed analysis Evidence through the CLI. */ +export const registerAnalysisViewCommands = ( + cli: CliInstance, + logger: Logger, +): void => { + cli.command(CLI_COMMANDS.inspectAnalysisView, { + description: + "Project a selected view of completed layout or JavaScript application Evidence JSON", + args: z.object({ + inputJson: z.string().describe("Inline workflow JSON or JSON file path"), + }), + run: ({ args }) => + logCliCommand(logger, CLI_COMMANDS.inspectAnalysisView, async () => { + const input = await parseCliJsonInput( + args.inputJson, + CLI_COMMANDS.inspectAnalysisView, + ); + if (!input.ok) return input.error; + const parsed = inspectAnalysisViewInputSchema.safeParse(input.value); + if (!parsed.success) + return { + error: "Application workflow failed", + ...projectAnalysisError( + analysisInputErrorFromIssues( + CLI_COMMANDS.inspectAnalysisView, + parsed.error.issues, + input.value, + ), + ), + }; + const result = inspectAnalysisViewValidated(parsed.data); + return result.ok + ? jsonValueSchema.parse(result.value) + : { + error: "Application workflow failed", + ...projectAnalysisError(result.error), + }; + }), + }); +}; diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index 2cf700e5803..dd0329eb0b7 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -41,6 +41,7 @@ export const CLI_COMMANDS = Object.freeze({ inspectManagedArtifact: "inspect-managed-artifact", inspectRecordedCrash: "inspect-recorded-crash", inspectBinaryLayout: "inspect-binary-layout", + inspectAnalysisView: "inspect-analysis-view", inspectFirmwareRegions: "inspect-firmware-regions", extractFirmware: "extract-firmware", inspectAndroidPackage: "inspect-android-package", diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts new file mode 100644 index 00000000000..1378722b536 --- /dev/null +++ b/src/contracts/analysisViewToolContracts.ts @@ -0,0 +1,63 @@ +import { + analysisViewResultSchema, + inspectAnalysisViewInputSchema, +} from "../domain/analysisView/analysisView.js"; +import type { ToolContract } from "./toolContractTypes.js"; +import { toolContractMetadata } from "./toolEffects.js"; +import { evidenceResultOf } from "./toolOutputSchemas.js"; + +const RETAINED_EXAMPLE_ID = `ev_${"a".repeat(64)}`; + +/** Selected views of already completed layout and JavaScript analysis Evidence. */ +export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ + { + name: "inspect_analysis_view", + ...toolContractMetadata("inspect_analysis_view"), + kind: "application", + description: + "Inspect a selected view of completed binary-layout or JavaScript application Evidence without repeating analysis. Use an exact same-session retained evidence_id or portable inline Evidence. Select a summary, layout mitigations/linkage facet, one section/symbol/module, or a stable page with a positive limit. Module pages contain JavaScript assets, bundled modules and source modules; use node_id when paths are ambiguous or unavailable. Returns selected facts inline with artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the retained Evidence when needed.", + inputSchema: inspectAnalysisViewInputSchema, + outputSchema: evidenceResultOf(analysisViewResultSchema), + examples: [ + { + title: "Summarize retained JavaScript application Evidence", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { kind: "summary" }, + }, + }, + { + title: "Read one ELF section from retained layout Evidence", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { + kind: "item", + collection: "sections", + selector: { name: ".text" }, + }, + }, + }, + { + title: "Page module identities from retained application Evidence", + input: { + source: { + kind: "retained-evidence", + evidence_id: RETAINED_EXAMPLE_ID, + }, + view: { + kind: "page", + collection: "modules", + offset: 0, + limit: 32, + }, + }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/contractSnapshot.test.ts b/src/contracts/contractSnapshot.test.ts index f0e7a6ce94f..2861e966e47 100644 --- a/src/contracts/contractSnapshot.test.ts +++ b/src/contracts/contractSnapshot.test.ts @@ -1,4 +1,5 @@ import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; +import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; import { describe, expect, it } from "vitest"; import { z } from "zod"; @@ -72,6 +73,7 @@ describe("tool contract surface", () => { ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...BINARY_DIAGNOSTICS_TOOL_CONTRACTS, + ...ANALYSIS_VIEW_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...EVM_TOOL_CONTRACTS, ...ANDROID_TOOL_CONTRACTS, diff --git a/src/contracts/promptContracts.ts b/src/contracts/promptContracts.ts index 534515a7dbe..833ac172ae4 100644 --- a/src/contracts/promptContracts.ts +++ b/src/contracts/promptContracts.ts @@ -131,9 +131,9 @@ export const PROMPT_CONTRACTS = [ "Search relevant strings or procedures and analyze the matching procedures; pass names or addresses directly when already known.", }, { - tools: ["trace_application_feature"], + tools: ["inspect_analysis_view", "trace_application_feature"], instruction: - "When application-graph Evidence is relevant, trace a route, API, channel, module, string, or native export across layers.", + "When inspect_binary_layout or analyze_javascript_application Evidence is already retained, call inspect_analysis_view with that evidence_id for a summary, one section/symbol/module, or a stable page. Do not re-run the producer. Use trace_application_feature once a route, API, channel, module, string, or native export seed is known.", }, { tools: [ diff --git a/src/contracts/toolContracts.test.ts b/src/contracts/toolContracts.test.ts index 2c4431eba1a..b895cd78209 100644 --- a/src/contracts/toolContracts.test.ts +++ b/src/contracts/toolContracts.test.ts @@ -1,4 +1,5 @@ import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; +import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; import { describe, expect, it } from "vitest"; @@ -26,6 +27,7 @@ const GROUPS = { enhanced: ENHANCED_TOOL_CONTRACTS, native: NATIVE_TOOL_CONTRACTS, binary_diagnostics: BINARY_DIAGNOSTICS_TOOL_CONTRACTS, + analysis_view: ANALYSIS_VIEW_TOOL_CONTRACTS, artifact: ARTIFACT_TOOL_CONTRACTS, evm: EVM_TOOL_CONTRACTS, managed: MANAGED_TOOL_CONTRACTS, diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 304ef41d5fe..82a6c513dcd 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -1,4 +1,5 @@ import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; +import { ANALYSIS_VIEW_TOOL_CONTRACTS } from "./analysisViewToolContracts.js"; import { BINARY_DIAGNOSTICS_TOOL_CONTRACTS } from "./native/binaryDiagnosticsToolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "./native/nativeToolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "./artifactToolContracts.js"; @@ -26,6 +27,7 @@ export const TOOL_CONTRACTS = [ ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...BINARY_DIAGNOSTICS_TOOL_CONTRACTS, + ...ANALYSIS_VIEW_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...EVM_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index dbe2b3bc9af..35a51010bac 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -80,6 +80,7 @@ export const TOOL_EFFECTS: Readonly> = { writesFilesystem: true, launchesProcess: true, }), + inspect_analysis_view: evidence, inspect_firmware_regions: effects({ mutatesSession: true, launchesProcess: true, diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts new file mode 100644 index 00000000000..554209a79bd --- /dev/null +++ b/src/domain/analysisView/analysisView.test.ts @@ -0,0 +1,526 @@ +import { expect, it } from "vitest"; + +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; +import { + analysisViewBindJavaScriptGraphs, + analysisViewJavaScriptAnalysis, + analysisViewJavaScriptAnalysisWithSource, + analysisViewLayoutEvidence, + analysisViewLayoutFixture, +} from "../../../tests/fixtures/analysisView.js"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, + createImmutableJavaScriptApplicationGraphSteps, +} from "../javascript/javascriptApplicationGraph.js"; +import { + createJavaScriptSemanticGraph, + createImmutableJavaScriptSemanticGraphSteps, +} from "../javascript/javascriptSemanticGraph.js"; +import { + completeWithinViewCoverage, + inspectAnalysisViewInputSchema, + pageViewCoverage, + projectAnalysisView, + type AnalysisViewParent, +} from "./analysisView.js"; +import { javascriptModulePath } from "./javascriptApplicationView.js"; + +const layoutParent = (): AnalysisViewParent => { + const evidence = analysisViewLayoutEvidence(); + return { + evidenceId: evidence.evidence_id, + operation: evidence.operation, + normalizedResult: evidence.normalized_result, + limitations: evidence.limitations, + }; +}; + +const javascriptParent = ( + analysis = analysisViewJavaScriptAnalysis(), +): AnalysisViewParent => ({ + evidenceId: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.evidence_id, + operation: "analyze_javascript_application", + normalizedResult: analysis, + limitations: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.limitations, +}); + +it("accepts caller-selected page sizes and rejects malformed bounds", () => { + const input = { + source: { kind: "retained-evidence", evidence_id: `ev_${"a".repeat(64)}` }, + view: { kind: "page", collection: "sections", offset: 0, limit: 500 }, + }; + expect(inspectAnalysisViewInputSchema.safeParse(input).success).toBe(true); + for (const limit of [0, -1, 0.5, Infinity, Number.MAX_SAFE_INTEGER + 1]) + expect( + inspectAnalysisViewInputSchema.safeParse({ + ...input, + view: { ...input.view, limit }, + }).success, + ).toBe(false); +}); + +it("projects layout summary, facet, item, and stable pages", () => { + const parent = layoutParent(); + const layout = analysisViewLayoutFixture(); + const summary = projectAnalysisView(parent, { kind: "summary" }); + if (!summary.ok) throw summary.error; + expect(summary.value).toMatchObject({ + kind: "summary", + parent_operation: "inspect_binary_layout", + summary: { + format: "elf", + counts: { sections: 3, symbols: 3, segments: 0, relocations: 0 }, + }, + coverage: completeWithinViewCoverage(6, 6), + }); + const mitigations = projectAnalysisView(parent, { + kind: "facet", + facet: "mitigations", + }); + if (!mitigations.ok) throw mitigations.error; + if (mitigations.value.kind !== "facet") + throw new Error("expected facet view"); + expect(mitigations.value.facet).toEqual(layout.mitigations); + const data = projectAnalysisView(parent, { + kind: "item", + collection: "sections", + selector: { name: ".data" }, + }); + if (!data.ok) throw data.error; + if (data.value.kind !== "item") throw new Error("expected item view"); + expect(data.value.item).toMatchObject({ + index: 2, + name: { display: ".data" }, + }); + const indexed = projectAnalysisView(parent, { + kind: "item", + collection: "sections", + selector: { index: 0 }, + }); + if (!indexed.ok) throw indexed.error; + expect(indexed.value.view_digest).not.toBe(data.value.view_digest); + expect(indexed.value.view_digest).not.toBe(parent.evidenceId.slice(3)); + const page = projectAnalysisView(parent, { + kind: "page", + collection: "sections", + offset: 0, + limit: 2, + }); + if (!page.ok) throw page.error; + expect(page.value).toMatchObject({ + kind: "page", + coverage: { + status: "page", + examined: 2, + total: 3, + next_offset: 2, + exhausted: false, + }, + }); + if (page.value.kind !== "page") throw new Error("expected page view"); + expect(page.value.items).toHaveLength(2); + const exhausted = projectAnalysisView(parent, { + kind: "page", + collection: "symbols", + offset: 3, + limit: 8, + }); + if (!exhausted.ok) throw exhausted.error; + expect(exhausted.value).toMatchObject({ + items: [], + coverage: pageViewCoverage(3, 0, 3), + }); +}); + +it("rejects ambiguous layout names with candidate indexes", () => { + const parent = layoutParent(); + const result = projectAnalysisView(parent, { + kind: "item", + collection: "sections", + selector: { name: ".text" }, + }); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: [ + { + path: ["view", "selector", "name"], + expected: [{ index: 0 }, { index: 1 }], + }, + ], + }, + }); +}); + +it("projects JavaScript summary and preserves selected module observations", () => { + const analysis = analysisViewJavaScriptAnalysisWithSource(); + const parent = javascriptParent(analysis); + const summary = projectAnalysisView(parent, { kind: "summary" }); + if (!summary.ok) throw summary.error; + if (summary.value.kind !== "summary") throw new Error("expected summary"); + expect(summary.value.summary).toMatchObject({ + format: "directory", + limitation_count: 0, + }); + expect(JSON.stringify(summary.value)).not.toMatch(/"graph"/); + expect(summary.value.summary).not.toHaveProperty("semantic_graph"); + const item = projectAnalysisView(parent, { + kind: "item", + collection: "modules", + selector: { path: "renderer.js" }, + }); + if (!item.ok) throw item.error; + const moduleNode = analysis.graph.nodes[0]; + if (moduleNode === undefined) throw new Error("missing module"); + expect(javascriptModulePath(moduleNode)).toBe("renderer.js"); + if (item.value.kind !== "item") throw new Error("expected item view"); + expect(item.value.item).toMatchObject({ + path: "renderer.js", + kind: "javascript-asset", + }); + expect(JSON.stringify(item.value.item)).toContain("export const secret = 1;"); + expect(JSON.stringify(item.value.item)).toContain("should not leak"); + const page = projectAnalysisView(parent, { + kind: "page", + collection: "modules", + offset: 0, + limit: 8, + }); + if (!page.ok) throw page.error; + if (page.value.kind !== "page") throw new Error("expected page view"); + expect(page.value.items).toEqual([ + { + node_id: analysis.graph.nodes[0]?.node_id, + kind: "javascript-asset", + path: "renderer.js", + }, + ]); + expect(page.value.coverage).toMatchObject({ + status: "page", + examined: 1, + total: 1, + next_offset: null, + exhausted: true, + }); +}); + +it("rejects ambiguous JavaScript module paths with candidate node ids", () => { + const analysis = analysisViewJavaScriptAnalysis(); + const first = analysis.graph.nodes[0]; + if (first === undefined) throw new Error("missing module"); + const duplicate = createJavaScriptApplicationNode({ + kind: first.kind, + identity: { + strategy: "content-digest", + stability: "global-exact", + sha256: "c".repeat(64), + }, + observations: first.observations.map((observation) => ({ + label: observation.label, + properties: observation.properties, + evidence: { + ...observation.evidence, + artifact: { + ...observation.evidence.artifact, + sha256: "c".repeat(64), + artifact_id: `art_${"c".repeat(64)}`, + }, + }, + })), + }); + const graph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [first.node_id, duplicate.node_id], + nodes: [first, duplicate], + edges: [], + coverage: analysis.graph.coverage, + limitations: analysis.graph.limitations, + }); + const parent = javascriptParent( + analysisViewBindJavaScriptGraphs(analysis, graph), + ); + const result = projectAnalysisView(parent, { + kind: "item", + collection: "modules", + selector: { path: "renderer.js" }, + }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected ambiguous path"); + expect(result.error).toMatchObject({ + _tag: "AnalysisInputError", + issues: [ + { + path: ["view", "selector", "path"], + expected: expect.arrayContaining([ + { node_id: first.node_id }, + { node_id: duplicate.node_id }, + ]), + }, + ], + }); +}); + +it("pages actual modules, including unknown and empty paths, without unrelated roles", () => { + const analysis = analysisViewJavaScriptAnalysis(); + const original = analysis.graph.nodes[0]; + if (original === undefined) throw new Error("missing module"); + const nodes = [ + original, + ...( + [ + "asar-entry", + "electron-renderer", + "javascript-module", + "source-module", + ] as const + ).map((kind) => + createJavaScriptApplicationNode({ + kind, + identity: + kind === "source-module" + ? { + strategy: "source-map-original", + stability: "source-map-exact", + original_source: "", + source_map_sha256: "c".repeat(64), + source_sha256: null, + } + : kind === "javascript-module" + ? { + strategy: "artifact-local-key", + stability: "artifact-version", + artifact_sha256: "c".repeat(64), + namespace: kind, + key: "module", + } + : original.identity, + observations: original.observations.map((observation) => ({ + label: observation.label, + properties: {}, + evidence: + kind === "javascript-module" || kind === "source-module" + ? { + ...observation.evidence, + artifact: { + ...observation.evidence.artifact, + sha256: "c".repeat(64), + artifact_id: `art_${"c".repeat(64)}`, + }, + ...(kind === "javascript-module" + ? { + state: "unknown", + confidence: "unknown", + limitations: ["Source location unavailable"], + location: { + available: false, + reason: "unknown", + detail: "Source location unavailable", + }, + } + : {}), + } + : observation.evidence, + })), + }), + ), + ]; + const graph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: nodes.map((node) => node.node_id), + nodes, + edges: [], + coverage: analysis.graph.coverage, + limitations: [], + }); + const parent = javascriptParent( + analysisViewBindJavaScriptGraphs(analysis, graph), + ); + const page = projectAnalysisView(parent, { + kind: "page", + collection: "modules", + offset: 0, + limit: 1000, + }); + if (!page.ok) throw page.error; + expect(page.value).toMatchObject({ + coverage: { total: 3, examined: 3, exhausted: true }, + }); + if (page.value.kind !== "page") throw new Error("expected page"); + expect(page.value.items).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "javascript-module", path: null }), + expect.objectContaining({ kind: "source-module", path: "" }), + ]), + ); + const emptyPath = projectAnalysisView(parent, { + kind: "item", + collection: "modules", + selector: { path: "" }, + }); + expect(emptyPath.ok).toBe(true); + const role = nodes.find((node) => node.kind === "electron-renderer"); + if (role === undefined) throw new Error("missing role"); + expect( + projectAnalysisView(parent, { + kind: "item", + collection: "modules", + selector: { node_id: role.node_id }, + }).ok, + ).toBe(false); +}); + +const complete = (steps: Generator): Value => { + let step = steps.next(); + while (!step.done) step = steps.next(); + return step.value; +}; + +it("reuses authenticated graphs while rejecting malformed metadata and imported companions", () => { + const analysis = analysisViewJavaScriptAnalysis(); + const { graph_id: _applicationId, ...application } = analysis.graph; + const graph = complete( + createImmutableJavaScriptApplicationGraphSteps(application), + ); + const { graph_id: _semanticId, ...semantic } = analysis.semantic_graph; + const semanticGraph = complete( + createImmutableJavaScriptSemanticGraphSteps({ + ...semantic, + application_graph_id: graph.graph_id, + }), + ); + const owned = { ...analysis, graph, semantic_graph: semanticGraph }; + expect( + projectAnalysisView(javascriptParent(owned), { kind: "summary" }).ok, + ).toBe(true); + for (const malformed of [ + { ...owned, root_artifact_sha256: "d".repeat(64) }, + { + ...owned, + statistics: { ...owned.statistics, parsed_javascript_files: -1 }, + }, + { + ...owned, + semantic_graph: Object.freeze({ + ...semanticGraph, + application_graph_id: "jag_" + "d".repeat(64), + }), + }, + ]) { + const result = projectAnalysisView( + { ...javascriptParent(owned), normalizedResult: malformed }, + { kind: "summary" }, + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisInputError", + issues: expect.arrayContaining([ + expect.objectContaining({ + path: expect.arrayContaining(["source", "normalized_result"]), + }), + ]), + }, + }); + } +}); + +it("retains semantic coverage and limitations in a summary", () => { + const analysis = analysisViewJavaScriptAnalysis(); + const { graph_id: _graphId, ...semantic } = analysis.semantic_graph; + const semanticGraph = createJavaScriptSemanticGraph({ + ...semantic, + coverage: { + ...semantic.coverage, + status: "partial", + truncated: true, + omitted_nodes: null, + omitted_relations: null, + limits: [ + { name: "semantic_graph_node_ceiling", value: 100000, unit: "items" }, + ], + }, + limitations: ["Some semantic nodes could not be retained."], + }); + const result = projectAnalysisView( + javascriptParent({ ...analysis, semantic_graph: semanticGraph }), + { kind: "summary" }, + ); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + summary: { coverage: { semantic: { status: "partial", truncated: true } } }, + limitations: ["Some semantic nodes could not be retained."], + unknowns: expect.arrayContaining([ + "semantic_graph.coverage.status is partial", + ]), + }); +}); + +it("does not match unavailable layout names using a display placeholder", () => { + const layout = analysisViewLayoutFixture(); + const section = layout.sections[2]; + if (section === undefined) throw new Error("missing section"); + const result = projectAnalysisView( + { + ...layoutParent(), + normalizedResult: { + ...layout, + sections: [ + ...layout.sections.slice(0, 2), + { + ...section, + name: { + ...section.name, + display: "", + unknown_reason: "Name bytes unavailable", + bytes_base64: null, + location: null, + }, + }, + ], + }, + }, + { + kind: "item", + collection: "sections", + selector: { name: "" }, + }, + ); + expect(result.ok).toBe(false); +}); + +it("rejects unsupported parent operations and incompatible collections", () => { + const unsupported = projectAnalysisView( + { + evidenceId: `ev_${"d".repeat(64)}`, + operation: "inspect_recorded_crash", + normalizedResult: { path: "/artifacts/crash.core" }, + limitations: [], + }, + { kind: "summary" }, + ); + expect(unsupported).toMatchObject({ + ok: false, + error: { _tag: "AnalysisUnsupportedTargetError" }, + }); + const modulesOnLayout = projectAnalysisView(layoutParent(), { + kind: "page", + collection: "modules", + offset: 0, + limit: 1, + }); + expect(modulesOnLayout).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + const sectionsOnJs = projectAnalysisView(javascriptParent(), { + kind: "item", + collection: "sections", + selector: { name: ".text" }, + }); + expect(sectionsOnJs).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); +}); diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts new file mode 100644 index 00000000000..de778f89491 --- /dev/null +++ b/src/domain/analysisView/analysisView.ts @@ -0,0 +1,341 @@ +import { z } from "zod"; + +import { + AnalysisInputError, + type AnalysisInputIssue, + AnalysisUnsupportedTargetError, +} from "../analysisErrorCore.js"; +import { digestCanonicalValue } from "../canonicalDigest.js"; +import { digestSchema, prefixedDigestSchema } from "../digests.js"; +import { evidenceSchema } from "../evidence.js"; +import { jsonObjectSchema, jsonValueSchema } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { binaryLayoutSchema } from "../native/binaryLayout.js"; +import { + parseOwnedJavaScriptApplicationAnalysisSteps, + type JavaScriptApplicationAnalysisResult, +} from "../javascript/javascriptApplicationAnalysis.js"; +import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; +import { projectBinaryLayoutView } from "./binaryLayoutView.js"; +import { projectJavaScriptApplicationView } from "./javascriptApplicationView.js"; + +/** Public name of the selected-view workflow. */ +export const INSPECT_ANALYSIS_VIEW_OPERATION = "inspect_analysis_view" as const; + +const summaryViewSchema = z.strictObject({ kind: z.literal("summary") }); +const facetViewSchema = z.strictObject({ + kind: z.literal("facet"), + facet: z.enum(["mitigations", "linkage"]), +}); +const itemSelectorSchema = z.union([ + z.strictObject({ index: z.number().int().nonnegative() }), + z.strictObject({ name: z.string() }), + z.strictObject({ + path: z + .string() + .describe( + "Exact observed module path or original source-map reference from a module page; an empty reference is valid.", + ), + }), + z.strictObject({ node_id: prefixedDigestSchema("jag_node") }), +]); +const itemViewSchema = z + .strictObject({ + kind: z.literal("item"), + collection: z.enum(["sections", "symbols", "modules"]), + selector: itemSelectorSchema, + }) + .superRefine((value, context) => { + const selector = value.selector; + if (value.collection === "modules") { + if (!("path" in selector) && !("node_id" in selector)) + context.addIssue({ + code: "custom", + path: ["selector"], + message: + "Module items are selected by exact path or node_id, not index or name.", + }); + return; + } + if (!("index" in selector) && !("name" in selector)) + context.addIssue({ + code: "custom", + path: ["selector"], + message: + "Section and symbol items are selected by index or exact name, not path or node_id.", + }); + }); +const pageViewSchema = z.strictObject({ + kind: z.literal("page"), + collection: z.enum(["sections", "symbols", "modules"]), + offset: z.number().int().nonnegative(), + limit: z.number().int().positive(), +}); + +/** Caller-selected projection of already completed analysis Evidence. */ +export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ + summaryViewSchema, + facetViewSchema, + itemViewSchema, + pageViewSchema, +]); + +/** Inline Evidence or an exact same-session retained reference. */ +export const analysisViewSourceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("retained-evidence"), + evidence_id: prefixedDigestSchema("ev"), + }), + z.strictObject({ + kind: z.literal("inline"), + evidence: evidenceSchema, + }), +]); + +/** Public inspect_analysis_view input. */ +export const inspectAnalysisViewInputSchema = z.strictObject({ + source: analysisViewSourceSchema, + view: analysisViewRequestSchema, +}); + +const coverageSchema = z.strictObject({ + status: z.enum(["complete-within-view", "page", "empty"]), + examined: z.number().int().nonnegative(), + total: z.number().int().nonnegative(), + next_offset: z.number().int().nonnegative().nullable(), + exhausted: z.boolean(), +}); + +const artifactSchema = z.strictObject({ + path: z.string().min(1), + sha256: digestSchema, +}); + +const layoutSummarySchema = z.strictObject({ + bytes: z.number().int().nonnegative(), + format: z.literal("elf"), + architecture: z.strictObject({ + machine: z.literal("EM_X86_64"), + bits: z.literal(64), + byte_order: z.literal("little"), + }), + image_type: z.enum(["ET_EXEC", "ET_DYN", "ET_REL"]), + entry: z.strictObject({ + reported_value: z.string().min(1), + meaning: z.enum(["linked-virtual-address", "not-applicable", "absent"]), + execution_status: z.literal("unknown"), + }), + counts: z.strictObject({ + sections: z.number().int().nonnegative(), + segments: z.number().int().nonnegative(), + symbols: z.number().int().nonnegative(), + relocations: z.number().int().nonnegative(), + }), + limitation_count: z.number().int().nonnegative(), +}); + +const javascriptSummarySchema = z.strictObject({ + input_path: z.string().min(1), + format: z.enum(["asar", "directory"]), + root_artifact_sha256: digestSchema, + statistics: jsonObjectSchema, + electron: jsonObjectSchema, + coverage: jsonObjectSchema, + limitation_count: z.number().int().nonnegative(), +}); + +const viewResultBase = { + parent_evidence_id: prefixedDigestSchema("ev"), + parent_operation: z.enum([ + "inspect_binary_layout", + "analyze_javascript_application", + ]), + parent_digest: digestSchema, + view_digest: digestSchema, + artifact: artifactSchema, + coverage: coverageSchema, + limitations: z.array(z.string()), + unknowns: z.array(z.string()), +} as const; + +/** Projected facts for one selected view of retained analysis Evidence. */ +export const analysisViewResultSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("summary"), + view: summaryViewSchema, + summary: z.union([layoutSummarySchema, javascriptSummarySchema]), + ...viewResultBase, + }), + z.strictObject({ + kind: z.literal("facet"), + view: facetViewSchema, + facet: jsonValueSchema, + ...viewResultBase, + }), + z.strictObject({ + kind: z.literal("item"), + view: itemViewSchema, + item: jsonValueSchema, + ...viewResultBase, + }), + z.strictObject({ + kind: z.literal("page"), + view: pageViewSchema, + items: z.array(jsonValueSchema), + ...viewResultBase, + }), +]); + +export type AnalysisViewRequest = z.output; +export type InspectAnalysisViewInput = z.output< + typeof inspectAnalysisViewInputSchema +>; +export type AnalysisViewResult = z.output; +export type AnalysisViewCoverage = z.output; + +/** Parent record already resolved from inline Evidence or a session reference. */ +export interface AnalysisViewParent { + readonly evidenceId: string; + readonly operation: string; + readonly normalizedResult: unknown; + readonly limitations: readonly string[]; +} + +export { + completeWithinViewCoverage, + pageViewCoverage, +} from "./analysisViewCoverage.js"; + +/** Construct a typed input failure for one selected-view constraint. */ +export const analysisViewInputError = ( + issues: readonly AnalysisInputIssue[], +): AnalysisInputError => + new AnalysisInputError(INSPECT_ANALYSIS_VIEW_OPERATION, undefined, issues); + +/** SHA-256 of the canonical projected JSON, excluding this digest field. */ +export const analysisViewDigest = ( + projection: Omit, +): string => digestCanonicalValue(projection, "Analysis view"); + +/** View payload before the content digest is attached. */ +export type UnsignedAnalysisView = + | Omit< + Extract, + "view_digest" + > + | Omit, "view_digest"> + | Omit, "view_digest"> + | Omit, "view_digest">; + +/** Attach the content digest that must change when projected bytes change. */ +export const sealAnalysisView = ( + projection: UnsignedAnalysisView, +): AnalysisViewResult => + analysisViewResultSchema.parse({ + ...projection, + view_digest: analysisViewDigest(projection), + }); + +const parentDigest = (evidenceId: string): string => evidenceId.slice(3); + +const parseJavaScriptParent = ( + input: unknown, +): Result => { + try { + // Only the existing realm-private graph proofs permit reuse. Imported or + // merely frozen graphs still receive full canonical validation. + const steps = parseOwnedJavaScriptApplicationAnalysisSteps(input); + let step = steps.next(); + while (!step.done) step = steps.next(); + return ok(step.value); + } catch (cause: unknown) { + if (!(cause instanceof z.ZodError)) throw cause; + return err( + analysisInputErrorFromIssues( + INSPECT_ANALYSIS_VIEW_OPERATION, + cause.issues.map((issue) => ({ + ...issue, + path: ["source", "normalized_result", ...issue.path], + })), + { source: { normalized_result: input } }, + { cause }, + ), + ); + } +}; + +const unsupportedParent = ( + parent: AnalysisViewParent, +): AnalysisUnsupportedTargetError => + new AnalysisUnsupportedTargetError( + INSPECT_ANALYSIS_VIEW_OPERATION, + parentArtifactPath(parent), + `Parent Evidence operation ${JSON.stringify(parent.operation)} is not inspect_binary_layout or analyze_javascript_application.`, + ); + +const parentArtifactPath = (parent: AnalysisViewParent): string => { + if ( + typeof parent.normalizedResult === "object" && + parent.normalizedResult !== null && + "artifact" in parent.normalizedResult && + typeof parent.normalizedResult.artifact === "object" && + parent.normalizedResult.artifact !== null && + "path" in parent.normalizedResult.artifact && + typeof parent.normalizedResult.artifact.path === "string" + ) + return parent.normalizedResult.artifact.path; + if ( + typeof parent.normalizedResult === "object" && + parent.normalizedResult !== null && + "input_path" in parent.normalizedResult && + typeof parent.normalizedResult.input_path === "string" + ) + return parent.normalizedResult.input_path; + return parent.evidenceId; +}; + +/** Project one selected view from authenticated parent analysis Evidence. */ +export const projectAnalysisView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, +): Result => { + const digest = parentDigest(parent.evidenceId); + if (!digestSchema.safeParse(digest).success) + return err( + analysisViewInputError([ + { + path: ["source"], + reason: "invalid_format", + message: "Parent Evidence ID is not a digest-backed identifier.", + }, + ]), + ); + if (parent.operation === "inspect_binary_layout") { + const layout = binaryLayoutSchema.safeParse(parent.normalizedResult); + if (!layout.success) + return err( + analysisViewInputError([ + { + path: ["source"], + reason: "invalid_value", + message: + "Parent Evidence result does not match inspect_binary_layout.", + }, + ]), + ); + const projected = projectBinaryLayoutView(parent, layout.data, view); + return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; + } + if (parent.operation === "analyze_javascript_application") { + const analysis = parseJavaScriptParent(parent.normalizedResult); + if (!analysis.ok) return analysis; + const projected = projectJavaScriptApplicationView( + parent, + analysis.value, + view, + ); + return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; + } + return err(unsupportedParent(parent)); +}; diff --git a/src/domain/analysisView/analysisViewCoverage.ts b/src/domain/analysisView/analysisViewCoverage.ts new file mode 100644 index 00000000000..7f592a8827e --- /dev/null +++ b/src/domain/analysisView/analysisViewCoverage.ts @@ -0,0 +1,29 @@ +import type { AnalysisViewCoverage } from "./analysisView.js"; + +/** Coverage for a summary, facet, or single selected object. */ +export const completeWithinViewCoverage = ( + examined: number, + total: number, +): AnalysisViewCoverage => ({ + status: "complete-within-view", + examined, + total, + next_offset: null, + exhausted: true, +}); + +/** Coverage for a stable page, including an offset past the collection. */ +export const pageViewCoverage = ( + offset: number, + examined: number, + total: number, +): AnalysisViewCoverage => { + const exhausted = offset >= total || offset + examined >= total; + return { + status: examined === 0 ? "empty" : "page", + examined, + total, + next_offset: exhausted ? null : offset + examined, + exhausted, + }; +}; diff --git a/src/domain/analysisView/binaryLayoutView.ts b/src/domain/analysisView/binaryLayoutView.ts new file mode 100644 index 00000000000..5a52904a859 --- /dev/null +++ b/src/domain/analysisView/binaryLayoutView.ts @@ -0,0 +1,239 @@ +import { + AnalysisInputError, + type AnalysisInputIssue, +} from "../analysisErrorCore.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { uniqueSorted } from "../canonicalOrdering.js"; +import type { BinaryLayout } from "../native/binaryLayout.js"; +import { err, ok, type Result } from "../result.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; +import { + completeWithinViewCoverage as completeCoverage, + pageViewCoverage as pageCoverage, +} from "./analysisViewCoverage.js"; + +const OPERATION = "inspect_analysis_view"; + +type LayoutName = BinaryLayout["sections"][number]["name"]; + +const inputError = ( + issues: readonly AnalysisInputIssue[], +): AnalysisInputError => new AnalysisInputError(OPERATION, undefined, issues); + +const layoutUnknowns = (layout: BinaryLayout): readonly string[] => { + const unknowns = [ + "runtime_load_base is unknown", + "entry_point.execution_status is unknown", + "relocation_inventory_completeness is unknown", + "linkage.convenience_maps_completeness is unknown", + "linkage.runtime_library_paths are unknown", + ]; + if (layout.mitigations.nx_indicator === null) + unknowns.push("mitigations.nx_indicator is unknown"); + if (layout.mitigations.relro === null) + unknowns.push("mitigations.relro is unknown"); + return unknowns; +}; + +const displayName = (name: LayoutName): string => name.display; + +const namedMatches = ( + rows: readonly Row[], + name: string, +): readonly { readonly index: number; readonly row: Row }[] => + rows.flatMap((row, index) => + row.name.unknown_reason === null && displayName(row.name) === name + ? [{ index, row }] + : [], + ); + +const selectNamed = ( + collection: "sections" | "symbols", + rows: readonly Row[], + name: string, +): Result => { + const matches = namedMatches(rows, name); + if (matches.length === 1) { + const selected = matches[0]; + if (selected !== undefined) return ok(selected.row); + } + if (matches.length === 0) + return err( + inputError([ + { + path: ["view", "selector", "name"], + reason: "invalid_value", + message: `No ${collection} named ${JSON.stringify(name)}. Collection has ${String(rows.length)} entries; page it to inspect exact indexes and names.`, + }, + ]), + ); + return err( + inputError([ + { + path: ["view", "selector", "name"], + reason: "invalid_value", + message: `Multiple ${collection} named ${JSON.stringify(name)}; select one by index.`, + expected: matches.map(({ index }) => ({ index })), + }, + ]), + ); +}; + +const selectIndexed = ( + collection: "sections" | "symbols", + rows: readonly Row[], + index: number, +): Result => { + const row = rows[index]; + if (row !== undefined) return ok(row); + return err( + inputError([ + { + path: ["view", "selector", "index"], + reason: "out_of_range", + message: `${collection} index ${String(index)} is outside 0..${String(Math.max(0, rows.length - 1))} (${String(rows.length)} entries).`, + minimum: 0, + maximum: Math.max(0, rows.length - 1), + }, + ]), + ); +}; + +const pageRows = ( + rows: readonly Row[], + offset: number, + limit: number, +): readonly Row[] => rows.slice(offset, offset + limit); + +const layoutIncompatible = (detail: string): AnalysisError => + inputError([ + { + path: ["view"], + reason: "invalid_value", + message: detail, + }, + ]); + +const parentFields = (parent: AnalysisViewParent, layout: BinaryLayout) => ({ + parent_evidence_id: parent.evidenceId, + parent_operation: "inspect_binary_layout" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: { + path: layout.artifact.path, + sha256: layout.artifact.sha256, + }, + limitations: uniqueSorted([...parent.limitations, ...layout.limitations]), + unknowns: uniqueSorted([...layoutUnknowns(layout)]), +}); + +const layoutSummary = (layout: BinaryLayout) => ({ + bytes: layout.artifact.bytes, + format: layout.format, + architecture: layout.architecture, + image_type: layout.image_type, + entry: layout.entry_point, + counts: { + sections: layout.sections.length, + segments: layout.segments.length, + symbols: layout.symbols.length, + relocations: layout.relocations.length, + }, + limitation_count: layout.limitations.length, +}); + +/** Project one layout view without re-running the decoder. */ +export const projectBinaryLayoutView = ( + parent: AnalysisViewParent, + layout: BinaryLayout, + view: AnalysisViewRequest, +): Result => { + const shared = parentFields(parent, layout); + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + summary: layoutSummary(layout), + coverage: completeCoverage( + layout.sections.length + + layout.segments.length + + layout.symbols.length + + layout.relocations.length, + layout.sections.length + + layout.segments.length + + layout.symbols.length + + layout.relocations.length, + ), + ...shared, + }); + if (view.kind === "facet") + return ok({ + kind: "facet", + view, + facet: view.facet === "mitigations" ? layout.mitigations : layout.linkage, + coverage: completeCoverage(1, 1), + ...shared, + }); + if (view.kind === "item") { + if (view.collection === "modules") + return err( + layoutIncompatible( + "modules collection applies to analyze_javascript_application Evidence.", + ), + ); + const selected = + view.collection === "sections" + ? selectLayoutItem(view.collection, layout.sections, view.selector) + : selectLayoutItem(view.collection, layout.symbols, view.selector); + if (!selected.ok) return selected; + return ok({ + kind: "item", + view, + item: selected.value, + coverage: completeCoverage( + 1, + view.collection === "sections" + ? layout.sections.length + : layout.symbols.length, + ), + ...shared, + }); + } + if (view.collection === "modules") + return err( + layoutIncompatible( + "modules collection applies to analyze_javascript_application Evidence.", + ), + ); + const rows = + view.collection === "sections" ? layout.sections : layout.symbols; + const items = + view.collection === "sections" + ? pageRows(layout.sections, view.offset, view.limit) + : pageRows(layout.symbols, view.offset, view.limit); + return ok({ + kind: "page", + view, + items: [...items], + coverage: pageCoverage(view.offset, items.length, rows.length), + ...shared, + }); +}; + +const selectLayoutItem = ( + collection: "sections" | "symbols", + rows: readonly Row[], + selector: Extract["selector"], +): Result => { + if ("index" in selector) + return selectIndexed(collection, rows, selector.index); + if ("name" in selector) return selectNamed(collection, rows, selector.name); + return err( + layoutIncompatible( + "Section and symbol items are selected by index or exact name.", + ), + ); +}; diff --git a/src/domain/analysisView/javascriptApplicationView.ts b/src/domain/analysisView/javascriptApplicationView.ts new file mode 100644 index 00000000000..5216c72b444 --- /dev/null +++ b/src/domain/analysisView/javascriptApplicationView.ts @@ -0,0 +1,318 @@ +import { + AnalysisInputError, + type AnalysisInputIssue, +} from "../analysisErrorCore.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { uniqueSorted } from "../canonicalOrdering.js"; +import { jsonObjectSchema, type JsonValue } from "../jsonValue.js"; +import type { JavaScriptApplicationAnalysisResult } from "../javascript/javascriptApplicationAnalysis.js"; +import type { ApplicationNode } from "../javascript/javascriptApplicationGraphSchemas.js"; +import { err, ok, type Result } from "../result.js"; +import { + completeWithinViewCoverage as completeCoverage, + pageViewCoverage as pageCoverage, +} from "./analysisViewCoverage.js"; +import type { + AnalysisViewParent, + AnalysisViewRequest, + UnsignedAnalysisView, +} from "./analysisView.js"; + +const OPERATION = "inspect_analysis_view"; + +const inputError = ( + issues: readonly AnalysisInputIssue[], +): AnalysisInputError => new AnalysisInputError(OPERATION, undefined, issues); + +const jsIncompatible = (detail: string): AnalysisError => + inputError([ + { + path: ["view"], + reason: "invalid_value", + message: detail, + }, + ]); + +/** Exact module path or original source-map reference, when available. */ +export const javascriptModulePath = (node: ApplicationNode): string | null => { + if (node.identity.strategy === "canonical-path") return node.identity.path; + if (node.identity.strategy === "source-map-original") + return node.identity.original_source; + for (const observation of node.observations) { + const path = observation.properties.path; + if (typeof path === "string") return path; + const location = observation.evidence.location; + if (location.available && location.value.kind === "artifact-path") + return location.value.path; + if (location.available && location.value.kind === "source-range") + return location.value.source; + } + return null; +}; + +const moduleEntries = ( + nodes: readonly ApplicationNode[], +): readonly { + readonly node: ApplicationNode; + readonly path: string | null; +}[] => + nodes + .filter( + (node) => + node.kind === "javascript-asset" || + node.kind === "javascript-module" || + node.kind === "source-module", + ) + .map((node) => ({ node, path: javascriptModulePath(node) })); + +const exportNames = (node: ApplicationNode): readonly string[] => { + const names = new Set(); + for (const observation of node.observations) { + const value = + observation.properties.export_names ?? observation.properties.exports; + if (!Array.isArray(value)) continue; + for (const entry of value) if (typeof entry === "string") names.add(entry); + } + return [...names]; +}; + +const nodeHashes = (node: ApplicationNode): readonly string[] => { + const hashes = new Set(); + const { identity } = node; + if (identity.strategy === "content-digest") hashes.add(identity.sha256); + if (identity.strategy === "canonical-path") + hashes.add(identity.artifact_sha256); + if (identity.strategy === "source-map-original") { + hashes.add(identity.source_map_sha256); + if (identity.source_sha256 !== null) hashes.add(identity.source_sha256); + } + for (const observation of node.observations) { + for (const key of [ + "sha256", + "container_sha256", + "entry_sha256", + "source_sha256", + ] as const) { + const value = observation.properties[key]; + if (typeof value === "string" && /^[a-f0-9]{64}$/u.test(value)) + hashes.add(value); + } + } + return [...hashes]; +}; + +const sourceRanges = (node: ApplicationNode): readonly JsonValue[] => + node.observations.flatMap((observation) => { + const location = observation.evidence.location; + return location.available && location.value.kind === "source-range" + ? [location.value] + : []; + }); + +const moduleItem = (node: ApplicationNode): JsonValue => ({ + node_id: node.node_id, + kind: node.kind, + path: javascriptModulePath(node), + identity: node.identity, + hashes: [...nodeHashes(node)], + exports: [...exportNames(node)], + source_ranges: [...sourceRanges(node)], + observations: node.observations.map((observation) => ({ + observation_id: observation.observation_id, + label: observation.label, + properties: observation.properties, + evidence: { + ...observation.evidence, + location: observation.evidence.location, + }, + })), +}); + +const applicationUnknownCount = ( + analysis: JavaScriptApplicationAnalysisResult, +): number => + analysis.graph.nodes.reduce( + (count, node) => count + Number(node.kind === "unknown"), + 0, + ); + +const javascriptUnknowns = ( + analysis: JavaScriptApplicationAnalysisResult, +): readonly string[] => { + const unknowns = [ + ...analysis.limitations, + ...analysis.graph.limitations, + ...analysis.semantic_graph.limitations, + ]; + if (analysis.graph.coverage.status !== "complete") + unknowns.push(`graph.coverage.status is ${analysis.graph.coverage.status}`); + if (analysis.semantic_graph.coverage.status !== "complete") + unknowns.push( + `semantic_graph.coverage.status is ${analysis.semantic_graph.coverage.status}`, + ); + if (applicationUnknownCount(analysis) > 0) + unknowns.push( + `${String(applicationUnknownCount(analysis))} application graph unknowns remain in the parent Evidence.`, + ); + if (analysis.semantic_graph.unknowns.length > 0) + unknowns.push( + `${String(analysis.semantic_graph.unknowns.length)} semantic graph unknowns remain in the parent Evidence.`, + ); + return unknowns; +}; + +const parentFields = ( + parent: AnalysisViewParent, + analysis: JavaScriptApplicationAnalysisResult, +) => ({ + parent_evidence_id: parent.evidenceId, + parent_operation: "analyze_javascript_application" as const, + parent_digest: parent.evidenceId.slice(3), + artifact: { + path: analysis.input_path, + sha256: analysis.root_artifact_sha256, + }, + limitations: uniqueSorted([ + ...parent.limitations, + ...analysis.limitations, + ...analysis.graph.limitations, + ...analysis.semantic_graph.limitations, + ]), + unknowns: uniqueSorted([...javascriptUnknowns(analysis)]), +}); + +const selectModule = ( + modules: ReturnType, + selector: Extract["selector"], +): Result => { + if ("node_id" in selector) { + const matches = modules.filter( + ({ node }) => node.node_id === selector.node_id, + ); + if (matches.length === 1) { + const selected = matches[0]; + if (selected !== undefined) return ok(selected.node); + } + return err( + inputError([ + { + path: ["view", "selector", "node_id"], + reason: "invalid_value", + message: `No module has node_id ${JSON.stringify(selector.node_id)}. Page the modules collection to inspect available identities.`, + }, + ]), + ); + } + if ("path" in selector) { + const matches = modules.filter((entry) => entry.path === selector.path); + if (matches.length === 1) { + const selected = matches[0]; + if (selected !== undefined) return ok(selected.node); + } + if (matches.length === 0) + return err( + inputError([ + { + path: ["view", "selector", "path"], + reason: "invalid_value", + message: `No module has path ${JSON.stringify(selector.path)}. Page the modules collection to inspect exact paths and node_id values.`, + }, + ]), + ); + return err( + inputError([ + { + path: ["view", "selector", "path"], + reason: "invalid_value", + message: `Multiple modules have path ${JSON.stringify(selector.path)}; select one by node_id.`, + expected: matches.map((entry) => ({ node_id: entry.node.node_id })), + }, + ]), + ); + } + return err( + jsIncompatible("Module items are selected by exact path or node_id."), + ); +}; + +/** Project one JavaScript application view without re-running reconstruction. */ +export const projectJavaScriptApplicationView = ( + parent: AnalysisViewParent, + analysis: JavaScriptApplicationAnalysisResult, + view: AnalysisViewRequest, +): Result => { + const shared = parentFields(parent, analysis); + const modules = moduleEntries(analysis.graph.nodes); + if (view.kind === "summary") + return ok({ + kind: "summary", + view, + summary: { + input_path: analysis.input_path, + format: analysis.format, + root_artifact_sha256: analysis.root_artifact_sha256, + statistics: jsonObjectSchema.parse(analysis.statistics), + electron: jsonObjectSchema.parse(analysis.summary), + coverage: jsonObjectSchema.parse({ + application: analysis.graph.coverage, + semantic: { + ...analysis.semantic_graph.coverage, + families: analysis.semantic_graph.coverage.families.map( + ({ unknown_ids, ...family }) => ({ + ...family, + unknown_count: unknown_ids.length, + }), + ), + }, + application_unknowns: applicationUnknownCount(analysis), + semantic_unknowns: analysis.semantic_graph.unknowns.length, + }), + limitation_count: analysis.limitations.length, + }, + coverage: completeCoverage(modules.length, analysis.graph.nodes.length), + ...shared, + }); + if (view.kind === "facet") + return err( + jsIncompatible( + "facet views apply to inspect_binary_layout Evidence (mitigations or linkage).", + ), + ); + if (view.kind === "item") { + if (view.collection !== "modules") + return err( + jsIncompatible( + "Section and symbol collections apply to inspect_binary_layout Evidence.", + ), + ); + const selected = selectModule(modules, view.selector); + if (!selected.ok) return selected; + return ok({ + kind: "item", + view, + item: moduleItem(selected.value), + coverage: completeCoverage(1, modules.length), + ...shared, + }); + } + if (view.collection !== "modules") + return err( + jsIncompatible( + "Section and symbol collections apply to inspect_binary_layout Evidence.", + ), + ); + const items = modules + .slice(view.offset, view.offset + view.limit) + .map(({ node, path }) => ({ + node_id: node.node_id, + kind: node.kind, + path, + })); + return ok({ + kind: "page", + view, + items, + coverage: pageCoverage(view.offset, items.length, modules.length), + ...shared, + }); +}; diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index cbb8670a0ff..cad23764729 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "33", + skillVersion: "34", } as const; diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 86783185538..aafb7b8a505 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -4,6 +4,7 @@ import { registerEvmTools } from "./registerEvmTools.js"; import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; import { createRecordedCrashService } from "../composition/binaryDiagnostics.js"; import type { RecordedCrashService } from "../application/binaryDiagnostics/RecordedCrashService.js"; +import { registerAnalysisViewTool } from "./registerAnalysisViewTool.js"; import { registerBinaryDiagnosticsTools } from "./registerBinaryDiagnosticsTools.js"; import { createBinaryLayoutService } from "../composition/binaryDiagnostics.js"; import type { BinaryLayoutService } from "../application/binaryDiagnostics/BinaryLayoutService.js"; @@ -221,6 +222,7 @@ export const createServer = ( toolLogger, recordEvidence, ); + registerAnalysisViewTool(server, toolLogger, evidenceById, recordEvidence); registerEvmTools( server, options.evmInterface ?? createEvmInterfaceService(), diff --git a/src/server/registerAnalysisViewTool.ts b/src/server/registerAnalysisViewTool.ts new file mode 100644 index 00000000000..ccb25aa7929 --- /dev/null +++ b/src/server/registerAnalysisViewTool.ts @@ -0,0 +1,32 @@ +import type { McpServer } from "@modelcontextprotocol/server"; + +import { inspectAnalysisView } from "../application/analysisView/AnalysisViewService.js"; +import type { EvidenceLookup } from "../application/EvidenceInputResolver.js"; +import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { Logger } from "../logger.js"; +import { logToolExecution } from "./toolLogging.js"; +import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; +import { toCallToolResult, toEvidenceToolResult } from "./toolResult.js"; + +/** Bind selected-view projection to its named contract and session Evidence owner. */ +export const registerAnalysisViewTool = ( + server: McpServer, + logger: Logger, + evidenceById?: EvidenceLookup, + recordEvidence?: EvidenceWriter["recordEvidence"], +): void => { + const contract = toolContract("inspect_analysis_view"); + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input) => { + const result = await logToolExecution(logger, contract.name, () => + Promise.resolve(inspectAnalysisView(input, evidenceById)), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordEvidence?.(result.value); + return toEvidenceToolResult(result.value, contract, recorded); + }, + ); +}; diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts new file mode 100644 index 00000000000..08cea192be5 --- /dev/null +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -0,0 +1,82 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { analysisViewLayoutEvidence } from "../../fixtures/analysisView.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "projects one section from a JSON file of inline layout Evidence", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-analysis-view-cli-"); + const parent = analysisViewLayoutEvidence(); + const request = { + source: { kind: "inline", evidence: parent }, + view: { + kind: "item", + collection: "sections", + selector: { name: ".data" }, + }, + }; + const input = join(root, "view.json"); + await mkdir(root, { recursive: true }); + await writeFile(input, JSON.stringify(request)); + const result = await cli.run({ + arguments: ["inspect-analysis-view", input, "--json"], + environment: { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toMatchObject({ + operation: "inspect_analysis_view", + normalized_result: { + kind: "item", + parent_evidence_id: parent.evidence_id, + item: { name: { display: ".data" } }, + }, + }); + }, +); + +cliTest("rejects malformed JSON before projection", async ({ cli }) => { + const root = await createTestTempDirectory("rea-analysis-view-cli-bad-"); + const result = await cli.run({ + arguments: ["inspect-analysis-view", "{", "--json"], + environment: { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { operation: "inspect-analysis-view" }, + }); +}); + +cliTest("rejects a request that omits source", async ({ cli }) => { + const root = await createTestTempDirectory("rea-analysis-view-cli-missing-"); + const input = join(root, "missing-source.json"); + await writeFile(input, JSON.stringify({ view: { kind: "summary" } })); + const result = await cli.run({ + arguments: ["inspect-analysis-view", input, "--json"], + environment: { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { operation: "inspect-analysis-view" }, + }); +}); diff --git a/tests/boundary/cli/jsonInputStatus.test.ts b/tests/boundary/cli/jsonInputStatus.test.ts index 1acd8c61b0a..7e768d7ecd5 100644 --- a/tests/boundary/cli/jsonInputStatus.test.ts +++ b/tests/boundary/cli/jsonInputStatus.test.ts @@ -7,6 +7,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; const JSON_COMMANDS = [ + ["inspect-analysis-view", "inspect-analysis-view"], ["trace-application-feature", "trace-application-feature"], ["trace-javascript-semantics", "trace-javascript-semantics"], ["compare-application-versions", "compare-application-versions"], diff --git a/tests/boundary/cli/productCatalog.test.ts b/tests/boundary/cli/productCatalog.test.ts index e5b25c1e877..e0e09283766 100644 --- a/tests/boundary/cli/productCatalog.test.ts +++ b/tests/boundary/cli/productCatalog.test.ts @@ -10,6 +10,7 @@ import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { SUPPORTED_CLIENT_DEFINITIONS } from "../../../src/application/SupportedClients.js"; import { + ANALYSIS_VIEW_PROVIDER, ANDROID_APPLICATION_PROVIDER, APPLE_APPLICATION_PROVIDER, ARTIFACT_GRAPH_PROVIDER, @@ -132,6 +133,7 @@ describe("canonical product catalog", () => { JAVASCRIPT_APPLICATION_PROVIDER, JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER, JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, + ANALYSIS_VIEW_PROVIDER, WEB_SCRIPT_EXPORT_PROVIDER, ] .map(({ id }) => id) @@ -178,6 +180,10 @@ describe("canonical product catalog", () => { "trace_application_feature", "trace_javascript_semantics", ]); + expect( + catalog.providers.find(({ id }) => id === ANALYSIS_VIEW_PROVIDER.id) + ?.capabilities, + ).toEqual(["inspect_analysis_view"]); expect( catalog.providers.find(({ id }) => id === ANDROID_APPLICATION_PROVIDER.id) ?.capabilities, diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts new file mode 100644 index 00000000000..d85fa05ec7f --- /dev/null +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -0,0 +1,270 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { Ajv2020 } from "ajv/dist/2020.js"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { BinaryLayoutService } from "../../../src/application/binaryDiagnostics/BinaryLayoutService.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { ok } from "../../../src/domain/result.js"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "../../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { + analysisViewJavaScriptEvidence, + analysisViewJavaScriptAnalysisWithSource, + analysisViewBindJavaScriptGraphs, + analysisViewLayoutFixture, +} from "../../fixtures/analysisView.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { BINARY_LAYOUT_TEST_PROVIDER } from "../../fixtures/binaryDiagnostics/layout.js"; + +const connect = async (binaryLayout?: BinaryLayoutService) => { + const session = createTestBinarySession(() => { + throw new Error("selected views must not start a deep provider"); + }); + const server = createServer( + session, + session, + binaryLayout === undefined ? {} : { binaryLayout }, + ); + const client = new Client({ name: "analysis-view-mcp", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return { client, session }; +}; + +it("advertises exact schemas and projects one layout section from retained Evidence", async () => { + const layout = analysisViewLayoutFixture(); + const service = new BinaryLayoutService({ + identity: BINARY_LAYOUT_TEST_PROVIDER, + inspect: () => Promise.resolve(ok(layout)), + }); + const { client, session } = await connect(service); + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_analysis_view", + ); + if (advertised?.outputSchema === undefined) + throw new Error("inspect_analysis_view must publish both schemas"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + expect( + ajv.validate(inputSchema, { + source: { + kind: "retained-evidence", + evidence_id: `ev_${"a".repeat(64)}`, + }, + view: { kind: "summary", approval: true }, + }), + ).toBe(false); + const inspected = await client.callTool({ + name: "inspect_binary_layout", + arguments: { path: layout.artifact.path }, + }); + expect(inspected.isError).not.toBe(true); + const parent = toolContract("inspect_binary_layout").outputSchema.parse( + inspected.structuredContent, + ); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { + kind: "item", + collection: "sections", + selector: { name: ".data" }, + }, + }, + }); + expect(response.isError).not.toBe(true); + expect( + ajv.validate(outputSchema, response.structuredContent), + JSON.stringify(ajv.errors), + ).toBe(true); + const parsed = toolContract("inspect_analysis_view").outputSchema.parse( + response.structuredContent, + ); + expect(parsed.result).toMatchObject({ + kind: "item", + parent_evidence_id: parent.evidence_id, + item: { name: { display: ".data" } }, + }); + expect(parsed.result).toEqual( + parseEvidence(parsed.evidence).normalized_result, + ); + expect(session.evidenceById(parsed.evidence_id)).toEqual( + parseEvidence(parsed.evidence), + ); +}); + +it("retains an oversized selected observation and keeps subsequent views usable", async () => { + const { client, session } = await connect(); + const analysis = analysisViewJavaScriptAnalysisWithSource(); + const original = analysis.graph.nodes[0]; + if (original === undefined) throw new Error("missing module"); + const source = "\0".repeat(500_000); + const node = createJavaScriptApplicationNode({ + kind: original.kind, + identity: original.identity, + observations: original.observations.map((observation) => ({ + label: observation.label, + properties: { ...observation.properties, source }, + evidence: observation.evidence, + })), + }); + const graph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [node.node_id], + nodes: [node], + edges: [], + coverage: analysis.graph.coverage, + limitations: analysis.graph.limitations, + }); + const parent = analysisViewJavaScriptEvidence( + analysisViewBindJavaScriptGraphs(analysis, graph), + ); + expect(session.recordEvidence(parent).ok).toBe(true); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { + kind: "item", + collection: "modules", + selector: { node_id: node.node_id }, + }, + }, + }); + expect(response.isError).toBe(true); + const reference = z + .object({ + error: z.object({ + details: z.object({ + reported_limits: z.object({ + evidence_reference: z.object({ + kind: z.literal("retained-evidence"), + evidence_id: z.string(), + }), + }), + }), + }), + }) + .parse(response.structuredContent).error.details + .reported_limits.evidence_reference; + expect( + session.evidenceById(reference.evidence_id)?.normalized_result, + ).toMatchObject({ + item: { + observations: [ + expect.objectContaining({ + properties: expect.objectContaining({ source }), + }), + ], + }, + }); + await client.ping(); + const summary = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "summary" }, + }, + }); + expect(summary.isError).not.toBe(true); +}); + +it("returns a JavaScript summary without graph payloads from a retained reference", async () => { + const { client, session } = await connect(); + const parent = analysisViewJavaScriptEvidence(); + expect(session.recordEvidence(parent).ok).toBe(true); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: parent.evidence_id, + }, + view: { kind: "summary" }, + }, + }); + expect(response.isError).not.toBe(true); + const parsed = toolContract("inspect_analysis_view").outputSchema.parse( + response.structuredContent, + ); + expect(parsed.result).toMatchObject({ + kind: "summary", + parent_operation: "analyze_javascript_application", + summary: { format: "directory" }, + }); + expect(parsed.result).not.toHaveProperty("summary.semantic_graph"); + const stale = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: `ev_${"e".repeat(64)}`, + }, + view: { kind: "summary" }, + }, + }); + expect(stale).toMatchObject({ + isError: true, + structuredContent: { + error: { details: { reason: "missing" } }, + }, + }); +}); + +it("accepts a transport-constraint retained reference as the view source", async () => { + const { client, session } = await connect(); + const parent = analysisViewJavaScriptEvidence(); + expect(session.recordEvidence(parent).ok).toBe(true); + const evidenceReference = { + kind: "retained-evidence" as const, + evidence_id: parent.evidence_id, + }; + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: evidenceReference, + view: { + kind: "page", + collection: "modules", + offset: 0, + limit: 8, + }, + }, + }); + expect(response.isError).not.toBe(true); + const parsed = toolContract("inspect_analysis_view").outputSchema.parse( + response.structuredContent, + ); + expect(parsed.result).toMatchObject({ + kind: "page", + parent_evidence_id: parent.evidence_id, + coverage: { exhausted: true }, + }); + if (parsed.result.kind !== "page") throw new Error("expected page view"); + expect(parsed.result.items).toEqual([ + { + node_id: expect.any(String), + kind: "javascript-asset", + path: "renderer.js", + }, + ]); +}); diff --git a/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts b/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts index 12ed431d963..b6bf0e3cbae 100644 --- a/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts +++ b/tests/boundary/mcp/binaryDiagnosticsMcp.test.ts @@ -65,6 +65,26 @@ it("advertises exact valid schemas and records inline layout without a binary ta expect(parsed.result).toEqual(value); expect(parsed.result).toEqual(evidence.normalized_result); expect(session.evidenceById(evidence.evidence_id)).toEqual(evidence); + const viewed = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "retained-evidence", + evidence_id: evidence.evidence_id, + }, + view: { kind: "summary" }, + }, + }); + expect(viewed.isError).not.toBe(true); + expect( + toolContract("inspect_analysis_view").outputSchema.parse( + viewed.structuredContent, + ).result, + ).toMatchObject({ + kind: "summary", + parent_evidence_id: evidence.evidence_id, + summary: { format: "elf", bytes: value.artifact.bytes }, + }); expect( ( await client.callTool({ diff --git a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts index 304bc9b143d..c01fd212fd0 100644 --- a/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts +++ b/tests/boundary/mcp/retainedApplicationEvidenceMcp.test.ts @@ -215,3 +215,27 @@ it("preserves validation for the referenced record's operation, predicate, and s }); expect(response.isError).toBe(true); }); + +it("projects a retained JavaScript summary without graph or semantic_graph", async () => { + const { client, session } = await connect(); + expect(session.recordEvidence(application).ok).toBe(true); + const inline = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "inline", evidence: application }, + view: { kind: "summary" }, + }, + }); + const referenced = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: retained(application.evidence_id), + view: { kind: "summary" }, + }, + }); + expect(inline.isError).not.toBe(true); + expect(referenced.structuredContent).toEqual(inline.structuredContent); + expect(inline.structuredContent).not.toHaveProperty( + "result.summary.semantic_graph", + ); +}); diff --git a/tests/fixtures/analysisView.ts b/tests/fixtures/analysisView.ts new file mode 100644 index 00000000000..0f928a22a28 --- /dev/null +++ b/tests/fixtures/analysisView.ts @@ -0,0 +1,195 @@ +import { BINARY_LAYOUT_TEST_PROVIDER } from "./binaryDiagnostics/layout.js"; +import { binaryLayoutFixture } from "./binaryDiagnostics/layout.js"; +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../src/contracts/javascript/javascriptRuntimeReconciliationExample.js"; +import { createEvidence, type Evidence } from "../../src/domain/evidence.js"; +import type { JavaScriptApplicationAnalysisResult } from "../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "../../src/domain/javascript/javascriptApplicationGraph.js"; +import { createJavaScriptSemanticGraph } from "../../src/domain/javascript/javascriptSemanticGraph.js"; +import type { BinaryLayout } from "../../src/domain/native/binaryLayout.js"; +import { binaryLayoutSchema } from "../../src/domain/native/binaryLayout.js"; + +const hex = (value: number): string => + value === 0 ? "0x0" : `0x${value.toString(16)}`; + +const layoutName = (display: string, offset: number) => ({ + display, + bytes_base64: Buffer.from(display, "utf8").toString("base64"), + location: { offset: hex(offset), bytes: hex(display.length + 1) }, + unknown_reason: null, +}); + +const layoutSection = ( + index: number, + display: string, + nameOffset: number, +): BinaryLayout["sections"][number] => ({ + index, + name: layoutName(display, nameOffset), + name_offset: hex(nameOffset), + type: 1, + header_location: { offset: hex(index * 16), bytes: "0x10" }, + address: "0x0", + offset: hex(64 + index * 16), + size: "0x10", + alignment: "0x1", + flags: "0x6", + link: 0, + info: 0, + entry_size: "0x0", + file_backing: "file", +}); + +const layoutSymbol = ( + entryIndex: number, + display: string, + nameOffset: number, +): BinaryLayout["symbols"][number] => ({ + table_index: 0, + entry_index: entryIndex, + name: layoutName(display, nameOffset), + name_offset: hex(nameOffset), + location: { offset: hex(nameOffset), bytes: hex(display.length + 1) }, + value: "0x0", + value_meaning: "no-address", + size: "0x0", + binding: 1, + type: 2, + visibility: 0, + section_index: 1, +}); + +/** Layout fixture with duplicate section/symbol names for selected-view tests. */ +export const analysisViewLayoutFixture = (): BinaryLayout => + binaryLayoutSchema.parse({ + ...binaryLayoutFixture("/artifacts/view-source.elf"), + artifact: { + path: "/artifacts/view-source.elf", + sha256: "b".repeat(64), + bytes: 512, + }, + sections: [ + layoutSection(0, ".text", 256), + layoutSection(1, ".text", 262), + layoutSection(2, ".data", 268), + ], + symbols: [ + layoutSymbol(0, "main", 280), + layoutSymbol(1, "main", 285), + layoutSymbol(2, "start", 290), + ], + }); + +/** Wrap a layout observation as inspect_binary_layout Evidence. */ +export const analysisViewLayoutEvidence = ( + layout: BinaryLayout = analysisViewLayoutFixture(), +): Evidence => + createEvidence( + { + path: layout.artifact.path, + format: "elf", + architecture: "x86_64", + sha256: layout.artifact.sha256, + }, + BINARY_LAYOUT_TEST_PROVIDER, + { + operation: "inspect_binary_layout", + parameters: { path: layout.artifact.path }, + result: layout, + rawResult: layout, + confidence: "observed", + limitations: layout.limitations, + locations: [{ kind: "artifact-path", path: layout.artifact.path }], + }, + ); + +/** Minimal authenticated JavaScript application analysis used by view tests. */ +export const analysisViewJavaScriptAnalysis = + (): JavaScriptApplicationAnalysisResult => + javascriptApplicationAnalysisResultSchema.parse( + JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result, + ); + +/** Application analysis whose module observations include source text. */ +export const analysisViewJavaScriptAnalysisWithSource = + (): JavaScriptApplicationAnalysisResult => { + const analysis = analysisViewJavaScriptAnalysis(); + const original = analysis.graph.nodes[0]; + if (original === undefined) + throw new Error("JavaScript application example is missing its module"); + const withSource = createJavaScriptApplicationNode({ + kind: original.kind, + identity: original.identity, + observations: original.observations.map((observation) => ({ + label: observation.label, + properties: { + ...observation.properties, + source: "export const secret = 1;\n", + text: "should not leak", + }, + evidence: observation.evidence, + })), + }); + const graph = createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [withSource.node_id], + nodes: [withSource], + edges: [], + coverage: analysis.graph.coverage, + limitations: analysis.graph.limitations, + }); + const { graph_id: _graphId, ...semantic } = analysis.semantic_graph; + return javascriptApplicationAnalysisResultSchema.parse({ + ...analysis, + graph, + semantic_graph: createJavaScriptSemanticGraph({ + ...semantic, + application_graph_id: graph.graph_id, + }), + }); + }; + +/** Rebind a companion semantic graph after replacing the application graph. */ +export const analysisViewBindJavaScriptGraphs = ( + analysis: JavaScriptApplicationAnalysisResult, + graph: ReturnType, +): JavaScriptApplicationAnalysisResult => { + const { graph_id: _graphId, ...semantic } = analysis.semantic_graph; + return javascriptApplicationAnalysisResultSchema.parse({ + ...analysis, + graph, + semantic_graph: createJavaScriptSemanticGraph({ + ...semantic, + application_graph_id: graph.graph_id, + }), + }); +}; + +/** Wrap JavaScript application analysis as producer Evidence. */ +export const analysisViewJavaScriptEvidence = ( + analysis: JavaScriptApplicationAnalysisResult = analysisViewJavaScriptAnalysis(), +): Evidence => + createEvidence( + { + path: analysis.input_path, + format: analysis.format, + sha256: analysis.root_artifact_sha256, + }, + { + id: "rea-javascript-application", + name: "REA JavaScript application analyzer", + version: "1", + }, + { + predicateType: "rea.javascript-application-analysis", + operation: "analyze_javascript_application", + parameters: { input_path: analysis.input_path, format: analysis.format }, + result: analysis, + confidence: "derived", + authority: "shipped-artifact", + limitations: analysis.limitations, + }, + ); From 864160d8858fad3c0afebff81c31b090a1ebf823 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 12:46:17 +0800 Subject: [PATCH 133/791] ci(website): allow explicitly requested Pages-only publication --- .github/workflows/website-pages.yml | 38 ++++++++++++++++++++++++++--- website/AGENTS.md | 6 +++-- website/README.md | 20 +++++++++++---- website/style-guide.md | 6 +++-- 4 files changed, 57 insertions(+), 13 deletions(-) diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml index 881a66dc858..980b065f198 100644 --- a/.github/workflows/website-pages.yml +++ b/.github/workflows/website-pages.yml @@ -2,6 +2,15 @@ name: Publish REA website on: workflow_dispatch: + inputs: + target: + description: Publish both hosts, or Pages while configuring Cloudflare + type: choice + required: true + default: both + options: + - both + - pages permissions: contents: read @@ -19,7 +28,16 @@ jobs: contents: read pages: read steps: + - name: Validate the publication target + env: + PUBLISH_TARGET: ${{ inputs.target }} + run: | + case "$PUBLISH_TARGET" in + both|pages) ;; + *) echo "::error::Unknown publication target: $PUBLISH_TARGET"; exit 1 ;; + esac - name: Require Cloudflare publishing credentials + if: inputs.target == 'both' env: CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -67,21 +85,26 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - name: Check out publishing configuration + if: inputs.target == 'both' uses: actions/checkout@v4 - name: Set up Node.js for Wrangler + if: inputs.target == 'both' uses: actions/setup-node@v4 with: node-version-file: .nvmrc - name: Download the verified website + if: inputs.target == 'both' uses: actions/download-artifact@v4 with: name: github-pages path: ${{ runner.temp }}/website-archive - name: Extract the shared publishing artifact + if: inputs.target == 'both' run: | mkdir -p "$RUNNER_TEMP/rea-website-public" tar -xf "$RUNNER_TEMP/website-archive/artifact.tar" -C "$RUNNER_TEMP/rea-website-public" - name: Publish the website to Cloudflare + if: inputs.target == 'both' uses: cloudflare/wrangler-action@v4 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} @@ -92,9 +115,10 @@ jobs: - name: Publish to GitHub Pages id: deployment uses: actions/deploy-pages@v4 - - name: Verify both published revisions + - name: Verify the selected published revisions env: PAGES_URL: ${{ steps.deployment.outputs.page_url }} + PUBLISH_TARGET: ${{ inputs.target }} run: | python3 - <<'PY' import json @@ -109,7 +133,13 @@ jobs: pages_url = os.environ["PAGES_URL"] if not pages_url: sys.exit("GitHub Pages returned no published URL.") - pending = {"Cloudflare": "https://rea.tools/", "GitHub Pages": pages_url} + target = os.environ["PUBLISH_TARGET"] + if target not in ("both", "pages"): + sys.exit("Unknown publication target: " + target) + hosts = {"GitHub Pages": pages_url} + if target == "both": + hosts["Cloudflare"] = "https://rea.tools/" + pending = dict(hosts) for attempt in range(24): for name, base in list(pending.items()): url = urljoin(base, "deployment.json") + "?commit=" + expected @@ -131,6 +161,6 @@ jobs: if pending: sys.exit("Published revision did not match on: " + ", ".join(pending)) with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: - summary.write(f"Published commit `{expected}` to [rea.tools](https://rea.tools/) " - f"and [GitHub Pages]({pages_url}).\n") + links = " and ".join(f"[{name}]({url})" for name, url in hosts.items()) + summary.write(f"Published commit `{expected}` to {links}.\n") PY diff --git a/website/AGENTS.md b/website/AGENTS.md index 47bee247a5b..da4f38b27a7 100644 --- a/website/AGENTS.md +++ b/website/AGENTS.md @@ -20,5 +20,7 @@ for asset notes and the source of case-study claims. copying and downloads. Run `scripts/prepare-website.py` followed by `scripts/verify-website.py`. - Keep `website-pages.yml` as the sole, manual production publisher for - Cloudflare and Pages. Both hosts must receive the same verified artifact and - pass the published-commit check. Follow the website README. + Cloudflare and Pages. Default to `both`: both hosts receive the same verified + artifact and pass the published-commit check. Use `pages` only when explicitly + requested while Cloudflare credentials are being configured. Follow the + website README. diff --git a/website/README.md b/website/README.md index 92b0be51d5b..c011e643547 100644 --- a/website/README.md +++ b/website/README.md @@ -261,14 +261,14 @@ Configure these repository **Actions secrets** before publishing: - `CLOUDFLARE_ACCOUNT_ID`: that Cloudflare account's ID. See Cloudflare's [GitHub Actions authentication instructions](https://developers.cloudflare.com/workers/ci-cd/external-cicd/github-actions/). -The workflow checks that both secrets are present before preparing a release. -Cloudflare deploys first; GitHub Pages deploys after that step succeeds. +The default `both` target checks that both secrets are present before preparing +a release. Cloudflare deploys first; GitHub Pages deploys after that step succeeds. Preparation adds `deployment.json` with the workflow's commit SHA. Cloudflare receives the extracted Pages artifact through Wrangler's explicit assets path; the local custom-build output is not the uploaded directory. The workflow then -checks both public version markers and reports success only when they match -that commit. A failed publication remains failed; fix its cause and rerun the +checks the selected public version markers and reports success only when they +match that commit. A failed publication remains failed; fix its cause and rerun the workflow to complete both deployments. The two hosts can update at different times while a run is in progress. @@ -294,12 +294,22 @@ website** from the Actions tab on `main`. This publishes both production hosts. The workflow uses the `github-pages` environment, the official Pages actions and Cloudflare's Wrangler action. -The equivalent CLI command is: +The equivalent CLI command publishes both hosts: ```sh gh workflow run website-pages.yml --repo morluto/rea --ref main ``` +While Cloudflare credentials are being configured, an explicitly requested +Pages-only publication uses the same workflow with `target=pages`: + +```sh +gh workflow run website-pages.yml --repo morluto/rea --ref main -f target=pages +``` + +This mode verifies the Pages commit marker. The default remains `both` for +synchronized production releases. + Local development does not change Pages settings or run the deployment workflow. Any environment protection rules are configured separately when publication is approved. diff --git a/website/style-guide.md b/website/style-guide.md index 5a05299c60c..de682049f07 100644 --- a/website/style-guide.md +++ b/website/style-guide.md @@ -233,8 +233,10 @@ New analysis or behavior claims need the real evidence that supports them; a moc or a page-loading check cannot establish those claims. Follow the PR review and CI process, then publish using the existing manual -`website-pages.yml` workflow on `main`. It publishes the same verified artifact -to Cloudflare and GitHub Pages and checks both published commit markers. +`website-pages.yml` workflow on `main`. Its default `both` target publishes the +same verified artifact to Cloudflare and GitHub Pages and checks both published +commit markers. Use `pages` only for an explicitly requested publication while +Cloudflare credentials are being configured. Ordinary commits should not publish the site. After deployment, check both live pages and confirm they match the reviewed version. The legacy VitePress build remains separate from the Pages publisher. From 6051cb2c0c723d88dc43ee5ef8a96971bf84d49e Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:57:00 +0800 Subject: [PATCH 134/791] fix(setup): preserve initial backups and Codex comments (#1204) --- docs/installation.md | 5 +- scripts/verify-package-setup.mjs | 27 ++-- .../ClientConfigurationDocument.ts | 54 +++---- src/application/Uninstall.ts | 17 ++- .../clientConfigurationTransactions.test.ts | 139 ++++++++++++++++++ .../filesystem/clientLifecycle.test.ts | 25 +++- 6 files changed, 221 insertions(+), 46 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 2028ea38dfa..5b4f4b83eaf 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -195,8 +195,9 @@ Before applying changes, REA checks your current configuration. The plan lists: Malformed or unsafe existing configuration blocks the whole transaction before Hopper installation or any file write. Declining or pressing Ctrl-C makes no -changes. Agent configuration writes preserve unrelated -entries, create backups, use atomic replacement, and verify their result. +changes. Agent configuration writes preserve unrelated entries and comments, +create backups, use atomic replacement, and verify their result. Setup and +uninstall retain an existing `.rea.backup` rather than replacing the first snapshot. After setup, REA reports which agents, analysis tools, and workflow files passed its final checks. Restart any agent named in the completion message, then begin diff --git a/scripts/verify-package-setup.mjs b/scripts/verify-package-setup.mjs index 9db11edba1b..b415965e071 100644 --- a/scripts/verify-package-setup.mjs +++ b/scripts/verify-package-setup.mjs @@ -31,6 +31,14 @@ const OPENCODE_ORIGINAL = `{ }, }\n`; +const CODEX_ORIGINAL = [ + "# Keep Codex preferences and their spelling.", + 'model = "gpt-5"', + 'notify = ["a", "b"]', + "literal = 'C:\\demo\\path'", + "", +].join("\n"); + const verifyMcpAdd = async ({ cli, environment, npxLog }) => { await run(cli, ["mcp", "add"], environment); const mcpRegistration = await readFile(npxLog, "utf8"); @@ -248,7 +256,7 @@ const verifySetupPlan = async ({ !planned.plannedActions.some(({ kind }) => kind === "configure_client") || !planned.plannedActions.some(({ kind }) => kind === "install_skill") || plannedClaudeConfig !== '{"existing":true}\n' || - plannedCodexConfig !== 'model = "gpt-5"\n' || + plannedCodexConfig !== CODEX_ORIGINAL || plannedCursorConfig !== '{"existing":true}\n' || plannedOpenCodeConfig !== opencodeOriginal ) @@ -459,8 +467,8 @@ const assertCodexSymlink = async ({ !codex.includes("[mcp_servers.rea]") || !codex.includes(`command = "${cli}"`) || !codex.includes("startup_timeout_sec = 30") || - (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== - 'model = "gpt-5"\n' || + !codex.startsWith(CODEX_ORIGINAL) || + (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL || (await readFile(`${cursorConfig}.rea.backup`, "utf8")) !== '{"existing":true}\n' ) @@ -631,7 +639,6 @@ const verifyUninstall = async ({ codexTarget, opencodeConfig, }) => { - const openCodeBeforeUninstall = await readFile(opencodeConfig, "utf8"); const uninstallExecution = await runWithStatus( cli, ["uninstall", "--json"], @@ -657,7 +664,7 @@ const verifyUninstall = async ({ !(await lstat(codexConfig)).isSymbolicLink() || cursorAfterUninstall.existing !== true || cursorAfterUninstall.mcpServers?.rea !== undefined || - !codexAfterUninstall.includes('model = "gpt-5"') || + codexAfterUninstall !== CODEX_ORIGINAL || codexAfterUninstall.includes("mcp_servers.rea") || openCodeErrors.length !== 0 || openCodeAfterUninstall?.model !== "provider/model" || @@ -670,11 +677,10 @@ const verifyUninstall = async ({ "Keep this independently managed server", ) || (await readFile(`${opencodeConfig}.rea.backup`, "utf8")) !== - openCodeBeforeUninstall || - !(await readFile(`${cursorConfig}.rea.backup`, "utf8")).includes('"rea"') || - !(await readFile(`${codexConfig}.rea.backup`, "utf8")).includes( - "[mcp_servers.rea]", - ) + OPENCODE_ORIGINAL || + (await readFile(`${cursorConfig}.rea.backup`, "utf8")) !== + '{"existing":true}\n' || + (await readFile(`${codexConfig}.rea.backup`, "utf8")) !== CODEX_ORIGINAL ) throw new Error( `packaged uninstall did not preserve config symlinks: ${JSON.stringify(uninstall)}`, @@ -704,6 +710,7 @@ export async function verifyPackageSetup({ ); const siblingSkillPath = join(home, ".agents/skills/unrelated/SKILL.md"); if (supportedSetupHost) { + await writeFile(codexTarget, CODEX_ORIGINAL); await mkdir(join(home, ".agents/skills/unrelated"), { recursive: true }); await writeFile(siblingSkillPath, "unrelated skill\n"); } diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index 991dcd0b4ea..038243d8e54 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -832,15 +832,15 @@ const disabledServerListLines = ( /** * Replace REA's TOML server tables and assignments. Comments that introduce a - * following table stay. Text inside strings is not treated as a header. Only - * Grok Build reads the root `disabled_mcp_servers` list that registration - * also edits. + * following table stay. Codex also retains comments around REA's own table. + * Text inside strings is not treated as a header. Only Grok Build reads the + * root `disabled_mcp_servers` list that registration also edits. */ const upsertTomlServerSection = ( originalText: string, serverKey: string, entry: unknown, - editDisabledList: boolean, + format: "toml" | "grok", ): string => { // smol-toml accepts a leading BOM. Leave it in place or the first header is missed. const bom = originalText.startsWith("\uFEFF"); @@ -917,24 +917,25 @@ const upsertTomlServerSection = ( mode = "none"; continue; } - const disabledLines = editDisabledList - ? disabledServerListLines( - disabledServerListEdit({ - text, - lineStart, - valueOffset, - valueEnd, - statementEnd, - tablePath, - segments: assignment.segments, - serverKey, - entry, - }), - lines, - lineIndex, - endLine, - ) - : undefined; + const disabledLines = + format === "grok" + ? disabledServerListLines( + disabledServerListEdit({ + text, + lineStart, + valueOffset, + valueEnd, + statementEnd, + tablePath, + segments: assignment.segments, + serverKey, + entry, + }), + lines, + lineIndex, + endLine, + ) + : undefined; if (disabledLines !== undefined) { flushPending(); kept.push(...disabledLines); @@ -975,7 +976,10 @@ const upsertTomlServerSection = ( continue; } if (isBlankOrComment(line)) { - pending.push(physical); + if (format === "toml" && line.trimStart().startsWith("#")) { + flushPending(); + kept.push(physical); + } else pending.push(physical); lineIndex += 1; continue; } @@ -1044,7 +1048,7 @@ const serializeGrokConfiguration = ( originalText, PRODUCT_IDENTITY.mcpServerKey, entry, - true, + "grok", ); }; @@ -1077,7 +1081,7 @@ const serializeCodexConfiguration = ( originalText, PRODUCT_IDENTITY.mcpServerKey, tomlServerEntry(document, PRODUCT_IDENTITY.mcpServerKey), - false, + "toml", ); // Removing a final table also drops the file's last line break. const edited = diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index 789bdc34e8c..e4e023d6319 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -7,6 +7,7 @@ import { type ClientConfigurationDocument, } from "./ClientConfigurationDocument.js"; import { copyFile, lstat, readFile, realpath, rm } from "node:fs/promises"; +import { constants as fsConstants } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -31,6 +32,7 @@ interface ManagedPathStats { /** Injectable filesystem operations used to prove uninstall failure recovery. */ export interface UninstallFileSystem { readText(path: string): Promise; + /** Preserve the first backup; reject an existing destination with EEXIST. */ copy(source: string, destination: string): Promise; writeText(path: string, contents: string): Promise; stat(path: string): Promise; @@ -40,7 +42,8 @@ export interface UninstallFileSystem { const systemFileSystem: UninstallFileSystem = { readText: (path) => readFile(path, "utf8"), - copy: (source, destination) => copyFile(source, destination), + copy: (source, destination) => + copyFile(source, destination, fsConstants.COPYFILE_EXCL), writeText: (path, contents) => writeFileAtomic(path, contents, { encoding: "utf8" }), stat: (path) => lstat(path), @@ -274,12 +277,12 @@ const removeClient = async ( try { await fileSystem.copy(transactionPath, backupPath); } catch (cause: unknown) { - void cause; - return item( - client.name, - "failed", - "Configuration could not be backed up, so no change was made. Check file permissions, then rerun uninstall.", - ); + if (!(cause instanceof Error && "code" in cause && cause.code === "EEXIST")) + return item( + client.name, + "failed", + "Configuration could not be backed up, so no change was made. Check file permissions, then rerun uninstall.", + ); } try { await fileSystem.writeText( diff --git a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts index 05576f5631f..4770853813c 100644 --- a/tests/boundary/filesystem/clientConfigurationTransactions.test.ts +++ b/tests/boundary/filesystem/clientConfigurationTransactions.test.ts @@ -2,8 +2,10 @@ import { chmod, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { parse as parseToml } from "smol-toml"; import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("client configuration write failures", () => { @@ -55,3 +57,140 @@ describe("TOML client configuration comparison", () => { }); }); }); + +const unrelatedToml = [ + "# Keep this explanation.", + 'notify = ["a", "b"]', + "literal = 'C:\\demo\\path'", + 'disabled_mcp_servers = ["rea", "other"] # Not a Codex registration.', + "", +].join("\n"); + +const unrelatedServer = [ + "# Keep the other server.", + '[mcp_servers."rea.env"]', + "command = 'other'", + "", +].join("\n"); + +const unrelatedValues = (text: string) => { + const document = parseToml(text); + const servers = document.mcp_servers; + if (typeof servers === "object" && servers !== null) { + Reflect.deleteProperty(servers, "rea"); + if (Object.keys(servers).length === 0) delete document.mcp_servers; + } + return document; +}; + +describe("Codex configuration text preservation", () => { + it.each([ + { + name: "new registration", + original: unrelatedToml, + preserved: [unrelatedToml], + }, + { + name: "quoted tables and nested environment", + original: `${unrelatedToml}[mcp_servers."rea"]\ncommand = "old"\n[mcp_servers.rea.env]\nOLD = "value"\n${unrelatedServer}`, + preserved: [unrelatedToml, unrelatedServer], + }, + { + name: "dotted registration keys", + original: `${unrelatedToml}mcp_servers.rea.command = "old"\n${unrelatedServer}`, + preserved: [unrelatedToml, unrelatedServer], + }, + { + name: "inline server table", + original: `${unrelatedToml}mcp_servers = { rea = { command = "old" }, other = { command = 'other' } } # Shared table.\n`, + preserved: [ + unrelatedToml, + "other = { command = 'other' }", + "# Shared table.", + ], + }, + { + name: "BOM, CRLF, and multiline text resembling a registration", + original: + `\uFEFF${unrelatedToml}banner = '''\n[mcp_servers.rea]\n# This is string content.\n'''\n`.replaceAll( + "\n", + "\r\n", + ), + preserved: [ + unrelatedToml.replaceAll("\n", "\r\n"), + "banner = '''\r\n[mcp_servers.rea]\r\n# This is string content.\r\n'''", + ], + }, + { + name: "comments without settings", + original: "# Keep this explanation.\n# And this trailing comment.\n\n", + preserved: ["# Keep this explanation.", "# And this trailing comment."], + }, + ])( + "preserves unrelated text through $name setup and uninstall", + async ({ original, preserved }) => { + const root = await createTestTempDirectory("rea-codex-preserve-"); + const configPath = join(root, "config.toml"); + const client = { name: "codex", format: "toml", configPath } as const; + await writeFile(configPath, original); + + expect( + await configureClientConfiguration(client, {}, ["rea", "mcp"]), + ).toMatchObject({ status: "configured" }); + const configured = await readFile(configPath, "utf8"); + for (const text of preserved) expect(configured).toContain(text); + expect(unrelatedValues(configured)).toEqual(unrelatedValues(original)); + expect(configured.startsWith("\uFEFF")).toBe( + original.startsWith("\uFEFF"), + ); + expect( + await configureClientConfiguration(client, {}, ["rea", "mcp"]), + ).toEqual({ status: "unchanged" }); + expect(await readFile(configPath, "utf8")).toBe(configured); + + expect( + (await systemUninstallHost(root).removeClient(client)).status, + ).toBe("removed"); + const removed = await readFile(configPath, "utf8"); + for (const text of preserved) expect(removed).toContain(text); + expect(unrelatedValues(removed)).toEqual(unrelatedValues(original)); + expect(parseToml(removed)).not.toHaveProperty("mcp_servers.rea"); + expect(removed.startsWith("\uFEFF")).toBe(original.startsWith("\uFEFF")); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe(original); + }, + ); + + it("retains the first backup after uninstall and a later setup", async () => { + const root = await createTestTempDirectory("rea-codex-backup-"); + const configPath = join(root, "config.toml"); + const client = { name: "codex", format: "toml", configPath } as const; + const original = 'notify = ["a", "b"]\n'; + await writeFile(configPath, original); + await configureClientConfiguration(client, {}, ["rea", "mcp"]); + expect((await systemUninstallHost(root).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe(original); + await configureClientConfiguration(client, {}, ["rea", "mcp"]); + expect(await readFile(`${configPath}.rea.backup`, "utf8")).toBe(original); + }); + + it("refuses malformed Codex TOML unchanged without creating a backup", async () => { + const root = await createTestTempDirectory("rea-codex-malformed-"); + const configPath = join(root, "config.toml"); + const client = { name: "codex", format: "toml", configPath } as const; + const original = + "# Keep this explanation.\n[mcp_servers.rea]\ncommand = [broken\n"; + await writeFile(configPath, original); + expect( + await configureClientConfiguration(client, {}, ["rea", "mcp"]), + ).toEqual({ status: "failed", reason: "readback" }); + expect((await systemUninstallHost(root).removeClient(client)).status).toBe( + "failed", + ); + expect(await readFile(configPath, "utf8")).toBe(original); + await expect( + readFile(`${configPath}.rea.backup`, "utf8"), + ).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 6c314eec45d..1333f465c70 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -1,3 +1,4 @@ +import { constants as fsConstants } from "node:fs"; import { copyFile, lstat, @@ -1587,7 +1588,7 @@ describe("client configuration filesystem failure reporting", () => { copy: (source, destination) => failure === "backup" ? Promise.reject(new Error("SECRET backup failure")) - : copyFile(source, destination), + : copyFile(source, destination, fsConstants.COPYFILE_EXCL), writeText: async (path, contents) => { writes += 1; if ((failure === "update" && writes === 1) || failure === "restore") @@ -1607,6 +1608,25 @@ describe("client configuration filesystem failure reporting", () => { }, ); + it("restores the current configuration without replacing its initial backup", async () => { + const { home, config } = await uninstallFixture(); + const current = await readFile(config, "utf8"); + const initial = '{"existing":true}\n'; + await writeFile(`${config}.rea.backup`, initial); + let writes = 0; + const result = await systemUninstallHost(home, { + ...testFileSystem, + writeText: async (path, contents) => { + writes += 1; + await writeFile(path, contents); + if (writes === 1) throw new Error("update failed after writing"); + }, + }).removeClient({ name: "cursor", format: "json", configPath: config }); + expect(result.status).toBe("failed"); + expect(await readFile(config, "utf8")).toBe(current); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(initial); + }); + it("reports a managed-path removal failure", async () => { const home = await createTestTempDirectory("rea-uninstall-remove-"); roots.push(home); @@ -1631,7 +1651,8 @@ describe("client configuration filesystem failure reporting", () => { const testFileSystem: UninstallFileSystem = { readText: (path) => readFile(path, "utf8"), - copy: (source, destination) => copyFile(source, destination), + copy: (source, destination) => + copyFile(source, destination, fsConstants.COPYFILE_EXCL), writeText: (path, contents) => writeFile(path, contents), stat: (path) => lstat(path), realpath: (path) => realpath(path), From beca9977eb2b18fc7b9844cff0ca70004b6ff21a Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:58:16 +0800 Subject: [PATCH 135/791] =?UTF-8?q?docs:=20celebrate=2030,000=20GitHub=20s?= =?UTF-8?q?tars=20=F0=9F=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++++ README_ar.md | 4 ++++ README_de.md | 4 ++++ README_es.md | 4 ++++ README_fa.md | 4 ++++ README_fr.md | 4 ++++ README_ja.md | 4 ++++ README_ko.md | 4 ++++ README_pl.md | 4 ++++ README_pt-BR.md | 4 ++++ README_ru.md | 4 ++++ README_th.md | 4 ++++ README_tr.md | 4 ++++ README_uk.md | 4 ++++ README_vi.md | 4 ++++ README_zh-TW.md | 4 ++++ README_zh.md | 4 ++++ 17 files changed, 68 insertions(+) diff --git a/README.md b/README.md index 2e6172b8ca1..0a521f31069 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30,000 GitHub stars — thank you!** + +Thanks to everyone using REA, reporting bugs, testing builds, and contributing fixes. + morluto%2Frea | Trendshift **[Website](https://rea.tools/) · [Guides](https://rea.tools/guides/) · [Showcases](https://rea.tools/showcase/)** diff --git a/README_ar.md b/README_ar.md index 29b5bde2d86..dbeb384d8f2 100644 --- a/README_ar.md +++ b/README_ar.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **وصلنا إلى 30,000 نجمة على GitHub، شكرًا لكم!** + +شكرًا لكل من يستخدم REA ويبلّغ عن الأخطاء ويختبر الإصدارات ويساهم في إصلاحها. + morluto%2Frea | Trendshift **[الموقع](https://rea.tools/) · [الأدلة](https://rea.tools/guides/) · [دراسات الحالة](https://rea.tools/showcase/)** diff --git a/README_de.md b/README_de.md index e15c5a0d3ce..f646b05b865 100644 --- a/README_de.md +++ b/README_de.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30.000 GitHub-Sterne — vielen Dank!** + +Danke an alle, die REA nutzen, Fehler melden, Builds testen und Fehlerbehebungen beisteuern. + morluto%2Frea | Trendshift **[Website](https://rea.tools/) · [Anleitungen](https://rea.tools/guides/) · [Fallstudien](https://rea.tools/showcase/)** diff --git a/README_es.md b/README_es.md index 5b65d7347f7..c3ea22408ef 100644 --- a/README_es.md +++ b/README_es.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30.000 estrellas en GitHub — ¡gracias!** + +Gracias a todas las personas que usan REA, reportan errores, prueban las compilaciones y contribuyen con correcciones. + morluto%2Frea | Trendshift **[Sitio web](https://rea.tools/) · [Guías](https://rea.tools/guides/) · [Casos prácticos](https://rea.tools/showcase/)** diff --git a/README_fa.md b/README_fa.md index 246f64e6cf9..1390defac7e 100644 --- a/README_fa.md +++ b/README_fa.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **۳۰٬۰۰۰ ستاره در GitHub — سپاسگزاریم!** + +از همهٔ کسانی که از REA استفاده می‌کنند، خطاها را گزارش می‌دهند، نسخه‌ها را آزمایش می‌کنند و در رفع مشکلات مشارکت دارند، سپاسگزاریم. + morluto%2Frea | Trendshift **[وب‌سایت](https://rea.tools/) · [راهنماها](https://rea.tools/guides/) · [نمونه‌های عملی](https://rea.tools/showcase/)** diff --git a/README_fr.md b/README_fr.md index 304fa03466d..b5332444b7f 100644 --- a/README_fr.md +++ b/README_fr.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30 000 étoiles sur GitHub — merci !** + +Merci à toutes les personnes qui utilisent REA, signalent des bugs, testent les builds et contribuent aux corrections. + morluto%2Frea | Trendshift **[Site web](https://rea.tools/) · [Guides](https://rea.tools/guides/) · [Études de cas](https://rea.tools/showcase/)** diff --git a/README_ja.md b/README_ja.md index 245885ccb00..64489adafc2 100644 --- a/README_ja.md +++ b/README_ja.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **GitHub スターが 30,000 を突破しました。ありがとうございます!** + +REA を使い、問題を報告し、ビルドをテストし、修正を届けてくださる皆さんに感謝します。 + morluto%2Frea | Trendshift **[ウェブサイト](https://rea.tools/) · [ガイド](https://rea.tools/guides/) · [事例](https://rea.tools/showcase/)** diff --git a/README_ko.md b/README_ko.md index 9c17e0755c6..0125d485f68 100644 --- a/README_ko.md +++ b/README_ko.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **GitHub 스타 30,000개, 감사합니다!** + +REA를 사용하고, 버그를 제보하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. + morluto%2Frea | Trendshift **[웹사이트](https://rea.tools/) · [가이드](https://rea.tools/guides/) · [사례](https://rea.tools/showcase/)** diff --git a/README_pl.md b/README_pl.md index 81d85f62d63..14d76120824 100644 --- a/README_pl.md +++ b/README_pl.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30 000 gwiazdek na GitHubie — dziękujemy!** + +Dziękujemy wszystkim, którzy korzystają z REA, zgłaszają błędy, testują kompilacje i przesyłają poprawki. + morluto%2Frea | Trendshift **[Strona internetowa](https://rea.tools/) · [Przewodniki](https://rea.tools/guides/) · [Przykłady zastosowań](https://rea.tools/showcase/)** diff --git a/README_pt-BR.md b/README_pt-BR.md index 1dc863b00e0..d916cbcac55 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30.000 estrelas no GitHub — obrigado!** + +Obrigado a todos que usam o REA, relatam bugs, testam builds e contribuem com correções. + morluto%2Frea | Trendshift **[Site](https://rea.tools/) · [Guias](https://rea.tools/guides/) · [Casos de uso](https://rea.tools/showcase/)** diff --git a/README_ru.md b/README_ru.md index 9729c2483d8..2372a61efa5 100644 --- a/README_ru.md +++ b/README_ru.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30 000 звёзд на GitHub — спасибо!** + +Спасибо всем, кто использует REA, сообщает об ошибках, тестирует сборки и помогает с исправлениями. + morluto%2Frea | Trendshift **[Сайт](https://rea.tools/) · [Руководства](https://rea.tools/guides/) · [Примеры](https://rea.tools/showcase/)** diff --git a/README_th.md b/README_th.md index 5969984f1a7..4af19c9bd62 100644 --- a/README_th.md +++ b/README_th.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30,000 ดาวบน GitHub — ขอบคุณทุกคน!** + +ขอบคุณทุกคนที่ใช้ REA รายงานบั๊ก ทดสอบบิลด์ และร่วมแก้ไขปัญหา + morluto%2Frea | Trendshift **[เว็บไซต์](https://rea.tools/) · [คู่มือ](https://rea.tools/guides/) · [ตัวอย่างการใช้งาน](https://rea.tools/showcase/)** diff --git a/README_tr.md b/README_tr.md index 374b70463a4..644b4181f00 100644 --- a/README_tr.md +++ b/README_tr.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **GitHub'da 30.000 yıldız — teşekkürler!** + +REA'yı kullanan, hata bildiren, derlemeleri test eden ve düzeltmelere katkıda bulunan herkese teşekkürler. + morluto%2Frea | Trendshift **[Web sitesi](https://rea.tools/) · [Rehberler](https://rea.tools/guides/) · [Örnek çalışmalar](https://rea.tools/showcase/)** diff --git a/README_uk.md b/README_uk.md index c34d5afc484..2af0af864d7 100644 --- a/README_uk.md +++ b/README_uk.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30 000 зірок на GitHub — дякуємо!** + +Дякуємо всім, хто користується REA, повідомляє про помилки, тестує збірки та допомагає з виправленнями. + morluto%2Frea | Trendshift **[Сайт](https://rea.tools/) · [Посібники](https://rea.tools/guides/) · [Приклади](https://rea.tools/showcase/)** diff --git a/README_vi.md b/README_vi.md index bdb25d5f86b..b20a424b4e8 100644 --- a/README_vi.md +++ b/README_vi.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **30.000 sao trên GitHub — cảm ơn các bạn!** + +Cảm ơn tất cả những ai sử dụng REA, báo lỗi, thử nghiệm các bản dựng và đóng góp bản sửa lỗi. + morluto%2Frea | Trendshift **[Trang web](https://rea.tools/) · [Hướng dẫn](https://rea.tools/guides/) · [Các nghiên cứu điển hình](https://rea.tools/showcase/)** diff --git a/README_zh-TW.md b/README_zh-TW.md index 29f81d29c49..c17bdd3c79b 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **GitHub Star 突破 30,000,感謝大家!** + +感謝每一位使用 REA、回報問題、測試版本和貢獻修正的朋友。 + morluto%2Frea | Trendshift **[網站](https://rea.tools/) · [指南](https://rea.tools/guides/) · [案例](https://rea.tools/showcase/)** diff --git a/README_zh.md b/README_zh.md index a4f35c24228..a365dce3841 100644 --- a/README_zh.md +++ b/README_zh.md @@ -16,6 +16,10 @@ [![MIT license](https://img.shields.io/badge/license-MIT-f4c430?style=flat-square)](LICENSE) [![Discord](https://img.shields.io/discord/1556595354999332884?logo=discord&logoColor=white&label=Discord&color=5865F2)](https://discord.gg/GkcryMnJDM) +🎉 **GitHub Star 突破 30,000,感谢大家!** + +感谢每一位使用 REA、反馈问题、测试构建和贡献修复的朋友。 + morluto%2Frea | Trendshift **[网站](https://rea.tools/) · [指南](https://rea.tools/guides/) · [案例](https://rea.tools/showcase/)** From 9db1fd0bd7b17ed1dd21b9c72dabfa4ec6c23a3f Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:02:26 +0800 Subject: [PATCH 136/791] docs: thank feature request contributors --- README.md | 2 +- README_ar.md | 2 +- README_de.md | 2 +- README_es.md | 2 +- README_fa.md | 2 +- README_fr.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_pl.md | 2 +- README_pt-BR.md | 2 +- README_ru.md | 2 +- README_th.md | 2 +- README_tr.md | 2 +- README_uk.md | 2 +- README_vi.md | 2 +- README_zh-TW.md | 2 +- README_zh.md | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0a521f31069..28f95dbd26e 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ 🎉 **30,000 GitHub stars — thank you!** -Thanks to everyone using REA, reporting bugs, testing builds, and contributing fixes. +Thanks to everyone using REA, reporting bugs, requesting features, testing builds, and contributing fixes. morluto%2Frea | Trendshift diff --git a/README_ar.md b/README_ar.md index dbeb384d8f2..fe3dad406df 100644 --- a/README_ar.md +++ b/README_ar.md @@ -18,7 +18,7 @@ 🎉 **وصلنا إلى 30,000 نجمة على GitHub، شكرًا لكم!** -شكرًا لكل من يستخدم REA ويبلّغ عن الأخطاء ويختبر الإصدارات ويساهم في إصلاحها. +شكرًا لكل من يستخدم REA ويبلّغ عن الأخطاء ويقترح ميزات ويختبر الإصدارات ويساهم في إصلاحها. morluto%2Frea | Trendshift diff --git a/README_de.md b/README_de.md index f646b05b865..32b3fb7d86b 100644 --- a/README_de.md +++ b/README_de.md @@ -18,7 +18,7 @@ 🎉 **30.000 GitHub-Sterne — vielen Dank!** -Danke an alle, die REA nutzen, Fehler melden, Builds testen und Fehlerbehebungen beisteuern. +Danke an alle, die REA nutzen, Fehler melden, Funktionswünsche äußern, Builds testen und Fehlerbehebungen beisteuern. morluto%2Frea | Trendshift diff --git a/README_es.md b/README_es.md index c3ea22408ef..fd23701c0a0 100644 --- a/README_es.md +++ b/README_es.md @@ -18,7 +18,7 @@ 🎉 **30.000 estrellas en GitHub — ¡gracias!** -Gracias a todas las personas que usan REA, reportan errores, prueban las compilaciones y contribuyen con correcciones. +Gracias a todas las personas que usan REA, reportan errores, solicitan funciones, prueban las compilaciones y contribuyen con correcciones. morluto%2Frea | Trendshift diff --git a/README_fa.md b/README_fa.md index 1390defac7e..09c9ac2e8a4 100644 --- a/README_fa.md +++ b/README_fa.md @@ -18,7 +18,7 @@ 🎉 **۳۰٬۰۰۰ ستاره در GitHub — سپاسگزاریم!** -از همهٔ کسانی که از REA استفاده می‌کنند، خطاها را گزارش می‌دهند، نسخه‌ها را آزمایش می‌کنند و در رفع مشکلات مشارکت دارند، سپاسگزاریم. +از همهٔ کسانی که از REA استفاده می‌کنند، خطاها را گزارش می‌دهند، قابلیت‌های جدید پیشنهاد می‌کنند، نسخه‌ها را آزمایش می‌کنند و در رفع مشکلات مشارکت دارند، سپاسگزاریم. morluto%2Frea | Trendshift diff --git a/README_fr.md b/README_fr.md index b5332444b7f..c7626a63d60 100644 --- a/README_fr.md +++ b/README_fr.md @@ -18,7 +18,7 @@ 🎉 **30 000 étoiles sur GitHub — merci !** -Merci à toutes les personnes qui utilisent REA, signalent des bugs, testent les builds et contribuent aux corrections. +Merci à toutes les personnes qui utilisent REA, signalent des bugs, proposent des fonctionnalités, testent les builds et contribuent aux corrections. morluto%2Frea | Trendshift diff --git a/README_ja.md b/README_ja.md index 64489adafc2..a2c14ba28ee 100644 --- a/README_ja.md +++ b/README_ja.md @@ -18,7 +18,7 @@ 🎉 **GitHub スターが 30,000 を突破しました。ありがとうございます!** -REA を使い、問題を報告し、ビルドをテストし、修正を届けてくださる皆さんに感謝します。 +REA を使い、問題を報告し、機能の要望を寄せ、ビルドをテストし、修正を届けてくださる皆さんに感謝します。 morluto%2Frea | Trendshift diff --git a/README_ko.md b/README_ko.md index 0125d485f68..7afe75df68e 100644 --- a/README_ko.md +++ b/README_ko.md @@ -18,7 +18,7 @@ 🎉 **GitHub 스타 30,000개, 감사합니다!** -REA를 사용하고, 버그를 제보하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. +REA를 사용하고, 버그를 제보하고, 기능을 요청하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다. morluto%2Frea | Trendshift diff --git a/README_pl.md b/README_pl.md index 14d76120824..39b397735fd 100644 --- a/README_pl.md +++ b/README_pl.md @@ -18,7 +18,7 @@ 🎉 **30 000 gwiazdek na GitHubie — dziękujemy!** -Dziękujemy wszystkim, którzy korzystają z REA, zgłaszają błędy, testują kompilacje i przesyłają poprawki. +Dziękujemy wszystkim, którzy korzystają z REA, zgłaszają błędy, proponują nowe funkcje, testują kompilacje i przesyłają poprawki. morluto%2Frea | Trendshift diff --git a/README_pt-BR.md b/README_pt-BR.md index d916cbcac55..ce843976363 100644 --- a/README_pt-BR.md +++ b/README_pt-BR.md @@ -18,7 +18,7 @@ 🎉 **30.000 estrelas no GitHub — obrigado!** -Obrigado a todos que usam o REA, relatam bugs, testam builds e contribuem com correções. +Obrigado a todos que usam o REA, relatam bugs, sugerem funcionalidades, testam builds e contribuem com correções. morluto%2Frea | Trendshift diff --git a/README_ru.md b/README_ru.md index 2372a61efa5..585632910ba 100644 --- a/README_ru.md +++ b/README_ru.md @@ -18,7 +18,7 @@ 🎉 **30 000 звёзд на GitHub — спасибо!** -Спасибо всем, кто использует REA, сообщает об ошибках, тестирует сборки и помогает с исправлениями. +Спасибо всем, кто использует REA, сообщает об ошибках, предлагает новые функции, тестирует сборки и помогает с исправлениями. morluto%2Frea | Trendshift diff --git a/README_th.md b/README_th.md index 4af19c9bd62..901f954354a 100644 --- a/README_th.md +++ b/README_th.md @@ -18,7 +18,7 @@ 🎉 **30,000 ดาวบน GitHub — ขอบคุณทุกคน!** -ขอบคุณทุกคนที่ใช้ REA รายงานบั๊ก ทดสอบบิลด์ และร่วมแก้ไขปัญหา +ขอบคุณทุกคนที่ใช้ REA รายงานบั๊ก เสนอฟีเจอร์ ทดสอบบิลด์ และร่วมแก้ไขปัญหา morluto%2Frea | Trendshift diff --git a/README_tr.md b/README_tr.md index 644b4181f00..558f21c8d66 100644 --- a/README_tr.md +++ b/README_tr.md @@ -18,7 +18,7 @@ 🎉 **GitHub'da 30.000 yıldız — teşekkürler!** -REA'yı kullanan, hata bildiren, derlemeleri test eden ve düzeltmelere katkıda bulunan herkese teşekkürler. +REA'yı kullanan, hata bildiren, özellik talep eden, derlemeleri test eden ve düzeltmelere katkıda bulunan herkese teşekkürler. morluto%2Frea | Trendshift diff --git a/README_uk.md b/README_uk.md index 2af0af864d7..0c77c2a05b4 100644 --- a/README_uk.md +++ b/README_uk.md @@ -18,7 +18,7 @@ 🎉 **30 000 зірок на GitHub — дякуємо!** -Дякуємо всім, хто користується REA, повідомляє про помилки, тестує збірки та допомагає з виправленнями. +Дякуємо всім, хто користується REA, повідомляє про помилки, пропонує нові функції, тестує збірки та допомагає з виправленнями. morluto%2Frea | Trendshift diff --git a/README_vi.md b/README_vi.md index b20a424b4e8..ff4a6a61d6c 100644 --- a/README_vi.md +++ b/README_vi.md @@ -18,7 +18,7 @@ 🎉 **30.000 sao trên GitHub — cảm ơn các bạn!** -Cảm ơn tất cả những ai sử dụng REA, báo lỗi, thử nghiệm các bản dựng và đóng góp bản sửa lỗi. +Cảm ơn tất cả những ai sử dụng REA, báo lỗi, đề xuất tính năng, thử nghiệm các bản dựng và đóng góp bản sửa lỗi. morluto%2Frea | Trendshift diff --git a/README_zh-TW.md b/README_zh-TW.md index c17bdd3c79b..8cfce823f24 100644 --- a/README_zh-TW.md +++ b/README_zh-TW.md @@ -18,7 +18,7 @@ 🎉 **GitHub Star 突破 30,000,感謝大家!** -感謝每一位使用 REA、回報問題、測試版本和貢獻修正的朋友。 +感謝每一位使用 REA、回報問題、提出功能需求、測試版本和貢獻修正的朋友。 morluto%2Frea | Trendshift diff --git a/README_zh.md b/README_zh.md index a365dce3841..61d262ebda5 100644 --- a/README_zh.md +++ b/README_zh.md @@ -18,7 +18,7 @@ 🎉 **GitHub Star 突破 30,000,感谢大家!** -感谢每一位使用 REA、反馈问题、测试构建和贡献修复的朋友。 +感谢每一位使用 REA、反馈问题、提出功能需求、测试构建和贡献修复的朋友。 morluto%2Frea | Trendshift From 7e9c91199ecd133d26a51cb33b6e63e1f942df7e Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:28:48 +0800 Subject: [PATCH 137/791] fix(cli): resolve xrefs symbol selectors after address validation (#1209) * fix(cli): resolve xrefs symbol selectors after address validation * style(cli): format xrefs fallback call --- scripts/lib/real-ghidra-entry-aliases.mjs | 64 +++++ src/application/DirectAnalysis.ts | 31 ++- src/application/XrefsAddressResolution.ts | 81 ++++++ tests/composition/directAnalysisXrefs.test.ts | 238 ++++++++++++++++++ tests/conformance/ghidra/entry-aliases.c | 5 +- 5 files changed, 417 insertions(+), 2 deletions(-) create mode 100644 src/application/XrefsAddressResolution.ts create mode 100644 tests/composition/directAnalysisXrefs.test.ts diff --git a/scripts/lib/real-ghidra-entry-aliases.mjs b/scripts/lib/real-ghidra-entry-aliases.mjs index 3230ed6d1c2..5a38c70610d 100644 --- a/scripts/lib/real-ghidra-entry-aliases.mjs +++ b/scripts/lib/real-ghidra-entry-aliases.mjs @@ -45,6 +45,21 @@ export async function verifyGhidraEntryAliases({ ({ address }) => address === alias.address, ); assert.ok(canonical, "Fixture alias is not at a function entry"); + const xrefsSelectors = []; + for (const value of [ + canonical.value, + alias.value, + `${prefix}rea_xrefs_data`, + `${prefix}rea_xrefs_unreferenced`, + ]) { + const symbol = names.find((item) => item.value === value); + assert.ok(symbol, `Missing xrefs selector ${value}`); + const expected = await call("xrefs", { address: symbol.address }); + if (value === `${prefix}rea_xrefs_unreferenced`) + assert.deepEqual(expected, []); + else assert.ok(expected.length > 0, `Missing references for ${value}`); + xrefsSelectors.push({ symbol, expected }); + } for (const procedure of [ alias.value, "dead", @@ -118,6 +133,55 @@ export async function verifyGhidraEntryAliases({ JSON.parse(stdout).normalized_result.procedure.address, alias.address, ); + for (const { symbol, expected } of xrefsSelectors) { + for (const selector of [ + [symbol.value], + ["--address", symbol.value], + [symbol.address], + ]) { + const { stdout } = await exec( + process.execPath, + [ + entrypoint, + "xrefs", + path, + ...selector, + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const evidence = JSON.parse(stdout); + assert.deepEqual(evidence.normalized_result, expected); + assert.equal(evidence.parameters.address, selector.at(-1)); + } + } + await assert.rejects( + exec( + process.execPath, + [ + entrypoint, + "xrefs", + path, + "rea_missing_symbol", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ), + (error) => { + const failure = JSON.parse(error.stdout); + assert.equal(failure.code, "invalid_request"); + assert.equal(failure.details.operation, "xrefs"); + assert.match( + JSON.stringify(failure.details.issues), + /Unknown analyzed symbol name.*rea_missing_symbol/u, + ); + return true; + }, + ); assert.deepEqual(await readFile(path), bytes); await call("open_binary", { path: target.path, provider_id: "ghidra" }); } finally { diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index f9f15244ffb..1f66ec8c84d 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -48,6 +48,8 @@ import { } from "./WorkflowEvidence.js"; import type { AnalysisProviderSelector } from "../contracts/providerSelection.js"; import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; +import { resolveXrefsAddress } from "./XrefsAddressResolution.js"; +import { AnalysisInputError } from "../domain/analysisErrorCore.js"; type DirectAnalysisTool = | "annotate_native_function" @@ -356,7 +358,34 @@ const executeAnalysisTool = async (input: { }); return { output: evidence, evidence }; } - const result = await session.execute(tool, input.arguments, { signal }); + let result = await session.execute(tool, input.arguments, { signal }); + if ( + tool === "xrefs" && + typeof input.arguments.address === "string" && + !result.ok && + result.error instanceof AnalysisInputError && + !result.error.cleanupIncomplete && + result.error.issues.length > 0 && + result.error.issues.every( + ({ path, reason }) => + path.length === 1 && + path[0] === "address" && + reason === "invalid_format", + ) + ) { + const address = await resolveXrefsAddress( + session, + input.arguments.address, + signal, + ); + if (!address.ok) return { output: cliError(address.error) }; + if (address.value !== input.arguments.address) + result = await session.execute( + tool, + { ...input.arguments, address: address.value }, + { signal }, + ); + } if (!result.ok) return { output: cliError(result.error) }; const evidence = createEvidence( result.value.subject ?? input.openedTarget, diff --git a/src/application/XrefsAddressResolution.ts b/src/application/XrefsAddressResolution.ts new file mode 100644 index 00000000000..c7eef8bdc81 --- /dev/null +++ b/src/application/XrefsAddressResolution.ts @@ -0,0 +1,81 @@ +import { addressedValue } from "../contracts/toolOutputSchemaPrimitives.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisInputError, + AnalysisOutputError, +} from "../domain/analysisErrorCore.js"; +import { err, ok, type Result } from "../domain/result.js"; +import type { AnalysisOperationPort } from "./AnalysisProvider.js"; +import { resolveProcedureAddress } from "./ProcedureAddressResolution.js"; + +/** Resolve a CLI symbol selector without changing existing address spellings. */ +export const resolveXrefsAddress = async ( + analysis: AnalysisOperationPort, + selector: string, + signal: AbortSignal, +): Promise> => { + if ( + selector.length === 0 || + /^[0-9a-f]+$/iu.test(selector) || + /^0x/iu.test(selector) || + /:0x/iu.test(selector) + ) + return ok(selector); + + const inventory = await analysis.execute("list_names", {}, { signal }); + if (!inventory.ok) { + if ( + inventory.error instanceof AnalysisCapabilityUnavailableError && + inventory.error.operation === "list_names" && + !inventory.error.cleanupIncomplete + ) + return resolveProcedureAddress( + async (operation, parameters, signal) => { + const result = await analysis.execute( + operation, + parameters, + signal === undefined ? {} : { signal }, + ); + return result.ok ? ok(result.value.result) : result; + }, + selector, + signal, + ); + return inventory; + } + if (!Array.isArray(inventory.value.result)) + return err( + new AnalysisOutputError( + "list_names", + "Expected a symbol inventory array", + ), + ); + + const addresses = new Set(); + for (const [index, item] of inventory.value.result.entries()) { + const parsed = addressedValue.safeParse(item); + if (!parsed.success || parsed.data.address.length === 0) + return err( + new AnalysisOutputError( + "list_names", + `Invalid symbol inventory entry at index ${index}`, + ), + ); + if (parsed.data.value === selector) addresses.add(parsed.data.address); + } + const [address] = addresses; + if (addresses.size === 1 && address !== undefined) return ok(address); + return err( + new AnalysisInputError("xrefs", undefined, [ + { + path: ["address"], + reason: "invalid_value", + message: + addresses.size === 0 + ? `Unknown analyzed symbol name ${JSON.stringify(selector)}. Use an exact analyzed name or a hexadecimal address.` + : `Analyzed symbol name ${JSON.stringify(selector)} is ambiguous at ${[...addresses].join(", ")}. Select an exact address.`, + }, + ]), + ); +}; diff --git a/tests/composition/directAnalysisXrefs.test.ts b/tests/composition/directAnalysisXrefs.test.ts new file mode 100644 index 00000000000..5e1d998a603 --- /dev/null +++ b/tests/composition/directAnalysisXrefs.test.ts @@ -0,0 +1,238 @@ +import { describe, expect, it } from "vitest"; +import { runDirectAnalysis } from "../../src/application/DirectAnalysis.js"; +import type { DirectAnalysisDependencies } from "../../src/application/DirectAnalysisDependencies.js"; +import type { AnalysisError } from "../../src/domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisCapabilityUnavailableError, + AnalysisInputError, +} from "../../src/domain/analysisErrorCore.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { err } from "../../src/domain/result.js"; +import { observed } from "../fixtures/analysisExecution.js"; +import { + createBinarySessionTargets, + createTestBinarySession, +} from "../fixtures/binarySession.js"; + +const inventory = [ + { address: "0x1000", value: "table_only" }, + { address: "0x1000", value: "entry_alias" }, + { address: "0x1000", value: "namespace::function" }, + { address: "0x2000", value: "data_symbol" }, + { address: "0x3000", value: "unreferenced" }, + { address: "0x1000", value: "same_address" }, + { address: "0x1000", value: "same_address" }, + { address: "0x1000", value: "ambiguous" }, + { address: "0x2000", value: "ambiguous" }, +]; + +const fixture = ( + options: { + readonly inventoryError?: AnalysisError; + readonly malformedInventory?: boolean; + readonly acceptsNamedSelectors?: boolean; + readonly xrefsError?: AnalysisInputError; + } = {}, +) => { + let closed = 0; + const createSession = () => + createTestBinarySession(() => ({ + execute: (operation, parameters) => { + if (operation === "list_names") + return Promise.resolve( + options.inventoryError !== undefined + ? err(options.inventoryError) + : observed( + options.malformedInventory ? "not an inventory" : inventory, + ), + ); + if (operation === "procedure_address") + return Promise.resolve(observed("0x1000")); + if (operation === "xrefs") { + if (options.xrefsError !== undefined) + return Promise.resolve(err(options.xrefsError)); + const address = parameters.address; + if ( + !options.acceptsNamedSelectors && + (typeof address !== "string" || + !/^(?:(?:0[xX])?[0-9a-fA-F]+|space%20name:0[xX][0-9a-fA-F]+)$/u.test( + address, + )) + ) + return Promise.resolve( + err( + new AnalysisInputError("xrefs", undefined, [ + { path: ["address"], reason: "invalid_format" }, + ]), + ), + ); + return Promise.resolve( + observed(address === "0x3000" ? [] : ["0x4000"]), + ); + } + return Promise.resolve(observed(null)); + }, + close: () => { + closed += 1; + return Promise.resolve(); + }, + })); + const dependencies: DirectAnalysisDependencies = { + createBinarySession: createSession, + createManagedBinarySession: createSession, + }; + return { dependencies, closed: () => closed }; +}; + +describe("direct CLI cross-reference selectors", () => { + it("retains validation issues unrelated to symbol resolution", async () => { + const [path] = await createBinarySessionTargets(); + const issues = [ + { path: ["address"], reason: "invalid_format" }, + { path: ["extra"], reason: "unknown_argument" }, + ] as const; + const provider = fixture({ + xrefsError: new AnalysisInputError("xrefs", undefined, issues), + }); + const result = await runDirectAnalysis( + provider.dependencies, + path, + "xrefs", + { address: "table_only", extra: true }, + ); + expect(result).toMatchObject({ + code: "invalid_request", + details: { operation: "xrefs", issues }, + }); + expect(provider.closed()).toBe(1); + }); + it.each(["hidden_alias", "dead"])( + "retains the provider's existing interpretation of %s", + async (address) => { + const [path] = await createBinarySessionTargets(); + const provider = fixture({ + acceptsNamedSelectors: true, + malformedInventory: true, + }); + const evidence = parseEvidence( + await runDirectAnalysis(provider.dependencies, path, "xrefs", { + address, + }), + ); + expect(evidence.parameters).toEqual({ address }); + expect(evidence.normalized_result).toEqual(["0x4000"]); + expect(provider.closed()).toBe(1); + }, + ); + it.each([ + "table_only", + "entry_alias", + "namespace::function", + "data_symbol", + "same_address", + "0x1000", + "1000", + "0X1000", + "space%20name:0x1000", + ])( + "inspects %s and retains the original selector in Evidence", + async (address) => { + const [path] = await createBinarySessionTargets(); + const provider = fixture(); + const evidence = parseEvidence( + await runDirectAnalysis(provider.dependencies, path, "xrefs", { + address, + }), + ); + expect(evidence.parameters).toEqual({ address }); + expect(evidence.normalized_result).toEqual(["0x4000"]); + expect(provider.closed()).toBe(1); + }, + ); + + it("distinguishes a resolved unreferenced symbol from an unknown name", async () => { + const [path] = await createBinarySessionTargets(); + const provider = fixture(); + const empty = parseEvidence( + await runDirectAnalysis(provider.dependencies, path, "xrefs", { + address: "unreferenced", + }), + ); + expect(empty.normalized_result).toEqual([]); + const missing = await runDirectAnalysis( + provider.dependencies, + path, + "xrefs", + { + address: "missing_symbol", + }, + ); + expect(missing).toMatchObject({ + code: "invalid_request", + details: { operation: "xrefs", issues: [{ path: ["address"] }] }, + }); + expect(JSON.stringify(missing)).toContain("missing_symbol"); + expect(provider.closed()).toBe(2); + }); + + it("reports every candidate for an ambiguous exact name", async () => { + const [path] = await createBinarySessionTargets(); + const provider = fixture(); + const result = await runDirectAnalysis( + provider.dependencies, + path, + "xrefs", + { + address: "ambiguous", + }, + ); + expect(result).toMatchObject({ code: "invalid_request" }); + expect(JSON.stringify(result)).toContain("0x1000"); + expect(JSON.stringify(result)).toContain("0x2000"); + expect(provider.closed()).toBe(1); + }); + + it("uses procedure resolution when the provider has no symbol inventory", async () => { + const [path] = await createBinarySessionTargets(); + const provider = fixture({ + inventoryError: new AnalysisCapabilityUnavailableError( + "fixture", + "list_names", + "only procedure inventory is supported", + ), + }); + const evidence = parseEvidence( + await runDirectAnalysis(provider.dependencies, path, "xrefs", { + address: "table_only", + }), + ); + expect(evidence.parameters).toEqual({ address: "table_only" }); + expect(evidence.normalized_result).toEqual(["0x4000"]); + expect(provider.closed()).toBe(1); + }); + + it.each([ + { + options: { inventoryError: new AnalysisCancelledError("list_names") }, + code: "cancelled", + }, + { options: { malformedInventory: true }, code: "unreadable_output" }, + ])( + "preserves $code from inventory resolution and releases the session", + async ({ options, code }) => { + const [path] = await createBinarySessionTargets(); + const provider = fixture(options); + const result = await runDirectAnalysis( + provider.dependencies, + path, + "xrefs", + { + address: "table_only", + }, + ); + expect(result).toMatchObject({ code }); + expect(provider.closed()).toBe(1); + }, + ); +}); diff --git a/tests/conformance/ghidra/entry-aliases.c b/tests/conformance/ghidra/entry-aliases.c index 3f99820445a..25493f73773 100644 --- a/tests/conformance/ghidra/entry-aliases.c +++ b/tests/conformance/ghidra/entry-aliases.c @@ -5,10 +5,13 @@ #define REA_SYMBOL(name) name #endif +volatile int rea_xrefs_data = 17; +__attribute__((used)) volatile int rea_xrefs_unreferenced = 41; + __attribute__((noinline, used)) int rea_alias_target(void) { __asm__ volatile(".globl " REA_SYMBOL("rea_interior") "\n" REA_SYMBOL("rea_interior") ":"); - return 17; + return rea_xrefs_data; } __asm__(".globl " REA_SYMBOL("rea_entry_alias") "\n" From c94f06d98f895eb8b6c5ffa5e999ef1d4bb5eea2 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Fri, 9 Oct 2026 14:03:00 +0800 Subject: [PATCH 138/791] feat(website): add homepage navigation and faster reading paths --- website/README.md | 11 +- website/public/assets/home.js | 74 ++++++++++++ website/public/assets/styles.css | 141 ++++++++++++++++++++++ website/public/index.html | 195 ++++++------------------------- website/style-guide.md | 7 ++ 5 files changed, 266 insertions(+), 162 deletions(-) diff --git a/website/README.md b/website/README.md index c011e643547..6f9af6887ed 100644 --- a/website/README.md +++ b/website/README.md @@ -5,7 +5,7 @@ > — N0zoM1z0 An English static website with explanatory figures, worked guides and DX-Ball, Notion, TH04 and CTF investigations. -The public files are in `website/public/`. The site uses HTML, CSS and a small +The public files are in `website/public/`. The site uses HTML, CSS and small scripts for copying code, following code comparisons and playing the dinosaur speed reconstruction. Python packages the downloadable example; there is no frontend bundler or npm dependency. @@ -26,7 +26,7 @@ Open . Refresh the browser after editing a file. ## Pages -- `public/index.html`: compact setup, reverse-engineering definition and purpose, manual/agent comparison, dinosaur and Calculator examples, and case-study overviews. +- `public/index.html`: compact setup, reverse-engineering definition and purpose, manual/agent comparison, dinosaur and Calculator examples, and analysis-guide links. - `public/examples/dino-lab/index.html`: adjustable-speed mini-game, recovered rule, original-game check and browser analysis steps. - `public/first-investigation/index.html`: a guided Notes export investigation, from setup to a checked CSV prediction. - `public/showcase/index.html`: the case-study index. @@ -100,6 +100,13 @@ and complete private Evidence records are not website assets. The homepage starts with a short installation prompt and the ordinary setup command, then explains what reverse engineering is and why someone would use it. The manual/agent comparison introduces REA’s role before the examples. +Its opening links directly to Showcases and lets experienced readers skip to +the analysis guides. Case-study previews live on the Showcases page. +A right-side table of contents stays visible at widths of 1440px and above. +On narrower screens it becomes a sticky, native disclosure; selecting a link +closes the menu and focuses the destination. The homepage script follows +the reading position and marks the current link. Without JavaScript, the +menu stays open and its fragment links remain usable. The comparison uses the same Calculator question on both sides: three selectable manual steps pair original instructions with an analyst illustration and thought bubbles; the agent side begins with “One prompt.” and a copyable diff --git a/website/public/assets/home.js b/website/public/assets/home.js index 27cd1b32f22..c5e102da603 100644 --- a/website/public/assets/home.js +++ b/website/public/assets/home.js @@ -62,5 +62,79 @@ function initializeExpertAssembly() { } } +function initializeHomeNavigation() { + const menu = document.querySelector("[data-home-nav]"); + if (menu === null) return; + const currentLabel = menu.querySelector("[data-home-nav-current]"); + if (currentLabel === null) return; + const wideScreen = window.matchMedia("(min-width: 1440px)"); + const entries = Array.from(menu.querySelectorAll('a[href^="#"]')) + .map((link) => ({ + link, + target: document.getElementById(link.getAttribute("href").slice(1)), + })) + .filter((entry) => entry.target !== null); + if (entries.length === 0) return; + + const updatePosition = () => { + const offset = wideScreen.matches ? 40 : 88; + let current = entries[0]; + entries.forEach((entry) => { + if (entry.target.getBoundingClientRect().top <= offset) current = entry; + }); + if ( + window.scrollY + window.innerHeight >= + document.documentElement.scrollHeight - 2 + ) { + const selected = entries.find( + (entry) => entry.link.hash === window.location.hash, + ); + const selectedTop = selected?.target.getBoundingClientRect().top; + current = + selectedTop !== undefined && + selectedTop >= 0 && + selectedTop < window.innerHeight + ? selected + : entries[entries.length - 1]; + } + entries.forEach((entry) => { + if (entry === current) + entry.link.setAttribute("aria-current", "location"); + else entry.link.removeAttribute("aria-current"); + }); + currentLabel.textContent = current.link.textContent; + }; + const updateLayout = () => { + menu.open = wideScreen.matches; + updatePosition(); + }; + let scheduled = false; + const schedulePosition = () => { + if (scheduled) return; + scheduled = true; + window.requestAnimationFrame(() => { + scheduled = false; + updatePosition(); + }); + }; + entries.forEach(({ link, target }) => { + link.addEventListener("click", () => { + if (!wideScreen.matches) menu.open = false; + const heading = target.querySelector("h2") ?? target; + heading.setAttribute("tabindex", "-1"); + window.requestAnimationFrame(() => + heading.focus({ preventScroll: true }), + ); + }); + }); + wideScreen.addEventListener("change", updateLayout); + window.addEventListener("scroll", schedulePosition, { passive: true }); + window.addEventListener("resize", schedulePosition, { passive: true }); + window.addEventListener("hashchange", schedulePosition); + window.addEventListener("load", schedulePosition); + updateLayout(); +} + initializeExpertAssembly(); initializePercentDemo(); +initializeHomeNavigation(); diff --git a/website/public/assets/styles.css b/website/public/assets/styles.css index 3bcc84d9e05..fc9850f6961 100644 --- a/website/public/assets/styles.css +++ b/website/public/assets/styles.css @@ -1218,6 +1218,137 @@ pre code { border: 0; } +.home-page-nav { + position: sticky; + top: 0; + z-index: 6; + background: var(--page); + border-bottom: 1px solid var(--line); +} +.home-page-nav > summary { + display: flex; + align-items: center; + gap: 0.75rem; + min-height: 52px; + color: var(--ink); + font-size: 0.8rem; + font-weight: 600; + list-style: none; + cursor: pointer; +} +.home-page-nav > summary::-webkit-details-marker { + display: none; +} +.home-nav-current { + margin-left: auto; + color: var(--muted); + font-weight: 400; +} +.home-page-nav[open] > summary svg { + transform: rotate(180deg); +} +.home-nav-links { + position: absolute; + inset: 100% 0 auto; + max-height: calc(100svh - 5rem); + overflow-y: auto; + padding-block: 0.4rem 1rem; + background: var(--page); + border-bottom: 1px solid var(--line); +} +.home-nav-title { + display: none; +} +.home-nav-links ul { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.2rem 1rem; + margin: 0; + padding: 0; + list-style: none; +} +.home-nav-links li + li { + margin: 0; +} +.home-nav-links a { + display: flex; + align-items: center; + min-height: 44px; + padding: 0.5rem 0.65rem; + border-left: 2px solid transparent; + color: var(--muted); + font-size: 0.825rem; + line-height: 1.5; + text-decoration: none; +} +.home-nav-links a:hover { + color: var(--ink); + background: var(--soft); +} +.home-nav-links a[aria-current="location"] { + border-left-color: var(--accent); + background: #f1f5fa; + color: var(--accent); + font-weight: 600; +} +.home-page section[id], +.home-role-comparison[id] { + scroll-margin-top: 5.25rem; +} +.home-expert-shortcut { + margin-block: 1.5rem 0; + color: var(--muted); + font-size: 0.85rem; +} +.home-expert-shortcut a { + white-space: nowrap; +} +@media (min-width: 1440px) { + .home-page-nav { + position: fixed; + top: 7.5rem; + right: max(1.5rem, calc((100vw - 1080px) / 2 - 156px)); + width: 132px; + max-height: calc(100svh - 9rem); + overflow-y: auto; + border: 0; + z-index: 2; + } + .home-page-nav > summary { + display: none; + } + .home-nav-links { + position: static; + max-height: none; + overflow: visible; + padding: 0; + border: 0; + } + .home-nav-links .container { + width: 100%; + } + .home-nav-title { + display: block; + margin-bottom: 0.7rem; + padding-left: 0.75rem; + color: var(--muted); + font-size: 0.75rem; + font-weight: 600; + } + .home-nav-links ul { + display: block; + border-left: 1px solid var(--line); + } + .home-nav-links a { + margin-left: -1px; + font-size: 0.78rem; + } + .home-page section[id], + .home-role-comparison[id] { + scroll-margin-top: 2rem; + } +} + .home-start-grid { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1fr); @@ -1765,6 +1896,16 @@ pre code { } } @media (max-width: 640px) { + .home-start-grid .actions { + display: grid; + grid-template-columns: max-content max-content; + justify-content: start; + gap: 0.9rem 1.4rem; + } + .home-start-grid .actions .button { + grid-column: 1 / -1; + justify-self: start; + } .expert-analysis-grid { grid-template-columns: minmax(0, 1fr); } diff --git a/website/public/index.html b/website/public/index.html index 26f8d2b0a8e..e3c9a7b0e56 100644 --- a/website/public/index.html +++ b/website/public/index.html @@ -15,7 +15,7 @@ - + +
      + + On this page + + + + +
      -
      +

      Reverse engineering, with your coding agent

      @@ -64,7 +89,12 @@

      Find out how
      software works.

      +

      + Already know RE? + Skip to analysis guides → +

      Set up REA

      @@ -132,7 +162,7 @@

      What is reverse engineering?

      One question: why does Calculator give 220 for 200 + 10%?

      -
      +
      Try REA on a small app.
      -
      +
      -

      What REA can analyze

      +

      Analyze your software

      Choose a guide for your binary, JavaScript app or browser session.

      @@ -866,161 +896,6 @@

      Browser & runtime activity

      - -
      -
      -
      -

      Case study · Native

      -

      Rebuilding DX-Ball

      -

      - DX-Ball is a 1996 Windows game. The project is reconstructing it - in C, using REA to inspect the executable and independent tests to - check the recovered behavior. -

      -

      - Follow the investigation -

      -

      - Work so far covers boards, resources, brick hits, animations, - particles and bonus production. The playable game is still in - progress. -

      -
      -
      -
      -
      55
      -
      functions in maintained C
      -
      -
      -
      45,380
      -
      original-x86 differential cases
      -
      -
      -
      33
      -
      complete functions with matching compiled bytes
      -
      -
      -
      -
      - -
      -
      -
      -

      Case study · Electron

      -

      Notion's Electron clipboard

      -

      - A copy operation crosses from the page to Electron's main process. - Inspect Notion's packaged JavaScript with REA, then follow one - channel through the wrappers to the system clipboard. -

      -

      - Short code excerpts show each handoff. A second figure explains - how an HTML marker connects ordinary clipboard text to Notion's - structured block data. -

      - Follow the clipboard bridge -
      -
      -

      Clipboard IPC channel

      -
      -
      Preload · send
      -
      invokerInMain(
      -  "notion:clipboard:write"
      -)
      -
      -
      -
      Main process · receive
      -
      addListener(
      -  "notion:clipboard:write",
      -  handler
      -)
      -
      -
      - Matching names give the investigation a path to follow. -
      -
      -
      -
      - -
      -
      -
      -

      Case study · PC-98 DOS

      -

      TH04's aimed bullet rings

      -

      - Inspect a 16-bit DOS function with REA. Follow its angle - calculation from original instructions to readable C++, then see - how adding the player's direction turns a ring of bullets. -

      - See the bullet calculation -
      -
      -

      Fixed and aimed rings

      - TH04's fixed and aimed 16-bullet rings side by side. The first bullet is highlighted in blue; adding the player direction rotates every bullet while preserving the spacing. -
      - Adding the player's direction rotates every bullet by the same - angle. -
      -
      -
      -
      - -
      -
      -
      -

      Case study · CTF

      -

      Solve a CTF flag checker

      -

      - Use REA to inspect an easy DownUnderCTF binary. Extract its - checking rules, solve for the flag and confirm that the original - program accepts it. -

      - See the CTF solution -
      -
      -

      One check reveals one character

      -
      -
      - From REA's code and data -
      -
      Mask: skip 21, select 1, skip 14
      -Target: 55
      -
      -code[21] = 55  →  '7'
      -
      -
      - Each mask selects characters whose codes must add up to a stored - number. This one selects a single character. -
      -
      -
      -
      - -
      -
      -
      -

      What would you like to understand?

      -

      - Connect REA to your coding agent, or run your first analysis from - the terminal. -

      -
      - Get started -
      -