From 407dba25e0c9644473835b8fe2b4d1d2e0066ce0 Mon Sep 17 00:00:00 2001 From: hawkff <109485367+hawkff@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:20:36 -0500 Subject: [PATCH 1/5] Support GPT Image 2.6 Register GPT Image 2.6 Sunburst, Flare, and the subscription-only plain model from CLIProxyAPI 8.0.56, and make 2.6 Sunburst the automatic OpenAI image default. Add complexity, GIF, and SVG output for 2.6 Sunburst and Flare, and markup edits for 2.5 and 2.6 Sunburst and Flare. Update Biome to 2.6.5 and pnpm/action-setup to 6.1.1. --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- biome.json | 2 +- package.json | 4 +- pnpm-lock.yaml | 92 +++++++-------- src/catalog.ts | 44 +++++--- src/media-defaults.ts | 2 +- src/media.ts | 180 +++++++++++++++++++++++------ test/auth-routes.test.ts | 29 ++--- test/media.test.ts | 207 +++++++++++++++++++++++++++++++++- test/picker.test.ts | 8 +- test/provider.test.ts | 7 ++ 12 files changed, 459 insertions(+), 120 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d78904d..7de1023 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: ${{ matrix.node }} - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 with: version: 12.16.1 - run: pnpm install --frozen-lockfile --ignore-scripts diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 971f3ac..cc736a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: 24 - - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + - uses: pnpm/action-setup@73e46992e05b19abd31f2f3c20e0e7370d762115 # v6.1.1 with: version: 12.16.1 - run: test "v$(jq -r .version package.json)" = "$GITHUB_REF_NAME" diff --git a/biome.json b/biome.json index 7089237..a11038a 100644 --- a/biome.json +++ b/biome.json @@ -1,5 +1,5 @@ { - "$schema": "https://biomejs.dev/schemas/2.6.4/schema.json", + "$schema": "https://biomejs.dev/schemas/2.6.5/schema.json", "vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true }, "formatter": { "indentStyle": "space", "indentWidth": 2, "lineWidth": 110 }, "linter": { "enabled": true } diff --git a/package.json b/package.json index 4b953c3..e68f7a5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-cliproxyapi-native", - "version": "0.3.4", + "version": "0.3.5", "description": "CLIProxyAPI chat discovery, image generation and editing, and video generation with Pi's native adapters", "publishConfig": { "access": "public" @@ -38,7 +38,7 @@ "check": "pnpm typecheck && pnpm lint && pnpm test" }, "devDependencies": { - "@biomejs/biome": "^2.6.4", + "@biomejs/biome": "^2.6.5", "@earendil-works/pi-ai": "^1.5.0", "@earendil-works/pi-coding-agent": "^1.5.0", "@earendil-works/pi-tui": "^1.5.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec61809..6498c2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: devDependencies: '@biomejs/biome': - specifier: ^2.6.4 - version: 2.6.4 + specifier: ^2.6.5 + version: 2.6.5 '@earendil-works/pi-ai': specifier: ^1.5.0 version: 1.5.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.12.3)(ws@8.21.3) @@ -125,71 +125,71 @@ packages: resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} - '@biomejs/biome@2.6.4': - resolution: {integrity: sha512-8sKKwaNW4ot22awZ3lBjwWLHnUFXgXnrBPYS2gNlBAXRbDC++7kqjK2H7dR/FgCDtPvO1xfKpgEhJkGxAKYkxA==} + '@biomejs/biome@2.6.5': + resolution: {integrity: sha512-n0sp74QbSVEkWXlH659TKv4oJNylwAvDeZ8K8vt/PbcF1xneuNWe57z/G0Bxfh00cufhWXF6DiStLM8+jcLUpg==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-android-arm64@2.6.4': - resolution: {integrity: sha512-jwrwigP4tBcHxzuV8izioX/BOUhJL/4e2rNSOE5nCjBkVC018LQAQUGiH+CNkFYvLtQ1dcZo5/xXERmaAGc7JQ==} + '@biomejs/cli-android-arm64@2.6.5': + resolution: {integrity: sha512-fkLzc1WRU7wRIUNmAm7cxILb4dd6eESz/VycWvVFKVn6tXx0gFQKdUbPnwT42xys8AEvIJQpFGcK00kURwqorg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [android] - '@biomejs/cli-android-x64@2.6.4': - resolution: {integrity: sha512-HdGInPJ4nZFwHxzvEtcp2nmSBtUyJrurmtIK43upFckuS3+ZqVM05HtsxQWnqGsuy/pxckIAjyKWlEKRlBXUyA==} + '@biomejs/cli-android-x64@2.6.5': + resolution: {integrity: sha512-fkZ9dPnDAzAcaPtPCiwqtkub5C9FQK/oGCGaU9sq+DysTaq13pE9FxBIMcEZ3ydn8/FfcMdXi/DDzKDP+qyn2Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [android] - '@biomejs/cli-darwin-arm64@2.6.4': - resolution: {integrity: sha512-0VvcUz46wbg59Q4TZ59KkIjDJrxEyu4Eoe79+F4SChxoWe+W7AEyIp6F5r72C9+p6MsB6dIovnZt1D30lGM3XA==} + '@biomejs/cli-darwin-arm64@2.6.5': + resolution: {integrity: sha512-bKd0EOREgnVjeZm4s9knQ+WdPOkWS3JHB+eWGHlx+Pq66bqQA1/KybwU0O+zuyAyXiphSCBcrLruZt3YtnJoaQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.6.4': - resolution: {integrity: sha512-fxmvkF81srlS3DiMyfm+aG3J0eG+6/BjaxRyjy57KbLSafnq0PO+hHfVWmtH2ty0S7nm6yDXBRwySfFeaRJKRQ==} + '@biomejs/cli-darwin-x64@2.6.5': + resolution: {integrity: sha512-UkT9BECNOXSiW6SGddjyaRACy362Sc9n3vDaTFkccL0bC2gW7JzM16i4+JKPbEaDn1Sa+oETHV8KungnI/y40Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.6.4': - resolution: {integrity: sha512-31ZaH/MHqPX+O+evRwRrCHQ8W5/9adY+/eMytGGWZnRx1T41QsWIF09+/WujynLBhWZUxNGS5CtsWLwgOFTAyg==} + '@biomejs/cli-linux-arm64-musl@2.6.5': + resolution: {integrity: sha512-kfc3AePW1a0/d/gUpcIz95nf3Ce8YY2/xL0Fk3fLqAhKsotqUi6PN6BaVWss4p6ijWbiuacZ9NQ4NiQtpjbYpw==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [musl] - '@biomejs/cli-linux-arm64@2.6.4': - resolution: {integrity: sha512-6+bK1Y/evmIUiJr7auertuMbiaYVicJhqA1bZyL47lCMWugv8l0gFQCqvR/gN1vZMHYm+HwoTKgeMxdNFCoQTg==} + '@biomejs/cli-linux-arm64@2.6.5': + resolution: {integrity: sha512-H9cVMWBqW/lkFK9NHJdt0bpM2q7PacUDEjQoff9+m0dCV09H/jmBykfTCkbDthHf8+jCuWDJWiQFJ7bDkiGOcA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] libc: [glibc] - '@biomejs/cli-linux-x64-musl@2.6.4': - resolution: {integrity: sha512-bomSa2XdQAGGj6Cwdm4M13qE+lN9+AOlW8KtI4hdyVyRRF/Z71qPAi95T4wtdZK0QpBbkTW8hgbZZCwPi4xzIw==} + '@biomejs/cli-linux-x64-musl@2.6.5': + resolution: {integrity: sha512-0mHVpLNt7/FugfZ2VPugUIEIiooWIyHGGJUqU9StxamBln9hz+maXRFIECU50jaf5/ZB93veGIViN5VKgw2AEA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [musl] - '@biomejs/cli-linux-x64@2.6.4': - resolution: {integrity: sha512-tbFZMdL6dP/ws4edXxYG9WhvaVa1ZDzL4G5DzahnWW7Psib9H+VTdyU4XXVxpJk5j/nYZroSbvSNyYlpW3yCIw==} + '@biomejs/cli-linux-x64@2.6.5': + resolution: {integrity: sha512-e6/WMSN/+FhH+ZOoBevps/euxFQmv6BXqYNuTn1+ITR6O4XYFjHF5PeVSNZnjkgSYmvhcDkfepSX5lXczMijGA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] libc: [glibc] - '@biomejs/cli-win32-arm64@2.6.4': - resolution: {integrity: sha512-eCsqt8Ma3eVap+oyJCDnJZH60SIU464B66n3y38EUTJdM1T/HfZol1FJC9vb+sHToX0ix8XzqPcwJnOzyAJz0g==} + '@biomejs/cli-win32-arm64@2.6.5': + resolution: {integrity: sha512-uBbfpdR/cH5up1iRJFGAPP97xKMzrgMs685CSrY7NM+Q4Unw7chY0o3JulEvJdmU6c6hGQxwqotMq7DDH0DzxA==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.6.4': - resolution: {integrity: sha512-pYsMy5AThiHFOZAsngg8qN4PurM6nInKBhVX7sbj3eNpRQqo1Hb0MgCwPL7VqVMrmcEEMj9Ks3qbkallR65e8g==} + '@biomejs/cli-win32-x64@2.6.5': + resolution: {integrity: sha512-07LaCeN9vvmbAAMOMsUjMfL5UbsCDkvVeyMJIgwNRc0xYqhizBGkjVzx05LYEwCXl2AiHcdzqVPR4IOYAsI1QQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -920,47 +920,47 @@ snapshots: '@babel/runtime@7.29.7': {} - '@biomejs/biome@2.6.4': + '@biomejs/biome@2.6.5': optionalDependencies: - '@biomejs/cli-android-arm64': 2.6.4 - '@biomejs/cli-android-x64': 2.6.4 - '@biomejs/cli-darwin-arm64': 2.6.4 - '@biomejs/cli-darwin-x64': 2.6.4 - '@biomejs/cli-linux-arm64': 2.6.4 - '@biomejs/cli-linux-arm64-musl': 2.6.4 - '@biomejs/cli-linux-x64': 2.6.4 - '@biomejs/cli-linux-x64-musl': 2.6.4 - '@biomejs/cli-win32-arm64': 2.6.4 - '@biomejs/cli-win32-x64': 2.6.4 - - '@biomejs/cli-android-arm64@2.6.4': + '@biomejs/cli-android-arm64': 2.6.5 + '@biomejs/cli-android-x64': 2.6.5 + '@biomejs/cli-darwin-arm64': 2.6.5 + '@biomejs/cli-darwin-x64': 2.6.5 + '@biomejs/cli-linux-arm64': 2.6.5 + '@biomejs/cli-linux-arm64-musl': 2.6.5 + '@biomejs/cli-linux-x64': 2.6.5 + '@biomejs/cli-linux-x64-musl': 2.6.5 + '@biomejs/cli-win32-arm64': 2.6.5 + '@biomejs/cli-win32-x64': 2.6.5 + + '@biomejs/cli-android-arm64@2.6.5': optional: true - '@biomejs/cli-android-x64@2.6.4': + '@biomejs/cli-android-x64@2.6.5': optional: true - '@biomejs/cli-darwin-arm64@2.6.4': + '@biomejs/cli-darwin-arm64@2.6.5': optional: true - '@biomejs/cli-darwin-x64@2.6.4': + '@biomejs/cli-darwin-x64@2.6.5': optional: true - '@biomejs/cli-linux-arm64-musl@2.6.4': + '@biomejs/cli-linux-arm64-musl@2.6.5': optional: true - '@biomejs/cli-linux-arm64@2.6.4': + '@biomejs/cli-linux-arm64@2.6.5': optional: true - '@biomejs/cli-linux-x64-musl@2.6.4': + '@biomejs/cli-linux-x64-musl@2.6.5': optional: true - '@biomejs/cli-linux-x64@2.6.4': + '@biomejs/cli-linux-x64@2.6.5': optional: true - '@biomejs/cli-win32-arm64@2.6.4': + '@biomejs/cli-win32-arm64@2.6.5': optional: true - '@biomejs/cli-win32-x64@2.6.4': + '@biomejs/cli-win32-x64@2.6.5': optional: true '@earendil-works/pi-agent-core@1.5.0(@aws-sdk/credential-provider-node@3.972.83)(@smithy/signature-v4@5.7.3)(undici@8.12.3)(ws@8.21.3)': diff --git a/src/catalog.ts b/src/catalog.ts index 65189fe..a0a75db 100644 --- a/src/catalog.ts +++ b/src/catalog.ts @@ -118,6 +118,15 @@ const mediaModels = new Map< price?: readonly [number, number, number]; } >([ + [ + "gpt-image-2.6-flare", + { name: "GPT Image 2.6 Flare", purpose: "image", route: "openai", price: gptImage2Price }, + ], + [ + "gpt-image-2.6-sunburst", + { name: "GPT Image 2.6 Sunburst", purpose: "image", route: "openai", price: gptImage2Price }, + ], + ["gpt-image-2.6", { name: "GPT Image 2.6", purpose: "image", route: "openai" }], [ "gpt-image-2.5-flare", { name: "GPT Image 2.5 Flare", purpose: "image", route: "openai", price: gptImage2Price }, @@ -165,13 +174,12 @@ export function mediaCapability(id: string) { if (backend && (capability?.route === "xai" || capability?.route === "openai")) { return { ...capability, route: undefined, disabledReason: `Unsupported media execution on ${backend}.` }; } - // CLIProxyAPI 8.0.48 advertises it for API keys, but OpenAI's Images API rejects the bare 2.5 name. - if (capability && authType === "api-key" && metadataId === "gpt-image-2.5") { + // CLIProxyAPI advertises the plain names for API keys, but OpenAI's Images API has only Sunburst and Flare. + if (capability && authType === "api-key" && /^gpt-image-2\.[56]$/.test(metadataId)) { return { ...capability, route: undefined, - disabledReason: - "OpenAI's API has no gpt-image-2.5 model. Use api/gpt-image-2.5-sunburst or api/gpt-image-2.5-flare.", + disabledReason: `OpenAI's API has no ${metadataId} model. Use api/${metadataId}-sunburst or api/${metadataId}-flare.`, }; } return capability; @@ -185,6 +193,9 @@ export function mediaControls(id: string) { const capability = mediaCapability(id); const { metadataId, authType } = modelRoute(id); const openai = capability?.route === "openai"; + // Sunburst and Flare add quality tiers and markup edits; 2.6 adds complexity plus GIF and SVG output. + const tiered = openai && /^gpt-image-2\.[56]-(sunburst|flare)$/.test(metadataId); + const image26 = tiered && metadataId.startsWith("gpt-image-2.6-"); const commonRatios = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"]; let resolution: string[] = []; let aspect_ratio: string[] = []; @@ -214,24 +225,27 @@ export function mediaControls(id: string) { : undefined, resolution, aspect_ratio, - quality: openai - ? [ - "auto", - "low", - "medium", - "high", - ...(/^gpt-image-2\.5-(sunburst|flare)$/.test(metadataId) ? ["xhigh", "max"] : []), - ] - : [], + quality: openai ? ["auto", "low", "medium", "high", ...(tiered ? ["xhigh", "max"] : [])] : [], + complexity: image26 ? ["low", "medium", "high"] : [], background: openai ? ["auto", "opaque", "transparent"] : [], - output_format: openai ? ["png", "jpeg", "webp"] : [], + output_format: openai ? ["png", "jpeg", "webp", ...(image26 ? ["gif", "svg"] : [])] : [], + // GIF sizes are per frame and skip min_pixels; OpenAI checks the animation against its pixel budget. + gif: image26 + ? { frame_count: { min: 2, max: 16 }, frame_duration_ms: { min: 20, max: 2000 } } + : undefined, output_compression: openai ? { min: 0, max: 100, formats: ["jpeg", "webp"] } : undefined, moderation: openai ? ["auto", "low"] : [], // Subscription image-tool translation does not forward n. n: openai ? { min: 1, max: authType === "api-key" ? 10 : 1 } : undefined, partial_images: openai ? { min: 0, max: 3 } : undefined, editing: openai - ? { max_images: 16, max_file_bytes: 14 * 1024 * 1024, max_total_bytes: 20 * 1024 * 1024, mask: "png" } + ? { + max_images: 16, + max_file_bytes: 14 * 1024 * 1024, + max_total_bytes: 20 * 1024 * 1024, + mask: "png", + ...(tiered ? { markup: "png" } : {}), + } : undefined, input_fidelity: openai && metadataId === "gpt-image-1.5" ? ["low", "high"] : [], }; diff --git a/src/media-defaults.ts b/src/media-defaults.ts index 75f5dac..e656e0b 100644 --- a/src/media-defaults.ts +++ b/src/media-defaults.ts @@ -3,7 +3,7 @@ import { type MediaPurpose, mediaCapability, mediaPurpose, modelRoute } from "./ import { type Config, isModelId, isRecord, PROVIDER_ID } from "./config.ts"; export const MEDIA_DEFAULTS_ENTRY = "cliproxyapi-media-defaults"; -export const OPENAI_IMAGE_DEFAULT = "gpt-image-2.5-sunburst"; +export const OPENAI_IMAGE_DEFAULT = "gpt-image-2.6-sunburst"; // null blocks fallback for invalid selections, including after unrelated picker saves. export type MediaDefaults = Partial>; export type DefaultsContext = { diff --git a/src/media.ts b/src/media.ts index cb1d387..491758c 100644 --- a/src/media.ts +++ b/src/media.ts @@ -14,6 +14,7 @@ import type { TextContent, Usage, } from "@earendil-works/pi-ai"; +import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { type Static, Type } from "typebox"; import { fetchCatalog, mapMediaCatalog, mediaCapability, mediaControls } from "./catalog.ts"; @@ -65,13 +66,22 @@ const imageParameters = Type.Object({ description: "OpenAI quality from controls.quality; Sunburst and Flare also support xhigh and max.", }), ), + complexity: Type.Optional( + Type.String({ + description: + "GPT Image 2.6 Sunburst/Flare planning effort: low, medium, or high (upstream default high). Low suits quick edits.", + }), + ), background: Type.Optional( Type.String({ description: "OpenAI background: auto, opaque, or transparent. Transparency requires PNG or WebP.", }), ), output_format: Type.Optional( - Type.String({ description: "OpenAI output format: png, jpeg, or webp. Omit for the backend default." }), + Type.String({ + description: + "OpenAI output format: png, jpeg, or webp. GPT Image 2.6 Sunburst/Flare also generate gif and svg (svg needs n of 1). Omit for the backend default.", + }), ), output_compression: Type.Optional( Type.Integer({ @@ -80,6 +90,20 @@ const imageParameters = Type.Object({ description: "OpenAI JPEG/WebP compression, 0–100. Requires an explicit jpeg or webp output_format.", }), ), + frame_count: Type.Optional( + Type.Integer({ + minimum: 2, + maximum: 16, + description: "GIF output only: number of frames, 2–16 (upstream default 9).", + }), + ), + frame_duration_ms: Type.Optional( + Type.Integer({ + minimum: 20, + maximum: 2000, + description: "GIF output only: milliseconds per frame, 20–2000, rounded to 10 (upstream default 200).", + }), + ), moderation: Type.Optional(Type.String({ description: "OpenAI moderation: auto or low." })), n: Type.Optional( Type.Integer({ @@ -112,6 +136,13 @@ const imageParameters = Type.Object({ "Optional local PNG mask for images[0]. The first image must also be PNG with matching dimensions; mask must have an alpha channel. Transparent regions are edited.", }), ), + markup: Type.Optional( + Type.String({ + minLength: 1, + description: + "Optional local PNG annotation layer for images[0] with GPT Image 2.5/2.6 Sunburst or Flare: transparent except for marks showing where to edit. Same size as images[0]; cannot be combined with mask.", + }), + ), input_fidelity: Type.Optional( Type.String({ description: "Editing fidelity for GPT Image 1.5 only: low or high. Omit for other models.", @@ -122,7 +153,7 @@ const imageParameters = Type.Object({ minLength: 1, maxLength: 32, description: - "OpenAI image size: auto or WIDTHxHEIGHT. Use controls.size and controls.size_limits from cliproxyapi_media_models. Omit to preserve the backend default.", + "OpenAI image size: auto or WIDTHxHEIGHT. Use controls.size and controls.size_limits from cliproxyapi_media_models; GIF sizes are per frame and may go below min_pixels. Omit to preserve the backend default.", }), ), }); @@ -141,6 +172,7 @@ function mediaOutputOptions(model: string, params: Partial= limits.min_pixels && + (gif || pixels >= limits.min_pixels) && pixels <= limits.max_pixels ) continue; @@ -196,11 +234,24 @@ function mediaOutputOptions(model: string, params: Partial controls.partial_images.max || - (options.partial_images > 0 && (options.n ?? 1) !== 1)) + (options.partial_images > 0 && ((options.n ?? 1) !== 1 || generationOnly))) ) throw new Error( - "CLIProxyAPI partial_images requires an OpenAI image model, n of 1, and an integer from 0 to 3.", + "CLIProxyAPI partial_images requires an OpenAI image model, n of 1, PNG, JPEG, or WebP output, and an integer from 0 to 3.", ); + for (const field of ["frame_count", "frame_duration_ms"] as const) { + const value = options[field]; + const range = controls.gif?.[field]; + if ( + value !== undefined && + (!gif || !range || !Number.isInteger(value) || value < range.min || value > range.max) + ) + throw new Error(`CLIProxyAPI ${field} requires gif output_format and a value within controls.gif.`); + } + if (options.output_format === "svg" && (options.n ?? 1) !== 1) + throw new Error("CLIProxyAPI svg output_format requires n of 1."); + if (generationOnly && params.images !== undefined) + throw new Error("CLIProxyAPI gif and svg output_format are for generation, not editing."); if ( options.output_compression !== undefined && (!controls.output_compression || @@ -226,7 +277,8 @@ async function imageInputs( signal: AbortSignal, ) { if (params.images === undefined) { - if (params.mask !== undefined) throw new Error("CLIProxyAPI mask requires editing images."); + if (params.mask !== undefined || params.markup !== undefined) + throw new Error("CLIProxyAPI mask and markup require editing images."); return {}; } const limits = mediaControls(model).editing; @@ -237,14 +289,18 @@ async function imageInputs( params.images.length > limits.max_images ) throw new Error("CLIProxyAPI editing requires 1–16 local images and a supported OpenAI image model."); + if (params.markup !== undefined && (params.mask !== undefined || !limits.markup)) + throw new Error( + "CLIProxyAPI markup requires GPT Image 2.5 or 2.6 Sunburst or Flare and cannot be combined with mask.", + ); + // The mask or markup layer follows the input images. + const layer = params.mask ?? params.markup; const images: { image_url: string }[] = []; let mask: { image_url: string } | undefined; + let markup: { image_url: string } | undefined; let firstPngDimensions: Buffer | undefined; let total = 0; - for (const [index, path] of [ - ...params.images, - ...(params.mask === undefined ? [] : [params.mask]), - ].entries()) { + for (const [index, path] of [...params.images, ...(layer === undefined ? [] : [layer])].entries()) { signal.throwIfAborted(); if ( typeof path !== "string" || @@ -285,22 +341,25 @@ async function imageInputs( } catch { throw new Error(`CLIProxyAPI editing input "${path}" is not a supported PNG, JPEG, or WebP image.`); } - const isMask = index === params.images.length; + const isLayer = index === params.images.length; if (index === 0 && mimeType === "image/png") firstPngDimensions = bytes.subarray(16, 24); + // Masks need a matching PNG first image; OpenAI checks markup on JPEG or WebP images after orientation. + const sameSize = firstPngDimensions?.equals(bytes.subarray(16, 24)); if ( - isMask && + isLayer && (mimeType !== "image/png" || - !firstPngDimensions?.equals(bytes.subarray(16, 24)) || - ![4, 6].includes(bytes[25])) + ![4, 6].includes(bytes[25]) || + (sameSize === undefined ? params.mask !== undefined : !sameSize)) ) throw new Error( - "CLIProxyAPI mask and first image must be matching-size PNGs; the mask needs an alpha channel.", + "CLIProxyAPI mask and markup layers must be alpha-channel PNGs matching a PNG first image's size; masks need a PNG first image.", ); const input = { image_url: `data:${mimeType};base64,${bytes.toString("base64")}` }; - if (isMask) mask = input; - else images.push(input); + if (!isLayer) images.push(input); + else if (params.mask !== undefined) mask = input; + else markup = input; } - return { images, mask }; + return { images, mask, markup }; } export async function mediaKey(ctx: Pick, signal: AbortSignal) { @@ -642,7 +701,31 @@ function imageType(bytes: Buffer) { return { extension, mimeType }; } -function parseImage(entry: unknown) { +// OpenAI documents static SVG; reject scripts, animation, entities, and references outside the document. +const unsafeSvg = + /<(?:script|foreignObject|image|iframe|embed|object|set|animate\w*|!DOCTYPE|!ENTITY)\b|\son[a-z]+\s*=|javascript:|@import|(?:href|src)\s*=\s*(?:"(?!#)|'(?!#)|[^"'\s#>])|url\(\s*(?:"(?!#)|'(?!#)|[^"'\s#)])/i; + +// Accept GIF or SVG only when the request asked GPT Image 2.6 for that format. +function outputType(bytes: Buffer, format?: string) { + if (format === "gif") { + if (bytes.length > 13 && /^GIF8[79]a$/.test(bytes.toString("ascii", 0, 6)) && bytes.at(-1) === 0x3b) + return { extension: "gif", mimeType: "image/gif" }; + throw new Error("CLIProxyAPI returned a malformed GIF."); + } + if (format !== "svg") return imageType(bytes); + let text: string; + try { + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new Error("CLIProxyAPI returned an SVG that is not valid UTF-8."); + } + const root = text.replace(/^\s*(?:<\?xml[^>]*\?>\s*)?(?:\s*)*/, "").trimEnd(); + if (!/^]/.test(root) || !root.endsWith("") || unsafeSvg.test(text)) + throw new Error("CLIProxyAPI returned a malformed SVG or one with active or external content."); + return { extension: "svg", mimeType: "image/svg+xml" }; +} + +function parseImage(entry: unknown, format?: string) { if (!isRecord(entry) || typeof entry.b64_json !== "string") { throw new Error("CLIProxyAPI did not return b64_json image data. URL-only images are not downloaded."); } @@ -651,12 +734,12 @@ function parseImage(entry: unknown) { if (!encoded || bytes.toString("base64") !== encoded) { throw new Error("CLIProxyAPI returned invalid base64 image data."); } - const { extension, mimeType } = imageType(bytes); + const { extension, mimeType } = outputType(bytes, format); const image: ImageContent = { type: "image", data: encoded, mimeType }; return { bytes, extension, image }; } -function parseImages(value: unknown, count: number) { +function parseImages(value: unknown, count: number, format?: string) { if ( !isRecord(value) || value.error !== undefined || @@ -664,7 +747,7 @@ function parseImages(value: unknown, count: number) { value.data.length !== count ) throw new Error(`CLIProxyAPI image response must contain ${count} image(s).`); - return value.data.map(parseImage); + return value.data.map((entry: unknown) => parseImage(entry, format)); } function parseGoogleImages(value: unknown) { @@ -727,7 +810,7 @@ function parseGoogleImages(value: unknown) { } // OpenAI reports image token usage. Costs use list prices and stay zero for unpriced models. -function imageUsage(model: string, value: unknown) { +function imageUsage(model: string, value: unknown, svg?: unknown) { const tokens = (field: unknown) => typeof field === "number" && Number.isSafeInteger(field) && field >= 0 ? field : undefined; if (!isRecord(value)) return undefined; @@ -746,8 +829,22 @@ function imageUsage(model: string, value: unknown) { cacheWrite: 0, total: 0, }; - cost.total = cost.input + cost.output; const usage: Usage = { input, output, cacheRead: 0, cacheWrite: 0, totalTokens: input + output, cost }; + // SVG output adds the authoring model's tokens, priced from Pi's OpenAI metadata when Pi knows it. + if (isRecord(svg)) { + const author = svg.model; + const svgInput = tokens(svg.input_tokens); + const svgOutput = tokens(svg.output_tokens); + if (svgInput !== undefined && svgOutput !== undefined) { + const price = getBuiltinModels("openai").find(({ id }) => id === author)?.cost; + usage.input += svgInput; + usage.output += svgOutput; + usage.totalTokens += svgInput + svgOutput; + cost.input += (svgInput * (price?.input ?? 0)) / 1e6; + cost.output += (svgOutput * (price?.output ?? 0)) / 1e6; + } + } + cost.total = cost.input + cost.output; return usage; } @@ -761,13 +858,14 @@ async function requestImages( output: Partial>; images?: { image_url: string }[]; mask?: { image_url: string }; + markup?: { image_url: string }; }, signal: AbortSignal, onPartial?: (image: ImageContent, index: number) => void, { onPayload, ...transport }: TransportOptions & { onPayload?: (payload: unknown) => unknown } = {}, ) { const route = mediaCapability(model)?.route; - const { prompt, images, mask } = request; + const { prompt, images, mask, markup } = request; const { partial_images = 0, ...output } = request.output; const n = output.n ?? 1; const stream = route === "openai" && partial_images > 0; @@ -782,7 +880,7 @@ async function requestImages( aspect_ratio: output.aspect_ratio, } : route === "openai" - ? { model, prompt, ...output, n, images, mask, ...(stream ? { stream, partial_images } : {}) } + ? { model, prompt, ...output, n, images, mask, markup, ...(stream ? { stream, partial_images } : {}) } : { contents: [{ role: "user", parts: [{ text: prompt }] }], generationConfig: { @@ -808,8 +906,14 @@ async function requestImages( openaiImage: route === "openai", }); return { - images: route === "gemini" ? parseGoogleImages(result) : parseImages(result, n), - usage: route === "openai" && isRecord(result) ? imageUsage(model, result.usage) : undefined, + images: + route === "gemini" + ? parseGoogleImages(result) + : parseImages(result, n, route === "openai" ? output.output_format : undefined), + usage: + route === "openai" && isRecord(result) + ? imageUsage(model, result.usage, result.svg_usage) + : undefined, }; } // Each frame carries one full image, so the stream may hold the final image plus every preview. @@ -839,7 +943,10 @@ async function requestImages( (event) => isRecord(event) && typeof event.type === "string" && event.type.endsWith(".completed"), ); if (completed.length !== n) throw new Error(`CLIProxyAPI image stream must complete ${n} image(s).`); - return { images: completed.map(parseImage), usage: imageUsage(model, completed[0].usage) }; + return { + images: completed.map((event) => parseImage(event)), + usage: imageUsage(model, completed[0].usage), + }; } export async function generateNativeImages( @@ -967,11 +1074,14 @@ export async function generateImage( let previewBudget = MAX_INLINE_IMAGE_BASE64_BYTES; const content: (TextContent | ImageContent)[] = []; for (const [index, { image }] of images.entries()) { - const previewOmitted = !ctx.model?.input.includes("image") - ? "Preview omitted: the current chat model does not support images." - : image.data.length > previewBudget - ? "Preview omitted: image base64 exceeds the 4 MiB inline budget. Use read on the saved file." - : undefined; + // Chat providers disagree on animated GIF input and reject SVG, so both skip inline previews. + const previewOmitted = ["image/gif", "image/svg+xml"].includes(image.mimeType) + ? "Preview omitted: GIF and SVG results are saved without an inline preview." + : !ctx.model?.input.includes("image") + ? "Preview omitted: the current chat model does not support images." + : image.data.length > previewBudget + ? "Preview omitted: image base64 exceeds the 4 MiB inline budget. Use read on the saved file." + : undefined; content.push({ type: "text", text: `Saved image: ${files[index].path}${previewOmitted ? `\n${previewOmitted}` : ""}`, @@ -1076,7 +1186,7 @@ export function registerMediaTools(pi: ExtensionAPI, config: Config) { name: "cliproxyapi_generate_image", label: "CLIProxyAPI image", description: - "Generate images, or edit local images with OpenAI models by supplying images and an optional mask. Supports OpenAI size, quality, background, format/compression, image count and streamed partial previews from cliproxyapi_media_models. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB in total, plus 32 MiB per streamed preview; inline base64: 4 MiB. No automatic retries.", + "Generate images, or edit local images with OpenAI models by supplying images and an optional mask or markup layer. Supports OpenAI size, quality, complexity, background, format/compression, GIF/SVG output, image count and streamed partial previews; check cliproxyapi_media_models for each model's controls. Uses the selected model or automatic OpenAI/GPT default without changing chat. Saves final images under a new .pi/ directory with optional previews. Maximum response: 32 MiB in total, plus 32 MiB per streamed preview; inline base64: 4 MiB. No automatic retries.", promptSnippet: "Generate requested raster images through CLIProxyAPI using the effective image default.", promptGuidelines: [ "Use cliproxyapi_generate_image for requested raster images. Omit model to use the effective default unless a specific image model is requested; keep the current chat model.", diff --git a/test/auth-routes.test.ts b/test/auth-routes.test.ts index 7599a1e..071d983 100644 --- a/test/auth-routes.test.ts +++ b/test/auth-routes.test.ts @@ -88,17 +88,20 @@ test("automatic media selection remains separate from the chat credential route" } }); -test("the picker explains why API-key gpt-image-2.5 is disabled while other routes stay usable", () => { - const ids = ["api/gpt-image-2.5", "gpt-image-2.5", "oauth/gpt-image-2.5"]; - const catalog = { data: ids.map((id) => ({ id })) }; - assert.deepEqual( - mapMediaCatalog(catalog).map(({ id }) => id), - ids.slice(1), - ); - const row = pickerCatalog(catalog)[0]; - assert.equal(row.supported, false); - assert.match(row.disabledReason ?? "", /no gpt-image-2.5 model.*sunburst/); - assert.deepEqual(mediaControls(ids[0]).quality, []); - assert.throws(() => resolveMediaModel(parseConfig({}), {}, "image", ids[0]), /no gpt-image-2.5 model/); - assert.equal(resolveMediaModel(parseConfig({}), {}, "image", ids[2]), ids[2]); +test("the picker explains why API-key plain GPT Image 2.5 and 2.6 are disabled while other routes stay usable", () => { + for (const plain of ["gpt-image-2.5", "gpt-image-2.6"]) { + const ids = [`api/${plain}`, plain, `oauth/${plain}`]; + const catalog = { data: ids.map((id) => ({ id })) }; + assert.deepEqual( + mapMediaCatalog(catalog).map(({ id }) => id), + ids.slice(1), + ); + const row = pickerCatalog(catalog)[0]; + const reason = `OpenAI's API has no ${plain} model. Use api/${plain}-sunburst or api/${plain}-flare.`; + assert.equal(row.supported, false); + assert.equal(row.disabledReason, reason); + assert.deepEqual(mediaControls(ids[0]).quality, []); + assert.throws(() => resolveMediaModel(parseConfig({}), {}, "image", ids[0]), { message: reason }); + assert.equal(resolveMediaModel(parseConfig({}), {}, "image", ids[2]), ids[2]); + } }); diff --git a/test/media.test.ts b/test/media.test.ts index a4574bb..a8dd198 100644 --- a/test/media.test.ts +++ b/test/media.test.ts @@ -14,6 +14,7 @@ import { setTimeout as sleep } from "node:timers/promises"; import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import type { ImageContent } from "@earendil-works/pi-ai"; +import { getBuiltinModels } from "@earendil-works/pi-ai/providers/all"; import { type ExtensionAPI, type ExtensionToolContext, @@ -48,6 +49,9 @@ const png = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwM const jpeg = "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAP//////////////////////////////////////////////////////////////////////////////////////2wBDAf//////////////////////////////////////////////////////////////////////////////////////wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAAAAX/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCdAAr/2Q=="; const webp = "UklGRiIAAABXRUJQVlA4IBYAAAAwAQCdASoBAAEADsD+JaQAA3AAAAAA"; +const gif = "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7"; +const svgDocument = (root: string) => `\n\n${root}\n`; +const svg = (root: string) => Buffer.from(svgDocument(root)).toString("base64"); async function home(t: TestContext) { const path = await mkdtemp(join(tmpdir(), "pi-cliproxyapi-media-test-")); @@ -856,6 +860,9 @@ test("recognized backends do not enable xAI media and unknown prefixes do not in }); const openaiIds = [ + "gpt-image-2.6-flare", + "gpt-image-2.6-sunburst", + "gpt-image-2.6", "gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "gpt-image-2.5", @@ -899,7 +906,7 @@ test("OpenAI image IDs are image-only, preserve all wire IDs, and reject unsuppo } }); -test("all five OpenAI IDs POST one image without response_format and reuse bounded private artifact validation", async (t) => { +test("all eight OpenAI IDs POST one image without response_format and reuse bounded private artifact validation", async (t) => { const cwd = await home(t); const ctx = context(cwd); let response: unknown; @@ -1472,6 +1479,24 @@ test("media discovery reports model-specific output controls and preserves quali max_pixels: 8294400, max_aspect_ratio: 3, }); + for (const id of ["gpt-image-2.6-sunburst", "gpt-image-2.6-flare"]) { + assert.deepEqual(controls(id).size_limits, controls("gpt-image-2.5-sunburst").size_limits); + assert.deepEqual(controls(id).quality, controls("gpt-image-2.5-sunburst").quality); + assert.deepEqual(controls(id).complexity, ["low", "medium", "high"]); + assert.deepEqual(controls(id).output_format, ["png", "jpeg", "webp", "gif", "svg"]); + assert.deepEqual(controls(id).gif, { + frame_count: { min: 2, max: 16 }, + frame_duration_ms: { min: 20, max: 2000 }, + }); + assert.equal(controls(id).editing?.markup, "png"); + } + for (const id of ["gpt-image-2.6", "gpt-image-2.5-sunburst", "gpt-image-2.5", "gpt-image-2"]) { + assert.deepEqual(controls(id).complexity, []); + assert.deepEqual(controls(id).output_format, ["png", "jpeg", "webp"]); + assert.equal(controls(id).gif, undefined); + assert.equal(controls(id).editing?.markup, id === "gpt-image-2.5-sunburst" ? "png" : undefined); + } + assert.deepEqual(controls("gpt-image-2.6").quality, ["auto", "low", "medium", "high"]); assert.deepEqual(controls("gemini-2.5-flash-image").resolution, []); assert.deepEqual(controls("gemini-3-pro-image").resolution, ["1K", "2K", "4K"]); assert.deepEqual(controls("gemini-3.1-flash-image").resolution, ["512", "1K", "2K", "4K"]); @@ -1749,7 +1774,53 @@ test("OpenAI editing uploads bounded local files and a mask to the exact edit ro assert.equal((await readFile(join(ctx.cwd, name))).toString("base64"), data); }); +test("markup edits send a transparent PNG layer to Sunburst and Flare without a mask", async (t) => { + const ids = ["gpt-image-2.6-flare", "oauth/gpt-image-2.5-sunburst"]; + const posted: unknown[] = []; + const config = parseConfig({ + baseUrl: await server(t, (req, res) => { + if (req.method === "GET") return void res.end(JSON.stringify({ data: ids.map((id) => ({ id })) })); + assert.equal(req.url, "/v1/images/edits"); + void body(req).then((value) => { + posted.push(value); + res.end(JSON.stringify({ data: [{ b64_json: png }] })); + }); + }), + }); + const ctx = context(await home(t)); + for (const [name, data] of [ + ["kitchen.jpg", jpeg], + ["kitchen.png", png], + ["overlay.png", png], + ]) + await writeFile(join(ctx.cwd, name), Buffer.from(data, "base64")); + // OpenAI checks markup on a JPEG first image after orientation; the extension checks PNG sizes. + await generateImage( + config, + { model: ids[0], prompt: "fixture", images: ["kitchen.jpg"], markup: "overlay.png", complexity: "low" }, + ctx, + ); + await generateImage( + config, + { model: ids[1], prompt: "fixture", images: ["kitchen.png"], markup: "overlay.png" }, + ctx, + ); + const layer = { image_url: `data:image/png;base64,${png}` }; + assert.deepEqual(posted, [ + { + model: ids[0], + prompt: "fixture", + complexity: "low", + n: 1, + images: [{ image_url: `data:image/jpeg;base64,${jpeg}` }], + markup: layer, + }, + { model: ids[1], prompt: "fixture", n: 1, images: [layer], markup: layer }, + ]); +}); + test("invalid OpenAI controls and edit inputs fail before authentication or network access", async (t) => { + const image26 = "api/gpt-image-2.6-sunburst"; const config = parseConfig({}); const ctx = { ...context(await home(t)), @@ -1790,6 +1861,28 @@ test("invalid OpenAI controls and edit inputs fail before authentication or netw { partial_images: 1, n: 2 }, { model: imageModel, partial_images: 1 }, { model: "gemini-3-pro-image", partial_images: 1 }, + { complexity: "low" }, + { output_format: "svg" }, + { model: image26, complexity: "max" }, + { model: "oauth/gpt-image-2.6", complexity: "low" }, + { model: "api/gpt-image-2", output_format: "gif" }, + { model: image26, output_format: "svg", n: 2 }, + { model: image26, output_format: "gif", partial_images: 1 }, + { model: image26, output_format: "svg", partial_images: 2 }, + { model: image26, output_format: "gif", output_compression: 50 }, + { model: image26, frame_count: 4 }, + { model: image26, output_format: "png", frame_duration_ms: 100 }, + { model: image26, output_format: "gif", frame_count: 1 }, + { model: image26, output_format: "gif", frame_count: 17 }, + { model: image26, output_format: "gif", frame_count: 2.5 }, + { model: image26, output_format: "gif", frame_duration_ms: 10 }, + { model: image26, output_format: "gif", frame_duration_ms: 2001 }, + { model: image26, output_format: "gif", size: "520x512" }, + { model: image26, output_format: "gif", images: ["source.png"] }, + { model: image26, output_format: "svg", images: ["source.png"] }, + { markup: "mask.png" }, + { model: "api/gpt-image-2", images: ["source.png"], markup: "mask.png" }, + { images: ["source.png"], mask: "mask.png", markup: "mask.png" }, ]) await assert.rejects( generateImage(config, { model: "api/gpt-image-2.5-sunburst", prompt: "fixture", ...params }, ctx), @@ -1810,6 +1903,10 @@ test("invalid OpenAI controls and edit inputs fail before authentication or netw ctx, ), ); + for (const markup of ["invalid.png", "wrong-size.png", "no-alpha.png"]) + await assert.rejects( + generateImage(config, { model: image26, prompt: "fixture", images: ["source.png"], markup }, ctx), + ); await assert.rejects( generateImage(config, { model: "api/gpt-image-2", prompt: "fixture", images: ["invalid.png"] }, ctx), ); @@ -2068,6 +2165,114 @@ test("OpenAI image usage reaches the tool result with list-price estimates", asy } }); +test("GPT Image 2.6 GIF and SVG output saves validated files without previews and counts SVG authoring", async (t) => { + const model = "api/gpt-image-2.6-sunburst"; + const author = getBuiltinModels("openai").find(({ cost }) => cost.input > 0 && cost.output > 0); + assert.ok(author); + const root = + ''; + const usage = { + input_tokens: 60, + input_tokens_details: { text_tokens: 10, image_tokens: 50 }, + output_tokens: 1000, + total_tokens: 1060, + }; + let response: unknown; + const posted: unknown[] = []; + const config = parseConfig({ + baseUrl: await server(t, (req, res) => { + if (req.method === "GET") return void res.end(JSON.stringify({ data: [{ id: model }] })); + assert.equal(req.url, "/v1/images/generations"); + void body(req).then((value) => { + posted.push(value); + res.end(JSON.stringify(response)); + }); + }), + }); + const ctx = context(await home(t)); + response = { data: [{ b64_json: gif }, { b64_json: gif }] }; + // GIF frames may go below the still-image minimum pixel count. + const params = { + model, + prompt: "fixture", + output_format: "gif", + frame_count: 4, + frame_duration_ms: 120, + size: "256x256", + complexity: "low", + n: 2, + }; + const animated = await generateImage(config, params, ctx); + assert.deepEqual(posted, [params]); + assert.deepEqual( + animated.details.files.map(({ mimeType }) => mimeType), + ["image/gif", "image/gif"], + ); + assert.ok(animated.details.files[1].path.endsWith("image-2.gif")); + assert.equal((await readFile(animated.details.files[1].path)).toString("base64"), gif); + assert.ok(animated.content.every((part) => part.type === "text")); + assert.match(JSON.stringify(animated.content), /GIF and SVG results are saved without an inline preview/); + for (const [svgModel, authorCost] of [ + [author.id, (100 * author.cost.input + 2000 * author.cost.output) / 1e6], + ["unknown-author", 0], + ] as const) { + response = { + data: [{ b64_json: svg(root) }], + usage, + svg_usage: { + model: svgModel, + input_tokens: 100, + output_tokens: 2000, + reasoning_tokens: 1500, + total_tokens: 2100, + }, + }; + const vector = await generateImage( + config, + { model, prompt: "fixture", output_format: "svg", quality: "low" }, + ctx, + ); + assert.deepEqual(posted.at(-1), { model, prompt: "fixture", output_format: "svg", quality: "low", n: 1 }); + assert.equal(vector.details.files[0].mimeType, "image/svg+xml"); + assert.ok(vector.details.files[0].path.endsWith("image.svg")); + assert.equal(await readFile(vector.details.files[0].path, "utf8"), svgDocument(root)); + assert.equal(vector.content.length, 1); + assert.equal(vector.usage?.totalTokens, 1060 + 2100); + assert.equal(vector.usage?.output, 3000); + assert.ok( + Math.abs((vector.usage?.cost.total ?? 0) - ((10 * 5 + 50 * 8 + 1000 * 30) / 1e6 + authorCost)) < 1e-12, + ); + } + const before = await readdir(join(ctx.cwd, ".pi")); + for (const [output_format, data] of [ + ["gif", png], + ["gif", Buffer.from("GIF89a").toString("base64")], + ["svg", png], + ["svg", Buffer.from("\u00ff", "latin1").toString("base64")], + ["svg", svg('')], + ["svg", svg('')], + ["svg", Buffer.from(']>&x;').toString("base64")], + ...[ + "", + '', + '', + '', + '', + "", + '', + '', + "
", + ].map((element) => ["svg", svg(`${element}`)]), + ]) { + response = { data: [{ b64_json: data }] }; + await assert.rejects( + generateImage(config, { model, prompt: "fixture", output_format }, ctx), + /CLIProxyAPI returned/, + ); + } + assert.deepEqual(await readdir(join(ctx.cwd, ".pi")), before); +}); + test("partial previews stream over the image transport and only the final image is saved", async (t) => { let mode = "success"; const posted: unknown[] = []; diff --git a/test/picker.test.ts b/test/picker.test.ts index 94b03b8..35f5de5 100644 --- a/test/picker.test.ts +++ b/test/picker.test.ts @@ -1005,8 +1005,8 @@ test("media picker and listings show derived defaults without footer output or u () => {}, ); const rendered = picker.render(200).join("\n"); - assert.match(rendered, /gpt-image-2.5-sunburst.*automatic default/); - assert.match(rendered, /gpt-image-2.5-sunburst.*selected/); + assert.match(rendered, /gpt-image-2.6-sunburst.*automatic default/); + assert.match(rendered, /gpt-image-2.6-sunburst.*selected/); return undefined; }, }, @@ -1021,10 +1021,10 @@ test("media picker and listings show derived defaults without footer output or u await command.handler("list", { ...ctx, model: chat }); assert.match(output.at(-1) ?? "", /Image: none/); await command.handler("list", ctx); - assert.match(output.at(-1) ?? "", /gpt-image-2.5-sunburst.*automatic default/); + assert.match(output.at(-1) ?? "", /gpt-image-2.6-sunburst.*automatic default/); for (const mode of ["rpc", "json", "print"] as const) { await command.handler("list", { ...ctx, mode, hasUI: mode === "rpc" }); - assert.match(output.at(-1) ?? "", /gpt-image-2.5-sunburst.*automatic default/); + assert.match(output.at(-1) ?? "", /gpt-image-2.6-sunburst.*automatic default/); } await command.handler("", { ...ctx, mode: "tui" }); await command.handler(`select ${video}`, ctx); diff --git a/test/provider.test.ts b/test/provider.test.ts index d01ae82..66058be 100644 --- a/test/provider.test.ts +++ b/test/provider.test.ts @@ -2165,6 +2165,9 @@ test("previous-policy caches cannot restore GPT-image chat aliases", async (t) = "gpt-image-2.5-flare", "gpt-image-2.5-sunburst", "gpt-image-2.5", + "gpt-image-2.6-flare", + "gpt-image-2.6-sunburst", + "gpt-image-2.6", ]) { const config = parseConfig({ aliases: { [id]: "openai/gpt-fixture" } }); const provider = createCliproxyProvider(config, known); @@ -2233,6 +2236,8 @@ test("refresh registers available image models for Pi and reports unmapped chat "gemini-3-pro-image", "grok-imagine-video", "imagen-4.0-generate-001", + "gpt-image-2.6-flare", + "api/gpt-image-2.6", ]; const baseUrl = await server(t, (_req, res) => res.end( @@ -2250,10 +2255,12 @@ test("refresh registers available image models for Pi and reports unmapped chat ["gpt-image-2.5-sunburst", ["text", "image"]], ["api/gpt-image-2", ["text", "image"]], ["gemini-3-pro-image", ["text"]], + ["gpt-image-2.6-flare", ["text", "image"]], ].map(([id, input]) => ({ id, api: IMAGE_API, input, output: ["image"], baseUrl })), ); assert.deepEqual(images[0].cost, { input: 5, output: 30, cacheRead: 0, cacheWrite: 0 }); assert.deepEqual(images[2].cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }); + assert.deepEqual(images[3].cost, images[0].cost); }); test("Pi image calls reuse proxy validation, honor payload hooks, report usage, and return errors as results", async (t) => { From 1ad49c115ffb9c55e2a7303537b5da558b6b9809 Mon Sep 17 00:00:00 2001 From: hawkff <109485367+hawkff@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:35:41 -0500 Subject: [PATCH 2/5] Validate SVG output against static elements Accept only static SVG drawing elements and same-document references, decode character references before checking, and reject CSS escapes, foreign namespaces, and CDATA. Decode the format a native payload hook requests, require PNG or WebP for transparent backgrounds, and compare file signatures byte for byte. --- src/media.ts | 55 +++++++++++++++++++++++++++++++++---------- test/media.test.ts | 19 ++++++++++++++- test/provider.test.ts | 24 +++++++++++++++---- 3 files changed, 80 insertions(+), 18 deletions(-) diff --git a/src/media.ts b/src/media.ts index 491758c..9cb4115 100644 --- a/src/media.ts +++ b/src/media.ts @@ -263,7 +263,11 @@ function mediaOutputOptions(model: string, params: Partial 24 && bytes.subarray(0, 8).equals(Buffer.from("89504e470d0a1a0a", "hex")) && - bytes.toString("ascii", 12, 16) === "IHDR" + bytes.toString("latin1", 12, 16) === "IHDR" ) { extension = "png"; mimeType = "image/png"; @@ -689,8 +693,8 @@ function imageType(bytes: Buffer) { mimeType = "image/jpeg"; } else if ( bytes.length > 20 && - bytes.toString("ascii", 0, 4) === "RIFF" && - bytes.toString("ascii", 8, 12) === "WEBP" && + bytes.toString("latin1", 0, 4) === "RIFF" && + bytes.toString("latin1", 8, 12) === "WEBP" && bytes.readUInt32LE(4) + 8 === bytes.length ) { extension = "webp"; @@ -701,14 +705,24 @@ function imageType(bytes: Buffer) { return { extension, mimeType }; } -// OpenAI documents static SVG; reject scripts, animation, entities, and references outside the document. +// OpenAI documents static SVG without scripts, external resources, or raster images. Only static drawing +// elements and same-document references pass; anything the checks cannot read fails closed. +const svgElements = new Set( + `svg g defs title desc symbol use style path rect circle ellipse line polyline polygon text tspan textPath + linearGradient radialGradient stop pattern clipPath mask marker filter feBlend feColorMatrix feComponentTransfer + feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap feDistantLight feDropShadow feFlood feFuncA + feFuncB feFuncG feFuncR feGaussianBlur feMerge feMergeNode feMorphology feOffset fePointLight + feSpecularLighting feSpotLight feTile feTurbulence`.split(/\s+/), +); +// Matches declarations, CDATA, processing instructions, event handlers, CSS escapes and imports, foreign +// namespaces, and references outside the document. const unsafeSvg = - /<(?:script|foreignObject|image|iframe|embed|object|set|animate\w*|!DOCTYPE|!ENTITY)\b|\son[a-z]+\s*=|javascript:|@import|(?:href|src)\s*=\s*(?:"(?!#)|'(?!#)|[^"'\s#>])|url\(\s*(?:"(?!#)|'(?!#)|[^"'\s#)])/i; + /<[!?]|\son[a-z]+\s*=|\\|@import|image-set\(|xmlns(?::[\w.-]+)?\s*=\s*["'](?!http:\/\/www\.w3\.org\/(?:2000\/svg|1999\/xlink)["'])|(?:href|src)\s*=\s*(?:"(?!#)|'(?!#)|[^"'\s#>])|url\(\s*(?:"(?!#)|'(?!#)|[^"'\s#)])/i; // Accept GIF or SVG only when the request asked GPT Image 2.6 for that format. function outputType(bytes: Buffer, format?: string) { if (format === "gif") { - if (bytes.length > 13 && /^GIF8[79]a$/.test(bytes.toString("ascii", 0, 6)) && bytes.at(-1) === 0x3b) + if (bytes.length > 13 && /^GIF8[79]a$/.test(bytes.toString("latin1", 0, 6)) && bytes.at(-1) === 0x3b) return { extension: "gif", mimeType: "image/gif" }; throw new Error("CLIProxyAPI returned a malformed GIF."); } @@ -719,8 +733,21 @@ function outputType(bytes: Buffer, format?: string) { } catch { throw new Error("CLIProxyAPI returned an SVG that is not valid UTF-8."); } - const root = text.replace(/^\s*(?:<\?xml[^>]*\?>\s*)?(?:\s*)*/, "").trimEnd(); - if (!/^]/.test(root) || !root.endsWith("") || unsafeSvg.test(text)) + // Comments cannot run, and decoding character references keeps them from hiding text from the checks. + const body = text + .replace(/^\s*<\?xml[^>]*\?>/, "") + .replace(//g, "") + .replace(/&#(x[\da-f]+|\d+);?/gi, (_, code: string) => { + const point = /^x/i.test(code) ? Number.parseInt(code.slice(1), 16) : Number(code); + return point <= 0x10ffff ? String.fromCodePoint(point) : "\\"; + }) + .trim(); + if ( + !/^]/.test(body) || + !body.endsWith("") || + unsafeSvg.test(body) || + [...body.matchAll(/<\/?([^\s/>]*)/g)].some(([, name]) => !svgElements.has(name)) + ) throw new Error("CLIProxyAPI returned a malformed SVG or one with active or external content."); return { extension: "svg", mimeType: "image/svg+xml" }; } @@ -905,11 +932,13 @@ async function requestImages( body: payload, openaiImage: route === "openai", }); + // Decode the format actually requested, which a native payload hook may have changed. + const format = + route === "openai" && isRecord(payload) && typeof payload.output_format === "string" + ? payload.output_format + : undefined; return { - images: - route === "gemini" - ? parseGoogleImages(result) - : parseImages(result, n, route === "openai" ? output.output_format : undefined), + images: route === "gemini" ? parseGoogleImages(result) : parseImages(result, n, format), usage: route === "openai" && isRecord(result) ? imageUsage(model, result.usage, result.svg_usage) diff --git a/test/media.test.ts b/test/media.test.ts index a8dd198..4d0b464 100644 --- a/test/media.test.ts +++ b/test/media.test.ts @@ -1870,6 +1870,8 @@ test("invalid OpenAI controls and edit inputs fail before authentication or netw { model: image26, output_format: "gif", partial_images: 1 }, { model: image26, output_format: "svg", partial_images: 2 }, { model: image26, output_format: "gif", output_compression: 50 }, + { model: image26, output_format: "gif", background: "transparent" }, + { model: image26, output_format: "svg", background: "transparent" }, { model: image26, frame_count: 4 }, { model: image26, output_format: "png", frame_duration_ms: 100 }, { model: image26, output_format: "gif", frame_count: 1 }, @@ -2170,7 +2172,12 @@ test("GPT Image 2.6 GIF and SVG output saves validated files without previews an const author = getBuiltinModels("openai").find(({ cost }) => cost.input > 0 && cost.output > 0); assert.ok(author); const root = - ''; + 'Field & FlourBAKERY ©'; + const corrupt = (data: string, offset: number) => { + const bytes = Buffer.from(data, "base64"); + bytes[offset] |= 0x80; + return bytes.toString("base64"); + }; const usage = { input_tokens: 60, input_tokens_details: { text_tokens: 10, image_tokens: 50 }, @@ -2247,6 +2254,9 @@ test("GPT Image 2.6 GIF and SVG output saves validated files without previews an for (const [output_format, data] of [ ["gif", png], ["gif", Buffer.from("GIF89a").toString("base64")], + ["gif", corrupt(gif, 0)], + ["png", corrupt(png, 12)], + ["webp", corrupt(webp, 0)], ["svg", png], ["svg", Buffer.from("\u00ff", "latin1").toString("base64")], ["svg", svg('')], @@ -2262,6 +2272,13 @@ test("GPT Image 2.6 GIF and SVG output saves validated files without previews an '', '', "
", + 'alert(1)', + '