From beca839abc34d95c2604efc42bfafcb4abb410f1 Mon Sep 17 00:00:00 2001 From: Siubeng Date: Thu, 12 Nov 2026 06:35:35 +0000 Subject: [PATCH 1/4] feat: add CarPlay OEM browser settings and debug overlay --- common/src/main/assets/web/app.js | 187 ++++++++++++++++-- common/src/main/assets/web/index.html | 63 +++++- common/src/main/assets/web/style.css | 79 +++++++- .../shilapi/xcertplay/CarPlayHostActivity.kt | 1 + .../shilapi/xcertplay/web/BrowserStrings.kt | 17 ++ .../com/shilapi/xcertplay/web/LanWebServer.kt | 9 + .../com/shilapi/xcertplay/web/WebSession.kt | 5 + .../shilapi/xcertplay/web/WebStreamSender.kt | 2 +- .../shilapi/xcertplay/web/LanWebServerTest.kt | 36 ++++ docs/PROJECTION_IMPLEMENTATION.md | 10 + scripts/test-media-presentation.mjs | 158 +++++++++++++-- .../orchestration/CarPlayController.kt | 14 +- shared/src/main/res/values-zh-rCN/strings.xml | 23 ++- shared/src/main/res/values-zh-rTW/strings.xml | 23 ++- shared/src/main/res/values/strings.xml | 23 ++- .../orchestration/CarPlayHostUiTest.kt | 48 +++++ test/debug-stats.test.js | 19 ++ test/fixtures/pairing-qr.svg | 1 + test/pairing-code.test.js | 53 +++++ test/pairing.test.js | 5 +- test/stream-client.test.js | 182 ++++++++++++++--- 21 files changed, 869 insertions(+), 89 deletions(-) create mode 100644 shared/src/test/java/com/shilapi/xcertplay/orchestration/CarPlayHostUiTest.kt create mode 100644 test/debug-stats.test.js create mode 100644 test/fixtures/pairing-qr.svg create mode 100644 test/pairing-code.test.js diff --git a/common/src/main/assets/web/app.js b/common/src/main/assets/web/app.js index c26fea31..b5c6462e 100644 --- a/common/src/main/assets/web/app.js +++ b/common/src/main/assets/web/app.js @@ -42,6 +42,68 @@ export function pairingCodeFromUrl(search) { return /^[0-9]{6}$/.test(code) ? code : null; } +// Six visible digits share one canonical value for the existing pairing API. +export function createPairingCodeInput(inputs, value) { + const numbers = text => String(text || '').replace(/[^0-9]/g, ''); + const sync = () => { value.value = inputs.map(input => input.value).join(''); }; + const focus = index => { inputs[index].focus(); inputs[index].select?.(); }; + function fill(text, index = 0, advance = false) { + const digits = numbers(text).slice(0, 6); + if (digits.length === 6) index = 0; + for (let offset = 0; offset < digits.length && index + offset < inputs.length; offset++) { + inputs[index + offset].value = digits[offset]; + } + sync(); + if (advance && digits.length) focus(Math.min(5, index + digits.length)); + } + inputs.forEach((input, index) => { + input.addEventListener('focus', () => input.select?.()); + input.addEventListener('input', () => { + const text = numbers(input.value); + input.value = ''; + fill(text, index, true); + }); + input.addEventListener('paste', event => { + const text = numbers(event.clipboardData?.getData('text')); + event.preventDefault(); + if (text) fill(text, index, true); + }); + input.addEventListener('keydown', event => { + let next = index; + if (event.key === 'Backspace' && !input.value && index > 0) { + next--; inputs[next].value = ''; sync(); + } else if (event.key === 'ArrowLeft') next = Math.max(0, index - 1); + else if (event.key === 'ArrowRight') next = Math.min(5, index + 1); + else if (event.key === 'Home') next = 0; + else if (event.key === 'End') next = 5; + else return; + event.preventDefault(); focus(next); + }); + }); + const set = text => { inputs.forEach(input => { input.value = ''; }); fill(text); }; + set(value.value); + return { set, enabled(enabled) { + inputs.forEach(input => { input.disabled = !enabled; input.required = enabled; }); + } }; +} + +// Browser-local measurements: committed images per rolling second and receive-to-decode time. +export function createImageDebugStats(now = () => performance.now()) { + let started = now(), frames = [], decodeMs = null, totalFrames = 0; + return { + reset() { started = now(); frames = []; decodeMs = null; totalFrames = 0; }, + frame(receivedAt) { + const at = now(); frames.push(at); totalFrames++; + if (frames.length > 240) frames.shift(); + decodeMs = Number.isFinite(receivedAt) ? Math.max(0, at - receivedAt) : null; + }, + sample() { + const at = now(); frames = frames.filter(frame => frame > at - 1000); + return { fps: at - started >= 1000 ? frames.length : null, decodeMs, totalFrames }; + }, + }; +} + // Source availability, a committed picture and input readiness are independent. // Silence is normal for a static screen; only an explicit gap starts this deadline. export function createPresentationController({ @@ -509,7 +571,7 @@ export function createRtcReceiver(createRenderer, { const caps = RTCRtpReceiver.getCapabilities('video'); return !caps || caps.codecs.some(c => c.mimeType.toLowerCase() === `video/${codec.toLowerCase()}`); }, - schedule = setTimeout, cancel = clearTimeout, now = () => Date.now(), + schedule = setTimeout, cancel = clearTimeout, now = () => performance.now(), } = {}) { let renderer = null, track = null, previousStats = null, renderStalledSince = null, lastRendered = 0; let codec = 'H264'; @@ -590,6 +652,7 @@ export function createRtcReceiver(createRenderer, { presentedFps: presented !== null && previous?.presented !== null && elapsed ? Math.max(0, presented - previous.presented) * 1000 / elapsed : null, renderedFps: rendered !== null && previous?.rendered !== null && elapsed ? Math.max(0, rendered - previous.rendered) * 1000 / elapsed : null, renderedFrames: rendered, + presentedFrames: presented, decodeMs: averageMs('totalDecodeTime', 'framesDecoded'), jitterBufferMs: averageMs('jitterBufferDelay', 'jitterBufferEmittedCount'), processingMs: averageMs('totalProcessingDelay', 'framesDecoded'), @@ -680,16 +743,26 @@ import { createBrowserMediaSession } from './browser-media-session.js'; if (typeof document !== 'undefined') { const $ = id => document.getElementById(id); + const pairingCode = createPairingCodeInput(Array.from({ length: 6 }, (_, index) => $(`code-digit-${index + 1}`)), $('code')); const screen = $('screen'); const screens = [screen, $('screen-next'), $('screen-third')]; const heldFrame = $('held-frame'); let video = null; const canvas = $('video-canvas'); const display = $('display'); + const settingsDialog = $('settings-dialog'); const touchSurface = $('touch-surface'); // Read-only diagnostic snapshots; counters never include pairing credentials or video payloads. const rendererStats = { selectedMode: 'native', activeMode: 'waiting', renderedFps: null, renderedFrames: null, fallbackReason: null }; const performanceStats = { source: null, transport: null, receiver: null, image: null, presentation: rendererStats }; + const imageDebugStats = createImageDebugStats(); + let debugRevision = null; + const debugOption = $('settings-debug'); + try { debugOption.checked = window.localStorage.getItem('wheelplay.debugOverlay') === 'true'; } catch (_) {} + debugOption.addEventListener('change', () => { + try { window.localStorage.setItem('wheelplay.debugOverlay', String(debugOption.checked)); } catch (_) {} + renderDebug(); + }); const webpSupport = probeWebp(); Object.defineProperty(window, 'wheelplayStats', { configurable: true, get: () => JSON.parse(JSON.stringify(performanceStats)) }); let rtc = null, rtcRequested = false, rtcFps = null; @@ -700,13 +773,19 @@ if (typeof document !== 'undefined') { let launchPresentation = null; const locationOption = $('browser-location'); locationOption.checked = false; // Never inherit form restoration or grant consent through storage. + $('settings-browser-location').checked = false; const browserLocation = createBrowserLocation({ send: data => send(data), onState(enabled, message, key) { locationOption.checked = enabled; + $('settings-browser-location').checked = enabled; $('location-status').textContent = message; + $('settings-location-status').textContent = message; $('location-message').textContent = message; $('location-message').hidden = !message || ['web_location_ready', 'web_location_active'].includes(key); } }); locationOption.addEventListener('change', () => { void browserLocation.setEnabled(locationOption.checked); }); + $('settings-browser-location').addEventListener('change', () => { + void browserLocation.setEnabled($('settings-browser-location').checked); + }); const devicePreference = 'wheelplay.browserDevice'; let pairingToken = null, qrGeneration = 0, qrTimer = null, qrRequest = null; let viewportReportTimer = null, lastViewportSignature = ''; @@ -728,9 +807,15 @@ if (typeof document !== 'undefined') { for (const name of ['left', 'top', 'width', 'height', 'transform']) display.style.removeProperty(name); } } - ignoreZoomOption.addEventListener('change', () => { + function updateIgnoreZoom() { + $('settings-ignore-browser-zoom').checked = ignoreZoomOption.checked; try { window.localStorage.setItem(ignoreZoomPreference, String(ignoreZoomOption.checked)); } catch (_) {} release(); syncDisplayZoom(); scheduleViewportReport(); + } + ignoreZoomOption.addEventListener('change', updateIgnoreZoom); + $('settings-ignore-browser-zoom').addEventListener('change', () => { + ignoreZoomOption.checked = $('settings-ignore-browser-zoom').checked; + updateIgnoreZoom(); }); const presentation = createPresentationController({ onChange: renderPresentation }); const jpeg = createJpegPresenter(screens, { @@ -738,6 +823,8 @@ if (typeof document !== 'undefined') { if (meta.width && (meta.width !== frameWidth || meta.height !== frameHeight)) return false; if (!presentation.present(meta, frameWidth, frameHeight, 'jpeg', meta.revision)) return false; width = frameWidth; height = frameHeight; + imageDebugStats.frame(meta.receivedAt); + renderDebug(); clearHeldFrame(); return true; }, onError: meta => { @@ -810,6 +897,7 @@ if (typeof document !== 'undefined') { $('device-name').textContent = name; $('qr-pairing').hidden = true; $('pair-instructions').hidden = true; $('code-entry').hidden = true; $('code').required = false; $('code').disabled = true; + pairingCode.enabled(false); $('connect-label').textContent = t("web_start_display"); $('connect').disabled = false; $('status').textContent = t("web_paired_ready_to_start"); $('message').textContent = ''; @@ -818,6 +906,7 @@ if (typeof document !== 'undefined') { $('remembered-device').hidden = true; $('qr-pairing').hidden = false; $('pair-instructions').hidden = false; $('code-entry').hidden = false; $('code').required = true; $('code').disabled = false; + pairingCode.enabled(true); $('connect-label').textContent = t("web_pair_start_display"); $('connect').disabled = false; } @@ -867,7 +956,7 @@ if (typeof document !== 'undefined') { } } if (linkedCode) { - $('code').value = linkedCode; + pairingCode.set(linkedCode); try { const result = await pairApi('code', { code: linkedCode }); if (generation !== authGeneration) return; @@ -1012,11 +1101,25 @@ if (typeof document !== 'undefined') { ? { type: 'touch', contacts: [] } : { type: 'input', kind: 'release' }); } + function openSettings() { + if (display.hidden || settingsDialog.open) return; + release(); + $('settings-ignore-browser-zoom').checked = ignoreZoomOption.checked; + settingsDialog.showModal(); + presentation.refresh(); + } + $('settings-close').addEventListener('click', () => settingsDialog.close()); + settingsDialog.addEventListener('close', () => presentation.refresh()); + $('settings-fullscreen').addEventListener('click', toggleFullscreen); + $('settings-disconnect').addEventListener('click', () => disconnect()); function renderPresentation(value) { + if (debugRevision !== value.revision) { + debugRevision = value.revision; imageDebugStats.reset(); performanceStats.receiver = null; + } if (value.revision !== desktopRevision) { desktop.release(false); desktopRevision = value.revision; } - live = value.interactive; + live = value.interactive && !settingsDialog.open; for (const surface of [...screens, video, canvas, heldFrame].filter(Boolean)) surface.classList.toggle('stale', !value.visible); touchSurface.classList.toggle('stale', !live); $('waiting').hidden = value.visible; @@ -1032,6 +1135,41 @@ if (typeof document !== 'undefined') { t('web_video_input_unavailable', transport) : t('web_connected_waiting_for_iphone_video'); $('media-message').hidden = !value.visible || value.interactive; $('media-message').textContent = value.visible && !value.interactive ? $('status').textContent : ''; + renderDebug(); + } + function renderDebug() { + const enabled = debugOption.checked && !display.hidden; + $('debug-overlay').hidden = !enabled; + display.classList.toggle('debug-visible', enabled); + if (!enabled) return; + const current = presentation.current, showing = current.state === 'showing'; + const isRtc = current.frame?.transport === 'rtc'; + const receiver = showing && isRtc ? performanceStats.receiver : null; + const image = imageDebugStats.sample(); + const number = (value, suffix = '') => Number.isFinite(value) && value >= 0 ? `${value.toFixed(1)}${suffix}` : '—'; + $('debug-connection').textContent = $('status').textContent; + $('debug-endpoint').textContent = `${location.protocol === 'https:' ? 'WSS' : 'WS'} · ${location.host}`; + $('debug-video').textContent = current.frame + ? `${current.frame.width} × ${current.frame.height}${isRtc ? ` · WebRTC / ${rendererStats.activeMode}` : ''}` : '—'; + $('debug-format').textContent = current.frame + ? isRtc ? rtc?.codec === 'H265' ? 'HEVC' : rtc?.codec === 'H264' ? 'H.264' : '—' : current.frame.mime === 'image/webp' ? 'WebP' : 'JPEG' : '—'; + const presentedFps = launchMode === 'native' ? receiver?.presentedFps : receiver?.renderedFps; + $('debug-fps-summary').textContent = number(showing ? isRtc ? presentedFps : image.fps : null, ' fps'); + $('debug-fps-row').hidden = !isRtc; + $('debug-fps-label').textContent = t(isRtc ? 'web_debug_rtc_fps' : 'web_debug_image_fps'); + $('debug-fps').textContent = isRtc + ? `${number(receiver?.receivedFps)} / ${number(receiver?.decodedFps)} / ${number(launchMode === 'native' ? receiver?.presentedFps : receiver?.renderedFps)} fps` + : number(showing ? image.fps : null, ' fps'); + const totalFrames = isRtc ? launchMode === 'native' ? receiver?.presentedFrames : receiver?.renderedFrames : showing ? image.totalFrames : null; + $('debug-frames').textContent = Number.isSafeInteger(totalFrames) && totalFrames >= 0 ? String(totalFrames) : '—'; + $('debug-latency-label').textContent = t(isRtc ? 'web_debug_rtt' : 'web_debug_image_ack'); + $('debug-summary-latency').title = $('debug-latency-label').textContent; + const ack = performanceStats.image?.lastAckMs; + $('debug-latency').textContent = number(isRtc ? receiver?.rttMs : showing && ack > 0 ? ack : null, ' ms'); + $('debug-decode-label').textContent = t(isRtc ? 'web_debug_decode' : 'web_debug_image_decode'); + $('debug-decode').textContent = number(isRtc ? receiver?.decodeMs : showing ? image.decodeMs : null, ' ms'); + $('debug-buffer-row').hidden = !isRtc; + $('debug-buffer').textContent = number(receiver?.jitterBufferMs, ' ms'); } function clearHeldFrame() { heldFrame.hidden = true; heldFrame.width = heldFrame.height = 0; @@ -1053,13 +1191,14 @@ if (typeof document !== 'undefined') { if (rtc) rtc.close(); rtc = null; rtcRequested = false; rtcFps = null; video = null; performanceStats.source = performanceStats.transport = performanceStats.receiver = performanceStats.image = null; + imageDebugStats.reset(); rendererStats.activeMode = 'waiting'; } function exitLaunchFullscreen() { try { Promise.resolve(document.exitFullscreen()).catch(() => {}); } catch (_) {} } function prepareLaunchPresentation() { - const presentation = { immersive: $('fullscreen-on-start').checked, ownsFullscreen: false, cancelled: false }; + const presentation = { immersive: true, ownsFullscreen: false, cancelled: false }; launchPresentation = presentation; if (!presentation.immersive || document.fullscreenElement || !document.documentElement?.requestFullscreen) return; presentation.ownsFullscreen = true; @@ -1077,6 +1216,7 @@ if (typeof document !== 'undefined') { if (launchPresentation.ownsFullscreen && document.fullscreenElement) exitLaunchFullscreen(); } function disconnect(refreshQr = true) { + if (settingsDialog.open) settingsDialog.close(); stopQr(); authGeneration++; restoreLaunchPresentation(); @@ -1089,8 +1229,9 @@ if (typeof document !== 'undefined') { browserLocation.close(); $('topbar').hidden = false; $('pairing').hidden = false; $('display').hidden = true; syncDisplayZoom(); - $('disconnect').hidden = true; $('connect').disabled = false; + $('connect').disabled = false; $('status').textContent = t("web_disconnected"); + renderDebug(); if (refreshQr) initializePairing(); } function connect() { @@ -1139,6 +1280,7 @@ if (typeof document !== 'undefined') { onStop: reason => { if (socket !== ws) return; video = null; rtcFps = null; rendererStats.fallbackReason = reason; rendererStats.renderedFps = null; + performanceStats.receiver = null; presentation.invalidate('rtc'); }, onStats: (fps, metrics) => { @@ -1151,11 +1293,12 @@ if (typeof document !== 'undefined') { $('message').textContent = ''; $('phone-connect').disabled = false; $('phone-connect-message').textContent = t("web_keep_wheelplay_in_the_foreground_on_android_tap_connect"); - $('pairing').hidden = true; $('display').hidden = false; $('disconnect').hidden = false; - $('topbar').hidden = $('hide-toolbar').checked || Boolean(launchPresentation?.immersive); + $('pairing').hidden = true; $('display').hidden = false; + $('topbar').hidden = true; lastViewportSignature = ''; scheduleViewportReport(); $('status').textContent = t("web_connected_waiting_for_video"); + renderDebug(); send({ type: 'ping' }); clearInterval(heartbeat); heartbeat = setInterval(() => { @@ -1170,6 +1313,7 @@ if (typeof document !== 'undefined') { if (socket !== ws || ws.readyState !== WebSocket.OPEN) return; if (typeof event.data === 'string') { const data = JSON.parse(event.data); + if (data.type === 'open-settings') { openSettings(); return; } if (browserMedia.receive(data)) return; if (browserAudio.control(data)) return; if (data.type === 'rtc-offer' || data.type === 'rtc-stop') { @@ -1235,6 +1379,7 @@ if (typeof document !== 'undefined') { if (previousRevision !== presentation.current.revision) { rtc.close(); rtcRequested = false; } if (rtc?.active) { acknowledge(); return; } meta.revision = presentation.current.revision; + meta.receivedAt = performance.now(); const payload = packet ? packet.jpeg : event.data; jpeg.load(payload instanceof ArrayBuffer ? new Blob([payload], { type: packet?.mime || 'image/jpeg' }) : payload, meta, () => socket === ws && ws.readyState === WebSocket.OPEN && !rtc?.active && presentation.current.revision === meta.revision, @@ -1243,6 +1388,7 @@ if (typeof document !== 'undefined') { ws.onclose = () => { clearTimeout(connectTimer); if (socket !== ws) return; + if (settingsDialog.open) settingsDialog.close(); if (outbox) { outbox.close(); outbox = null; } cancelMove(); socket = null; pointers.clear(); desktop.release(false); pointerInput = false; keyboardInput = false; backButton = false; $('desktop-back').hidden = true; @@ -1257,6 +1403,7 @@ if (typeof document !== 'undefined') { } $('status').textContent = t("web_connection_interrupted_retrying"); $('stage').textContent = t("web_check_that_the_server_and_head_unit_are_still"); + renderDebug(); const generation = authGeneration; retry = setTimeout(async () => { if (deviceCredential) { @@ -1316,20 +1463,26 @@ if (typeof document !== 'undefined') { $('phone-connect').disabled = true; $('phone-connect-message').textContent = t("web_requesting_iphone_connection"); }); - $('disconnect').addEventListener('click', () => disconnect()); - $('fullscreen').addEventListener('click', async () => { + async function toggleFullscreen() { try { if (document.fullscreenElement) await document.exitFullscreen(); - else if (document.documentElement.requestFullscreen) await document.documentElement.requestFullscreen(); + else if (document.documentElement.requestFullscreen) { + const launch = launchPresentation; + await document.documentElement.requestFullscreen(); + if (launch && !launch.cancelled && launch === launchPresentation) launch.ownsFullscreen = true; + else if (launch?.cancelled && !launchPresentation?.immersive) exitLaunchFullscreen(); + } else $('status').textContent = t("web_this_browser_does_not_support_full_screen"); } catch (_) { $('status').textContent = t("web_use_the_browser_menu_to_toggle_full_screen"); } - }); + } + $('fullscreen').addEventListener('click', toggleFullscreen); document.addEventListener('fullscreenchange', () => { $('fullscreen').textContent = document.fullscreenElement ? t("web_exit_full_screen") : t("web_full_screen"); - if (!document.fullscreenElement && launchPresentation?.immersive) { - launchPresentation.immersive = false; - launchPresentation.ownsFullscreen = false; - $('topbar').hidden = false; + $('settings-fullscreen').textContent = $('fullscreen').textContent; + if (launchPresentation && !launchPresentation.cancelled) { + launchPresentation.immersive = Boolean(document.fullscreenElement); + if (!document.fullscreenElement) launchPresentation.ownsFullscreen = false; + $('topbar').hidden = paired && !display.hidden; } scheduleViewportReport(); }); @@ -1383,7 +1536,7 @@ if (typeof document !== 'undefined') { window.addEventListener('pageshow', event => { if (event.persisted) initializePairing(); }); const linkedCode = pairingCodeFromUrl(window.location?.search || ''); if (linkedCode) { - $('code').value = linkedCode; + pairingCode.set(linkedCode); // Keep the short-lived code out of copied URLs and browser history entries. window.history?.replaceState(null, '', `${window.location.pathname}${window.location.hash}`); } diff --git a/common/src/main/assets/web/index.html b/common/src/main/assets/web/index.html index 3e3c1846..7616476b 100644 --- a/common/src/main/assets/web/index.html +++ b/common/src/main/assets/web/index.html @@ -10,14 +10,35 @@ -
WheelPlay
{{ui_waiting_for_connection}}
+
WheelPlay
{{ui_waiting_for_connection}}
{{web_local_network_head_unit_display}}

{{web_connect_your_carplay_display}}

-

{{web_for_the_first_connection_tap_scan_qr_code_to}}
{{web_then_scan_the_qr_code_below_and_allow_the}}

- - +
+
+ +
+
+

{{web_scan_to_connect_the_head_unit}}

+

{{web_for_the_first_connection_tap_scan_qr_code_to}} {{web_then_scan_the_qr_code_below_and_allow_the}}

+

{{web_generating_qr_code}}

+
+
{{web_six_digit_pairing_code_on_the_device}} +
+ + + + +
+ +
+ +
{{web_connect_the_device_and_head_unit_to_the_same}}
{{web_audio_and_microphone_use_android_by_default_enable_browser}}
+
+
+

{{ui_settings}}

+
{{web_video_renderer_title}}
@@ -27,13 +48,37 @@

{{web_connect_your_carplay_display}}

{{web_browser_microphone_requires_https}}

{{web_local_http_pages_cannot_request_microphone_permission_in_the}}

{{web_loading}}

- -

{{web_scan_to_connect_the_head_unit}}

{{web_generating_qr_code}}

-

{{web_you_can_also_enter_the_pairing_code}}

- -
{{web_connect_the_device_and_head_unit_to_the_same}}
{{web_audio_and_microphone_use_android_by_default_enable_browser}}
+
+
+
+ + diff --git a/common/src/main/assets/web/style.css b/common/src/main/assets/web/style.css index e5a35ecd..0a05823f 100644 --- a/common/src/main/assets/web/style.css +++ b/common/src/main/assets/web/style.css @@ -96,3 +96,16 @@ #debug-overlay .debug-summary svg{display:block;width:16px;height:16px;fill:none;stroke:#b4d9b1;stroke-width:1.8;stroke-linecap:round;stroke-linejoin:round} #debug-overlay .debug-summary dd{text-align:left;white-space:nowrap;overflow:hidden;text-overflow:ellipsis;font-size:13px;font-weight:600} @media(max-width:480px),(max-height:480px){#debug-overlay .debug-summary{gap:8px;padding-bottom:10px}#debug-overlay .debug-summary>div{gap:4px}#debug-overlay .debug-summary dd{font-size:12px}} + +#code-entry .pin-digit.keypad-active{border-color:#b4d9b1;box-shadow:0 0 0 2px #b4d9b1;background:#24352c} +#pairing-keypad{position:fixed;inset:auto;margin:0;padding:12px;width:304px;max-width:calc(100% - 24px);overflow:auto;overscroll-behavior:contain;z-index:10;border:1px solid #58645e;border-radius:16px;background:#19221f;color:#eef2ef;box-shadow:0 12px 48px #0009;--keypad-key-height:48px} +.keypad-heading{display:flex;align-items:center;justify-content:space-between;gap:8px;min-height:40px;margin-bottom:10px} +#keypad-title{font-size:12px;font-weight:500;color:#b5beb9} +#keypad-code{font:18px/1.4 ui-monospace,SFMono-Regular,Consolas,monospace;white-space:nowrap;color:#b4d9b1} +#keypad-done{min-width:44px;min-height:40px;padding:0 8px;border:0;color:#b4d9b1;font-size:13px} +.keypad-keys{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:6px} +#pairing-keypad .keypad-keys button{min-width:0;height:var(--keypad-key-height);min-height:var(--keypad-key-height);padding:0 8px;border:1px solid #46514e;border-radius:8px;background:#25312c;font-size:22px;line-height:1;touch-action:manipulation;-webkit-tap-highlight-color:transparent} +#pairing-keypad .keypad-keys button:active{background:#b4d9b1;color:#17251b} +#pairing-keypad #keypad-clear{font-size:13px;background:transparent;color:#cbd5cf} +#keypad-delete svg{display:block;width:24px;height:24px;margin:auto;fill:none;stroke:currentColor;stroke-width:1.8;stroke-linecap:round;stroke-linejoin:round} +@media(max-height:480px){body.pin-keypad-open #topbar{display:none}} diff --git a/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt b/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt index 139dc8c2..b59df082 100644 --- a/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt +++ b/common/src/main/java/com/shilapi/xcertplay/web/BrowserStrings.kt @@ -9,6 +9,11 @@ import org.json.JSONObject internal object BrowserStrings { private val resources = mapOf( + "web_keypad_title" to UiR.string.web_keypad_title, + "web_keypad_done" to UiR.string.web_keypad_done, + "web_keypad_clear" to UiR.string.web_keypad_clear, + "web_keypad_delete" to UiR.string.web_keypad_delete, + "web_pairing_code_incomplete" to UiR.string.web_pairing_code_incomplete, "web_debug_format" to UiR.string.web_debug_format, "web_debug_image_decode" to UiR.string.web_debug_image_decode, "web_debug_frames" to UiR.string.web_debug_frames, diff --git a/docs/PROJECTION_IMPLEMENTATION.md b/docs/PROJECTION_IMPLEMENTATION.md index 5b04c1e1..34253f9c 100644 --- a/docs/PROJECTION_IMPLEMENTATION.md +++ b/docs/PROJECTION_IMPLEMENTATION.md @@ -12,6 +12,8 @@ Web 服务和浏览器配对不再依赖 MFi 身份。CarPlay 的连接入口仍执行原有认证检查,包括 USB 外部启动。 +浏览器手动配对使用六个明文只读输入框和页面内的浮动数字键盘,避免触发系统软键盘;实体数字键、退格、方向键和粘贴仍使用同一配对码控制器。完整性校验放在提交按钮上,防止只读字段跳过浏览器校验。浮层优先位于输入框下方,再尝试上方或侧边,按 visual viewport 的大小、偏移和缩放定位;短屏临时收起首页顶栏、压缩按键高度并按需补充滚动空间,让输入框和键盘同时可见;键盘关闭后恢复顶栏。极短视口下浮层自身可滚动,顶部始终显示当前六位内容。点击外部、按 Escape、收起、配对成功或启动显示时关闭;主动滚离输入框不会被浮层反复拉回。 + 在手机连接页面的「投影输入源」卡片选择「选择诊断输入源」: | 源 | 视频路径 | 交互 | diff --git a/scripts/test-media-presentation.mjs b/scripts/test-media-presentation.mjs index 79378bc0..b9e55470 100644 --- a/scripts/test-media-presentation.mjs +++ b/scripts/test-media-presentation.mjs @@ -420,12 +420,70 @@ try { assert.ok(home.rightTop >= home.leftBottom, 'Narrow home places settings below pairing'); assert.equal(home.overflow, false); await screenshot('home-mobile.png'); + const keypadGeometry = `(() => { + const panel=document.getElementById('pairing-keypad'), anchor=document.getElementById('pin-inputs'), visual=visualViewport; + return {hidden:panel.hidden,panel:panel.getBoundingClientRect().toJSON(),anchor:anchor.getBoundingClientRect().toJSON(),content:document.getElementById('main-content').getBoundingClientRect().toJSON(), + viewport:{left:visual?.offsetLeft||0,top:visual?.offsetTop||0,width:visual?.width||innerWidth,height:visual?.height||innerHeight}, + placement:panel.dataset.placement,readonly:[...anchor.querySelectorAll('input')].every(input=>input.readOnly&&input.inputMode==='none')}; + })()`; + const tapKey = async id => { + const point=await evaluate(`(()=>{const r=document.getElementById('${id}').getBoundingClientRect();return{x:r.left+r.width/2,y:r.top+r.height/2}})()`); + await command('Input.dispatchTouchEvent',{type:'touchStart',touchPoints:[{...point,id:1}]}); + await command('Input.dispatchTouchEvent',{type:'touchEnd',touchPoints:[]}); + await pause(30); + }; + for (const [width,height,name] of [[1360,1000,'wide'],[390,844,'mobile'],[800,360,'landscape'],[320,320,'short']]) { + await evaluate(`document.getElementById('keypad-done').click()`); + await command('Emulation.setDeviceMetricsOverride',{width,height,deviceScaleFactor:1,mobile:false}); + await evaluate(`{document.getElementById('main-content').scrollTop=0;const field=document.getElementById('code-digit-1');field.focus({preventScroll:true});field.click()}`); + await wait(`!document.getElementById('pairing-keypad').hidden`); + await pause(80); + const geometry=await evaluate(keypadGeometry), {panel,anchor,viewport}=geometry; + assert.equal(geometry.hidden,false,`${name}: keypad remains open after bringing the field into view`); + assert.equal(geometry.readonly,true,'PIN fields cannot summon the system keyboard'); + assert.ok(panel.left>=viewport.left+10&&panel.top>=viewport.top+10&&panel.right<=viewport.left+viewport.width-10&&panel.bottom<=viewport.top+viewport.height-10,`${name}: keypad fits the visual viewport`); + assert.ok(panel.right<=anchor.left||panel.left>=anchor.right||panel.bottom<=anchor.top||panel.top>=anchor.bottom,`${name}: keypad does not cover the six-digit input`); + assert.ok(anchor.top>=Math.max(viewport.top,geometry.content.top)+10&&anchor.bottom<=Math.min(viewport.top+viewport.height,geometry.content.bottom)-10,`${name}: input remains visible below the header`); + await tapKey('keypad-clear'); + for (const digit of '012') await tapKey('keypad-'+digit); + assert.equal(await evaluate(`document.getElementById('code').value`),'012'); + assert.equal(await evaluate(`document.getElementById('keypad-code').textContent`),'0 1 2 · · ·'); + await screenshot('keypad-'+name+'.png'); + if (name === 'wide') { + await command('Browser.grantPermissions',{origin:`http://127.0.0.1:${server.address().port}`,permissions:['clipboardReadWrite','clipboardSanitizedWrite']}); + await evaluate(`navigator.clipboard.writeText('098765')`,{userGesture:true}); + await evaluate(`document.getElementById('code-digit-1').focus({preventScroll:true})`); + await command('Input.dispatchKeyEvent',{type:'keyDown',key:'v',code:'KeyV',modifiers:2,windowsVirtualKeyCode:86}); + await command('Input.dispatchKeyEvent',{type:'keyUp',key:'v',code:'KeyV',modifiers:2,windowsVirtualKeyCode:86}); + await wait(`document.getElementById('code').value==='098765'`); + await tapKey('keypad-clear'); + for (const digit of '012') await tapKey('keypad-'+digit); + } + for (const digit of '345') await tapKey('keypad-'+digit); + assert.equal(await evaluate(`document.getElementById('code').value`),'012345'); + assert.equal(await evaluate(`document.getElementById('connect-form').checkValidity()`),true); + await tapKey('keypad-delete'); + assert.equal(await evaluate(`document.getElementById('code').value`),'01234'); + await tapKey('keypad-done'); + assert.equal(await evaluate(`document.getElementById('pairing-keypad').hidden`),true); + } + await command('Emulation.setPageScaleFactor',{pageScaleFactor:1.5}); + await evaluate(`document.getElementById('code-digit-1').click()`); + await pause(100); + const zoomKeypad=await evaluate(keypadGeometry); + assert.equal(zoomKeypad.hidden,false); + assert.ok(zoomKeypad.panel.left>=zoomKeypad.viewport.left&&zoomKeypad.panel.right<=zoomKeypad.viewport.left+zoomKeypad.viewport.width); + assert.ok(zoomKeypad.panel.top>=zoomKeypad.viewport.top&&zoomKeypad.panel.bottom<=zoomKeypad.viewport.top+zoomKeypad.viewport.height); + await evaluate(`document.getElementById('setup-title').dispatchEvent(new PointerEvent('pointerdown',{bubbles:true}));document.getElementById('setup-title').click()`); + assert.equal(await evaluate(`document.getElementById('pairing-keypad').hidden`),true,'Outside tap dismisses the keypad'); + await command('Emulation.setPageScaleFactor',{pageScaleFactor:1}); await command('Emulation.clearDeviceMetricsOverride'); assert.deepEqual(errors, []); console.log(JSON.stringify({ passed: true, connections, imageAcks: acks, webpProbes, jpegFallbacks, nativeFullscreenEntered, blockedChecks: nativeFullscreenEntered ? [] : ['Native fullscreen did not enter in this browser environment; OEM fullscreen transitions were not exercised.'], - checks: ['responsive home columns and stacking', 'six-digit PIN validation and paste', 'default fullscreen request and hidden toolbar', + checks: ['responsive home columns and stacking', 'six-digit PIN validation and paste', 'floating keypad on wide, narrow, short and zoomed viewports', + 'trusted keypad touches, deletion, leading zeroes and outside dismissal', 'default fullscreen request and hidden toolbar', ...(nativeFullscreenEntered ? ['OEM native fullscreen toggle'] : []), 'OEM debug toggle, top-right position, contrast over white video and touch passthrough', 'OEM settings centered modal, focus, input isolation and Escape', 'five-touch source', 'read-only source', 'WebP capability probe', 'WebP touch with zoom compensation', 'JPEG fallback after corrupt WebP', 'two-image pipeline with exact ACKs', diff --git a/shared/src/main/res/values-zh-rCN/strings.xml b/shared/src/main/res/values-zh-rCN/strings.xml index b71c02b7..9eb86570 100644 --- a/shared/src/main/res/values-zh-rCN/strings.xml +++ b/shared/src/main/res/values-zh-rCN/strings.xml @@ -718,4 +718,9 @@ 已呈现帧数 图像收帧至呈现 画面格式 + 数字键盘 + 收起 + 清空 + 删除上一位 + 请完整输入六位配对码。 diff --git a/shared/src/main/res/values-zh-rTW/strings.xml b/shared/src/main/res/values-zh-rTW/strings.xml index 3f841371..6dfb33b8 100644 --- a/shared/src/main/res/values-zh-rTW/strings.xml +++ b/shared/src/main/res/values-zh-rTW/strings.xml @@ -718,4 +718,9 @@ 已呈現影格數 影像收幀至呈現 畫面格式 + 數字鍵盤 + 收起 + 清空 + 刪除上一位 + 請完整輸入六位配對碼。 diff --git a/shared/src/main/res/values/strings.xml b/shared/src/main/res/values/strings.xml index adb2f5ab..dc1baf57 100644 --- a/shared/src/main/res/values/strings.xml +++ b/shared/src/main/res/values/strings.xml @@ -718,4 +718,9 @@ Presented frames Image receive-to-present time Video format + Number pad + Done + Clear + Delete last digit + Enter all six digits of the pairing code. diff --git a/test/pairing-code.test.js b/test/pairing-code.test.js index a698ff6d..070fc445 100644 --- a/test/pairing-code.test.js +++ b/test/pairing-code.test.js @@ -1,6 +1,6 @@ import test from 'node:test'; import assert from 'node:assert/strict'; -import { createPairingCodeInput } from '../common/src/main/assets/web/app.js'; +import { createPairingCodeInput, placePairingKeypad } from '../common/src/main/assets/web/app.js'; function fixture() { let focused = -1; @@ -51,3 +51,52 @@ test('remembered pairing disables hidden required inputs and link prefill replac f.control.enabled(true); assert.ok(f.inputs.every(input => !input.disabled && input.required)); }); + +test('virtual digits, backspace and clear share the same six-digit value and validation', () => { + const f = fixture(); + for (const digit of '012345') f.control.digit(digit); + assert.equal(f.value.value, '012345'); assert.equal(f.control.complete, true); + f.control.backspace(); + assert.equal(f.value.value, '01234'); assert.equal(f.control.complete, false); + assert.equal(f.focused(), 5); + f.control.backspace(); assert.equal(f.value.value, '0123'); assert.equal(f.focused(), 4); + f.control.clear(); assert.equal(f.value.value, ''); assert.equal(f.focused(), 0); + f.control.enabled(false); f.control.digit('9'); + assert.equal(f.value.value, ''); +}); + +test('hardware digits and Delete work without making the fields editable', () => { + const f = fixture(); + for (let i = 0; i < 6; i++) f.key(i, String(i)); + assert.equal(f.value.value, '012345'); + f.key(2, 'Delete'); assert.equal(f.inputs[2].value, ''); + f.key(2, '9'); assert.equal(f.value.value, '019345'); +}); + +test('keypad chooses an unobstructed side and stays inside the visible viewport', () => { + const viewport = { left: 0, top: 0, width: 800, height: 360 }, size = { width: 304, height: 264 }; + const cases = [ + [{ left: 40, right: 340, top: 100, bottom: 158 }, viewport, size, 'right'], + [{ left: 450, right: 750, top: 100, bottom: 158 }, viewport, size, 'left'], + [{ left: 40, right: 340, top: 100, bottom: 158 }, { ...viewport, height: 1000 }, size, 'below'], + [{ left: 40, right: 340, top: 700, bottom: 758 }, { ...viewport, height: 1000 }, size, 'above'], + [{ left: 135, right: 485, top: 112, bottom: 170 }, { left: 120, top: 90, width: 390, height: 300 }, { width: 304, height: 200 }, 'below'], + ]; + for (const [anchor, view, panel, expected] of cases) { + const position = placePairingKeypad(anchor, view, panel); + assert.equal(position.placement, expected); + assert.ok(position.left >= view.left + 12 && position.top >= view.top + 12); + assert.ok(position.left + panel.width <= view.left + view.width - 12); + assert.ok(position.top + panel.height <= view.top + view.height - 12); + assert.ok(position.left >= anchor.right || position.left + panel.width <= anchor.left || + position.top >= anchor.bottom || position.top + panel.height <= anchor.top); + } +}); + +test('keypad requests scrolling when the anchor is clipped or no full-size placement fits', () => { + const view = { left: 0, top: 0, width: 320, height: 320 }; + assert.equal(placePairingKeypad({ left: 12, right: 308, top: 280, bottom: 338 }, view, { width: 296, height: 230 }), null); + assert.equal(placePairingKeypad({ left: 12, right: 308, top: 130, bottom: 188 }, view, { width: 296, height: 270 }), null); + assert.deepEqual(placePairingKeypad({ left: 12, right: 308, top: 12, bottom: 70 }, view, { width: 296, height: 230 }), + { left: 12, top: 78, placement: 'below' }); +}); From 9c6cf2562d47e190c43e01757e07573f9db87db3 Mon Sep 17 00:00:00 2001 From: Siubeng Date: Thu, 12 Nov 2026 15:18:31 +0000 Subject: [PATCH 3/4] test: update Android browser pairing markup assertions --- .../xcertplay/web/BrowserPageRuntimeTest.kt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/common/src/androidTest/java/com/shilapi/xcertplay/web/BrowserPageRuntimeTest.kt b/common/src/androidTest/java/com/shilapi/xcertplay/web/BrowserPageRuntimeTest.kt index 93c1a2d7..534d51d4 100644 --- a/common/src/androidTest/java/com/shilapi/xcertplay/web/BrowserPageRuntimeTest.kt +++ b/common/src/androidTest/java/com/shilapi/xcertplay/web/BrowserPageRuntimeTest.kt @@ -53,7 +53,20 @@ class BrowserPageRuntimeTest { html.contains(owner.getString(UiR.string.web_connect_your_carplay_display)) ) assertFalse(html.contains("{{")) - assertTrue(html.contains("pattern=\"[0-9]{6}\"")) + val inputs = Regex("""]*>""").findAll(html).map { it.value }.toList() + val pinFields = inputs.filter { Regex("""\bid="code-digit-[0-9]+"""").containsMatchIn(it) } + assertEquals("$tag $path: six PIN fields", 6, pinFields.size) + for (digit in 1..6) { + val field = pinFields.single { it.contains("id=\"code-digit-$digit\"") } + assertTrue("$tag $path: digit $digit accepts one number", field.contains("pattern=\"[0-9]\"")) + assertTrue("$tag $path: digit $digit suppresses the system keyboard", field.contains("inputmode=\"none\"")) + assertTrue("$tag $path: digit $digit is readonly", Regex("""\sreadonly(?:\s|>)""").containsMatchIn(field)) + assertTrue("$tag $path: digit $digit opens the page keypad", field.contains("aria-controls=\"pairing-keypad\"")) + } + assertTrue("$tag $path: canonical pairing value", inputs.any { + it.contains("id=\"code\"") && it.contains("name=\"code\"") && it.contains("type=\"hidden\"") + }) + assertTrue("$tag $path: floating keypad", html.contains("id=\"pairing-keypad\"")) assertTrue(html.contains("")) assertTrue(html.contains("")) } From cefdf5ce455d469c1d2eca267db7c70a6c1f968e Mon Sep 17 00:00:00 2001 From: Siubeng Date: Thu, 12 Nov 2026 15:52:33 +0000 Subject: [PATCH 4/4] test: await AirPlay listener port release before rebinding --- .../network/CarPlayAirPlayPortTest.kt | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/shared/src/test/java/com/shilapi/xcertplay/network/CarPlayAirPlayPortTest.kt b/shared/src/test/java/com/shilapi/xcertplay/network/CarPlayAirPlayPortTest.kt index e3a68ef6..8b47a634 100644 --- a/shared/src/test/java/com/shilapi/xcertplay/network/CarPlayAirPlayPortTest.kt +++ b/shared/src/test/java/com/shilapi/xcertplay/network/CarPlayAirPlayPortTest.kt @@ -17,6 +17,7 @@ import com.shilapi.xcertplay.transport.Iap2WirelessCarPlayEndpoint import com.shilapi.xcertplay.transport.Iap2WirelessControlClient import com.shilapi.xcertplay.transport.Iap2WirelessSecurity import java.io.IOException +import java.net.BindException import java.net.InetAddress import java.net.InetSocketAddress import java.net.ServerSocket @@ -80,8 +81,7 @@ class CarPlayAirPlayPortTest { service.detach() assertTrue(server.isClosed) assertReleased(service) - ServerSocket().use { replacement -> - replacement.bind(InetSocketAddress(loopback, port)) + bindReleasedPort(port).use { replacement -> assertEquals(port, replacement.localPort) } } @@ -121,6 +121,10 @@ class CarPlayAirPlayPortTest { service.detach() assertTrue(first.isClosed) + // A closed Java socket can retain its native descriptor until the blocked + // accept call unwinds. Establish availability before testing port preference. + // This reservation never accepts, so its close has no pending reader. + bindReleasedPort(preferredPort).close() assertStarted(service, attach(service, preferredPort)) assertEquals(preferredPort, listenerSocket(service).localPort) } @@ -182,8 +186,7 @@ class CarPlayAirPlayPortTest { service.detach() assertTrue(firstServer.isClosed) - ServerSocket().use { occupied -> - occupied.bind(InetSocketAddress(loopback, firstPort)) + bindReleasedPort(firstPort).use { occupied -> val replacement = assertStarted(service, attach(service, firstPort)) assertNotEquals(firstPort, replacement.port) assertEquals(firstPort, first.port) @@ -197,6 +200,25 @@ class CarPlayAirPlayPortTest { try { check(lifecycle.get()) } finally { lifecycle.destroy() } } + /** Wait for kernel release and retain the successful binding for the caller. */ + private fun bindReleasedPort(port: Int): ServerSocket { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + var lastFailure: BindException? = null + do { + val socket = ServerSocket() + try { + socket.bind(InetSocketAddress(loopback, port)) + return socket + } catch (error: Exception) { + socket.close() + if (error !is BindException) throw error + lastFailure = error + } + Thread.sleep(10) + } while (System.nanoTime() < deadline) + throw AssertionError("Detached AirPlay port $port did not become bindable within 5s", lastFailure) + } + private fun attach( service: CarPlayVpnService, port: Int,