From c9a7c55531533f60e24651a99d509f721e195051 Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sun, 1 Nov 2026 02:09:04 -0500 Subject: [PATCH 01/10] Build bounded camera-to-action learning workflow and local Studio --- .../camera-action-v01/independent_audit.py | 276 ++++++ docs/camera-action-v01.md | 102 +++ pyproject.toml | 2 +- src/dvidia_training/arm_skill_install.py | 13 + src/dvidia_training/camera_action_capture.py | 560 ++++++++++++ src/dvidia_training/camera_action_export.py | 128 +++ src/dvidia_training/camera_action_model.py | 463 ++++++++++ src/dvidia_training/camera_action_skill.py | 131 +++ src/dvidia_training/camera_action_studio.html | 44 + src/dvidia_training/camera_action_studio.py | 230 +++++ src/dvidia_training/camera_action_study.py | 861 ++++++++++++++++++ src/dvidia_training/footage_pipeline.py | 3 + src/dvidia_training/piper_action_capture.py | 13 +- src/dvidia_training/training_studio.html | 4 +- src/dvidia_training/training_studio.py | 26 +- tests/test_camera_action_audit.py | 155 ++++ tests/test_camera_action_capture.py | 288 ++++++ tests/test_camera_action_export.py | 145 +++ tests/test_camera_action_model.py | 239 +++++ tests/test_camera_action_skill.py | 102 +++ tests/test_camera_action_studio.py | 201 ++++ tests/test_camera_action_study.py | 324 +++++++ 22 files changed, 4299 insertions(+), 11 deletions(-) create mode 100644 benchmarks/camera-action-v01/independent_audit.py create mode 100644 docs/camera-action-v01.md create mode 100644 src/dvidia_training/camera_action_capture.py create mode 100644 src/dvidia_training/camera_action_export.py create mode 100644 src/dvidia_training/camera_action_model.py create mode 100644 src/dvidia_training/camera_action_skill.py create mode 100644 src/dvidia_training/camera_action_studio.html create mode 100644 src/dvidia_training/camera_action_studio.py create mode 100644 src/dvidia_training/camera_action_study.py create mode 100644 tests/test_camera_action_audit.py create mode 100644 tests/test_camera_action_capture.py create mode 100644 tests/test_camera_action_export.py create mode 100644 tests/test_camera_action_model.py create mode 100644 tests/test_camera_action_skill.py create mode 100644 tests/test_camera_action_studio.py create mode 100644 tests/test_camera_action_study.py diff --git a/benchmarks/camera-action-v01/independent_audit.py b/benchmarks/camera-action-v01/independent_audit.py new file mode 100644 index 0000000..b4cdc58 --- /dev/null +++ b/benchmarks/camera-action-v01/independent_audit.py @@ -0,0 +1,276 @@ +"""Read-only diagnostic audit; imports no controller, renderer or environment. + +Recomputes numeric evidence and identities, not the simulator itself. Passing +this audit means the reported failures or successes agree with saved evidence; +it does not imply that the diagnostic or learned-skill gates passed. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import math +from pathlib import Path + +import numpy as np + +from dvidia_training import piper_feedback_score as scoring + + +SCHEMA = 'dvidia.camera-action-diagnostic-audit/v1' + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128*1024**2: + raise ValueError('Expected a bounded ordinary JSON artifact: '+path.name) + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON field.') + result[key] = value + return result + def invalid(value): + raise ValueError('Nonfinite JSON value.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + + +def identity(path): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ValueError('Artifact is not an ordinary file.') + h = sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(1024*1024), b''): + h.update(block) + return dict(bytes=path.stat().st_size, sha256=h.hexdigest()) + + +def relative(root, name): + path = Path(name) + if path.is_absolute() or '..' in path.parts or not path.parts or str(path) != name: + raise ValueError('Artifact paths must be canonical relative paths.') + result = root/path + if any(p.is_symlink() for p in (result, *result.parents) if str(p) not in ('/tmp', '/var')): + raise ValueError('Artifact path traverses a symlink.') + return result + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def inventory(root): + manifest = read(root/'integrity.json') + require(manifest.get('schema') == 'dvidia.camera-action-rollout/v2/integrity', 'Unexpected episode integrity schema.') + files = manifest['files'] + actual = {str(p.relative_to(root)) for p in root.rglob('*') if p.is_file() and p.name != 'integrity.json'} + require(set(files) == actual, 'Episode inventory does not match its seal.') + for name, expected in files.items(): + require(identity(relative(root, name)) == expected, 'Artifact identity changed: '+name) + return files + + +def _reference(root, ref, files): + require(type(ref) is dict and set(ref) == {'path', 'bytes', 'sha256'}, 'Malformed artifact reference.') + require(files.get(ref['path']) == {key: ref[key] for key in ('bytes', 'sha256')}, 'Artifact reference does not match sealed bytes.') + return read(relative(root, ref['path'])) + + +def _close(a, b, name, tolerance=1e-8): + require(type(a) in (int, float) and type(b) in (int, float) + and math.isfinite(a) and math.isfinite(b) and math.isclose(a, b, abs_tol=tolerance, rel_tol=0.), name) + + +def _transform(metadata): + raw = metadata.get('world_from_robot_base') + require(metadata.get('world_from_robot_base_sha256') == digest(raw), 'Fixed-base transform hash differs.') + value = np.asarray(raw, dtype=float) + if value.shape == (16,): + value = value.reshape(4, 4) + require(value.shape == (4, 4) and np.isfinite(value).all(), 'Actual world-from-base transform is absent.') + require(np.allclose(value[3], [0., 0., 0., 1.], atol=1e-8, rtol=0) + and np.allclose(value[:3, :3].T@value[:3, :3], np.eye(3), atol=1e-8, rtol=0) + and math.isclose(float(np.linalg.det(value[:3, :3])), 1., abs_tol=1e-8), 'Invalid fixed-base transform.') + return value + + +def audit_episode(root, expected_case): + root = Path(root) + files = inventory(root) + summary = read(root/'rollout.json') + require(summary.get('case') == expected_case and summary.get('mode') == 'teacher' + and summary.get('negative_control') is None, 'Diagnostic identity differs from registration.') + result = dict(case_id=expected_case['id'], files_verified=len(files), status=summary['status'], + strict_success=False, complete_intervals=0, native_ticks=0, camera_pairs=0, + initial_anchor_error_m=None, available_position_error_p95_m=None, + maximum_native_object_displacement_m=None) + if 'recording.json' not in files: + require(summary['status'] == 'refused' and summary['recorded_complete_intervals'] == 0 + and summary['strict_success'] is False, 'Setup failure claimed completed data or success.') + return result + recording = read(root/'recording.json') + require(recording.get('schema') == 'dvidia.camera-action-rollout/v2' + and recording.get('control_mode') == 'teacher', 'Diagnostic recording schema/control changed.') + require(recording['provenance']['source_group'] == expected_case['group_id'] + and recording['provenance']['case_id'] == expected_case['id'] + and recording['goal_position_m'] == expected_case['goal_position_m'], 'Recording group/goal identity changed.') + steps = recording['steps'] + require(len(steps) == summary['recorded_complete_intervals'], 'Recorded interval count differs.') + context = read(root/'context.json') + transform = _transform(context['metadata']) + native_episode = read(root/'native_episode.json') + native_metadata = read(root/'native_metadata.json') + with np.load(root/'native.npz', allow_pickle=False) as handle: + arrays = {name: handle[name] for name in handle.files} + require(all(value.dtype.kind in 'iuf' for value in arrays.values()), 'Native arrays are not numeric-only.') + score = scoring.score_feedback_episode(native_episode, arrays, native_metadata, expected_case['goal_position_m'][:2]) + require(score == summary['score'], 'Saved independent score differs from recomputation.') + require(len(arrays['native_tick']) == native_metadata['rows'], 'Native row count differs.') + require(np.array_equal(arrays['native_tick'], np.arange(1, native_metadata['rows']+1)), 'Native tick ordering differs.') + require(np.allclose(arrays['t_seconds'], np.arange(1, native_metadata['rows']+1)*.001, atol=1e-8, rtol=0), 'Native 1 ms cadence differs.') + initial_world = np.asarray(native_episode['trajectory'][0]['object_position_m'], dtype=float) + initial_base = transform[:3, :3].T@(initial_world-transform[:3, 3]) + native_position = np.asarray(arrays['object_position_m']) + prior = recording['initial']; estimates = []; visible_errors = []; cameras = {} + for index, snapshot in enumerate([prior]+[s['after'] for s in steps]): + _close(snapshot['timestamp_seconds'], index*.02, 'Encoder 50 Hz cadence differs.') + camera = snapshot['camera']; frame = camera['value'] + require(frame['index'] == index//2 and snapshot['vision_frame_index'] == index//2, 'Camera source index differs from 25 Hz cadence.') + _close(camera['captured_at_seconds'], (index//2)*.04, 'Camera capture time differs.') + _close(camera['age_seconds'], index*.02-(index//2)*.04, 'Camera age was reset or changed.') + require(camera['captured_at_seconds'] <= camera['available_at_seconds'] <= snapshot['timestamp_seconds']+1e-8, + 'Camera timestamps are noncausal.') + if frame['index'] in cameras: + require(cameras[frame['index']] == frame, 'A reused camera frame changed its bytes.') + cameras[frame['index']] = frame + for channel in ('rgb', 'depth'): + reference = frame[channel] + require(files.get(reference['path']) == {key: reference[key] for key in ('bytes', 'sha256')}, 'Frame source identity changed.') + require(digest(recording['calibration']) == recording['binding']['calibration_sha256'], 'Camera calibration binding differs.') + for index, step in enumerate(steps): + require(step['index'] == index and step['before'] == prior, 'Transition prefix continuity differs.') + require(step['command']['authority'] == 'teacher' and step['command']['operator_correction'] is False, + 'Diagnostic issued authority is not its teacher.') + perception = _reference(root, step['perception'], files) + query = _reference(root, step['teacher_query'], files) + for sidecar in (perception, query): + require(sidecar['index'] == index and sidecar['snapshot_sha256'] == digest(step['before']), 'Sidecar is bound to a different decision snapshot.') + require(query['action'] == step['command']['issued'] == step['command']['requested'] + and query['origin'] == 'privileged_simulator_teacher', 'Teacher query/issued targets differ.') + require(perception['frame'] == step['before']['camera']['value'] + and perception['calibration_sha256'] == recording['binding']['calibration_sha256'] + and perception['tracker_sha256'] == recording['tracker_sha256'], 'Perception frame/calibration/tracker binding differs.') + estimate = perception['estimate']; estimates.append(estimate) + camera = step['before']['camera'] + require(estimate['frame_index'] == camera['value']['index'], 'Estimate frame differs from camera.') + for field in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(estimate[field], camera[field], 'Estimate and source camera timestamps differ.') + _close(estimate['decision_at_seconds'], index*.02, 'Estimate decision clock differs.') + tick = (index+1)*20-1 + require(tick < native_metadata['rows'], 'Complete motor interval has no native endpoint.') + np.testing.assert_allclose(step['applied']['ramped_targets']['joint_targets_rad'], + arrays['arm_actuator_controls_rad'][tick], atol=1e-9, rtol=0) + _close(step['applied']['ramped_targets']['gripper_position_m'], float(arrays['applied_jaw_target_m'][tick]), 'Applied gripper endpoint differs from native data.') + if estimate['status'] == 'available': + position = np.asarray(estimate['object_position_estimate_m']) + captured_tick = round(estimate['captured_at_seconds']/.001) + world = initial_world if captured_tick == 0 else native_position[captured_tick-1] + truth = transform[:3, :3].T@(world-transform[:3, 3]) + visible_errors.append(float(np.linalg.norm(position-truth))) + for field in ('object_position_estimate_m', 'object_position_uncertainty_m'): + signal = estimate['signals'][field] + require(signal['status'] == 'available', 'Available estimate has unavailable signals.') + for stamp in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(signal[stamp], estimate[stamp], 'Estimate signal age/source differs.') + else: + for field in ('object_position_estimate_m', 'object_position_uncertainty_m'): + require(estimate[field] is None and estimate['signals'][field]['value'] is None, + 'Unavailable estimate forwards a position or uncertainty.') + prior = step['after'] + query_count = sum(name.startswith('teacher/') and name.endswith('.json') for name in files) + require(summary['teacher_queries'] == query_count and len(steps) <= query_count <= len(steps)+1 and summary['student_calls'] == 0, + 'Diagnostic teacher query count or student intervention differs.') + require(summary['camera_pairs'] == len(cameras), 'Unique camera pair count differs.') + expected_success = summary['status'] == 'complete' and len(steps) == 1000 and score.get('success') is True + require(summary['strict_success'] is expected_success, 'Diagnostic strict success differs from complete native scoring.') + initial_error = None + if estimates and estimates[0]['status'] == 'available': + initial_error = float(np.linalg.norm(np.asarray(estimates[0]['object_position_estimate_m'])-initial_base)) + _close(summary['initial_anchor_error_m'], initial_error, 'Initial camera anchor error differs from native reconstruction.') + else: + require(summary['initial_anchor_error_m'] is None, 'Unavailable anchor has a fabricated error.') + missing = [e['status'] != 'available' for e in estimates] + longest = streak = 0 + for value in missing: + streak = streak+1 if value else 0; longest = max(longest, streak) + expected_perception = dict(decisions=len(estimates), available=sum(not m for m in missing), + missing=sum(missing), longest_missing_seconds=longest*.02) + require(summary['perception'] == expected_perception, 'Perception coverage/missing-duration summary differs.') + result.update(strict_success=expected_success, complete_intervals=len(steps), native_ticks=native_metadata['rows'], + camera_pairs=len(cameras), initial_anchor_error_m=initial_error, perception=expected_perception, + available_position_error_p95_m=float(np.quantile(visible_errors, .95)) if visible_errors else None, + maximum_native_object_displacement_m=float(np.max(np.linalg.norm(native_position-initial_world, axis=1))) if len(native_position) else 0., + score=score) + return result + + +def audit(directory): + root = Path(directory) + errors, results = [], [] + try: + protocol, registration = read(root/'protocol.json'), read(root/'registration.json') + require(identity(root/'protocol.json')['sha256'] == registration['protocol_sha256'], 'Study protocol changed after registration.') + require(protocol['source_sha256'] == registration['source_sha256'], 'Registration/runtime source lists disagree.') + for name, expected in protocol['source_sha256'].items(): + require(identity(Path(scoring.__file__).with_name(name))['sha256'] == expected, 'Registered source changed: '+name) + roster = protocol['rosters']['diagnostic'] + require(len(roster) == 4 and len({r['id'] for r in roster}) == 4, 'Expected exactly four diagnostic attempts.') + stage = read(root/'stages/diagnostic.json') + seal = read(root/'stages/diagnostic.integrity.json') + require(seal == dict(sha256=identity(root/'stages/diagnostic.json')['sha256'], + protocol_sha256=identity(root/'protocol.json')['sha256']), 'Diagnostic stage seal changed.') + require({row['id'] for row in stage['rows']} == {row['id'] for row in roster} + and len(stage['rows']) == 4, 'Diagnostic denominator differs from its roster.') + teacher_binding = digest(protocol['source_sha256']) + require(stage.get('model_bindings') == {'teacher': teacher_binding}, 'Diagnostic teacher runtime binding differs.') + for attempt in roster: + try: + row = next(r for r in stage['rows'] if r['id'] == attempt['id']) + require(row['case'] == attempt['case'], 'Stage row is associated with a different case.') + bound_case = {**attempt['case'], 'policy_artifact_sha256': teacher_binding} + result = audit_episode(root/'episodes/diagnostic'/attempt['id'], bound_case) + require(row['strict_success'] is result['strict_success'], 'Stage strict success differs from raw evidence.') + results.append(result) + except (ValueError, OSError, KeyError, TypeError, AssertionError) as exc: + errors.append(dict(attempt=attempt['id'], reason=str(exc))) + except (ValueError, OSError, KeyError, TypeError) as exc: + errors.append(dict(attempt=None, reason=str(exc))) + return dict(schema=SCHEMA, passed=not errors and len(results) == 4, errors=errors, + attempts_verified=len(results), strict_successes=sum(r['strict_success'] for r in results), results=results, + audit_source_sha256=identity(Path(__file__))['sha256'], scorer_source_sha256=identity(Path(scoring.__file__))['sha256'], + physics_executed=False, estimator_executed=False, source_authenticated=False, physical_robot_ready=False, + scope='Independent byte/clock/control and numeric task reconstruction; synthetic depth accuracy and hardware transfer remain unverified.') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study') + args = parser.parse_args(argv) + result = audit(args.study) + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 if result['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/docs/camera-action-v01.md b/docs/camera-action-v01.md new file mode 100644 index 0000000..ea93fd2 --- /dev/null +++ b/docs/camera-action-v01.md @@ -0,0 +1,102 @@ +# Camera-to-action skill v0.1 + +This local research workflow pairs rendered RGB-D with explicitly synthetic +PiPER teacher commands, fits a small CPU policy and compares complete tasks on +unseen starts. A model or successful installation does not establish qualification. +The four diagnostic trajectories and teacher-data gate must pass before fitting. +Failures are preserved and block dependent stages rather than triggering retries. + +## Local setup + +Python 3.12, the pinned NumPy and MuJoCo dependencies, existing verified PiPER +assets and a working OpenGL context are required. The tested renderer is macOS +CGL. No hosted inference or CUDA GPU is required; other operating systems and +hardware minimums remain unqualified. Native stepping is CPU work; rendering +uses the graphics driver. After dependencies/assets are provisioned, the workflow +is offline. The study reserves 30 GiB of free disk and caps its evidence at 120 GiB. + +```sh +python3.12 -m venv .venv +source .venv/bin/activate +python -m pip install '.[arm]' +dvidia-train robot prepare --study runs/camera-action --fixed-model /path/to/piper-base-model.json +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage diagnostic +dvidia-train robot inspect --study runs/camera-action +dvidia-training-studio --directory runs/studio --port 8363 +``` + +Open `http://127.0.0.1:8363/robot`, choose the prepared study and assets folders, +check the inputs, then run the remaining registered stages. The UI and Terminal +read the same immutable stage receipts. The UI stops when a gate blocks progress. +It installs a model only after a completed benchmark; installation remains a +simulation-candidate action. Ordinary video without paired actions is not an +accepted motor-learning input. + +For manual execution after a passing diagnostic: + +```sh +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage demonstrations +dvidia-train robot train --study runs/camera-action --stage initial +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage development-initial +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage corrections +dvidia-train robot train --study runs/camera-action --stage corrected +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage development-corrected +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage confirmation +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage negative +dvidia-train robot package --study runs/camera-action --output runs/camera-action-summary +``` + +The CLI package contains summary evidence and any fitted model JSON; it explicitly +does not include raw recordings or the executable runtime. Published research +archives additionally carry the independently audited evidence available for the +reported stage. Existing destination directories are never overwritten. + +## What the learner sees + +The frozen RGB-D tracker receives current pixels, depth and calibration only. +The policy receives a timestamped initial visual position, goal, elapsed time, +encoders, encoder-derived tool pose, previous applied servo references and 160 ms +of causal history. Missing live estimates retain availability masks and source +frame age; their numerical tensor fills are not measurements. Normalization uses +training data, including these explicit missing fills and masks. + +The policy never receives native object/contact truth, teacher phase, current +teacher target or future frames. Historical initial position is never copied into +the current-object channel. Native truth is used only by the teacher and offline +scorer. The first tracker only recognizes a known upright block: tilt and gripper +occlusion remain measured gaps, not invented observations. + +Teacher queries and issued commands are separate: one registered 50% mixed-control +round can request teacher labels on student-visited states. Those queries are +synthetic suggestions, not proven recoveries or operator corrections. Confirmation +has no teacher fallback. All candidates retain the native actuator and task bounds. + +## Registered comparison + +72 fresh initial-XY groups, split 32/8/32, share two known goals within each group. +Starts use a 3 mm inset from the original primitive's operating envelope to buffer +camera error, with no clipping or simulator-truth substitution. The fixed comparator +gets the same initial visual position. Comparisons also require at least 24 common +admitted groups so comparator refusals alone cannot establish an improvement. + +The acceptance gates are at least 58/64 strict placements, positive 97.5% grouped +paired intervals versus fixed and blind controls, no increase in critical-contact +frequency, all sixteen negative controls behaving correctly and no privileged +learner inputs or hidden intervention. All failures remain in scheduled denominators. +These are proposed acceptance thresholds; the study report records whether they +were actually met. Offline host timings are not physical sensor-to-actuator latency. + +## Candidate installation + +For a fitted model from a completed study: + +```sh +dvidia-arm check --model /path/to/model.json --robot piper +dvidia-arm install --model /path/to/model.json --robot piper --assets /path/to/robot-assets --output installed-camera-skill +dvidia-arm run installed-camera-skill --assets /path/to/robot-assets --output runs/installed-camera-attempt --start-x .30 --start-y -.03 --goal 0 +``` + +Only the exact local PiPER simulation interface is accepted. No physical driver, +arbitrary-arm transfer, shoe skill, human-video motor recovery or real-time guarantee +is introduced. Original code and generated evidence are MIT; upstream asset notices +remain in the repository and assets are provisioned separately. diff --git a/pyproject.toml b/pyproject.toml index 487c341..e981c8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "dvidia-training" -version = "0.1.3.dev3" +version = "0.1.3.dev4" description = "Local CPU training from Skillspace footage, with optional simulation movement candidates" readme = "README.md" license = "MIT AND Apache-2.0" diff --git a/src/dvidia_training/arm_skill_install.py b/src/dvidia_training/arm_skill_install.py index 2bbc3a3..6dfc730 100644 --- a/src/dvidia_training/arm_skill_install.py +++ b/src/dvidia_training/arm_skill_install.py @@ -54,6 +54,10 @@ def check_model(model_path, robot, *, interface=INTERFACE): raise ValueError('This learned policy requires PiPER. Other arms need their own trained ' 'policy and task benchmark; matching joint counts do not transfer it.') raw = _read(model_path, MAX_MODEL_BYTES) + decoded = _decode(raw) + if type(decoded) is dict and decoded.get('schema') == 'dvidia.camera-action-mlp/v1': + from .camera_action_skill import check + return check(model_path, robot, interface=interface) model = validate_model(raw) return { 'compatible': True, 'runner': RUNNER, 'robot': robot, 'interface_id': interface, @@ -83,6 +87,9 @@ def _record(check, binding): def install_model(model, robot, assets, output, *, interface=INTERFACE): """Verify model/assets, then reserve a fresh installation without stepping.""" check = check_model(model, robot, interface=interface) + if check['runner'] == 'piper-camera-action-v1': + from .camera_action_skill import install + return install(model, robot, assets, output, interface=interface) raw = _read(model, MAX_MODEL_BYTES) if sha256(raw).hexdigest() != check['model_sha256']: raise ValueError('Model changed during compatibility check.') @@ -105,6 +112,9 @@ def inspect_installation(directory, assets): if root.is_symlink() or not root.is_dir(): raise ValueError('Installation must be a regular directory.') stored = _decode(_read(root/'installation.json', 256*1024)) + if type(stored) is dict and stored.get('schema') == 'dvidia.camera-action-installation/v1': + from .camera_action_skill import inspect + return inspect(root, assets) if type(stored) is not dict or stored.get('schema') != SCHEMA: raise ValueError('Unknown arm skill installation.') compatibility = stored.get('compatibility') @@ -122,6 +132,9 @@ def inspect_installation(directory, assets): def run_installation(directory, assets, output, *, start_x=.30, start_y=-.03, goal=0): """Revalidate, then dispatch only to the built-in bounded simulation runner.""" record = inspect_installation(directory, assets) + if record['compatibility']['runner'] == 'piper-camera-action-v1': + from .camera_action_skill import run + return run(directory, assets, output, start_x=start_x, start_y=start_y, goal=goal) from .piper_learned_skill import run_learned_skill result = run_learned_skill(Path(directory)/'model.json', assets, output, start_x=start_x, start_y=start_y, goal=goal) diff --git a/src/dvidia_training/camera_action_capture.py b/src/dvidia_training/camera_action_capture.py new file mode 100644 index 0000000..da50936 --- /dev/null +++ b/src/dvidia_training/camera_action_capture.py @@ -0,0 +1,560 @@ +"""Causal camera/action rollout v2 for the bounded simulated PiPER task. + +Rendered observations, issued commands, privileged teacher suggestions, and +native scoring are distinct artifacts. This is an offline simulation recorder; +it makes no real-time delivery or physical-hardware qualification claim. +""" +from __future__ import annotations + +from copy import deepcopy +from dataclasses import asdict, dataclass +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import random +import time + +from . import action_demonstration as tape +from . import piper_action_capture as base +from . import piper_rgbd_tracker as tracker + +SCHEMA = 'dvidia.camera-action-rollout/v2' +PERCEPTION_SCHEMA = 'dvidia.camera-action-perception/v1' +TEACHER_SCHEMA = 'dvidia.camera-action-teacher-query/v1' +MAX_ROLLOUT_BYTES = 64 * 1024 * 1024 +MODES = ('teacher', 'student', 'mixed', 'fixed') +NEGATIVES = (None, 'initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper') +SCOPE = ('Offline native simulation with authored teacher demonstrations and ' + 'current-frame RGB-D estimates. No hardware commands, physical delivery ' + 'latency, human-video motor recovery or general object tracking.') + + +@dataclass(frozen=True) +class RolloutConfig: + duration_s: float = 20. + seed: int = 0 + mode: str = 'authored_task' + width: int = 320 + height: int = 240 + camera_every_controls: int = 2 + interrupt_after: int | None = None + + def __post_init__(self): + value = self.duration_s + if (type(value) not in (int, float) or not math.isfinite(value) or not .1 <= value <= 20. + or not math.isclose(value / tape.DT, round(value / tape.DT), abs_tol=1e-9, rel_tol=0)): + raise ValueError('Duration requires complete 20 ms intervals in [0.1,20] seconds.') + if type(self.seed) is not int or not 0 <= self.seed <= 999999: + raise ValueError('Seed must be an integer in [0,999999].') + if (self.mode != 'authored_task' or type(self.width) is not int or type(self.height) is not int + or self.width != 320 or self.height != 240 + or type(self.camera_every_controls) is not int or self.camera_every_controls != 2): + raise ValueError('The registered camera contract is 320x240 RGB-D at 25 Hz.') + count = round(value / tape.DT) + if self.interrupt_after is not None and (type(self.interrupt_after) is not int + or not 0 <= self.interrupt_after < count): + raise ValueError('Interruption must precede a scheduled interval.') + pairs = count // self.camera_every_controls + 1 + if (self.width * self.height * 7 + 4096) * pairs > tape.MAX_FRAME_TOTAL: + raise ValueError('Unique camera frames exceed the existing 512 MiB frame budget.') + + +def _case(case): + if type(case) is not dict: + raise ValueError('A registered case dictionary is required.') + result = deepcopy(case) + for name in ('id', 'group_id'): + if not isinstance(case.get(name), str) or not case[name] or len(case[name]) > 200: + raise ValueError('Case requires bounded nonempty '+name) + if type(case.get('seed')) is not int or not 0 <= case['seed'] <= 999999: + raise ValueError('Case requires a bounded seed.') + from .piper_task import PiperTaskConfig + config = PiperTaskConfig(object_position_m=case.get('object_position_m'), + target_position_m=case.get('goal_position_m')) + result['object_position_m'] = list(config.object_position_m) + result['goal_position_m'] = list(config.target_position_m) + return result + + +def _identity(path): + path = Path(path) + return dict(bytes=path.stat().st_size, sha256=sha256(path.read_bytes()).hexdigest()) + + +def _artifact(root, relative, document): + identity = base._json(root / relative, document) + identity['path'] = relative + return identity + + +def _sample_at(sample, seconds): + result = deepcopy(sample) + if result['status'] == 'available': + result['age_seconds'] = max(0., seconds - result['captured_at_seconds']) + return result + + +def _estimate_at(estimate, seconds): + """Reuse one captured frame, retaining its original acquisition timestamp.""" + result = deepcopy(estimate) + result['decision_at_seconds'] = seconds + if result.get('captured_at_seconds') is not None: + result['age_seconds'] = max(0., seconds - result['captured_at_seconds']) + for name, sample in result.get('signals', {}).items(): + result['signals'][name] = _sample_at(sample, seconds) + if result.get('status') == 'available' and result['age_seconds'] > .1 + 1e-12: + result['status'], result['reason'] = 'stale', 'source_frame_too_old' + for name in tracker.SIGNALS: + result[name] = None + result['signals'][name] = base._missing() + return result + + +class _Session(base._NativeSession): + def __init__(self, assets, config, case): + super().__init__(assets, config) + self.case = case + + def _task_configuration(self, task): + return task.PiperTaskConfig(timeout_s=self.config.duration_s, + object_position_m=self.case['object_position_m'], + target_position_m=self.case['goal_position_m']) + + def _environment(self, task): + from .piper_feedback_env import FeedbackPiperTaskEnv + return FeedbackPiperTaskEnv(self.assets, self.task_config) + + def open(self): + header, targets, metadata = super().open() + rotation = self.env.data.xmat[self.base].reshape(3, 3) + transform = self.np.eye(4) + transform[:3, :3] = rotation + transform[:3, 3] = self.env.data.xpos[self.base] + if (not self.np.isfinite(transform).all() + or not self.np.allclose(rotation.T @ rotation, self.np.eye(3), atol=1e-8, rtol=0) + or not math.isclose(float(self.np.linalg.det(rotation)), 1., abs_tol=1e-8, rel_tol=0)): + raise ValueError('Native robot base transform is not finite and rigid.') + metadata['world_from_robot_base'] = transform.reshape(16).tolist() + metadata['world_from_robot_base_sha256'] = tape.digest(metadata['world_from_robot_base']) + metadata['base_transform_scope'] = 'Fixed native world/base diagnostic transform; excluded from policy inputs.' + return header, targets, metadata + + def measured(self, previous_camera): + """50 Hz encoders/FK and the explicitly old 25 Hz camera frame.""" + seconds = float(self.env.elapsed) + if not math.isclose(float(self.env.data.time)-self.native_origin, seconds, + abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise RuntimeError('Native and declared clocks differ.') + rotation = self.env.data.xmat[self.base].reshape(3, 3) + tool_rotation = rotation.T @ self.env.data.xmat[self.tool].reshape(3, 3) + quaternion = self.np.empty(4) + self.mj.mju_mat2Quat(quaternion, tool_rotation.reshape(9)) + values = dict(joint_position_rad=self.env.data.qpos[:6].tolist(), + joint_velocity_rad_s=self.env.data.qvel[:6].tolist(), + gripper_joint_position_m=self.env.data.qpos[6:8].tolist(), + gripper_joint_velocity_m_s=self.env.data.qvel[6:8].tolist(), + tcp_position_estimate_m=(rotation.T @ (self.env.data.xpos[self.tool]-self.env.data.xpos[self.base])).tolist(), + tcp_quaternion_estimate_wxyz=quaternion.tolist()) + signals = {name: base._missing() for name in tape.SIGNALS} + signals.update({name: base._sample(value, seconds) for name, value in values.items()}) + snapshot = dict(timestamp_seconds=seconds, signals=signals, + camera=_sample_at(previous_camera, seconds), vision_frame_index=previous_camera['value']['index']) + diagnostic = dict(scope='Privileged native diagnostics; excluded from policy inputs.', + elapsed_seconds=seconds, native_data_time_seconds=float(self.env.data.time), + native_info=deepcopy(self.info), policy_observation=deepcopy(self.observation)) + return snapshot, {}, diagnostic, dict(render_encode_wall_seconds=0.) + + def native_evidence(self): + return self.env.native_arrays(), self.env.native_metadata() + + def anchor_error(self, estimate): + if estimate.get('status') != 'available': + return None + position = self.env.data.xmat[self.base].reshape(3, 3).T @ ( + self.env.data.xpos[self.env.block]-self.env.data.xpos[self.base]) + return float(self.np.linalg.norm(position-estimate['object_position_estimate_m'])) + + +def _initial_negative(snapshot, rgb, depth, calibration, negative): + """Faults affect only the perception input; no hidden object pose changes.""" + local = deepcopy(calibration) + captured = snapshot['camera']['captured_at_seconds'] + available = snapshot['camera']['available_at_seconds'] + if negative == 'initial_missing': + rgb = rgb.copy(); rgb[:] = 0 + elif negative == 'initial_stale': + captured = available = -.2 + elif negative == 'altered_calibration': + local['base_from_camera'][0] *= 1.1 + return rgb, depth, local, captured, available + + +def _fresh_estimate(session, snapshot, calibration, negative=None): + rgb, depth = session.latest_rgbd + rgb, depth, actual, captured, available = _initial_negative(snapshot, rgb, depth, calibration, negative) + began = time.perf_counter() + estimate = tracker.estimate_pose(rgb, depth, actual, + frame_index=snapshot['vision_frame_index'], captured_at_seconds=captured, + available_at_seconds=available, decision_at_seconds=snapshot['timestamp_seconds']) + return estimate, dict(estimator_wall_seconds=time.perf_counter()-began, + input_calibration_sha256=tape.digest(actual), negative_control=negative) + + +def _native_episode(session, states, actions): + info = deepcopy(session.info) + return dict(schema='dvidia.piper-task-episode/v1', config=asdict(session.task_config), + initial_object_position_m=deepcopy(states[0]['object_position_m']) if states else None, + trajectory=states, actions=actions, valid=bool(info.get('valid')), + success=bool(info.get('success')), reason=info.get('reason'), + metrics={**info, 'contact_pair_native_ticks': deepcopy(session.env.contact_pair_native_ticks)}, + physical_robot_ready=False, simulated_seconds=float(session.env.elapsed)) + + +def _score(root, goal): + import numpy as np + from .piper_feedback_score import score_feedback_episode + if not (root / 'native_episode.json').exists(): + return dict(valid=False, success=False, reason='native_not_initialized') + episode = json.loads((root / 'native_episode.json').read_text()) + metadata = json.loads((root / 'native_metadata.json').read_text()) + with np.load(root / 'native.npz', allow_pickle=False) as data: + arrays = {name: data[name] for name in data.files} + return score_feedback_episode(episode, arrays, metadata, goal[:2]) + + +def run_rollout(assets, output, *, case, mode, policy=None, negative_control=None, + duration_s=20., interrupt_after=None): + """Run a new attempt; failures seal the exact complete prefix, never retry. + + ``policy(context)`` receives only ``snapshot``, ``estimate``, + ``previous_applied`` and ``goal_position_m``. It returns the existing seven + target action dictionary. Teacher queries follow the actual trajectory and + are called once per boundary only in teacher/mixed modes. + """ + case = _case(case) + if mode not in MODES or negative_control not in NEGATIVES: + raise ValueError('Unknown controller mode or negative control.') + if mode != 'teacher' and not callable(policy): + raise ValueError('Student, mixed and fixed modes require a policy callback.') + config = RolloutConfig(duration_s=duration_s, seed=case['seed'], interrupt_after=interrupt_after) + root = Path(output).absolute() + if any(p.is_symlink() and str(p) not in ('/var', '/tmp') for p in (root, *root.parents)): + raise ValueError('Output cannot traverse symlinks.') + root.mkdir(parents=True, exist_ok=False) + tracker_hash = sha256(Path(tracker.__file__).read_bytes()).hexdigest() + base._json(root / 'started.json', dict(schema=SCHEMA, status='started', case=case, + mode=mode, config=asdict(config), negative_control=negative_control, scope=SCOPE)) + session = _Session(assets, config, case) + started = time.perf_counter() + steps, states, actions, timings, estimates = [], [], [], [], [] + header = initial = metadata = initial_targets = None + error = None; reason = 'capture_budget_complete'; started_intervals = 0 + frame_bytes = 0; camera_pairs = 0; anchor_available = False; anchor_error = None + native_status = {}; native_saved = False; rng = random.Random(case['seed']) + teacher_calls = student_calls = 0 + try: + header, initial_targets, metadata = session.open() + metadata = deepcopy(metadata) + metadata.update(tracker_implemented=True, + policy_observation_origin='teacher: privileged native state; student/fixed: whitelist of measured signals and RGB-D estimates', + rendered_camera_cadence_seconds=.04, measured_control_cadence_seconds=.02) + header = deepcopy(header); header['schema'] = SCHEMA + header['provenance']['kind'] = 'simulation_'+mode+'_controller' + header['provenance']['source_group'] = case['group_id'] + header['provenance']['case_id'] = case['id'] + header['goal_position_m'] = deepcopy(case['goal_position_m']) + header['tracker_sha256'] = tracker_hash + header['camera_cadence_seconds'] = .04 + header['control_mode'] = mode + snapshot, frames, diagnostic, timing = session.sample(0) + initial = deepcopy(snapshot) + current_estimate, estimate_timing = _fresh_estimate(session, snapshot, header['calibration'], negative_control) + timing.update(estimate_timing); timings.append(timing) + anchor_available = current_estimate['status'] == 'available' + anchor_error = session.anchor_error(current_estimate) + for path, body in frames.items(): + frame_bytes += len(body); base._record(root / path, body) + camera_pairs += 1 + if frame_bytes > tape.MAX_FRAME_TOTAL: + raise ValueError('Actual unique frame bytes exceed the 512 MiB limit.') + base._json(root / 'context.json', dict(header=header, initial=snapshot, + initial_applied_targets=initial_targets, metadata=metadata, case=case)) + base._json(root / 'native/0000.json', diagnostic) + states.append(deepcopy(diagnostic['policy_observation'])) + previous_applied = deepcopy(initial_targets) + for index in range(round(duration_s / tape.DT)): + if config.interrupt_after is not None and index >= config.interrupt_after: + raise KeyboardInterrupt('Declared engineering interruption.') + boundary = time.perf_counter() + seconds = snapshot['timestamp_seconds'] + estimate = _estimate_at(current_estimate, seconds) + perception = dict(schema=PERCEPTION_SCHEMA, index=index, + decision_at_seconds=seconds, snapshot_sha256=tape.digest(snapshot), + frame=deepcopy(snapshot['camera']['value']), + calibration_sha256=header['binding']['calibration_sha256'], + tracker_sha256=tracker_hash, estimate=estimate, + input_calibration_sha256=estimate_timing['input_calibration_sha256'], + negative_control=estimate_timing['negative_control']) + perception_ref = _artifact(root, f'perception/{index:04}.json', perception) + estimates.append(estimate) + if index == 0 and not anchor_available and mode != 'teacher': + reason = 'initial_visual_anchor_unavailable' + break + teacher_action = None; teacher_ref = None + if mode in ('teacher', 'mixed'): + teacher_calls += 1; teacher_action, teacher_decision = session.decide() + tape._action(teacher_action, header, 'queried teacher targets') + teacher_ref = _artifact(root, f'teacher/{index:04}.json', dict( + schema=TEACHER_SCHEMA, index=index, snapshot_sha256=tape.digest(snapshot), + computed_at_seconds=seconds, action=teacher_action, + origin='privileged_simulator_teacher', phase=teacher_decision.get('phase'))) + if mode == 'teacher': + requested, authority = deepcopy(teacher_action), 'teacher' + else: + context = dict(snapshot=deepcopy(snapshot), estimate=deepcopy(estimate), + previous_applied=deepcopy(previous_applied), goal_position_m=deepcopy(case['goal_position_m'])) + inference = time.perf_counter(); student_calls += 1; student_action = policy(context) + inference_wall = time.perf_counter()-inference + tape._action(student_action, header, 'policy targets') + if mode == 'mixed' and rng.random() < .5: + requested, authority = deepcopy(teacher_action), 'teacher' + else: + requested, authority = deepcopy(student_action), ('fixed' if mode == 'fixed' else 'student') + issued = deepcopy(requested) + if negative_control == 'open_gripper': + issued['gripper_position_m'] = .035 + tape._action(issued, header, 'issued targets') + command = dict(issued_at_seconds=seconds, requested=requested, issued=issued, + authority=authority, operator_correction=False, + issue_reason='open_gripper_negative_control' if issued != requested else 'unchanged') + base._json(root / f'decisions/{index:04}.json', dict(index=index, command=command, + perception=perception_ref, teacher_query=teacher_ref)) + started_intervals += 1 + applied = session.advance(issued) + next_index = index + 1 + if next_index % config.camera_every_controls == 0: + after, frames, diagnostic, timing = session.sample(next_index // config.camera_every_controls) + for path, body in frames.items(): + frame_bytes += len(body); base._record(root / path, body) + camera_pairs += 1 + if frame_bytes > tape.MAX_FRAME_TOTAL: + raise ValueError('Actual unique frame bytes exceed the 512 MiB limit.') + current_estimate, estimate_timing = _fresh_estimate(session, after, header['calibration']) + timing.update(estimate_timing) + else: + after, frames, diagnostic, timing = session.measured(snapshot['camera']) + step = dict(index=index, before=snapshot, command=command, applied=applied, + after=after, perception=perception_ref, teacher_query=teacher_ref) + base._json(root / f'transitions/{index:04}.json', step) + base._json(root / f'native/{next_index:04}.json', diagnostic) + steps.append(step) + action_row = dict(t_seconds=seconds, **issued, phase=authority) + states[-1].update(issued_joint_targets_rad=issued['joint_targets_rad'], + issued_gripper_position_m=issued['gripper_position_m'], command=action_row) + actions.append(action_row); states.append(deepcopy(diagnostic['policy_observation'])) + timing['full_boundary_wall_seconds'] = time.perf_counter()-boundary + timing['policy_inference_wall_seconds'] = 0. if mode == 'teacher' else inference_wall + timings.append(timing) + snapshot = after; previous_applied = deepcopy(applied['ramped_targets']) + except (Exception, KeyboardInterrupt) as exc: + reason = 'producer_interrupted' if isinstance(exc, KeyboardInterrupt) else 'producer_error' + error = dict(type=type(exc).__name__, message=str(exc)[:500]) + finally: + try: + native_status = session.status() + if header is not None: + import numpy as np + arrays, native_metadata = session.native_evidence() + buffer = io.BytesIO(); np.savez_compressed(buffer, **arrays) + base._record(root / 'native.npz', buffer.getvalue()) + base._json(root / 'native_metadata.json', native_metadata) + base._json(root / 'native_episode.json', _native_episode(session, states, actions)) + native_saved = True + except Exception as exc: + if error is None: + error = dict(type=type(exc).__name__, message=str(exc)[:500]); reason = 'native_evidence_error' + try: + session.close() + except Exception as exc: + if error is None: + error = dict(type=type(exc).__name__, message=str(exc)[:500]); reason = 'renderer_close_error' + complete = error is None and len(steps) == round(duration_s / tape.DT) + status = 'complete' if complete else 'interrupted' if steps else 'refused' + if header is not None: + recording = {**header, 'initial': initial, 'initial_applied_targets': initial_targets, + 'steps': steps, 'outcome': dict(status=status, reason=reason, + end_seconds=steps[-1]['after']['timestamp_seconds'] if steps else 0., + task_result='unverified' if complete else 'unknown')} + base._json(root / 'recording.json', recording) + score = _score(root, case['goal_position_m']) if native_saved else dict(valid=False, success=False, reason='native_evidence_unavailable') + missing = [estimate['status'] != 'available' for estimate in estimates] + longest = streak = 0 + for absent in missing: + streak = streak + 1 if absent else 0; longest = max(longest, streak) + critical_ticks = (native_status.get('native_info', {}).get('unexpected_contact_native_ticks') + if native_status.get('native_info') else None) + report = dict(schema=SCHEMA, status=status, reason=reason, error=error, case=case, mode=mode, + scope=SCOPE, physical_robot_ready=False, negative_control=negative_control, + requested_intervals=round(duration_s / tape.DT), started_intervals=started_intervals, + recorded_complete_intervals=len(steps), uncommitted_native_intervals=started_intervals-len(steps), + camera_pairs=camera_pairs, frame_bytes=frame_bytes, + initial_anchor_available=anchor_available, initial_anchor_error_m=anchor_error, + initial_anchor_binding_valid=bool(header is not None and anchor_available and negative_control not in ('initial_stale','altered_calibration')), + teacher_queries=teacher_calls, student_calls=student_calls, + no_privileged_features_or_teacher_intervention=bool(mode in ('student', 'fixed') and teacher_calls == 0), + policy_boundary_scope='Whitelisted deep-copied callback inputs; no process sandbox or external callback code attestation.', + score=score, strict_success=bool(complete and score.get('success')), + critical_contacts=critical_ticks, critical_contact_ticks=critical_ticks, + native_status=native_status, tracker_sha256=tracker_hash, + perception=dict(decisions=len(estimates), available=sum(not value for value in missing), + missing=sum(missing), longest_missing_seconds=longest*tape.DT), + timing=dict(wall_seconds=time.perf_counter()-started, boundaries=timings, + scope='Offline paused-simulator acquisition and computation. Synthetic capture/availability times; no physical transport or end-to-end real-time latency.')) + base._json(root / 'rollout.json', report) + files = {str(path.relative_to(root)): _identity(path) for path in sorted(root.rglob('*')) if path.is_file()} + base._json(root / 'integrity.json', dict(schema=SCHEMA+'/integrity', files=files)) + return report + + +def _read(root, relative, maximum=MAX_ROLLOUT_BYTES): + return json.loads(tape._ordinary_bytes(root / str(tape._relative(relative)), maximum)) + + +def inspect_rollout(directory): + """Read-only byte, source-frame, authority and independent-score checks.""" + root = Path(directory) + manifest = _read(root, 'integrity.json') + if manifest.get('schema') != SCHEMA+'/integrity' or type(manifest.get('files')) is not dict: + raise ValueError('Unsealed or unknown rollout manifest.') + observed = {str(p.relative_to(root)) for p in root.rglob('*') if p.is_file() and p.name != 'integrity.json'} + if observed != set(manifest['files']): + raise ValueError('Rollout inventory differs.') + for name, expected in manifest['files'].items(): + body = tape._ordinary_bytes(root / str(tape._relative(name)), MAX_ROLLOUT_BYTES) + if expected != dict(bytes=len(body), sha256=sha256(body).hexdigest()): + raise ValueError('Rollout byte identity differs: '+name) + report = _read(root, 'rollout.json') + if report.get('schema') != SCHEMA or report.get('status') not in ('complete', 'interrupted', 'refused'): + raise ValueError('Unknown rollout receipt.') + if 'recording.json' in manifest['files']: + recording = _read(root, 'recording.json') + if recording.get('schema') != SCHEMA or len(recording.get('steps', [])) != report['recorded_complete_intervals']: + raise ValueError('Recording schema or completeness differs.') + previous = recording['initial'] + native_episode = _read(root, 'native_episode.json') if 'native_episode.json' in manifest['files'] else None + native_arrays = None + if native_episode is not None and 'actions' in native_episode: + import numpy as np + with np.load(root / 'native.npz', allow_pickle=False) as archive: + native_arrays = {name: archive[name] for name in ('arm_actuator_controls_rad', + 'applied_jaw_target_m', 'normal_rate_limited_jaw_target_m')} + tape._snapshot(previous, recording['calibration']) + if tape.digest(recording['calibration']) != recording['binding']['calibration_sha256']: + raise ValueError('Bound calibration differs.') + for index, step in enumerate(recording['steps']): + if step['index'] != index or step['before'] != previous: + raise ValueError('Recording transition prefix differs.') + seconds = index*tape.DT + if not math.isclose(step['before']['timestamp_seconds'], seconds, abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise ValueError('Decision cadence differs.') + if not math.isclose(step['after']['timestamp_seconds'], seconds+tape.DT, abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise ValueError('Applied interval is incomplete.') + tape._snapshot(step['after'], recording['calibration'], previous) + if any(step['before']['signals'][name]['status'] != 'missing' for name in tracker.SIGNALS): + raise ValueError('Raw sensor snapshots cannot substitute tracker or native object truth.') + frame = step['before']['camera']['value'] + if frame['index'] != index // 2: + raise ValueError('Camera cadence differs from 25 Hz.') + for channel in ('rgb', 'depth'): + reference = frame[channel] + if manifest['files'].get(reference['path']) != {key: reference[key] for key in ('bytes', 'sha256')}: + raise ValueError('Camera source byte binding differs.') + for field in ('requested', 'issued'): + tape._action(step['command'][field], recording, field) + expected_issued = deepcopy(step['command']['requested']) + if report['negative_control'] == 'open_gripper': + expected_issued['gripper_position_m'] = .035 + if step['command']['issued'] != expected_issued: + raise ValueError('Actual issued command differs from declared intervention.') + if native_episode is not None and 'actions' in native_episode: + actual = native_episode['actions'][index] + if ({name: actual[name] for name in ('joint_targets_rad', 'gripper_position_m')} + != step['command']['issued']): + raise ValueError('Issued command differs from native action evidence.') + native_state = native_episode['trajectory'][index] + for signal in ('joint_position_rad', 'joint_velocity_rad_s', 'gripper_joint_position_m'): + if step['before']['signals'][signal]['value'] != native_state[signal]: + raise ValueError('Measured inputs differ from native encoder evidence.') + boundary = (index+1)*20-1 + controls = [*native_arrays['arm_actuator_controls_rad'][boundary].tolist(), + float(native_arrays['applied_jaw_target_m'][boundary])] + ramped = dict(joint_targets_rad=controls[:6], + gripper_position_m=float(native_arrays['normal_rate_limited_jaw_target_m'][boundary])) + if step['applied']['native_position_controls'] != controls or step['applied']['ramped_targets'] != ramped: + raise ValueError('Previous-applied learning inputs differ from native servo references.') + for field in ('perception', 'teacher_query'): + reference = step[field] + if reference is None: + if field == 'perception' or report['mode'] in ('teacher', 'mixed'): + raise ValueError('Required sidecar missing.') + continue + identity = manifest['files'].get(reference['path']) + if identity != {key: reference[key] for key in ('bytes', 'sha256')}: + raise ValueError('Sidecar binding differs.') + sidecar = _read(root, reference['path']) + if sidecar['index'] != index or sidecar['snapshot_sha256'] != tape.digest(step['before']): + raise ValueError('Sidecar source snapshot differs.') + if field == 'perception': + if (sidecar['tracker_sha256'] != recording['tracker_sha256'] + or sidecar['calibration_sha256'] != recording['binding']['calibration_sha256'] + or sidecar['frame'] != step['before']['camera']['value']): + raise ValueError('Perception frame/tracker/calibration binding differs.') + if sidecar['estimate']['decision_at_seconds'] != step['before']['timestamp_seconds']: + raise ValueError('Perception decision time differs.') + estimate = sidecar['estimate'] + if estimate['frame_index'] != frame['index']: + raise ValueError('Estimated source frame index differs.') + if estimate['status'] == 'available': + camera = step['before']['camera'] + if (estimate['captured_at_seconds'] != camera['captured_at_seconds'] + or estimate['available_at_seconds'] != camera['available_at_seconds'] + or estimate['age_seconds'] != camera['age_seconds']): + raise ValueError('Estimated source frame acquisition timing differs.') + for name in tracker.SIGNALS: + signal = estimate['signals'][name] + if (signal['status'] != 'available' or any(signal[key] != estimate[key] for key in + ('captured_at_seconds', 'available_at_seconds', 'age_seconds'))): + raise ValueError('Estimated source signal timing differs.') + expected = estimate[name] + if name.endswith('_uncertainty_m') or name.endswith('_uncertainty_rad'): + expected = [expected] + if signal['value'] != expected: + raise ValueError('Estimated source signal value differs.') + elif any(estimate[name] is not None or estimate['signals'][name]['value'] is not None + for name in tracker.SIGNALS): + raise ValueError('Missing estimates must not carry substituted values.') + elif step['command']['authority'] == 'teacher' and sidecar['action'] != step['command']['requested']: + raise ValueError('Teacher authority differs from its query.') + authority = step['command']['authority'] + allowed = {'teacher': {'teacher'}, 'student': {'student'}, 'mixed': {'teacher', 'student'}, 'fixed': {'fixed'}}[report['mode']] + if authority not in allowed or step['command']['operator_correction'] is not False: + raise ValueError('Invalid control authority.') + previous = step['after'] + if report['status'] == 'complete' and len(recording['steps']) != report['requested_intervals']: + raise ValueError('Complete receipt omits scheduled intervals.') + recomputed = _score(root, report['case']['goal_position_m']) if 'native_episode.json' in manifest['files'] else dict(valid=False, success=False, reason='native_evidence_unavailable') + if recomputed != report['score']: + raise ValueError('Independent task score differs.') + if report['strict_success'] != bool(report['status'] == 'complete' and recomputed.get('success')): + raise ValueError('Reported strict success differs from complete native scoring.') + return dict(schema=SCHEMA+'/inspection', status=report['status'], files_verified=len(manifest['files']), + source_authenticated=False, physics_executed=False, physical_robot_ready=False, + score=recomputed, strict_success=report['strict_success'], + recorded_complete_intervals=report['recorded_complete_intervals']) + + +def load_rollout(directory): + inspect_rollout(directory) + return _read(Path(directory), 'recording.json') diff --git a/src/dvidia_training/camera_action_export.py b/src/dvidia_training/camera_action_export.py new file mode 100644 index 0000000..2ce7208 --- /dev/null +++ b/src/dvidia_training/camera_action_export.py @@ -0,0 +1,128 @@ +"""Mask-aware v2 exports, separate from the intentionally strict v1 exporter. + +Only pre-action measurements and separately sealed perception enter inputs. +Teacher queries are explicit supervision on mixed trajectories; they are never +relabeled as commands that the producer actually executed. +""" +from __future__ import annotations + +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import numpy as np + +from . import camera_action_model as model + + +def _document(root, reference): + from . import action_demonstration as tape + body = tape._ordinary_bytes(root / str(tape._relative(reference['path'])), 2*1024*1024) + if reference['bytes'] != len(body) or reference['sha256'] != sha256(body).hexdigest(): + raise ValueError('Learning sidecar bytes no longer match their sealed reference.') + return model.core._decode(body) + + +def export_episode(directory, *, blind=False, label_source='issued'): + """Read and verify one sealed episode; no rendering, physics or relabeling. + + ``issued`` uses teacher-only producer commands. ``teacher_query`` requires + mixed control and labels each pre-action snapshot using its one separate + teacher query, whether the teacher or student actually controlled that step. + Interrupted complete prefixes remain exportable; a fresh t=0 anchor is + always required. Source selection is the experiment orchestrator's job. + """ + from . import camera_action_capture as capture + root = Path(directory) + recording = capture.load_rollout(root) + recording_body = (root/'recording.json').read_bytes() + if recording_body.strip() != model.canonical(recording): + raise ValueError('Recording changed after rollout verification.') + if label_source not in ('issued', 'teacher_query'): + raise ValueError('Choose issued or teacher_query labels explicitly.') + expected = 'teacher' if label_source == 'issued' else 'mixed' + if recording.get('control_mode') != expected: + raise ValueError('Issued supervision requires teacher control; queried correction supervision requires mixed control.') + if not recording['steps']: + raise ValueError('No complete pre-action intervals are available.') + encoder = model.CameraActionFeatures(blind=blind) + previous_applied = recording['initial_applied_targets'] + features, targets, perception_hashes, query_hashes, authority = [], [], [], [], [] + for step in recording['steps']: + perception = _document(root, step['perception']) + context = dict(snapshot=step['before'], estimate=perception['estimate'], + previous_applied=previous_applied, goal_position_m=recording['goal_position_m']) + features.append(encoder(context)) + perception_hashes.append(step['perception']['sha256']) + if label_source == 'issued': + if step['command']['authority'] != 'teacher': + raise ValueError('Issued imitation labels must actually come from the teacher.') + target = step['command']['issued'] + else: + reference = step.get('teacher_query') + if reference is None: + raise ValueError('A separately sealed teacher query is required for every correction label.') + query = _document(root, reference) + if query.get('origin') != 'privileged_simulator_teacher': + raise ValueError('Unknown teacher-query supervision origin.') + target = query['action'] + query_hashes.append(reference['sha256']) + targets.append(model.action_vector(target)) + authority.append(step['command']['authority']) + previous_applied = step['applied']['ramped_targets'] + source_group = recording['provenance']['source_group'] + source = dict(source_group=source_group, case_id=recording['provenance']['case_id'], + recording_sha256=sha256(recording_body).hexdigest(), + integrity_sha256=sha256((root/'integrity.json').read_bytes()).hexdigest(), + calibration_sha256=recording['binding']['calibration_sha256'], + tracker_sha256=recording['tracker_sha256'], label_source=label_source, + sample_count=len(features), perception_references_sha256=model.digest(perception_hashes), + teacher_query_references_sha256=model.digest(query_hashes), + actual_authority_sha256=model.digest(authority)) + sources = [source] + provenance = dict(feature_contract_sha256=model.FEATURE_CONTRACT_SHA256, blind=blind, + episodes=sources, source_identity_sha256=model.digest(sources), + export_source_sha256=sha256(Path(__file__).read_bytes()).hexdigest(), + label_definition='absolute issued teacher targets' if label_source == 'issued' + else 'separate teacher query suggestions; actual control authority retained independently') + return dict(schema=model.DATASET_SCHEMA, features=np.asarray(features, dtype=np.float32), + targets=np.asarray(targets, dtype=np.float64), groups=[source_group]*len(features), provenance=provenance) + + +def combine_datasets(datasets): + """Concatenate already exported episodes once; retain whole-start group IDs.""" + if type(datasets) not in (list, tuple) or not datasets: + raise ValueError('At least one exported dataset is required.') + blind = datasets[0]['provenance']['blind'] + features, targets, groups, sources, identities = [], [], [], [], set() + for dataset in datasets: + x, y, g, p, _ = model._dataset(dataset, blind=blind) + if (p.get('blind') is not blind or p.get('feature_contract_sha256') != model.FEATURE_CONTRACT_SHA256 + or p.get('source_identity_sha256') != model.digest(p.get('episodes'))): + raise ValueError('Combined exports must have the same exact feature contract and feedback masking.') + for episode in p['episodes']: + identity = episode['recording_sha256'] + if identity in identities: + raise ValueError('An episode cannot be duplicated or relabeled within one training dataset.') + identities.add(identity) + sources.append(episode) + features.append(x); targets.append(y); groups.extend(g) + if len(groups) > 1_000_000: + raise ValueError('Combined training data exceeds its bounded sample budget.') + provenance = dict(feature_contract_sha256=model.FEATURE_CONTRACT_SHA256, blind=blind, + episodes=sources, source_identity_sha256=model.digest(sources), + export_source_sha256=sha256(Path(__file__).read_bytes()).hexdigest(), + label_definition='per-episode explicit issued teacher targets or separately sealed teacher query suggestions') + return dict(schema=model.DATASET_SCHEMA, features=np.concatenate(features), targets=np.concatenate(targets), + groups=groups, provenance=provenance) + + +def blind_dataset(dataset): + """Mask a verified full export without rereading its potentially large frames.""" + x, y, groups, provenance, _ = model._dataset(dataset, blind=False) + if provenance.get('blind') is not False: + raise ValueError('Blind masking expects a full-feedback exported dataset.') + x = x.copy(); x[:, model.LIVE_FEATURE_START:] = 0. + provenance = deepcopy(provenance); provenance['blind'] = True + return dict(schema=model.DATASET_SCHEMA, features=x, targets=y.copy(), + groups=groups, provenance=provenance) diff --git a/src/dvidia_training/camera_action_model.py b/src/dvidia_training/camera_action_model.py new file mode 100644 index 0000000..d7b96eb --- /dev/null +++ b/src/dvidia_training/camera_action_model.py @@ -0,0 +1,463 @@ +"""Small causal RGB-D/encoder policy, trained on explicit native action labels. + +The network never receives a simulator, teacher, task phase, current target, +contact truth or a future snapshot. Its visual anchor is historical context, +not a held-forward current measurement. Missing live estimates stay missing. +""" +from __future__ import annotations + +from collections import deque +from copy import deepcopy +from hashlib import sha256 +import math +from pathlib import Path +import time + +import numpy as np + +from . import piper_imitation as core +from . import piper_rgbd_tracker as tracker + + +SCHEMA = 'dvidia.camera-action-mlp/v1' +DATASET_SCHEMA = 'dvidia.camera-action-learning/v1' +FEATURE_CONTRACT = 'camera-encoder-absolute-causal160/v1' +SCOPE = ('Simulation-only camera/encoder behavioral cloning of a privileged synthetic ' + 'teacher; known upright block, no human-video motor recovery or physical qualification.') +LAGS = (0., .04, .08, .12, .16) +SENSOR_SPECS = ( + ('joint_position_rad', 6, 3.), ('joint_velocity_rad_s', 6, 3.), + ('gripper_joint_position_m', 2, .035), ('gripper_joint_velocity_m_s', 2, .035), + ('tcp_position_estimate_m', 3, .5), ('tcp_quaternion_estimate_wxyz', 4, 1.), +) +CONTEXT_NAMES = tuple(f'initial_visual_{a}_m/.5' for a in 'xyz') + ( + 'initial_visual_captured_at_s/20', 'initial_visual_uncertainty_m/.02', +) + tuple(f'goal_{a}_m/.5' for a in 'xyz') + ('elapsed_s/20',) +COMMAND_NAMES = tuple(f'previous_applied_joint_{i}_rad/3' for i in range(1, 7)) + ( + 'previous_applied_jaw_m/.035', 'history_present', 'history_age_s/.16',) +SENSOR_NAMES = tuple(name for field, size, scale in SENSOR_SPECS + for name in tuple(f'{field}_{i}/{scale:g}' for i in range(size)) + + (field+'_available', field+'_age_s/.1')) + ( + 'visual_x_m/.5', 'visual_y_m/.5', 'visual_z_m/.5', + 'visual_uncertainty_m/.02', 'visual_available', 'visual_age_s/.1', + 'history_present', 'history_age_s/.16',) +FEATURE_NAMES = CONTEXT_NAMES + tuple(f'lag{round(lag*1000)}:{name}' + for lag in LAGS for name in COMMAND_NAMES) + tuple(f'lag{round(lag*1000)}:{name}' + for lag in LAGS for name in SENSOR_NAMES) +FEATURE_COUNT = len(FEATURE_NAMES) +LIVE_FEATURE_START = len(CONTEXT_NAMES) + len(LAGS)*len(COMMAND_NAMES) +ACTION_SCALE_NATIVE = np.asarray([3.]*6 + [core.JAW_MAX_M], dtype=np.float64) +MODEL_FIELDS = {'schema', 'scope', 'physical_robot_ready', 'observation_origin', + 'supervision_origin', 'feature_contract', 'feature_names', 'feature_contract_sha256', + 'blind', 'hidden_units', 'joint_limits_rad', 'jaw_limit_m', 'action_representation', + 'feature_mean', 'feature_scale', 'action_mean', 'action_scale', 'weights', 'biases', 'training'} +TRAINING_FIELDS = {'algorithm', 'stage', 'seed', 'epochs', 'batch_size', 'learning_rate', + 'sample_count', 'train_groups', 'dataset_sha256', 'provenance', 'provenance_sha256', + 'model_source_sha256', 'initial_loss', 'final_loss', 'validation'} + + +def canonical(document): + return core.canonical(document) + + +def digest(document): + return sha256(canonical(document)).hexdigest() + + +FEATURE_CONTRACT_SHA256 = digest({'contract': FEATURE_CONTRACT, 'names': FEATURE_NAMES, + 'lags': LAGS, 'history_seconds': .16, 'camera_maximum_age_seconds': .1, + 'blind_live_start': LIVE_FEATURE_START, + 'previous_applied': 'preceding interval final native servo references; never measured q', + 'initial_anchor': 'fresh t=0 visual xyz and heuristic uncertainty; historical thereafter'}) + + +def _hash(value, label): + if type(value) is not str or len(value) != 64 or any(c not in '0123456789abcdef' for c in value): + raise ValueError(label+' must be a lowercase SHA-256 digest.') + return value + + +def action_vector(action): + """Validate seven *absolute* issued/query targets, without interpreting authority.""" + if type(action) is not dict or set(action) != {'joint_targets_rad', 'gripper_position_m'}: + raise ValueError('An action must contain only six joint targets and one jaw target.') + q = core._array(action['joint_targets_rad'], (6,), 'joint targets', 10.) + limits = np.asarray(core.JOINT_LIMITS) + if np.any(q < limits[:, 0]) or np.any(q > limits[:, 1]): + raise ValueError('Action exceeds the original native joint bounds.') + jaw = core._number(action['gripper_position_m'], 'jaw target', 0., core.JAW_MAX_M) + return np.concatenate((q, [jaw])) + + +def _time(value, name): + return core._number(value, name, 0., 20.) + + +def _sample(sample, size, now, name): + if type(sample) is not dict or sample.get('status') not in ('available', 'missing', 'stale', 'refused'): + raise ValueError(name+' has an invalid availability state.') + if sample['status'] != 'available': + if sample.get('value') is not None: + raise ValueError(name+' must not forward-fill unavailable values.') + return np.zeros(size), 0., 0. + captured = _time(sample.get('captured_at_seconds'), name+' capture time') + available = _time(sample.get('available_at_seconds'), name+' availability time') + age = _time(sample.get('age_seconds'), name+' age') + if (captured > available+1e-10 or available > now+1e-10 + or abs(age-(now-captured)) > 1e-8 or age > .1+1e-10): + raise ValueError(name+' is future, stale or incorrectly age-stamped.') + return core._array(sample.get('value'), (size,), name, 1e4), 1., age + + +def _visual(estimate, now, snapshot): + if type(estimate) is not dict or estimate.get('schema') != tracker.SCHEMA: + raise ValueError('A frozen tracker result is required.') + if estimate.get('status') not in ('available', 'missing', 'stale', 'refused'): + raise ValueError('Invalid tracker status.') + decision = _time(estimate.get('decision_at_seconds'), 'tracker decision time') + if abs(decision-now) > 1e-8: + raise ValueError('Tracker decision time must match this motor boundary.') + camera = snapshot.get('camera') + if type(camera) is not dict or camera.get('status') != 'available' or type(camera.get('value')) is not dict: + raise ValueError('Visual estimates require their available source camera frame.') + index = camera['value'].get('index') + if (type(index) is not int or index < 0 or estimate.get('frame_index') != index + or snapshot.get('vision_frame_index') != index): + raise ValueError('Tracker frame index differs from the source camera snapshot.') + captured = _time(camera.get('captured_at_seconds'), 'camera capture time') + available = _time(camera.get('available_at_seconds'), 'camera availability time') + camera_age = _time(camera.get('age_seconds'), 'camera age') + if (captured > available+1e-10 or available > now+1e-10 + or abs(camera_age-(now-captured)) > 1e-8): + raise ValueError('Source camera timestamps are noncausal or incorrectly aged.') + if any(estimate.get(field) != camera.get(field) + for field in ('captured_at_seconds', 'available_at_seconds', 'age_seconds')): + raise ValueError('Tracker source timestamps differ from the camera snapshot.') + signals = estimate.get('signals') + if type(signals) is not dict: + raise ValueError('Tracker signals are required.') + p, present, age = _sample(signals.get('object_position_estimate_m'), 3, now, 'visual position') + u, uncertainty_present, uncertainty_age = _sample(signals.get('object_position_uncertainty_m'), 1, now, 'visual uncertainty') + if present != uncertainty_present or abs(age-uncertainty_age) > 1e-8: + raise ValueError('Visual position and uncertainty must share availability and age.') + if bool(present) != (estimate['status'] == 'available'): + raise ValueError('Tracker status and visual signal availability disagree.') + if present: + if u[0] < 0. or u[0] > 1.: + raise ValueError('Visual uncertainty must be a nonnegative bounded scale.') + top = core._array(estimate.get('object_position_estimate_m'), (3,), 'tracker position', 1e4) + if not np.array_equal(p, top) or estimate.get('object_position_uncertainty_m') != float(u[0]): + raise ValueError('Tracker position/uncertainty receipt disagrees with signals.') + for name in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + if (estimate.get(name) != signals['object_position_estimate_m'].get(name) + or estimate.get(name) != signals['object_position_uncertainty_m'].get(name)): + raise ValueError('Tracker top-level and signal times disagree.') + else: + if estimate.get('object_position_estimate_m') is not None or estimate.get('object_position_uncertainty_m') is not None: + raise ValueError('Unavailable visual estimates must be null.') + # An abstention has no pose age, but its source image still has a known + # acquisition age. Preserve that age next to the zero position and mask. + return p, float(u[0]), present, camera_age + + +class CameraActionFeatures: + """Fresh per episode. Five causal slots span 160 ms; warmup slots are absent. + + The blind comparator retains the same initial camera anchor and previous + applied command history. It receives no live sensor values, masks or ages. + Its command history is the preceding native reference, not measured joints. + """ + def __init__(self, *, blind=False): + if type(blind) is not bool: + raise ValueError('blind must be boolean.') + self.blind = blind + self.history = deque() + self.initial = self.goal = None + self.last_time = None + + def features(self, context): + if type(context) is not dict or type(context.get('snapshot')) is not dict: + raise ValueError('Camera-action context and pre-action snapshot are required.') + snapshot = context['snapshot'] + now = _time(snapshot.get('timestamp_seconds'), 'snapshot time') + if self.last_time is None and now != 0.: + raise ValueError('Camera-action episodes must start at t=0.') + if self.last_time is not None and not math.isclose(now-self.last_time, .02, abs_tol=1e-8, rel_tol=0.): + raise ValueError('Camera-action observations require increasing 20 ms boundaries.') + goal = core._array(context.get('goal_position_m'), (3,), 'goal position', 1.) + if self.goal is not None and not np.array_equal(goal, self.goal): + raise ValueError('The destination cannot change during an episode.') + command = action_vector(context.get('previous_applied'))/ACTION_SCALE_NATIVE + visual = None + if self.initial is None or not self.blind: + visual = _visual(context.get('estimate'), now, snapshot) + initial = self.initial + if initial is None: + p, uncertainty, present, age = visual + if not present or age > 1e-10: + raise ValueError('A fresh initial visual anchor is required before the first student action.') + initial = np.concatenate((p/.5, [0., uncertainty/.02])) + if self.blind: + sensed = np.zeros(len(SENSOR_NAMES)-2) + else: + signals = snapshot.get('signals') + if type(signals) is not dict: + raise ValueError('Measured encoder/FK signals are required.') + values = [] + for name, size, scale in SENSOR_SPECS: + value, present, age = _sample(signals.get(name), size, now, name) + if present and name == 'tcp_quaternion_estimate_wxyz': + value = core._quaternion(value, name) + values.extend(np.clip(value/scale, -10., 10.)) + values.extend((present, age/.1)) + p, uncertainty, present, age = visual + values.extend(np.clip(p/.5, -10., 10.)) + values.extend((uncertainty/.02, present, age/.1)) + sensed = np.asarray(values) + pending = [row for row in self.history if row[0] >= now-.16-1e-8] + pending.append((now, command.copy(), sensed.copy())) + selected = [] + for lag in LAGS: + eligible = [row for row in pending if row[0] <= now-lag+1e-8] + selected.append(eligible[-1] if eligible else None) + command_slots, sensed_slots = [], [] + for row in selected: + if row is None: + command_slots.extend(np.zeros(len(COMMAND_NAMES))) + sensed_slots.extend(np.zeros(len(SENSOR_NAMES))) + else: + age = max(0., now-row[0])/.16 + command_slots.extend([*row[1], 1., age]) + sensed_slots.extend(np.zeros(len(SENSOR_NAMES)) if self.blind else [*row[2], 1., age]) + result = np.concatenate((initial, goal/.5, [now/20.], command_slots, sensed_slots)) + if result.shape != (FEATURE_COUNT,) or not np.isfinite(result).all() or np.any(np.abs(result) > 10.+1e-8): + raise ValueError('Camera-action features exceed the bounded finite contract.') + self.history = deque(pending) + self.initial, self.goal, self.last_time = initial.copy(), goal.copy(), now + return result.astype(np.float32) + + __call__ = features + + +def _dataset(dataset, *, blind): + if type(dataset) is not dict or set(dataset) != {'schema', 'features', 'targets', 'groups', 'provenance'} or dataset['schema'] != DATASET_SCHEMA: + raise ValueError('Use a camera-action mask-aware dataset with provenance.') + groups = dataset['groups'] + if type(groups) not in (list, tuple) or not 1 <= len(groups) <= 1_000_000: + raise ValueError('Dataset requires 1 to 1000000 grouped samples.') + groups = list(groups) + if any(type(group) is not str for group in groups): + raise ValueError('Every training sample requires a string source-group ID.') + core._case_ids(list(dict.fromkeys(groups))) + x = core._array(dataset['features'], (len(groups), FEATURE_COUNT), 'features', 10.+1e-7) + y = core._array(dataset['targets'], (len(groups), 7), 'absolute action targets', 10.) + limits = np.asarray(core.JOINT_LIMITS) + if (np.any(y[:, :6] < limits[:, 0]) or np.any(y[:, :6] > limits[:, 1]) + or np.any(y[:, 6] < 0.) or np.any(y[:, 6] > core.JAW_MAX_M)): + raise ValueError('Training targets exceed original native bounds.') + if blind and np.any(x[:, LIVE_FEATURE_START:] != 0.): + raise ValueError('Blind datasets must zero all live/history feedback features.') + provenance = dataset['provenance'] + if type(provenance) is not dict or not provenance: + raise ValueError('Dataset provenance is required.') + provenance = core._decode(canonical(provenance)) + h = sha256(canonical({'schema': DATASET_SCHEMA, 'features': list(x.shape), + 'targets': list(y.shape), 'groups': groups, 'provenance': provenance})) + for value in (x, y): + h.update(np.asarray(value, dtype=' + +Simulation skill training · DVIDIA + + + + +
Camera → action

Teach an arm, then test it.

Train a compact block-placement policy from paired camera recordings and robot actions. Compare it on unseen starts before installing a simulation candidate.

Local files · Offline computation · One job at a time

+

01Choose paired demonstrations

Open an existing prepared camera-to-action study. It keeps the demonstration groups, fixed experiment budget and benchmark results together.

+ + +

Paths refer to this computer. Plain videos alone do not contain the demonstrated motor targets required by this pilot. The study collects paired recordings from its explicitly labeled simulator teacher.

+
Choose both folders to begin.
+
First time?

Prepare the registered study in Terminal with the existing fixed comparator model. Then enter the study and verified PiPER asset folders above.

dvidia-train robot prepare --study /path/to/new-study --fixed-model /path/to/fixed-model.json
+
+

02Train and benchmark

The same workflow as Terminal: verify demonstrations, train the model, collect one bounded correction round, and compare it with fixed and feedback-blind controls.

+
Check the study before starting.
+
Connecting to the local studio…
+

Completed stages are preserved. A failed prerequisite stops the workflow; it does not retry or replace difficult scenes. Graphics support is required for native RGB-D rendering.

+
+

03Review the evidence

The checked study and its benchmark results will appear here.

+ +
DVIDIA · Measured simulation evidence, with failures retained.
hello@dvidia.org
+ diff --git a/src/dvidia_training/camera_action_studio.py b/src/dvidia_training/camera_action_studio.py new file mode 100644 index 0000000..6accad7 --- /dev/null +++ b/src/dvidia_training/camera_action_studio.py @@ -0,0 +1,230 @@ +"""Local Training Studio bridge for registered camera/action studies. + +It shares the footage server's one worker and lock. A browser never supplies +Python code, arbitrary commands or hardware endpoints. Stage order comes from +the same verified study inspection used by the Terminal workflow. +""" +from __future__ import annotations + +from hashlib import sha256 +import json +from pathlib import Path +import time +import uuid + +SCHEMA = 'dvidia.camera-action-studio/v1' + + +def _local_path(value, label): + if not isinstance(value, str) or not value.strip() or len(value) > 8192 or any(ord(c) < 32 for c in value): + raise ValueError('Supply a local '+label+' folder.') + if '://' in value: + raise ValueError('Use local folders; the robot study does not download inputs.') + path = Path(value).expanduser().absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (path, *path.parents)): + raise ValueError('The '+label+' folder cannot traverse symlinks.') + if not path.is_dir(): + raise ValueError('The '+label+' folder does not exist on this server.') + return path.resolve() + + +def check_study(study, assets): + from . import camera_action_study, robot_profiles + root = _local_path(study, 'prepared study') + asset_root = _local_path(assets, 'verified robot assets') + state = camera_action_study.inspect(root) + camera_action_study._protocol(root, runtime=True) + metadata = robot_profiles.verify_assets('piper', asset_root) + binding = dict(study=str(root), assets=str(asset_root), protocol_sha256=state['protocol_sha256'], + assets_sha256=sha256(json.dumps(metadata, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest()) + stage = state.get('report', {}).get('selected_stage') + model = root/'models'/str(stage)/'full'/'model.json' if stage in ('initial', 'corrected') else None + install_available = bool(state['status'] == 'complete' and model is not None and model.is_file() and not model.is_symlink()) + return dict(schema=SCHEMA, binding=binding, inspection=state, install_available=install_available, + selected_model=str(model) if install_available else None, physical_robot_ready=False) + + +def _write_progress(output, status, message, **extra): + output = Path(output) + value = dict(schema=SCHEMA, status=status, message=message, **extra) + target = output/'robot-progress.json' + temporary = output/'robot-progress.tmp' + temporary.write_text(json.dumps(value, allow_nan=False)) + temporary.replace(target) + + +def _record(output, value): + with (Path(output)/'robot-receipt.json').open('x') as stream: + json.dump(value, stream, allow_nan=False) + + +def run_worker(kind, study, assets, output, expected_binding=None, installation=None): + """Worker-only orchestration. Each registered stage is attempted at most once.""" + from . import camera_action_study + steps = []; started = time.perf_counter() + try: + _write_progress(output, 'checking', 'Verifying the registered study and robot assets.') + checked = check_study(study, assets) + if expected_binding is not None and checked['binding'] != expected_binding: + raise ValueError('The checked study, protocol or assets changed. Check the inputs again.') + if kind == 'check': + status = 'blocked' if checked['inspection']['status'] == 'blocked' else 'checked' + elif kind == 'run': + for _ in range(len(camera_action_study.STAGES)+1): + state = checked['inspection'] + if state['status'] in ('blocked', 'complete'): + break + actions = state.get('next_actions', []) + if len(actions) != 1: + raise ValueError('The study did not declare exactly one next stage.') + action = actions[0] + command, stage = action['command'], action['stage'] + if command not in ('collect', 'train', 'evaluate'): + raise ValueError('Unknown registered study stage.') + key = (command, stage) + if key in steps: + raise ValueError('The study requested a repeated stage; no automatic retry is allowed.') + _write_progress(output, 'running', 'Running '+command+' / '+stage+'.', command=command, stage=stage) + steps.append(key) + if command == 'train': + camera_action_study.train(study, stage=stage) + else: + getattr(camera_action_study, command)(study, assets, stage=stage) + checked = check_study(study, assets) + if checked['binding'] != expected_binding: + raise ValueError('The bound study protocol or assets changed during execution.') + else: + raise ValueError('Registered study stage limit exceeded.') + status = checked['inspection']['status'] + elif kind == 'install': + if not checked['install_available']: + raise ValueError('Complete the benchmark before installing its simulation candidate.') + from .arm_skill_install import install_model, inspect_installation + destination = Path(installation) + if destination.exists() or destination.is_relative_to(Path(study)): + raise ValueError('Installation must use a fresh directory outside the study.') + install_model(checked['selected_model'], 'piper', assets, destination) + installation_record = inspect_installation(destination, assets) + checked['installation'] = dict(path=str(destination), record=installation_record) + status = 'installed' + else: + raise ValueError('Unknown robot studio action.') + result = dict(**checked, status=status, executed_stages=[list(step) for step in steps], + elapsed_seconds=time.perf_counter()-started) + _write_progress(output, status, 'Simulation study '+status+'.') + _record(output, result) + return result + except Exception as exc: + result = dict(schema=SCHEMA, status='failed', error=str(exc)[:2000], + executed_stages=[list(step) for step in steps], elapsed_seconds=time.perf_counter()-started, + install_available=False, physical_robot_ready=False) + _write_progress(output, 'failed', str(exc)[:600]) + _record(output, result) + raise + + +class CameraActionStudio: + """One robot job, protected by the parent server's shared lock and pool.""" + def __init__(self, server, *, worker=None, checker=None): + self.server = server + self.worker = worker or run_worker + self.checker = checker or check_study + self.job = None; self.checked = None + + def refresh(self): + job = self.job + if job is None or job['status'] != 'running' or not job['future'].done(): + return + try: + value = job['future'].result() + if type(value) is not dict or value.get('status') not in ('checked', 'blocked', 'complete', 'installed'): + raise ValueError('The robot worker did not return a verified study state.') + verified = self.checker(job['study'], job['assets']) + if verified['binding'] != value['binding']: + raise ValueError('The study or assets changed after the worker completed.') + if value['status'] == 'blocked' or verified['inspection']['status'] == 'blocked': + self.checked = None + value = {**verified, 'status': 'blocked', 'install_available': False} + else: + self.checked = verified + value.update(verified) + job.update(status=value['status'], summary=value, error=None, finished=time.monotonic()) + except Exception as exc: + self.checked = None + job.update(status='failed', summary=None, error=str(exc)[:2000], finished=time.monotonic()) + + def busy(self): + self.refresh() + return self.job is not None and self.job['status'] == 'running' + + def submit(self, kind, study, assets): + with self.server.lock: + self.server.refresh(); self.refresh() + if self.busy() or (self.server.job is not None and self.server.job['status'] == 'running'): + raise RuntimeError('One CPU job is already running. Wait for its outcome.') + try: + study = str(_local_path(study, 'prepared study')); assets = str(_local_path(assets, 'robot assets')) + binding = None + if kind in ('run', 'install'): + if self.checked is None: + raise ValueError('Check the paired demonstrations and assets before continuing.') + verified = self.checker(study, assets) + if verified['binding'] != self.checked['binding']: + raise ValueError('The checked protocol or assets changed. Check the inputs again.') + if kind == 'run' and verified['inspection']['status'] != 'ready': + raise ValueError('This study has no runnable next stage.') + if kind == 'install' and not verified['install_available']: + raise ValueError('No completed simulation candidate is available to install.') + binding = verified['binding'] + elif kind == 'check': + self.checked = None + else: + raise ValueError('Unknown robot studio action.') + except (OSError, ValueError, RuntimeError): + self.checked = None + self.job = None + raise + identity = uuid.uuid4().hex + output = self.server.directory/'robot-jobs'/identity + output.mkdir(parents=True, exist_ok=False) + installation = str(self.server.directory/'installations'/identity) if kind == 'install' else None + future = self.server.pool.submit(self.worker, kind, study, assets, str(output), binding, installation) + self.job = dict(id=identity, kind=kind, output=output, study=study, assets=assets, + status='running', future=future, summary=None, error=None, + started=time.monotonic(), finished=None) + return dict(status='running', job_id=identity, kind=kind) + + def state(self): + with self.server.lock: + self.server.refresh(); self.refresh() + if self.checked is not None and not (self.job and self.job['status'] == 'running'): + try: + protocol = Path(self.checked['binding']['study'])/'protocol.json' + if protocol.is_symlink() or sha256(protocol.read_bytes()).hexdigest() != self.checked['binding']['protocol_sha256']: + raise ValueError('The checked protocol changed. Check the inputs again.') + except (OSError, ValueError) as exc: + self.checked = None + if self.job is not None: + self.job.update(status='failed', summary=None, error=str(exc)[:2000], finished=time.monotonic()) + footage_busy = self.server.job is not None and self.server.job['status'] == 'running' + result = dict(schema=SCHEMA, status='idle', busy=footage_busy, job=None, + summary=None, error=None, checked=False, can_run=False, can_install=False, + physical_robot_ready=False) + if self.job is not None: + job = self.job + progress = None + path = job['output']/'robot-progress.json' + if path.is_file() and path.stat().st_size <= 8192: + try: + progress = json.loads(path.read_text()) + except (ValueError, OSError): + pass + result.update(status=job['status'], busy=footage_busy or job['status']=='running', + summary=job['summary'], error=job['error'], job=dict(id=job['id'],kind=job['kind'], + elapsed_seconds=(job['finished'] or time.monotonic())-job['started'], progress=progress, + study=job['study'], assets=job['assets'])) + if self.checked is not None: + result.update(checked=True, + can_run=not result['busy'] and self.checked['inspection']['status']=='ready', + can_install=not result['busy'] and self.checked['install_available']) + return result diff --git a/src/dvidia_training/camera_action_study.py b/src/dvidia_training/camera_action_study.py new file mode 100644 index 0000000..b12816d --- /dev/null +++ b/src/dvidia_training/camera_action_study.py @@ -0,0 +1,861 @@ +"""Registered local camera-to-action study; failures never become qualifications. + +Every native attempt has a literal registered identity. Repeating a stage may +finish its previously unattempted roster, but never overwrites or retries an +existing attempt. Study inspection imports no renderer and issues no commands. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +from hashlib import sha256 +from functools import wraps +import json +import math +import os +from pathlib import Path +import platform +import shutil +import tempfile +import time +import zipfile + +import numpy as np + +SCHEMA = 'dvidia.camera-action-study/v1' +SEED = 311401 +BOOTSTRAP_SEED = 311402 +GIB = 1024 ** 3 +STUDY_MAX_BYTES = 120 * GIB +DISK_RESERVE_BYTES = 30 * GIB +ATTEMPT_HEADROOM_BYTES = 320 * 1024 ** 2 +GOALS = ((.35, .08, .08), (.40, -.08, .08)) +SOURCES = ('camera_action_study.py', 'camera_action_capture.py', + 'camera_action_model.py', 'camera_action_export.py', + 'piper_rgbd_tracker.py', 'piper_action_capture.py', + 'action_demonstration.py', 'piper_task.py', 'reference_task.py', + 'reference_sim.py', 'piper_imitation_score.py', 'piper_motor_primitive.py', + 'piper_feedback_env.py', 'piper_feedback_score.py', + 'piper_imitation.py', 'robot_profiles.py', 'robot_profiles.lock.json', + 'episode_tape.py', 'embodiment_calibration.py') +STAGES = ('diagnostic', 'demonstrations', 'train-initial', + 'development-initial', 'corrections', 'train-corrected', + 'development-corrected', 'confirmation', 'negative') +LIMITATIONS = [ + 'Synthetic robot demonstrations from an authored teacher; not human-video motor recovery.', + 'One simulated PiPER, a known rigid block, two known goals and bounded starting-position interpolation.', + 'The frozen visual estimator can abstain under gripper occlusion or object tilt; historical anchors are explicitly historical.', + 'Synthetic camera time and host computation are recorded separately; no physical sensor delivery or real-time hardware claim.', + 'Contact, friction, depth accuracy and actuator models remain unqualified on physical hardware.', +] + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def file_hash(path): + h = sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + h.update(chunk) + return h.hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128 * 1024 ** 2: + raise ValueError('Expected a bounded ordinary JSON artifact.') + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON field.') + result[key] = value + return result + def reject(value): + raise ValueError('Nonfinite JSON number.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=reject) + + +def write(path, value): + _create(Path(path), canonical(value) + b'\n') + + +def _create(path, body): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, prefix='.camera-action-', delete=False) as stream: + temporary = Path(stream.name) + stream.write(body) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def _root(root): + root = Path(root).absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (root, *root.parents)): + raise ValueError('Study paths cannot traverse symbolic links.') + return root + + +def runtime_hashes(): + return {name: file_hash(Path(__file__).with_name(name)) for name in SOURCES} + + +def _historical_starts(): + # Previously published original imitation starts; random fresh positions + # use micrometre precision and are explicitly checked against this roster. + return {(x, y) for x in (.284, .294, .304, .314) for y in (-.045, -.035, -.025, -.015)} | { + (x, y) for x in (.289, .309) for y in (-.04, -.02)} | { + (.287, -.032), (.291, -.018), (.299, -.041), (.303, -.028), (.311, -.038), (.307, -.022)} + + +def registered_cases(seed=SEED): + if type(seed) is not int or seed != SEED: + raise ValueError('This v0.1 study uses the registered seed 311401.') + rng = np.random.default_rng(seed) + seen = _historical_starts() + points = [] + while len(points) < 72: + point = tuple(np.round(rng.uniform([.287, -.042], [.311, -.018]), 6).tolist()) + if point not in seen: + seen.add(point) + points.append(point) + splits = {} + offset = 0 + for split, count in (('train', 32), ('development', 8), ('confirmation', 32)): + splits[split] = [] + for index, (x, y) in enumerate(points[offset:offset + count]): + group = f'{split}-{index:02}' + for goal, position in enumerate(GOALS): + splits[split].append(dict(id=f'{group}-goal{goal}', group_id=group, + split=split, goal_id=goal, seed=seed + 2 * (offset + index) + goal, + object_position_m=[x, y, .082], goal_position_m=list(position))) + offset += count + return splits + + +def _attempt(case, controller, *, prefix='', negative=None): + return dict(id=f'{prefix}{case["id"]}-{controller}' + (f'-{negative}' if negative else ''), + case=deepcopy(case), controller=controller, negative_control=negative) + + +def prepare(root, *, fixed_model, seed=SEED): + """Register every scene/controller before capture and pin the comparator.""" + from .piper_motor_primitive import validate_model + root = _root(root) + if root.exists(): + raise ValueError('Use a fresh study directory; registered runs are immutable.') + model = validate_model(read(fixed_model)) + sources = runtime_hashes() + cases = registered_cases(seed) + dev_evaluation = cases['development'] + rosters = { + 'diagnostic': [_attempt(c, 'teacher', prefix='diagnostic-') for c in dev_evaluation[:4]], + 'demonstrations': [_attempt(c, 'teacher') for c in cases['train'] + cases['development']], + 'corrections': [_attempt(cases['train'][2 * i + i % 2], 'mixed') for i in range(32)], + } + for stage in ('initial', 'corrected'): + rosters[f'development-{stage}'] = [_attempt(c, controller, prefix=stage + '-') + for c in dev_evaluation for controller in ('student', 'blind')] + rosters['confirmation'] = [_attempt(c, controller) for c in cases['confirmation'] + for controller in ('student', 'blind', 'fixed', 'teacher')] + rosters['negative'] = [_attempt(c, 'student', prefix='negative-', negative=negative) + for c in cases['confirmation'][:4] + for negative in ('initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper')] + protocol = dict(schema=SCHEMA, seed=seed, cases=cases, rosters=rosters, + goals=[list(g) for g in GOALS], source_sha256=sources, fixed_model_sha256=digest(model), + historical_start_exclusion=[list(p) for p in sorted(_historical_starts())], + start_domain=dict(x_m=[.287, .311], y_m=[-.042, -.018], + reason='Three millimetre inset from the original fixed primitive envelope buffers observed camera estimation error; estimates are never clipped or replaced by native truth.'), + grouping='Both goals, all cameras, controls and corrections stay in their exact starting-position group.', + capture=dict(duration_s=20., width=320, height=240, camera_hz=25, control_hz=50, + physics_timestep_s=.001, camera_pairs_per_complete_episode=501), + fitting=dict(hidden_units=[64, 64], epochs=50, batch_size=512, learning_rate=.003, seed=17, + normalization='training-only', corrections_rounds=1, teacher_mix_probability=.5), + data_quality=dict(minimum_successful_episodes_per_goal=28, all_training_groups_covered=True), + diagnostic=dict(all_four_strict_successful=True, maximum_initial_camera_error_m=.010, + error_source='Privileged native diagnostic only; never a learner feature.'), + selection='full learner development strict success descending, critical contacts ascending, common physical-scale prediction loss ascending, then initial before corrected; matched blind from selected stage', + acceptance=dict(student_successes_minimum=58, student_attempts=64, confidence=.975, + bootstrap_resamples=10000, bootstrap_seed=BOOTSTRAP_SEED, bootstrap_unit='starting-position group; both goals paired', + comparisons=['student-minus-fixed', 'student-minus-blind'], no_critical_contact_increase=True, + minimum_common_admitted_groups=24, + common_admission='Both goals must have an available bound initial camera and at least one completed control interval for both compared controllers. Require a positive paired interval on this subset as well; initial comparator refusal alone cannot establish improvement.', + all_negative_controls_correct=True, no_privileged_features_or_teacher_intervention=True), + storage=dict(maximum_study_bytes=STUDY_MAX_BYTES, disk_reserve_bytes=DISK_RESERVE_BYTES, + next_attempt_headroom_bytes=ATTEMPT_HEADROOM_BYTES), + physical_robot_ready=False, limitations=LIMITATIONS) + root.mkdir(parents=True) + write(root / 'fixed-model.json', model) + write(root / 'protocol.json', protocol) + write(root / 'registration.json', dict(schema=SCHEMA + '/registration', + protocol_sha256=file_hash(root / 'protocol.json'), fixed_model_file_sha256=file_hash(root / 'fixed-model.json'), + source_sha256=sources, registered_before_any_capture=True)) + return inspect(root) + + +def _protocol(root, *, runtime=False): + root = _root(root) + registration = read(root / 'registration.json') + if registration.get('schema') != SCHEMA + '/registration': + raise ValueError('Unknown registration schema.') + if file_hash(root / 'protocol.json') != registration['protocol_sha256']: + raise ValueError('Registered protocol bytes changed.') + if file_hash(root / 'fixed-model.json') != registration['fixed_model_file_sha256']: + raise ValueError('Pinned fixed comparator changed.') + protocol = read(root / 'protocol.json') + if protocol.get('schema') != SCHEMA or protocol.get('source_sha256') != registration['source_sha256']: + raise ValueError('Invalid or inconsistent protocol.') + if runtime and runtime_hashes() != registration['source_sha256']: + raise ValueError('Runtime changed after registration; preserve this study and register a new one.') + return protocol + + +def _tree_bytes(root): + size = 0 + for directory, subdirs, files in os.walk(root): + if any((Path(directory) / name).is_symlink() for name in subdirs + files): + raise ValueError('Study artifacts cannot contain symbolic links.') + size += sum((Path(directory) / name).stat().st_size for name in files) + return size + + +def storage_preflight(root, *, known_study_bytes=None): + root = _root(root) + size = _tree_bytes(root) if known_study_bytes is None else known_study_bytes + free = shutil.disk_usage(root).free + if size + ATTEMPT_HEADROOM_BYTES > STUDY_MAX_BYTES: + raise ValueError('Study storage budget would exceed 120 GiB; existing evidence retained.') + if free - ATTEMPT_HEADROOM_BYTES < DISK_RESERVE_BYTES: + raise ValueError('Next attempt would consume the protected 30 GiB disk reserve.') + return dict(study_bytes=size, free_bytes=free, maximum_study_bytes=STUDY_MAX_BYTES, + disk_reserve_bytes=DISK_RESERVE_BYTES) + + +def _single_worker(function): + @wraps(function) + def wrapped(root, *args, **kwargs): + _protocol(root, runtime=True) + lock = _root(root) / 'worker.lock' + try: + descriptor = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as exc: + raise ValueError('This study already has an active or interrupted worker; no concurrent execution or automatic lock removal.') from exc + try: + with os.fdopen(descriptor, 'wb') as stream: + stream.write(canonical(dict(pid=os.getpid(), operation=function.__name__))) + stream.flush() + os.fsync(stream.fileno()) + return function(root, *args, **kwargs) + finally: + lock.unlink(missing_ok=True) + return wrapped + + +def _stage(root, name): + path = Path(root) / 'stages' / (name + '.json') + if not path.exists(): + return None + seal = read(path.with_name(name + '.integrity.json')) + if seal != dict(sha256=file_hash(path), protocol_sha256=file_hash(Path(root) / 'protocol.json')): + raise ValueError('Stage receipt identity changed.') + return read(path) + + +def _seal_stage(root, name, receipt): + path = Path(root) / 'stages' / (name + '.json') + write(path, receipt) + write(path.with_name(name + '.integrity.json'), dict(sha256=file_hash(path), + protocol_sha256=file_hash(Path(root) / 'protocol.json'))) + + +def _require(root, stage, status='complete'): + result = _stage(root, stage) + if result is None or result.get('status') != status: + raise ValueError(f'Stage {stage} must be {status} before continuing.') + return result + + +def _begin(root, name, protocol, *, model_bindings=None): + target = Path(root) / 'stages' / (name + '.json') + if target.exists(): + raise ValueError('Stage already sealed; existing attempts cannot be retried.') + started = target.with_name(name + '.started.json') + if not started.exists(): + write(started, dict(schema=SCHEMA + '/stage-start', stage=name, + protocol_sha256=file_hash(Path(root) / 'protocol.json'), source_sha256=protocol['source_sha256'], + model_bindings=model_bindings)) + elif read(started)['protocol_sha256'] != file_hash(Path(root) / 'protocol.json'): + raise ValueError('Stage start is bound to a different protocol.') + elif read(started).get('model_bindings') != model_bindings: + raise ValueError('Model identity differs from the started stage.') + + +def _critical(summary): + # The capture receipt reports the native per-millisecond count explicitly. + value = summary.get('critical_contact_ticks') + if value is None: + value = summary.get('native_status', {}).get('native_info', {}).get('metrics', {}).get('critical_contact_ticks') + if value is None: + value = summary.get('score', {}).get('critical_contact_ticks') + return value if type(value) is int and value >= 0 else None + + +def _row(root, attempt, summary, directory, *, model_sha256=None): + expected_mode = 'student' if attempt['controller'] == 'blind' else attempt['controller'] + expected_case = {**attempt['case'], 'policy_artifact_sha256': model_sha256} if model_sha256 else attempt['case'] + if (summary.get('case') != expected_case or summary.get('mode') != expected_mode + or summary.get('negative_control') != attempt['negative_control']): + raise ValueError('Rollout identity differs from its registered case and control.') + score = summary.get('score') or {} + complete = summary.get('status') == 'complete' and summary.get('recorded_complete_intervals') == 1000 + return dict(id=attempt['id'], case=attempt['case'], controller=attempt['controller'], + negative_control=attempt['negative_control'], path=str(directory.relative_to(root)), + policy_artifact_sha256=model_sha256, + status=summary.get('status', 'unknown'), complete=complete, strict_success=complete and score.get('success') is True, + score=score, critical_contact_ticks=_critical(summary), + initial_anchor_available=summary.get('initial_anchor_available') is True, + initial_anchor_binding_valid=summary.get('initial_anchor_binding_valid') is True, + initial_anchor_error_m=summary.get('initial_anchor_error_m'), + recorded_complete_intervals=summary.get('recorded_complete_intervals', 0), + provenance_ok=summary.get('no_privileged_features_or_teacher_intervention', + summary.get('provenance', {}).get('learner_input_whitelist_verified', False)) is True, + perception=summary.get('perception'), timing=_timing_summary(summary.get('timing') or {}), + receipt_sha256=file_hash(directory / 'rollout.json') if (directory / 'rollout.json').exists() else None, + integrity_sha256=file_hash(directory / 'integrity.json') if (directory / 'integrity.json').exists() else None) + + +def _timing_summary(timing): + result = {name: timing.get(name) for name in ('wall_seconds', 'scope')} + boundaries = timing.get('boundaries') or [] + for metric in ('estimator_wall_seconds', 'policy_inference_wall_seconds', 'full_boundary_wall_seconds', + 'render_encode_wall_seconds'): + values = [row[metric] for row in boundaries if type(row.get(metric)) in (int, float) + and math.isfinite(row[metric]) and row[metric] >= 0.] + result[metric] = (dict(samples=len(values), p50=float(np.quantile(values, .5)), + p95=float(np.quantile(values, .95)), maximum=max(values)) if values else None) + return result + + +def _peak_rss(): + try: + import resource + return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * (1 if platform.system() == 'Darwin' else 1024) + except ImportError: + return None + + +class _FixedPolicy: + """Adapt the pinned open-loop comparator using only allowed camera context.""" + def __init__(self, model): + from .piper_motor_primitive import MotorPrimitivePolicy + self.policy = MotorPrimitivePolicy(model) + self.anchor = None + + def __call__(self, context): + snapshot = context['snapshot'] + signals = snapshot['signals'] + if self.anchor is None: + estimate = context['estimate'] + candidate = estimate.get('object_position_estimate_m') + if candidate is None: + candidate = estimate.get('signals', {}).get('object_position_estimate_m') + if isinstance(candidate, dict): + candidate = candidate.get('value') if candidate.get('status') == 'available' else None + if candidate is None: + raise ValueError('Fixed comparator requires its initial camera position.') + self.anchor = list(candidate) + return self.policy(dict(t_seconds=snapshot['timestamp_seconds'], + joint_position_rad=signals['joint_position_rad']['value'], + object_position_m=self.anchor, target_position_m=context['goal_position_m'])) + + +def _model_path(root, stage, controller): + return Path(root) / 'models' / stage / ('blind' if controller == 'blind' else 'full') / 'model.json' + + +def _model_document(root, stage, controller): + receipt = _require(root, 'train-' + stage) + variant = 'blind' if controller == 'blind' else 'full' + path = _model_path(root, stage, controller) + if receipt['models'][variant]['model_sha256'] != file_hash(path): + raise ValueError('Model bytes differ from the sealed training receipt.') + return read(path) + + +def _model_bindings(root, name, model_stage): + protocol = _protocol(root) + result = {} + for controller in {a['controller'] for a in protocol['rosters'][name]}: + if controller == 'teacher': + result[controller] = digest(protocol['source_sha256']) + elif controller == 'fixed': + result[controller] = file_hash(Path(root) / 'fixed-model.json') + else: + _model_document(root, model_stage, controller) + result[controller] = file_hash(_model_path(root, model_stage, controller)) + return result + + +def _policy(root, controller, model_stage): + if controller == 'teacher': + return None + if controller == 'fixed': + return _FixedPolicy(read(Path(root) / 'fixed-model.json')) + from .camera_action_model import ModelPolicy + return ModelPolicy(_model_document(root, model_stage, controller)) + + +def _run_stage(root, assets, name, *, model_stage=None): + from .camera_action_capture import run_rollout, inspect_rollout + root = _root(root) + protocol = _protocol(root, runtime=True) + model_bindings = _model_bindings(root, name, model_stage) + _begin(root, name, protocol, model_bindings=model_bindings) + rows = [] + blocked = None + study_bytes = _tree_bytes(root) + for attempt in protocol['rosters'][name]: + directory = root / 'episodes' / name / attempt['id'] + if blocked is not None: + rows.append(dict(**attempt, status='unattempted', complete=False, strict_success=False, + critical_contact_ticks=None, reason=blocked)) + continue + if directory.exists(): + if not (directory / 'rollout.json').exists() or not (directory / 'integrity.json').exists(): + rows.append(dict(**attempt, status='unknown', complete=False, strict_success=False, + critical_contact_ticks=None, path=str(directory.relative_to(root)), reason='Unsealed prior attempt retained; never retried.')) + continue + inspect_rollout(directory) + summary = read(directory / 'rollout.json') + else: + try: + storage_preflight(root, known_study_bytes=study_bytes) + except ValueError as exc: + blocked = str(exc) + rows.append(dict(**attempt, status='unattempted', complete=False, strict_success=False, + critical_contact_ticks=None, reason=blocked)) + continue + controller = attempt['controller'] + policy = _policy(root, controller, model_stage) if controller != 'teacher' else None + print(json.dumps(dict(stage=name, attempt=attempt['id'], status='starting')), flush=True) + bound_case = {**attempt['case'], 'policy_artifact_sha256': model_bindings[controller]} + summary = run_rollout(assets, directory, case=bound_case, + mode='student' if controller == 'blind' else controller, policy=policy, + negative_control=attempt['negative_control'], duration_s=20.) + inspect_rollout(directory) + study_bytes += _tree_bytes(directory) + rows.append(_row(root, attempt, summary, directory, model_sha256=model_bindings[attempt['controller']])) + print(json.dumps(dict(stage=name, attempt=attempt['id'], status=rows[-1]['status'], + strict_success=rows[-1]['strict_success'])), flush=True) + receipt = dict(schema=SCHEMA + '/stage', stage=name, + protocol_sha256=file_hash(root / 'protocol.json'), rows=rows, model_bindings=model_bindings, + scheduled=len(protocol['rosters'][name]), strict_successes=sum(r['strict_success'] for r in rows), + status='blocked' if blocked or any(r['status'] in ('unknown', 'unattempted') for r in rows) else 'complete', + reason=blocked, physical_robot_ready=False) + return receipt + + +def data_quality(rows, protocol): + training = [r for r in rows if r['case']['split'] == 'train'] + successes = [r for r in training if r.get('strict_success') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid')] + required = {c['group_id'] for c in protocol['cases']['train']} + covered = {r['case']['group_id'] for r in successes} + goals = {str(g): sum(r['case']['goal_id'] == g for r in successes) for g in range(2)} + return dict(passed=covered == required and all(n >= 28 for n in goals.values()), + covered_training_groups=len(covered), required_training_groups=32, + missing_training_groups=sorted(required - covered), successful_episodes_per_goal=goals, + minimum_successful_episodes_per_goal=28, + successful_with_available_bound_initial_camera=len(successes)) + + +@_single_worker +def collect(root, assets, *, stage='diagnostic'): + root = _root(root) + if stage not in ('diagnostic', 'demonstrations', 'corrections'): + raise ValueError('Unknown collection stage.') + if stage != 'diagnostic': + _require(root, 'diagnostic') + if stage == 'corrections': + _require(root, 'train-initial') + _require(root, 'development-initial') + receipt = _run_stage(root, assets, stage, model_stage='initial' if stage == 'corrections' else None) + if stage == 'diagnostic': + passed = all(r.get('strict_success') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid') + and type(r.get('initial_anchor_error_m')) in (int, float) + and math.isfinite(r['initial_anchor_error_m']) + and 0. <= r['initial_anchor_error_m'] <= .010 for r in receipt['rows']) + receipt['diagnostic_passed'] = passed + if not passed: + receipt.update(status='blocked', reason='Diagnostic did not demonstrate complete teacher success and a bound initial visual anchor within 10 mm of native diagnostic truth in all four cases.') + elif stage == 'demonstrations': + quality = data_quality(receipt['rows'], _protocol(root)) + receipt['data_quality'] = quality + if not quality['passed']: + receipt.update(status='blocked', reason='Teacher demonstration quality gate failed; learning and confirmation are blocked.') + _seal_stage(root, stage, receipt) + if receipt['status'] == 'blocked' and not (root / 'report.json').exists(): + write(root / 'report.json', report_study(root)) + return receipt + + +def _dataset(root, rows, *, blind, label_source): + from .camera_action_export import export_episode, combine_datasets + selected = [r for r in rows if r.get('complete') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid') + and (label_source == 'teacher_query' or r.get('strict_success'))] + if not selected: + raise ValueError('No complete, camera-bound episodes are eligible for learning.') + return combine_datasets([export_episode(Path(root) / r['path'], blind=blind, + label_source=label_source) for r in selected]), [r['id'] for r in selected] + + +@_single_worker +def train(root, *, stage='initial'): + from .camera_action_export import combine_datasets, blind_dataset + from .camera_action_model import train as fit + root = _root(root) + if stage not in ('initial', 'corrected'): + raise ValueError('Unknown model fitting stage.') + protocol = _protocol(root, runtime=True) + demos = _require(root, 'demonstrations') + corrections = _require(root, 'corrections') if stage == 'corrected' else None + name = 'train-' + stage + _begin(root, name, protocol) + training_rows = [r for r in demos['rows'] if r['case']['split'] == 'train'] + dev_rows = [r for r in demos['rows'] if r['case']['split'] == 'development'] + export_began = time.perf_counter() + full_dataset, selected = _dataset(root, training_rows, blind=False, label_source='issued') + if corrections: + correction_data, correction_ids = _dataset(root, corrections['rows'], blind=False, label_source='teacher_query') + full_dataset = combine_datasets([full_dataset, correction_data]) + selected += correction_ids + full_development, development_ids = _dataset(root, dev_rows, blind=False, label_source='issued') + export_seconds = time.perf_counter() - export_began + models = {} + for variant in ('full', 'blind'): + target = root / 'models' / stage / variant + if target.exists(): + raise ValueError('Model fitting attempt already exists; never silently refit it.') + target.mkdir(parents=True) + write(target / 'started.json', dict(stage=stage, variant=variant, + protocol_sha256=file_hash(root / 'protocol.json'))) + began = time.perf_counter() + dataset = blind_dataset(full_dataset) if variant == 'blind' else full_dataset + development = blind_dataset(full_development) if variant == 'blind' else full_development + model = fit(dataset, blind=variant == 'blind', stage=stage, seed=17, epochs=50, + batch_size=512, learning_rate=.003, validation_dataset=development) + write(target / 'model.json', model) + models[variant] = dict(path=str((target / 'model.json').relative_to(root)), + model_sha256=file_hash(target / 'model.json'), model_bytes=(target / 'model.json').stat().st_size, + training_wall_seconds=time.perf_counter() - began, training_episodes=selected, + shared_export_wall_seconds=export_seconds, process_peak_rss_bytes=_peak_rss(), + memory_scope='Process high-water RSS including shared dataset export and any preceding fit.', + development_episodes=development_ids, training=model.get('training'), + dataset_provenance=dataset.get('provenance'), physical_robot_ready=False) + write(target / 'receipt.json', models[variant]) + receipt = dict(schema=SCHEMA + '/training', stage=name, status='complete', models=models, + protocol_sha256=file_hash(root / 'protocol.json'), physical_robot_ready=False) + _seal_stage(root, name, receipt) + return receipt + + +def _loss(model): + training = model.get('training') or {} + validation = training.get('validation') or {} + value = validation.get('action_scale_mse') + if type(value) in (int, float) and math.isfinite(value): + return float(value) + raise ValueError('Model must report a finite development prediction loss for registered selection.') + + +def freeze(root): + root = _root(root) + protocol = _protocol(root, runtime=True) + existing = root / 'freeze.json' + if existing.exists(): + _verify_freeze(root) + return read(existing) + candidates = [] + for index, stage in enumerate(('initial', 'corrected')): + receipt = _require(root, 'development-' + stage) + if receipt.get('model_bindings') != _model_bindings(root, 'development-' + stage, stage): + raise ValueError('Development outcomes do not match the trained model identities.') + rows = [r for r in receipt['rows'] if r['controller'] == 'student'] + if len(rows) != 16 or any(r.get('critical_contact_ticks') is None for r in rows): + raise ValueError('Development comparison requires all 16 learner outcomes and contact counts.') + candidates.append(dict(stage=stage, strict_successes=sum(r['strict_success'] for r in rows), + critical_contact_ticks=sum(r['critical_contact_ticks'] for r in rows), + prediction_loss=_loss(_model_document(root, stage, 'student')), order=index)) + chosen = min(candidates, key=lambda c: (-c['strict_successes'], c['critical_contact_ticks'], c['prediction_loss'], c['order'])) + identities = {} + for stage in ('initial', 'corrected'): + for controller in ('student', 'blind'): + _model_document(root, stage, controller) + path = _model_path(root, stage, controller) + identities[str(path.relative_to(root))] = file_hash(path) + for stage in STAGES[:7]: + path = root / 'stages' / (stage + '.json') + identities[str(path.relative_to(root))] = file_hash(path) + result = dict(schema=SCHEMA + '/freeze', selected_stage=chosen['stage'], candidates=candidates, + protocol_sha256=file_hash(root / 'protocol.json'), source_sha256=protocol['source_sha256'], + artifacts=identities, fixed_model_sha256=file_hash(root / 'fixed-model.json'), + frozen_before_confirmation=True, physical_robot_ready=False) + write(existing, result) + return result + + +def _verify_freeze(root, *, runtime=True): + frozen = read(Path(root) / 'freeze.json') + protocol = _protocol(root, runtime=runtime) + if frozen.get('schema') != SCHEMA + '/freeze' or frozen['source_sha256'] != protocol['source_sha256']: + raise ValueError('Frozen runtime identity changed.') + if frozen['protocol_sha256'] != file_hash(Path(root) / 'protocol.json'): + raise ValueError('Frozen protocol identity changed.') + if frozen['fixed_model_sha256'] != file_hash(Path(root) / 'fixed-model.json'): + raise ValueError('Frozen comparator identity changed.') + for name, expected in frozen['artifacts'].items(): + path = Path(name) + if path.is_absolute() or '..' in path.parts or file_hash(Path(root) / path) != expected: + raise ValueError('Frozen artifact changed.') + return frozen + + +@_single_worker +def evaluate(root, assets, *, stage='development-initial'): + root = _root(root) + if stage in ('development-initial', 'development-corrected'): + model_stage = stage.split('-', 1)[1] + _require(root, 'train-' + model_stage) + elif stage in ('confirmation', 'negative'): + frozen = freeze(root) + model_stage = frozen['selected_stage'] + if stage == 'negative': + _require(root, 'confirmation') + else: + raise ValueError('Unknown evaluation stage.') + receipt = _run_stage(root, assets, stage, model_stage=model_stage) + if stage in ('confirmation', 'negative'): + receipt['freeze_sha256'] = file_hash(root / 'freeze.json') + _seal_stage(root, stage, receipt) + if stage == 'negative': + report = report_study(root) + write(root / 'report.json', report) + return receipt + + +def _admitted(row): + return (row.get('initial_anchor_available') is True and row.get('initial_anchor_binding_valid') is True + and type(row.get('recorded_complete_intervals')) is int and row['recorded_complete_intervals'] > 0) + + +def paired_bootstrap(rows, comparator, *, admitted_only=False): + grouped = {} + for row in rows: + if row['controller'] not in ('student', comparator): + continue + group = row['case']['group_id'] + key = (row['controller'], row['case']['goal_id']) + if key in grouped.setdefault(group, {}): + raise ValueError('Duplicate paired outcome.') + grouped[group][key] = row + keys = {(c, g) for c in ('student', comparator) for g in (0, 1)} + if len(grouped) != 32 or any(set(g) != keys for g in grouped.values()): + raise ValueError('Paired interval requires exactly 32 complete starting-position groups.') + retained = {key: value for key, value in grouped.items() + if not admitted_only or all(_admitted(row) for row in value.values())} + differences = np.asarray([sum(float(g[('student', goal)].get('strict_success') is True) + - float(g[(comparator, goal)].get('strict_success') is True) + for goal in (0, 1)) / 2. for _, g in sorted(retained.items())]) + rng = np.random.default_rng(BOOTSTRAP_SEED) + count = len(differences) + sampled = differences[rng.integers(0, count, size=(10000, count))].mean(axis=1) if count else None + bounds = np.quantile(sampled, [.0125, .9875]).tolist() if count else [None, None] + return dict(comparison='student-minus-' + comparator, difference=float(differences.mean()) if count else None, + confidence=.975, lower=bounds[0], upper=bounds[1], resamples=10000, seed=BOOTSTRAP_SEED, + groups=count, excluded_groups=32-count, admitted_only=admitted_only, + pairing='both goals within exact starting-position group', positive=bounds[0] is not None and bounds[0] > 0.) + + +def _negative_correct(row): + if row.get('negative_control') == 'open_gripper': + score = row.get('score') or {} + return (row.get('complete') is True and row.get('strict_success') is False + and score.get('valid') is True + and (score.get('strict_v01') or {}).get('trajectory_complete') is True) + return (row.get('status') == 'refused' and row.get('recorded_complete_intervals') == 0 + and row.get('strict_success') is False) + + +def report_study(root): + root = _root(root) + protocol = _protocol(root) + confirmation = _stage(root, 'confirmation') + negatives = _stage(root, 'negative') + report = dict(schema=SCHEMA + '/report', physical_robot_ready=False, qualified=False, + qualification='research_candidate', protocol_sha256=file_hash(root / 'protocol.json'), + limitations=LIMITATIONS, status='incomplete', controllers={}, comparisons=[], admitted_comparisons=[], gates={}) + if confirmation is None or negatives is None: + blocked = [name for name in STAGES if (_stage(root, name) or {}).get('status') == 'blocked'] + report.update(status='blocked' if blocked else 'incomplete', blocked_stages=blocked) + return report + frozen = _verify_freeze(root, runtime=False) + if any(stage.get('freeze_sha256') != file_hash(root / 'freeze.json') for stage in (confirmation, negatives)): + raise ValueError('Evaluation stage is not bound to the frozen selection.') + rows = confirmation['rows'] + for stage, receipt in (('confirmation', confirmation), ('negative', negatives)): + expected = {a['id']: a for a in protocol['rosters'][stage]} + for row in receipt['rows']: + original = expected.get(row['id']) + if original is None or any(row.get(key) != original[key] for key in ('case', 'controller', 'negative_control')): + raise ValueError('Reported outcome identity differs from the registered roster.') + complete_roster = (len(rows) == 256 and {r['id'] for r in rows} == {r['id'] for r in protocol['rosters']['confirmation']} + and len(negatives['rows']) == 16 + and {r['id'] for r in negatives['rows']} == {r['id'] for r in protocol['rosters']['negative']} + and confirmation['status'] == 'complete' and negatives['status'] == 'complete') + for controller in ('student', 'blind', 'fixed', 'teacher'): + selected = [r for r in rows if r['controller'] == controller] + contacts = [r.get('critical_contact_ticks') for r in selected] + report['controllers'][controller] = dict(scheduled=len(selected), + strict_successes=sum(r.get('strict_success') is True for r in selected), + complete=sum(r.get('complete') is True for r in selected), + admitted_cases=sum(_admitted(r) for r in selected), + initial_refusals=sum(r.get('status') in ('refused', 'interrupted') + and r.get('recorded_complete_intervals') == 0 for r in selected), + critical_contact_episodes=sum(v > 0 for v in contacts if v is not None), + critical_contact_ticks=sum(v for v in contacts if v is not None), + contact_counts_complete=all(v is not None for v in contacts)) + if complete_roster: + report['comparisons'] = [paired_bootstrap(rows, comparator) for comparator in ('fixed', 'blind')] + report['admitted_comparisons'] = [paired_bootstrap(rows, comparator, admitted_only=True) for comparator in ('fixed', 'blind')] + student = report['controllers']['student'] + fixed = report['controllers']['fixed'] + report['gates'] = dict( + all_registered_attempts_accounted=complete_roster, + at_least_58_of_64=student['scheduled'] == 64 and student['strict_successes'] >= 58, + positive_paired_comparisons=len(report['comparisons']) == 2 and all(c['positive'] for c in report['comparisons']), + positive_common_admitted_comparisons=len(report['admitted_comparisons']) == 2 + and all(c['groups'] >= 24 and c['positive'] for c in report['admitted_comparisons']), + no_critical_contact_increase=student['contact_counts_complete'] and fixed['contact_counts_complete'] + and student['critical_contact_episodes'] <= fixed['critical_contact_episodes'], + all_negative_controls_correct=len(negatives['rows']) == 16 and all(_negative_correct(r) for r in negatives['rows']), + no_privileged_features_or_teacher_intervention=all(r.get('provenance_ok') is True for r in rows if r['controller'] in ('student', 'blind', 'fixed'))) + report['qualified'] = all(report['gates'].values()) + report['qualification'] = 'bounded_simulation_skill' if report['qualified'] else 'research_candidate' + report['status'] = 'complete' if complete_roster else 'incomplete' + report['selected_stage'] = frozen['selected_stage'] + return report + + +def inspect(root): + """Read-only UI/CLI state; does not import native simulation or run models.""" + root = _root(root) + protocol = _protocol(root) + stages = {name: _stage(root, name) for name in STAGES} + blocked = [name for name, value in stages.items() if value and value.get('status') == 'blocked'] + current = next((name for name, value in stages.items() if value is None), None) + report = report_study(root) + command = ('collect' if current in ('diagnostic', 'demonstrations', 'corrections') else + 'train' if current and current.startswith('train-') else 'evaluate') + actions = [] if blocked or current is None else [dict(command=command, + stage=current.removeprefix('train-') if command == 'train' else current, + requires_assets=command != 'train')] + return dict(schema=SCHEMA + '/inspection', status='blocked' if blocked else ('complete' if current is None else 'ready'), + study=str(root), protocol_sha256=file_hash(root / 'protocol.json'), + stage_summaries={name: None if value is None else {key: value.get(key) for key in + ('status', 'scheduled', 'strict_successes', 'reason', 'diagnostic_passed', 'data_quality')} for name, value in stages.items()}, + next_stage=None if blocked else current, next_actions=actions, blocked_stages=blocked, report=report, + qualification=report['qualification'], qualified=report['qualified'], + registered_attempts={name: len(rows) for name, rows in protocol['rosters'].items()}, + scope='Simulation-only camera-to-action research; no physical commands or human-video motor recovery.', + physical_robot_ready=False, physics_executed=False, robot_commands_sent=False) + + +def package(root, output): + """Create an evidence package, including blockers; installer bundles are separate.""" + root, output = _root(root), _root(output) + if output.exists() or output.is_relative_to(root): + raise ValueError('Choose a fresh package directory outside the study.') + state = inspect(root) + output.mkdir(parents=True) + files = [root / 'protocol.json', root / 'registration.json', root / 'fixed-model.json'] + files += sorted((root / 'stages').glob('*.json')) if (root / 'stages').exists() else [] + files += sorted((root / 'models').rglob('*.json')) if (root / 'models').exists() else [] + if (root / 'freeze.json').exists(): + files.append(root / 'freeze.json') + inventory = {} + for source in files: + relative = source.relative_to(root) + raw = source.read_bytes() + _create(output / 'evidence' / relative, raw) + inventory[str(Path('evidence') / relative)] = dict(bytes=len(raw), sha256=sha256(raw).hexdigest()) + write(output / 'report.json', state['report']) + inventory['report.json'] = dict(bytes=(output / 'report.json').stat().st_size, sha256=file_hash(output / 'report.json')) + write(output / 'manifest.json', dict(schema=SCHEMA + '/evidence-package', files=inventory, + qualification=state['report']['qualification'], qualified=state['report']['qualified'], + raw_episode_data_included=False, model_runtime_installation_included=False, + source_study_protocol_sha256=file_hash(root / 'protocol.json'), physical_robot_ready=False)) + with zipfile.ZipFile(output / 'camera-action-evidence.zip', 'x', zipfile.ZIP_DEFLATED) as archive: + for name in [*inventory, 'manifest.json']: + archive.write(output / name, name) + return dict(status='complete', output=str(output), qualification=state['report']['qualification'], + qualified=state['report']['qualified'], evidence_sha256=file_hash(output / 'camera-action-evidence.zip'), + physical_robot_ready=False) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + create = sub.add_parser('prepare') + create.add_argument('--study', required=True) + create.add_argument('--fixed-model', required=True) + create.add_argument('--seed', type=int, default=SEED) + for name, choices, default in ( + ('collect', ('diagnostic', 'demonstrations', 'corrections'), 'diagnostic'), + ('train', ('initial', 'corrected'), 'initial'), + ('evaluate', ('development-initial', 'development-corrected', 'confirmation', 'negative'), 'development-initial')): + cmd = sub.add_parser(name) + cmd.add_argument('--study', required=True) + cmd.add_argument('--stage', choices=choices, default=default) + if name != 'train': + cmd.add_argument('--assets', required=True) + check = sub.add_parser('inspect') + check.add_argument('--study', required=True) + packed = sub.add_parser('package') + packed.add_argument('--study', required=True) + packed.add_argument('--output', required=True) + args = parser.parse_args(argv) + try: + if args.command == 'prepare': + result = prepare(args.study, fixed_model=args.fixed_model, seed=args.seed) + elif args.command in ('collect', 'evaluate'): + result = globals()[args.command](args.study, args.assets, stage=args.stage) + elif args.command == 'train': + result = train(args.study, stage=args.stage) + elif args.command == 'package': + result = package(args.study, args.output) + else: + result = inspect(args.study) + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 if result.get('status') not in ('blocked', 'refused', 'incomplete') else 1 + except (ValueError, OSError, RuntimeError, TypeError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc), physical_robot_ready=False), allow_nan=False)) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/dvidia_training/footage_pipeline.py b/src/dvidia_training/footage_pipeline.py index f2b1cd5..265a2f9 100644 --- a/src/dvidia_training/footage_pipeline.py +++ b/src/dvidia_training/footage_pipeline.py @@ -337,6 +337,9 @@ def inspect_run(path): def main(): + if len(sys.argv) > 1 and sys.argv[1] == 'robot': + from .camera_action_study import main as robot_main + raise SystemExit(robot_main(sys.argv[2:])) parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) for name in ('prepare', 'run'): diff --git a/src/dvidia_training/piper_action_capture.py b/src/dvidia_training/piper_action_capture.py index 775fd31..f5d429c 100644 --- a/src/dvidia_training/piper_action_capture.py +++ b/src/dvidia_training/piper_action_capture.py @@ -132,12 +132,18 @@ def __init__(self, assets, config): self.assets=assets;self.config=config;self.renderer=None;self.env=None self.physics_attempted=False;self.initialized=False;self.renderer_warnings=[] + def _task_configuration(self, task): + return task.PiperTaskConfig(timeout_s=self.config.duration_s) + + def _environment(self, task): + return task.PiperTaskEnv(self.assets,self.task_config) + def open(self): from . import piper_task as task, reference_sim as sim, robot_profiles self.mj,self.np=sim._native_imports();self.sim=sim - self.task_config=task.PiperTaskConfig(timeout_s=self.config.duration_s) + self.task_config=self._task_configuration(task) self.assets_metadata=robot_profiles.verify_assets('piper',task._asset_root(self.assets)) - self.env=task.PiperTaskEnv(self.assets,self.task_config);self.physics_attempted=True + self.env=self._environment(task);self.physics_attempted=True self.observation,self.info=self.env.reset(self.config.seed);self.initialized=True self.native_origin=float(self.env.data.time) self.base=self.mj.mj_name2id(self.env.model,self.mj.mjtObj.mjOBJ_BODY,'base_link') @@ -219,6 +225,9 @@ def sample(self,index): or depth.shape!=(self.config.height,self.config.width) or depth.dtype!=self.np.float32 or not self.np.isfinite(depth).all() or not (depth>0).all()):raise RuntimeError('Renderer did not supply finite aligned RGB and metric depth arrays.') if native_time!=float(self.env.data.time) or seconds!=float(self.env.elapsed):raise RuntimeError('Physics advanced during paired RGB-D capture.') + # Only the latest pair is retained for optional observation-sidecar + # consumers. Existing v1 capture files and behavior are unchanged. + self.latest_rgbd=(rgb,depth) depth_buffer=io.BytesIO();self.np.save(depth_buffer,depth.astype(' @@ -60,7 +60,7 @@ function visualMetrics(report,parent){const test=report&&report.test;if(!test)return;for(const [key,label] of [['learned_rgb_mse','Learned prediction error'],['persistence_rgb_mse','Last-frame baseline'],['train_mean_rgb_mse','Training-mean baseline']])if(typeof test[key]==='number'&&Number.isFinite(test[key]))metric(parent,label,Number(test[key].toPrecision(5)));} function showResult(summary){clearResult();$('result-content').hidden=false;$('result-empty').hidden=true;const metrics=$('result-metrics');metrics.replaceChildren();const counts=summary.counts||{};metric(metrics,'Held-out episodes',counts.test);metric(metrics,'Source groups',counts.groups);metric(metrics,'Elapsed seconds',typeof summary.elapsed_seconds==='number'?Math.round(summary.elapsed_seconds*100)/100:summary.elapsed_seconds);const cap=capability(summary);const readiness=$('readiness');readiness.replaceChildren();for(const statement of ['Visual representation: '+(cap.visual_trained?'trained':'not confirmed')+'.','Movement candidate: '+(cap.movement_candidate_trained?'trained; requires task qualification':'not trained')+'.','Acquired robot skill: no. Physical robot readiness: no.']){const p=document.createElement('p');text(p,statement);readiness.append(p);}const vm=$('visual-metrics');vm.replaceChildren();const dl=document.createElement('dl');dl.className='metrics';visualMetrics(summary.visual,dl);if(dl.children.length){const title=document.createElement('p');text(title,'Held-out next-frame prediction');const note=document.createElement('p');note.className='caption';text(note,'Normalized pixel mean squared error; lower is better. This measures visual prediction, not task completion.');vm.append(title,dl,note);}const list=$('limitations');list.replaceChildren();for(const message of summary.limitations||[]){const li=document.createElement('li');text(li,String(message));list.append(li);}text($('record'),JSON.stringify(summary,null,2));$('download').hidden=false;text($('train-hint'),'Completed locally. Inspect the held-out measurements below.');} function schedule(){clearTimeout(pollTimer);pollTimer=setTimeout(poll,800);} - async function poll(){try{const state=await request('/api/state');connected=true;$('online').dataset.allowed=state.allow_online?'true':'';text($('runtime'),state.allow_online?'CPU · local training\nHTTPS intake available':'CPU · offline server');text($('online-help'),state.allow_online?'HTTPS intake is optional. Leave unchecked to use local data without network access.':'This server is offline. Start with --allow-online only when HTTPS intake is needed.');busy=state.status==='running';const job=state.job;if(job){const elapsed=Number(job.elapsed_seconds)||0;text($('job-state'),state.status==='running'?((job.progress&&job.progress.message)||'Running locally…')+' '+Math.floor(elapsed)+' s elapsed.':state.status==='prepared'?'Footage check complete.':state.status==='complete'?'Training complete. The result has been verified.':'The job failed.');if(state.status==='prepared'&&lastJob!==job.id){showCheck(state.summary||{});checkedBinding=visualReady(state.summary||{})?binding():null;lastJob=job.id;text($('train-hint'),checkedBinding?'Ready for local visual training.':'Visual readiness was not established. Resolve the dataset check before training.');}else if(state.status==='complete'&&lastJob!==job.id){const result=await request('/api/result');showResult(result);lastJob=job.id;}else if(state.status==='failed'){checkedBinding=null;error(state.error||'The job failed. No result is available.');}}controls();if(busy)schedule();}catch(exc){connected=false;busy=false;checkedBinding=null;error(exc.message);controls();}} + async function poll(){try{const state=await request('/api/state');connected=true;$('online').dataset.allowed=state.allow_online?'true':'';text($('runtime'),state.allow_online?'CPU · local training\nHTTPS intake available':'CPU · offline server');text($('online-help'),state.allow_online?'HTTPS intake is optional. Leave unchecked to use local data without network access.':'This server is offline. Start with --allow-online only when HTTPS intake is needed.');busy=state.status==='running'||state.robot_job_running===true;const job=state.job;if(job){const elapsed=Number(job.elapsed_seconds)||0;text($('job-state'),state.status==='running'?((job.progress&&job.progress.message)||'Running locally…')+' '+Math.floor(elapsed)+' s elapsed.':state.status==='prepared'?'Footage check complete.':state.status==='complete'?'Training complete. The result has been verified.':'The job failed.');if(state.status==='prepared'&&lastJob!==job.id){showCheck(state.summary||{});checkedBinding=visualReady(state.summary||{})?binding():null;lastJob=job.id;text($('train-hint'),checkedBinding?'Ready for local visual training.':'Visual readiness was not established. Resolve the dataset check before training.');}else if(state.status==='complete'&&lastJob!==job.id){const result=await request('/api/result');showResult(result);lastJob=job.id;}else if(state.status==='failed'){checkedBinding=null;error(state.error||'The job failed. No result is available.');}}controls();if(busy)schedule();}catch(exc){connected=false;busy=false;checkedBinding=null;error(exc.message);controls();}} async function start(kind){$('error').hidden=true;clearResult();if(kind==='prepare')clearCheck();const seed=Number($('seed').value);if(!Number.isInteger(seed)||seed<0||seed>999999){error('Use an integer seed between 0 and 999999.');return;}busy=true;controls();text($('job-state'),kind==='prepare'?'Starting footage check…':'Starting local training…');try{await post('/api/'+kind,{source:$('source').value.trim(),online:$('online').checked,seed});await poll();}catch(exc){busy=false;checkedBinding=null;error(exc.message);controls();}} async function upload(file){if(!file)return;$('error').hidden=true;clearCheck();if(file.size>24*1024*1024){error('ZIP upload limit is 24 MiB. Enter the local folder path for larger footage.');return;}busy=true;controls();text($('job-state'),'Saving ZIP in the local workspace…');try{const value=await request('/api/upload',{method:'POST',headers:{'Content-Type':'application/zip'},body:file});$('source').value=value.source;$('online').checked=false;text($('job-state'),'ZIP saved locally. Check its footage before training.');}catch(exc){error(exc.message);}busy=false;$('upload').value='';controls();} $('prepare').addEventListener('click',()=>start('prepare'));$('train-button').addEventListener('click',()=>start('train'));for(const id of ['source','online','seed'])$(id).addEventListener('input',clearCheck);$('upload').addEventListener('change',event=>upload(event.target.files[0]));const drop=$('drop');for(const name of ['dragenter','dragover'])drop.addEventListener(name,event=>{event.preventDefault();if(!busy)drop.classList.add('dragging');});for(const name of ['dragleave','drop'])drop.addEventListener(name,event=>{event.preventDefault();drop.classList.remove('dragging');});drop.addEventListener('drop',event=>{if(!busy){const files=event.dataTransfer.files;if(files.length!==1){error('Drop one Skillspace ZIP.');return;}upload(files[0]);}});poll(); diff --git a/src/dvidia_training/training_studio.py b/src/dvidia_training/training_studio.py index 50721b3..4f06eb1 100644 --- a/src/dvidia_training/training_studio.py +++ b/src/dvidia_training/training_studio.py @@ -106,7 +106,8 @@ def validate_zip(raw): class TrainingStudioServer(ThreadingHTTPServer): daemon_threads = True - def __init__(self, address, directory, offline=True, *, runner=None, preparer=None, inspector=None): + def __init__(self, address, directory, offline=True, *, runner=None, preparer=None, inspector=None, + robot_worker=None, robot_checker=None): if address[0] != '127.0.0.1': raise ValueError('The training studio must bind to 127.0.0.1.') if type(offline) is not bool: @@ -119,8 +120,10 @@ def __init__(self, address, directory, offline=True, *, runner=None, preparer=No self.lock = threading.RLock() self.job = None self.prepared = None - self.pool = (ThreadPoolExecutor(max_workers=1) if runner is not None or preparer is not None + self.pool = (ThreadPoolExecutor(max_workers=1) if runner is not None or preparer is not None or robot_worker is not None else ProcessPoolExecutor(max_workers=1, mp_context=multiprocessing.get_context('spawn'))) + from .camera_action_studio import CameraActionStudio + self.robot = CameraActionStudio(self, worker=robot_worker, checker=robot_checker) try: super().__init__(address, TrainingStudioHandler) except Exception: @@ -204,6 +207,7 @@ def state(self): result = {'status': 'idle', 'allow_online': not self.offline, 'scope': 'visual learning; movement readiness is separate', 'robot_skill_acquired': False, 'physical_robot_ready': False, + 'robot_job_running': self.robot.busy(), 'job': None, 'summary': None, 'error': None} if self.job is not None: job = self.job @@ -228,7 +232,7 @@ def state(self): def submit(self, kind, source, online, seed): with self.lock: self.refresh() - if self.job is not None and self.job['status'] == 'running': + if (self.job is not None and self.job['status'] == 'running') or self.robot.busy(): raise RuntimeError('One CPU job is already running. Wait for its outcome.') offline = not online actual_source = source @@ -338,6 +342,10 @@ def do_GET(self): self.respond(Path(__file__).with_name('training_studio.html').read_bytes(), mime='text/html; charset=utf-8') elif self.path == '/coverage': self.respond(Path(__file__).with_name('coverage_studio.html').read_bytes(), mime='text/html; charset=utf-8') + elif self.path == '/robot': + self.respond(Path(__file__).with_name('camera_action_studio.html').read_bytes(), mime='text/html; charset=utf-8') + elif self.path == '/api/robot/state': + self.respond(self.server.robot.state()) elif self.path == '/api/coverage/example': from .coverage import example_plan self.respond(example_plan()) @@ -369,6 +377,11 @@ def do_POST(self): source, online, seed = self.train_request() kind = 'prepare' if self.path == '/api/prepare' else 'train' self.respond(self.server.submit(kind, source, online, seed), 202) + elif self.path in ('/api/robot/check', '/api/robot/run', '/api/robot/install'): + body = _json(self.body('application/json', JSON_LIMIT)) + if type(body) is not dict or set(body) != {'study', 'assets'}: + raise ValueError('Supply local prepared study and robot assets folders.') + self.respond(self.server.robot.submit(self.path.rsplit('/', 1)[-1], body['study'], body['assets']), 202) elif self.path == '/api/coverage/inspect': from .coverage import MAX_PLAN_BYTES, summarize_plan, validate_plan plan = validate_plan(self.body('application/json', MAX_PLAN_BYTES)) @@ -378,7 +391,7 @@ def do_POST(self): validate_zip(raw) with self.server.lock: self.server.refresh() - if self.server.job is not None and self.server.job['status'] == 'running': + if (self.server.job is not None and self.server.job['status'] == 'running') or self.server.robot.busy(): raise RuntimeError('One CPU job is already running. Wait before replacing its source.') self.server.clear_completed() root = self.server.directory / 'uploads' @@ -398,11 +411,12 @@ def do_POST(self): def configure(directory=Path('runs/training-studio'), *, port=8270, offline=True, - runner=None, preparer=None, inspector=None): + runner=None, preparer=None, inspector=None, robot_worker=None, robot_checker=None): if type(port) is not int or not 0 <= port <= 65535: raise ValueError('Use a valid local port.') return TrainingStudioServer(('127.0.0.1', port), directory, offline, - runner=runner, preparer=preparer, inspector=inspector) + runner=runner, preparer=preparer, inspector=inspector, + robot_worker=robot_worker, robot_checker=robot_checker) def run_server(directory=Path('runs/training-studio'), *, port=8270, offline=True): diff --git a/tests/test_camera_action_audit.py b/tests/test_camera_action_audit.py new file mode 100644 index 0000000..8909789 --- /dev/null +++ b/tests/test_camera_action_audit.py @@ -0,0 +1,155 @@ +from copy import deepcopy +import importlib.util +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from tests.test_camera_action_model import context + + +SCRIPT = Path(__file__).parents[1]/'benchmarks/camera-action-v01/independent_audit.py' +SPEC = importlib.util.spec_from_file_location('camera_action_independent_audit', SCRIPT) +audit = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(audit) + + +def put(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(audit.canonical(document)+b'\n') + + +def seal(root): + files = {str(p.relative_to(root)): audit.identity(p) for p in root.rglob('*') + if p.is_file() and p.name != 'integrity.json'} + put(root/'integrity.json', {'schema': 'dvidia.camera-action-rollout/v2/integrity', 'files': files}) + + +def episode(root, case): + root.mkdir(parents=True) + before = context()['snapshot']; after = context(.02)['snapshot'] + for name in ('rgb', 'depth'): + path = root/('frames/'+name); path.parent.mkdir(exist_ok=True) + path.write_bytes(b'fixture bytes '+name.encode()) + reference = {'path': str(path.relative_to(root)), **audit.identity(path)} + before['camera']['value'][name] = reference + after['camera']['value'][name] = deepcopy(reference) + calibration = {'fixture': True} + estimate = context()['estimate'] + action = context()['previous_applied'] + perception = dict(index=0, snapshot_sha256=audit.digest(before), frame=before['camera']['value'], + calibration_sha256=audit.digest(calibration), tracker_sha256='a'*64, estimate=estimate) + query = dict(index=0, snapshot_sha256=audit.digest(before), action=action, origin='privileged_simulator_teacher') + put(root/'perception/0000.json', perception); put(root/'teacher/0000.json', query) + step = dict(index=0, before=before, after=after, + command=dict(authority='teacher', operator_correction=False, requested=action, issued=action), + perception=dict(path='perception/0000.json', **audit.identity(root/'perception/0000.json')), + teacher_query=dict(path='teacher/0000.json', **audit.identity(root/'teacher/0000.json')), + applied={'ramped_targets': action}) + recording = dict(schema='dvidia.camera-action-rollout/v2', control_mode='teacher', initial=before, + provenance={'source_group': case['group_id'], 'case_id': case['id']}, + goal_position_m=case['goal_position_m'], calibration=calibration, + binding={'calibration_sha256': audit.digest(calibration)}, tracker_sha256='a'*64, steps=[step]) + transform = np.eye(4).reshape(-1).tolist() + put(root/'context.json', {'metadata': {'world_from_robot_base': transform, + 'world_from_robot_base_sha256': audit.digest(transform)}}) + put(root/'recording.json', recording) + put(root/'native_episode.json', {'trajectory': [{'object_position_m': [.3, 0., .08]}]}) + put(root/'native_metadata.json', {'rows': 20}) + arrays = dict(native_tick=np.arange(1, 21), t_seconds=np.arange(1, 21)*.001, + object_position_m=np.tile([.3, 0., .08], (20, 1)), + arm_actuator_controls_rad=np.tile(action['joint_targets_rad'], (20, 1)), + applied_jaw_target_m=np.full(20, .02)) + with (root/'native.npz').open('wb') as stream: + np.savez_compressed(stream, **arrays) + summary = dict(case=case, mode='teacher', negative_control=None, status='interrupted', + recorded_complete_intervals=1, strict_success=False, score={'success': False, 'reason': 'fixture'}, + teacher_queries=1, student_calls=0, camera_pairs=1, initial_anchor_error_m=0., + perception={'decisions': 1, 'available': 1, 'missing': 0, 'longest_missing_seconds': 0.}) + put(root/'rollout.json', summary) + seal(root) + return summary + + +class AuditTests(unittest.TestCase): + def case(self, index=0): + return {'id': f'dev-{index}', 'group_id': f'dev-{index}', 'goal_position_m': [.35, .08, .08]} + + def test_partial_failure_is_audited_without_becoming_success(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + with patch.object(audit.scoring, 'score_feedback_episode', return_value=summary['score']): + result = audit.audit_episode(root, case) + self.assertFalse(result['strict_success']) + self.assertEqual(result['complete_intervals'], 1) + self.assertEqual(result['native_ticks'], 20) + self.assertEqual(result['initial_anchor_error_m'], 0.) + self.assertEqual(result['available_position_error_p95_m'], 0.) + + def test_anchor_error_recomputed_and_cannot_be_claimed(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + summary['initial_anchor_error_m'] = .1 + put(root/'rollout.json', summary); seal(root) + with patch.object(audit.scoring, 'score_feedback_episode', return_value=summary['score']): + with self.assertRaisesRegex(ValueError, 'anchor error'): + audit.audit_episode(root, case) + + def test_changed_bytes_and_nonrigid_transform_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + (root/'frames/rgb').write_bytes(b'changed') + with self.assertRaisesRegex(ValueError, 'identity changed'): + audit.audit_episode(root, case) + seal(root) + context_doc = audit.read(root/'context.json') + context_doc['metadata']['world_from_robot_base'][0] = 2. + context_doc['metadata']['world_from_robot_base_sha256'] = audit.digest(context_doc['metadata']['world_from_robot_base']) + put(root/'context.json', context_doc); seal(root) + with self.assertRaisesRegex(ValueError, 'fixed-base transform'): + audit.audit_episode(root, case) + + def test_failed_setup_can_pass_consistency_audit(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); case = self.case() + put(root/'rollout.json', {'case': case, 'mode': 'teacher', 'negative_control': None, + 'status': 'refused', 'recorded_complete_intervals': 0, 'strict_success': False}) + seal(root) + result = audit.audit_episode(root, case) + self.assertEqual(result['native_ticks'], 0) + self.assertFalse(result['strict_success']) + + def test_four_attempt_denominator_and_source_bytes(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + sources = {'piper_feedback_score.py': audit.identity(Path(audit.scoring.__file__))['sha256']} + roster, rows = [], [] + for index in range(4): + case = self.case(index); attempt = {'id': f'diagnostic-{index}', 'case': case} + directory = root/'episodes/diagnostic'/attempt['id'] + episode(directory, {**case, 'policy_artifact_sha256': audit.digest(sources)}) + roster.append(attempt); rows.append({**attempt, 'strict_success': False}) + protocol = {'source_sha256': sources, 'rosters': {'diagnostic': roster}} + put(root/'protocol.json', protocol) + protocol_sha = audit.identity(root/'protocol.json')['sha256'] + put(root/'registration.json', {'source_sha256': sources, 'protocol_sha256': protocol_sha}) + put(root/'stages/diagnostic.json', {'rows': rows, 'model_bindings': {'teacher': audit.digest(sources)}}) + put(root/'stages/diagnostic.integrity.json', { + 'sha256': audit.identity(root/'stages/diagnostic.json')['sha256'], 'protocol_sha256': protocol_sha}) + with patch.object(audit.scoring, 'score_feedback_episode', return_value={'success': False, 'reason': 'fixture'}): + result = audit.audit(root) + self.assertTrue(result['passed'], result['errors']) + self.assertEqual(result['attempts_verified'], 4) + self.assertEqual(result['strict_successes'], 0) + self.assertFalse(result['physics_executed']) + rows.pop(); put(root/'stages/diagnostic.json', {'rows': rows, 'model_bindings': {'teacher': audit.digest(sources)}}) + put(root/'stages/diagnostic.integrity.json', { + 'sha256': audit.identity(root/'stages/diagnostic.json')['sha256'], 'protocol_sha256': protocol_sha}) + self.assertFalse(audit.audit(root)['passed']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_capture.py b/tests/test_camera_action_capture.py new file mode 100644 index 0000000..36ff819 --- /dev/null +++ b/tests/test_camera_action_capture.py @@ -0,0 +1,288 @@ +"""Data-only rollout tests. No renderer, native physics or hardware commands.""" +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import action_demonstration as tape +from dvidia_training import camera_action_capture as capture +from dvidia_training import piper_rgbd_tracker as tracker +from tests.test_action_demonstration import fixture + +CASE = dict(id='test-case', group_id='test-start', seed=4, + object_position_m=[.30, -.03, .082], goal_position_m=[.35, .08, .08]) +ACTION = dict(joint_targets_rad=[.01, .5, -.5, 0., 0., 0.], gripper_position_m=.024) +STUDENT = dict(joint_targets_rad=[.03, .5, -.5, 0., 0., 0.], gripper_position_m=.021) + + +class FakeSession: + instances = [] + fail_at = None + + def __init__(self, assets, config, case): + self.config, self.case = config, case + self.position = 0; self.teacher_calls = 0; self.closed = False + self.demo, self.assets = fixture(round(config.duration_s / .02)) + self.header = {key: deepcopy(self.demo[key]) for key in tape.HEADER} + self.initialized = False; self.last_camera = None + self.latest_rgbd = (np.zeros((240, 320, 3), dtype=np.uint8), np.ones((240, 320), dtype=np.float32)) + self.instances.append(self) + + def open(self): + if self.fail_at == 'open': + raise RuntimeError('Authored setup failure') + self.initialized = True + return self.header, deepcopy(self.demo['initial_applied_targets']), {'authored_test_double': True} + + def _snapshot(self): + snapshot = deepcopy(self.demo['initial'] if self.position == 0 else self.demo['steps'][self.position-1]['after']) + for name in tracker.SIGNALS: + snapshot['signals'][name] = capture.base._missing() + return snapshot + + def sample(self, index): + if self.fail_at == 'sample' and self.position == 4: + raise RuntimeError('Authored camera failure') + snapshot = self._snapshot() + camera = snapshot['camera']; old_index = camera['value']['index'] + frames = {} + for channel in ('rgb', 'depth'): + old_name = camera['value'][channel]['path'] + name = old_name.replace(f'{old_index:04}', f'{index:04}') + body = self.assets[old_name] + camera['value'][channel] = dict(path=name, bytes=len(body), sha256=sha256(body).hexdigest()) + frames[name] = body + camera['value']['index'] = snapshot['vision_frame_index'] = index + self.last_camera = deepcopy(camera) + return snapshot, frames, {'policy_observation': {'test': True}}, {'render_encode_wall_seconds': 0.} + + def measured(self, previous_camera): + snapshot = self._snapshot() + snapshot['camera'] = capture._sample_at(previous_camera, self.position*.02) + snapshot['vision_frame_index'] = previous_camera['value']['index'] + return snapshot, {}, {'policy_observation': {'test': True}}, {'render_encode_wall_seconds': 0.} + + def decide(self): + self.teacher_calls += 1 + return deepcopy(ACTION), {'phase': 'fake_teacher'} + + def advance(self, action): + before = self.position*.02 + self.position += 1 + if self.fail_at == 'advance' and self.position == 3: + raise RuntimeError('Authored partial interval failure') + ramped = deepcopy(action); ramped['joint_targets_rad'][0] += .001 + return dict(interval_start_seconds=before, interval_end_seconds=self.position*.02, + ramped_targets=ramped, native_position_controls=[*ramped['joint_targets_rad'],ramped['gripper_position_m']], + rate_limited=True, scope='final_native_tick_only') + + def status(self): + return dict(physics_attempted=False, native_initialized=False, + actual_native_elapsed_seconds=self.position*.02, native_info={'unexpected_contact_native_ticks': 0}, + authored_test_double=True) + + def native_evidence(self): + return {'test_only': np.zeros((self.position*20,))}, {'test_only': True} + + def anchor_error(self, estimate): + return .001 if estimate['status'] == 'available' else None + + def close(self): + self.closed = True + + +def fake_estimate(session, snapshot, calibration, negative=None): + seconds = snapshot['timestamp_seconds'] + status = 'missing' if negative in ('initial_missing', 'initial_stale', 'altered_calibration') else 'available' + estimate = tracker._empty(snapshot['vision_frame_index'], seconds, seconds, seconds) + estimate['status'] = status; estimate['age_seconds'] = 0. + if status == 'available': + values = dict(object_position_estimate_m=[.30, -.03, .08], + object_quaternion_estimate_wxyz=[1., 0., 0., 0.], object_position_uncertainty_m=[.001], + object_orientation_uncertainty_rad=[.01]) + for name, value in values.items(): + estimate[name] = value[0] if 'uncertainty' in name else value + estimate['signals'][name] = capture.base._sample(value, seconds) + return estimate, dict(estimator_wall_seconds=0., input_calibration_sha256=tape.digest(calibration), negative_control=negative) + + +class CameraCaptureTests(unittest.TestCase): + def setUp(self): + FakeSession.instances = []; FakeSession.fail_at = None + self.temp = TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + for target, value in (('_Session', FakeSession), ('_fresh_estimate', fake_estimate), + ('_native_episode', lambda *args: {'test_only': True}), + ('_score', lambda *args: dict(valid=False, success=False, reason='authored_fixture_not_physics'))): + self.enterContext(patch.object(capture, target, value)) + + def run_case(self, name='run', **kwargs): + root = Path(self.temp.name) / name + report = capture.run_rollout('unused', root, case=CASE, mode=kwargs.pop('mode', 'teacher'), + duration_s=kwargs.pop('duration_s', .2), **kwargs) + return root, report + + def test_25hz_camera_50hz_measurements_and_unique_frame_budget(self): + self.assertEqual(capture.RolloutConfig().duration_s, 20.) + self.assertEqual(20*25+1, 501) + root, report = self.run_case() + self.assertEqual(report['status'], 'complete'); self.assertEqual(report['camera_pairs'], 6) + recording = capture.load_rollout(root) + self.assertEqual(len(recording['steps']), 10) + for i, step in enumerate(recording['steps']): + self.assertAlmostEqual(step['before']['timestamp_seconds'], i*.02) + camera = step['before']['camera'] + self.assertEqual(camera['value']['index'], i//2) + self.assertAlmostEqual(camera['captured_at_seconds'], (i//2)*.04) + self.assertAlmostEqual(camera['age_seconds'], (i % 2)*.02) + for name in tracker.SIGNALS: + self.assertEqual(step['before']['signals'][name]['status'], 'missing') + self.assertEqual(len(list((root / 'frames').iterdir())), 12) + self.assertFalse(report['strict_success']) + + def test_queries_once_per_boundary_actual_mixed_authority_separate(self): + observed = [] + def student(context): + self.assertEqual(set(context), {'snapshot', 'estimate', 'previous_applied', 'goal_position_m'}) + observed.append(deepcopy(context)); return deepcopy(STUDENT) + root, report = self.run_case(mode='mixed', policy=student, duration_s=.4) + self.assertEqual(report['teacher_queries'], 20); self.assertEqual(report['student_calls'], 20) + self.assertEqual(FakeSession.instances[-1].teacher_calls, 20) + recording = capture.load_rollout(root) + authorities = {step['command']['authority'] for step in recording['steps']} + self.assertEqual(authorities, {'student', 'teacher'}) + for step in recording['steps']: + query = json.loads((root / step['teacher_query']['path']).read_text()) + self.assertEqual(query['action'], ACTION) + expected = ACTION if step['command']['authority'] == 'teacher' else STUDENT + self.assertEqual(step['command']['issued'], expected) + self.assertNotIn('reference', step) + self.assertEqual(observed[1]['previous_applied'], recording['steps'][0]['applied']['ramped_targets']) + other, _ = self.run_case('other', mode='mixed', policy=student, duration_s=.4) + self.assertEqual([s['command']['authority'] for s in recording['steps']], + [s['command']['authority'] for s in capture.load_rollout(other)['steps']]) + + def test_student_and_fixed_have_no_teacher_queries(self): + for mode in ('student', 'fixed'): + root, report = self.run_case(mode, mode=mode, policy=lambda context: deepcopy(STUDENT)) + self.assertEqual(report['teacher_queries'], 0) + self.assertFalse((root / 'teacher').exists()) + self.assertEqual(FakeSession.instances[-1].teacher_calls, 0) + self.assertTrue(all(s['teacher_query'] is None for s in capture.load_rollout(root)['steps'])) + + def test_initial_missing_refuses_without_any_native_action_or_teacher_query(self): + for control in ('initial_missing', 'initial_stale', 'altered_calibration'): + root, report = self.run_case(control, mode='mixed', policy=lambda _: deepcopy(STUDENT), negative_control=control) + self.assertEqual(report['status'], 'refused'); self.assertEqual(report['started_intervals'], 0) + self.assertEqual(report['teacher_queries'], 0); self.assertEqual(report['student_calls'], 0) + self.assertEqual(FakeSession.instances[-1].position, 0) + self.assertFalse(report['initial_anchor_available']) + self.assertTrue((root / 'perception/0000.json').exists()) + capture.inspect_rollout(root) + + def test_teacher_diagnostic_retains_missing_and_continues(self): + root, report = self.run_case(negative_control='initial_missing') + self.assertEqual(report['status'], 'complete'); self.assertFalse(report['initial_anchor_available']) + first = json.loads((root / 'perception/0000.json').read_text()) + second = json.loads((root / 'perception/0001.json').read_text()) + third = json.loads((root / 'perception/0002.json').read_text()) + self.assertEqual(first['estimate']['status'], 'missing') + self.assertEqual(second['negative_control'], 'initial_missing') + self.assertEqual(third['estimate']['status'], 'available') + + def test_open_gripper_records_requested_and_issued_truthfully(self): + root, report = self.run_case(mode='student', policy=lambda _: deepcopy(STUDENT), negative_control='open_gripper') + for step in capture.load_rollout(root)['steps']: + self.assertEqual(step['command']['requested']['gripper_position_m'], STUDENT['gripper_position_m']) + self.assertEqual(step['command']['issued']['gripper_position_m'], .035) + self.assertEqual(step['applied']['native_position_controls'][-1], .035) + self.assertEqual(step['command']['issue_reason'], 'open_gripper_negative_control') + + def test_interrupt_failure_and_uncommitted_interval_are_retained(self): + root, report = self.run_case(interrupt_after=3) + self.assertEqual(report['status'], 'interrupted'); self.assertEqual(report['recorded_complete_intervals'], 3) + self.assertEqual(report['uncommitted_native_intervals'], 0) + capture.inspect_rollout(root) + FakeSession.fail_at = 'advance' + root, report = self.run_case('failed') + self.assertEqual(report['status'], 'interrupted'); self.assertEqual(report['recorded_complete_intervals'], 2) + self.assertEqual(report['started_intervals'], 3); self.assertEqual(report['uncommitted_native_intervals'], 1) + self.assertTrue((root / 'decisions/0002.json').exists()) + self.assertFalse((root / 'transitions/0002.json').exists()) + capture.inspect_rollout(root) + + def test_camera_failure_retains_prior_prefix_and_unique_written_frames(self): + FakeSession.fail_at = 'sample' + root, report = self.run_case() + self.assertEqual(report['recorded_complete_intervals'], 3) + self.assertEqual(report['uncommitted_native_intervals'], 1) + self.assertEqual(report['camera_pairs'], 2) + self.assertTrue(FakeSession.instances[-1].closed) + capture.inspect_rollout(root) + + def test_setup_failure_is_sealed_and_output_cannot_be_reused(self): + FakeSession.fail_at = 'open' + root, report = self.run_case() + self.assertEqual(report['status'], 'refused') + capture.inspect_rollout(root) + with self.assertRaises(FileExistsError): + self.run_case() + + def test_tampered_sidecar_and_rebound_snapshot_are_rejected(self): + root, report = self.run_case() + path = root / 'perception/0000.json' + document = json.loads(path.read_text()); document['snapshot_sha256'] = '0'*64 + body = tape.canonical(document)+b'\n'; path.write_bytes(body) + with self.assertRaisesRegex(ValueError, 'byte identity'): + capture.inspect_rollout(root) + manifest = json.loads((root / 'integrity.json').read_text()) + identity = dict(bytes=len(body), sha256=sha256(body).hexdigest()) + manifest['files']['perception/0000.json'] = identity + (root / 'integrity.json').write_bytes(tape.canonical(manifest)+b'\n') + with self.assertRaisesRegex(ValueError, 'Sidecar binding'): + capture.inspect_rollout(root) + + def test_reused_estimate_has_historical_time_and_no_missing_forward_fill(self): + snap = dict(timestamp_seconds=0.,vision_frame_index=0) + estimate, _ = fake_estimate(None, snap, {}) + aged = capture._estimate_at(estimate, .02) + self.assertEqual(aged['captured_at_seconds'], 0.) + self.assertEqual(aged['age_seconds'], .02) + self.assertEqual(aged['signals']['object_position_estimate_m']['age_seconds'], .02) + stale = capture._estimate_at(estimate, .12) + self.assertEqual(stale['status'], 'stale') + self.assertIsNone(stale['object_position_estimate_m']) + self.assertIsNone(stale['signals']['object_position_estimate_m']['value']) + self.assertEqual(estimate['age_seconds'], 0.) + + def test_registered_configuration_rejects_silent_cadence_and_scope_changes(self): + for changes in ({'width': 160}, {'camera_every_controls': 1}, {'duration_s': .11}, + {'duration_s': True}, {'duration_s': float('nan')}, {'duration_s': 21.}, + {'seed': True}, {'interrupt_after': -1}, {'interrupt_after': 1000}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + capture.RolloutConfig(**changes) + with self.assertRaises(ValueError): + capture.run_rollout('unused',Path(self.temp.name)/'bad',case={**CASE,'object_position_m':[.3,-.03,.3]},mode='teacher') + + def test_sealed_capture_exports_actual_commands_and_separate_mixed_queries(self): + from dvidia_training.camera_action_export import export_episode + root, _ = self.run_case() + exported = export_episode(root) + self.assertEqual(exported['features'].shape, (10, 269)) + np.testing.assert_array_equal(exported['targets'], np.tile( + [*ACTION['joint_targets_rad'], ACTION['gripper_position_m']], (10, 1))) + root, _ = self.run_case('mixed-export', mode='mixed', policy=lambda _: deepcopy(STUDENT)) + exported = export_episode(root, label_source='teacher_query') + np.testing.assert_array_equal(exported['targets'], np.tile( + [*ACTION['joint_targets_rad'], ACTION['gripper_position_m']], (10, 1))) + with self.assertRaisesRegex(ValueError, 'requires teacher control'): + export_episode(root, label_source='issued') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_export.py b/tests/test_camera_action_export.py new file mode 100644 index 0000000..f55d761 --- /dev/null +++ b/tests/test_camera_action_export.py @@ -0,0 +1,145 @@ +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import camera_action_export as export +from dvidia_training import camera_action_model as m +from tests.test_camera_action_model import context, missing_visual + + +def fixture(root, *, mode='teacher', group='train-01', anchor_missing=False): + root.mkdir(exist_ok=True) + recording = dict(control_mode=mode, goal_position_m=[.35, .08, .08], + initial_applied_targets=context()['previous_applied'], + provenance={'source_group': group, 'case_id': group+'-goal0'}, + binding={'calibration_sha256': 'a'*64}, tracker_sha256='b'*64, steps=[]) + for index in range(3): + c = context(index*.02) + if index == 0 and anchor_missing: + c = missing_visual(c) + elif index == 1: + c = missing_visual(c) + before = c['snapshot'] + def save(relative, document): + path = root/relative + path.parent.mkdir(exist_ok=True) + body = m.canonical(document)+b'\n'; path.write_bytes(body) + return dict(path=relative, bytes=len(body), sha256=sha256(body).hexdigest()) + perception = save(f'perception/{index:04}.json', {'estimate': c['estimate']}) + query_action = dict(joint_targets_rad=[.8, 1.4, -1.3, 0., 0., 0.], gripper_position_m=.01) + issued_action = dict(joint_targets_rad=[.2, 1.4, -1.3, 0., 0., 0.], gripper_position_m=.03) + query = save(f'teacher/{index:04}.json', {'action': query_action, 'origin': 'privileged_simulator_teacher'}) + step = dict(index=index, before=before, perception=perception, teacher_query=query, + command={'issued': issued_action, 'authority': 'teacher' if mode == 'teacher' else 'student'}, + applied={'ramped_targets': c['previous_applied']}, + reference={'action': query_action}, native_truth={'object_position': [9, 9, 9]}) + recording['steps'].append(step) + (root/'recording.json').write_bytes(m.canonical(recording)+b'\n') + (root/'integrity.json').write_text('{"fixture":true}\n') + return recording + + +class ExportTests(unittest.TestCase): + def test_export_uses_issued_labels_and_explicit_missing_masks(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + dataset = export.export_episode(folder) + self.assertEqual(dataset['features'].shape, (3, m.FEATURE_COUNT)) + self.assertEqual(dataset['targets'][:, 0].tolist(), [.2]*3) + self.assertEqual(dataset['groups'], ['train-01']*3) + i = m.FEATURE_NAMES.index('lag0:visual_available') + self.assertEqual(dataset['features'][:, i].tolist(), [1., 0., 1.]) + self.assertEqual(dataset['provenance']['episodes'][0]['label_source'], 'issued') + + def test_queries_are_labels_not_claimed_execution(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder), mode='mixed') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + dataset = export.export_episode(folder, label_source='teacher_query') + with self.assertRaisesRegex(ValueError, 'teacher control'): + export.export_episode(folder) + self.assertEqual(dataset['targets'][:, 0].tolist(), [.8]*3) + self.assertEqual(recording['steps'][0]['command']['issued']['joint_targets_rad'][0], .2) + self.assertEqual(recording['steps'][0]['command']['authority'], 'student') + self.assertIn('separate teacher query', dataset['provenance']['label_definition']) + + def test_full_blind_same_source_identity_and_different_feature_masks(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + full = export.export_episode(folder) + blind = export.export_episode(folder, blind=True) + self.assertEqual(full['provenance']['source_identity_sha256'], blind['provenance']['source_identity_sha256']) + np.testing.assert_array_equal(full['features'][:, :m.LIVE_FEATURE_START], blind['features'][:, :m.LIVE_FEATURE_START]) + np.testing.assert_array_equal(full['targets'], blind['targets']) + self.assertTrue(np.all(blind['features'][:, m.LIVE_FEATURE_START:] == 0.)) + masked = export.blind_dataset(full) + np.testing.assert_array_equal(masked['features'], blind['features']) + self.assertEqual(masked['provenance'], blind['provenance']) + self.assertTrue(np.any(full['features'][:, m.LIVE_FEATURE_START:] != 0.)) + with self.assertRaisesRegex(ValueError, 'full-feedback'): + export.blind_dataset(blind) + + def test_current_labels_truth_reference_and_future_do_not_change_inputs(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + recording = fixture(root) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + before = export.export_episode(root) + changed = deepcopy(recording) + for row in changed['steps']: + row['command']['issued']['joint_targets_rad'][0] = .5 + row['native_truth'] = {'pose': [100]*9} + row['reference'] = {'action': [99]*7} + row['after'] = {'timestamp_seconds': 20., 'object': [99]*3} + (root/'recording.json').write_bytes(m.canonical(changed)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=changed): + after = export.export_episode(root) + np.testing.assert_array_equal(before['features'], after['features']) + self.assertFalse(np.array_equal(before['targets'], after['targets'])) + + def test_tampered_sidecar_or_changed_recording_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); recording = fixture(root) + (root/'perception/0001.json').write_text('{}') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'sidecar bytes'): + export.export_episode(root) + (root/'recording.json').write_text('{}') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'changed after'): + export.export_episode(root) + + def test_initial_missing_prevents_learning_pair_export(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder), anchor_missing=True) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'initial visual anchor'): + export.export_episode(folder) + + def test_combine_rejects_duplicates_and_mixed_feature_contracts(self): + with tempfile.TemporaryDirectory() as folder: + a, b = Path(folder)/'a', Path(folder)/'b' + ra, rb = fixture(a, group='train-a'), fixture(b, group='train-b') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=ra): + left = export.export_episode(a) + blind = export.export_episode(a, blind=True) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=rb): + right = export.export_episode(b) + result = export.combine_datasets([left, right]) + self.assertEqual(result['groups'], ['train-a']*3+['train-b']*3) + self.assertEqual(len(result['provenance']['episodes']), 2) + with self.assertRaisesRegex(ValueError, 'duplicated'): + export.combine_datasets([left, left]) + with self.assertRaisesRegex(ValueError, 'feedback masking'): + export.combine_datasets([left, blind]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_model.py b/tests/test_camera_action_model.py new file mode 100644 index 0000000..84a14da --- /dev/null +++ b/tests/test_camera_action_model.py @@ -0,0 +1,239 @@ +from copy import deepcopy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import camera_action_model as m + + +def sample(value, now, captured=None): + captured = now if captured is None else captured + return dict(status='available', captured_at_seconds=captured, + available_at_seconds=captured, age_seconds=now-captured, value=value) + + +def context(now=0., x=.3, y=0.): + signals = {} + values = ([.2, 1.4, -1.3, 0., 0., 0.], [0.]*6, [.02, -.02], [0., 0.], + [.3, 0., .2], [1., 0., 0., 0.]) + for (name, _, _), value in zip(m.SENSOR_SPECS, values): + signals[name] = sample(value, now) + captured = round(int(round(now/.02))//2*.04, 8) + position = [x, y, .08] + estimate = dict(schema=m.tracker.SCHEMA, status='available', decision_at_seconds=now, + frame_index=int(round(captured/.04)), + captured_at_seconds=captured, available_at_seconds=captured, age_seconds=now-captured, + object_position_estimate_m=position, object_position_uncertainty_m=.002, + signals={'object_position_estimate_m': sample(position, now, captured), + 'object_position_uncertainty_m': sample([.002], now, captured)}) + return dict(snapshot={'timestamp_seconds': now, 'signals': signals, + 'camera': sample({'index': int(round(captured/.04))}, now, captured), + 'vision_frame_index': int(round(captured/.04))}, estimate=estimate, + previous_applied={'joint_targets_rad': values[0], 'gripper_position_m': .02}, + goal_position_m=[.35, .08, .08]) + + +def missing_visual(value): + value = deepcopy(value) + value['estimate']['status'] = 'missing' + for key in ('object_position_estimate_m', 'object_position_uncertainty_m'): + value['estimate'][key] = None + value['estimate']['signals'][key] = dict(status='missing', captured_at_seconds=None, + available_at_seconds=None, age_seconds=None, value=None) + return value + + +def dataset(blind=False, prefix='train'): + rows, targets, groups = [], [], [] + for group in range(4): + encoder = m.CameraActionFeatures(blind=blind) + for index in range(16): + c = context(index*.02, x=.28+group*.015, y=.01*group) + c['snapshot']['signals']['joint_position_rad']['value'][0] = .1+index*.01 + rows.append(encoder(c)) + targets.append([.1+group*.03+index*.001, 1.42, -1.28, 0., 0., 0., .015+group*.001]) + groups.append(prefix+'-'+str(group)) + return dict(schema=m.DATASET_SCHEMA, features=np.asarray(rows), targets=np.asarray(targets), + groups=groups, provenance={'test_fixture': True, 'origin': prefix}) + + +class FeatureTests(unittest.TestCase): + def test_dimensions_and_source_whitelist(self): + clean = context() + poisoned = deepcopy(clean) + poisoned.update(reference={'action': {'jaw': 999}}, teacher_phase='release', + current_issued=[999]*7, after={'object_position': [9]*3}, native_truth={'contact': 999}) + poisoned['snapshot'].update(reference={'action': [99]*7}, future={'time': .02}) + poisoned['snapshot']['signals']['object_position_estimate_m'] = sample([8, 8, 8], 0.) + poisoned['estimate'].update(object_quaternion_estimate_wxyz=[0, 0, 0, 0], + diagnostics={'native_truth': [99]*3}) + a = m.CameraActionFeatures()(clean) + b = m.CameraActionFeatures()(poisoned) + self.assertEqual(m.FEATURE_COUNT, 269) + self.assertEqual(a.shape, (269,)) + np.testing.assert_array_equal(a, b) + self.assertFalse(any('reference' in name or 'object_quaternion' in name for name in m.FEATURE_NAMES)) + + def test_missing_live_estimate_stays_missing_and_anchor_remains_historical(self): + encoder = m.CameraActionFeatures() + first = encoder(context()) + live = encoder(missing_visual(context(.02, x=.8))) + np.testing.assert_array_equal(first[:5], live[:5]) + index = m.FEATURE_NAMES.index('lag0:visual_available') + self.assertEqual(first[index], 1.) + self.assertEqual(live[index], 0.) + self.assertEqual(live[m.FEATURE_NAMES.index('lag0:visual_x_m/.5')], 0.) + self.assertAlmostEqual(float(live[m.FEATURE_NAMES.index('lag0:visual_age_s/.1')]), .2) + self.assertEqual(live[m.FEATURE_NAMES.index('lag40:history_present')], 0.) + + def test_camera_age_is_original_and_no_future_history(self): + encoder = m.CameraActionFeatures() + encoder(context()) + result = encoder(context(.02)) + self.assertAlmostEqual(float(result[m.FEATURE_NAMES.index('lag0:visual_age_s/.1')]), .2) + encoder(context(.04)); encoder(context(.06)) + result = encoder(context(.08)) + # Duplicate names refer to command and sensor lag presence; both are causal. + i = m.LIVE_FEATURE_START+len(m.SENSOR_NAMES)*2 + self.assertEqual(result[i+len(m.SENSOR_NAMES)-2], 1.) + self.assertAlmostEqual(float(result[i+len(m.SENSOR_NAMES)-1]), .5) + + def test_initial_missing_or_future_or_stale_refused(self): + with self.assertRaisesRegex(ValueError, 'initial visual anchor'): + m.CameraActionFeatures()(missing_visual(context())) + bad = context() + for key in ('object_position_estimate_m', 'object_position_uncertainty_m'): + bad['estimate']['signals'][key]['captured_at_seconds'] = .04 + bad['estimate']['signals'][key]['available_at_seconds'] = .04 + with self.assertRaisesRegex(ValueError, 'future'): + m.CameraActionFeatures()(bad) + bad = context(); bad['estimate']['decision_at_seconds'] = .02 + with self.assertRaisesRegex(ValueError, 'decision time'): + m.CameraActionFeatures()(bad) + + def test_blind_live_values_masks_and_ages_cannot_change_features(self): + a, b = m.CameraActionFeatures(blind=True), m.CameraActionFeatures(blind=True) + np.testing.assert_array_equal(a(context()), b(context())) + altered = missing_visual(context(.02)) + altered['snapshot']['signals'] = {'untrusted_native_truth': [100]*9} + left, right = a(context(.02)), b(altered) + np.testing.assert_array_equal(left, right) + self.assertTrue(np.all(left[m.LIVE_FEATURE_START:] == 0.)) + altered = context(.04) + altered['previous_applied']['joint_targets_rad'][0] = .3 + self.assertFalse(np.array_equal(a(context(.04)), b(altered))) + + def test_bounds_goal_and_timestamp_failures_do_not_advance_history(self): + encoder = m.CameraActionFeatures(); encoder(context()) + bad = context(.02); bad['previous_applied']['joint_targets_rad'][1] = -1. + with self.assertRaisesRegex(ValueError, 'native joint bounds'): + encoder(bad) + bad = context(.02); bad['goal_position_m'][0] = .4 + with self.assertRaisesRegex(ValueError, 'destination'): + encoder(bad) + with self.assertRaisesRegex(ValueError, '20 ms'): + encoder(context(.04)) + encoder(context(.02)) + self.assertAlmostEqual(encoder.last_time, .02) + + def test_unavailable_nonnull_and_inconsistent_receipts_refused(self): + bad = context(); bad['estimate']['signals']['object_position_estimate_m']['status'] = 'missing' + with self.assertRaisesRegex(ValueError, 'forward-fill'): + m.CameraActionFeatures()(bad) + bad = context(); bad['estimate']['object_position_estimate_m'] = [.8, 0., .08] + with self.assertRaisesRegex(ValueError, 'disagrees'): + m.CameraActionFeatures()(bad) + + def test_camera_frame_binding_and_uncertainty_capture_cannot_be_rebound(self): + bad = context(); bad['estimate']['frame_index'] = 99 + with self.assertRaisesRegex(ValueError, 'frame index'): + m.CameraActionFeatures()(bad) + encoder = m.CameraActionFeatures(); encoder(context()) + bad = context(.02) + bad['estimate']['signals']['object_position_uncertainty_m']['available_at_seconds'] = .01 + with self.assertRaisesRegex(ValueError, 'signal times'): + encoder(bad) + bad = context(.02); bad['estimate']['captured_at_seconds'] = .01 + with self.assertRaisesRegex(ValueError, 'source timestamps'): + encoder(bad) + + +class ModelTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.data = dataset() + cls.model = m.train(cls.data, validation_dataset=dataset(prefix='development')) + + def test_learning_absolute_targets_and_recipe(self): + model = self.model + self.assertEqual(model['hidden_units'], [64, 64]) + self.assertEqual(model['action_representation'], 'absolute_native_targets') + self.assertLess(model['training']['final_loss'], model['training']['initial_loss']/10.) + self.assertLess(model['training']['validation']['normalized_mse'], .1) + validation = model['training']['validation'] + expected = float(np.mean((np.asarray(validation['native_rmse'])/m.ACTION_SCALE_NATIVE)**2)) + self.assertAlmostEqual(validation['action_scale_mse'], expected, places=12) + with self.assertRaisesRegex(ValueError, 'registered recipe'): + m.train(self.data, epochs=51) + + def test_training_only_normalization_and_split_isolation(self): + altered = dataset(prefix='different-development') + altered['features'] = np.clip(altered['features']+2., -10., 10.) + alternative = m.train(self.data, validation_dataset=altered) + for key in ('weights', 'biases', 'feature_mean', 'feature_scale', 'action_mean', 'action_scale'): + self.assertEqual(self.model[key], alternative[key]) + with self.assertRaisesRegex(ValueError, 'disjoint'): + m.train(self.data, validation_dataset=self.data) + + def test_policy_has_no_teacher_and_does_not_read_truth(self): + clean, poisoned = context(), context() + poisoned['reference'] = {'action': [100]*7} + poisoned['current_teacher_query'] = {'phase': 'release'} + with patch('dvidia_training.piper_task.PiperTaskPolicy', side_effect=AssertionError('No teacher')): + a, b = m.ModelPolicy(self.model), m.ModelPolicy(self.model) + self.assertEqual(a(clean), b(poisoned)) + self.assertEqual(a.diagnostics()['teacher_fallback_calls'], 0) + self.assertEqual(a.diagnostics()['student_inference_calls'], 1) + + def test_strict_json_roundtrip_and_provenance(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder)/'model.json' + saved = m.save_model(self.model, path) + self.assertEqual(saved, m.digest(m.load_model(path))) + self.assertEqual(m.ModelPolicy(self.model)(context()), m.ModelPolicy(m.load_model(path))(context())) + with self.assertRaises(FileExistsError): + m.save_model(self.model, path) + for key, value in (('schema', 'arbitrary'), ('physical_robot_ready', True), ('jaw_limit_m', .2)): + bad = deepcopy(self.model); bad[key] = value + with self.assertRaises(ValueError): + m.validate_model(bad) + bad = deepcopy(self.model); bad['training']['provenance']['test_fixture'] = False + with self.assertRaisesRegex(ValueError, 'provenance hash'): + m.validate_model(bad) + with self.assertRaisesRegex(ValueError, 'Duplicate'): + m.validate_model('{"schema":"x","schema":"x"}') + + def test_native_action_bounds_and_blind_mask_training(self): + extreme = deepcopy(self.model) + extreme['weights'] = [np.zeros_like(w).tolist() for w in extreme['weights']] + extreme['biases'][-1] = [64., -64., 64., -64., 64., -64., 64.] + action = m.ModelPolicy(extreme)(context()) + m.action_vector(action) + blind = m.train(dataset(blind=True), blind=True) + self.assertTrue(blind['blind']) + with self.assertRaisesRegex(ValueError, 'zero all'): + m.train(self.data, blind=True) + + def test_dataset_hash_and_fixed_training_are_reproducible(self): + again = m.train(self.data, validation_dataset=dataset(prefix='development')) + self.assertEqual(self.model, again) + changed = deepcopy(self.data); changed['targets'][0, 0] += .01 + self.assertNotEqual(self.model['training']['dataset_sha256'], m.train(changed)['training']['dataset_sha256']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_skill.py b/tests/test_camera_action_skill.py new file mode 100644 index 0000000..a586e64 --- /dev/null +++ b/tests/test_camera_action_skill.py @@ -0,0 +1,102 @@ +"""Data-only camera policy installation cannot grant success or change arms.""" +from copy import deepcopy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from dvidia_training import arm_skill_install as dispatch +from dvidia_training import camera_action_model as model +from dvidia_training import camera_action_skill as skill + + +class CameraSkillTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + data = dict(schema=model.DATASET_SCHEMA, features=[[0.]*model.FEATURE_COUNT]*16, + targets=[[0.]*6+[.02]]*16, groups=['training-fixture']*16, + provenance={'kind': 'unit-test-only'}) + cls.document = model.train(data) + cls.raw = model.canonical(cls.document)+b'\n' + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root/'model.json' + self.source.write_bytes(self.raw) + self.output = self.root/'installed' + self.binding = {'test_fixture': True} + self.native = patch.object(skill.arm_interfaces, 'build', return_value=(None, None, self.binding)).start() + self.addCleanup(patch.stopall) + + def test_dispatch_checks_camera_schema_without_motion_or_qualification(self): + result = dispatch.check_model(self.source, 'piper') + self.assertEqual(result['runner'], skill.RUNNER) + self.assertFalse(result['task_qualified_by_installation']) + self.assertFalse(result['physical_robot_ready']) + self.native.assert_not_called() + + def test_other_arms_interfaces_and_invalid_model_fail_before_motion(self): + for robot, interface in [('so101', skill.INTERFACE), ('ur5e-2f85', skill.INTERFACE), ('piper', 'ros2')]: + with self.assertRaises(ValueError): + dispatch.install_model(self.source, robot, 'unused', self.output, interface=interface) + self.native.assert_not_called() + bad = deepcopy(self.document); bad['physical_robot_ready'] = True + self.source.write_bytes(model.canonical(bad)) + with self.assertRaises(ValueError): + dispatch.check_model(self.source, 'piper') + + def test_exact_copy_inspection_and_fresh_install(self): + result = dispatch.install_model(self.source, 'piper', 'unused', self.output) + self.assertEqual((self.output/'model.json').read_bytes(), self.raw) + self.assertEqual(dispatch.inspect_installation(self.output, 'unused'), result) + self.assertEqual(result['status'], 'installed_simulation_candidate') + with self.assertRaises(ValueError): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + + def test_tampered_receipt_model_and_runtime_are_rejected(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + path = self.output/'installation.json'; original = path.read_bytes() + receipt = json.loads(original) + for key, value in [('status', 'qualified'), ('runtime_sha256', {}), ('native_binding', {})]: + bad = deepcopy(receipt); bad[key] = value + path.write_bytes(skill.canonical(bad)) + with self.assertRaises(ValueError): + dispatch.inspect_installation(self.output, 'unused') + path.write_bytes(original) + with patch.object(skill, 'runtime', return_value={}): + with self.assertRaisesRegex(ValueError, 'Runtime|runtime'): + dispatch.inspect_installation(self.output, 'unused') + (self.output/'model.json').write_bytes(self.raw+b' ') + with self.assertRaises(ValueError): + dispatch.inspect_installation(self.output, 'unused') + + def test_run_dispatch_preserves_failure_and_native_rollout_inventory(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + out = self.root/'attempt' + def run(assets, output, **kwargs): + self.assertEqual(kwargs['mode'], 'student') + self.assertNotIn('teacher', kwargs) + self.assertEqual(kwargs['case']['object_position_m'], [.30, -.03, .082]) + output.mkdir() + (output/'integrity.json').write_text('{}') + return {'strict_success': False, 'status': 'complete', 'reason': 'failed_task'} + with patch('dvidia_training.camera_action_capture.run_rollout', side_effect=run): + result = dispatch.run_installation(self.output, 'unused', out) + self.assertFalse(result['success']) + self.assertEqual([p.name for p in out.iterdir()], ['integrity.json']) + self.assertTrue((self.root/'attempt-installation.json').is_file()) + + def test_out_of_envelope_refuses_before_execution(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + with patch('dvidia_training.camera_action_capture.run_rollout') as runner: + for kwargs in ({'start_x': .284}, {'goal': True}, {'start_y': float('nan')}): + with self.assertRaises(ValueError): + dispatch.run_installation(self.output, 'unused', self.root/'attempt', **kwargs) + runner.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_studio.py b/tests/test_camera_action_studio.py new file mode 100644 index 0000000..2f53206 --- /dev/null +++ b/tests/test_camera_action_studio.py @@ -0,0 +1,201 @@ +"""Real loopback API tests with data-only injected workers; no native physics.""" +from hashlib import sha256 +import http.client +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from dvidia_training import camera_action_studio as robot +from dvidia_training.training_studio import configure +from tests.test_training_studio import artifact_run, artifact_inspect, prepare_data + + +def fake_check(study, assets): + root = Path(study) + state = (root/'state.txt').read_text() + return dict(schema=robot.SCHEMA, + binding=dict(study=str(root.resolve()), assets=str(Path(assets).resolve()), + protocol_sha256=sha256((root/'protocol.json').read_bytes()).hexdigest(), assets_sha256='a'*64), + inspection=dict(status=state, next_actions=[] if state!='ready' else [dict(command='collect',stage='diagnostic',requires_assets=True)], + stage_summaries={}, report=dict(qualified=False, qualification='research_candidate'), + blocked_stages=['diagnostic'] if state=='blocked' else []), + install_available=state=='complete', selected_model=str(root/'model.json') if state=='complete' else None, + physical_robot_ready=False) + + +def fake_worker(kind, study, assets, output, expected_binding=None, installation=None): + result = fake_check(study, assets) + if expected_binding is not None and result['binding'] != expected_binding: + raise ValueError('Inputs changed') + if kind == 'run': + (Path(study)/'state.txt').write_text('complete') + result = fake_check(study, assets) + status = 'checked' if kind=='check' else 'complete' if kind=='run' else 'installed' + if result['inspection']['status']=='blocked': status='blocked' + result['status'] = status + if kind=='install': + destination = Path(installation); destination.mkdir(parents=True) + (destination/'test-only.txt').write_text('No real model installation') + result['installation'] = dict(path=str(destination),record={'test_only':True}) + return result + + +class CameraStudioHTTPTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory(); self.root=Path(self.temp.name) + self.study=self.root/'study'; self.study.mkdir(); (self.study/'protocol.json').write_text('{"test":true}') + (self.study/'state.txt').write_text('ready') + self.assets=self.root/'assets'; self.assets.mkdir() + self.events=[]; self.servers=[] + self.server=self.make_server() + + def make_server(self, worker=fake_worker, runner=artifact_run): + server=configure(self.root/('server'+str(len(self.servers))),port=0, + runner=runner,preparer=prepare_data,inspector=artifact_inspect, + robot_worker=worker,robot_checker=fake_check) + thread=threading.Thread(target=server.serve_forever,kwargs={'poll_interval':.01},daemon=True) + thread.start();self.servers.append((server,thread));return server + + def tearDown(self): + for event in self.events:event.set() + for server,thread in self.servers: + server.shutdown();server.server_close();thread.join(timeout=2) + self.temp.cleanup() + + def request(self,path,body=None,*,method='POST',headers=None,server=None): + server=server or self.server;port=server.server_address[1] + connection=http.client.HTTPConnection('127.0.0.1',port,timeout=3) + h={'Content-Type':'application/json','Origin':f'http://127.0.0.1:{port}'};h.update(headers or {}) + connection.request(method,path,body=None if body is None else json.dumps(body),headers=h) + response=connection.getresponse();raw=response.read();code=response.status;mime=response.getheader('Content-Type','');connection.close() + return code,json.loads(raw) if mime.startswith('application/json') else raw + + def payload(self):return dict(study=str(self.study),assets=str(self.assets)) + + def done(self,server=None): + for _ in range(200): + status,state=self.request('/api/robot/state',method='GET',server=server) + self.assertEqual(status,200) + if state['status']!='running':return state + time.sleep(.01) + self.fail('Injected worker did not finish') + + def checked(self,server=None): + status,body=self.request('/api/robot/check',self.payload(),server=server) + self.assertEqual(status,202,body);return self.done(server) + + def test_simple_ui_and_empty_state_do_not_claim_robot_readiness(self): + code,html=self.request('/robot',method='GET') + self.assertEqual(code,200);self.assertIn(b'Teach an arm, then test it.',html) + self.assertIn(b'id="install" disabled',html);self.assertIn(b'Plain videos alone',html) + self.assertNotIn(b'innerHTML',html) + code,footage=self.request('/',method='GET');self.assertIn(b'href="/robot"',footage) + state=self.done();self.assertFalse(state['can_install']);self.assertFalse(state['can_run']) + self.assertFalse(state['physical_robot_ready']) + + def test_existing_host_origin_and_cross_site_boundary_applies_to_robot_routes(self): + for route,method,body in (('/robot','GET',None),('/api/robot/state','GET',None),('/api/robot/check','POST',self.payload())): + for headers in ({'Host':'evil.example'},{'Origin':'https://evil.example'},{'Sec-Fetch-Site':'cross-site'}): + code,_=self.request(route,body,method=method,headers=headers) + self.assertEqual(code,400) + self.assertIsNone(self.server.robot.job) + + def test_check_run_install_uses_same_worker_and_installs_outside_study(self): + state=self.checked();self.assertTrue(state['checked']);self.assertTrue(state['can_run']);self.assertFalse(state['can_install']) + code,_=self.request('/api/robot/run',self.payload());self.assertEqual(code,202) + state=self.done();self.assertEqual(state['status'],'complete');self.assertTrue(state['can_install']) + code,_=self.request('/api/robot/install',self.payload());self.assertEqual(code,202) + state=self.done();self.assertEqual(state['status'],'installed') + installation=Path(state['summary']['installation']['path']) + self.assertTrue(installation.is_relative_to(self.server.directory/'installations')) + self.assertFalse(installation.is_relative_to(self.study));self.assertTrue(installation.is_dir()) + + def test_train_before_check_and_plain_video_input_are_refused(self): + self.assertEqual(self.request('/api/robot/run',self.payload())[0],400) + video=self.root/'video.mp4';video.write_bytes(b'not a study') + self.assertEqual(self.request('/api/robot/check',{**self.payload(),'study':str(video)})[0],400) + self.assertEqual(self.request('/api/robot/check',{**self.payload(),'study':'https://example.com/video'})[0],400) + + def test_one_shared_job_rejects_footage_during_robot_and_robot_during_footage(self): + event=threading.Event();self.events.append(event) + def slow_robot(*args):event.wait(3);return fake_worker(*args) + server=self.make_server(worker=slow_robot) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],202) + self.assertEqual(self.request('/api/train',{'source':'/fake'},server=server)[0],409) + code,state=self.request('/api/state',method='GET',server=server) + self.assertTrue(state['robot_job_running']) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],409) + event.set();self.done(server) + event2=threading.Event();self.events.append(event2) + def slow_footage(*args,**kwargs):event2.wait(3);return artifact_run(*args,**kwargs) + server=self.make_server(runner=slow_footage) + self.assertEqual(self.request('/api/train',{'source':'/fake'},server=server)[0],202) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],409) + self.assertTrue(self.request('/api/robot/state',method='GET',server=server)[1]['busy']) + event2.set() + + def test_blocker_clears_stale_install_capability_and_never_retries(self): + (self.study/'state.txt').write_text('complete') + self.assertTrue(self.checked()['can_install']) + (self.study/'state.txt').write_text('blocked') + state=self.checked();self.assertEqual(state['status'],'blocked') + self.assertFalse(state['checked']);self.assertFalse(state['can_install']);self.assertFalse(state['can_run']) + self.assertEqual(self.request('/api/robot/install',self.payload())[0],400) + + def test_protocol_change_invalidates_check_and_result(self): + (self.study/'state.txt').write_text('complete') + self.assertTrue(self.checked()['can_install']) + (self.study/'protocol.json').write_text('{"test":"changed"}') + state=self.done();self.assertEqual(state['status'],'failed');self.assertFalse(state['can_install']) + self.assertIsNone(state['summary']);self.assertIn('protocol changed',state['error']) + self.assertEqual(self.request('/api/robot/install',self.payload())[0],400) + + def test_worker_error_preserves_error_without_stale_success(self): + def fail(*args):raise ValueError('Fake renderer unavailable') + server=self.make_server(worker=fail) + state=self.checked(server) + self.assertEqual(state['status'],'failed');self.assertIn('renderer unavailable',state['error']) + self.assertFalse(state['can_install']);self.assertIsNone(state['summary']) + + +class CameraWorkerTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.root=Path(self.temp.name);self.output=self.root/'job';self.output.mkdir() + self.binding=dict(study='test-study',assets='test-assets',protocol_sha256='a'*64,assets_sha256='b'*64) + + def value(self,status,action=None): + return dict(binding=self.binding,inspection=dict(status=status,next_actions=[] if action is None else [action]),install_available=False) + + def test_worker_dispatches_registered_terminal_stages_without_shell(self): + from dvidia_training import camera_action_study as study + checks=[self.value('ready',dict(command='collect',stage='diagnostic',requires_assets=True)), + self.value('ready',dict(command='train',stage='initial',requires_assets=False)), + self.value('ready',dict(command='evaluate',stage='development-initial',requires_assets=True)),self.value('complete')] + with patch.object(robot,'check_study',side_effect=checks),patch.object(study,'collect') as collect,patch.object(study,'train') as train,patch.object(study,'evaluate') as evaluate: + result=robot.run_worker('run','test-study','test-assets',self.output,self.binding) + collect.assert_called_once_with('test-study','test-assets',stage='diagnostic') + train.assert_called_once_with('test-study',stage='initial') + evaluate.assert_called_once_with('test-study','test-assets',stage='development-initial') + self.assertEqual(result['status'],'complete');self.assertEqual(len(result['executed_stages']),3) + self.assertTrue((self.output/'robot-receipt.json').is_file()) + + def test_worker_stops_at_blocker_without_retry(self): + from dvidia_training import camera_action_study as study + with patch.object(robot,'check_study',side_effect=[self.value('ready',dict(command='collect',stage='diagnostic',requires_assets=True)),self.value('blocked')]),patch.object(study,'collect') as collect: + result=robot.run_worker('run','test-study','test-assets',self.output,self.binding) + self.assertEqual(result['status'],'blocked');collect.assert_called_once() + + def test_worker_rejects_changed_binding_before_stage_and_preserves_failure(self): + with patch.object(robot,'check_study',return_value=self.value('ready')): + with self.assertRaisesRegex(ValueError,'changed'): + robot.run_worker('run','test-study','test-assets',self.output,{**self.binding,'protocol_sha256':'c'*64}) + receipt=json.loads((self.output/'robot-receipt.json').read_text()) + self.assertEqual(receipt['status'],'failed');self.assertEqual(receipt['executed_stages'],[]) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_camera_action_study.py b/tests/test_camera_action_study.py new file mode 100644 index 0000000..75379bb --- /dev/null +++ b/tests/test_camera_action_study.py @@ -0,0 +1,324 @@ +"""Pure orchestration checks; these tests never initialize native physics.""" +from copy import deepcopy +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from dvidia_training import camera_action_study as study + + +class StudyTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name) + self.root = self.base / 'study' + self.fixed = self.base / 'fixed.json' + study.write(self.fixed, {'fixture': 'fixed comparator'}) + self.runtime = patch.object(study, 'runtime_hashes', return_value={'fixture': '0' * 64}) + self.runtime.start() + self.addCleanup(self.runtime.stop) + with patch('dvidia_training.piper_motor_primitive.validate_model', side_effect=lambda value: value): + study.prepare(self.root, fixed_model=self.fixed) + + def protocol(self): + return study._protocol(self.root) + + def row(self, attempt, *, success=True, anchor=True, contacts=0): + return dict(**deepcopy(attempt), status='complete', complete=True, + strict_success=success, initial_anchor_available=anchor, + initial_anchor_binding_valid=anchor, initial_anchor_error_m=.002, critical_contact_ticks=contacts, + provenance_ok=attempt['controller'] != 'teacher', recorded_complete_intervals=1000, + score=dict(valid=True, success=success, strict_v01=dict(trajectory_complete=True))) + + def receipt(self, stage, *, success=True): + return dict(schema=study.SCHEMA + '/stage', stage=stage, status='complete', + protocol_sha256=study.file_hash(self.root / 'protocol.json'), + rows=[self.row(attempt, success=success) for attempt in self.protocol()['rosters'][stage]]) + + def test_roster_counts_and_complete_split_isolation(self): + p = self.protocol() + self.assertEqual({name: len(rows) for name, rows in p['cases'].items()}, + dict(train=64, development=16, confirmation=64)) + self.assertEqual({name: len(rows) for name, rows in p['rosters'].items()}, + dict(diagnostic=4, demonstrations=80, corrections=32, + **{'development-initial': 32, 'development-corrected': 32}, confirmation=256, negative=16)) + groups, points = {}, {} + for split, cases in p['cases'].items(): + for case in cases: + point = tuple(case['object_position_m'][:2]) + self.assertTrue(.287 <= point[0] <= .311 and -.042 <= point[1] <= -.018) + self.assertNotIn(point, study._historical_starts()) + self.assertEqual(groups.setdefault(case['group_id'], split), split) + self.assertEqual(points.setdefault(point, split), split) + self.assertEqual(len(points), 72) + self.assertEqual(len(groups), 72) + correction = p['rosters']['corrections'] + self.assertEqual(len({a['case']['group_id'] for a in correction}), 32) + self.assertTrue(all(a['case']['split'] == 'train' for a in correction)) + self.assertEqual([a['case']['goal_id'] for a in correction], [i % 2 for i in range(32)]) + self.assertEqual(p['cases'], study.registered_cases()) + + def test_registration_is_exclusive_and_detects_runtime_or_protocol_changes(self): + with self.assertRaises(ValueError): + study.prepare(self.root, fixed_model=self.fixed) + with patch.object(study, 'runtime_hashes', return_value={'fixture': '1' * 64}): + with self.assertRaisesRegex(ValueError, 'Runtime changed'): + study._protocol(self.root, runtime=True) + protocol = self.root / 'protocol.json' + protocol.write_bytes(protocol.read_bytes() + b' ') + with self.assertRaisesRegex(ValueError, 'protocol bytes changed'): + study.inspect(self.root) + + def test_group_and_per_goal_quality_gates_are_both_required(self): + rows = self.receipt('demonstrations')['rows'] + self.assertTrue(study.data_quality(rows, self.protocol())['passed']) + # Sixty-two successes are insufficient if a whole starting group is absent. + rows[0]['strict_success'] = rows[1]['strict_success'] = False + result = study.data_quality(rows, self.protocol()) + self.assertFalse(result['passed']) + self.assertEqual(result['missing_training_groups'], ['train-00']) + rows = self.receipt('demonstrations')['rows'] + for row in rows[:10:2]: + row['strict_success'] = False + result = study.data_quality(rows, self.protocol()) + self.assertEqual(result['covered_training_groups'], 32) + self.assertEqual(result['successful_episodes_per_goal']['0'], 27) + self.assertFalse(result['passed']) + + def test_missing_initial_anchor_is_not_usable_training_data(self): + rows = self.receipt('demonstrations')['rows'] + rows[0]['initial_anchor_available'] = False + rows[1]['initial_anchor_binding_valid'] = False + self.assertFalse(study.data_quality(rows, self.protocol())['passed']) + + def test_diagnostic_rejects_available_but_inaccurate_or_missing_camera_error(self): + receipt = self.receipt('diagnostic') + receipt['rows'][0]['initial_anchor_error_m'] = .010001 + with patch.object(study, '_run_stage', return_value=receipt): + result = study.collect(self.root, '/unused/assets') + self.assertEqual(result['status'], 'blocked') + self.assertIn('10 mm', result['reason']) + + def test_diagnostic_requires_finite_native_error_measurement(self): + receipt = self.receipt('diagnostic') + receipt['rows'][0]['initial_anchor_error_m'] = None + with patch.object(study, '_run_stage', return_value=receipt): + result = study.collect(self.root, '/unused/assets') + self.assertEqual(result['status'], 'blocked') + + def test_blocked_diagnostic_stops_training_and_exposes_ui_reason(self): + receipt = self.receipt('diagnostic') + receipt['rows'][0]['strict_success'] = False + with patch.object(study, '_run_stage', return_value=receipt): + result = study.collect(self.root, '/unused/assets') + self.assertEqual(result['status'], 'blocked') + state = study.inspect(self.root) + self.assertEqual(state['status'], 'blocked') + self.assertEqual(state['next_actions'], []) + self.assertFalse(state['qualified']) + with self.assertRaisesRegex(ValueError, 'diagnostic must be complete'): + study.collect(self.root, '/unused/assets', stage='demonstrations') + with self.assertRaisesRegex(ValueError, 'demonstrations must be complete'): + study.train(self.root) + + def test_inspection_is_read_only_and_has_one_action(self): + before = {str(p.relative_to(self.root)): study.file_hash(p) + for p in self.root.rglob('*') if p.is_file()} + state = study.inspect(self.root) + self.assertEqual(state['next_actions'], [dict(command='collect', stage='diagnostic', requires_assets=True)]) + self.assertFalse(state['physics_executed']) + after = {str(p.relative_to(self.root)): study.file_hash(p) + for p in self.root.rglob('*') if p.is_file()} + self.assertEqual(before, after) + + def test_stage_receipt_tamper_is_detected(self): + study._seal_stage(self.root, 'diagnostic', self.receipt('diagnostic')) + target = self.root / 'stages/diagnostic.json' + target.write_bytes(target.read_bytes() + b' ') + with self.assertRaisesRegex(ValueError, 'receipt identity changed'): + study.inspect(self.root) + + def test_disk_reserve_and_study_budget_refuse_before_next_attempt(self): + with patch.object(study.shutil, 'disk_usage', return_value=SimpleNamespace(free=study.DISK_RESERVE_BYTES)): + with self.assertRaisesRegex(ValueError, '30 GiB'): + study.storage_preflight(self.root) + + def test_second_worker_is_refused_and_never_clears_existing_lock(self): + lock = self.root / 'worker.lock' + study.write(lock, {'pid': 123, 'operation': 'another worker'}) + with self.assertRaisesRegex(ValueError, 'active or interrupted worker'): + study.collect(self.root, '/unused/assets') + self.assertTrue(lock.exists()) + with patch.object(study, 'STUDY_MAX_BYTES', 1): + with self.assertRaisesRegex(ValueError, '120 GiB'): + study.storage_preflight(self.root) + + def test_prior_unsealed_attempt_is_unknown_and_never_retried(self): + p = self.protocol() + for attempt in p['rosters']['diagnostic']: + directory = self.root / 'episodes/diagnostic' / attempt['id'] + directory.mkdir(parents=True) + study.write(directory / 'started.json', {'status': 'interrupted before receipt'}) + with patch('dvidia_training.camera_action_capture.run_rollout') as run: + result = study.collect(self.root, '/unused/assets') + run.assert_not_called() + self.assertEqual(result['status'], 'blocked') + self.assertTrue(all(r['status'] == 'unknown' for r in result['rows'])) + + def test_paired_bootstrap_keeps_both_goals_in_one_group(self): + rows = self.receipt('confirmation')['rows'] + for row in rows: + row['strict_success'] = row['controller'] == 'student' + interval = study.paired_bootstrap(rows, 'fixed') + self.assertEqual(interval['lower'], 1.) + self.assertEqual(interval['upper'], 1.) + self.assertTrue(interval['positive']) + for row in rows: + row['strict_success'] = row['case']['goal_id'] == 0 + tied = study.paired_bootstrap(rows, 'fixed') + self.assertEqual(tied['lower'], 0.) + self.assertFalse(tied['positive']) + with self.assertRaisesRegex(ValueError, '32 complete'): + study.paired_bootstrap(rows[4:], 'fixed') + + def test_qualification_requires_improvement_not_just_58_successes(self): + confirmation = self.receipt('confirmation') + negatives = self.receipt('negative', success=False) + for row in negatives['rows']: + if row['negative_control'] != 'open_gripper': + row.update(status='refused', complete=False, recorded_complete_intervals=0) + study.write(self.root / 'freeze.json', dict(schema=study.SCHEMA + '/freeze', selected_stage='initial', + source_sha256=self.protocol()['source_sha256'], protocol_sha256=study.file_hash(self.root/'protocol.json'), + fixed_model_sha256=study.file_hash(self.root/'fixed-model.json'), artifacts={})) + for receipt in (confirmation, negatives): + receipt['freeze_sha256'] = study.file_hash(self.root / 'freeze.json') + study._seal_stage(self.root, receipt['stage'], receipt) + report = study.report_study(self.root) + self.assertEqual(report['controllers']['student']['strict_successes'], 64) + self.assertTrue(report['gates']['at_least_58_of_64']) + self.assertFalse(report['gates']['positive_paired_comparisons']) + self.assertFalse(report['qualified']) + + def test_negative_controls_must_actually_refuse_or_run_and_fail(self): + a = self.protocol()['rosters']['negative'] + missing = self.row(a[0], success=False) + missing.update(status='refused', complete=False, recorded_complete_intervals=0) + self.assertTrue(study._negative_correct(missing)) + missing['recorded_complete_intervals'] = 1 + self.assertFalse(study._negative_correct(missing)) + jaw = self.row(a[3], success=False) + self.assertTrue(study._negative_correct(jaw)) + jaw['score']['valid'] = False + self.assertFalse(study._negative_correct(jaw)) + jaw['score']['valid'] = True + jaw['score']['strict_v01']['trajectory_complete'] = False + self.assertFalse(study._negative_correct(jaw)) + jaw['score']['strict_v01']['trajectory_complete'] = True + jaw.update(status='refused', complete=False, recorded_complete_intervals=0) + self.assertFalse(study._negative_correct(jaw)) + + def test_development_loss_selection_never_falls_back_to_training_loss(self): + self.assertEqual(study._loss({'training': {'validation': {'action_scale_mse': .2}, 'final_loss': .001}}), .2) + with self.assertRaisesRegex(ValueError, 'development prediction loss'): + study._loss({'training': {'final_loss': .001}}) + + def test_rollout_identity_cannot_be_swapped_between_groups(self): + attempt = self.protocol()['rosters']['diagnostic'][0] + summary = dict(case=deepcopy(attempt['case']), mode='teacher', negative_control=None, + status='complete', recorded_complete_intervals=1000, score={'success': True}) + summary['case']['id'] = 'other-scene' + with self.assertRaisesRegex(ValueError, 'registered case'): + study._row(self.root, attempt, summary, self.root / 'nonexistent') + + def test_package_does_not_claim_an_unrun_skill_is_qualified(self): + destination = self.base / 'package' + result = study.package(self.root, destination) + self.assertFalse(result['qualified']) + manifest = study.read(destination / 'manifest.json') + self.assertEqual(manifest['qualification'], 'research_candidate') + self.assertFalse(manifest['model_runtime_installation_included']) + with self.assertRaises(ValueError): + study.package(self.root, destination) + + def test_registered_selection_and_confirmation_lifecycle_without_native_execution(self): + def fake_run(root, assets, stage, model_stage=None): + receipt = self.receipt(stage) + receipt['model_bindings'] = study._model_bindings(root, stage, model_stage) + for row in receipt['rows']: + if stage.startswith('development-'): + row['strict_success'] = stage.endswith('corrected') or row['case']['goal_id'] == 0 + elif stage == 'confirmation': + row['strict_success'] = (row['controller'] in ('student', 'teacher') + or int(row['case']['group_id'].split('-')[-1]) < 12) + elif stage == 'negative': + row['strict_success'] = False + if row['negative_control'] != 'open_gripper': + row.update(status='refused', complete=False, recorded_complete_intervals=0) + return receipt + + def fake_export(root, rows, blind, label_source): + return dict(features=[], targets=[], groups=[], provenance={'blind': blind, + 'source_identity_sha256': 'same'}), [r['id'] for r in rows] + + def fake_fit(dataset, **kwargs): + self.assertEqual(kwargs['epochs'], 50) + self.assertEqual(kwargs['batch_size'], 512) + self.assertEqual(kwargs['learning_rate'], .003) + self.assertEqual(kwargs['seed'], 17) + self.assertIsNotNone(kwargs['validation_dataset']) + return dict(training={'validation': {'action_scale_mse': .1}, 'final_loss': .001}) + + with patch.object(study, '_run_stage', side_effect=fake_run), \ + patch.object(study, '_dataset', side_effect=fake_export), \ + patch('dvidia_training.camera_action_export.blind_dataset', side_effect=lambda value: value), \ + patch('dvidia_training.camera_action_export.combine_datasets', side_effect=lambda values: values[0]), \ + patch('dvidia_training.camera_action_model.train', side_effect=fake_fit): + study.collect(self.root, '/unused/assets') + study.collect(self.root, '/unused/assets', stage='demonstrations') + study.train(self.root) + study.evaluate(self.root, '/unused/assets') + study.collect(self.root, '/unused/assets', stage='corrections') + study.train(self.root, stage='corrected') + study.evaluate(self.root, '/unused/assets', stage='development-corrected') + study.evaluate(self.root, '/unused/assets', stage='confirmation') + frozen = study.read(self.root / 'freeze.json') + self.assertEqual(frozen['selected_stage'], 'corrected') + study.evaluate(self.root, '/unused/assets', stage='negative') + report = study.read(self.root / 'report.json') + self.assertTrue(report['qualified']) + self.assertTrue(all(report['gates'].values())) + self.assertEqual(study.inspect(self.root)['next_actions'], []) + self.assertEqual(len(study._stage(self.root, 'confirmation')['rows']), 256) + model_path = study._model_path(self.root, 'corrected', 'student') + model_path.write_bytes(model_path.read_bytes() + b' ') + with self.assertRaisesRegex(ValueError, 'Frozen artifact changed'): + study.report_study(self.root) + + def test_refusal_alone_cannot_win_common_admitted_comparison(self): + rows = self.receipt('confirmation')['rows'] + for row in rows: + if row['controller'] == 'fixed': + row.update(status='refused', complete=False, strict_success=False, + recorded_complete_intervals=0) + self.assertTrue(study.paired_bootstrap(rows, 'fixed')['positive']) + admitted = study.paired_bootstrap(rows, 'fixed', admitted_only=True) + self.assertEqual(admitted['groups'], 0) + self.assertIsNone(admitted['lower']) + self.assertFalse(admitted['positive']) + + def test_policy_load_rejects_model_replaced_after_training(self): + path = study._model_path(self.root, 'initial', 'student') + study.write(path, {'fixture': 'trained'}) + receipt = dict(status='complete', models={'full': {'model_sha256': study.file_hash(path)}}) + study._seal_stage(self.root, 'train-initial', receipt) + self.assertEqual(study._model_document(self.root, 'initial', 'student'), {'fixture': 'trained'}) + path.write_bytes(b'{"fixture":"replacement"}\n') + with self.assertRaisesRegex(ValueError, 'sealed training receipt'): + study._model_document(self.root, 'initial', 'student') + + +if __name__ == '__main__': + unittest.main() From 6fb15fb4246c0b3f43bc30424841b7f6462a615b Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sun, 1 Nov 2026 02:46:58 -0500 Subject: [PATCH 02/10] Add independent camera-action audits and reviewed local delivery tools --- README.md | 10 + .../diagnostic_phase_coverage.py | 122 +++ benchmarks/camera-action-v01/study_audit.py | 703 ++++++++++++++++++ .../camera-action-v01/summarize_results.py | 503 +++++++++++++ docs/camera-action-v01-report-draft.md | 155 ++++ docs/camera-action-v01.md | 7 +- publishing/CAMERA-ACTION-V01.md | 99 +++ .../huggingface/camera_action_hardware.py | 32 + .../huggingface/publish_camera_action_v01.py | 270 +++++++ .../test_camera_action_hardware.py | 33 + .../test_publish_camera_action_v01.py | 156 ++++ publishing/prepare_camera_action_v01.py | 520 +++++++++++++ publishing/test_prepare_camera_action_v01.py | 232 ++++++ src/dvidia_training/camera_action_skill.py | 7 +- src/dvidia_training/camera_action_studio.html | 4 +- src/dvidia_training/camera_action_studio.py | 40 +- tests/test_camera_action_skill.py | 12 +- tests/test_camera_action_studio.py | 20 + tests/test_camera_action_study_audit.py | 285 +++++++ tests/test_camera_action_summary.py | 218 ++++++ 20 files changed, 3420 insertions(+), 8 deletions(-) create mode 100644 benchmarks/camera-action-v01/diagnostic_phase_coverage.py create mode 100644 benchmarks/camera-action-v01/study_audit.py create mode 100644 benchmarks/camera-action-v01/summarize_results.py create mode 100644 docs/camera-action-v01-report-draft.md create mode 100644 publishing/CAMERA-ACTION-V01.md create mode 100644 publishing/huggingface/camera_action_hardware.py create mode 100644 publishing/huggingface/publish_camera_action_v01.py create mode 100644 publishing/huggingface/test_camera_action_hardware.py create mode 100644 publishing/huggingface/test_publish_camera_action_v01.py create mode 100644 publishing/prepare_camera_action_v01.py create mode 100644 publishing/test_prepare_camera_action_v01.py create mode 100644 tests/test_camera_action_study_audit.py create mode 100644 tests/test_camera_action_summary.py diff --git a/README.md b/README.md index aa7694a..53c87af 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,16 @@ [Recorded demo](https://huggingface.co/spaces/Dvidia/dvidia-training) · [Synthetic datasets](https://huggingface.co/datasets/Dvidia/dvidia-training-examples) · [Pilot models](https://huggingface.co/Dvidia/dvidia-training-pilot) +**[Camera-to-action skill v0.1](docs/camera-action-v01.md).** +The **0.1.3.dev4 preview** adds a local demonstration-to-action learning loop: +25 Hz RGB-D, 50 Hz measured joints and demonstrated targets, a compact CPU +policy, one bounded correction round, and a fixed unseen-start benchmark. +Use `dvidia-train robot` in Terminal or the Training Studio's `/robot` page. +The first task is simulated PiPER block placement with an explicitly synthetic +teacher. Missing object estimates remain masked. Each installation remains a +simulation candidate and sends no physical commands. +[Registered experiment and diagnostic evidence](docs/camera-action-v01-report-draft.md). + **[Arm interfaces and installation v0.1](docs/arm-interfaces-v01.md).** The **0.1.3.dev3 preview** adds `dvidia-arm`: three explicit native simulation adapters, exact PiPER policy compatibility checks, separate install/run commands, diff --git a/benchmarks/camera-action-v01/diagnostic_phase_coverage.py b/benchmarks/camera-action-v01/diagnostic_phase_coverage.py new file mode 100644 index 0000000..15eeb92 --- /dev/null +++ b/benchmarks/camera-action-v01/diagnostic_phase_coverage.py @@ -0,0 +1,122 @@ +"""Read-only phase coverage characterization of the four audited diagnostics. + +Teacher phase is used only for this offline analysis. It is never exported as +a learner input. Motor decisions reuse a 25 Hz camera at 50 Hz, so decision +counts are correlated observations, not independent tracking trials. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from hashlib import sha256 +import json +from pathlib import Path + + +MANIPULATION_PHASES = ('close', 'lift', 'transfer', 'place', 'release') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128*1024**2: + raise ValueError('Expected an ordinary bounded JSON artifact.') + return json.loads(path.read_bytes()) + + +def hash_file(path): + return sha256(Path(path).read_bytes()).hexdigest() + + +def sidecar(root, reference, inventory): + name = reference['path']; relative = Path(name) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError('Invalid source path.') + path = root/relative + if path.is_symlink(): + raise ValueError('Source cannot be a symlink.') + raw = path.read_bytes() + identity = {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + if (inventory.get(name) != identity + or {key: reference[key] for key in ('bytes', 'sha256')} != identity): + raise ValueError('Teacher/perception sidecar identity changed.') + return json.loads(raw) + + +def analyze(study, audited): + root = Path(study) + evidence = read(audited) + if (evidence.get('schema') != 'dvidia.camera-action-diagnostic-audit/v1' + or evidence.get('passed') is not True or evidence.get('attempts_verified') != 4): + raise ValueError('A passing four-diagnostic independent audit is required.') + stage = read(root/'stages/diagnostic.json') + protocol = read(root/'protocol.json') + registration = read(root/'registration.json') + if hash_file(root/'protocol.json') != registration['protocol_sha256']: + raise ValueError('Registered protocol changed.') + expected = {row['id']: row for row in protocol['rosters']['diagnostic']} + if len(stage['rows']) != 4 or {row['id'] for row in stage['rows']} != set(expected): + raise ValueError('Expected the exact four diagnostic rows.') + audited_cases = {row['case_id'] for row in evidence['results']} + if audited_cases != {row['case']['id'] for row in stage['rows']}: + raise ValueError('Audit and diagnostic case identities differ.') + total = defaultdict(lambda: {'decisions': 0, 'available': 0}) + results = [] + for row in stage['rows']: + relative = Path(row['path']) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError('Episode path is not study-relative.') + episode = root/relative + if (hash_file(episode/'integrity.json') != row['integrity_sha256'] + or hash_file(episode/'rollout.json') != row['receipt_sha256']): + raise ValueError('Episode binding changed after stage sealing.') + inventory = read(episode/'integrity.json')['files'] + raw = (episode/'recording.json').read_bytes() + if inventory['recording.json'] != {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()}: + raise ValueError('Recording bytes differ from the episode seal.') + recording = json.loads(raw) + counts = defaultdict(lambda: {'decisions': 0, 'available': 0}) + for step in recording['steps']: + query = sidecar(episode, step['teacher_query'], inventory) + perception = sidecar(episode, step['perception'], inventory) + if (query['snapshot_sha256'] != digest(step['before']) + or perception['snapshot_sha256'] != digest(step['before']) + or query['index'] != step['index'] or perception['index'] != step['index']): + raise ValueError('Phase and perception are not aligned to the same decision.') + phase = query.get('phase') + if type(phase) is not str or len(phase) > 80: + raise ValueError('Expected a bounded offline teacher phase label.') + available = int(perception['estimate']['status'] == 'available') + counts[phase]['decisions'] += 1; counts[phase]['available'] += available + total[phase]['decisions'] += 1; total[phase]['available'] += available + results.append(dict(case_id=row['case']['id'], source_group=row['case']['group_id'], + phases=dict(counts), integrity_sha256=row['integrity_sha256'])) + contact_count = sum(total[phase]['decisions'] for phase in MANIPULATION_PHASES) + contact_available = sum(total[phase]['available'] for phase in MANIPULATION_PHASES) + return dict(schema='dvidia.camera-action-phase-coverage/v1', + protocol_sha256=hash_file(root/'protocol.json'), diagnostic_audit_sha256=hash_file(audited), + analysis_source_sha256=hash_file(__file__), cases=4, starting_position_groups=2, + total_motor_decisions=sum(v['decisions'] for v in total.values()), phases=dict(total), + manipulation_phases=list(MANIPULATION_PHASES), manipulation_decisions=contact_count, + manipulation_available=contact_available, results=results, + evidence_scope='Offline teacher phase analysis only; phases are excluded from learner inputs. Repeated 50 Hz decisions share 25 Hz camera frames.', + physics_executed=False, estimator_executed=False, physical_robot_ready=False) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study'); parser.add_argument('--audit', required=True) + args = parser.parse_args(argv) + print(json.dumps(analyze(args.study, args.audit), indent=2, allow_nan=False)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/study_audit.py b/benchmarks/camera-action-v01/study_audit.py new file mode 100644 index 0000000..5f9670d --- /dev/null +++ b/benchmarks/camera-action-v01/study_audit.py @@ -0,0 +1,703 @@ +"""Independent saved-evidence audit across the camera/action study lifecycle. + +Imports numeric scoring and the diagnostic auditor's byte readers only. No +environment, runner, controller, tracker or study-report function is imported. +The audit does not initialize physics, infer poses or execute learned actions. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import math +from pathlib import Path +import random + +import numpy as np + +from independent_audit import (canonical, digest, read, identity, relative, require, + inventory, _reference, _close, _transform, scoring) + +SCHEMA = 'dvidia.camera-action-study-audit/v1' +STAGES = ('diagnostic', 'demonstrations', 'train-initial', 'development-initial', + 'corrections', 'train-corrected', 'development-corrected', 'confirmation', 'negative') +VISION = ('object_position_estimate_m', 'object_quaternion_estimate_wxyz', + 'object_position_uncertainty_m', 'object_orientation_uncertainty_rad') +ENCODERS = ('joint_position_rad', 'joint_velocity_rad_s', 'gripper_joint_position_m', + 'gripper_joint_velocity_m_s', 'tcp_position_estimate_m', 'tcp_quaternion_estimate_wxyz') + + +def _action(value, limits): + require(type(value) is dict and set(value) == {'joint_targets_rad', 'gripper_position_m'}, 'Unknown or missing action fields.') + q = np.asarray(value['joint_targets_rad'], dtype=float) + bounds = np.asarray(limits['joint_limits_rad'], dtype=float) + require(q.shape == (6,) and bounds.shape == (6, 2) and np.isfinite(q).all() + and np.all(q >= bounds[:, 0]) and np.all(q <= bounds[:, 1]), 'Action joint targets violate native bounds.') + jaw = value['gripper_position_m']; low, high = limits['gripper_position_limits_m'] + require(type(jaw) in (int, float) and math.isfinite(jaw) and low <= jaw <= high, 'Action gripper target violates native bounds.') + + +def command_check(step, query, mode, negative, rng, limits): + command = step['command'] + require(command.get('operator_correction') is False, 'Synthetic control cannot claim operator correction.') + for name in ('requested', 'issued'): + _action(command[name], limits) + expected_authority = ('teacher' if mode == 'teacher' else + ('teacher' if rng.random() < .5 else 'student') if mode == 'mixed' else mode) + require(command['authority'] == expected_authority, 'Command authority differs from registered controller or seeded mixture.') + expected_issued = deepcopy(command['requested']) + if negative == 'open_gripper': + expected_issued['gripper_position_m'] = .035 + require(command['issued'] == expected_issued, 'Issued command differs from its explicitly registered intervention.') + require(command['issue_reason'] == ('open_gripper_negative_control' if expected_issued != command['requested'] else 'unchanged'), + 'Command intervention reason is inconsistent.') + if mode in ('teacher', 'mixed'): + require(type(query) is dict and query.get('origin') == 'privileged_simulator_teacher', 'Missing or mislabeled teacher query.') + _action(query['action'], limits) + if expected_authority == 'teacher': + require(query['action'] == command['requested'], 'Executed teacher request differs from its separately sealed query.') + else: + require(query is None, 'A student or fixed rollout contains teacher intervention.') + + +def _native(root, files, case, summary): + required = {'native_episode.json', 'native_metadata.json', 'native.npz'} + if not required <= set(files): + require(not required.intersection(files) and summary.get('recorded_complete_intervals') == 0, + 'Partial native evidence cannot justify completed intervals.') + require(summary['score'].get('valid') is False and summary['score'].get('success') is False, + 'No-native-evidence attempt claims a valid native score.') + return None, None, None, dict(valid=False, success=False, reason='native_evidence_unavailable') + episode = read(root / 'native_episode.json'); metadata = read(root / 'native_metadata.json') + with np.load(root / 'native.npz', allow_pickle=False) as handle: + arrays = {key: handle[key] for key in handle.files} + require(set(arrays) == {name for name, width in scoring.FIELDS}, 'Unknown or missing native arrays.') + count = metadata.get('rows') + require(type(count) is int and 0 <= count <= 20000, 'Native trace exceeds the registered time budget.') + for name, width in scoring.FIELDS: + values = arrays[name] + require(values.dtype.kind in 'iuf' and values.shape == ((count,) if width == 1 else (count, width)) + and np.isfinite(values).all(), 'Native array shape or finite values differ: ' + name) + require(np.array_equal(arrays['native_tick'], np.arange(1, count + 1)), 'Native tick ordering differs.') + require(np.allclose(arrays['t_seconds'], np.arange(1, count + 1) * .001, atol=1e-8, rtol=0), 'Native 1 ms cadence differs.') + require(episode['config']['object_position_m'] == case['object_position_m'] + and episode['config']['target_position_m'] == case['goal_position_m'], 'Native scene differs from registered start or goal.') + score = scoring.score_feedback_episode(episode, arrays, metadata, case['goal_position_m'][:2]) + require(score == summary['score'], 'Saved native score differs from independent numeric reconstruction.') + ticks = int(np.count_nonzero(arrays['unexpected_contact_count'] > 0)) + require(summary.get('critical_contact_ticks') == ticks, 'Critical contact count differs from native ticks.') + return episode, arrays, metadata, score + + +def _snapshot(snapshot, index, files, cameras, native_state=None): + _close(snapshot['timestamp_seconds'], index * .02, 'Measured snapshot is not on the 50 Hz clock.') + require(set(snapshot['signals']) == set(VISION) | set(ENCODERS), 'Raw sensor channels differ from the registered whitelist.') + for name in VISION: + signal = snapshot['signals'][name] + require(signal['status'] == 'missing' and signal['value'] is None, 'Raw snapshot substitutes native object truth for perception.') + for name in ENCODERS: + signal = snapshot['signals'][name] + require(signal['status'] == 'available', 'Registered native encoder channel is unavailable.') + for stamp in ('captured_at_seconds', 'available_at_seconds'): + _close(signal[stamp], snapshot['timestamp_seconds'], 'Encoder signal timestamp differs.') + _close(signal['age_seconds'], 0., 'Encoder signal age differs.') + if native_state and name in native_state: + require(signal['value'] == native_state[name], 'Encoder signal differs from saved native measurement: ' + name) + camera = snapshot['camera']; frame = camera['value']; expected_index = index // 2 + require(frame['index'] == expected_index and snapshot['vision_frame_index'] == expected_index, 'Camera source index differs from 25 Hz cadence.') + _close(camera['captured_at_seconds'], expected_index * .04, 'Camera capture time differs.') + _close(camera['available_at_seconds'], expected_index * .04, 'Synthetic camera availability differs from its captured frame.') + _close(camera['age_seconds'], index * .02 - expected_index * .04, 'Camera age was reset or changed.') + if expected_index in cameras: + require(frame == cameras[expected_index], 'Reused camera frame changed its identity.') + cameras[expected_index] = frame + for channel in ('rgb', 'depth'): + reference = frame[channel] + require(files.get(reference['path']) == {key: reference[key] for key in ('bytes', 'sha256')}, 'Camera frame is not byte-bound to the recording.') + + +def _rotation(quaternion): + q = np.asarray(quaternion, dtype=float) + require(q.shape == (4,) and np.isfinite(q).all() and abs(float(q @ q)-1.) < 1e-7, + 'Encoder pose quaternion is not a finite unit quaternion.') + w, x, y, z = q + return np.asarray([[1-2*(y*y+z*z), 2*(x*y-z*w), 2*(x*z+y*w)], + [2*(x*y+z*w), 1-2*(x*x+z*z), 2*(y*z-x*w)], + [2*(x*z-y*w), 2*(y*z+x*w), 1-2*(x*x+y*y)]]) + + +def _encoder_pose(snapshot, native_state, world_from_base): + signals = snapshot['signals'] + require(signals['gripper_joint_velocity_m_s']['value'] == native_state['qvel'][6:8], + 'Gripper encoder velocity differs from saved native state.') + # Frozen capture names the link6 body origin "tcp_position_estimate_m". + # The task scorer uses its distal authored site at link6 [0,0,.12003]. + # Reconstruct this explicit site-to-body offset; do not execute kinematics. + world_rotation = _rotation(native_state['tcp_quaternion_wxyz']) + world_link6 = np.asarray(native_state['tcp_position_m'])-world_rotation @ np.asarray([0., 0., .12003]) + expected = world_from_base[:3, :3].T @ (world_link6-world_from_base[:3, 3]) + require(np.allclose(signals['tcp_position_estimate_m']['value'], expected, atol=1e-8, rtol=0), + 'Encoder tool position differs from saved native pose and base transform.') + expected_rotation = world_from_base[:3, :3].T @ world_rotation + require(np.allclose(_rotation(signals['tcp_quaternion_estimate_wxyz']['value']), expected_rotation, atol=1e-8, rtol=0), + 'Encoder tool orientation differs from saved native pose and base transform.') + + +def _perception(document, snapshot, recording, negative, index): + require(document['index'] == index and document['snapshot_sha256'] == digest(snapshot), 'Perception is bound to a different motor snapshot.') + _close(document['decision_at_seconds'], index * .02, 'Perception decision time differs.') + require(document['frame'] == snapshot['camera']['value'] + and document['calibration_sha256'] == recording['binding']['calibration_sha256'] + and document['tracker_sha256'] == recording['tracker_sha256'], 'Perception frame/calibration/tracker identity differs.') + expected_fault = negative if snapshot['camera']['value']['index'] == 0 else None + require(document.get('negative_control') == expected_fault, 'Perception intervention differs from its registered initial fault.') + calibration = deepcopy(recording['calibration']) + if expected_fault == 'altered_calibration': + calibration['base_from_camera'][0] *= 1.1 + require(document['input_calibration_sha256'] == digest(calibration), 'Actual estimator input calibration differs from the declared intervention.') + estimate = document['estimate']; camera = snapshot['camera'] + require(estimate['frame_index'] == camera['value']['index'], 'Estimate source frame differs from the camera.') + captured = -.2 if expected_fault == 'initial_stale' else camera['captured_at_seconds'] + available = -.2 if expected_fault == 'initial_stale' else camera['available_at_seconds'] + _close(estimate['captured_at_seconds'], captured, 'Estimate capture time differs from its declared source.') + _close(estimate['available_at_seconds'], available, 'Estimate availability differs from its declared source.') + _close(estimate['decision_at_seconds'], index * .02, 'Estimate decision time differs.') + _close(estimate['age_seconds'], max(0., index * .02 - captured), 'Estimate age differs from its source.') + if estimate['status'] == 'available': + require(available <= index * .02 + 1e-8 and index * .02 - captured <= .1 + 1e-8, 'Available pose is stale or from the future.') + for field in VISION: + signal = estimate['signals'][field] + expected = estimate[field] + if field in ('object_position_uncertainty_m', 'object_orientation_uncertainty_rad'): + expected = [expected] + require(signal['status'] == 'available' and signal['value'] == expected, 'Available pose and sensor values disagree.') + for stamp in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(signal[stamp], estimate[stamp], 'Estimate signal age or source differs.') + else: + for field in VISION: + require(estimate[field] is None and estimate['signals'][field]['status'] == 'missing' + and estimate['signals'][field]['value'] is None, 'Unavailable perception forwards a live object value.') + return estimate + + +def audit_episode(root, attempt, binding, *, tracker_sha256=None): + root = Path(root); files = inventory(root); summary = read(root / 'rollout.json') + mode = 'student' if attempt['controller'] == 'blind' else attempt['controller'] + case = {**attempt['case'], 'policy_artifact_sha256': binding} + negative = attempt['negative_control'] + require(summary['case'] == case and summary['mode'] == mode and summary['negative_control'] == negative, + 'Episode controller, scene or intervention differs from registration.') + if tracker_sha256 is not None: + require(summary['tracker_sha256'] == tracker_sha256, 'Episode used a different tracker implementation.') + started = read(root / 'started.json') + require(started['case'] == case and started['mode'] == mode and started['negative_control'] == negative + and started['config']['duration_s'] == 20. and started['config']['seed'] == case['seed'] + and started['config']['width'] == 320 and started['config']['height'] == 240 + and started['config']['camera_every_controls'] == 2, 'Started rollout differs from its registered scene, controller or capture budget.') + episode, arrays, metadata, score = _native(root, files, case, summary) + result = dict(id=attempt['id'], case=attempt['case'], controller=attempt['controller'], negative_control=negative, + status=summary['status'], complete=False, strict_success=False, score=score, + initial_anchor_available=False, initial_anchor_binding_valid=False, initial_anchor_error_m=None, + recorded_complete_intervals=0, critical_contact_ticks=summary.get('critical_contact_ticks'), + provenance_ok=mode in ('student', 'fixed') and summary.get('teacher_queries') == 0, + files_verified=len(files), native_ticks=metadata['rows'] if metadata else 0, + native_score_recomputed=episode is not None, available_position_error_p95_m=None, + integrity_sha256=identity(root / 'integrity.json')['sha256'], receipt_sha256=identity(root / 'rollout.json')['sha256']) + require(summary['no_privileged_features_or_teacher_intervention'] is result['provenance_ok'], 'Reported no-teacher provenance differs from query evidence.') + if 'recording.json' not in files: + require(summary['status'] == 'refused' and summary['recorded_complete_intervals'] == 0 + and summary['strict_success'] is False, 'Setup failure claims completed task evidence.') + return result + recording = read(root / 'recording.json'); steps = recording['steps']; initial = recording['initial'] + require(recording['tracker_sha256'] == summary['tracker_sha256'], 'Recording and receipt tracker identities differ.') + require(recording['schema'] == 'dvidia.camera-action-rollout/v2' and recording['control_mode'] == mode, + 'Recording schema or authority differs.') + require(recording['provenance']['case_id'] == case['id'] and recording['provenance']['source_group'] == case['group_id'] + and recording['goal_position_m'] == case['goal_position_m'], 'Recording group, goal or case differs.') + require(len(steps) == summary['recorded_complete_intervals'] and len(steps) <= 1000, 'Completed interval count differs.') + if initial is None: + require(not steps and summary['status'] == 'refused' and not summary['initial_anchor_available'], 'Missing initial snapshot cannot support motion or a visual anchor.') + return result + context = read(root / 'context.json'); transform = _transform(context['metadata']) + require(context['case'] == case and context['initial'] == initial, 'Initial context differs from the sealed recording.') + require(digest(recording['calibration']) == recording['binding']['calibration_sha256'], 'Camera calibration binding differs.') + require(digest(recording['limits']) == recording['binding']['limits_sha256'], 'Recorded native action limits changed.') + cameras = {}; snapshots = [initial] + [step['after'] for step in steps] + for index, snapshot in enumerate(snapshots): + state = episode['trajectory'][index] if episode and len(episode['trajectory']) > index else None + _snapshot(snapshot, index, files, cameras, state) + if state is not None: + _encoder_pose(snapshot, state, transform) + require(summary['camera_pairs'] == len(cameras), 'Unique camera pair count differs.') + prior = initial; rng = random.Random(case['seed']) + for index, step in enumerate(steps): + require(step['index'] == index and step['before'] == prior, 'Transition sequence is discontinuous.') + _close(step['command']['issued_at_seconds'], index * .02, 'Command issue clock differs.') + query = _reference(root, step['teacher_query'], files) if step['teacher_query'] is not None else None + if query: + require(query['index'] == index and query['snapshot_sha256'] == digest(step['before']), 'Teacher query uses a different input snapshot.') + _close(query['computed_at_seconds'], index * .02, 'Teacher query timing differs.') + command_check(step, query, mode, negative, rng, recording['limits']) + perception = _reference(root, step['perception'], files) + _perception(perception, step['before'], recording, negative, index) + require(episode is not None and arrays is not None, 'Complete interval has no independently reconstructable native trace.') + native_action = episode['actions'][index] + require({key: native_action[key] for key in ('joint_targets_rad', 'gripper_position_m')} == step['command']['issued'], + 'Command label differs from actual issued native evidence.') + tick = (index + 1) * 20 - 1 + require(tick < metadata['rows'], 'Complete interval lacks its native endpoint.') + expected_controls = [*arrays['arm_actuator_controls_rad'][tick].tolist(), float(arrays['applied_jaw_target_m'][tick])] + require(step['applied']['native_position_controls'] == expected_controls, 'Actual native position control differs.') + require(step['applied']['ramped_targets'] == dict(joint_targets_rad=expected_controls[:6], + gripper_position_m=float(arrays['normal_rate_limited_jaw_target_m'][tick])), 'Previous applied references differ from native controls.') + prior = step['after'] + perception_files = sorted(name for name in files if name.startswith('perception/') and name.endswith('.json')) + teacher_files = sorted(name for name in files if name.startswith('teacher/') and name.endswith('.json')) + require(perception_files == [f'perception/{i:04}.json' for i in range(len(perception_files))] + and len(steps) <= len(perception_files) <= len(steps) + 1, 'Perception decision prefix is discontinuous.') + require(teacher_files == [f'teacher/{i:04}.json' for i in range(len(teacher_files))], 'Teacher query prefix is discontinuous.') + if mode in ('teacher', 'mixed'): + require(len(steps) <= len(teacher_files) <= len(steps) + 1, 'Teacher queries are missing or repeated.') + else: + require(not teacher_files, 'Student or fixed controller contains teacher-query evidence.') + require(summary['teacher_queries'] == len(teacher_files), 'Teacher query counter differs from sealed evidence.') + for index, filename in enumerate(teacher_files): + query = read(root / filename) + require(query['index'] == index and query['snapshot_sha256'] == digest(snapshots[index]) + and query['origin'] == 'privileged_simulator_teacher', 'Teacher query prefix has an invalid origin or source snapshot.') + _close(query['computed_at_seconds'], index * .02, 'Teacher query prefix clock differs.') + _action(query['action'], recording['limits']) + require(summary['student_calls'] == 0 if mode == 'teacher' else len(steps) <= summary['student_calls'] <= len(steps) + 1, + 'Student inference count differs from the completed decision prefix.') + require(summary['no_privileged_features_or_teacher_intervention'] is result['provenance_ok'], 'Reported no-teacher provenance differs from query evidence.') + estimates, visible_errors = [], [] + initial_world = np.asarray(episode['trajectory'][0]['object_position_m']) if episode and episode['trajectory'] else None + for index, filename in enumerate(perception_files): + document = read(root / filename) + estimate = _perception(document, snapshots[index], recording, negative, index) + estimates.append(estimate) + if estimate['status'] == 'available' and initial_world is not None: + tick = round(estimate['captured_at_seconds'] / .001) + truth_world = initial_world if tick == 0 else arrays['object_position_m'][tick - 1] + truth_base = transform[:3, :3].T @ (truth_world - transform[:3, 3]) + visible_errors.append(float(np.linalg.norm(np.asarray(estimate['object_position_estimate_m']) - truth_base))) + anchor_available = bool(estimates and estimates[0]['status'] == 'available') + anchor_bound = anchor_available and negative not in ('initial_stale', 'altered_calibration') + require(summary['initial_anchor_available'] is anchor_available and summary['initial_anchor_binding_valid'] is anchor_bound, + 'Initial visual anchor availability or binding differs.') + anchor_error = visible_errors[0] if anchor_available and visible_errors else None + if anchor_error is None: + require(summary['initial_anchor_error_m'] is None, 'Unavailable anchor claims native diagnostic accuracy.') + else: + _close(summary['initial_anchor_error_m'], anchor_error, 'Initial anchor error differs from native reconstruction.') + missing = [estimate['status'] != 'available' for estimate in estimates] + longest = streak = 0 + for absent in missing: + streak = streak + 1 if absent else 0; longest = max(longest, streak) + perception = dict(decisions=len(estimates), available=sum(not value for value in missing), + missing=sum(missing), longest_missing_seconds=longest * .02) + require(summary['perception'] == perception, 'Perception coverage or longest missing interval differs.') + complete = summary['status'] == 'complete' and len(steps) == 1000 + strict = complete and score.get('success') is True + require(summary['strict_success'] is strict, 'Reported strict success differs from complete native scoring.') + result.update(complete=complete, strict_success=strict, initial_anchor_available=anchor_available, + initial_anchor_binding_valid=anchor_bound, initial_anchor_error_m=anchor_error, + recorded_complete_intervals=len(steps), perception=perception, + available_position_error_p95_m=float(np.quantile(visible_errors, .95)) if visible_errors else None) + return result + + +def _stage(root, name, protocol_hash, inputs): + path = root / 'stages' / (name + '.json') + if not path.exists(): + return None + seal_path = path.with_name(name + '.integrity.json') + record = identity(path) + require(read(seal_path) == dict(sha256=record['sha256'], protocol_sha256=protocol_hash), 'Stage receipt seal differs: ' + name) + receipt = read(path) + require(receipt['stage'] == name and receipt['protocol_sha256'] == protocol_hash, 'Stage receipt names a different stage or protocol.') + started_path = path.with_name(name + '.started.json') + started = read(started_path) + require(started['stage'] == name and started['protocol_sha256'] == protocol_hash + and started.get('model_bindings') == receipt.get('model_bindings'), 'Started stage and completed receipt bindings differ.') + inputs[str(started_path.relative_to(root))] = identity(started_path) + inputs[str(path.relative_to(root))] = record + inputs[str(seal_path.relative_to(root))] = identity(seal_path) + return receipt + + +def validate_roster(protocol): + groups, points = {}, {} + for split, expected in (('train', 64), ('development', 16), ('confirmation', 64)): + cases = protocol['cases'][split] + require(len(cases) == expected and len({case['id'] for case in cases}) == expected, 'Case denominator or identity differs.') + by_group = {} + for case in cases: + require(case['split'] == split and case['goal_id'] in (0, 1), 'Case split or goal differs.') + require(groups.setdefault(case['group_id'], split) == split, 'Starting group leaks between splits.') + point = tuple(case['object_position_m']) + require(len(point) == 3 and protocol['start_domain']['x_m'][0] <= point[0] <= protocol['start_domain']['x_m'][1] + and protocol['start_domain']['y_m'][0] <= point[1] <= protocol['start_domain']['y_m'][1] + and point[2] == .082 and case['goal_position_m'] == protocol['goals'][case['goal_id']], 'Case position or destination differs from the registered domain.') + require(list(point[:2]) not in protocol['historical_start_exclusion'], 'Fresh start duplicates an explicitly excluded historical start.') + require(points.setdefault(point, split) == split, 'Exact starting position leaks between splits.') + by_group.setdefault(case['group_id'], []).append(case) + require(len(by_group) == expected // 2 and all({c['goal_id'] for c in row} == {0, 1} + and len({tuple(c['object_position_m']) for c in row}) == 1 for row in by_group.values()), + 'Both goals must share one starting-position group.') + require(len(points) == 72, 'Expected 72 unique starting positions.') + expected_counts = dict(diagnostic=4, demonstrations=80, corrections=32, + **{'development-initial': 32, 'development-corrected': 32}, confirmation=256, negative=16) + require({name: len(rows) for name, rows in protocol['rosters'].items()} == expected_counts, 'Registered attempt budgets differ.') + for name, roster in protocol['rosters'].items(): + require(len({r['id'] for r in roster}) == len(roster), 'Duplicate attempt identity.') + for attempt in roster: + split = attempt['case']['split'] + require(attempt['case'] in protocol['cases'][split], 'Attempt contains an unregistered scene.') + expected_split = ('train' if name == 'corrections' else 'confirmation' if name in ('confirmation', 'negative') else + 'development' if name == 'diagnostic' or name.startswith('development-') else None) + require(expected_split is None or split == expected_split, 'Attempt crossed its registered split.') + expected_controllers = ({'teacher': 4} if name == 'diagnostic' else {'teacher': 80} if name == 'demonstrations' + else {'mixed': 32} if name == 'corrections' else {'student': 16, 'blind': 16} if name.startswith('development-') + else {'student': 64, 'blind': 64, 'fixed': 64, 'teacher': 64} if name == 'confirmation' else {'student': 16}) + require({c: sum(a['controller'] == c for a in roster) for c in {a['controller'] for a in roster}} == expected_controllers, + 'Controller budgets differ from the approved experiment.') + require(all(a['negative_control'] is None for a in roster) if name != 'negative' else + {c: sum(a['negative_control'] == c for a in roster) for c in {a['negative_control'] for a in roster}} + == dict(initial_missing=4, initial_stale=4, altered_calibration=4, open_gripper=4), 'Negative fault budgets differ.') + pairs = [(a['case']['id'], a['controller'], a['negative_control']) for a in roster] + require(len(set(pairs)) == len(pairs), 'Same scene/controller/fault appears more than once in a stage.') + if name == 'corrections': + require(len({a['case']['group_id'] for a in roster}) == 32 + and all(a['case']['goal_id'] == i % 2 for i, a in enumerate(roster)), 'Corrections must cover each training group once with alternating destinations.') + if name == 'negative': + require(len({a['case']['id'] for a in roster}) == 4 + and all(sum(a['case']['id'] == case_id for a in roster) == 4 for case_id, _, _ in pairs), 'Every negative scene must receive all four faults.') + + +def data_quality(rows, protocol): + successful = [r for r in rows if r['case']['split'] == 'train' and r['strict_success'] + and r['initial_anchor_available'] and r['initial_anchor_binding_valid']] + expected = {case['group_id'] for case in protocol['cases']['train']} + covered = {row['case']['group_id'] for row in successful} + counts = {str(goal): sum(r['case']['goal_id'] == goal for r in successful) for goal in (0, 1)} + minimum = protocol['data_quality']['minimum_successful_episodes_per_goal'] + return dict(passed=covered == expected and all(n >= minimum for n in counts.values()), + covered_training_groups=len(covered), required_training_groups=len(expected), + missing_training_groups=sorted(expected - covered), successful_episodes_per_goal=counts, + minimum_successful_episodes_per_goal=minimum, + successful_with_available_bound_initial_camera=len(successful)) + + +def admitted(row): + return row.get('initial_anchor_available') is True and row.get('initial_anchor_binding_valid') is True and row.get('recorded_complete_intervals', 0) > 0 + + +def bootstrap(rows, comparator, acceptance, *, admitted_only=False): + groups = {} + for row in rows: + if row['controller'] in ('student', comparator): + key = (row['controller'], row['case']['goal_id']) + group = groups.setdefault(row['case']['group_id'], {}) + require(key not in group, 'Duplicate paired outcome.') + group[key] = row + require(len(groups) == 32 and all(set(group) == {(c, g) for c in ('student', comparator) for g in (0, 1)} for group in groups.values()), + 'Paired comparison lacks complete starting groups.') + retained = {key: group for key, group in groups.items() if not admitted_only or all(admitted(row) for row in group.values())} + differences = np.asarray([sum(int(group[('student', goal)]['strict_success']) - int(group[(comparator, goal)]['strict_success']) + for goal in (0, 1)) / 2 for key, group in sorted(retained.items())]) + n = len(differences); confidence = acceptance['confidence']; alpha = (1 - confidence) / 2 + if n: + rng = np.random.default_rng(acceptance['bootstrap_seed']) + sample = differences[rng.integers(0, n, size=(acceptance['bootstrap_resamples'], n))].mean(axis=1) + low, high = np.quantile(sample, [alpha, 1 - alpha]).tolist() + else: + low = high = None + return dict(comparison='student-minus-' + comparator, difference=float(differences.mean()) if n else None, + confidence=confidence, lower=low, upper=high, resamples=acceptance['bootstrap_resamples'], + seed=acceptance['bootstrap_seed'], groups=n, excluded_groups=32 - n, admitted_only=admitted_only, + pairing='both goals within exact starting-position group', positive=low is not None and low > 0.) + + +def negative_correct(row): + if row['negative_control'] == 'open_gripper': + return (row['complete'] is True and row['strict_success'] is False and row['score'].get('valid') is True + and row['score'].get('strict_v01', {}).get('trajectory_complete') is True) + return row['status'] == 'refused' and row['recorded_complete_intervals'] == 0 and row['strict_success'] is False + + +def final_gates(rows, negatives, protocol, *, stages_complete): + acceptance = protocol['acceptance']; controllers = {} + for controller in ('student', 'blind', 'fixed', 'teacher'): + selected = [row for row in rows if row['controller'] == controller] + contact = [row['critical_contact_ticks'] for row in selected] + controllers[controller] = dict(scheduled=len(selected), strict_successes=sum(r['strict_success'] for r in selected), + complete=sum(r['complete'] for r in selected), admitted_cases=sum(admitted(r) for r in selected), + initial_refusals=sum(r['status'] in ('refused', 'interrupted') and r['recorded_complete_intervals'] == 0 for r in selected), + critical_contact_episodes=sum(n > 0 for n in contact if n is not None), + critical_contact_ticks=sum(n for n in contact if n is not None), contact_counts_complete=all(n is not None for n in contact)) + complete = stages_complete and len(rows) == 256 and len(negatives) == 16 + comparisons = [bootstrap(rows, other, acceptance) for other in ('fixed', 'blind')] if complete else [] + common = [bootstrap(rows, other, acceptance, admitted_only=True) for other in ('fixed', 'blind')] if complete else [] + student, fixed = controllers['student'], controllers['fixed'] + gates = dict(all_registered_attempts_accounted=complete, + at_least_58_of_64=student['scheduled'] == acceptance['student_attempts'] and student['strict_successes'] >= acceptance['student_successes_minimum'], + positive_paired_comparisons=len(comparisons) == 2 and all(c['positive'] for c in comparisons), + positive_common_admitted_comparisons=len(common) == 2 and all(c['groups'] >= acceptance['minimum_common_admitted_groups'] and c['positive'] for c in common), + no_critical_contact_increase=student['contact_counts_complete'] and fixed['contact_counts_complete'] + and student['critical_contact_episodes'] <= fixed['critical_contact_episodes'], + all_negative_controls_correct=len(negatives) == 16 and all(negative_correct(row) for row in negatives), + no_privileged_features_or_teacher_intervention=all(row['provenance_ok'] is True for row in rows if row['controller'] in ('student', 'blind', 'fixed'))) + return dict(controllers=controllers, comparisons=comparisons, admitted_comparisons=common, gates=gates, + qualified=all(gates.values()), status='complete' if complete else 'incomplete') + + +def _model_hash(root, stage, controller, stages, inputs): + require(stage in ('initial', 'corrected') and stages.get('train-' + stage) is not None, 'Model has no sealed fitting stage.') + variant = 'blind' if controller == 'blind' else 'full' + path = f'models/{stage}/{variant}/model.json' + record = identity(root / path); inputs[path] = record + require(stages['train-' + stage]['models'][variant]['model_sha256'] == record['sha256'], 'Model differs from its sealed training receipt.') + return record['sha256'] + + +def _eligible(rows, *, queried=False): + return [row for row in rows if row.get('complete') is True and row.get('initial_anchor_available') is True + and row.get('initial_anchor_binding_valid') is True and (queried or row.get('strict_success') is True)] + + +def _export_source(root, row, label): + directory = relative(root, row['path']); recording = read(directory / 'recording.json'); steps = recording['steps'] + require(len(steps) == row['recorded_complete_intervals'], 'Training source sample count differs from completed intervals.') + require(recording['provenance']['source_group'] == row['case']['group_id'] + and recording['provenance']['case_id'] == row['case']['id'], 'Training source group differs from its registered scene.') + require(recording['control_mode'] == ('teacher' if label == 'issued' else 'mixed'), 'Training label origin differs from actual control mode.') + return dict(source_group=row['case']['group_id'], case_id=row['case']['id'], + recording_sha256=identity(directory / 'recording.json')['sha256'], + integrity_sha256=identity(directory / 'integrity.json')['sha256'], + calibration_sha256=recording['binding']['calibration_sha256'], tracker_sha256=recording['tracker_sha256'], + label_source=label, sample_count=len(steps), + perception_references_sha256=digest([s['perception']['sha256'] for s in steps]), + teacher_query_references_sha256=digest([s['teacher_query']['sha256'] for s in steps] if label == 'teacher_query' else []), + actual_authority_sha256=digest([s['command']['authority'] for s in steps])) + + +def _training(root, stages, protocol, inputs): + training_groups = {case['group_id'] for case in protocol['cases']['train']} + development_groups = {case['group_id'] for case in protocol['cases']['development']} + source_cache = {} + for stage in ('initial', 'corrected'): + receipt = stages.get('train-' + stage) + if receipt is None: + continue + require((stages.get('demonstrations') or {}).get('data_quality', {}).get('passed') is True, + 'Training proceeded without the demonstration quality gate.') + demos = stages['demonstrations']['rows'] + selected = _eligible([row for row in demos if row['case']['split'] == 'train']) + development = _eligible([row for row in demos if row['case']['split'] == 'development']) + tagged = [(row, 'issued') for row in selected] + if stage == 'corrected': + require(stages.get('corrections') is not None, 'Corrected fitting has no correction collection.') + corrections = _eligible(stages['corrections']['rows'], queried=True) + selected = selected + corrections + tagged += [(row, 'teacher_query') for row in corrections] + require(bool(selected) and bool(development), 'Fitting lacks eligible training or development recordings.') + expected_sources = [] + for row, label in tagged: + key = (row['path'], label) + if key not in source_cache: source_cache[key] = _export_source(root, row, label) + expected_sources.append(source_cache[key]) + require(len({s['recording_sha256'] for s in expected_sources}) == len(expected_sources), 'A training recording is duplicated.') + source_identities = [] + for controller, variant in (('student', 'full'), ('blind', 'blind')): + _model_hash(root, stage, controller, stages, inputs) + model = read(root / f'models/{stage}/{variant}/model.json'); training = model['training'] + receipt_entry = receipt['models'][variant] + expected_path = f'models/{stage}/{variant}/model.json' + require(receipt_entry['path'] == expected_path and receipt_entry['model_bytes'] == inputs[expected_path]['bytes'], 'Model receipt path or byte count differs.') + require(receipt_entry['training_episodes'] == [r['id'] for r in selected] + and receipt_entry['development_episodes'] == [r['id'] for r in development], 'Model fitting or validation episode roster differs from all eligible recordings.') + for name in ('receipt.json', 'started.json'): + path = f'models/{stage}/{variant}/{name}'; inputs[path] = identity(root / path) + require(read(root / f'models/{stage}/{variant}/receipt.json') == receipt_entry, 'Individual model receipt differs from sealed training stage.') + require(read(root / f'models/{stage}/{variant}/started.json') == dict(stage=stage, variant=variant, + protocol_sha256=identity(root / 'protocol.json')['sha256']), 'Model fitting start names a different stage or protocol.') + require(receipt_entry['training'] == training and receipt_entry['dataset_provenance'] == training['provenance'], 'Model metadata differs from its sealed fitting receipt.') + require(model['schema'] == 'dvidia.camera-action-mlp/v1' and model['blind'] is (variant == 'blind') + and model['observation_origin'] == 'rendered_rgbd_and_simulated_encoders' + and model['feature_contract'] == 'camera-encoder-absolute-causal160/v1' + and model['action_representation'] == 'absolute_native_targets' + and model['hidden_units'] == protocol['fitting']['hidden_units'], 'Learned model observation, architecture or action contract differs.') + require(training['stage'] == stage and training['model_source_sha256'] == protocol['source_sha256']['camera_action_model.py'], 'Training source/stage differs.') + for key in ('epochs', 'batch_size', 'learning_rate', 'seed'): + require(training[key] == protocol['fitting'][key], 'Training hyperparameter differs from registration.') + require(set(training['train_groups']) <= training_groups + and set(training['validation']['groups']) <= development_groups, 'Model fitting or validation crosses its split.') + require(training['train_groups'] == list(dict.fromkeys(r['case']['group_id'] for r in selected)) + and training['validation']['groups'] == list(dict.fromkeys(r['case']['group_id'] for r in development)), 'Model training or validation group roster differs.') + require(training['sample_count'] == sum(r['recorded_complete_intervals'] for r in selected) + and training['validation']['sample_count'] == sum(r['recorded_complete_intervals'] for r in development), 'Model training or validation sample count differs.') + provenance = training['provenance'] + require(training['provenance_sha256'] == digest(provenance) + and provenance['source_identity_sha256'] == digest(provenance['episodes']), 'Training provenance identity differs.') + require(provenance['blind'] is (variant == 'blind') + and provenance['export_source_sha256'] == protocol['source_sha256']['camera_action_export.py'] + and provenance['feature_contract_sha256'] == model['feature_contract_sha256'], 'Training export, feature contract or feedback masking identity differs.') + require(provenance['episodes'] == expected_sources, 'Training provenance omits, duplicates or changes eligible source recordings or supervision labels.') + source_identities.append(provenance['source_identity_sha256']) + require(source_identities[0] == source_identities[1], 'Full and blind models used different source demonstrations.') + + +def audit(directory): + root = Path(directory); errors = []; results = []; inputs = {}; stages = {}; stage_results = {}; qualification = None + protocol_hash = None + try: + require(not (root / 'worker.lock').exists(), 'Refusing to audit a running or interrupted study worker.') + protocol = read(root / 'protocol.json'); registration = read(root / 'registration.json') + protocol_hash = identity(root / 'protocol.json')['sha256'] + require(registration['protocol_sha256'] == protocol_hash and registration['source_sha256'] == protocol['source_sha256'], 'Registration identity differs.') + require(identity(root / 'fixed-model.json')['sha256'] == registration['fixed_model_file_sha256'], 'Pinned comparator identity differs.') + for name in ('protocol.json', 'registration.json', 'fixed-model.json'): + inputs[name] = identity(root / name) + for name, expected in protocol['source_sha256'].items(): + require(Path(name).name == name and identity(Path(scoring.__file__).with_name(name))['sha256'] == expected, 'Registered source bytes changed: ' + name) + validate_roster(protocol) + stages = {name: _stage(root, name, protocol_hash, inputs) for name in STAGES} + gap = blocked = False + for name in STAGES: + receipt = stages[name] + if receipt is None: + gap = True + continue + require(not gap and not blocked, 'Study continued after a missing or blocked prerequisite stage.') + blocked = receipt['status'] == 'blocked' + start = read(root / 'stages' / (name + '.started.json')) + require(start['source_sha256'] == protocol['source_sha256'], 'Stage started with different registered runtime bytes.') + frozen = read(root / 'freeze.json') if (root / 'freeze.json').exists() else None + if frozen: + inputs['freeze.json'] = identity(root / 'freeze.json') + require(frozen['protocol_sha256'] == protocol_hash and frozen['source_sha256'] == protocol['source_sha256'], 'Frozen protocol/source binding differs.') + for name, expected in frozen['artifacts'].items(): + record = identity(relative(root, name)); inputs[name] = record + require(record['sha256'] == expected, 'Frozen selected artifact changed.') + for stage in STAGES: + receipt = stages[stage] + if receipt is None or stage.startswith('train-'): + continue + roster = protocol['rosters'][stage]; rows = receipt['rows']; by_id = {row['id']: row for row in rows} + require(len(rows) == len(by_id) == len(roster) and set(by_id) == {a['id'] for a in roster}, 'Stage denominator or attempt IDs differ: ' + stage) + model_stage = ('initial' if stage in ('corrections', 'development-initial') else 'corrected' if stage == 'development-corrected' + else frozen['selected_stage'] if frozen and stage in ('confirmation', 'negative') else None) + expected_bindings = {} + for controller in {a['controller'] for a in roster}: + expected_bindings[controller] = (digest(protocol['source_sha256']) if controller == 'teacher' else + registration['fixed_model_file_sha256'] if controller == 'fixed' else + _model_hash(root, model_stage, controller, stages, inputs)) + require(receipt['model_bindings'] == expected_bindings, 'Stage model identities differ: ' + stage) + if stage in ('confirmation', 'negative'): + require(frozen is not None and receipt['freeze_sha256'] == inputs['freeze.json']['sha256'], 'Evaluation is not bound to the frozen model selection.') + audited = [] + for attempt in roster: + row = by_id[attempt['id']] + try: + require(all(row.get(key) == attempt[key] for key in ('case', 'controller', 'negative_control')), 'Stage row substitutes an unregistered scene or controller.') + path = f'episodes/{stage}/{attempt["id"]}' + if row['status'] in ('unknown', 'unattempted'): + require(row.get('strict_success') is False and row.get('complete') is False, 'Unknown/unattempted outcome claims success.') + require((root / path).exists() is (row['status'] == 'unknown'), 'Unknown/unattempted directory presence differs.') + result = dict(**attempt, status=row['status'], strict_success=False, complete=False, + initial_anchor_available=False, initial_anchor_binding_valid=False, initial_anchor_error_m=None, + recorded_complete_intervals=0, critical_contact_ticks=None, provenance_ok=False, + score=dict(valid=False, success=False), files_verified=0, native_score_recomputed=False) + else: + require(row['path'] == path, 'Sealed episode path differs from its attempt identity.') + require(row['policy_artifact_sha256'] == expected_bindings[attempt['controller']], 'Stage row model identity differs from its registered controller.') + result = audit_episode(root / path, attempt, expected_bindings[attempt['controller']], + tracker_sha256=protocol['source_sha256']['piper_rgbd_tracker.py']) + for key in ('status', 'strict_success', 'complete', 'initial_anchor_available', 'initial_anchor_binding_valid', + 'recorded_complete_intervals', 'critical_contact_ticks', 'provenance_ok', 'receipt_sha256', 'integrity_sha256', 'score'): + require(row.get(key) == result[key], 'Stage summary differs from independent evidence: ' + key) + if result['initial_anchor_error_m'] is None: + require(row.get('initial_anchor_error_m') is None, 'Stage summary invents an anchor error.') + else: + _close(row.get('initial_anchor_error_m'), result['initial_anchor_error_m'], 'Stage initial anchor error differs.') + for name in ('integrity.json', 'rollout.json', 'recording.json'): + if (root / path / name).exists(): + inputs[path + '/' + name] = identity(root / path / name) + result['stage'] = stage; audited.append(result); results.append(result) + except (ValueError, OSError, KeyError, TypeError, IndexError, AssertionError) as exc: + errors.append(dict(stage=stage, attempt=attempt['id'], reason=str(exc))) + stage_results[stage] = audited + if len(audited) != len(roster): + continue + require(receipt['strict_successes'] == sum(r['strict_success'] for r in audited), 'Stage success count differs.') + if stage == 'diagnostic': + threshold = protocol['diagnostic']['maximum_initial_camera_error_m'] + passed = all(r['strict_success'] and r['initial_anchor_available'] and r['initial_anchor_binding_valid'] + and type(r['initial_anchor_error_m']) in (int, float) and math.isfinite(r['initial_anchor_error_m']) + and 0 <= r['initial_anchor_error_m'] <= threshold for r in audited) + require(receipt['diagnostic_passed'] is passed, 'Diagnostic gate differs from reconstructed evidence.') + if stage == 'demonstrations': + require(receipt['data_quality'] == data_quality(audited, protocol), 'Demonstration quality gate differs from reconstructed evidence.') + _training(root, stages, protocol, inputs) + if frozen: + candidates = [] + for order, name in enumerate(('initial', 'corrected')): + rows = [row for row in stage_results['development-' + name] if row['controller'] == 'student'] + require(len(rows) == 16 and all(r['critical_contact_ticks'] is not None for r in rows), 'Development selection lacks complete outcomes.') + model = read(root / f'models/{name}/full/model.json') + candidates.append(dict(stage=name, strict_successes=sum(r['strict_success'] for r in rows), + critical_contact_ticks=sum(r['critical_contact_ticks'] for r in rows), + prediction_loss=model['training']['validation']['action_scale_mse'], order=order)) + winner = min(candidates, key=lambda c: (-c['strict_successes'], c['critical_contact_ticks'], c['prediction_loss'], c['order'])) + require(frozen['candidates'] == candidates and frozen['selected_stage'] == winner['stage'], 'Frozen selection differs from independent development comparison.') + if stages.get('confirmation') and stages.get('negative') and not errors: + qualification = final_gates(stage_results['confirmation'], stage_results['negative'], protocol, + stages_complete=all(stages[name]['status'] == 'complete' for name in ('confirmation', 'negative'))) + report = read(root / 'report.json'); inputs['report.json'] = identity(root / 'report.json') + for key, value in qualification.items(): + require(_equivalent(report[key], value), 'Final benchmark report differs from independent reconstruction: ' + key) + require(report['qualification'] == ('bounded_simulation_skill' if qualification['qualified'] else 'research_candidate') + and report['selected_stage'] == frozen['selected_stage'], 'Report qualification label or selected model differs.') + elif (root / 'report.json').exists(): + report = read(root / 'report.json'); inputs['report.json'] = identity(root / 'report.json') + blocked = [name for name in STAGES if stages[name] and stages[name]['status'] == 'blocked'] + require(report.get('qualified') is False and report.get('qualification') == 'research_candidate' + and report.get('status') == ('blocked' if blocked else 'incomplete'), 'Incomplete study claims qualification or an inconsistent blocker.') + except (ValueError, OSError, KeyError, TypeError, IndexError, AssertionError) as exc: + errors.append(dict(stage=None, attempt=None, reason=str(exc))) + return dict(schema=SCHEMA, passed=not errors, errors=errors, protocol_sha256=protocol_hash, + stages_verified=[name for name in STAGES if name in stage_results], + training_stages_verified=[name for name in STAGES if name.startswith('train-') and stages.get(name)], + attempts_verified=len(results), native_scores_recomputed=sum(r.get('native_score_recomputed') is True for r in results), + files_verified=sum(r['files_verified'] for r in results), results=results, qualification=qualification, + evidence_inventory=inputs, evidence_inventory_sha256=digest(inputs), + audit_source_sha256=identity(Path(__file__))['sha256'], + diagnostic_helper_source_sha256=identity(Path(__file__).with_name('independent_audit.py'))['sha256'], + scorer_source_sha256=identity(Path(scoring.__file__))['sha256'], + physics_executed=False, estimator_executed=False, model_executed=False, + source_authenticated=False, physical_robot_ready=False, + encoder_pose_reference='Capture estimates use the link6 body origin in robot_base. Native scoring uses the authored site at link6 local [0,0,0.12003] m; this audit explicitly reconstructs that fixed offset.', + scope='Independent byte, roster, clock, control-authority and numeric outcome reconstruction for all sealed completed stages. Missing stages remain unrun; no physical qualification.') + + +def _equivalent(left, right): + if isinstance(left, dict) and isinstance(right, dict): + return set(left) == set(right) and all(_equivalent(left[key], right[key]) for key in left) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(_equivalent(a, b) for a, b in zip(left, right)) + if type(left) in (int, float) and type(right) in (int, float): + return math.isfinite(left) and math.isfinite(right) and math.isclose(left, right, abs_tol=1e-12, rel_tol=0) + return type(left) is type(right) and left == right + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study') + args = parser.parse_args(argv) + result = audit(args.study) + print(canonical(result).decode()) + return 0 if result['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/summarize_results.py b/benchmarks/camera-action-v01/summarize_results.py new file mode 100644 index 0000000..e20fa3d --- /dev/null +++ b/benchmarks/camera-action-v01/summarize_results.py @@ -0,0 +1,503 @@ +#!/usr/bin/env python3 +"""Read-only posthoc tables from sealed camera/action study receipts. + +This tool never imports a policy, simulator, trainer or the frozen study runtime. +It verifies evidence receipts and model bytes, then summarizes recorded native +scores and reconstructs terminal components numerically. It never reruns physics, +changes qualification gates or rehashes image archives. Missing evidence stays unknown. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import zipfile + +import numpy as np + +SCHEMA = 'dvidia.camera-action-posthoc-summary/v1' +MAX_JSON_BYTES = 32 * 1024 * 1024 +TIMING_FIELDS = ('estimator_wall_seconds', 'policy_inference_wall_seconds', + 'full_boundary_wall_seconds', 'render_encode_wall_seconds') +COMPONENTS = ( + ('grasp', ('native_recomputed', 'grasp_seen'), True, None), + ('lift', ('native_recomputed', 'lift_seen'), True, ('native_recomputed', 'grasp_seen')), + ('carry', ('native_recomputed', 'carry_complete'), True, ('native_recomputed', 'lift_seen')), + ('retention', ('native_recomputed', 'retention_lost'), False, ('native_recomputed', 'lift_seen')), + ('terminal_composite', ('terminal_only', 'success'), True, ('native_recomputed', 'carry_complete')), + ('release', None, True, ('native_recomputed', 'carry_complete')), + ('settling', None, True, ('native_recomputed', 'carry_complete')), + ('retreat', None, True, ('native_recomputed', 'carry_complete')), +) +NOTES = [ + 'Posthoc description only: no tuning, model inference, new episodes or qualification changes.', + 'All-attempt success denominators include scheduled refusals, interrupted attempts and missing receipts.', + 'Component counts use valid independent native score receipts. Missing or invalid evidence remains unassessed.', + 'Conditional component denominators require the preceding grasp, lift or supported carry predicate; they are not all-attempt success rates.', + 'Release, settling and retreat are separately reconstructed posthoc from sealed native arrays using the frozen scorer predicates. Final predicate and consecutive dwell are diagnostic localization, not new qualification gates.', + 'Latencies pool recorded boundary samples, never averages of episode percentiles. The simulator pauses for computation on a shared host; these are not physical sensor-to-actuator latencies.', + 'Camera acquisition and availability timestamps are synthetic. Depth and physical hardware accuracy remain unqualified.', + 'Peak RSS is a process high-water mark including dataset export and any preceding fit, not an isolated model memory measurement.', + 'Evidence verification covers protocol/stage seals, referenced rollout/context receipts, native numeric evidence and model bytes. Image archives and full policy/scoring correctness remain the scope of the separately published independent audit.', +] + + +def digest(raw): + return sha256(raw).hexdigest() + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON key: '+key) + result[key] = value + return result + + +def _read(path, maximum=MAX_JSON_BYTES): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: + raise ValueError('Expected a bounded regular evidence file: '+path.name) + raw = path.read_bytes() + value = json.loads(raw, object_pairs_hook=_pairs, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Nonfinite JSON value'))) + return value, raw + + +def _relative(root, value): + if not isinstance(value, str): + raise ValueError('Evidence path must be relative.') + relative = Path(value) + if relative.is_absolute() or '..' in relative.parts or not relative.parts: + raise ValueError('Evidence path escapes the study.') + path = root/relative + if any(p.is_symlink() for p in (path, *path.parents) if p != root.parent): + raise ValueError('Evidence cannot traverse symlinks.') + if not path.resolve().is_relative_to(root.resolve()): + raise ValueError('Evidence path escapes the study.') + return path + + +def _number(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0. + + +def _integer(value): + return type(value) is int and value >= 0 + + +def _at(value, path): + for key in path or (): + if not isinstance(value, dict): + return None + value = value.get(key) + return value if path else None + + +def _quantile(values, q): + ordered = sorted(values) + if not ordered: + return None + point = (len(ordered)-1)*q + lower = int(point); upper = min(lower+1, len(ordered)-1) + return ordered[lower]+(ordered[upper]-ordered[lower])*(point-lower) + + +def distribution(values): + return dict(samples=len(values), p50=_quantile(values, .5), p95=_quantile(values, .95), + minimum=min(values) if values else None, maximum=max(values) if values else None) + + +def _verify_identity(path, expected): + value, raw = _read(path) + if expected != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Bound evidence identity differs: '+path.name) + return value + + +def native_terminal_components(arrays, metadata, episode): + """Exact terminal predicates from the frozen scorer, recomputed numerically. + + No grasp attachment, action, teacher phase or scored success flag is used. + Full 1 ms evidence is required; missing arrays produce an explicit refusal. + """ + config = episode.get('config', {}) + dt, timeout = config.get('timestep'), config.get('timeout_s') + required = config.get('goal_dwell_s') + retreat = config.get('retreat_clearance_m') + tolerance = config.get('goal_position_tolerance_m') + if (not _number(dt) or dt != .001 or not _number(timeout) or not .1 <= timeout <= 30. + or not math.isclose(timeout/dt, round(timeout/dt), abs_tol=1e-8, rel_tol=0) + or not _number(required) or not .1 <= required <= .6 + or not _number(retreat) or not .08 <= retreat <= .15 + or not _number(tolerance) or not .01 <= tolerance <= .025): + raise ValueError('Invalid native component task config.') + count = round(timeout/dt) + if (metadata.get('schema') != 'dvidia.piper-feedback-native/v1' + or metadata.get('native_timestep_s') != dt or metadata.get('rows') != count + or metadata.get('side_order') != ['left', 'right']): + raise ValueError('Native component metadata/cadence differs.') + widths = dict(t_seconds=1, native_tick=1, grasp_contact_counts=2, + gripper_joint_position_m=2, object_position_m=3, tcp_position_m=3, + object_linear_velocity_m_s=3, object_angular_velocity_rad_s=3, + table_object_contact_count=1) + values = {} + for name, width in widths.items(): + value = np.asarray(arrays.get(name)) + if (value.dtype.kind not in 'fiu' or value.shape != ((count,) if width == 1 else (count, width)) + or not np.isfinite(value).all()): + raise ValueError('Missing or malformed native component field: '+name) + values[name] = value + if (not np.array_equal(values['native_tick'], np.arange(1, count+1)) + or not np.allclose(values['t_seconds'], np.arange(1, count+1)*dt, atol=1e-8, rtol=0)): + raise ValueError('Native component time/tick evidence differs.') + for name in ('grasp_contact_counts', 'table_object_contact_count'): + if np.any(values[name] < 0) or np.any(values[name] != np.floor(values[name])): + raise ValueError('Native component contact count is not a nonnegative integer.') + goal = np.asarray(config.get('target_position_m')) + if goal.shape != (3,) or goal.dtype.kind not in 'fiu' or not np.isfinite(goal).all(): + raise ValueError('Native component destination is malformed.') + jaw = values['gripper_joint_position_m']; obj = values['object_position_m'] + released = (np.all(values['grasp_contact_counts'] == 0, axis=1) + & (jaw[:, 0] > .028) & (jaw[:, 1] < -.028)) + settled = ((np.linalg.norm(values['object_linear_velocity_m_s'], axis=1) < .035) + & (np.linalg.norm(values['object_angular_velocity_rad_s'], axis=1) < .3)) + clear = values['tcp_position_m'][:, 2] >= obj[:, 2]+retreat + supported = values['table_object_contact_count'] > 0 + at_destination = np.linalg.norm(obj-goal, axis=1) <= tolerance + predicates = dict(release=released, settling=settled, retreat=clear, + supported=supported, at_destination=at_destination, + joint_terminal=released & settled & clear & supported & at_destination) + result = {} + for name, mask in predicates.items(): + failures = np.flatnonzero(~mask) + terminal_ticks = count-int(failures[-1])-1 if len(failures) else count + dwell = terminal_ticks*dt + result[name] = dict(final_predicate=bool(mask[-1]), final_consecutive_dwell_seconds=dwell, + required_dwell_seconds=required, dwell_met=bool(dwell+1e-12 >= required)) + return dict(status='available', native_ticks=count, predicates=result, + scope='Posthoc native terminal-component diagnosis; does not replace the frozen task qualification gates.') + + +def _native_components(episode_root, files, receipt): + if receipt.get('score', {}).get('valid') is not True: + return dict(status='unassessed', reason='No valid independent native score receipt.') + required = ('native.npz', 'native_metadata.json', 'native_episode.json') + if any(name not in files for name in required): + return dict(status='unassessed', reason='Required native component evidence is missing.') + metadata = _verify_identity(episode_root/'native_metadata.json', files['native_metadata.json']) + episode = _verify_identity(episode_root/'native_episode.json', files['native_episode.json']) + path = episode_root/'native.npz' + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_JSON_BYTES: + raise ValueError('Native numeric artifact must be a bounded regular file.') + raw = path.read_bytes() + if files['native.npz'] != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Native numeric evidence differs from its bound inventory.') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entries = archive.infolist() + if len(entries) > 64 or sum(entry.file_size for entry in entries) > 64*1024*1024: + raise ValueError('Native numeric expansion exceeds the bounded study artifact.') + names = ('t_seconds', 'native_tick', 'grasp_contact_counts', 'gripper_joint_position_m', + 'object_position_m', 'tcp_position_m', 'object_linear_velocity_m_s', + 'object_angular_velocity_rad_s', 'table_object_contact_count') + with np.load(io.BytesIO(raw), allow_pickle=False) as archive: + arrays = {name: archive[name] for name in names if name in archive.files} + result = native_terminal_components(arrays, metadata, episode) + terminal = receipt['score'].get('terminal_only', {}) + reconstructed = result['predicates']['joint_terminal'] + if (type(terminal.get('success')) is not bool or not _number(terminal.get('dwell_s')) + or terminal['success'] != reconstructed['dwell_met'] + or not math.isclose(terminal['dwell_s'], reconstructed['final_consecutive_dwell_seconds'], abs_tol=1e-8, rel_tol=0)): + raise ValueError('Posthoc joint terminal predicate differs from the sealed independent scorer.') + result['source_sha256'] = {name: files[name]['sha256'] for name in required} + return result + + +def _episode(root, row): + """Only read files already bound by the sealed stage and episode inventory.""" + if row.get('path') is None or row.get('receipt_sha256') is None: + if row.get('strict_success') is True or row.get('complete') is True or row.get('score', {}).get('valid') is True: + raise ValueError('Successful or valid outcomes require a bound rollout receipt.') + return None, None + episode_root = _relative(root, row['path']) + receipt, raw = _read(episode_root/'rollout.json') + if digest(raw) != row['receipt_sha256']: + raise ValueError('Rollout receipt differs from sealed stage.') + inventory, inventory_raw = _read(episode_root/'integrity.json') + if digest(inventory_raw) != row.get('integrity_sha256'): + raise ValueError('Episode inventory differs from sealed stage.') + files = inventory.get('files') + if not isinstance(files, dict): + raise ValueError('Episode inventory is missing.') + if files.get('rollout.json') != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Rollout receipt differs from episode inventory.') + if receipt.get('score') != row.get('score') or receipt.get('perception') != row.get('perception'): + raise ValueError('Stage score/perception differs from bound rollout.') + if receipt.get('status') != row.get('status'): + raise ValueError('Stage completion differs from bound rollout.') + complete = receipt.get('status') == 'complete' and receipt.get('recorded_complete_intervals') == 1000 + strict = bool(complete and receipt.get('score', {}).get('success') is True) + if row.get('complete') is not complete or row.get('strict_success') is not strict: + raise ValueError('Stage strict outcome differs from recorded completion and score.') + context = None + if 'context.json' in files: + context = _verify_identity(episode_root/'context.json', files['context.json']) + receipt['_posthoc_components'] = _native_components(episode_root, files, receipt) + return receipt, context + + +def component_counts(rows, expected_count, component): + name, path, success_value, prerequisite = component + known = passed = failed = eligible = conditional_known = conditional_failed = unknown_prerequisite = 0 + for row in rows: + score = row.get('score') or {} + if score.get('valid') is not True: + unknown_prerequisite += prerequisite is not None + continue + value = (_at(row.get('_posthoc_components', {}), ('predicates', name, 'dwell_met')) + if path is None else _at(score, path)) + admitted = True if prerequisite is None else _at(score, prerequisite) + if type(admitted) is not bool: + unknown_prerequisite += 1 + if admitted is True: + eligible += 1 + if type(value) is bool: + known += 1 + passes = value is success_value + passed += passes; failed += not passes + if admitted is True: + conditional_known += 1; conditional_failed += not passes + terminal_values = [_at(row.get('_posthoc_components', {}), ('predicates', name)) for row in rows] if path is None else [] + terminal_values = [value for value in terminal_values if isinstance(value, dict)] + final = [value['final_predicate'] for value in terminal_values if type(value.get('final_predicate')) is bool] + dwell = [value['final_consecutive_dwell_seconds'] for value in terminal_values if _number(value.get('final_consecutive_dwell_seconds'))] + return dict(metric=name, all_attempts=expected_count, assessed=known, passed=passed, failed=failed, + unassessed=expected_count-known, conditional_eligible=eligible, + conditional_assessed=conditional_known, conditional_failed=conditional_failed, + conditional_unassessed=eligible-conditional_known, + prerequisite_unknown=unknown_prerequisite+(expected_count-len(rows) if prerequisite else 0), + condition='all valid native score receipts' if prerequisite is None else '.'.join(prerequisite)+' == true', + availability='posthoc_reconstructed_native_predicate' if path is None else 'recorded_independent_native_predicate', + final_predicate=dict(assessed=len(final), true=sum(final), false=sum(not value for value in final)) if path is None else None, + final_consecutive_dwell_seconds=distribution(dwell) if path is None else None, + required_dwell_seconds=sorted({value['required_dwell_seconds'] for value in terminal_values + if _number(value.get('required_dwell_seconds'))}) if path is None else None) + + +def _controller(stage_name, controller, roster, rows, episodes): + expected = len(roster) + values = [{**row, '_posthoc_components': (episodes.get(row['id'], (None, None))[0] or {}).get('_posthoc_components', {})} + for row in rows if row['controller'] == controller] + actual = [episodes.get(row['id'], (None, None))[0] for row in values] + bound = [value for value in actual if value is not None] + success = sum(row.get('strict_success') is True for row in values) + complete = sum(row.get('complete') is True for row in values) + native_valid = sum(row.get('score', {}).get('valid') is True for row in values) + perception_rows = [row['perception'] for row in values if isinstance(row.get('perception'), dict) + and all(_integer(row['perception'].get(name)) for name in ('decisions', 'available', 'missing'))] + if any(row['available']+row['missing'] != row['decisions'] for row in perception_rows): + raise ValueError('Perception counts disagree.') + decisions = sum(row['decisions'] for row in perception_rows) + available = sum(row['available'] for row in perception_rows) + missing_spans = [row['longest_missing_seconds'] for row in perception_rows if _number(row.get('longest_missing_seconds'))] + latencies = {} + for metric in TIMING_FIELDS: + samples = [boundary[metric] for value in bound for boundary in value.get('timing', {}).get('boundaries', []) + if isinstance(boundary, dict) and _number(boundary.get(metric))] + applicable = not (metric == 'policy_inference_wall_seconds' and controller == 'teacher') + result = distribution(samples if applicable else []) + result.update(applicable=applicable, episode_receipts=len(bound), unit='seconds') + if metric == 'full_boundary_wall_seconds': + result['wall_samples_over_20ms'] = sum(sample > .02 for sample in samples) + result['deadline_claim'] = False + latencies[metric] = result + durations = [value['timing']['wall_seconds'] for value in bound + if _number(value.get('timing', {}).get('wall_seconds'))] + critical_counts = [_at(row.get('score'), ('native_recomputed', 'unexpected_contact_native_ticks')) + for row in values if row.get('score', {}).get('valid') is True] + critical_counts = [value for value in critical_counts if _integer(value)] + return dict(stage=stage_name, controller=controller, scheduled=expected, + retained_rows=len(values), missing_rows=expected-len(values), complete=complete, + native_assessable=native_valid, strict_successes=success, + strict_success_rate_all_attempts=success/expected if expected else None, + strict_success_rate_complete=success/complete if complete else None, + not_confirmed_successful=expected-success, + critical_contacts=dict(assessed_attempts=len(critical_counts), + unassessed_attempts=expected-len(critical_counts), + attempts_with_contacts=sum(value > 0 for value in critical_counts), + affected_fraction_assessed=sum(value > 0 for value in critical_counts)/len(critical_counts) if critical_counts else None, + native_ticks=sum(critical_counts), native_timestep_seconds=.001, + scope='Independent native unexpected-contact ticks; repeated contact is counted at each 1 ms physics tick.'), + statuses=dict(Counter(row.get('status', 'unknown') for row in values)), + components={item[0]: component_counts(values, expected, item) for item in COMPONENTS}, + terminal_components_by_attempt=[dict(id=row['id'], **row['_posthoc_components']) for row in values], + perception=dict(episodes_reported=len(perception_rows), episodes_unknown=expected-len(perception_rows), + available_decisions=available, assessed_decisions=decisions, + available_fraction=available/decisions if decisions else None, + maximum_missing_seconds=max(missing_spans) if missing_spans else None), + timing=latencies, episode_wall_seconds=distribution(durations)) + + +def summarize(study): + root = Path(study).resolve() + registration, _ = _read(root/'registration.json') + protocol, protocol_raw = _read(root/'protocol.json') + protocol_hash = digest(protocol_raw) + if registration.get('protocol_sha256') != protocol_hash: + raise ValueError('Protocol differs from registration.') + rosters = protocol.get('rosters') + if not isinstance(rosters, dict): + raise ValueError('Registered attempt rosters are required.') + result = dict(schema=SCHEMA, protocol_sha256=protocol_hash, posthoc=True, + physical_robot_ready=False, evidence_scope=NOTES[-1], notes=list(NOTES), + controllers=[], stages=[], training=[], hardware=[], source_receipts={}, qualification=None) + hardware = {} + stage_names = list(dict.fromkeys([*rosters, 'train-initial', 'train-corrected'])) + for name in stage_names: + path = root/'stages'/(name+'.json') + if not path.exists(): + result['stages'].append(dict(stage=name, status='pending_receipt', scheduled=len(rosters.get(name, [])))) + continue + stage, raw = _read(path) + seal, _ = _read(path.with_name(name+'.integrity.json')) + if seal != dict(sha256=digest(raw), protocol_sha256=protocol_hash): + raise ValueError('Stage seal differs: '+name) + if stage.get('stage') != name or stage.get('protocol_sha256') != protocol_hash: + raise ValueError('Stage protocol/name binding differs.') + result['source_receipts'][str(path.relative_to(root))] = digest(raw) + result['stages'].append(dict(stage=name, status=stage.get('status'), reason=stage.get('reason'), + scheduled=stage.get('scheduled'), strict_successes=stage.get('strict_successes'))) + if name.startswith('train-'): + for variant, record in stage.get('models', {}).items(): + model_path = _relative(root, record['path']) + _, model_raw = _read(model_path) + if digest(model_raw) != record['model_sha256'] or len(model_raw) != record['model_bytes']: + raise ValueError('Training model bytes differ from its receipt.') + result['training'].append(dict(stage=name, variant=variant, model_bytes=len(model_raw), + model_sha256=digest(model_raw), training_wall_seconds=record.get('training_wall_seconds'), + shared_export_wall_seconds=record.get('shared_export_wall_seconds'), + process_peak_rss_bytes=record.get('process_peak_rss_bytes'), memory_scope=record.get('memory_scope'), + training_episodes=len(record.get('training_episodes', [])))) + continue + roster = rosters.get(name, []) + expected = {row['id']: row for row in roster} + if len(expected) != len(roster) or stage.get('scheduled') != len(roster): + raise ValueError('Registered or sealed attempt counts disagree.') + rows = stage.get('rows', []) + if not isinstance(rows, list) or len({row['id'] for row in rows}) != len(rows): + raise ValueError('Stage attempt IDs are duplicated or invalid.') + if {row['id'] for row in rows} != set(expected): + raise ValueError('Stage attempt ID set differs from the registered roster; unattempted rows must be retained.') + episodes = {} + for row in rows: + registered = expected.get(row['id']) + if registered is None or any(row.get(key) != registered.get(key) + for key in ('controller', 'case', 'negative_control')): + raise ValueError('Attempt differs from the registered roster.') + receipt, context = _episode(root, row) + episodes[row['id']] = receipt, context + if context is not None: + runtime = context.get('metadata', {}).get('runtime') + if isinstance(runtime, dict): + key = json.dumps(runtime, sort_keys=True, allow_nan=False) + hardware.setdefault(key, dict(recorded_runtime=runtime, episode_receipts=0, + cpu_model=None, gpu_model=None, physical_sensor_model=None, + unreported_fields='CPU/GPU device identity and physical sensor identity were not captured in the runtime receipt.'))['episode_receipts'] += 1 + if stage.get('strict_successes') != sum(row.get('strict_success') is True for row in rows): + raise ValueError('Stage strict-success total disagrees with its rows.') + for controller in dict.fromkeys(row['controller'] for row in roster): + registered = [row for row in roster if row['controller'] == controller] + result['controllers'].append(_controller(name, controller, registered, rows, episodes)) + result['hardware'] = list(hardware.values()) + report_path = root/'report.json' + if report_path.exists(): + report, raw = _read(report_path) + if report.get('protocol_sha256') != protocol_hash: + raise ValueError('Study report protocol binding differs.') + result['qualification'] = dict(status=report.get('status'), qualified=report.get('qualified'), + qualification=report.get('qualification'), report_sha256=digest(raw), + source='existing study report; not newly awarded by this posthoc tool') + return result + + +def _cell(value): + return str(value).replace('|', '\\|').replace('\n', ' ') if value is not None else 'Not reported' + + +def _percent(value): + return 'Not reported' if value is None else f'{value*100:.1f}%' + + +def markdown(summary): + lines = ['# Camera-to-action study: posthoc evidence tables', '', + 'These tables summarize sealed evidence without running or tuning policies.', '', + '| Stage / controller | Strict success / scheduled | Complete | Native assessable | All-attempt success |', + '|---|---:|---:|---:|---:|'] + for row in summary['controllers']: + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {row['strict_successes']} / {row['scheduled']} | {row['complete']} | {row['native_assessable']} | {_percent(row['strict_success_rate_all_attempts'])} |") + lines += ['', 'Component failures are conditional on the stated prerequisite. Unknown counts are retained.', '', + '| Stage / controller | Component | Prerequisite | Failed / conditional assessed | Eligible but unassessed | All-attempt unassessed |', + '|---|---|---|---:|---:|---:|'] + for row in summary['controllers']: + for name, component in row['components'].items(): + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {_cell(name)} | {_cell(component['condition'])} | {component['conditional_failed']} / {component['conditional_assessed']} | {component['conditional_unassessed']} | {component['unassessed']} |") + lines += ['', '| Stage / controller | Contact-affected / native assessed | Unassessed | Critical 1 ms ticks |', + '|---|---:|---:|---:|'] + for row in summary['controllers']: + contact=row['critical_contacts'] + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {contact['attempts_with_contacts']} / {contact['assessed_attempts']} | {contact['unassessed_attempts']} | {contact['native_ticks']} |") + lines += ['', 'Posthoc terminal predicates are measured separately from the frozen task gate.', '', + '| Stage / controller | Predicate | False at final tick / assessed | Final dwell min / median | Required dwell |', + '|---|---|---:|---:|---:|'] + for row in summary['controllers']: + for name in ('release', 'settling', 'retreat'): + component=row['components'][name];final=component['final_predicate'];dwell=component['final_consecutive_dwell_seconds'] + observed='Not reported' if dwell['p50'] is None else f"{dwell['minimum']:.3f} / {dwell['p50']:.3f} s" + required=', '.join(f'{value:.3f} s' for value in component['required_dwell_seconds']) or 'Not reported' + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {name} | {final['false']} / {final['assessed']} | {observed} | {required} |") + lines += ['', '| Stage / controller | Visual decisions available | Maximum missing span | Inference p95 | Full boundary p95 |', + '|---|---:|---:|---:|---:|'] + for row in summary['controllers']: + perception=row['perception']; timing=row['timing'] + def ms(value):return 'Not reported' if value is None else f'{value*1000:.3f} ms' + span='Not reported' if perception['maximum_missing_seconds'] is None else f"{perception['maximum_missing_seconds']:.3f} s" + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {perception['available_decisions']} / {perception['assessed_decisions']} | {span} | {ms(timing['policy_inference_wall_seconds']['p95'])} | {ms(timing['full_boundary_wall_seconds']['p95'])} |") + lines += ['', '| Fit | Model bytes | Fit/save seconds | Shared export seconds | Process peak RSS bytes |', + '|---|---:|---:|---:|---:|'] + for row in summary['training']: + lines.append('| '+' | '.join(_cell(v) for v in (row['stage']+' / '+row['variant'],row['model_bytes'], + row['training_wall_seconds'],row['shared_export_wall_seconds'],row['process_peak_rss_bytes']))+' |') + lines += ['', 'Recorded execution environments:', ''] + if summary['hardware']: + for row in summary['hardware']: + lines.append('- '+_cell(json.dumps(row['recorded_runtime'], sort_keys=True))+ + f"; {row['episode_receipts']} episode receipts. CPU/GPU device identities were not recorded.") + else: + lines.append('- No bound runtime receipt is available.') + lines += ['', *('- '+note for note in summary['notes']), ''] + return '\n'.join(lines) + + +def main(argv=None): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study',required=True);parser.add_argument('--output',required=True) + args=parser.parse_args(argv) + root=Path(args.study).resolve();output=Path(args.output).absolute() + if output.exists() or output.is_symlink() or output.resolve().is_relative_to(root): + parser.error('Choose a fresh output directory outside the study.') + summary=summarize(root) + output.mkdir(parents=True,exist_ok=False) + (output/'summary.json').write_text(json.dumps(summary,indent=2,sort_keys=True,allow_nan=False)+'\n') + (output/'summary.md').write_text(markdown(summary)) + print(json.dumps(dict(status='complete',output=str(output),controller_rows=len(summary['controllers']), + physical_robot_ready=False,posthoc=True))) + return 0 + + +if __name__=='__main__': + raise SystemExit(main()) diff --git a/docs/camera-action-v01-report-draft.md b/docs/camera-action-v01-report-draft.md new file mode 100644 index 0000000..b2fbd8f --- /dev/null +++ b/docs/camera-action-v01-report-draft.md @@ -0,0 +1,155 @@ +# Camera-to-action skill v0.1 — research report draft + +**Status: diagnostic evidence only; student results are pending.** This document +does not claim a learned placement success rate, improvement over either +comparator, a qualified downloadable skill or physical robot readiness. + +## Question and bounded experiment + +Can a small local model learn a complete PiPER block-placement task from +synchronized RGB-D, measured robot state and explicit synthetic action labels? +The intended learned sequence is approach, grasp, lift, carry, release and +retreat. The study uses one simulated PiPER, its existing gripper, a known rigid +40 mm block and two fixed destinations. It does not recover motor actions from +ordinary human video. + +All starts, controller assignments, training settings and acceptance gates were +registered before collection. The runtime is frozen at +`c9a7c55531533f60e24651a99d509f721e195051`; the protocol SHA-256 is +`576851e321034a0f6d3cb4d0818b3a5ca97975fcc6f2c7fa30b07ffd203a2d76`. + +There are 72 fresh starting-position groups: 32 training, eight development and +32 confirmation. Both destinations remain together in their starting-position +group. Starts span x=0.287–0.311 m and y=−0.042–−0.018 m, with the configured +initial block height 0.082 m. This small interpolation domain is a deliberate +first proof, not general spatial manipulation. A 3 mm inset from the prior +fixed primitive's envelope buffers camera error; estimates are not clipped or +replaced with simulator truth. + +| Registered stage | Budget | Role | +| --- | ---: | --- | +| Initial teacher diagnostic | 4 trajectories | Verify actual motion, initial camera acquisition and complete teacher placement | +| Training teacher demonstrations | 64 episodes | Two goals for each of 32 training starts | +| Development teacher demonstrations | 16 episodes | Prediction diagnostics; excluded from fitting and normalization | +| Mixed-control corrections | 32 episodes | One training-only round; teacher chosen with probability 0.5 | +| Model development | 64 rollouts | Initial/corrected × full/blind × 16 development cases | +| Frozen confirmation | 256 attempts | 64 cases each for learner, matched blind model, fixed primitive and teacher | +| Negative controls | 16 attempts | Blank initial RGB, invalid old initial timestamps, malformed extrinsics and forced-open gripper | + +The maximum registered native workload is 452 attempts, with no replacement of +failed starts or hidden retry. A 20-second complete episode contains 501 +320×240 RGB-D pairs at 25 Hz, 1,000 motor intervals at 50 Hz, and 20,000 native +physics ticks at 1 ms. Camera capture/availability timestamps are simulated +clocks; wall-clock processing is recorded separately. They do not establish +physical camera-to-actuator latency. + +## Inputs, supervision and acceptance + +The CPU NumPy model has two 64-unit hidden layers and predicts six absolute +joint targets plus one jaw target. Its 269 features contain an explicitly +historical initial visual anchor, destination, elapsed time, encoders, jaw +measurements, encoder-derived link6 body pose, previous applied servo references and +160 ms of causal history. Current missing object estimates have zero numerical +fills, an unavailable mask and their source frame's original age. The historical +anchor is never presented as a new live observation. The frozen recording names +the link6 body-origin pose `tcp_position_estimate_m` and +`tcp_quaternion_estimate_wxyz`; these legacy field names do not identify a +calibrated gripper contact point. The native task scorer instead uses a distal +site offset by 0.12003 m along link6’s local z axis. The independent auditor +reconstructs this explicit offset when comparing the two coordinate records. + +Teacher phase, current teacher command, native object/contact truth and future +observations are excluded. Teacher commands are synthetic labels from a +privileged authored controller. In the mixed-control round, the one teacher +query per step is stored separately from the command actually executed; it is +not described as an operator correction or a proven recovery. The blind model +uses identical source episodes and initial context but has all live and +historical sensor feedback masked out. It retains past command history. + +Each fit uses seed 17, 50 epochs, batch size 512 and Adam learning rate 0.003. +Normalization uses training data only, including explicit missing fills and +masks. Candidate selection uses development task success, critical contacts, +then prediction error normalized by the same physical action scales across +candidates. The matched blind model is taken from the selected stage. + +Before training, every training start must have at least one successful usable +teacher demonstration, and each goal must have at least 28 successes. Final +qualification requires all of the following: + +- At least 58/64 strict learned placements on the scheduled unseen cases. +- Positive 97.5% paired bootstrap intervals against both fixed and blind + comparators, resampling whole starting-position groups with both goals paired. +- Positive comparisons also on at least 24 common admitted groups, preventing + initial comparator refusals from being the only basis for improvement. +- No increase in the frequency of critical-contact episodes versus the fixed + comparator; all negative controls behave as specified. +- Complete attempt accounting, independent native scoring, no privileged + learner inputs and no teacher intervention during evaluation. + +The teacher comparator has privileged simulation observations and is an upper +reference, not evidence of human-video learning. Installation alone cannot +award qualification. Failed candidates remain publishable research findings. + +The fault definitions are deliberately narrow: `initial_missing` zeros the +initial RGB image while the physical object remains in the scene; +`initial_stale` sets capture and availability times to −0.2 s at episode time +zero; `altered_calibration` scales one rotation-matrix element by 1.1; and +`open_gripper` overrides issued jaw targets to 0.035 m. The first three exercise +input-contract refusal. They do not establish detection of a physically absent +object, valid-but-old frames or small valid rigid calibration errors. + +## Completed diagnostic evidence + +An independent reader verified **24,040 files**, exact camera/control/native +cadences, source and frame bindings, issued/query separation, camera-position +errors and independently recomputed task scores. All four teacher diagnostics +completed strict placement successfully. They represent **two development +starting positions, each with two goals**, not four independent starting groups. + +| Diagnostic | Initial camera error | Available-pose error p95 | Available decisions | Longest missing stretch | +| --- | ---: | ---: | ---: | ---: | +| Development 00, goal 0 | 0.196 mm | 0.314 mm | 620/1,000 | 7.24 s | +| Development 00, goal 1 | 0.196 mm | 0.504 mm | 618/1,000 | 7.28 s | +| Development 01, goal 0 | 0.249 mm | 0.816 mm | 602/1,000 | 7.96 s | +| Development 01, goal 1 | 0.249 mm | 0.816 mm | 604/1,000 | 7.92 s | + +Native object displacement reached 138.6–160.0 mm, confirming actual +manipulation rather than a static-camera or replay-only check. Errors above +are conditional on the tracker returning an estimate. Adjacent 50 Hz decisions +often reuse the same 25 Hz frame and are not independent tracking trials. + +The major diagnostic finding is **where visibility disappears**. An offline +analysis of sealed teacher-phase and perception sidecars found zero available +object estimates during close, lift, transfer, place and release: **0/749 motor +decisions** across the four trajectories. Approach and final verification were +fully visible. Teacher phase was used only to analyze this evidence, never as +a model input. + +Therefore sub-millimetre available-pose errors must not be described as tracking +throughout manipulation. During the contact-heavy sequence the proposed +learner must rely on measured robot state and its historical initial visual +context. This is a measured limitation of the frozen upright-block tracker. +The current experiment will test whether those remaining signals are enough; +it must not silently add camera views or substitute native object positions. + +Evidence files retained for publication: + +- `runs/camera-action-v01-proofs-20261101/diagnostic-audit.json` +- `runs/camera-action-v01-proofs-20261101/diagnostic-phase-coverage.json` +- The registered protocol, stage receipts and complete sealed diagnostic recordings. + +## Pending results and interpretation + +**Pending:** training-data quality, model fitting, initial and corrected +development rollouts, frozen confirmation, negative controls, measured model +size/fit cost/inference cost, and fresh-install execution. Fill these sections +only from sealed receipts after independent readback; do not infer learned +success from successful teacher demonstrations. + +This study can establish a reproducible camera-and-action learning loop for a +bounded synthetic task. It cannot establish arbitrary video-to-skill transfer, +arbitrary-arm compatibility, real-time hardware operation, real-world depth or +contact calibration, shoe manipulation or physical qualification. A useful next +experiment after the fixed results is to measure a second camera or another +explicitly uncertain observation method during grasp and carry, where the +present tracker provides no object measurements. diff --git a/docs/camera-action-v01.md b/docs/camera-action-v01.md index ea93fd2..2387a26 100644 --- a/docs/camera-action-v01.md +++ b/docs/camera-action-v01.md @@ -55,11 +55,16 @@ reported stage. Existing destination directories are never overwritten. The frozen RGB-D tracker receives current pixels, depth and calibration only. The policy receives a timestamped initial visual position, goal, elapsed time, -encoders, encoder-derived tool pose, previous applied servo references and 160 ms +encoders, encoder-derived link6 pose, previous applied servo references and 160 ms of causal history. Missing live estimates retain availability masks and source frame age; their numerical tensor fills are not measurements. Normalization uses training data, including these explicit missing fills and masks. +The legacy `tcp_position_estimate_m` field represents the link6 body origin in +this pilot. The native task scorer uses a distal site 0.12003 m along link6's +local z axis. These are distinct points; the audit accounts for that fixed +transform rather than treating the feature as a calibrated tool-tip estimate. + The policy never receives native object/contact truth, teacher phase, current teacher target or future frames. Historical initial position is never copied into the current-object channel. Native truth is used only by the teacher and offline diff --git a/publishing/CAMERA-ACTION-V01.md b/publishing/CAMERA-ACTION-V01.md new file mode 100644 index 0000000..6c79df8 --- /dev/null +++ b/publishing/CAMERA-ACTION-V01.md @@ -0,0 +1,99 @@ +# Camera-to-action evidence publication + +This workflow publishes a stopped, sealed simulation study to +`Dvidia/piper-camera-action-v01`. It does not train a model, run physics, alter a +Space, or turn a failed study into a qualified skill. + +Build the wheel and a `git archive` tar.gz from the reviewed source commit. +The archive must retain Git's PAX commit comment. Both artifacts must contain +the exact registered runtime bytes, including unchanged native scorers. +Provide the reviewed dataset card and the independently generated diagnostic +audit. The audit's scope is explicitly the four diagnostic episodes. +The optional `--phase-coverage` argument includes the diagnostic-only phase +coverage proof after verifying its protocol and audit bindings. + +After the study stops, independently audit every sealed stage with the numeric +scorer and saved evidence. This command does not import or execute the native +environment, tracker, teacher or learned policy. It verifies frame identities, +50 Hz command and 25 Hz camera clocks, 1 ms native traces, command authority, +teacher-query separation, model identities and split membership. It reconstructs +data-quality gates and, if confirmation exists, grouped bootstrap comparisons +and the final qualification gates. Missing stages remain unrun. + +```sh +PYTHONPATH=src:benchmarks/camera-action-v01 python \ + benchmarks/camera-action-v01/study_audit.py /absolute/stopped-study \ + > /absolute/full-study-audit.json +``` + +Pass `--study-audit /absolute/full-study-audit.json` to preparation. It must +pass, cover every currently sealed stage and match the protocol, model, stage +and episode identities. This proof supplements the diagnostic audit. An audit +passing means reported outcomes agree with saved evidence, including failures; +it does not mean the skill passed its qualification gates. +The reviewed source archive must include the exact full-study auditor and its +diagnostic byte-reader helper; use the final tooling commit while preserving +all registered runtime bytes. + +Add `--hardware /absolute/hardware-native.json` to include the measured host, +software versions, native worker count and graphics/timing scope as +`hardware.json`. The field whitelist permits public CPU/model/memory details +and rejects hostname, serial, account or other arbitrary fields. The entire +JSON must fit within 16 KiB. These details describe the measured run; they do +not establish minimum hardware requirements or physical sensor latency. + +```sh +PYTHONPATH=src python publishing/prepare_camera_action_v01.py \ + --study /absolute/stopped-study \ + --output /absolute/new-publication-directory \ + --audit /absolute/diagnostic-audit.json \ + --card /absolute/README.md \ + --wheel /absolute/dvidia_training.whl \ + --source-archive /absolute/reviewed-source.tar.gz \ + --source-commit EXACT_40_CHARACTER_COMMIT + +python publishing/huggingface/publish_camera_action_v01.py stage \ + /absolute/new-publication-directory --plan /absolute/reviewed-plan.json +``` + +Review the public roster and returned plan digest before publication. The +explicit `publish` command reads only the existing macOS Keychain entry +`dvidia-huggingface`, account `dvidia-publisher`. No credential argument, +environment variable, credential file, or login cache is required. + +```sh +python publishing/huggingface/publish_camera_action_v01.py publish \ + --plan /absolute/reviewed-plan.json --sha256 REVIEWED_PLAN_DIGEST \ + --cache /absolute/publication-cache + +python publishing/huggingface/publish_camera_action_v01.py verify \ + --plan /absolute/reviewed-plan.json --sha256 REVIEWED_PLAN_DIGEST \ + --cache /absolute/anonymous-readback-cache +``` + +Preparation streams one deterministic ZIP64 archive per sealed episode, +checks each file against the original integrity manifest, and decompresses +every member to verify its bytes. It refuses a worker lock, unsealed episode, +unexpected study file, symbolic link, missing source binding, or detected +credential-shaped text. The 30 GiB disk reserve is preserved. Existing output +directories and files are never overwritten. + +The publisher freezes reviewed bytes into a private temporary directory before +credential lookup, uses a single commit bound to the empty dataset's exact +parent revision, and refuses existing nonempty or private repositories. +Anonymous verification downloads every published file at the exact resulting +commit and checks its byte identity. Large files are streamed, not loaded into +one large in-memory payload. + +To reconstruct the recorded study, verify `SHA256SUMS`, then extract +`study-metadata.zip` and every `episodes/**/*.zip` into one new directory. +The resulting layout contains the original protocol, stage receipts, camera +frames, encoders, issued controls, perception and teacher-query sidecars, +native traces, and each episode's original `integrity.json`. Raw files inside +an episode must not be edited or supplemented before inspection. + +The dataset includes separate model JSON files only when sealed training +receipts bind those exact files. A blocked data-quality gate therefore +publishes evidence and source without inventing a learned policy. The package +retains the study's qualification result; installation itself does not award +task qualification or physical-hardware readiness. diff --git a/publishing/huggingface/camera_action_hardware.py b/publishing/huggingface/camera_action_hardware.py new file mode 100644 index 0000000..511dfd9 --- /dev/null +++ b/publishing/huggingface/camera_action_hardware.py @@ -0,0 +1,32 @@ +"""Small public hardware context; hostnames, serials and arbitrary fields excluded.""" +import re + +MAX_BYTES = 16 * 1024 +FIELDS = {'scope', 'system', 'release', 'machine', 'python', 'versions', 'hardware', 'native_workers', 'gpu_use'} +HARDWARE = {'hw.model', 'hw.memsize', 'hw.physicalcpu', 'hw.logicalcpu', 'machdep.cpu.brand_string'} + + +def validate_hardware(value): + if type(value) is not dict or set(value) != FIELDS: + raise ValueError('Hardware context must use the public field whitelist; no hostname, serial or account fields.') + for key in ('scope', 'system', 'release', 'machine', 'python', 'gpu_use'): + text=value[key] + if type(text) is not str or not 1 <= len(text) <= (1024 if key in ('scope','gpu_use') else 80) or any(ord(c)<32 for c in text): + raise ValueError('Hardware context contains invalid or unbounded text.') + if type(value['native_workers']) is not int or not 1 <= value['native_workers'] <= 1024: + raise ValueError('Hardware context requires a bounded native worker count.') + versions=value['versions']; hardware=value['hardware'] + if type(versions) is not dict or set(versions) != {'numpy','mujoco'} or type(hardware) is not dict or set(hardware) != HARDWARE: + raise ValueError('Hardware/software details must use the public field whitelist.') + for version in [value['python'],*versions.values()]: + if type(version) is not str or not re.fullmatch(r'[0-9][0-9A-Za-z.+_-]{0,79}',version): + raise ValueError('Software version must be a plain bounded version string.') + for key in ('hw.memsize','hw.physicalcpu','hw.logicalcpu'): + number=hardware[key] + if type(number) is not str or not re.fullmatch(r'[1-9][0-9]{0,18}',number) or int(number)>(2**60 if key=='hw.memsize' else 4096): + raise ValueError('Hardware memory and CPU counts must be positive bounded decimal strings.') + for key in ('hw.model','machdep.cpu.brand_string'): + text=hardware[key] + if type(text) is not str or not 1<=len(text)<=120 or any(ord(c)<32 for c in text) or any(c in text for c in ('/','\\','@')): + raise ValueError('Hardware product name must be a bounded public model identifier.') + return value diff --git a/publishing/huggingface/publish_camera_action_v01.py b/publishing/huggingface/publish_camera_action_v01.py new file mode 100644 index 0000000..5bbb851 --- /dev/null +++ b/publishing/huggingface/publish_camera_action_v01.py @@ -0,0 +1,270 @@ +"""Publish one reviewed, byte-bound camera/action evidence dataset. + +Stage and verify never read credentials. Publish copies the reviewed payload +into a private temporary directory before Keychain access, then makes one +parent-bound commit to the named empty public dataset. No Space is modified. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import os +from pathlib import Path +import re +import shutil +import stat +import sys +import tempfile + +from publish import PublicationError, client +from publish_task_environments_v02 import encoded, plain, public_client, relative, scan +from camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware + +REPO = 'Dvidia/piper-camera-action-v01' +SCHEMA = 'dvidia.camera-action-publication/v1' +EVIDENCE_SCHEMA = 'dvidia.camera-action-evidence/v1' +MAX_JSON = 32 * 1024 ** 2 +MAX_FILE = 1024 ** 3 +MAX_TOTAL = 120 * 1024 ** 3 +RESERVE = 30 * 1024 ** 3 +OID = re.compile(r'(?:[a-f0-9]{40}|[a-f0-9]{64})\Z') +REQUIRED = {'README.md', 'LICENSE', 'manifest.json', 'SHA256SUMS', 'report.json', + 'protocol.json', 'registration.json', 'independent-audit.json', 'study-metadata.zip'} + + +def identity(path, *, maximum=MAX_FILE): + path = plain(path) + with path.open('rb') as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= maximum: + raise PublicationError('Publication file is missing, nonregular or oversized.') + h, count, tail = sha256(), 0, b'' + for raw in iter(lambda: stream.read(1024 * 1024), b''): + count += len(raw) + if count > maximum: + raise PublicationError('Publication payload grew beyond its bound.') + h.update(raw) + scan(tail + raw) + tail = raw[-256:] + after = os.fstat(stream.fileno()) + if count != before.st_size or any(getattr(before, key) != getattr(after, key) + for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise PublicationError('Publication bytes changed during inspection.') + return dict(bytes=count, sha256=h.hexdigest()) + + +def read(path): + path = plain(path) + identity(path, maximum=MAX_JSON) + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise PublicationError('Duplicate JSON field in reviewed publication artifact.') + result[key] = value + return result + def invalid(_value): + raise PublicationError('Nonfinite reviewed publication artifact.') + try: + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + except (UnicodeError, ValueError, TypeError) as exc: + raise PublicationError('Malformed reviewed publication artifact.') from exc + + +def allowed(name): + relative(name) + return (name in REQUIRED or name in ('diagnostic-phase-coverage.json', 'full-study-audit.json', 'hardware.json') + or re.fullmatch(r'episodes/[a-z-]+/[a-z0-9-]+\.zip', name) is not None + or re.fullmatch(r'models/(initial|corrected)/(full|blind)/model\.json', name) is not None + or re.fullmatch(r'source/[A-Za-z0-9_.+-]+\.(whl|tar\.gz)', name) is not None) + + +def _record(value): + if (not isinstance(value, dict) or set(value) != {'bytes', 'sha256'} + or type(value['bytes']) is not int or not 0 <= value['bytes'] <= MAX_FILE + or not isinstance(value['sha256'], str) or not re.fullmatch('[a-f0-9]{64}', value['sha256'])): + raise PublicationError('Malformed reviewed file identity.') + return value + + +def files(root): + root = plain(root) + manifest = read(root / 'manifest.json') + if (manifest.get('schema') != EVIDENCE_SCHEMA or manifest.get('repo_id') != REPO + or manifest.get('repo_type') != 'dataset' or manifest.get('physical_robot_ready') is not False + or manifest.get('original_footage_included') is not False + or manifest.get('status') not in ('blocked', 'complete', 'incomplete') + or manifest.get('pipeline_status', manifest.get('status')) not in ('blocked', 'complete') + or type(manifest.get('qualified')) is not bool): + raise PublicationError('Expected the reviewed simulation-only camera/action evidence manifest.') + expected = manifest.get('files') + if not isinstance(expected, dict) or not REQUIRED - {'manifest.json', 'SHA256SUMS'} <= set(expected): + raise PublicationError('Required evidence files are missing.') + if any(not allowed(name) or name in ('manifest.json', 'SHA256SUMS') for name in expected): + raise PublicationError('Unrecognized public evidence path.') + if manifest.get('hardware_context_file') != ('hardware.json' if 'hardware.json' in expected else None): + raise PublicationError('Hardware context file differs from its manifest binding.') + if 'hardware.json' in expected: + identity(root / 'hardware.json', maximum=MAX_HARDWARE_BYTES) + try: + validate_hardware(read(root / 'hardware.json')) + except ValueError as exc: + raise PublicationError('Hardware context contains unknown, private or unbounded fields.') from exc + observed = [] + for path in sorted(root.rglob('*')): + plain(path) + if not (path.is_file() or path.is_dir()): + raise PublicationError('Public evidence contains a special file.') + if path.is_file(): + observed.append(path.relative_to(root).as_posix()) + if not 10 <= len(observed) <= 1024 or set(observed) != set(expected) | {'manifest.json', 'SHA256SUMS'}: + raise PublicationError('Public file roster differs from the reviewed manifest.') + source_files = [name for name in expected if name.startswith('source/')] + if len(source_files) != 2 or sum(name.endswith('.whl') for name in source_files) != 1 or sum(name.endswith('.tar.gz') for name in source_files) != 1: + raise PublicationError('Exact source archive and installable wheel are required.') + episodes = manifest.get('episodes') + if not isinstance(episodes, list) or len(episodes) != manifest.get('episode_count'): + raise PublicationError('Episode shard roster is malformed.') + if {r.get('archive') for r in episodes} != {name for name in expected if name.startswith('episodes/')}: + raise PublicationError('Episode archives differ from the declared roster.') + if set(manifest.get('trained_model_files', [])) != {name for name in expected if name.startswith('models/')}: + raise PublicationError('Model artifacts differ from the actual training roster.') + report = read(root / 'report.json') + if (report.get('qualification') != manifest.get('qualification') or report.get('qualified') is not manifest['qualified'] + or report.get('physical_robot_ready') is not False or report.get('status') != manifest['status']): + raise PublicationError('Public report and qualification manifest disagree.') + if manifest['qualified']: + if 'full-study-audit.json' not in expected: + raise PublicationError('A qualified skill requires the full-study independent audit.') + proof = read(root / 'full-study-audit.json') + if (proof.get('schema') != 'dvidia.camera-action-study-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or (proof.get('qualification') or {}).get('qualified') is not True): + raise PublicationError('Full-study audit does not support the qualified-skill label.') + inventory = [] + for name in observed: + record = identity(root / name) + if name in expected and record != _record(expected[name]): + raise PublicationError('Payload bytes differ from the reviewed evidence manifest.') + inventory.append(dict(path=name, **record, + representation='lfs' if name.endswith(('.zip', '.gz', '.whl')) else 'git')) + if sum(row['bytes'] for row in inventory) > MAX_TOTAL: + raise PublicationError('Public evidence exceeds the bounded dataset size.') + sums = ''.join(f'{row["sha256"]} {row["path"]}\n' for row in inventory if row['path'] != 'SHA256SUMS') + if (root / 'SHA256SUMS').read_text() != sums: + raise PublicationError('Published checksum list does not match all payload bytes.') + return inventory + + +def stage(root, plan): + root, plan = plain(root), plain(plan) + value = dict(schema=SCHEMA, repo_id=REPO, repo_type='dataset', private=False, + root=str(root), files=files(root)) + with plan.open('xb') as stream: + stream.write(encoded(value)) + return dict(plan_sha256=identity(plan, maximum=MAX_JSON)['sha256'], files=len(value['files']), + bytes=sum(r['bytes'] for r in value['files'])) + + +def checked(plan, expected_digest): + if identity(plan, maximum=MAX_JSON)['sha256'] != expected_digest: + raise PublicationError('Reviewed publication plan digest changed.') + value = read(plan) + if (set(value) != {'schema', 'repo_id', 'repo_type', 'private', 'root', 'files'} + or value['schema'] != SCHEMA or value['repo_id'] != REPO or value['repo_type'] != 'dataset' + or value['private'] is not False or type(value['root']) is not str or type(value['files']) is not list): + raise PublicationError('Reviewed publication destination or scope changed.') + if files(value['root']) != value['files']: + raise PublicationError('Reviewed publication inventory changed.') + return value + + +def _remote(info): + if (getattr(info, 'private', None) is not False or not OID.fullmatch(getattr(info, 'sha', '') or '') + or getattr(info, 'siblings', None) is None): + raise PublicationError('Expected an exact initialized public dataset head.') + + +def publish(plan, expected_digest, cache): + value = checked(plan, expected_digest) + cache = plain(cache) + cache.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(cache).free - sum(r['bytes'] for r in value['files']) < RESERVE: + raise PublicationError('Freezing the publication would consume the protected disk reserve.') + with tempfile.TemporaryDirectory(prefix='camera-action-frozen-', dir=cache) as temporary: + frozen = Path(temporary) + for row in value['files']: + source = plain(Path(value['root']) / row['path']) + target = frozen / relative(row['path']) + target.parent.mkdir(parents=True, exist_ok=True) + with source.open('rb') as source_stream, target.open('xb') as destination: + shutil.copyfileobj(source_stream, destination, 1024 * 1024) + if identity(target) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Payload changed while freezing reviewed publication bytes.') + # Originals are never reopened after the credential boundary. The + # private immutable copy bounds memory while preserving reviewed bytes. + api, token = client(cache) + from huggingface_hub import CommitOperationAdd + from huggingface_hub.errors import RepositoryNotFoundError + try: + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + except RepositoryNotFoundError: + api.create_repo(REPO, repo_type='dataset', private=False, exist_ok=False) + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + _remote(info) + if len(info.siblings) != 1 or {item.rfilename for item in info.siblings} != {'.gitattributes'}: + raise PublicationError('Dataset already contains content; verify it instead of overwriting it.') + operations = [CommitOperationAdd(path_in_repo=row['path'], path_or_fileobj=str(frozen / row['path'])) + for row in value['files']] + commit = api.create_commit(REPO, repo_type='dataset', operations=operations, + parent_commit=info.sha, commit_message='Publish camera-to-action v0.1 reproducible simulation evidence') + del token, api + return dict(repo_id=REPO, commit=commit.oid, files_added=len(operations)) + + +def download(name, revision, cache): + from huggingface_hub import hf_hub_download + return Path(hf_hub_download(REPO, name, repo_type='dataset', revision=revision, + token=False, endpoint='https://huggingface.co', cache_dir=str(cache), force_download=True)).resolve() + + +def verify(plan, expected_digest, cache): + value = checked(plan, expected_digest) + api = public_client() + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True, token=False) + _remote(info) + expected = {row['path'] for row in value['files']} + if len(info.siblings) != len(expected) + 1 or {item.rfilename for item in info.siblings} != expected | {'.gitattributes'}: + raise PublicationError('Public dataset roster differs from the reviewed plan.') + for row in value['files']: + path = download(row['path'], info.sha, cache) + if identity(path) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Anonymous public bytes differ from reviewed evidence.') + return dict(repo_id=REPO, commit=info.sha, anonymous_files_verified=len(value['files'])) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + create = commands.add_parser('stage') + create.add_argument('root') + create.add_argument('--plan', required=True) + for name in ('publish', 'verify'): + command = commands.add_parser(name) + command.add_argument('--plan', required=True) + command.add_argument('--sha256', required=True) + command.add_argument('--cache', required=True) + args = parser.parse_args(argv) + try: + result = stage(args.root, args.plan) if args.command == 'stage' else globals()[args.command](args.plan, args.sha256, args.cache) + print(json.dumps(result, indent=2)) + return 0 + except PublicationError as exc: + print(str(exc), file=sys.stderr) + except Exception: + print('Publication operation failed; no credential or remote response details displayed.', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/huggingface/test_camera_action_hardware.py b/publishing/huggingface/test_camera_action_hardware.py new file mode 100644 index 0000000..5c0a123 --- /dev/null +++ b/publishing/huggingface/test_camera_action_hardware.py @@ -0,0 +1,33 @@ +from copy import deepcopy +import unittest + +from camera_action_hardware import validate_hardware + + +def fixture(): + return dict(scope='Shared host; offline simulator timings only.', system='Darwin', release='25.5.0', + machine='arm64', python='3.12.9', versions={'numpy':'2.5.3','mujoco':'3.15.0'}, + hardware={'hw.model':'Mac17,4','hw.memsize':'25769803776','hw.physicalcpu':'10', + 'hw.logicalcpu':'10','machdep.cpu.brand_string':'Apple M5'},native_workers=1, + gpu_use='Existing macOS CGL rendering; CPU policy and physics.') + + +class HardwareContextTests(unittest.TestCase): + def test_public_hardware_is_accepted_and_private_fields_rejected(self): + validate_hardware(fixture()) + for key in ('hostname','serial','account','machine_uuid'): + for nested in (False,True): + value=fixture(); target=value['hardware'] if nested else value;target[key]='private fixture' + with self.subTest(key=key,nested=nested),self.assertRaisesRegex(ValueError,'whitelist'): + validate_hardware(value) + + def test_text_worker_counts_and_numeric_hardware_are_bounded(self): + for key,invalid in (('scope','x'*1025),('python','/private/version'),('native_workers',True),('native_workers',0)): + value=fixture();value[key]=invalid + with self.subTest(key=key),self.assertRaises(ValueError):validate_hardware(value) + for key,invalid in (('hw.memsize','0'),('hw.physicalcpu','4097'),('hw.logicalcpu',10),('hw.model','/Users/private')): + value=fixture();value['hardware'][key]=invalid + with self.subTest(key=key),self.assertRaises(ValueError):validate_hardware(value) + + +if __name__=='__main__':unittest.main() diff --git a/publishing/huggingface/test_publish_camera_action_v01.py b/publishing/huggingface/test_publish_camera_action_v01.py new file mode 100644 index 0000000..11241c1 --- /dev/null +++ b/publishing/huggingface/test_publish_camera_action_v01.py @@ -0,0 +1,156 @@ +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import Mock, patch + +import publish_camera_action_v01 as publisher + + +class CameraPublicationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.dataset = self.root / 'dataset' + self.dataset.mkdir() + names = (publisher.REQUIRED - {'manifest.json', 'SHA256SUMS'}) | { + 'source/fixture.whl', 'source/fixture.tar.gz'} + for name in names: + path = self.dataset / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'Synthetic test evidence only.') + (self.dataset / 'report.json').write_text(json.dumps(dict( + qualification='research_candidate', qualified=False, physical_robot_ready=False, status='blocked'))) + files = {name: publisher.identity(self.dataset / name) for name in sorted(names)} + manifest = dict(schema=publisher.EVIDENCE_SCHEMA, repo_id=publisher.REPO, repo_type='dataset', + status='blocked', qualification='research_candidate', qualified=False, + original_footage_included=False, physical_robot_ready=False, + files=files, episodes=[], episode_count=0, trained_model_files=[]) + (self.dataset / 'manifest.json').write_text(json.dumps(manifest)) + files['manifest.json'] = publisher.identity(self.dataset / 'manifest.json') + (self.dataset / 'SHA256SUMS').write_text(''.join(f'{row["sha256"]} {name}\n' for name, row in sorted(files.items()))) + self.plan = self.root / 'plan.json' + self.receipt = publisher.stage(self.dataset, self.plan) + class Operation: + def __init__(self, path_in_repo, path_or_fileobj): + self.path_in_repo, self.path_or_fileobj = path_in_repo, path_or_fileobj + class NotFound(Exception): + pass + patches = patch.dict(sys.modules, {'huggingface_hub': SimpleNamespace(CommitOperationAdd=Operation), + 'huggingface_hub.errors': SimpleNamespace(RepositoryNotFoundError=NotFound)}) + patches.start() + self.addCleanup(patches.stop) + + def info(self, names=('.gitattributes',), *, private=False, sha='a' * 40): + return SimpleNamespace(private=private, sha=sha, + siblings=[SimpleNamespace(rfilename=name) for name in names]) + + def test_stage_check_and_full_public_readback_never_access_credentials(self): + value = publisher.checked(self.plan, self.receipt['plan_sha256']) + api = Mock() + api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) + with patch.object(publisher, 'client', side_effect=AssertionError('Unexpected credential access')), \ + patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', side_effect=lambda name, revision, cache: self.dataset / name): + proof = publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(proof['anonymous_files_verified'], len(value['files'])) + api.repo_info.assert_called_once_with(publisher.REPO, repo_type='dataset', files_metadata=True, token=False) + + def test_mutated_payload_and_changed_destination_fail_before_keychain(self): + (self.dataset / 'README.md').write_bytes(b'changed') + with patch.object(publisher, 'client') as client: + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + client.assert_not_called() + value = json.loads(self.plan.read_bytes()) + value['repo_id'] = 'Dvidia/robot-simulation-lab' + self.plan.write_bytes(publisher.encoded(value)) + with self.assertRaisesRegex(publisher.PublicationError, 'destination'): + publisher.checked(self.plan, sha256(self.plan.read_bytes()).hexdigest()) + + def test_existing_private_or_nonempty_dataset_cannot_be_overwritten(self): + for info in (self.info(['README.md']), self.info(private=True), self.info(sha='main')): + api = Mock() + api.repo_info.return_value = info + with self.subTest(info=info), patch.object(publisher, 'client', return_value=(api, 'fixture')): + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + api.create_commit.assert_not_called() + + def test_payload_is_frozen_before_credential_boundary_and_uses_parent_cas(self): + api = Mock() + api.repo_info.return_value = self.info() + uploaded = {} + def commit(repo, **kwargs): + self.assertEqual(repo, publisher.REPO) + self.assertEqual(kwargs['parent_commit'], 'a' * 40) + for operation in kwargs['operations']: + uploaded[operation.path_in_repo] = Path(operation.path_or_fileobj).read_bytes() + return SimpleNamespace(oid='b' * 40) + api.create_commit.side_effect = commit + original = (self.dataset / 'README.md').read_bytes() + def credential_boundary(_cache): + (self.dataset / 'README.md').write_bytes(b'late mutation must not be uploaded') + return api, 'fixture' + with patch.object(publisher, 'client', side_effect=credential_boundary): + result = publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(uploaded['README.md'], original) + self.assertEqual(result['commit'], 'b' * 40) + self.assertEqual(list((self.root / 'cache').glob('camera-action-frozen-*')), []) + + def test_extra_file_symlink_and_false_qualification_are_refused(self): + target = self.dataset / 'secret.env' + target.write_text('not a credential') + with self.assertRaises(publisher.PublicationError): + publisher.files(self.dataset) + target.unlink() + readme = self.dataset / 'README.md' + readme.unlink() + readme.symlink_to(self.dataset / 'LICENSE') + with self.assertRaises(publisher.PublicationError): + publisher.files(self.dataset) + readme.unlink() + readme.write_bytes(b'Synthetic test evidence only.') + report = json.loads((self.dataset / 'report.json').read_bytes()) + report['qualified'] = True + (self.dataset / 'report.json').write_text(json.dumps(report)) + with self.assertRaisesRegex(publisher.PublicationError, 'qualification'): + publisher.files(self.dataset) + + def test_public_readback_detects_changed_remote_bytes(self): + value = publisher.checked(self.plan, self.receipt['plan_sha256']) + api = Mock() + api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) + changed = self.root / 'changed' + changed.write_bytes(b'not the published bytes') + with patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', return_value=changed): + with self.assertRaisesRegex(publisher.PublicationError, 'Anonymous public bytes'): + publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + + def test_qualified_label_requires_full_study_audit_even_when_report_matches_manifest(self): + for name in ('manifest.json','report.json'): + path=self.dataset/name; value=json.loads(path.read_bytes()) + value.update(qualified=True,qualification='bounded_simulation_skill',status='complete') + path.write_text(json.dumps(value)) + with self.assertRaisesRegex(publisher.PublicationError,'requires the full-study'): + publisher.files(self.dataset) + + def test_private_hardware_fields_are_rejected_before_upload(self): + from test_camera_action_hardware import fixture + value=fixture();value['hardware']['serial']='private fixture' + target=self.dataset/'hardware.json';target.write_text(json.dumps(value)) + manifest=json.loads((self.dataset/'manifest.json').read_bytes()) + manifest['hardware_context_file']='hardware.json';manifest['files']['hardware.json']=publisher.identity(target) + (self.dataset/'manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(publisher.PublicationError,'private or unbounded'): + publisher.files(self.dataset) + + +if __name__ == '__main__': + unittest.main() diff --git a/publishing/prepare_camera_action_v01.py b/publishing/prepare_camera_action_v01.py new file mode 100644 index 0000000..8577328 --- /dev/null +++ b/publishing/prepare_camera_action_v01.py @@ -0,0 +1,520 @@ +"""Build deterministic raw-evidence shards from a stopped, sealed local study. + +This is an offline packaging command. It performs no simulation, training, +credential lookup or upload, and never changes study artifacts. A diagnostic +audit remains explicitly diagnostic-only when a larger study is packaged. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import tarfile +import zipfile + +from huggingface.camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware + +SCHEMA = 'dvidia.camera-action-evidence/v1' +MAX_FILE = 256 * 1024 ** 2 +MAX_SOURCE = 512 * 1024 ** 2 +MAX_EPISODE = 1024 ** 3 +MAX_TOTAL = 120 * 1024 ** 3 +DISK_RESERVE = 30 * 1024 ** 3 +TEXT_SUFFIXES = {'.py', '.js', '.ts', '.json', '.html', '.md', '.txt', '.toml', '.yaml', '.yml'} +SECRET = re.compile(rb'(?:hf_|gh[pousr]_|apikey_)[A-Za-z0-9_]{24,}') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def plain(path): + path = Path(path).absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (path, *path.parents)): + raise ValueError('Evidence paths cannot traverse symbolic links.') + return path + + +def relative(name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name + or str(PurePosixPath(name)) != name or PurePosixPath(name).is_absolute() + or any(p in ('', '.', '..') or p.startswith('.') for p in PurePosixPath(name).parts) + or any(ord(c) < 32 for c in name)): + raise ValueError('Expected a plain relative evidence path.') + return PurePosixPath(name) + + +def scan(raw): + if SECRET.search(raw): + raise ValueError('Credential-shaped bytes found in proposed public evidence.') + + +def identity(path, *, maximum=MAX_FILE, secret_scan=False): + path = plain(path) + with path.open('rb') as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= maximum: + raise ValueError('Missing, nonregular or oversized evidence file.') + digest, size, tail = sha256(), 0, b'' + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + size += len(chunk) + if size > maximum: + raise ValueError('Evidence exceeded its byte limit while reading.') + digest.update(chunk) + if secret_scan: + scan(tail + chunk) + tail = chunk[-256:] + after = os.fstat(stream.fileno()) + if any(getattr(before, key) != getattr(after, key) for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise ValueError('Evidence changed while reading.') + if size != before.st_size: + raise ValueError('Evidence byte count changed.') + return dict(bytes=size, sha256=digest.hexdigest()) + + +def read(path): + path = plain(path) + if path.stat().st_size > MAX_FILE: + raise ValueError('JSON evidence exceeds the bounded format.') + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON evidence field.') + result[key] = value + return result + def invalid(_value): + raise ValueError('Nonfinite JSON evidence.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + + +def write(path, value): + path = plain(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('xb') as stream: + stream.write(canonical(value) + b'\n') + + +def files(root): + root = plain(root) + paths = [] + for path in sorted(root.rglob('*')): + plain(path) + if not (path.is_dir() or path.is_file()): + raise ValueError('Special files are not public evidence.') + if path.is_file(): + relative(path.relative_to(root).as_posix()) + paths.append(path) + return paths + + +def checked_identity(value): + if (not isinstance(value, dict) or set(value) != {'bytes', 'sha256'} + or type(value['bytes']) is not int or not 0 <= value['bytes'] <= MAX_FILE + or not isinstance(value['sha256'], str) or not re.fullmatch('[a-f0-9]{64}', value['sha256'])): + raise ValueError('Malformed sealed evidence identity.') + return value + + +def episode_inventory(root): + root = plain(root) + manifest = read(root / 'integrity.json') + if manifest.get('schema') != 'dvidia.camera-action-rollout/v2/integrity': + raise ValueError('Only sealed v2 native rollout evidence can be archived.') + expected = manifest.get('files') + if not isinstance(expected, dict) or not 2 <= len(expected) <= 9000: + raise ValueError('Episode file inventory is outside the bounded format.') + for name, record in expected.items(): + relative(name) + checked_identity(record) + observed = {p.relative_to(root).as_posix() for p in files(root)} + if observed != set(expected) | {'integrity.json'}: + raise ValueError('Episode contains unsealed, missing or extra files.') + result = dict(expected, **{'integrity.json': identity(root / 'integrity.json')}) + if sum(row['bytes'] for row in result.values()) > MAX_EPISODE: + raise ValueError('Episode exceeds its raw byte budget.') + return result + + +def archive(root, inventory, destination, *, prefix=''): + """Stream byte-bound files into a deterministic ZIP, then read every member.""" + root, destination = plain(root), plain(destination) + if destination.exists(): + raise ValueError('Archive exists; existing publication artifacts are immutable.') + if prefix: + relative(prefix) + destination.parent.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(destination.parent).free - sum(r['bytes'] for r in inventory.values()) < DISK_RESERVE: + raise ValueError('Archive preparation would consume the protected disk reserve.') + expected = {} + with zipfile.ZipFile(destination, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=6, allowZip64=True) as output: + for name, record in sorted(inventory.items()): + relative(name) + checked_identity(record) + path = plain(root / name) + member = (prefix + '/' if prefix else '') + name + relative(member) + info = zipfile.ZipInfo(member, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + info._compresslevel = 6 + h, size, tail = sha256(), 0, b'' + with path.open('rb') as source, output.open(info, 'w', force_zip64=True) as target: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode) or before.st_size != record['bytes']: + raise ValueError('Source file identity changed before archiving.') + for chunk in iter(lambda: source.read(1024 * 1024), b''): + size += len(chunk) + if size > record['bytes']: + raise ValueError('Source file grew while archiving.') + if path.suffix in TEXT_SUFFIXES: + scan(tail + chunk) + tail = chunk[-256:] + h.update(chunk) + target.write(chunk) + after = os.fstat(source.fileno()) + if (dict(bytes=size, sha256=h.hexdigest()) != record or any(getattr(before, key) != getattr(after, key) + for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns'))): + raise ValueError('Sealed source bytes changed while archiving.') + expected[member] = record + with zipfile.ZipFile(destination) as verification: + if len(verification.infolist()) != len(expected) or set(verification.namelist()) != set(expected): + raise ValueError('Archive member roster differs from sealed evidence.') + for member, record in expected.items(): + h, size = sha256(), 0 + with verification.open(member) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + size += len(chunk) + h.update(chunk) + if dict(bytes=size, sha256=h.hexdigest()) != record: + raise ValueError('Archive decompression differs from sealed evidence.') + return dict(**identity(destination, maximum=MAX_EPISODE), files=len(expected), + unpacked_bytes=sum(r['bytes'] for r in expected.values()), + inventory_sha256=sha256(canonical(expected)).hexdigest()) + + +def _runtime_members(path, sources, *, wheel): + """Check the actual wheel/source archive contains each registered runtime byte.""" + identities = {} + total = 0 + def source_name(name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name + or PurePosixPath(name).is_absolute() or str(PurePosixPath(name)) != name + or any(part in ('', '.', '..', '.git', '.env', '.venv', 'node_modules') + or part.startswith('.env.') for part in PurePosixPath(name).parts) + or any(ord(c) < 32 for c in name)): + raise ValueError('Unsafe or private source artifact path.') + if wheel: + with zipfile.ZipFile(path) as archive_file: + entries = archive_file.infolist() + if len(entries) > 50000 or len({e.filename for e in entries}) != len(entries): + raise ValueError('Invalid source wheel member roster.') + for entry in entries: + if entry.is_dir(): + source_name(entry.filename.rstrip('/')) + continue + source_name(entry.filename) + if stat.S_ISLNK(entry.external_attr >> 16) or entry.file_size > MAX_FILE: + raise ValueError('Source wheel contains an unsupported member.') + total += entry.file_size + if total > 2 * MAX_EPISODE: + raise ValueError('Expanded source wheel exceeds its bounded size.') + name = PurePosixPath(entry.filename).name + raw = archive_file.read(entry) if PurePosixPath(entry.filename).suffix in TEXT_SUFFIXES or name in sources else None + if raw is not None: + scan(raw) + if entry.filename.startswith('dvidia_training/') and name in sources: + if name in identities: + raise ValueError('Duplicate registered runtime member.') + identities[name] = sha256(raw).hexdigest() + else: + with tarfile.open(path, mode='r:*') as archive_file: + seen = set() + for entry in archive_file: + source_name(entry.name.rstrip('/')) + if entry.name in seen or len(seen) >= 50000: + raise ValueError('Invalid source archive member roster.') + seen.add(entry.name) + if entry.isdir(): + continue + if not entry.isfile() or entry.size > MAX_FILE: + raise ValueError('Source archive contains links or special members.') + total += entry.size + if total > 2 * MAX_EPISODE: + raise ValueError('Expanded source archive exceeds its bounded size.') + name = PurePosixPath(entry.name).name + raw = None + if PurePosixPath(entry.name).suffix in TEXT_SUFFIXES or name in sources: + with archive_file.extractfile(entry) as stream: + raw = stream.read(MAX_FILE + 1) + scan(raw) + if '/src/dvidia_training/' in '/' + entry.name and name in sources: + if name in identities: + raise ValueError('Duplicate registered runtime member.') + identities[name] = sha256(raw).hexdigest() + if identities != sources: + raise ValueError('Wheel/source archive does not contain the exact registered runtime.') + return identities + + +def _copy(source, destination, *, maximum=MAX_FILE): + expected = identity(source, maximum=maximum, secret_scan=Path(source).suffix in TEXT_SUFFIXES) + destination = plain(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + with plain(source).open('rb') as read_stream, destination.open('xb') as write_stream: + shutil.copyfileobj(read_stream, write_stream, 1024 * 1024) + if identity(destination, maximum=maximum) != expected: + raise ValueError('Copied publication bytes differ from reviewed source.') + return expected + + +def verify_study_audit(study, proof_path, receipts, state): + """Bind an optional independent audit to every currently sealed stage.""" + proof = read(proof_path) + if (proof.get('schema') != 'dvidia.camera-action-study-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or any(proof.get(k) is not False for k in + ('physics_executed', 'estimator_executed', 'model_executed', 'source_authenticated', 'physical_robot_ready')) + or proof.get('protocol_sha256') != identity(study / 'protocol.json')['sha256']): + raise ValueError('Full-study audit does not pass or is bound to a different protocol.') + inventory = proof['evidence_inventory'] + if sha256(canonical(inventory)).hexdigest() != proof['evidence_inventory_sha256']: + raise ValueError('Full-study audit inventory digest differs.') + for name, expected in inventory.items(): + if identity(study / relative(name)) != expected: + raise ValueError('Full-study audit evidence changed after auditing.') + expected_rows = {(name, row['id']): row for name, receipt in receipts.items() for row in receipt.get('rows', [])} + observed = {(row['stage'], row['id']): row for row in proof['results']} + expected_stages = {name for name, receipt in receipts.items() if 'rows' in receipt} + if (len(observed) != len(proof['results']) or set(observed) != set(expected_rows) + or proof['attempts_verified'] != len(expected_rows) or set(proof['stages_verified']) != expected_stages): + raise ValueError('Full-study audit omits or invents a sealed stage or attempt.') + if set(proof.get('training_stages_verified', [])) != {name for name in receipts if name.startswith('train-')}: + raise ValueError('Full-study audit omits or invents a sealed fitting stage.') + required = {'protocol.json', 'registration.json', 'fixed-model.json'} + for name in ('report.json', 'freeze.json'): + if (study / name).exists(): required.add(name) + for stage, receipt in receipts.items(): + required.update('stages/' + stage + suffix for suffix in ('.json', '.integrity.json', '.started.json')) + for model in receipt.get('models', {}).values(): required.add(model['path']) + for key, row in expected_rows.items(): + result = observed[key] + for name in ('case', 'controller', 'negative_control', 'status', 'strict_success', 'complete'): + if result.get(name) != row.get(name): + raise ValueError('Full-study audit outcome or registered scene differs.') + if row.get('path'): + for name in ('integrity.json', 'rollout.json', 'recording.json'): + path = str(relative(row['path'])) + '/' + name + if (study / path).exists(): required.add(path) + if not required <= set(inventory): + raise ValueError('Full-study audit lacks essential source evidence identities.') + qualification = proof.get('qualification') + if qualification is not None and qualification.get('qualified') is not state['qualified']: + raise ValueError('Full-study audit and publication qualification differ.') + if state['qualified'] and qualification is None: + raise ValueError('Qualified publication requires independently reconstructed qualification.') + helpers = Path(__file__).resolve().parents[1] / 'benchmarks/camera-action-v01' + if proof.get('scorer_source_sha256') != read(study / 'protocol.json')['source_sha256']['piper_feedback_score.py']: + raise ValueError('Independent audit used a different native scorer.') + for path, field in ((helpers / 'study_audit.py', 'audit_source_sha256'), + (helpers / 'independent_audit.py', 'diagnostic_helper_source_sha256')): + if proof.get(field) != identity(path)['sha256']: + raise ValueError('Independent audit implementation differs from the reviewed proof.') + return proof + + +def verify_audit_source_archive(source_archive, proof): + """The published source snapshot must actually include this audit code.""" + expected = {'benchmarks/camera-action-v01/study_audit.py': proof['audit_source_sha256'], + 'benchmarks/camera-action-v01/independent_audit.py': proof['diagnostic_helper_source_sha256']} + with tarfile.open(source_archive, 'r:*') as archive: + members = archive.getmembers() + for name, expected_hash in expected.items(): + matches = [m for m in members if m.name == name or m.name.endswith('/' + name)] + if len(matches) != 1 or not matches[0].isfile() or matches[0].size > MAX_FILE: + raise ValueError('Reviewed source archive must contain each independent audit implementation exactly once.') + with archive.extractfile(matches[0]) as stream: + digest = sha256() + for chunk in iter(lambda: stream.read(1024*1024), b''): digest.update(chunk) + if digest.hexdigest() != expected_hash: + raise ValueError('Reviewed source archive differs from the independent audit implementation.') + + +def prepare(study, output, *, audit, card, wheel, source_archive, source_commit, phase_coverage=None, study_audit=None, hardware=None): + from dvidia_training import camera_action_study as workflow + study, output = plain(study), plain(output) + if output.exists() or output == study or output.is_relative_to(study): + raise ValueError('Use a fresh publication directory outside the study.') + if (study / 'worker.lock').exists(): + raise ValueError('A running or interrupted study worker must be resolved before packaging.') + if not re.fullmatch('[a-f0-9]{40}', source_commit): + raise ValueError('An exact source commit is required.') + state = workflow.inspect(study) + if state['status'] not in ('blocked', 'complete'): + raise ValueError('Publish only a stopped blocked or completed study, not a partial active pipeline.') + if state['qualified'] and study_audit is None: + raise ValueError('A qualified-skill publication requires a full-study independent audit.') + if hardware is not None: + identity(hardware, maximum=MAX_HARDWARE_BYTES, secret_scan=True) + validate_hardware(read(hardware)) + protocol = read(study / 'protocol.json') + proof = read(audit) + if (proof.get('schema') != 'dvidia.camera-action-diagnostic-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or proof.get('attempts_verified') != 4 + or proof.get('physics_executed') is not False or proof.get('estimator_executed') is not False): + raise ValueError('The explicit passed four-episode diagnostic audit is required.') + diagnostic = read(study / 'stages/diagnostic.json') + if {r['case_id'] for r in proof['results']} != {r['case']['id'] for r in diagnostic['rows']}: + raise ValueError('Diagnostic audit cases differ from this study.') + for result in proof['results']: + row = next(r for r in diagnostic['rows'] if r['case']['id'] == result['case_id']) + if row['strict_success'] is not result['strict_success']: + raise ValueError('Diagnostic audit and study outcomes differ.') + if phase_coverage is not None: + phase_proof = read(phase_coverage) + if (phase_proof.get('schema') != 'dvidia.camera-action-phase-coverage/v1' + or phase_proof.get('protocol_sha256') != identity(study / 'protocol.json')['sha256'] + or phase_proof.get('diagnostic_audit_sha256') != identity(audit)['sha256']): + raise ValueError('Phase coverage proof is not bound to this protocol and diagnostic audit.') + _runtime_members(wheel, protocol['source_sha256'], wheel=True) + _runtime_members(source_archive, protocol['source_sha256'], wheel=False) + with tarfile.open(source_archive, 'r:*') as source_snapshot: + if source_snapshot.pax_headers.get('comment') != source_commit: + raise ValueError('Source archive must carry the exact git-archive commit in its PAX comment.') + metadata = {} + allowed_root = {'protocol.json', 'registration.json', 'fixed-model.json', 'freeze.json', 'report.json'} + episode_rows = [] + models = [] + for path in files(study): + relative_path = path.relative_to(study).as_posix() + parts = PurePosixPath(relative_path).parts + if parts[0] == 'episodes': + continue + allowed = (len(parts) == 1 and relative_path in allowed_root + or len(parts) == 2 and parts[0] == 'stages' and path.suffix == '.json' + or len(parts) == 4 and parts[0] == 'models' and parts[1] in ('initial', 'corrected') + and parts[2] in ('full', 'blind') and parts[3] in ('model.json', 'receipt.json', 'started.json')) + if not allowed: + raise ValueError('Unexpected study artifact; review it before publication.') + metadata[relative_path] = identity(path, secret_scan=True) + if parts[0] == 'models' and parts[-1] == 'model.json': + training = workflow._require(study, 'train-' + parts[1]) + if training['models'][parts[2]]['model_sha256'] != metadata[relative_path]['sha256']: + raise ValueError('Model does not match completed training evidence.') + models.append(relative_path) + all_expected = {}; receipts = {} + for stage in workflow.STAGES: + receipt = workflow._stage(study, stage) + if receipt is None: + continue + receipts[stage] = receipt + for row in receipt.get('rows', []): + if 'path' not in row: + continue + path = relative(row['path']) + if len(path.parts) != 3 or path.parts[0] != 'episodes' or path.parts[1] != stage or path.parts[2] != row['id']: + raise ValueError('Episode path differs from the sealed stage identity.') + directory = study / path + if row.get('receipt_sha256') != identity(directory / 'rollout.json')['sha256'] or row.get('integrity_sha256') != identity(directory / 'integrity.json')['sha256']: + raise ValueError('Episode bytes differ from the sealed stage receipt.') + inventory = episode_inventory(directory) + all_expected[str(path)] = inventory + episode_rows.append(dict(stage=stage, id=row['id'], group_id=row['case']['group_id'], + split=row['case']['split'], controller=row['controller'], negative_control=row['negative_control'], + status=row['status'], strict_success=row['strict_success'], source_directory=str(path), + integrity_sha256=row['integrity_sha256'], receipt_sha256=row['receipt_sha256'])) + observed_episode_dirs = {p.parent.relative_to(study).as_posix() for p in (study / 'episodes').glob('*/*/started.json')} + if observed_episode_dirs != set(all_expected): + raise ValueError('Unsealed, unregistered or missing episode attempts cannot be silently omitted.') + if study_audit is not None: + full_proof = verify_study_audit(study, study_audit, receipts, state) + verify_audit_source_archive(source_archive, full_proof) + raw_bytes = sum(v['bytes'] for inventory in all_expected.values() for v in inventory.values()) + if raw_bytes > MAX_TOTAL: + raise ValueError('Study exceeds the registered raw evidence budget.') + output.mkdir(parents=True) + inventory = {} + repo = Path(__file__).resolve().parents[1] + for source, destination in ((card, 'README.md'), (repo / 'LICENSE', 'LICENSE'), + (audit, 'independent-audit.json'), (study / 'protocol.json', 'protocol.json'), + (study / 'registration.json', 'registration.json')): + inventory[destination] = _copy(source, output / destination) + if phase_coverage is not None: + inventory['diagnostic-phase-coverage.json'] = _copy(phase_coverage, output / 'diagnostic-phase-coverage.json') + if study_audit is not None: + inventory['full-study-audit.json'] = _copy(study_audit, output / 'full-study-audit.json') + if hardware is not None: + inventory['hardware.json'] = _copy(hardware, output / 'hardware.json', maximum=MAX_HARDWARE_BYTES) + write(output / 'report.json', state['report']) + inventory['report.json'] = identity(output / 'report.json') + for source, label in ((wheel, 'wheel'), (source_archive, 'source')): + source = plain(source) + if source.name.startswith('.') or len(PurePosixPath(source.name).parts) != 1: + raise ValueError('Unexpected source artifact filename.') + name = 'source/' + source.name + inventory[name] = _copy(source, output / name, maximum=MAX_SOURCE) + metadata_archive = archive(study, metadata, output / 'study-metadata.zip') + inventory['study-metadata.zip'] = {k: metadata_archive[k] for k in ('bytes', 'sha256')} + for row in episode_rows: + name = row['source_directory'] + '.zip' + sealed = archive(study / row['source_directory'], all_expected[row['source_directory']], + output / name, prefix=row['source_directory']) + row['archive'] = name + row['archive_receipt'] = sealed + inventory[name] = {k: sealed[k] for k in ('bytes', 'sha256')} + print(json.dumps(dict(stage='archive_verified', episode=row['id'], bytes=sealed['bytes'])), flush=True) + for name in models: + inventory[name] = _copy(study / name, output / name) + value = dict(schema=SCHEMA, repo_id='Dvidia/piper-camera-action-v01', repo_type='dataset', + source_commit=source_commit, registered_runtime_sha256=protocol['source_sha256'], + status=state['report']['status'], pipeline_status=state['status'], + qualification=state['qualification'], qualified=state['qualified'], + physical_robot_ready=False, original_footage_included=False, + recorded_evidence_origin='Synthetic MuJoCo RGB-D, encoders, controls and native scorer traces.', + independent_audit_scope=('All sealed stages are independently byte-, authority-, clock- and numerically audited in full-study-audit.json. Missing stages remain unrun; no physics, estimator or policy was re-executed.' if study_audit is not None else + 'Four diagnostic episodes only. Other shards are checked against sealed manifests; this is not a full-study independent audit.'), + raw_evidence_bytes=raw_bytes, episode_count=len(episode_rows), episodes=episode_rows, + trained_model_files=models, files=inventory, + hardware_context_file='hardware.json' if hardware is not None else None, + reconstruction='Extract study-metadata.zip and every episodes/**/*.zip into one new directory. Compare SHA256SUMS first. Each recording retains its original integrity.json.', + archive_format='Deterministic ZIP64, sorted members, 1980 timestamp, regular 0644 files, DEFLATE level 6; all members read back byte-for-byte.') + write(output / 'manifest.json', value) + inventory_with_manifest = dict(inventory, **{'manifest.json': identity(output / 'manifest.json')}) + sums = ''.join(f'{record["sha256"]} {name}\n' for name, record in sorted(inventory_with_manifest.items())) + with (output / 'SHA256SUMS').open('x') as stream: + stream.write(sums) + return dict(status='complete', output=str(output), episode_count=len(episode_rows), raw_evidence_bytes=raw_bytes, + public_bytes=sum(v['bytes'] for v in inventory_with_manifest.values()), trained_models=len(models), + qualification=state['qualification'], manifest_sha256=identity(output / 'manifest.json')['sha256']) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--audit', required=True) + parser.add_argument('--card', required=True) + parser.add_argument('--wheel', required=True) + parser.add_argument('--source-archive', required=True) + parser.add_argument('--source-commit', required=True) + parser.add_argument('--phase-coverage', help='Optional diagnostic-only phase coverage proof bound to the same protocol and audit.') + parser.add_argument('--study-audit', help='Optional independent audit of all sealed study stages, byte-bound to this stopped study.') + parser.add_argument('--hardware', help='Optional bounded public hardware/software and timing-scope JSON; no hostname or serial fields.') + args = parser.parse_args(argv) + try: + print(json.dumps(prepare(args.study, args.output, audit=args.audit, card=args.card, + wheel=args.wheel, source_archive=args.source_archive, source_commit=args.source_commit, + phase_coverage=args.phase_coverage, study_audit=args.study_audit, hardware=args.hardware), indent=2)) + return 0 + except (ValueError, OSError, KeyError, TypeError, zipfile.BadZipFile, tarfile.TarError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc)))) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/test_prepare_camera_action_v01.py b/publishing/test_prepare_camera_action_v01.py new file mode 100644 index 0000000..61a8848 --- /dev/null +++ b/publishing/test_prepare_camera_action_v01.py @@ -0,0 +1,232 @@ +from hashlib import sha256 +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +import prepare_camera_action_v01 as package + + +class EvidencePackageTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.episode = self.root / 'episode' + self.episode.mkdir() + package.write(self.episode / 'started.json', {'status': 'started'}) + package.write(self.episode / 'rollout.json', {'status': 'complete', 'synthetic': True}) + inventory = {p.name: package.identity(p) for p in self.episode.iterdir()} + package.write(self.episode / 'integrity.json', + {'schema': 'dvidia.camera-action-rollout/v2/integrity', 'files': inventory}) + + def test_deterministic_archives_restore_exact_original_integrity_inventory(self): + inventory = package.episode_inventory(self.episode) + first = package.archive(self.episode, inventory, self.root / 'first.zip', prefix='episodes/diagnostic/example') + second = package.archive(self.episode, inventory, self.root / 'second.zip', prefix='episodes/diagnostic/example') + self.assertEqual(first, second) + self.assertEqual((self.root / 'first.zip').read_bytes(), (self.root / 'second.zip').read_bytes()) + with zipfile.ZipFile(self.root / 'first.zip') as archive: + for name, expected in inventory.items(): + raw = archive.read('episodes/diagnostic/example/' + name) + self.assertEqual({'bytes': len(raw), 'sha256': sha256(raw).hexdigest()}, expected) + self.assertEqual(archive.getinfo('episodes/diagnostic/example/' + name).date_time, (1980, 1, 1, 0, 0, 0)) + + def test_changed_sealed_source_refuses_during_streaming_copy(self): + inventory = package.episode_inventory(self.episode) + (self.episode / 'rollout.json').write_text('{"status":"different"}\n') + with self.assertRaisesRegex(ValueError, 'identity changed|bytes changed'): + package.archive(self.episode, inventory, self.root / 'changed.zip') + + def test_missing_extra_symlink_and_unsealed_attempt_are_refused(self): + (self.episode / 'extra.json').write_text('{}') + with self.assertRaisesRegex(ValueError, 'unsealed, missing or extra'): + package.episode_inventory(self.episode) + (self.episode / 'extra.json').unlink() + (self.episode / 'rollout.json').unlink() + (self.episode / 'rollout.json').symlink_to(self.episode / 'started.json') + with self.assertRaisesRegex(ValueError, 'symbolic links'): + package.episode_inventory(self.episode) + (self.episode / 'rollout.json').unlink() + (self.episode / 'integrity.json').unlink() + with self.assertRaises(OSError): + package.episode_inventory(self.episode) + + def test_path_escape_and_secret_shaped_json_are_refused(self): + for name in ('../secret', '/tmp/secret', 'frames/../../secret', './frame', 'frame\\secret'): + with self.subTest(name=name), self.assertRaises(ValueError): + package.relative(name) + (self.episode / 'rollout.json').write_text('"' + 'hf_' + 'A' * 32 + '"') + inventory = {p.name: package.identity(p) for p in self.episode.iterdir()} + with self.assertRaisesRegex(ValueError, 'Credential-shaped'): + package.archive(self.episode, inventory, self.root / 'secret.zip') + + def test_source_wheel_and_tar_bind_registered_runtime_bytes(self): + raw = b'"""Synthetic source fixture."""\n' + sources = {'camera_action_study.py': sha256(raw).hexdigest()} + wheel = self.root / 'fixture.whl' + with zipfile.ZipFile(wheel, 'w') as archive: + archive.writestr('dvidia_training/camera_action_study.py', raw) + source = self.root / 'fixture.tar.gz' + with tarfile.open(source, 'w:gz') as archive: + entry = tarfile.TarInfo('fixture/src/dvidia_training/camera_action_study.py') + entry.size = len(raw) + archive.addfile(entry, io.BytesIO(raw)) + entry = tarfile.TarInfo('fixture/.gitignore') + entry.size = 3 + archive.addfile(entry, io.BytesIO(b'foo')) + self.assertEqual(package._runtime_members(wheel, sources, wheel=True), sources) + self.assertEqual(package._runtime_members(source, sources, wheel=False), sources) + with self.assertRaisesRegex(ValueError, 'exact registered runtime'): + package._runtime_members(wheel, {'camera_action_study.py': '0' * 64}, wheel=True) + + def test_source_archive_cannot_include_private_environment_or_links(self): + source = self.root / 'private.tar.gz' + with tarfile.open(source, 'w:gz') as archive: + entry = tarfile.TarInfo('fixture/.env') + entry.size = 1 + archive.addfile(entry, io.BytesIO(b'x')) + with self.assertRaisesRegex(ValueError, 'private source'): + package._runtime_members(source, {}, wheel=False) + source = self.root / 'linked.tar.gz' + with tarfile.open(source, 'w:gz') as archive: + entry = tarfile.TarInfo('fixture/redirect') + entry.type = tarfile.SYMTYPE + entry.linkname = '/tmp/other' + archive.addfile(entry) + with self.assertRaisesRegex(ValueError, 'links or special'): + package._runtime_members(source, {}, wheel=False) + + def test_source_archive_must_include_exact_audit_implementation(self): + source=self.root/'audit-source.tar.gz'; proof={} + with tarfile.open(source,'w:gz') as archive: + for name,field in (('study_audit.py','audit_source_sha256'),('independent_audit.py','diagnostic_helper_source_sha256')): + raw=('synthetic '+name).encode(); proof[field]=sha256(raw).hexdigest() + info=tarfile.TarInfo('fixture/benchmarks/camera-action-v01/'+name); info.size=len(raw) + archive.addfile(info,io.BytesIO(raw)) + package.verify_audit_source_archive(source,proof) + proof['audit_source_sha256']='0'*64 + with self.assertRaisesRegex(ValueError,'differs from the independent'): + package.verify_audit_source_archive(source,proof) + + def test_running_worker_is_refused_before_inspection_or_archiving(self): + study = self.root / 'study' + study.mkdir() + (study / 'worker.lock').write_text('{}') + with patch('dvidia_training.camera_action_study.inspect') as inspect: + with self.assertRaisesRegex(ValueError, 'worker'): + package.prepare(study, self.root / 'payload', audit='unused', card='unused', + wheel='unused', source_archive='unused', source_commit='a' * 40) + inspect.assert_not_called() + self.assertFalse((self.root / 'payload').exists()) + + def test_full_study_audit_requires_every_current_stage_and_unchanged_evidence(self): + from copy import deepcopy + study = self.root / 'study'; study.mkdir() + package.write(study/'protocol.json', dict(source_sha256={'piper_feedback_score.py': 'e'*64})) + for name in ('registration.json', 'fixed-model.json', 'report.json'): + package.write(study/name, {'fixture': name}) + row = dict(id='unattempted', case={'id': 'scene'}, controller='teacher', negative_control=None, + status='unattempted', complete=False, strict_success=False) + receipt = dict(rows=[row]) + for suffix in ('.json', '.integrity.json', '.started.json'): + package.write(study/('stages/diagnostic'+suffix), receipt) + inventory = {p.relative_to(study).as_posix(): package.identity(p) for p in study.rglob('*') if p.is_file()} + helpers = Path(package.__file__).resolve().parents[1]/'benchmarks/camera-action-v01' + proof = dict(schema='dvidia.camera-action-study-audit/v1', passed=True, errors=[], + protocol_sha256=package.identity(study/'protocol.json')['sha256'], + physics_executed=False, estimator_executed=False, model_executed=False, source_authenticated=False, physical_robot_ready=False, + evidence_inventory=inventory, evidence_inventory_sha256=sha256(package.canonical(inventory)).hexdigest(), + stages_verified=['diagnostic'], attempts_verified=1, results=[dict(**row, stage='diagnostic')], qualification=None, + audit_source_sha256=package.identity(helpers/'study_audit.py')['sha256'], + diagnostic_helper_source_sha256=package.identity(helpers/'independent_audit.py')['sha256'], scorer_source_sha256='e'*64) + proof_path=self.root/'full-audit.json'; package.write(proof_path, proof) + state={'qualified': False} + package.verify_study_audit(study, proof_path, {'diagnostic': receipt}, state) + with self.assertRaisesRegex(ValueError, 'omits or invents'): + package.verify_study_audit(study, proof_path, {'diagnostic': receipt, 'demonstrations': receipt}, state) + with self.assertRaisesRegex(ValueError, 'requires independently'): + package.verify_study_audit(study, proof_path, {'diagnostic': receipt}, {'qualified': True}) + changed = deepcopy(proof); changed['results'][0]['strict_success']=True + proof_path.write_bytes(package.canonical(changed)) + with self.assertRaisesRegex(ValueError, 'outcome or registered scene'): + package.verify_study_audit(study, proof_path, {'diagnostic': receipt}, state) + proof_path.write_bytes(package.canonical(proof)); (study/'fixed-model.json').write_text('{}') + with self.assertRaisesRegex(ValueError, 'changed after auditing'): + package.verify_study_audit(study, proof_path, {'diagnostic': receipt}, state) + + def test_qualified_publication_requires_full_study_audit_before_reading_payload(self): + study=self.root/'study'; study.mkdir() + with patch('dvidia_training.camera_action_study.inspect', return_value={'status':'complete', 'qualified':True}): + with self.assertRaisesRegex(ValueError, 'requires a full-study'): + package.prepare(study,self.root/'payload',audit='unused',card='unused',wheel='unused', + source_archive='unused',source_commit='a'*40) + self.assertFalse((self.root/'payload').exists()) + + def test_blocked_study_packages_all_sealed_attempts_without_model_artifacts(self): + import shutil + from test_camera_action_hardware import fixture + from dvidia_training import camera_action_study as workflow + study = self.root / 'study' + study.mkdir() + source_bytes = b'"""Synthetic runtime fixture."""\n' + sources = {'camera_action_study.py': sha256(source_bytes).hexdigest()} + package.write(study / 'protocol.json', {'source_sha256': sources}) + package.write(study / 'registration.json', {'fixture': 'registration'}) + package.write(study / 'fixed-model.json', {'fixture': 'comparator'}) + rows = [] + for index in range(4): + identity = f'diagnostic-development-{index:02}-goal0-teacher' + directory = study / 'episodes/diagnostic' / identity + shutil.copytree(self.episode, directory) + rows.append(dict(id=identity, case=dict(id=identity, group_id=f'development-{index:02}', + split='development'), path=directory.relative_to(study).as_posix(), + controller='teacher', negative_control=None, status='complete', strict_success=False, + receipt_sha256=package.identity(directory / 'rollout.json')['sha256'], + integrity_sha256=package.identity(directory / 'integrity.json')['sha256'])) + diagnostic = dict(status='blocked', rows=rows) + package.write(study / 'stages/diagnostic.json', diagnostic) + proof = dict(schema='dvidia.camera-action-diagnostic-audit/v1', passed=True, + errors=[], attempts_verified=4, physics_executed=False, estimator_executed=False, + results=[dict(case_id=r['case']['id'], strict_success=False) for r in rows]) + audit = self.root / 'audit.json' + package.write(audit, proof) + card = self.root / 'README.md' + card.write_text('Synthetic blocked-study fixture; no trained policy.\n') + wheel = self.root / 'fixture.whl' + with zipfile.ZipFile(wheel, 'w') as archive: + archive.writestr('dvidia_training/camera_action_study.py', source_bytes) + source = self.root / 'fixture.tar.gz' + commit = 'a' * 40 + with tarfile.open(source, 'w:gz', format=tarfile.PAX_FORMAT, pax_headers={'comment': commit}) as archive: + entry = tarfile.TarInfo('fixture/src/dvidia_training/camera_action_study.py') + entry.size = len(source_bytes) + archive.addfile(entry, io.BytesIO(source_bytes)) + state = dict(status='blocked', qualification='research_candidate', qualified=False, + report=dict(status='blocked', qualification='research_candidate', qualified=False, physical_robot_ready=False)) + output = self.root / 'payload' + hardware=self.root/'hardware.json';package.write(hardware,fixture()) + with patch.object(workflow, 'inspect', return_value=state), \ + patch.object(workflow, '_stage', side_effect=lambda root, name: diagnostic if name == 'diagnostic' else None): + result = package.prepare(study, output, audit=audit, card=card, wheel=wheel, + source_archive=source, source_commit=commit, hardware=hardware) + self.assertEqual(result['episode_count'], 4) + self.assertEqual(result['trained_models'], 0) + self.assertFalse((output / 'models').exists()) + manifest = package.read(output / 'manifest.json') + self.assertIn('Four diagnostic episodes only', manifest['independent_audit_scope']) + self.assertFalse(manifest['qualified']) + self.assertEqual(manifest['hardware_context_file'],'hardware.json') + self.assertEqual((output/'hardware.json').read_bytes(),hardware.read_bytes()) + self.assertEqual(len(list((output / 'episodes').rglob('*.zip'))), 4) + import publish_camera_action_v01 + inventory = publish_camera_action_v01.files(output) + self.assertEqual(len(inventory), len(manifest['files']) + 2) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/dvidia_training/camera_action_skill.py b/src/dvidia_training/camera_action_skill.py index 167ac5b..a2f7498 100644 --- a/src/dvidia_training/camera_action_skill.py +++ b/src/dvidia_training/camera_action_skill.py @@ -111,21 +111,26 @@ def run(directory, assets, output, *, start_x=.30, start_y=-.03, goal=0): y = _point(start_y, 'start_y', *TRAINED_SCOPE['start_y_m']) if type(goal) is not int or goal not in (0, 1): raise ValueError('Select known goal 0 or 1.') + link = Path(output).with_name(Path(output).name+'-installation.json') + if link.exists() or link.is_symlink(): + raise ValueError('Use a fresh execution path and installation link.') raw = _read(Path(directory)/'model.json', MAX_MODEL_BYTES) if sha256(raw).hexdigest() != record['compatibility']['model_sha256']: raise ValueError('Model changed before execution.') policy = model_api.ModelPolicy(model_api.validate_model(raw)) case = dict(id=f'installed-camera-goal{goal}', group_id='installed-camera-example', seed=311499, + policy_artifact_sha256=record['compatibility']['model_sha256'], object_position_m=[x, y, .082], goal_position_m=[*TRAINED_SCOPE['known_goals_xy_m'][goal], .08]) report = run_rollout(assets, output, case=case, mode='student', policy=policy) result = dict(success=report['strict_success'], reason=report['reason'], status=report['status'], model_sha256=record['compatibility']['model_sha256'], installation_sha256=record['installation_sha256'], + rollout_sha256=sha256((Path(output)/'rollout.json').read_bytes()).hexdigest(), + integrity_sha256=sha256((Path(output)/'integrity.json').read_bytes()).hexdigest(), physical_robot_ready=False, report=report) # The raw rollout stays sealed. Its installation link is adjacent, not an # unregistered file added to its evidence inventory. - link = Path(output).with_name(Path(output).name+'-installation.json') with link.open('xb') as stream: stream.write(canonical({key: value for key, value in result.items() if key != 'report'})+b'\n') return result diff --git a/src/dvidia_training/camera_action_studio.html b/src/dvidia_training/camera_action_studio.html index bfc43cd..d2c2ea6 100644 --- a/src/dvidia_training/camera_action_studio.html +++ b/src/dvidia_training/camera_action_studio.html @@ -36,9 +36,9 @@ function failure(message){clear();text('error',message);$('error').hidden=false;} async function request(path,options={}){const response=await fetch(path,options);let value;try{value=await response.json();}catch(_){throw Error('The local server did not return a readable result.');}if(!response.ok)throw Error(value.error||'The request failed.');return value;} const quote=value=>"'"+String(value).replaceAll("'","'\"'\"'")+"'"; -function show(summary){const state=summary.inspection||{},report=state.report||{};$('results').hidden=false;$('empty').hidden=true;text('result-title',state.status==='blocked'?'A prerequisite needs work':state.status==='complete'?(report.qualified?'Bounded simulation gates passed':'Research candidate — benchmark complete'):'Study checked');text('result-description',state.status==='blocked'?'The evidence is preserved. Training has stopped at the failed prerequisite.':state.status==='complete'?'Review the complete task results and failure conditions before using this candidate.':'The registered stages will run in order using the existing experiment budget.');const tbody=$('stages');tbody.replaceChildren();for(const[name,row]of Object.entries(state.stage_summaries||{})){const tr=document.createElement('tr');for(const value of [name,row?(row.status||'recorded'):'Not run',row&&row.strict_successes!==null&&row.strict_successes!==undefined?String(row.strict_successes)+(row.scheduled!==null&&row.scheduled!==undefined?' / '+row.scheduled:''):'—']){const td=document.createElement('td');td.textContent=value;tr.append(td);}tbody.append(tr);}const blocked=state.blocked_stages||[];$('blocker').hidden=!blocked.length;text('blocker',blocked.length?'Blocked stages: '+blocked.join(', '):'');text('record',JSON.stringify(state,null,2));const commands=['dvidia-train robot inspect --study '+quote($('study').value.trim())];for(const action of state.next_actions||[])commands.push('dvidia-train robot '+action.command+' --study '+quote($('study').value.trim())+' --stage '+quote(action.stage)+(action.requires_assets?' --assets '+quote($('assets').value.trim()):''));text('terminal',commands.join('\n'));text('install-note',summary.install_available?'Installs locally for this PiPER simulation; it sends no physical commands.':'A completed benchmark and selected model are required.');if(summary.installation){text('installation','Installed at '+summary.installation.path);$('installation').hidden=false;}} +function show(summary){const state=summary.inspection||{},report=state.report||{};$('results').hidden=false;$('empty').hidden=true;text('result-title',state.status==='blocked'?'A prerequisite needs work':state.status==='complete'?(report.qualified?'Bounded simulation gates passed':'Research candidate — benchmark complete'):'Study checked');text('result-description',state.status==='blocked'?'The evidence is preserved. Training has stopped at the failed prerequisite.':state.status==='complete'?'Review the complete task results and failure conditions before using this candidate.':'The registered stages will run in order using the existing experiment budget.');const tbody=$('stages');tbody.replaceChildren();for(const[name,row]of Object.entries(state.stage_summaries||{})){const tr=document.createElement('tr');for(const value of [name,row?(row.status||'recorded'):'Pending receipt',row&&row.strict_successes!==null&&row.strict_successes!==undefined?String(row.strict_successes)+(row.scheduled!==null&&row.scheduled!==undefined?' / '+row.scheduled:''):'—']){const td=document.createElement('td');td.textContent=value;tr.append(td);}tbody.append(tr);}const blocked=state.blocked_stages||[];$('blocker').hidden=!blocked.length;text('blocker',blocked.length?'Blocked stages: '+blocked.join(', '):'');text('record',JSON.stringify(state,null,2));const commands=['dvidia-train robot inspect --study '+quote($('study').value.trim())];for(const action of state.next_actions||[])commands.push('dvidia-train robot '+action.command+' --study '+quote($('study').value.trim())+' --stage '+quote(action.stage)+(action.requires_assets?' --assets '+quote($('assets').value.trim()):''));text('terminal',commands.join('\n'));text('install-note',summary.install_available?'Installs locally for this PiPER simulation; it sends no physical commands.':'A completed benchmark and selected model are required.');if(summary.installation){text('installation','Installed at '+summary.installation.path);$('installation').hidden=false;}} function schedule(){clearTimeout(timer);timer=setTimeout(poll,1000);} -async function poll(){try{const state=await request('/api/robot/state');connected=true;busy=state.busy;canRun=state.can_run;canInstall=state.can_install;const job=state.job;if(job){if(!$('study').value&&!$('assets').value){$('study').value=job.study;$('assets').value=job.assets;}if(state.checked){$('study').value=job.study;$('assets').value=job.assets;checkedBinding=JSON.stringify({study:job.study,assets:job.assets});}const progress=job.progress;text('status',state.status==='running'?((progress&&progress.message)||'Running locally…')+' '+Math.floor(job.elapsed_seconds)+' s elapsed.':state.status==='failed'?'The job failed. Its evidence is preserved.':state.status==='blocked'?'Stopped at a failed prerequisite.':state.status==='installed'?'Simulation candidate installed.':state.status==='complete'?'Benchmark complete.':'Inputs checked.');if(state.summary&&lastJob!==job.id+state.status){show(state.summary);lastJob=job.id+state.status;}if(state.error)failure(state.error);}else text('status',busy?'Another local training job is running.':'Choose a study to begin.');text('check-note',state.checked?'Checked against the registered protocol and assets.':'Choose both folders to begin.');text('run-note',canRun?'Ready to run the remaining registered stages.':'Check the study before starting.');controls();if(busy)schedule();}catch(exc){connected=false;busy=false;failure(exc.message);controls();}} +async function poll(){try{const state=await request('/api/robot/state');connected=true;busy=state.busy;canRun=state.can_run;canInstall=state.can_install;const job=state.job;if(job){if(!$('study').value&&!$('assets').value){$('study').value=job.study;$('assets').value=job.assets;}if(state.checked){$('study').value=job.study;$('assets').value=job.assets;checkedBinding=JSON.stringify({study:job.study,assets:job.assets});}const progress=job.progress;text('status',state.status==='running'?((progress&&progress.message)||'Running locally…')+' '+Math.floor(job.elapsed_seconds)+' s elapsed.':state.status==='failed'?'The job failed. Its evidence is preserved.':state.status==='blocked'?'Stopped at a failed prerequisite.':state.status==='installed'?'Simulation candidate installed.':state.status==='complete'?'Benchmark complete.':'Inputs checked.');if(state.summary&&lastJob!==job.id+state.status){show(state.summary);lastJob=job.id+state.status;}if(state.error)failure(state.error);}else text('status',busy?'Another local training job is running.':'Choose a study to begin.');if(state.study_locked&&state.status!=='running'){text('status','Study running in Terminal');text('run-note','An existing worker lock is present. Recheck the inputs after that run ends.');}else if(state.recheck_required){text('status','Terminal worker lock cleared. Check inputs again to refresh the study.');text('run-note','Recheck before starting another stage.');}else{text('run-note',canRun?'Ready to run the remaining registered stages.':'Check the study before starting.');}text('check-note',state.checked?'Checked against the registered protocol and assets.':'Choose both folders to begin.');controls();if(busy)schedule();}catch(exc){connected=false;busy=false;failure(exc.message);controls();}} async function start(kind){$('error').hidden=true;const input={study:$('study').value.trim(),assets:$('assets').value.trim()};if(kind==='check')clear();busy=true;canInstall=false;controls();text('status',kind==='check'?'Checking study and assets…':kind==='install'?'Installing locally…':'Starting the registered workflow…');try{await request('/api/robot/'+kind,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(input)});await poll();}catch(exc){busy=false;failure(exc.message);controls();}} for(const id of ['study','assets'])$(id).addEventListener('input',()=>{clear();$('error').hidden=true;});for(const id of ['check','run','install'])$(id).addEventListener('click',()=>start(id));poll();})(); diff --git a/src/dvidia_training/camera_action_studio.py b/src/dvidia_training/camera_action_studio.py index 6accad7..1665b16 100644 --- a/src/dvidia_training/camera_action_studio.py +++ b/src/dvidia_training/camera_action_studio.py @@ -15,6 +15,22 @@ SCHEMA = 'dvidia.camera-action-studio/v1' +def _worker_lock(study): + """Read only the study's small lock record; never infer liveness or unlink it.""" + path = Path(study)/'worker.lock' + if not path.exists() and not path.is_symlink(): + return dict(present=False) + result = dict(present=True) + try: + if not path.is_symlink() and path.is_file() and path.stat().st_size <= 1024: + value = json.loads(path.read_text()) + if type(value) is dict and isinstance(value.get('operation'), str): + result['operation'] = value['operation'][:80] + except (OSError, ValueError): + pass + return result + + def _local_path(value, label): if not isinstance(value, str) or not value.strip() or len(value) > 8192 or any(ord(c) < 32 for c in value): raise ValueError('Supply a local '+label+' folder.') @@ -40,8 +56,10 @@ def check_study(study, assets): stage = state.get('report', {}).get('selected_stage') model = root/'models'/str(stage)/'full'/'model.json' if stage in ('initial', 'corrected') else None install_available = bool(state['status'] == 'complete' and model is not None and model.is_file() and not model.is_symlink()) + worker_lock = _worker_lock(root) return dict(schema=SCHEMA, binding=binding, inspection=state, install_available=install_available, - selected_model=str(model) if install_available else None, physical_robot_ready=False) + selected_model=str(model) if install_available else None, study_locked=worker_lock['present'], + study_worker_lock=worker_lock, physical_robot_ready=False) def _write_progress(output, status, message, **extra): @@ -67,6 +85,8 @@ def run_worker(kind, study, assets, output, expected_binding=None, installation= checked = check_study(study, assets) if expected_binding is not None and checked['binding'] != expected_binding: raise ValueError('The checked study, protocol or assets changed. Check the inputs again.') + if kind != 'check' and _worker_lock(study)['present']: + raise ValueError('This study has an existing worker lock. Wait for its Terminal run, then check the inputs again.') if kind == 'check': status = 'blocked' if checked['inspection']['status'] == 'blocked' else 'checked' elif kind == 'run': @@ -130,6 +150,7 @@ def __init__(self, server, *, worker=None, checker=None): self.worker = worker or run_worker self.checker = checker or check_study self.job = None; self.checked = None + self.external_lock_seen = False def refresh(self): job = self.job @@ -148,6 +169,8 @@ def refresh(self): else: self.checked = verified value.update(verified) + if job['kind'] == 'check': + self.external_lock_seen = _worker_lock(job['study'])['present'] job.update(status=value['status'], summary=value, error=None, finished=time.monotonic()) except Exception as exc: self.checked = None @@ -169,6 +192,8 @@ def submit(self, kind, study, assets): if self.checked is None: raise ValueError('Check the paired demonstrations and assets before continuing.') verified = self.checker(study, assets) + if self.external_lock_seen or _worker_lock(study)['present']: + raise ValueError('Study running in Terminal or awaiting a fresh check. Recheck after its worker lock clears.') if verified['binding'] != self.checked['binding']: raise ValueError('The checked protocol or assets changed. Check the inputs again.') if kind == 'run' and verified['inspection']['status'] != 'ready': @@ -178,6 +203,7 @@ def submit(self, kind, study, assets): binding = verified['binding'] elif kind == 'check': self.checked = None + self.external_lock_seen = False else: raise ValueError('Unknown robot studio action.') except (OSError, ValueError, RuntimeError): @@ -209,6 +235,7 @@ def state(self): footage_busy = self.server.job is not None and self.server.job['status'] == 'running' result = dict(schema=SCHEMA, status='idle', busy=footage_busy, job=None, summary=None, error=None, checked=False, can_run=False, can_install=False, + study_locked=False, recheck_required=False, study_worker_lock=dict(present=False), physical_robot_ready=False) if self.job is not None: job = self.job @@ -224,7 +251,14 @@ def state(self): elapsed_seconds=(job['finished'] or time.monotonic())-job['started'], progress=progress, study=job['study'], assets=job['assets'])) if self.checked is not None: + worker_lock = _worker_lock(self.checked['binding']['study']) + if worker_lock['present'] and not (self.job and self.job['status']=='running'): + self.external_lock_seen = True result.update(checked=True, - can_run=not result['busy'] and self.checked['inspection']['status']=='ready', - can_install=not result['busy'] and self.checked['install_available']) + study_locked=worker_lock['present'], study_worker_lock=worker_lock, + recheck_required=self.external_lock_seen, + can_run=not result['busy'] and not self.external_lock_seen and not worker_lock['present'] + and self.checked['inspection']['status']=='ready', + can_install=not result['busy'] and not self.external_lock_seen and not worker_lock['present'] + and self.checked['install_available']) return result diff --git a/tests/test_camera_action_skill.py b/tests/test_camera_action_skill.py index a586e64..7ee1e82 100644 --- a/tests/test_camera_action_skill.py +++ b/tests/test_camera_action_skill.py @@ -1,5 +1,6 @@ """Data-only camera policy installation cannot grant success or change arms.""" from copy import deepcopy +from hashlib import sha256 import json from pathlib import Path import tempfile @@ -80,14 +81,23 @@ def run(assets, output, **kwargs): self.assertEqual(kwargs['mode'], 'student') self.assertNotIn('teacher', kwargs) self.assertEqual(kwargs['case']['object_position_m'], [.30, -.03, .082]) + self.assertEqual(kwargs['case']['policy_artifact_sha256'], sha256(self.raw).hexdigest()) output.mkdir() (output/'integrity.json').write_text('{}') + (output/'rollout.json').write_text('{"fixture":true}') return {'strict_success': False, 'status': 'complete', 'reason': 'failed_task'} with patch('dvidia_training.camera_action_capture.run_rollout', side_effect=run): result = dispatch.run_installation(self.output, 'unused', out) self.assertFalse(result['success']) - self.assertEqual([p.name for p in out.iterdir()], ['integrity.json']) + self.assertEqual(sorted(p.name for p in out.iterdir()), ['integrity.json', 'rollout.json']) self.assertTrue((self.root/'attempt-installation.json').is_file()) + link = json.loads((self.root/'attempt-installation.json').read_bytes()) + self.assertEqual(link['rollout_sha256'], sha256((out/'rollout.json').read_bytes()).hexdigest()) + self.assertEqual(link['integrity_sha256'], sha256((out/'integrity.json').read_bytes()).hexdigest()) + with patch('dvidia_training.camera_action_capture.run_rollout') as runner: + with self.assertRaisesRegex(ValueError, 'fresh execution'): + dispatch.run_installation(self.output, 'unused', out) + runner.assert_not_called() def test_out_of_envelope_refuses_before_execution(self): dispatch.install_model(self.source, 'piper', 'unused', self.output) diff --git a/tests/test_camera_action_studio.py b/tests/test_camera_action_studio.py index 2f53206..206d50f 100644 --- a/tests/test_camera_action_studio.py +++ b/tests/test_camera_action_studio.py @@ -154,6 +154,26 @@ def test_protocol_change_invalidates_check_and_result(self): self.assertIsNone(state['summary']);self.assertIn('protocol changed',state['error']) self.assertEqual(self.request('/api/robot/install',self.payload())[0],400) + def test_terminal_worker_lock_allows_check_but_requires_recheck_before_execution(self): + lock=self.study/'worker.lock' + lock.write_text(json.dumps(dict(pid=999999,operation='collect'))) + state=self.checked() + self.assertTrue(state['checked']);self.assertTrue(state['study_locked']) + self.assertEqual(state['study_worker_lock']['operation'],'collect') + self.assertFalse(state['can_run']);self.assertFalse(state['can_install']) + self.assertTrue(lock.exists()) + # The test owns this fake lock; the implementation never removes it. + lock.unlink() + state=self.done();self.assertFalse(state['study_locked']) + self.assertTrue(state['recheck_required']);self.assertFalse(state['can_run']) + self.assertTrue(self.checked()['can_run']) + + def test_new_terminal_lock_after_check_blocks_direct_run_request(self): + self.assertTrue(self.checked()['can_run']) + lock=self.study/'worker.lock';lock.write_text('{"operation":"collect"}') + code,result=self.request('/api/robot/run',self.payload()) + self.assertEqual(code,400);self.assertIn('Terminal',result['error']);self.assertTrue(lock.exists()) + def test_worker_error_preserves_error_without_stale_success(self): def fail(*args):raise ValueError('Fake renderer unavailable') server=self.make_server(worker=fail) diff --git a/tests/test_camera_action_study_audit.py b/tests/test_camera_action_study_audit.py new file mode 100644 index 0000000..09d8d84 --- /dev/null +++ b/tests/test_camera_action_study_audit.py @@ -0,0 +1,285 @@ +"""Independent audit tests: no simulator, renderer, tracker or policy execution.""" +from copy import deepcopy +from pathlib import Path +import random +import os +import subprocess +import sys +import tempfile +import unittest + +HELPERS = Path(__file__).resolve().parents[1] / 'benchmarks/camera-action-v01' +sys.path.insert(0, str(HELPERS)) +import study_audit as audit + + +def protocol(): + value = dict(cases={}, rosters={}, goals=[[.35, .08, .08], [.40, -.08, .08]], + start_domain=dict(x_m=[.287, .311], y_m=[-.042, -.018]), historical_start_exclusion=[], + data_quality=dict(minimum_successful_episodes_per_goal=28), + acceptance=dict(student_successes_minimum=58, student_attempts=64, confidence=.975, + bootstrap_resamples=10000, bootstrap_seed=311402, minimum_common_admitted_groups=24)) + offset = 0 + for split, count in (('train', 32), ('development', 8), ('confirmation', 32)): + value['cases'][split] = [] + for i in range(count): + group = f'{split}-{i:02}' + for goal in (0, 1): + value['cases'][split].append(dict(id=f'{group}-goal{goal}', group_id=group, split=split, + goal_id=goal, seed=311401+2*(offset+i)+goal, + object_position_m=[.288+(offset+i)*.0003, -.03, .082], goal_position_m=value['goals'][goal])) + offset += count + def attempt(case, controller, negative=None): + return dict(id=case['id']+'-'+controller+('-'+negative if negative else ''), case=deepcopy(case), + controller=controller, negative_control=negative) + cases = value['cases']; roster = value['rosters'] + roster['diagnostic'] = [attempt(c, 'teacher') for c in cases['development'][:4]] + roster['demonstrations'] = [attempt(c, 'teacher') for c in cases['train']+cases['development']] + roster['corrections'] = [attempt(cases['train'][2*i+i%2], 'mixed') for i in range(32)] + for stage in ('initial', 'corrected'): + roster['development-'+stage] = [attempt(c, control) for c in cases['development'] for control in ('student', 'blind')] + roster['confirmation'] = [attempt(c, control) for c in cases['confirmation'] for control in ('student', 'blind', 'fixed', 'teacher')] + roster['negative'] = [attempt(c, 'student', fault) for c in cases['confirmation'][:4] + for fault in ('initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper')] + return value + + +def row(attempt, success=False): + return dict(**deepcopy(attempt), status='complete', complete=True, strict_success=success, + initial_anchor_available=True, initial_anchor_binding_valid=True, recorded_complete_intervals=1000, + critical_contact_ticks=0, provenance_ok=attempt['controller'] != 'teacher', + score=dict(valid=True, success=success, strict_v01=dict(trajectory_complete=True))) + + +class StudyAuditTests(unittest.TestCase): + def test_roster_rejects_split_leak_controller_substitution_and_missing_fault(self): + p = protocol(); audit.validate_roster(p) + for modify in ( + lambda x: x['rosters']['confirmation'][0].update(controller='teacher'), + lambda x: x['rosters']['negative'][0].update(negative_control='initial_stale'), + lambda x: x['rosters']['corrections'][0].update(case=x['cases']['confirmation'][0]), + lambda x: x['cases']['confirmation'][0].update(group_id='train-00'), + lambda x: x['cases']['train'][0].update(goal_position_m=[0, 0, 0]), + ): + changed = deepcopy(p); modify(changed) + with self.assertRaises(ValueError): audit.validate_roster(changed) + + def test_data_quality_requires_group_coverage_and_per_destination_success(self): + p = protocol(); rows = [row(a, True) for a in p['rosters']['demonstrations']] + self.assertTrue(audit.data_quality(rows, p)['passed']) + rows[0]['strict_success'] = rows[1]['strict_success'] = False + self.assertEqual(audit.data_quality(rows, p)['missing_training_groups'], ['train-00']) + rows = [row(a, True) for a in p['rosters']['demonstrations']] + for r in rows[:10:2]: r['initial_anchor_available'] = False + self.assertEqual(audit.data_quality(rows, p)['successful_episodes_per_goal']['0'], 27) + self.assertFalse(audit.data_quality(rows, p)['passed']) + + def test_mixed_authority_is_seeded_and_teacher_query_is_not_executed_student_command(self): + limits = dict(joint_limits_rad=[[-3, 3]]*6, gripper_position_limits_m=[0., .035]) + teacher = dict(joint_targets_rad=[.1]*6, gripper_position_m=.02) + student = dict(joint_targets_rad=[.2]*6, gripper_position_m=.01) + truth, checked = random.Random(311401), random.Random(311401) + for _ in range(20): + authority = 'teacher' if truth.random() < .5 else 'student' + requested = deepcopy(teacher if authority == 'teacher' else student) + step = dict(command=dict(operator_correction=False, authority=authority, requested=requested, + issued=deepcopy(requested), issue_reason='unchanged')) + query = dict(origin='privileged_simulator_teacher', action=deepcopy(teacher)) + audit.command_check(step, query, 'mixed', None, checked, limits) + step['command']['authority'] = 'teacher' + step['command']['requested'] = student + step['command']['issued'] = student + with self.assertRaisesRegex(ValueError, 'separately sealed query'): + audit.command_check(step, query, 'teacher', None, None, limits) + step['command']['authority'] = 'student' + with self.assertRaisesRegex(ValueError, 'teacher intervention'): + audit.command_check(step, query, 'student', None, None, limits) + + def test_open_gripper_intervention_and_nested_numeric_negative_score(self): + limits = dict(joint_limits_rad=[[-3, 3]]*6, gripper_position_limits_m=[0., .035]) + command = dict(joint_targets_rad=[0.]*6, gripper_position_m=.01) + step = dict(command=dict(operator_correction=False, authority='student', requested=command, + issued={**command, 'gripper_position_m': .035}, issue_reason='open_gripper_negative_control')) + audit.command_check(step, None, 'student', 'open_gripper', None, limits) + with self.assertRaisesRegex(ValueError, 'registered intervention'): + audit.command_check(step, None, 'student', None, None, limits) + r = row(protocol()['rosters']['negative'][3]); self.assertTrue(audit.negative_correct(r)) + r['score'] = dict(valid=True, trajectory_complete=True) + self.assertFalse(audit.negative_correct(r)) + r['score'] = dict(valid=False, strict_v01=dict(trajectory_complete=True)) + self.assertFalse(audit.negative_correct(r)) + r['negative_control'] = 'initial_missing'; r['status'] = 'refused' + self.assertFalse(audit.negative_correct(r)) + r['recorded_complete_intervals'] = 0; self.assertTrue(audit.negative_correct(r)) + + def test_perception_missing_masks_and_declared_initial_calibration_fault(self): + frame = {'index': 0, 'rgb': {'path': 'frames/rgb-0000.png', 'bytes': 1, 'sha256': 'a'*64}, + 'depth': {'path': 'frames/depth-0000.npy', 'bytes': 1, 'sha256': 'b'*64}} + snapshot = dict(timestamp_seconds=.02, camera=dict(value=frame, captured_at_seconds=0., available_at_seconds=0., age_seconds=.02)) + calibration = dict(base_from_camera=[1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1., 0., 0., 0., 0., 1.]) + recording = dict(calibration=calibration, binding=dict(calibration_sha256=audit.digest(calibration)), tracker_sha256='f'*64) + def document(fault=None): + actual=deepcopy(calibration) + if fault=='altered_calibration': actual['base_from_camera'][0]*=1.1 + captured=-.2 if fault=='initial_stale' else 0. + estimate=dict(status='missing', frame_index=0, captured_at_seconds=captured, available_at_seconds=captured, + decision_at_seconds=.02, age_seconds=.02-captured, signals={}) + for key in audit.VISION: + estimate[key]=None; estimate['signals'][key]=dict(status='missing', value=None) + return dict(index=1, snapshot_sha256=audit.digest(snapshot), decision_at_seconds=.02, + frame=frame, calibration_sha256=recording['binding']['calibration_sha256'], tracker_sha256='f'*64, + input_calibration_sha256=audit.digest(actual), negative_control=fault, estimate=estimate) + for fault in (None, 'initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper'): + audit._perception(document(fault), snapshot, recording, fault, 1) + changed=document(); changed['estimate']['signals']['object_position_estimate_m']['value']=[0, 0, 0] + with self.assertRaisesRegex(ValueError, 'Unavailable perception'): + audit._perception(changed, snapshot, recording, None, 1) + changed=document('altered_calibration'); changed['input_calibration_sha256']=audit.digest(calibration) + with self.assertRaisesRegex(ValueError, 'input calibration'): + audit._perception(changed, snapshot, recording, 'altered_calibration', 1) + changed=document('initial_stale'); changed['estimate']['age_seconds']=0. + with self.assertRaisesRegex(ValueError, 'age differs'): + audit._perception(changed, snapshot, recording, 'initial_stale', 1) + + def test_snapshot_cadence_frame_reuse_and_raw_truth_exclusion(self): + missing=dict(status='missing', value=None) + signals={name:deepcopy(missing) for name in audit.VISION} + for name in audit.ENCODERS: + signals[name]=dict(status='available', value=[0., 0.], captured_at_seconds=.02, available_at_seconds=.02, age_seconds=0.) + frame=dict(index=0, rgb=dict(path='frames/rgb.png', bytes=2, sha256='a'*64), depth=dict(path='frames/depth.npy', bytes=3, sha256='b'*64)) + snap=dict(timestamp_seconds=.02, signals=signals, vision_frame_index=0, + camera=dict(value=frame, captured_at_seconds=0., available_at_seconds=0., age_seconds=.02)) + files={ref['path']:{k:ref[k] for k in ('bytes','sha256')} for ref in (frame['rgb'],frame['depth'])} + audit._snapshot(snap, 1, files, {}) + changed=deepcopy(snap); changed['signals']['object_position_estimate_m']={'status':'available', 'value':[1, 2, 3]} + with self.assertRaisesRegex(ValueError, 'native object truth'): audit._snapshot(changed, 1, files, {}) + changed=deepcopy(snap); changed['camera']['captured_at_seconds']=.02 + with self.assertRaisesRegex(ValueError, 'capture time'): audit._snapshot(changed, 1, files, {}) + with self.assertRaisesRegex(ValueError, 'Reused camera'): audit._snapshot(snap, 1, files, {0:dict(index=0)}) + files['frames/rgb.png']['sha256']='c'*64 + with self.assertRaisesRegex(ValueError, 'byte-bound'): audit._snapshot(snap, 1, files, {}) + + def test_bootstrap_groups_both_goals_and_excludes_both_goals_if_one_not_admitted(self): + p = protocol(); rows = [row(a, a['controller']=='student') for a in p['rosters']['confirmation']] + result = audit.bootstrap(rows, 'fixed', p['acceptance']) + self.assertEqual((result['difference'], result['lower'], result['upper'], result['groups']), (1., 1., 1., 32)) + rows[2]['initial_anchor_available'] = False + result = audit.bootstrap(rows, 'fixed', p['acceptance'], admitted_only=True) + self.assertEqual((result['groups'], result['excluded_groups']), (31, 1)) + with self.assertRaisesRegex(ValueError, 'Duplicate'): + audit.bootstrap(rows+[rows[0]], 'fixed', p['acceptance']) + + def test_final_gates_fail_when_advantage_exists_only_through_refusals_or_contacts(self): + p = protocol(); rows = [row(a, a['controller']=='student') for a in p['rosters']['confirmation']] + negatives = [row(a) for a in p['rosters']['negative']] + for r in negatives: + if r['negative_control'] != 'open_gripper': + r.update(status='refused', recorded_complete_intervals=0, complete=False) + self.assertTrue(audit.final_gates(rows, negatives, p, stages_complete=True)['qualified']) + for r in rows: + if r['controller'] == 'fixed' and int(r['case']['group_id'][-2:]) < 9: + r.update(initial_anchor_available=False, recorded_complete_intervals=0) + result = audit.final_gates(rows, negatives, p, stages_complete=True) + self.assertFalse(result['gates']['positive_common_admitted_comparisons']) + rows[0]['critical_contact_ticks'] = 1 + self.assertFalse(audit.final_gates(rows, negatives, p, stages_complete=True)['gates']['no_critical_contact_increase']) + self.assertFalse(audit.final_gates(rows, negatives, p, stages_complete=False)['gates']['all_registered_attempts_accounted']) + + def test_stage_binding_and_model_replacement_are_detected_without_loading_models(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory); (root/'stages').mkdir() + def save(name, obj): + p=root/name; p.parent.mkdir(parents=True, exist_ok=True); p.write_bytes(audit.canonical(obj)) + receipt = dict(stage='diagnostic', protocol_sha256='a'*64, model_bindings={'teacher': 'b'*64}) + save('stages/diagnostic.json', receipt) + save('stages/diagnostic.started.json', receipt) + save('stages/diagnostic.integrity.json', dict(protocol_sha256='a'*64, sha256=audit.identity(root/'stages/diagnostic.json')['sha256'])) + audit._stage(root, 'diagnostic', 'a'*64, {}) + save('stages/diagnostic.started.json', {**receipt, 'model_bindings': {'teacher': 'c'*64}}) + with self.assertRaisesRegex(ValueError, 'Started stage'): + audit._stage(root, 'diagnostic', 'a'*64, {}) + save('models/initial/full/model.json', {'weights': 'original'}) + stages = {'train-initial': dict(models=dict(full=dict(model_sha256=audit.identity(root/'models/initial/full/model.json')['sha256'])))} + audit._model_hash(root, 'initial', 'student', stages, {}) + save('models/initial/full/model.json', {'weights': 'replacement'}) + with self.assertRaisesRegex(ValueError, 'sealed training receipt'): + audit._model_hash(root, 'initial', 'student', stages, {}) + + def test_active_worker_is_not_read_or_audited(self): + with tempfile.TemporaryDirectory() as directory: + (Path(directory)/'worker.lock').write_text('running') + result = audit.audit(directory) + self.assertFalse(result['passed']); self.assertEqual(result['attempts_verified'], 0) + self.assertIn('worker', result['errors'][0]['reason']) + + def test_training_audit_rejects_matched_full_blind_subset_and_validation_roster_drift(self): + with tempfile.TemporaryDirectory() as directory: + root=Path(directory); p=protocol() + p['fitting']=dict(epochs=50,batch_size=512,learning_rate=.003,seed=17,hidden_units=[64,64]) + p['source_sha256']={'camera_action_model.py':'a'*64,'camera_action_export.py':'b'*64} + def save(name,value): + target=root/name; target.parent.mkdir(parents=True,exist_ok=True); target.write_bytes(audit.canonical(value)) + save('protocol.json',p) + rows=[] + for case in p['cases']['train'][:2]+p['cases']['development'][:1]: + attempt=dict(id=case['id']+'-teacher',case=case,controller='teacher',negative_control=None) + r=row(attempt,True); r.update(recorded_complete_intervals=2,path='episodes/demonstrations/'+attempt['id']) + recording=dict(control_mode='teacher',provenance=dict(source_group=case['group_id'],case_id=case['id']), + binding=dict(calibration_sha256='c'*64),tracker_sha256='d'*64, + steps=[dict(perception=dict(sha256='e'*64),teacher_query=None,command=dict(authority='teacher')) for _ in range(2)]) + save(r['path']+'/recording.json',recording); save(r['path']+'/integrity.json',{'fixture':case['id']}) + rows.append(r) + stages={'demonstrations':dict(data_quality={'passed':True},rows=rows),'train-initial':dict(models={})} + original=[audit._export_source(root,r,'issued') for r in rows[:2]] + def models(sources, validation_ids=None): + for variant in ('full','blind'): + provenance=dict(episodes=sources,source_identity_sha256=audit.digest(sources),blind=variant=='blind', + export_source_sha256='b'*64,feature_contract_sha256='f'*64) + training=dict(stage='initial',model_source_sha256='a'*64,epochs=50,batch_size=512,learning_rate=.003,seed=17, + train_groups=['train-00'],sample_count=4,validation=dict(groups=['development-00'],sample_count=2), + provenance=provenance,provenance_sha256=audit.digest(provenance)) + model=dict(schema='dvidia.camera-action-mlp/v1',blind=variant=='blind', + observation_origin='rendered_rgbd_and_simulated_encoders',feature_contract='camera-encoder-absolute-causal160/v1', + action_representation='absolute_native_targets',hidden_units=[64,64],feature_contract_sha256='f'*64,training=training) + path=f'models/initial/{variant}/model.json';save(path,model); identity=audit.identity(root/path) + receipt=dict(path=path,model_bytes=identity['bytes'],model_sha256=identity['sha256'], + training_episodes=[r['id'] for r in rows[:2]],development_episodes=validation_ids or [rows[2]['id']], + training=training,dataset_provenance=provenance) + stages['train-initial']['models'][variant]=receipt + save(f'models/initial/{variant}/receipt.json',receipt) + save(f'models/initial/{variant}/started.json',dict(stage='initial',variant=variant, + protocol_sha256=audit.identity(root/'protocol.json')['sha256'])) + models(original);audit._training(root,stages,p,{}) + models(original[:1]) + with self.assertRaisesRegex(ValueError,'omits, duplicates or changes'): + audit._training(root,stages,p,{}) + models([original[0],original[0]]) + with self.assertRaisesRegex(ValueError,'omits, duplicates or changes'): + audit._training(root,stages,p,{}) + models(original,validation_ids=[rows[0]['id']]) + with self.assertRaisesRegex(ValueError,'episode roster differs'): + audit._training(root,stages,p,{}) + + def test_numeric_comparison_does_not_accept_booleans_as_numeric_gates(self): + self.assertTrue(audit._equivalent({'low': .1}, {'low': .10000000000000002})) + self.assertFalse(audit._equivalent({'passed': True}, {'passed': 1})) + + def test_imports_do_not_load_environment_tracker_controller_or_study_runtime(self): + env=dict(os.environ, PYTHONPATH=str(HELPERS)+os.pathsep+str(HELPERS.parents[1]/'src')) + completed=subprocess.run([sys.executable, '-c', + "import study_audit, sys; forbidden=('camera_action_study','camera_action_capture','camera_action_model','piper_feedback_env','piper_rgbd_tracker','piper_motor_primitive','mujoco'); assert not any(m.rsplit('.',1)[-1] in forbidden for m in sys.modules)"], + env=env, capture_output=True, text=True) + self.assertEqual(completed.returncode, 0, completed.stderr) + + def test_encoder_tool_pose_and_velocity_are_reconstructed_without_kinematic_runtime(self): + import numpy as np + transform=np.eye(4); transform[:3, 3]=[1., 2., 3.] + state=dict(qvel=[0.]*6+[.01, -.01], tcp_position_m=[1.2, 2.3, 3.52003], tcp_quaternion_wxyz=[1., 0., 0., 0.]) + snapshot=dict(signals=dict(gripper_joint_velocity_m_s={'value':[.01, -.01]}, + tcp_position_estimate_m={'value':[.2, .3, .4]}, tcp_quaternion_estimate_wxyz={'value':[-1., 0., 0., 0.]})) + audit._encoder_pose(snapshot,state,transform) + snapshot['signals']['tcp_position_estimate_m']['value']=[1.2, 2.3, 3.4] + with self.assertRaisesRegex(ValueError, 'tool position'): audit._encoder_pose(snapshot,state,transform) + + +if __name__ == '__main__': unittest.main() diff --git a/tests/test_camera_action_summary.py b/tests/test_camera_action_summary.py new file mode 100644 index 0000000..9eab89f --- /dev/null +++ b/tests/test_camera_action_summary.py @@ -0,0 +1,218 @@ +"""Posthoc receipt/terminal-component tests using fabricated numeric fixtures.""" +from copy import deepcopy +from contextlib import redirect_stdout, redirect_stderr +from hashlib import sha256 +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest + +import numpy as np + +PATH=Path(__file__).resolve().parents[1]/'benchmarks/camera-action-v01/summarize_results.py' +SPEC=importlib.util.spec_from_file_location('camera_action_posthoc',PATH) +summary=importlib.util.module_from_spec(SPEC);SPEC.loader.exec_module(summary) + + +def raw_json(value):return json.dumps(value,sort_keys=True,separators=(',',':'),allow_nan=False).encode()+b'\n' +def write(path,value): + path.parent.mkdir(parents=True,exist_ok=True);raw=raw_json(value);path.write_bytes(raw);return raw + +def identity(raw):return dict(bytes=len(raw),sha256=sha256(raw).hexdigest()) + + +def native_fixture(*,ticks=20000): + position=np.tile([.35,.08,.08],(ticks,1)) + arrays=dict(t_seconds=np.arange(1,ticks+1)*.001,native_tick=np.arange(1,ticks+1), + grasp_contact_counts=np.zeros((ticks,2)),gripper_joint_position_m=np.tile([.03,-.03],(ticks,1)), + object_position_m=position,tcp_position_m=position+np.array([0.,0.,.05]), + object_linear_velocity_m_s=np.zeros((ticks,3)),object_angular_velocity_rad_s=np.zeros((ticks,3)), + table_object_contact_count=np.ones(ticks)) + metadata=dict(schema='dvidia.piper-feedback-native/v1',native_timestep_s=.001,rows=ticks,side_order=['left','right']) + episode=dict(config=dict(timestep=.001,timeout_s=ticks*.001,goal_dwell_s=.25, + retreat_clearance_m=.10,goal_position_tolerance_m=.025,target_position_m=[.35,.08,.08])) + return arrays,metadata,episode + + +def study_fixture(root): + case=dict(id='held-out-0',group_id='held-out-0',goal_id=0) + roster=[dict(id='attempt-0',controller='student',case=case),dict(id='attempt-1',controller='student',case={**case,'id':'held-out-1'})] + protocol=dict(rosters={'confirmation':roster}) + protocol_raw=write(root/'protocol.json',protocol);protocol_hash=summary.digest(protocol_raw) + write(root/'registration.json',dict(protocol_sha256=protocol_hash)) + score=dict(valid=True,success=False,native_recomputed=dict(grasp_seen=True,lift_seen=True, + carry_complete=True,retention_lost=False,unexpected_contact_native_ticks=5),terminal_only=dict(success=False,dwell_s=0.)) + perception=dict(available=600,missing=400,decisions=1000,longest_missing_seconds=7.2) + receipt=dict(status='complete',recorded_complete_intervals=1000,score=score,perception=perception, + timing=dict(wall_seconds=30.,boundaries=[dict(policy_inference_wall_seconds=.001, + full_boundary_wall_seconds=.01),dict(policy_inference_wall_seconds=.003,full_boundary_wall_seconds=.03)])) + episode_root=root/'episodes'/'attempt-0';episode_root.mkdir(parents=True) + receipt_raw=write(episode_root/'rollout.json',receipt) + context_raw=write(episode_root/'context.json',dict(metadata=dict(runtime=dict(system='fixture',machine='no robot')))) + arrays,metadata,episode=native_fixture() + metadata_raw=write(episode_root/'native_metadata.json',metadata) + episode_raw=write(episode_root/'native_episode.json',episode) + stream=io.BytesIO();np.savez_compressed(stream,**arrays);native_raw=stream.getvalue();(episode_root/'native.npz').write_bytes(native_raw) + inventory=dict(files={name:identity(raw) for name,raw in ( + ('rollout.json',receipt_raw),('context.json',context_raw),('native.npz',native_raw), + ('native_metadata.json',metadata_raw),('native_episode.json',episode_raw))}) + inventory_raw=write(episode_root/'integrity.json',inventory) + row={**roster[0], 'path':'episodes/attempt-0','receipt_sha256':summary.digest(receipt_raw), + 'integrity_sha256':summary.digest(inventory_raw),'status':'complete','complete':True,'strict_success':False, + 'score':score,'perception':perception} + refused={**roster[1],'status':'refused','complete':False,'strict_success':False,'score':{'valid':False,'success':False}, + 'receipt_sha256':None,'path':None} + stage=dict(stage='confirmation',status='complete',protocol_sha256=protocol_hash,scheduled=2, + strict_successes=0,rows=[row,refused]) + stage_raw=write(root/'stages/confirmation.json',stage) + write(root/'stages/confirmation.integrity.json',dict(sha256=summary.digest(stage_raw),protocol_sha256=protocol_hash)) + return stage + + +class NativeTerminalTests(unittest.TestCase): + def test_released_and_settled_but_not_retreated_are_independent(self): + arrays,metadata,episode=native_fixture(ticks=1000) + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertTrue(result['release']['final_predicate']);self.assertTrue(result['release']['dwell_met']) + self.assertTrue(result['settling']['dwell_met']);self.assertFalse(result['retreat']['final_predicate']) + self.assertFalse(result['joint_terminal']['dwell_met']) + self.assertEqual(result['retreat']['final_consecutive_dwell_seconds'],0.) + + def test_dwell_counts_only_final_consecutive_ticks_at_exact_threshold(self): + arrays,metadata,episode=native_fixture(ticks=1000) + arrays['tcp_position_m'][-250:,2]=.18 + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertEqual(result['retreat']['final_consecutive_dwell_seconds'],.25) + self.assertTrue(result['retreat']['dwell_met']);self.assertTrue(result['joint_terminal']['dwell_met']) + arrays['tcp_position_m'][-250,2]=.179 + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertAlmostEqual(result['retreat']['final_consecutive_dwell_seconds'],.249) + self.assertTrue(result['retreat']['final_predicate']);self.assertFalse(result['retreat']['dwell_met']) + + def test_actual_jaw_contacts_speed_and_support_are_distinct(self): + arrays,metadata,episode=native_fixture(ticks=1000) + arrays['tcp_position_m'][:,2]=.2 + arrays['object_linear_velocity_m_s'][-1,0]=.035 + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertTrue(result['release']['dwell_met']);self.assertFalse(result['settling']['final_predicate']) + self.assertTrue(result['retreat']['dwell_met']) + arrays['object_linear_velocity_m_s'][-1,0]=0. + arrays['gripper_joint_position_m'][-1,0]=.028 + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertFalse(result['release']['final_predicate']);self.assertTrue(result['settling']['dwell_met']) + arrays['gripper_joint_position_m'][-1,0]=.03;arrays['grasp_contact_counts'][-1,0]=1 + self.assertFalse(summary.native_terminal_components(arrays,metadata,episode)['predicates']['release']['final_predicate']) + arrays['grasp_contact_counts'][-1,0]=0;arrays['table_object_contact_count'][-1]=0 + result=summary.native_terminal_components(arrays,metadata,episode)['predicates'] + self.assertTrue(result['release']['dwell_met']);self.assertFalse(result['joint_terminal']['final_predicate']) + + def test_missing_nonfinite_and_incomplete_native_evidence_is_refused(self): + arrays,metadata,episode=native_fixture(ticks=1000) + bad=deepcopy(arrays);del bad['object_linear_velocity_m_s'] + with self.assertRaisesRegex(ValueError,'Missing or malformed'): + summary.native_terminal_components(bad,metadata,episode) + bad=deepcopy(arrays);bad['object_position_m'][0,0]=float('nan') + with self.assertRaisesRegex(ValueError,'Missing or malformed'): + summary.native_terminal_components(bad,metadata,episode) + bad=deepcopy(arrays);bad['native_tick'][-1]=500 + with self.assertRaisesRegex(ValueError,'time/tick'): + summary.native_terminal_components(bad,metadata,episode) + + +class PosthocSummaryTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.root=Path(self.temp.name)/'study';self.root.mkdir();self.stage=study_fixture(self.root) + + def test_real_seal_readback_preserves_all_attempt_and_conditional_denominators(self): + result=summary.summarize(self.root);row=result['controllers'][0] + self.assertEqual(row['scheduled'],2);self.assertEqual(row['complete'],1) + self.assertEqual(row['native_assessable'],1);self.assertEqual(row['strict_successes'],0) + self.assertEqual(row['components']['release']['conditional_assessed'],1) + self.assertEqual(row['components']['release']['conditional_failed'],0) + self.assertEqual(row['components']['retreat']['conditional_failed'],1) + self.assertEqual(row['components']['release']['unassessed'],1) + self.assertEqual(row['critical_contacts']['assessed_attempts'],1) + self.assertEqual(row['critical_contacts']['unassessed_attempts'],1) + self.assertEqual(row['critical_contacts']['attempts_with_contacts'],1) + self.assertEqual(row['critical_contacts']['native_ticks'],5) + self.assertEqual(row['perception']['available_fraction'],.6) + self.assertEqual(row['perception']['episodes_unknown'],1) + self.assertEqual(row['terminal_components_by_attempt'][0]['source_sha256']['native.npz'], + summary.digest((self.root/'episodes/attempt-0/native.npz').read_bytes())) + self.assertIsNone(result['hardware'][0]['cpu_model']) + self.assertTrue(any(x['status']=='pending_receipt' for x in result['stages'])) + text=summary.markdown(result) + self.assertIn('0 / 2',text);self.assertIn('posthoc',text);self.assertIn('not physical sensor-to-actuator',text) + + def test_missing_metrics_stay_unassessed_not_guessed_from_overall_failure(self): + row=dict(score={'valid':True,'success':False,'native_recomputed':{}},strict_success=False) + result=summary.component_counts([row],2,summary.COMPONENTS[0]) + self.assertEqual(result['assessed'],0);self.assertEqual(result['failed'],0);self.assertEqual(result['unassessed'],2) + result=summary.component_counts([row],2,summary.COMPONENTS[5]) + self.assertEqual(result['conditional_assessed'],0);self.assertEqual(result['prerequisite_unknown'],2) + + def test_latency_pools_boundary_samples_instead_of_episode_percentiles(self): + roster=[dict(id='a'),dict(id='b')] + rows=[dict(id=key,controller='student',score={}) for key in ('a','b')] + episodes={'a':({'timing':{'boundaries':[{'full_boundary_wall_seconds':1.}]}},None), + 'b':({'timing':{'boundaries':[{'full_boundary_wall_seconds':.01}]*19}},None)} + record=summary._controller('test','student',roster,rows,episodes)['timing']['full_boundary_wall_seconds'] + self.assertEqual(record['samples'],20);self.assertAlmostEqual(record['p95'],.0595) + self.assertFalse(record['deadline_claim']) + + def test_modified_native_bytes_or_stage_receipts_are_rejected(self): + path=self.root/'episodes/attempt-0/native.npz';path.write_bytes(path.read_bytes()+b'changed') + with self.assertRaisesRegex(ValueError,'Native numeric evidence differs'): + summary.summarize(self.root) + stage_path=self.root/'stages/confirmation.json';stage_path.write_text('{}') + with self.assertRaisesRegex(ValueError,'Stage seal'): + summary.summarize(self.root) + + def test_resealed_missing_or_replaced_attempt_id_is_rejected(self): + for replacement in (None, 'unregistered-attempt'): + with self.subTest(replacement=replacement): + stage=deepcopy(self.stage) + if replacement is None: + stage['rows'].pop() + else: + stage['rows'][-1]['id']=replacement + raw=write(self.root/'stages/confirmation.json',stage) + write(self.root/'stages/confirmation.integrity.json',dict(sha256=summary.digest(raw), + protocol_sha256=stage['protocol_sha256'])) + with self.assertRaisesRegex(ValueError,'ID set differs'): + summary.summarize(self.root) + + def test_resealed_negative_control_change_is_rejected(self): + stage=deepcopy(self.stage);stage['rows'][-1]['negative_control']='open_gripper' + raw=write(self.root/'stages/confirmation.json',stage) + write(self.root/'stages/confirmation.integrity.json',dict(sha256=summary.digest(raw), + protocol_sha256=stage['protocol_sha256'])) + with self.assertRaisesRegex(ValueError,'registered roster'): + summary.summarize(self.root) + + def test_unattempted_rows_remain_explicit_and_count_in_denominator(self): + stage=deepcopy(self.stage);stage['status']='blocked' + stage['rows'][-1]=dict(id=stage['rows'][-1]['id'],controller='student', + case=stage['rows'][-1]['case'],status='unattempted',complete=False,strict_success=False) + raw=write(self.root/'stages/confirmation.json',stage) + write(self.root/'stages/confirmation.integrity.json',dict(sha256=summary.digest(raw), + protocol_sha256=stage['protocol_sha256'])) + result=summary.summarize(self.root)['controllers'][0] + self.assertEqual(result['scheduled'],2);self.assertEqual(result['retained_rows'],2) + self.assertEqual(result['statuses']['unattempted'],1) + self.assertEqual(result['components']['retreat']['unassessed'],1) + + def test_cli_writes_fresh_posthoc_files_outside_frozen_study(self): + output=Path(self.temp.name)/'report' + with redirect_stdout(io.StringIO()): + summary.main(['--study',str(self.root),'--output',str(output)]) + self.assertTrue((output/'summary.json').is_file());self.assertTrue((output/'summary.md').is_file()) + with redirect_stderr(io.StringIO()),self.assertRaises(SystemExit): + summary.main(['--study',str(self.root),'--output',str(output)]) + self.assertFalse((self.root/'summary.json').exists()) + + +if __name__=='__main__':unittest.main() From 9551fa1f2a1ee8c725237c8bff78c5e187578c05 Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sun, 1 Nov 2026 03:05:53 -0500 Subject: [PATCH 03/10] Bound camera-action publication storage and verify anonymous downloads --- publishing/CAMERA-ACTION-V01.md | 30 +++- .../huggingface/publish_camera_action_v01.py | 146 ++++++++++++++++-- .../test_publish_camera_action_v01.py | 137 +++++++++++++++- publishing/prepare_camera_action_v01.py | 12 +- publishing/test_prepare_camera_action_v01.py | 11 ++ 5 files changed, 310 insertions(+), 26 deletions(-) diff --git a/publishing/CAMERA-ACTION-V01.md b/publishing/CAMERA-ACTION-V01.md index 6c79df8..e52b292 100644 --- a/publishing/CAMERA-ACTION-V01.md +++ b/publishing/CAMERA-ACTION-V01.md @@ -73,17 +73,39 @@ python publishing/huggingface/publish_camera_action_v01.py verify \ Preparation streams one deterministic ZIP64 archive per sealed episode, checks each file against the original integrity manifest, and decompresses -every member to verify its bytes. It refuses a worker lock, unsealed episode, +every member to verify its bytes. DEFLATE level 1 is the publication default. +On the same sealed teacher episode, one sequential shared-host comparison +measured 40.36 seconds / 81,058,092 bytes at level 1 versus 54.02 seconds / +75,301,793 bytes at level 6. That is a 25.3% shorter archive-and-verification +time for 7.64% more bytes in this example, not a general throughput guarantee. +It refuses a worker lock, unsealed episode, unexpected study file, symbolic link, missing source binding, or detected credential-shaped text. The 30 GiB disk reserve is preserved. Existing output directories and files are never overwritten. The publisher freezes reviewed bytes into a private temporary directory before -credential lookup, uses a single commit bound to the empty dataset's exact +credential lookup. On a supporting macOS filesystem, it uses the documented +`fclonefileat(2)` copy-on-write operation from a regular-file descriptor. The +copy has its own inode and attributes, is marked read-only, and is checked +against the reviewed bytes. Later writes to either file are private to that +file. Hardlinks are never used. Unsupported filesystems use bounded streaming +copies with free-space checks before each file and each chunk, preserving the +30 GiB reserve. Every path rejects symlinks and existing destinations. +The uploader uses a single commit bound to the empty dataset's exact parent revision, and refuses existing nonempty or private repositories. Anonymous verification downloads every published file at the exact resulting -commit and checks its byte identity. Large files are streamed, not loaded into -one large in-memory payload. +commit and checks its byte identity. Each file uses a new owned temporary cache +under the explicit `--cache` directory, with its full byte budget plus metadata +headroom checked against the 30 GiB reserve before download. That temporary is +removed after verification, including on failure. Existing shared cache files +are never removed. The retained receipt records the pinned commit, reviewed +plan digest, inventory digest, byte count and largest single readback file. +The pinned remote file-size metadata must match every reviewed byte count +before any file download; missing or changed sizes fail closed. +Verification sets `HF_HUB_DISABLE_XET=1` before SDK import to use HTTP downloads +into the owned Hub cache, and refuses an already-imported SDK whose Xet +transport is still enabled. This avoids a separate shared Xet data cache. +Large files are streamed, not loaded into one large in-memory payload. To reconstruct the recorded study, verify `SHA256SUMS`, then extract `study-metadata.zip` and every `episodes/**/*.zip` into one new directory. diff --git a/publishing/huggingface/publish_camera_action_v01.py b/publishing/huggingface/publish_camera_action_v01.py index 5bbb851..5c3fbe9 100644 --- a/publishing/huggingface/publish_camera_action_v01.py +++ b/publishing/huggingface/publish_camera_action_v01.py @@ -7,6 +7,8 @@ from __future__ import annotations import argparse +import ctypes +import errno from hashlib import sha256 import json import os @@ -185,22 +187,112 @@ def _remote(info): raise PublicationError('Expected an exact initialized public dataset head.') +def _space_guard(directory, additional=0): + if shutil.disk_usage(directory).free - additional < RESERVE: + raise PublicationError('Freezing the publication would consume the protected disk reserve.') + + +def _clone_from_fd(source_fd, directory_fd, name): + """Darwin clonefile(2): new inode/attributes; subsequent writes are private. + + Source and destination directory are already open without following links. + No hardlink fallback is permitted. Local SDK clonefile.h defines the + CLONE_NOOWNERCOPY flag as 0x0002; source is a regular-file descriptor. + """ + if sys.platform != 'darwin': + return False + library = ctypes.CDLL(None, use_errno=True) + try: + clone = library.fclonefileat + except AttributeError: + return False + clone.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32) + clone.restype = ctypes.c_int + if clone(source_fd, directory_fd, os.fsencode(name), 0x0002) == 0: + return True + number = ctypes.get_errno() + if number in {errno.ENOTSUP, errno.EOPNOTSUPP, errno.EXDEV, errno.ENOSYS}: + return False + raise OSError(number, 'Private copy-on-write freeze failed.') + + +def freeze_file(source, target, expected): + """Create and verify one private snapshot, with bounded streaming fallback.""" + source, target = plain(source), plain(target) + _record(expected) + if target.exists(): + raise PublicationError('Private freeze destination already exists.') + target.parent.mkdir(parents=True, exist_ok=True) + plain(target.parent) + _space_guard(target.parent, 1024 * 1024) # metadata headroom; no full-data reservation for a clone + created = False + source_fd = directory_fd = None + try: + source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + directory_fd = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + before = os.fstat(source_fd) + if not stat.S_ISREG(before.st_mode) or before.st_size != expected['bytes']: + raise PublicationError('Freeze source is not the reviewed ordinary file.') + cloned = _clone_from_fd(source_fd, directory_fd, target.name) + if cloned: + created = True + else: + _space_guard(target.parent, expected['bytes']) + fd = os.open(target.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) + created = True + with os.fdopen(fd, 'wb') as destination: + copied = 0 + while True: + chunk = os.read(source_fd, 1024 * 1024) + if not chunk: break + copied += len(chunk) + if copied > expected['bytes']: + raise PublicationError('Freeze source grew beyond its reviewed byte count.') + _space_guard(target.parent, len(chunk)) + destination.write(chunk) + destination.flush() + _space_guard(target.parent) + if copied != expected['bytes']: + raise PublicationError('Freeze source length changed while copying.') + os.fsync(destination.fileno()) + after = os.fstat(source_fd) + if any(getattr(before, k) != getattr(after, k) for k in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise PublicationError('Source changed while freezing reviewed publication bytes.') + frozen_fd = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) + try: + frozen_stat = os.fstat(frozen_fd) + if (not stat.S_ISREG(frozen_stat.st_mode) or frozen_stat.st_nlink != 1 + or (frozen_stat.st_dev, frozen_stat.st_ino) == (before.st_dev, before.st_ino)): + raise PublicationError('Freeze must create an independent ordinary inode, never a hardlink.') + os.fchmod(frozen_fd, 0o400) + finally: + os.close(frozen_fd) + _space_guard(target.parent) + if identity(target) != expected: + raise PublicationError('Payload changed while freezing reviewed publication bytes.') + return 'apfs_clone' if cloned else 'stream_copy' + except BaseException: + if created: + os.unlink(target.name, dir_fd=directory_fd) + raise + finally: + if source_fd is not None: os.close(source_fd) + if directory_fd is not None: os.close(directory_fd) + + def publish(plan, expected_digest, cache): value = checked(plan, expected_digest) cache = plain(cache) cache.mkdir(parents=True, exist_ok=True) - if shutil.disk_usage(cache).free - sum(r['bytes'] for r in value['files']) < RESERVE: - raise PublicationError('Freezing the publication would consume the protected disk reserve.') + _space_guard(cache, 1024 * 1024) with tempfile.TemporaryDirectory(prefix='camera-action-frozen-', dir=cache) as temporary: frozen = Path(temporary) + freeze_methods = dict(apfs_clone=0, stream_copy=0) for row in value['files']: source = plain(Path(value['root']) / row['path']) target = frozen / relative(row['path']) - target.parent.mkdir(parents=True, exist_ok=True) - with source.open('rb') as source_stream, target.open('xb') as destination: - shutil.copyfileobj(source_stream, destination, 1024 * 1024) - if identity(target) != {key: row[key] for key in ('bytes', 'sha256')}: - raise PublicationError('Payload changed while freezing reviewed publication bytes.') + method = freeze_file(source, target, {key: row[key] for key in ('bytes', 'sha256')}) + freeze_methods[method] += 1 # Originals are never reopened after the credential boundary. The # private immutable copy bounds memory while preserving reviewed bytes. api, token = client(cache) @@ -219,10 +311,21 @@ def publish(plan, expected_digest, cache): commit = api.create_commit(REPO, repo_type='dataset', operations=operations, parent_commit=info.sha, commit_message='Publish camera-to-action v0.1 reproducible simulation evidence') del token, api - return dict(repo_id=REPO, commit=commit.oid, files_added=len(operations)) + return dict(repo_id=REPO, commit=commit.oid, files_added=len(operations), freeze_methods=freeze_methods) + + +def _http_readback(): + # Hub constants are initialized at import time. Fail closed if a caller + # pre-imported an SDK that would route data into a separate Xet cache. + os.environ.update(HF_HUB_DISABLE_XET='1', HF_DEBUG='0', HF_HUB_DISABLE_TELEMETRY='1', + HF_HUB_DISABLE_IMPLICIT_TOKEN='1', HF_HUB_DISABLE_PROGRESS_BARS='1', HF_HUB_DISABLE_UPDATE_CHECK='1') + from huggingface_hub import constants + if getattr(constants, 'HF_HUB_DISABLE_XET', None) is not True: + raise PublicationError('Start verification in a fresh process with HF_HUB_DISABLE_XET=1 before SDK import.') def download(name, revision, cache): + _http_readback() from huggingface_hub import hf_hub_download return Path(hf_hub_download(REPO, name, repo_type='dataset', revision=revision, token=False, endpoint='https://huggingface.co', cache_dir=str(cache), force_download=True)).resolve() @@ -230,17 +333,36 @@ def download(name, revision, cache): def verify(plan, expected_digest, cache): value = checked(plan, expected_digest) + cache = plain(cache) + cache.mkdir(parents=True, exist_ok=True) + _http_readback() api = public_client() info = api.repo_info(REPO, repo_type='dataset', files_metadata=True, token=False) _remote(info) expected = {row['path'] for row in value['files']} if len(info.siblings) != len(expected) + 1 or {item.rfilename for item in info.siblings} != expected | {'.gitattributes'}: raise PublicationError('Public dataset roster differs from the reviewed plan.') + remote_files = {item.rfilename: item for item in info.siblings} + for row in value['files']: + size = getattr(remote_files[row['path']], 'size', None) + if type(size) is not int or size != row['bytes']: + raise PublicationError('Revision-pinned remote size differs from the reviewed per-file download budget.') for row in value['files']: - path = download(row['path'], info.sha, cache) - if identity(path) != {key: row[key] for key in ('bytes', 'sha256')}: - raise PublicationError('Anonymous public bytes differ from reviewed evidence.') - return dict(repo_id=REPO, commit=info.sha, anonymous_files_verified=len(value['files'])) + _space_guard(cache, row['bytes'] + 1024 * 1024) + with tempfile.TemporaryDirectory(prefix='camera-action-readback-', dir=cache) as temporary: + owned = Path(temporary) + path = download(row['path'], info.sha, owned) + if not path.resolve().is_relative_to(owned.resolve()): + raise PublicationError('Anonymous readback escaped its owned temporary cache.') + if identity(path) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Anonymous public bytes differ from reviewed evidence.') + _space_guard(cache) + return dict(repo_id=REPO, commit=info.sha, anonymous_files_verified=len(value['files']), + anonymous_bytes_verified=sum(r['bytes'] for r in value['files']), + maximum_readback_file_bytes=max(r['bytes'] for r in value['files']), + reviewed_plan_sha256=expected_digest, + readback_inventory_sha256=sha256(encoded(value['files'])).hexdigest(), + cache_policy='One file per owned temporary cache, removed after full pinned-revision byte verification; shared caches untouched.') def main(argv=None): diff --git a/publishing/huggingface/test_publish_camera_action_v01.py b/publishing/huggingface/test_publish_camera_action_v01.py index 11241c1..c5b9244 100644 --- a/publishing/huggingface/test_publish_camera_action_v01.py +++ b/publishing/huggingface/test_publish_camera_action_v01.py @@ -1,8 +1,10 @@ from copy import deepcopy from hashlib import sha256 import json +import os from pathlib import Path import sys +import stat from types import SimpleNamespace import tempfile import unittest @@ -41,14 +43,22 @@ def __init__(self, path_in_repo, path_or_fileobj): self.path_in_repo, self.path_or_fileobj = path_in_repo, path_or_fileobj class NotFound(Exception): pass - patches = patch.dict(sys.modules, {'huggingface_hub': SimpleNamespace(CommitOperationAdd=Operation), + patches = patch.dict(sys.modules, {'huggingface_hub': SimpleNamespace(CommitOperationAdd=Operation, + constants=SimpleNamespace(HF_HUB_DISABLE_XET=True)), 'huggingface_hub.errors': SimpleNamespace(RepositoryNotFoundError=NotFound)}) patches.start() self.addCleanup(patches.stop) def info(self, names=('.gitattributes',), *, private=False, sha='a' * 40): return SimpleNamespace(private=private, sha=sha, - siblings=[SimpleNamespace(rfilename=name) for name in names]) + siblings=[SimpleNamespace(rfilename=name,size=(self.dataset/name).stat().st_size if (self.dataset/name).is_file() else 0) for name in names]) + + def downloaded(self,name,revision,cache,*,wrong=False): + self.assertEqual(revision,'a'*40) + target=Path(cache)/'synthetic-download'/name + target.parent.mkdir(parents=True,exist_ok=True) + target.write_bytes(b'not the published bytes' if wrong else (self.dataset/name).read_bytes()) + return target def test_stage_check_and_full_public_readback_never_access_credentials(self): value = publisher.checked(self.plan, self.receipt['plan_sha256']) @@ -56,9 +66,14 @@ def test_stage_check_and_full_public_readback_never_access_credentials(self): api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) with patch.object(publisher, 'client', side_effect=AssertionError('Unexpected credential access')), \ patch.object(publisher, 'public_client', return_value=api), \ - patch.object(publisher, 'download', side_effect=lambda name, revision, cache: self.dataset / name): + patch.object(publisher, 'download', side_effect=self.downloaded) as download: proof = publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') self.assertEqual(proof['anonymous_files_verified'], len(value['files'])) + self.assertEqual(proof['anonymous_bytes_verified'],sum(r['bytes'] for r in value['files'])) + self.assertEqual(proof['reviewed_plan_sha256'],self.receipt['plan_sha256']) + self.assertEqual(len({call.args[2] for call in download.call_args_list}),len(value['files'])) + self.assertTrue(all(not Path(call.args[2]).exists() for call in download.call_args_list)) + self.assertEqual(list((self.root/'cache').iterdir()),[]) api.repo_info.assert_called_once_with(publisher.REPO, repo_type='dataset', files_metadata=True, token=False) def test_mutated_payload_and_changed_destination_fail_before_keychain(self): @@ -126,12 +141,54 @@ def test_public_readback_detects_changed_remote_bytes(self): value = publisher.checked(self.plan, self.receipt['plan_sha256']) api = Mock() api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) - changed = self.root / 'changed' - changed.write_bytes(b'not the published bytes') with patch.object(publisher, 'public_client', return_value=api), \ - patch.object(publisher, 'download', return_value=changed): + patch.object(publisher, 'download', side_effect=lambda *args:self.downloaded(*args,wrong=True)): with self.assertRaisesRegex(publisher.PublicationError, 'Anonymous public bytes'): publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(list((self.root/'cache').iterdir()),[]) + + def test_readback_protects_reserve_and_never_deletes_shared_cache_files(self): + value=publisher.checked(self.plan,self.receipt['plan_sha256']);api=Mock() + api.repo_info.return_value=self.info([r['path'] for r in value['files']]+['.gitattributes']) + cache=self.root/'cache';cache.mkdir();shared=cache/'user-shared-cache';shared.write_bytes(b'keep existing user data') + with patch.object(publisher,'public_client',return_value=api), \ + patch.object(publisher.shutil,'disk_usage',return_value=SimpleNamespace(free=publisher.RESERVE)), \ + patch.object(publisher,'download') as download: + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.verify(self.plan,self.receipt['plan_sha256'],cache) + download.assert_not_called() + with patch.object(publisher,'public_client',return_value=api),patch.object(publisher,'download',return_value=shared): + with self.assertRaisesRegex(publisher.PublicationError,'escaped its owned'): + publisher.verify(self.plan,self.receipt['plan_sha256'],cache) + self.assertEqual(shared.read_bytes(),b'keep existing user data') + self.assertEqual(list(cache.iterdir()),[shared]) + + def test_download_is_anonymous_and_revision_pinned_in_explicit_private_cache(self): + cache=self.root/'private-readback';cache.mkdir();blob=cache/'blob';blob.write_bytes(b'fixture') + download=Mock(return_value=str(blob)) + with patch.dict(sys.modules,{'huggingface_hub':SimpleNamespace(hf_hub_download=download, + constants=SimpleNamespace(HF_HUB_DISABLE_XET=True))}): + self.assertEqual(publisher.download('source/fixture.whl','a'*40,cache),blob.resolve()) + download.assert_called_once_with(publisher.REPO,'source/fixture.whl',repo_type='dataset',revision='a'*40, + token=False,endpoint='https://huggingface.co',cache_dir=str(cache),force_download=True) + + def test_missing_or_changed_remote_size_refuses_before_any_download(self): + value=publisher.checked(self.plan,self.receipt['plan_sha256']) + for size in (None,True,-1,10**12): + api=Mock();info=self.info([r['path'] for r in value['files']]+['.gitattributes']) + info.siblings[0].size=size;api.repo_info.return_value=info + with self.subTest(size=size),patch.object(publisher,'public_client',return_value=api), \ + patch.object(publisher,'download') as download: + with self.assertRaisesRegex(publisher.PublicationError,'remote size differs'): + publisher.verify(self.plan,self.receipt['plan_sha256'],self.root/'cache') + download.assert_not_called() + + def test_preimported_xet_enabled_sdk_refuses_before_remote_calls(self): + with patch.dict(sys.modules,{'huggingface_hub':SimpleNamespace(constants=SimpleNamespace(HF_HUB_DISABLE_XET=False))}), \ + patch.object(publisher,'public_client') as client: + with self.assertRaisesRegex(publisher.PublicationError,'fresh process'): + publisher.verify(self.plan,self.receipt['plan_sha256'],self.root/'cache') + client.assert_not_called() def test_qualified_label_requires_full_study_audit_even_when_report_matches_manifest(self): for name in ('manifest.json','report.json'): @@ -151,6 +208,74 @@ def test_private_hardware_fields_are_rejected_before_upload(self): with self.assertRaisesRegex(publisher.PublicationError,'private or unbounded'): publisher.files(self.dataset) + def test_private_freeze_has_independent_inode_attributes_and_mutations(self): + source=self.root/'source';source.write_bytes(b'reviewed bytes'*1000);source.chmod(0o640) + target=self.root/'private'/'frozen';expected=publisher.identity(source) + method=publisher.freeze_file(source,target,expected) + self.assertIn(method,('apfs_clone','stream_copy')) + self.assertNotEqual(source.stat().st_ino,target.stat().st_ino) + self.assertEqual(target.stat().st_nlink,1) + self.assertEqual(stat.S_IMODE(target.stat().st_mode),0o400) + self.assertEqual(stat.S_IMODE(source.stat().st_mode),0o640) + source.write_bytes(b'original mutated after private freeze') + self.assertEqual(publisher.identity(target),expected) + target.chmod(0o600);target.write_bytes(b'private mutation') + self.assertEqual(source.read_bytes(),b'original mutated after private freeze') + + def test_freeze_rejects_destination_collision_and_symlinks_without_overwrite(self): + source=self.root/'source';source.write_bytes(b'reviewed bytes');expected=publisher.identity(source) + target=self.root/'frozen';target.write_bytes(b'existing destination') + with self.assertRaisesRegex(publisher.PublicationError,'already exists'): + publisher.freeze_file(source,target,expected) + self.assertEqual(target.read_bytes(),b'existing destination') + target.unlink();target.symlink_to(source) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source,target,expected) + target.unlink();source_link=self.root/'source-link';source_link.symlink_to(source) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source_link,target,expected) + directory=self.root/'linked-parent';directory.symlink_to(self.root,target_is_directory=True) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source,directory/'frozen',expected) + + def test_stream_fallback_is_guarded_per_file_and_during_copy(self): + source=self.root/'source';source.write_bytes(b'x'*(2*1024*1024));expected=publisher.identity(source) + target=self.root/'private'/'frozen' + def usage(free):return SimpleNamespace(free=publisher.RESERVE+free) + with patch.object(publisher,'_clone_from_fd',return_value=False), \ + patch.object(publisher.shutil,'disk_usage',return_value=usage(1536*1024)): + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.freeze_file(source,target,expected) + self.assertFalse(target.exists()) + with patch.object(publisher,'_clone_from_fd',return_value=False), \ + patch.object(publisher.shutil,'disk_usage',side_effect=[usage(5*1024*1024)]*3+[usage(-1)]): + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.freeze_file(source,target,expected) + self.assertFalse(target.exists()) + with patch.object(publisher,'_clone_from_fd',return_value=False): + self.assertEqual(publisher.freeze_file(source,target,expected),'stream_copy') + source.write_bytes(b'late change');self.assertEqual(publisher.identity(target),expected) + + def test_growing_stream_source_refuses_before_writing_excess_and_cleans_target(self): + source=self.root/'source';source.write_bytes(b'x'*(2*1024*1024));expected=publisher.identity(source) + target=self.root/'private'/'frozen';read=os.read;calls=[] + def growing(fd,count): + chunk=read(fd,count);calls.append(len(chunk)) + if len(calls)==1: + with source.open('ab') as stream:stream.write(b'g'*(3*1024*1024)) + return chunk + with patch.object(publisher,'_clone_from_fd',return_value=False),patch.object(publisher.os,'read',side_effect=growing): + with self.assertRaisesRegex(publisher.PublicationError,'grew beyond'): + publisher.freeze_file(source,target,expected) + self.assertEqual(calls,[1024*1024]*3) + self.assertFalse(target.exists()) + + def test_atomic_clone_collision_does_not_delete_the_existing_destination(self): + source=self.root/'source';source.write_bytes(b'reviewed');target=self.root/'private'/'frozen' + def collision(*_args): + target.write_bytes(b'racing destination') + raise FileExistsError('Destination already created.') + with patch.object(publisher,'_clone_from_fd',side_effect=collision): + with self.assertRaises(FileExistsError):publisher.freeze_file(source,target,publisher.identity(source)) + self.assertEqual(target.read_bytes(),b'racing destination') + if __name__ == '__main__': unittest.main() diff --git a/publishing/prepare_camera_action_v01.py b/publishing/prepare_camera_action_v01.py index 8577328..39035fd 100644 --- a/publishing/prepare_camera_action_v01.py +++ b/publishing/prepare_camera_action_v01.py @@ -25,6 +25,7 @@ MAX_EPISODE = 1024 ** 3 MAX_TOTAL = 120 * 1024 ** 3 DISK_RESERVE = 30 * 1024 ** 3 +ZIP_LEVEL = 1 TEXT_SUFFIXES = {'.py', '.js', '.ts', '.json', '.html', '.md', '.txt', '.toml', '.yaml', '.yml'} SECRET = re.compile(rb'(?:hf_|gh[pousr]_|apikey_)[A-Za-z0-9_]{24,}') @@ -141,9 +142,11 @@ def episode_inventory(root): return result -def archive(root, inventory, destination, *, prefix=''): +def archive(root, inventory, destination, *, prefix='', compresslevel=ZIP_LEVEL): """Stream byte-bound files into a deterministic ZIP, then read every member.""" root, destination = plain(root), plain(destination) + if compresslevel not in (1, 6): + raise ValueError('Use a reviewed ZIP compression level: 1 or 6.') if destination.exists(): raise ValueError('Archive exists; existing publication artifacts are immutable.') if prefix: @@ -152,7 +155,7 @@ def archive(root, inventory, destination, *, prefix=''): if shutil.disk_usage(destination.parent).free - sum(r['bytes'] for r in inventory.values()) < DISK_RESERVE: raise ValueError('Archive preparation would consume the protected disk reserve.') expected = {} - with zipfile.ZipFile(destination, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=6, allowZip64=True) as output: + with zipfile.ZipFile(destination, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=compresslevel, allowZip64=True) as output: for name, record in sorted(inventory.items()): relative(name) checked_identity(record) @@ -163,7 +166,7 @@ def archive(root, inventory, destination, *, prefix=''): info.compress_type = zipfile.ZIP_DEFLATED info.create_system = 3 info.external_attr = (stat.S_IFREG | 0o644) << 16 - info._compresslevel = 6 + info._compresslevel = compresslevel h, size, tail = sha256(), 0, b'' with path.open('rb') as source, output.open(info, 'w', force_zip64=True) as target: before = os.fstat(source.fileno()) @@ -196,6 +199,7 @@ def archive(root, inventory, destination, *, prefix=''): raise ValueError('Archive decompression differs from sealed evidence.') return dict(**identity(destination, maximum=MAX_EPISODE), files=len(expected), unpacked_bytes=sum(r['bytes'] for r in expected.values()), + compression_level=compresslevel, inventory_sha256=sha256(canonical(expected)).hexdigest()) @@ -482,7 +486,7 @@ def prepare(study, output, *, audit, card, wheel, source_archive, source_commit, trained_model_files=models, files=inventory, hardware_context_file='hardware.json' if hardware is not None else None, reconstruction='Extract study-metadata.zip and every episodes/**/*.zip into one new directory. Compare SHA256SUMS first. Each recording retains its original integrity.json.', - archive_format='Deterministic ZIP64, sorted members, 1980 timestamp, regular 0644 files, DEFLATE level 6; all members read back byte-for-byte.') + archive_format=f'Deterministic ZIP64, sorted members, 1980 timestamp, regular 0644 files, DEFLATE level {ZIP_LEVEL}; all members read back byte-for-byte.') write(output / 'manifest.json', value) inventory_with_manifest = dict(inventory, **{'manifest.json': identity(output / 'manifest.json')}) sums = ''.join(f'{record["sha256"]} {name}\n' for name, record in sorted(inventory_with_manifest.items())) diff --git a/publishing/test_prepare_camera_action_v01.py b/publishing/test_prepare_camera_action_v01.py index 61a8848..3b5bf80 100644 --- a/publishing/test_prepare_camera_action_v01.py +++ b/publishing/test_prepare_camera_action_v01.py @@ -29,6 +29,7 @@ def test_deterministic_archives_restore_exact_original_integrity_inventory(self) first = package.archive(self.episode, inventory, self.root / 'first.zip', prefix='episodes/diagnostic/example') second = package.archive(self.episode, inventory, self.root / 'second.zip', prefix='episodes/diagnostic/example') self.assertEqual(first, second) + self.assertEqual(first['compression_level'],1) self.assertEqual((self.root / 'first.zip').read_bytes(), (self.root / 'second.zip').read_bytes()) with zipfile.ZipFile(self.root / 'first.zip') as archive: for name, expected in inventory.items(): @@ -36,6 +37,16 @@ def test_deterministic_archives_restore_exact_original_integrity_inventory(self) self.assertEqual({'bytes': len(raw), 'sha256': sha256(raw).hexdigest()}, expected) self.assertEqual(archive.getinfo('episodes/diagnostic/example/' + name).date_time, (1980, 1, 1, 0, 0, 0)) + def test_explicit_reviewed_compression_level_preserves_identical_raw_inventory(self): + inventory=package.episode_inventory(self.episode) + first=package.archive(self.episode,inventory,self.root/'fast.zip',compresslevel=1) + second=package.archive(self.episode,inventory,self.root/'compact.zip',compresslevel=6) + self.assertEqual(first['inventory_sha256'],second['inventory_sha256']) + self.assertEqual(first['unpacked_bytes'],second['unpacked_bytes']) + self.assertEqual(second['compression_level'],6) + with self.assertRaisesRegex(ValueError,'reviewed ZIP'): + package.archive(self.episode,inventory,self.root/'unreviewed.zip',compresslevel=9) + def test_changed_sealed_source_refuses_during_streaming_copy(self): inventory = package.episode_inventory(self.episode) (self.episode / 'rollout.json').write_text('{"status":"different"}\n') From 28632a29f38b17168459a1278f3d9f089ddea6fb Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sun, 1 Nov 2026 03:18:01 -0500 Subject: [PATCH 04/10] Render honest camera-action model and data cards from sealed results --- publishing/CAMERA-ACTION-V01.md | 67 +++ publishing/render_camera_action_card.py | 516 +++++++++++++++++++ publishing/test_render_camera_action_card.py | 231 +++++++++ 3 files changed, 814 insertions(+) create mode 100644 publishing/render_camera_action_card.py create mode 100644 publishing/test_render_camera_action_card.py diff --git a/publishing/CAMERA-ACTION-V01.md b/publishing/CAMERA-ACTION-V01.md index e52b292..a133ce2 100644 --- a/publishing/CAMERA-ACTION-V01.md +++ b/publishing/CAMERA-ACTION-V01.md @@ -119,3 +119,70 @@ receipts bind those exact files. A blocked data-quality gate therefore publishes evidence and source without inventing a learned policy. The package retains the study's qualification result; installation itself does not award task qualification or physical-hardware readiness. + +## Stopping, interruption and publication scope + +Studio and Terminal use the same immutable stage records. Studio owns one local +worker, checks the protocol and assets between stages, and stops at a blocked +or completed study. It does not retry a stage after an exception. A browser +disconnect does not cancel the server's worker. Closing or killing the server +is not a supported pause mechanism. + +A native attempt that can write its final receipt retains its completed prefix +as `interrupted` or `refused`; it is not counted as a strict success. If a process +ends before sealing an episode, a subsequent collection/evaluation inspection +retains that existing directory as `unknown` and never reruns its case. Remaining +registered cases are not replaced with easier starts. A sealed stage with an +unknown or unattempted case blocks downstream work. + +Fitting has a stricter interruption boundary. Once a model variant's directory +has been created, the frozen trainer refuses to fit that variant again. A fit +exception can therefore leave a model prefix without a sealed training stage. +In this state, the small `inspect` response may still list training as the next +stage; that is not permission or evidence that an automatic retry is possible. +Preserve the directory, fitting log and original stage-start record. Do not +delete them, fabricate a completed receipt or award a model qualification. + +An ordinary exception releases the study worker lock. A hard process stop may +leave `worker.lock`; Studio deliberately neither guesses whether it is stale +nor removes it. Any operator investigation must first establish that the +original worker has stopped and preserve the interrupted evidence. Do not +remove a lock to start a second worker against a live study. + +The raw-evidence publisher accepts only stopped, sealed `blocked` or `complete` +studies. It refuses an interrupted fitting prefix or an unsealed episode; those +cannot silently disappear from this dataset. If such an interruption happens, +publish its separate, explicitly incomplete evidence report without changing +the registered study or retrying the fit. This is an operational blocker, not a +negative task-success measurement or a completed benchmark. + +## Reviewed source snapshot + +This change is stacked on `feat/piper-rgbd-tracker-v01`, whose exact local base +for this study is `769d666ecd684406ae228daf722e41f0bf9ac17d`. The camera/action +branch is `feat/piper-camera-action-v01`; use the tracker branch as its pull +request base, not the older stable `main`. Recheck remote ancestry before +publication rather than merging unrelated changes into the frozen study. + +Build release artifacts only after the final tooling and report changes are +reviewed and committed. Use that exact commit for both the source archive and +wheel. The archive's PAX comment must identify the commit; the packager checks +that the registered runtime and independent auditor bytes agree. Keep the +existing development version; do not replace a stable release. + +```sh +git status --short +git merge-base HEAD origin/feat/piper-rgbd-tracker-v01 +git rev-list --left-right --count origin/feat/piper-rgbd-tracker-v01...HEAD +git rev-parse HEAD +git archive --format=tar.gz --prefix=dvidia-training/ \ + --output=/absolute/new-release/reviewed-source.tar.gz EXACT_REVIEWED_COMMIT +``` + +Extract that reviewed Git archive into a fresh build directory, then build the +wheel from its `dvidia-training` directory with the existing local Python build +environment (`python -m build --wheel --no-isolation --outdir +/absolute/new-release`). This keeps uncommitted files and generated run data out +of the source release. Re-run the fresh-environment installation checks against +the final wheel, and retain its digest with the publication plan. Neither the +source snapshot nor a successful install proves task qualification. diff --git a/publishing/render_camera_action_card.py b/publishing/render_camera_action_card.py new file mode 100644 index 0000000..5a42be2 --- /dev/null +++ b/publishing/render_camera_action_card.py @@ -0,0 +1,516 @@ +"""Render publication cards from a stopped study, never from hand-entered metrics. + +No simulation, training, credentials or network calls. A supplied posthoc summary +is recomputed from sealed numeric receipts; a completed study additionally needs +the independent full-study audit. Blocked studies remain explicit candidates. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path +import re + +import prepare_camera_action_v01 as evidence +from huggingface.camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware + +SCHEMA = 'dvidia.camera-action-cards/v1' +REPO = Path(__file__).resolve().parents[1] +SUMMARY_SOURCE = REPO / 'benchmarks/camera-action-v01/summarize_results.py' +PHASE_SOURCE = REPO / 'benchmarks/camera-action-v01/diagnostic_phase_coverage.py' +SOURCE_URL = 'https://github.com/Dvidia-Inference/dvidia-training' +STAGES = ('diagnostic', 'demonstrations', 'train-initial', 'development-initial', + 'corrections', 'train-corrected', 'development-corrected', 'confirmation', 'negative') +BUDGETS = {'diagnostic': 4, 'demonstrations': 80, 'development-initial': 32, + 'corrections': 32, 'development-corrected': 32, 'confirmation': 256, 'negative': 16} + + +def recompute_summary(study): + spec = importlib.util.spec_from_file_location('camera_action_card_posthoc', SUMMARY_SOURCE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.summarize(study) + + +def recompute_phase(study, diagnostic_audit): + spec = importlib.util.spec_from_file_location('camera_action_card_phase', PHASE_SOURCE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.analyze(study, diagnostic_audit) + + +def _cell(value): + if value is None: + return 'Not reported' + return str(value).replace('&', '&').replace('<', '<').replace('>', '>').replace('|', '\\|').replace('`', '`').replace('\n', ' ').replace('\r', ' ') + + +def _number(value, digits=3): + return 'Not reported' if value is None else f'{value:.{digits}f}' + + +def _table(headers, rows): + return ['| ' + ' | '.join(headers) + ' |', '|' + '|'.join('---' for _ in headers) + '|', + *('| ' + ' | '.join(_cell(cell) for cell in row) + ' |' for row in rows), ''] + + +def _inputs(study, *, summary, source_commit, hardware=None, study_audit=None, phase_coverage=None, diagnostic_audit=None): + from dvidia_training import camera_action_study as workflow + study = evidence.plain(study) + if (study / 'worker.lock').exists(): + raise ValueError('Resolve the active or interrupted study worker before rendering publication cards.') + if not re.fullmatch('[a-f0-9]{40}', source_commit): + raise ValueError('An exact reviewed source commit is required.') + state = workflow.inspect(study) + if state['status'] not in ('complete', 'blocked'): + raise ValueError('Cards require a completed study or an explicitly blocked study.') + protocol = evidence.read(study / 'protocol.json') + if {name: len(rows) for name, rows in protocol['rosters'].items()} != BUDGETS: + raise ValueError('This card template requires the exact registered v0.1 stage budgets.') + if (protocol.get('capture') != dict(duration_s=20., width=320, height=240, camera_hz=25, + control_hz=50, physics_timestep_s=.001, camera_pairs_per_complete_episode=501) + or protocol.get('fitting') != dict(hidden_units=[64, 64], epochs=50, batch_size=512, + learning_rate=.003, seed=17, normalization='training-only', corrections_rounds=1, + teacher_mix_probability=.5) + or protocol.get('start_domain', {}).get('x_m') != [.287, .311] + or protocol.get('start_domain', {}).get('y_m') != [-.042, -.018] + or {key: len(value) for key, value in protocol.get('cases', {}).items()} != + dict(train=64, development=16, confirmation=64)): + raise ValueError('Capture, fitting or start-domain registration differs from this v0.1 methodology.') + report = evidence.read(study / 'report.json') + if report != state['report'] or report.get('physical_robot_ready') is not False: + raise ValueError('Saved report differs from sealed study inspection.') + if state['status'] == 'complete' and report['status'] != 'complete': + raise ValueError('Completed study lacks a complete result report.') + if state['status'] == 'blocked' and (report.get('status') != 'blocked' or report.get('qualified') is not False): + raise ValueError('Blocked studies must remain unqualified research candidates.') + if study_audit is None and state['status'] != 'blocked': + raise ValueError('Completed cards require the independent full-study audit.') + receipts = {name: receipt for name in STAGES if (receipt := workflow._stage(study, name)) is not None} + proof = evidence.verify_study_audit(study, study_audit, receipts, state) if study_audit is not None else None + document = evidence.read(summary) + if document != recompute_summary(study): + raise ValueError('Posthoc summary differs from fresh read-only reconstruction of sealed receipts.') + if document.get('schema') != 'dvidia.camera-action-posthoc-summary/v1' or document.get('physical_robot_ready') is not False: + raise ValueError('Expected the simulation-only posthoc summary format.') + expected_qualification = dict(status=report['status'], qualified=report['qualified'], + qualification=report['qualification'], report_sha256=evidence.identity(study / 'report.json')['sha256'], + source='existing study report; not newly awarded by this posthoc tool') + if document.get('qualification') != expected_qualification: + raise ValueError('Posthoc qualification is not bound to the final report.') + bindings = {'protocol.json': evidence.identity(study / 'protocol.json'), + 'registration.json': evidence.identity(study / 'registration.json'), + 'report.json': evidence.identity(study / 'report.json')} + for name, receipt in receipts.items(): + for suffix in ('.json', '.integrity.json', '.started.json'): + path = 'stages/' + name + suffix + bindings[path] = evidence.identity(study / path) + if document['source_receipts'].get('stages/' + name + '.json') != bindings['stages/' + name + '.json']['sha256']: + raise ValueError('Posthoc stage identity differs from the sealed receipt.') + if set(document['source_receipts']) != {'stages/' + name + '.json' for name in receipts}: + raise ValueError('Posthoc stage roster differs from current sealed evidence.') + selected = report.get('selected_stage') + if selected is not None: + frozen = evidence.read(study / 'freeze.json') + if selected not in ('initial', 'corrected') or frozen.get('selected_stage') != selected: + raise ValueError('Selected model stage differs from frozen selection.') + bindings['freeze.json'] = evidence.identity(study / 'freeze.json') + models = [] + for stage in ('initial', 'corrected'): + receipt = receipts.get('train-' + stage) + if receipt is None: + continue + if set(receipt['models']) != {'full', 'blind'}: + raise ValueError('Each sealed fit must retain the matched full and blind models.') + for variant in ('full', 'blind'): + record = receipt['models'][variant] + path = f'models/{stage}/{variant}/model.json' + if record['path'] != path: + raise ValueError('Model receipt path differs from its fitting stage.') + actual = evidence.identity(study / path) + if actual != dict(bytes=record['model_bytes'], sha256=record['model_sha256']): + raise ValueError('Model bytes differ from the sealed fitting receipt.') + from dvidia_training.camera_action_model import validate_model + model = validate_model(evidence.read(study / path)) + if (model.get('schema') != 'dvidia.camera-action-mlp/v1' or model.get('blind') is not (variant == 'blind') + or model.get('physical_robot_ready') is not False or model.get('hidden_units') != [64, 64] + or model['training'] != record.get('training') or model['training']['stage'] != stage): + raise ValueError('Model schema or architecture differs from this card template.') + for filename in ('model.json', 'receipt.json', 'started.json'): + name = f'models/{stage}/{variant}/{filename}' + bindings[name] = evidence.identity(study / name) + if evidence.read(study / f'models/{stage}/{variant}/receipt.json') != record: + raise ValueError('Individual model receipt differs from sealed fitting stage.') + models.append(dict(stage=stage, variant=variant, path=path, identity=actual, + selected=selected == stage, training=record, model=model)) + if selected and {row['variant'] for row in models if row['selected']} != {'full', 'blind'}: + raise ValueError('Selected full/blind pair is missing.') + expected_model_files = {name for name in bindings if name.startswith('models/')} + model_root = study / 'models' + if model_root.exists(): + actual_model_files = {path.relative_to(study).as_posix() for path in evidence.files(model_root)} + expected_directories = {str(Path(name).parent) for name in expected_model_files} + expected_directories |= {str(Path(name).parent) for name in expected_directories} + actual_directories = {path.relative_to(study).as_posix() for path in model_root.rglob('*') if path.is_dir()} + if actual_model_files != expected_model_files or actual_directories != expected_directories: + raise ValueError('Unsealed or orphan model prefixes cannot disappear from a publication card.') + hardware_doc = None + if hardware is not None: + evidence.identity(hardware, maximum=MAX_HARDWARE_BYTES, secret_scan=True) + hardware_doc = validate_hardware(evidence.read(hardware)) + declared_runtime = {name: hardware_doc[name] for name in ('system', 'machine', 'python')} + declared_runtime.update(hardware_doc['versions']) + if any(any(row['recorded_runtime'].get(name) != value for name, value in declared_runtime.items()) + for row in document['hardware']): + raise ValueError('Public hardware/software context differs from the recorded episode runtime.') + phase = None + if phase_coverage is not None: + if diagnostic_audit is None: + raise ValueError('Phase coverage requires its independently audited diagnostic source.') + phase = evidence.read(phase_coverage) + if (phase.get('schema') != 'dvidia.camera-action-phase-coverage/v1' + or phase.get('protocol_sha256') != bindings['protocol.json']['sha256'] + or phase.get('diagnostic_audit_sha256') != evidence.identity(diagnostic_audit)['sha256'] + or phase != recompute_phase(study, diagnostic_audit)): + raise ValueError('Diagnostic phase coverage differs from this protocol.') + elif diagnostic_audit is not None: + raise ValueError('A diagnostic audit is used only with its supplied phase coverage.') + return dict(study=study, source_commit=source_commit, state=state, protocol=protocol, + report=report, receipts=receipts, summary=document, models=models, hardware=hardware_doc, + audit=proof, phase=phase, study_bindings=bindings) + + +def _status(data): + report = data['report'] + if report['qualified']: + return 'Bounded simulation skill: all registered gates passed. Physical robot ready: **no**.' + if report['status'] == 'blocked': + return '**Research candidate — blocked.** The study stopped at a declared blocker; missing stages are unrun, not successful. Physical robot ready: **no**.' + return '**Research candidate — qualification gates failed.** A completed benchmark does not establish an installable qualified skill. Physical robot ready: **no**.' + + +def _results(data): + report, summary = data['report'], data['summary'] + lines = ['## Registered experiment and results', '', + '72 fresh starting-position groups are split 32 training / 8 development / 32 confirmation. ' + 'Both destinations and all related recordings stay within their group. ' + 'The tiny interpolation domain is x = 0.287–0.311 m, y = −0.042–−0.018 m, z = 0.082 m. ' + 'Demonstrations comprise 64 training and 16 development episodes. No failed scene is replaced.', ''] + rows = [] + for name in STAGES: + receipt = data['receipts'].get(name) + budget = BUDGETS.get(name, '2 models; 50 epochs each') + rows.append((name, budget, receipt['status'] if receipt else 'Unrun', + receipt.get('strict_successes') if receipt and name in BUDGETS else None)) + lines += _table(['Stage', 'Registered budget', 'Status', 'Strict successful attempts'], rows) + lines += ['All registered rollout budgets total **452 attempts**. Fitting does not add native attempts. ' + 'The 32 correction episodes use training groups only and seeded 50% teacher control. ' + 'Development evaluates two stages × two model variants × 16 cases = 64 rollouts. ' + 'Frozen confirmation is 64 cases × four controllers = 256 attempts. Negatives add 16 attempts.', ''] + lines += ['Before fitting, all four diagnostics must strictly succeed with a valid initial visual anchor ' + 'within 10 mm of native diagnostic truth. Teacher data must cover every training start-group ' + 'at least once and provide at least 28 successful usable episodes per destination.', ''] + quality = data['receipts'].get('demonstrations', {}).get('data_quality') + if quality is not None: + lines += ['Recorded teacher-data gate:', '', '```json', json.dumps(quality, indent=2, sort_keys=True), '```', ''] + rows = [(row['stage'] + ' / ' + row['controller'], f"{row['strict_successes']} / {row['scheduled']}", + row['complete'], row['native_assessable'], json.dumps(row['statuses'], sort_keys=True)) + for row in summary['controllers']] + lines += _table(['Stage / controller', 'Strict success / scheduled', 'Complete', 'Native assessable', 'Retained statuses'], rows) + lines += ['Contact-affected counts use the independent 1 ms native scorer; unassessed attempts stay explicit.', ''] + lines += _table(['Stage / controller', 'Contact-affected / assessed', 'Unassessed', 'Critical native ticks'], + [(r['stage'] + ' / ' + r['controller'], + f"{r['critical_contacts']['attempts_with_contacts']} / {r['critical_contacts']['assessed_attempts']}", + r['critical_contacts']['unassessed_attempts'], r['critical_contacts']['native_ticks']) + for r in summary['controllers']]) + lines += ['Component cells are **failed / conditionally assessed; unassessed overall / eligible / unknown prerequisite**. ' + 'Lift requires observed grasp, carry/retention require observed lift, and release/settling/retreat ' + 'require supported carry. These conditional diagnostics do not replace all-attempt task success.', ''] + component_rows = [] + for r in summary['controllers']: + cells = [r['stage'] + ' / ' + r['controller']] + for name in ('grasp', 'lift', 'carry', 'retention', 'release', 'settling', 'retreat'): + c = r['components'][name] + cells.append(f"{c['conditional_failed']} / {c['conditional_assessed']}; " + f"{c['unassessed']} / {c['conditional_unassessed']} / {c['prerequisite_unknown']}") + component_rows.append(cells) + lines += _table(['Stage / controller', 'Grasp', 'Lift', 'Carry', 'Retention', 'Release', 'Settling', 'Retreat'], component_rows) + lines += ['These posthoc tables retain unknown prerequisites and all-attempt unassessed counts; ' + 'an absent prerequisite does not become an observed component failure.', ''] + lines += ['### Qualification gates', '', + 'Required: ≥58/64 strict learned placements; positive 97.5% paired bootstrap intervals versus ' + 'both fixed and blind controllers, grouping both destinations by starting position; positive intervals ' + 'also on at least 24 common admitted groups; no increase in critical-contact episode frequency; ' + 'all 16 negative controls correct; no privileged learner inputs or teacher evaluation fallback.', ''] + lines += _table(['Gate', 'Result'], [(name, 'PASS' if passed else 'FAIL') for name, passed in report['gates'].items()]) if report['gates'] else ['Gates are **unassessed** because the study is blocked.', ''] + for label, key in (('All scheduled starting groups', 'comparisons'), ('Common admitted starting groups', 'admitted_comparisons')): + if report.get(key): + lines += [label + ' (verbatim structured result):', '', '```json', + json.dumps(report[key], indent=2, sort_keys=True, allow_nan=False), '```', ''] + for name, receipt in data['receipts'].items(): + if receipt.get('status') == 'blocked': + lines += [f"Blocked at **{name}**: {_cell(receipt.get('reason'))}.", ''] + return lines + + +def _observations(data): + lines = ['## Perception, speed and hardware', '', + 'RGB-D is captured at 320×240 and 25 Hz; encoders/commands at 50 Hz; independent native checks ' + 'at 1 ms. A complete 20-second episode has 501 camera pairs and 1,000 motor intervals. ' + 'Missing estimates remain missing with masks and frame age; the initial anchor is explicitly historical. ' + 'The known-upright-block tracker can abstain when the gripper occludes or tilts the block.', ''] + rows = [] + for row in data['summary']['controllers']: + p, t = row['perception'], row['timing'] + inference = t['policy_inference_wall_seconds'] + loop = t['full_boundary_wall_seconds'] + rows.append((row['stage'] + ' / ' + row['controller'], + f"{p['available_decisions']} / {p['assessed_decisions']}", p['episodes_unknown'], + _number(p['maximum_missing_seconds']), + _number(inference['p50'] * 1000 if inference['p50'] is not None else None) if inference['applicable'] else 'Not applicable', + _number(inference['p95'] * 1000 if inference['p95'] is not None else None) if inference['applicable'] else 'Not applicable', + _number(loop['p95'] * 1000 if loop['p95'] is not None else None))) + lines += _table(['Stage / controller', 'Available / assessed visual decisions', 'Episodes unreported', + 'Longest missing (s)', 'Inference p50 (ms)', 'Inference p95 (ms)', 'Full boundary p95 (ms)'], rows) + lines += ['Latencies pool individual recorded boundaries. A teacher zero is not a learned-policy inference measurement. ' + 'Synthetic camera time, source-frame age, simulation time and wall-clock computation are distinct. ' + 'The simulator pauses for computation on a shared host: these numbers do **not** establish physical ' + 'sensor-to-actuator latency, a hard deadline or hardware control frequency.', ''] + phase = data['phase'] + if phase: + lines += [f"**Diagnostic manipulation blind spot:** {phase['manipulation_available']} / " + f"{phase['manipulation_decisions']} visual estimates were available during " + + ', '.join(_cell(name) for name in phase['manipulation_phases']) + + f" across {phase['cases']} teacher diagnostic episodes. " + 'This is a diagnostic subset, not student confirmation. Teacher phase is used only in this ' + 'offline readback and is excluded from learner features; repeated 50 Hz decisions share 25 Hz camera frames.', ''] + rows = [(row['stage'] + ' / ' + row['variant'], row['model_bytes'], _number(row['training_wall_seconds']), + _number(row['shared_export_wall_seconds']), row['process_peak_rss_bytes'], row['training_episodes']) + for row in data['summary']['training']] + lines += _table(['Fit', 'Model bytes', 'Fit/save wall seconds', 'Shared export seconds', 'Process peak RSS bytes', 'Training episodes'], rows) + lines += ['Process peak RSS includes shared dataset export and any preceding fit; it is not isolated model memory. ' + 'Training uses CPU NumPy. Native rendering needs a working graphics driver/OpenGL context; no CUDA requirement ' + 'or minimum-hardware qualification is inferred from this host.', ''] + if data['hardware']: + lines += ['Public host context supplied with this release (`hardware.json`); software identifiers ' + 'are cross-checked against bound episode runtime receipts when present:', '', '```json', + json.dumps(data['hardware'], indent=2, sort_keys=True), '```', ''] + else: + lines += ['CPU model, memory capacity and GPU identity were not supplied in a bound public hardware context; ' + 'those details remain **not reported**.', ''] + for row in data['summary']['hardware']: + lines += [f"Recorded runtime in {row['episode_receipts']} episode receipts:", '', '```json', + json.dumps(row['recorded_runtime'], indent=2, sort_keys=True), '```', ''] + return lines + + +def dataset_card(data): + protocol_hash = data['study_bindings']['protocol.json']['sha256'] + lines = ['---', 'license: mit', 'task_categories:', '- robotics', 'language:', '- en', + 'tags:', '- robotics', '- simulation', '- behavior-cloning', '- synthetic-data', '- piper', '---', '', + '# DVIDIA · Camera-to-action v0.1', '', _status(data), '', + 'A local, reproducible PiPER block-placement experiment: learn seven absolute motor targets ' + 'from synchronized synthetic RGB-D, encoders and demonstrated actions; evaluate the complete ' + 'approach, grasp, lift, carry, release, settling and retreat task on unseen starting positions.', '', + f"[Reviewed source]({SOURCE_URL}/tree/{data['source_commit']}) · " + '[Research](https://research.dvidia.org) · [DVIDIA](https://dvidia.org)', '', + f"Protocol SHA-256: `{protocol_hash}`. Source commit: `{data['source_commit']}`.", '', + '## Dataset origin and intended use', '', + 'All footage is rendered in MuJoCo. Motor labels come from an authored teacher with privileged ' + 'simulator state. No human video, human teleoperation, private footage or physical robot measurements ' + 'are claimed. Initial demonstrations label issued targets; the correction round labels separately ' + 'recorded teacher suggestions, while the tape also preserves the commands actually executed.', '', + 'Use this evidence to study bounded camera/encoder imitation and its failures. ' + 'It does not demonstrate arbitrary-video action recovery, general world modeling, shoes, ' + 'arbitrary-arm transfer or physical hardware qualification.', '', + '## Model contract and selection', '', + 'A CPU NumPy MLP with two 64-unit tanh hidden layers predicts six joint targets and one gripper target. ' + 'Each fitting stage uses 50 epochs, batch size 512, Adam learning rate 0.003 and seed 17. ' + 'Training-only normalization includes explicit missing fills and masks. The model uses 160 ms ' + 'of causal history, current encoders, encoder-derived link6 pose, previous applied servo references, ' + 'goal, elapsed time and a historical initial visual position. It never receives teacher phase, ' + '`reference.action`, the current teacher target, native object/contact truth, future frames or ' + 'native object orientation.', '', + 'The matched blind model masks all live/historical sensor features, retaining the same initial ' + 'visual context, goal, time and causal prior-command history. It sees the same training episodes. ' + 'The legacy tcp_* feature fields are link6 body-origin estimates, not a calibrated tool point; ' + 'the native scorer uses a distal site at link6 local [0, 0, 0.12003] m.', '', + 'Selection uses full-learner development success, then critical-contact ticks, then common ' + 'physical-action-scale prediction loss, then initial before corrected. The blind comparator ' + 'comes from that same selected stage. Confirmation permits no teacher intervention.', ''] + models = data['models'] + lines += _table(['Stage', 'Variant', 'Selected', 'Model artifact', 'Bytes', 'SHA-256'], + [(r['stage'], r['variant'], r['selected'], r['path'], r['identity']['bytes'], r['identity']['sha256']) for r in models]) + if not models: + lines += ['No completed model fit is sealed before the blocker. No downloadable skill is claimed.', ''] + lines += _results(data) + _observations(data) + lines += ['## Limitations and independent verification', '', + 'Negative controls are deliberately narrow: initial_missing blanks the initial RGB while leaving ' + 'the object in the simulator; initial_stale sets capture/availability to −0.2 s; altered_calibration ' + 'scales one rotation element by 1.1; open_gripper forces the jaw open. Passing these checks does ' + 'not establish detection of physical absence, all stale-frame cases or small valid rigid calibration errors.', '', + 'Depth, contact, friction, actuator fidelity, sensor delivery and the sim-to-real gap remain ' + 'unqualified. A passing integrity audit certifies consistency of saved evidence, not task success. ' + 'The posthoc summary does not tune or award qualification.', '', + ('The passed full-study audit covers every currently sealed stage, roster, action authority, clocks ' + 'and independent native score reconstruction. No physics, tracker or policy was re-executed.' if data['audit'] else + '**No full-study independent audit was supplied.** This blocked publication is explicitly unaudited beyond ' + 'the sealed receipts and read-only posthoc reconstruction; it cannot award a qualified-skill label.'), '', + 'See `report.json`, `protocol.json`, `full-study-audit.json` when supplied, and the raw per-episode ' + 'integrity inventories. The table generator rechecks a local posthoc summary against sealed receipts; ' + 'its rendering receipt and standalone model-card drafts are local review artifacts, not additional release files. ' + 'All failed, refused, interrupted and unattempted outcomes remain in ' + 'their registered denominators. Unassessed component outcomes are not guessed from overall failure.', '', + '## Reconstruct the raw study and run locally', '', + 'The release contains source/wheel artifacts, `study-metadata.zip`, and one deterministic ZIP ' + 'per recorded attempt under `episodes/`. Verify the manifest and checksums before extracting. ' + 'Archives preserve each original recording integrity file; the compact diagnostic viewer is ' + 'not the full raw archive.', '', + '```sh', 'shasum -a256 -c SHA256SUMS', + 'python -m zipfile -e study-metadata.zip restored-study', '```', '', + 'Extract every episode archive listed in `manifest.json` into that same fresh `restored-study` ' + 'directory; each archive already contains its `episodes///` prefix. ' + 'Install the released wheel with its `arm` extra, or extract the reviewed source archive and run:', '', + '```sh', 'python3.12 -m venv .venv', 'source .venv/bin/activate', + "python -m pip install '.[arm]'", 'dvidia-train robot inspect --study /path/to/restored-study', + 'dvidia-training-studio --directory /path/to/new-local-runs --port 8363', '```', '', + 'The UI at `http://127.0.0.1:8363/robot` and Terminal share the same run record. ' + 'Reproduction uses a **fresh** study, the published fixed comparator and separately provisioned ' + 'verified PiPER assets; original study attempts are immutable:', '', + '```sh', 'dvidia-train robot prepare --study /path/to/new-study --fixed-model /path/to/restored-study/fixed-model.json', + 'dvidia-train robot collect --study /path/to/new-study --assets /path/to/robot-assets --stage diagnostic', + 'dvidia-train robot inspect --study /path/to/new-study', '```', '', + f"[Complete collect/train/evaluate/package commands]({SOURCE_URL}/blob/{data['source_commit']}/docs/camera-action-v01.md). " + 'After dependencies/assets are installed, inference and training are local. The exact PiPER ' + 'simulation adapter is the only accepted embodiment. Native stepping is CPU work; rendering ' + 'needs OpenGL. The tested renderer is macOS CGL. Preserve 30 GiB free disk; raw evidence is ' + 'bounded at 120 GiB. Other platforms and hardware minimums remain unqualified.', ''] + if data['report'].get('selected_stage'): + path = f"models/{data['report']['selected_stage']}/full/model.json" + lines += ['Install the selected model as a **simulation candidate**. Installation does not change its qualification:', '', + '```sh', f'dvidia-arm check --model {path} --robot piper', + f'dvidia-arm install --model {path} --robot piper --assets /path/to/robot-assets --output /path/to/new-installed-skill', + 'dvidia-arm run /path/to/new-installed-skill --assets /path/to/robot-assets --output /path/to/new-attempt --start-x .30 --start-y -.03 --goal 0', + '```', ''] + lines += ['## License and attribution', '', + 'Original DVIDIA code, generated dataset evidence and learned parameter JSON are MIT. ' + 'Upstream robot models, MuJoCo and other third-party components retain their own licenses ' + 'and notices; assets are provisioned separately. See `LICENSE` and the source third-party notices. ' + 'Individual contributor attribution is intentionally unset.', ''] + return '\n'.join(lines) + + +def model_card(data, record): + variant = record['variant'] + selected = record['selected'] + lines = ['---', 'license: mit', 'tags:', '- robotics', '- simulation', '- behavior-cloning', '- piper', '---', '', + f"# DVIDIA · PiPER camera-to-action · {record['stage']} / {variant}", '', + (_status(data) if selected and variant == 'full' else + '**Research candidate — comparison artifact.** This model has no separately awarded skill qualification. ' + 'The study gates concern the selected full-feedback model. Physical robot ready: **no**.'), '', + ('This is the selected full-feedback model.' if variant == 'full' else + 'This is the matched blind baseline, not the deployable full-feedback skill.') if selected else + 'This development candidate was not selected for frozen confirmation.', '', + f"Model file: `model.json` ({record['identity']['bytes']} bytes). SHA-256: `{record['identity']['sha256']}`.", '', + f"Reviewed source: [{data['source_commit']}]({SOURCE_URL}/tree/{data['source_commit']}). " + f"Protocol: `{data['study_bindings']['protocol.json']['sha256']}`.", '', + '## Inputs, output and training', '', + 'Data-only JSON, schema `dvidia.camera-action-mlp/v1`; no pickle or remote code. ' + 'CPU NumPy, two 64-unit tanh hidden layers, seven absolute targets (six joints and jaw). ' + 'Native joint/rate limits and independent task checks remain in the simulation runtime.', '', + ('Full feedback uses historical initial visual position, goal, time, current encoder/link6 measurements, ' + 'previous applied commands and 160 ms of causal sensor history with explicit masks and age.' if variant == 'full' else + 'All live and historical sensor-feedback tensor channels are masked. Initial visual context, goal, time ' + 'and causal previous-applied-command history remain, exactly matching the full model context.'), '', + 'Labels are authored privileged-simulator teacher actions; they are not recovered human actions. ' + 'The correction pass uses separately recorded teacher queries at mixed-controller visited states. ' + 'Executed commands and suggestions retain separate authority. No current teacher command, ' + 'reference.action, teacher phase, native object/contact truth or future frame is a learner input.', '', + 'Fitting budget: seed 17, 50 epochs, batch 512, Adam 0.003, training-only normalization including ' + 'missing zeros/masks. The full and blind models use identical source recordings.', '', + '## Measured fitting cost', ''] + training = record['training'] + lines += _table(['Measurement', 'Value'], [(key, training.get(key)) for key in + ('training_wall_seconds', 'shared_export_wall_seconds', 'process_peak_rss_bytes', 'memory_scope')]) + lines += [f"Training recordings: {len(training['training_episodes'])}; development recordings: {len(training['development_episodes'])}.", '', + 'Model provenance:', '', '```json', json.dumps({key: record['model']['training'].get(key) for key in + ('stage', 'sample_count', 'train_groups', 'dataset_sha256', 'provenance_sha256', 'model_source_sha256')}, + indent=2, sort_keys=True), '```', '', + '## Evaluation and use', '', + 'See the [dataset card](../../../README.md) and `report.json` at the release root ' + 'for all four confirmation controllers, failed gates, grouped intervals, negative controls and ' + 'perception/latency denominators. A model file or successful installation is not evidence of skill qualification.', ''] + matching = [r for r in data['summary']['controllers'] if r['controller'] == ('student' if variant == 'full' else 'blind') + and (r['stage'] == 'development-' + record['stage'] or selected and r['stage'] == 'confirmation')] + lines += _table(['Evaluation', 'Strict success / scheduled', 'Complete', 'Native assessable'], + [(r['stage'], f"{r['strict_successes']} / {r['scheduled']}", r['complete'], r['native_assessable']) for r in matching]) + lines += ['Only one simulated PiPER, known rigid upright block and two known destinations were studied. ' + 'Object estimates can disappear throughout manipulation. Link6 pose is not the calibrated distal tool point. ' + 'No shoe skill, human-video motor recovery, arbitrary-arm transfer, physical robot readiness or real-time guarantee. ' + 'Teacher latency zeroes are inapplicable; process RSS includes export/preceding fits.', '', + 'Original learned parameters and DVIDIA code are MIT; upstream dependencies retain their notices. ' + 'This artifact cannot command physical hardware through the provided installer.', ''] + return '\n'.join(lines) + + +def render(study, output, *, summary, source_commit, hardware=None, study_audit=None, phase_coverage=None, diagnostic_audit=None): + study, output = evidence.plain(study), evidence.plain(output) + if output.exists() or output == study or output.is_relative_to(study): + raise ValueError('Use a fresh cards directory outside the study.') + data = _inputs(study, summary=summary, source_commit=source_commit, hardware=hardware, + study_audit=study_audit, phase_coverage=phase_coverage, diagnostic_audit=diagnostic_audit) + documents = {'README.md': dataset_card(data)} + for record in data['models']: + documents[f"model-cards/{record['stage']}/{record['variant']}/README.md"] = model_card(data, record) + for text in documents.values(): + evidence.scan(text.encode()) + sources = {'summary.json': evidence.identity(summary), 'generator.py': evidence.identity(Path(__file__)), + 'summarize_results.py': evidence.identity(SUMMARY_SOURCE)} + for name, path in (('hardware.json', hardware), ('full-study-audit.json', study_audit), + ('diagnostic-phase-coverage.json', phase_coverage), ('independent-audit.json', diagnostic_audit)): + if path is not None: + sources[name] = evidence.identity(path) + if phase_coverage is not None: + sources['diagnostic_phase_coverage.py'] = evidence.identity(PHASE_SOURCE) + files = {} + output.mkdir(parents=True, exist_ok=False) + for name, text in documents.items(): + path = output / name + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('x', encoding='utf-8', newline='\n') as stream: + stream.write(text) + files[name] = evidence.identity(path) + receipt = dict(schema=SCHEMA, source_commit=source_commit, + protocol_sha256=data['study_bindings']['protocol.json']['sha256'], + report_sha256=data['study_bindings']['report.json']['sha256'], status=data['report']['status'], + qualification=data['report']['qualification'], qualified=data['report']['qualified'], + full_study_audited=data['audit'] is not None, physical_robot_ready=False, + summary_reconstructed=True, physics_executed=False, model_executed=False, + source_inputs=sources, study_bindings=data['study_bindings'], files=files, + model_cards=[dict(card=f"model-cards/{r['stage']}/{r['variant']}/README.md", model=r['path'], + model_sha256=r['identity']['sha256'], selected=r['selected']) for r in data['models']]) + evidence.write(output / 'card-receipt.json', receipt) + return dict(status='complete', output=str(output), cards=len(files), qualification=data['report']['qualification'], + receipt_sha256=evidence.identity(output / 'card-receipt.json')['sha256']) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('study', 'output', 'summary', 'source-commit'): + parser.add_argument('--' + name, required=True) + for name in ('hardware', 'study-audit', 'phase-coverage', 'diagnostic-audit'): + parser.add_argument('--' + name) + args = parser.parse_args(argv) + try: + print(json.dumps(render(args.study, args.output, summary=args.summary, + source_commit=args.source_commit, hardware=args.hardware, study_audit=args.study_audit, + phase_coverage=args.phase_coverage, diagnostic_audit=args.diagnostic_audit), indent=2)) + return 0 + except (ValueError, OSError, KeyError, TypeError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc)))) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/test_render_camera_action_card.py b/publishing/test_render_camera_action_card.py new file mode 100644 index 0000000..3b854da --- /dev/null +++ b/publishing/test_render_camera_action_card.py @@ -0,0 +1,231 @@ +"""Publication-card checks use fabricated sealed receipts, never native physics.""" +from copy import deepcopy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import render_camera_action_card as cards +from dvidia_training import camera_action_study as study + + +class CardTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.base = Path(self.temp.name).resolve() + self.root = self.base / 'study' + self.fixed = self.base / 'fixed.json' + study.write(self.fixed, {'fixture': 'fixed comparator'}) + with patch.object(study, 'runtime_hashes', return_value={'fixture.py': '0' * 64}), \ + patch('dvidia_training.piper_motor_primitive.validate_model', side_effect=lambda value: value): + study.prepare(self.root, fixed_model=self.fixed) + protocol = study.read(self.root / 'protocol.json') + rows = [{**deepcopy(a), 'status': 'unattempted', 'complete': False, 'strict_success': False, + 'score': {}, 'path': None, 'receipt_sha256': None} for a in protocol['rosters']['diagnostic']] + receipt = dict(schema=study.SCHEMA + '/stage', stage='diagnostic', status='blocked', + reason='Synthetic fixture: capture budget unavailable.', scheduled=4, strict_successes=0, + protocol_sha256=study.file_hash(self.root / 'protocol.json'), rows=rows) + study.write(self.root / 'stages/diagnostic.started.json', {'fixture': True}) + study._seal_stage(self.root, 'diagnostic', receipt) + study.write(self.root / 'report.json', study.report_study(self.root)) + self.summary = self.base / 'summary.json' + study.write(self.summary, cards.recompute_summary(self.root)) + + def render(self, **kwargs): + return cards.render(self.root, self.base / 'cards', summary=self.summary, + source_commit='a' * 40, **kwargs) + + def test_blocked_real_seals_render_honest_budget_missing_metrics_and_local_receipt(self): + result = self.render() + text = (self.base / 'cards/README.md').read_text() + self.assertEqual(result['qualification'], 'research_candidate') + self.assertIn('license: mit\ntask_categories:\n- robotics', text) + self.assertIn('Research candidate — blocked', text) + self.assertIn('452 attempts', text) + self.assertIn('0 / 4', text) + self.assertIn('No completed model fit is sealed before the blocker', text) + self.assertIn('No full-study independent audit was supplied', text) + self.assertIn('Not applicable', text) + self.assertIn('not reported', text) + self.assertIn('https://github.com/Dvidia-Inference/dvidia-training/tree/' + 'a' * 40, text) + self.assertIn('shasum -a256 -c SHA256SUMS', text) + self.assertIn('dvidia-train robot inspect', text) + self.assertNotIn('`summary.json`', text) + self.assertNotIn(str(self.root), text) + receipt = study.read(self.base / 'cards/card-receipt.json') + self.assertFalse(receipt['full_study_audited']) + self.assertFalse(receipt['qualified']) + self.assertFalse(receipt['physics_executed']) + self.assertTrue(receipt['summary_reconstructed']) + self.assertEqual(receipt['files']['README.md'], cards.evidence.identity(self.base / 'cards/README.md')) + + def test_fabricated_success_or_latency_summary_is_refused_before_output(self): + value = study.read(self.summary) + value['controllers'][0]['strict_successes'] = 4 + self.summary.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError, 'fresh read-only reconstruction'): + self.render() + self.assertFalse((self.base / 'cards').exists()) + value = cards.recompute_summary(self.root) + value['controllers'][0]['timing']['full_boundary_wall_seconds']['p95'] = .00001 + self.summary.write_text(json.dumps(value)) + with self.assertRaisesRegex(ValueError, 'fresh read-only reconstruction'): + self.render() + + def test_running_worker_and_unsealed_partial_study_are_not_publishable(self): + (self.root / 'worker.lock').write_text('{}') + with patch.object(study, 'inspect') as inspect, self.assertRaisesRegex(ValueError, 'worker'): + self.render() + inspect.assert_not_called() + (self.root / 'worker.lock').unlink() + state = study.inspect(self.root) + state['status'] = 'ready' + with patch.object(study, 'inspect', return_value=state), self.assertRaisesRegex(ValueError, 'explicitly blocked'): + self.render() + + def test_complete_state_requires_full_study_audit_even_when_gates_fail(self): + state = study.inspect(self.root) + state['status'] = 'complete' + state['report']['status'] = 'complete' + (self.root / 'report.json').write_bytes(study.canonical(state['report'])) + with patch.object(study, 'inspect', return_value=state), self.assertRaisesRegex(ValueError, 'full-study audit'): + self.render() + + def test_mismatched_report_or_stage_seal_cannot_render(self): + report_path = self.root / 'report.json' + report = study.read(report_path) + report['qualified'] = True + report_path.write_bytes(study.canonical(report)) + with self.assertRaisesRegex(ValueError, 'Saved report differs'): + self.render() + report_path.write_bytes(study.canonical(study.report_study(self.root))) + stage_path = self.root / 'stages/diagnostic.json' + stage_path.write_bytes(stage_path.read_bytes() + b' ') + with self.assertRaisesRegex(ValueError, 'receipt identity changed'): + self.render() + + def test_output_is_fresh_and_outside_study(self): + self.render() + with self.assertRaisesRegex(ValueError, 'fresh cards'): + self.render() + with self.assertRaisesRegex(ValueError, 'outside the study'): + cards.render(self.root, self.root / 'cards', summary=self.summary, source_commit='a' * 40) + + def test_hardware_requires_whitelisted_public_fields(self): + hardware = self.base / 'hardware.json' + study.write(hardware, {'hostname': 'private'}) + with self.assertRaisesRegex(ValueError, 'public field whitelist'): + self.render(hardware=hardware) + + def test_supplied_audit_failure_cannot_be_ignored_on_a_blocked_study(self): + audit = self.base / 'audit.json' + study.write(audit, dict(schema='dvidia.camera-action-study-audit/v1', passed=False, errors=['fixture'])) + with self.assertRaisesRegex(ValueError, 'Full-study audit does not pass'): + self.render(study_audit=audit) + self.assertFalse((self.base / 'cards').exists()) + + def test_static_methodology_cannot_describe_different_registered_capture(self): + protocol = study.read(self.root / 'protocol.json') + protocol['capture']['camera_hz'] = 20 + with patch.object(cards.evidence, 'read', side_effect=lambda path: protocol if Path(path).name == 'protocol.json' + else study.read(path)), self.assertRaisesRegex(ValueError, 'methodology'): + self.render() + + def test_blocked_study_cannot_silently_omit_interrupted_fit_prefix(self): + prefix = self.root / 'models/initial/full' + prefix.mkdir(parents=True) + study.write(prefix / 'started.json', {'stage': 'initial', 'variant': 'full'}) + with self.assertRaisesRegex(ValueError, 'Unsealed or orphan model'): + self.render() + (prefix / 'started.json').unlink() + with self.assertRaisesRegex(ValueError, 'Unsealed or orphan model'): + self.render() + + def test_model_header_alone_cannot_claim_a_trained_network(self): + models = {} + for variant in ('full', 'blind'): + prefix = self.root / 'models/initial' / variant + study.write(prefix / 'model.json', dict(schema='dvidia.camera-action-mlp/v1', + hidden_units=[64, 64], blind=variant == 'blind', physical_robot_ready=False)) + record = dict(path=f'models/initial/{variant}/model.json', + model_sha256=study.file_hash(prefix / 'model.json'), model_bytes=(prefix / 'model.json').stat().st_size, + training_episodes=[], development_episodes=[]) + study.write(prefix / 'receipt.json', record) + study.write(prefix / 'started.json', {'fixture': True}) + models[variant] = record + stage = dict(stage='train-initial', status='complete', models=models, + protocol_sha256=study.file_hash(self.root / 'protocol.json')) + study.write(self.root / 'stages/train-initial.started.json', {'fixture': True}) + study._seal_stage(self.root, 'train-initial', stage) + self.summary.write_bytes(study.canonical(cards.recompute_summary(self.root))) + with self.assertRaisesRegex(ValueError, 'Unsupported camera-action model schema'): + self.render() + + def test_phase_coverage_requires_bound_audit_and_reconstruction(self): + audit = self.base / 'diagnostic-audit.json' + study.write(audit, {'fixture': 'diagnostic proof'}) + phase = dict(schema='dvidia.camera-action-phase-coverage/v1', + protocol_sha256=study.file_hash(self.root / 'protocol.json'), + diagnostic_audit_sha256=study.file_hash(audit), manipulation_available=0, + manipulation_decisions=749, manipulation_phases=['close', 'lift', 'transfer', 'place', 'release'], cases=4) + phase_path = self.base / 'phase.json' + study.write(phase_path, phase) + with self.assertRaisesRegex(ValueError, 'independently audited diagnostic'): + self.render(phase_coverage=phase_path) + with patch.object(cards, 'recompute_phase', return_value=phase) as recompute: + self.render(phase_coverage=phase_path, diagnostic_audit=audit) + recompute.assert_called_once_with(self.root, audit) + text = (self.base / 'cards/README.md').read_text() + self.assertIn('0 / 749 visual estimates', text) + self.assertIn('diagnostic subset, not student confirmation', text) + phase['manipulation_available'] = 749 + phase_path.write_bytes(study.canonical(phase)) + with patch.object(cards, 'recompute_phase', return_value={**phase, 'manipulation_available': 0}), \ + self.assertRaisesRegex(ValueError, 'Diagnostic phase coverage differs'): + cards.render(self.root, self.base / 'cards-other', summary=self.summary, source_commit='a' * 40, + phase_coverage=phase_path, diagnostic_audit=audit) + + def test_complete_failed_gates_and_model_identities_are_visible(self): + data = cards._inputs(self.root, summary=self.summary, source_commit='a' * 40) + data['report'].update(status='complete', gates={'at_least_58_of_64': False, + 'no_critical_contact_increase': True}, selected_stage='initial') + training = dict(training_wall_seconds=12.5, shared_export_wall_seconds=3.0, + process_peak_rss_bytes=125000000, memory_scope='Process high-water RSS including export and previous fit.', + training_episodes=['training-0'], development_episodes=['dev-0']) + provenance = dict(stage='initial', sample_count=1000, train_groups=['train-0'], + dataset_sha256='1' * 64, provenance_sha256='2' * 64, model_source_sha256='3' * 64) + data['models'] = [dict(stage='initial', variant=variant, selected=True, + path=f'models/initial/{variant}/model.json', identity={'bytes': 471234, 'sha256': digit * 64}, + training=training, model={'training': provenance}) for variant, digit in [('full', '4'), ('blind', '5')]] + text = cards.dataset_card(data) + self.assertIn('Research candidate — qualification gates failed', text) + self.assertIn('at_least_58_of_64 | FAIL', text) + self.assertIn('no_critical_contact_increase | PASS', text) + self.assertIn('4' * 64, text) + self.assertIn('5' * 64, text) + self.assertIn('models/initial/full/model.json', text) + self.assertIn('simulation candidate', text) + model = cards.model_card(data, data['models'][1]) + self.assertIn('matched blind baseline', model) + self.assertIn('live and historical sensor-feedback tensor channels are masked', model) + self.assertIn('12.5', model) + self.assertIn('1' * 64, model) + self.assertNotIn('This is the selected full-feedback model', model) + data['report'].update(qualified=True, qualification='bounded_simulation_skill') + for record in (data['models'][1], {**data['models'][0], 'selected': False}): + text = cards.model_card(data, record) + self.assertIn('no separately awarded skill qualification', text) + self.assertNotIn('Bounded simulation skill: all registered gates passed', text) + + def test_text_values_cannot_add_html_or_new_table_rows(self): + self.assertEqual(cards._cell('