diff --git a/README.md b/README.md index aa7694a..0bc20bc 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,16 @@ [Recorded demo](https://huggingface.co/spaces/Dvidia/dvidia-training) · [Synthetic datasets](https://huggingface.co/datasets/Dvidia/dvidia-training-examples) · [Pilot models](https://huggingface.co/Dvidia/dvidia-training-pilot) +**[Camera-to-action skill v0.1](docs/camera-action-v01.md).** +The **0.1.3.dev4 preview** adds a local demonstration-to-action learning loop: +25 Hz RGB-D, 50 Hz measured joints and demonstrated targets, a compact CPU +policy, one bounded correction round, and a fixed unseen-start benchmark. +Use `dvidia-train robot` in Terminal or the Training Studio's `/robot` page. +The first task is simulated PiPER block placement with an explicitly synthetic +teacher. Missing object estimates remain masked. Each installation remains a +simulation candidate and sends no physical commands. +[Registered experiment and diagnostic evidence](docs/camera-action-v01-report-draft.md). + **[Arm interfaces and installation v0.1](docs/arm-interfaces-v01.md).** The **0.1.3.dev3 preview** adds `dvidia-arm`: three explicit native simulation adapters, exact PiPER policy compatibility checks, separate install/run commands, @@ -390,6 +400,9 @@ A folder of videos is accepted, with lineage/completeness warnings. Prefer an ex ## Hardware These are **provisional planning targets for the small default visual learner**, not verified minimum specifications or guarantees for arbitrary datasets. +They do not describe the camera-to-action native benchmark, which requires a +working graphics context, substantially more evidence storage and a separate +[setup and measured-host report](docs/camera-action-v01.md). | Resource | Proposed starting point | Recommended | | --- | --- | --- | diff --git a/benchmarks/camera-action-v01/development_failure_summary.py b/benchmarks/camera-action-v01/development_failure_summary.py new file mode 100644 index 0000000..007119d --- /dev/null +++ b/benchmarks/camera-action-v01/development_failure_summary.py @@ -0,0 +1,362 @@ +"""Descriptive comparison of all sealed initial/corrected development attempts. + +Reads no confirmation outcomes; executes no physics, policy or training. Saved +numeric outcomes are reconstructed, while media/perception hashes remain bound +by episode manifests rather than being re-audited in full. +""" +from __future__ import annotations + +import argparse +from collections import Counter +import importlib.util +import json +from pathlib import Path + +import numpy as np + + +def sibling(name): + spec = importlib.util.spec_from_file_location('camera_development_' + name, + Path(__file__).with_name(name + '.py')) + module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + return module + + +audit = sibling('study_audit') +first = sibling('diagnose_first_student') +SCHEMA = 'dvidia.camera-action-development-failures/v1' +DEVELOPMENT = ('development-initial', 'development-corrected') + + +def load(root, name, sources): + path = audit.relative(root, name) + value = audit.read(path); sources[name] = audit.identity(path) + return value + + +def stage(root, name, protocol, protocol_hash, sources): + value = load(root, 'stages/' + name + '.json', sources) + seal = load(root, 'stages/' + name + '.integrity.json', sources) + start = load(root, 'stages/' + name + '.started.json', sources) + audit.require(seal == dict(sha256=sources['stages/' + name + '.json']['sha256'], + protocol_sha256=protocol_hash), 'Stage seal differs: ' + name) + expected_schema='dvidia.camera-action-study/v1/'+('training' if name.startswith('train-') else 'stage') + audit.require(value['schema'] == expected_schema + and start['schema'] == 'dvidia.camera-action-study/v1/stage-start' + and value['stage'] == start['stage'] == name + and value['protocol_sha256'] == start['protocol_sha256'] == protocol_hash + and start['source_sha256'] == protocol['source_sha256'] + and value.get('status') in ('complete', 'blocked') + and start.get('model_bindings') == value.get('model_bindings'), 'Stage registration differs: ' + name) + if not name.startswith('train-'): + roster = protocol['rosters'][name]; rows = value['rows'] + audit.require(value['scheduled'] == len(roster) + and [r['id'] for r in rows] == [r['id'] for r in roster] + and len({r['id'] for r in rows}) == len(rows), 'Sealed stage roster differs: ' + name) + for row, registered in zip(rows, roster): + audit.require(all(row.get(key) == registered.get(key) for key in + ('id', 'case', 'controller', 'negative_control')), 'Registered attempt identity differs.') + return value + + +def measure(steps, arrays, native_score): + """Numeric measurements for a complete, validated trajectory only.""" + times = first.control_clock(steps) + for index, step in enumerate(steps): + for actual, expected in ((step['command']['issued_at_seconds'], index * .02), + (step['after']['timestamp_seconds'], (index + 1) * .02), + (step['applied']['interval_start_seconds'], index * .02), + (step['applied']['interval_end_seconds'], (index + 1) * .02)): + audit._close(actual, expected, 'Command/transition clock differs.') + audit.require(type(step['applied']['rate_limited']) is bool + and step['applied']['rate_limited'] == (step['applied']['ramped_targets'] != step['command']['issued']), + 'Recorded rate-limited flag differs from actual applied targets.') + tick = (index + 1) * 20 - 1 + np.testing.assert_allclose(step['applied']['ramped_targets']['joint_targets_rad'], + arrays['arm_actuator_controls_rad'][tick], atol=1e-10, rtol=0) + audit._close(step['applied']['ramped_targets']['gripper_position_m'], + float(arrays['normal_rate_limited_jaw_target_m'][tick]), 'Applied jaw reference differs.', tolerance=1e-10) + jaw = np.asarray([s['command']['issued']['gripper_position_m'] for s in steps]) + closure = None; indices = np.flatnonzero(jaw < .020) + if len(indices): + index = int(indices[0]); tick = index * 20 - 1 + closure = dict(time_seconds=float(times[index]), issued_single_finger_target_m=float(jaw[index]), + geometry_available=tick >= 0, tcp_object_xy_distance_m=None, + tcp_minus_object_z_m=None, actual_finger_positions_m=None) + if tick >= 0: + delta = arrays['tcp_position_m'][tick] - arrays['object_position_m'][tick] + closure.update(tcp_object_xy_distance_m=float(np.linalg.norm(delta[:2])), + tcp_minus_object_z_m=float(delta[2]), + actual_finger_positions_m=arrays['gripper_joint_position_m'][tick].tolist()) + critical = int(np.count_nonzero(arrays['unexpected_contact_count'] > 0)) + audit.require(critical == native_score['unexpected_contact_native_ticks'], 'Native critical count differs.') + return dict(control_intervals=1000, rate_limited_intervals=sum(s['applied']['rate_limited'] for s in steps), + critical_contact_ticks=critical, critical_contact_episode=critical > 0, + bilateral_contact_ticks=int(np.count_nonzero(np.all(arrays['grasp_contact_counts'] > 0, axis=1))), + strict_grasp_seen=native_score['grasp_seen'], lift_seen=native_score['lift_seen'], + maximum_object_lift_m=native_score['max_object_lift_m'], + no_closure_command=closure is None, first_closure_command=closure, + closure_command_beyond_20mm_xy=(False if closure is None else + closure['tcp_object_xy_distance_m'] > .020 if closure['geometry_available'] else None)) + + +def episode(root, stage_name, row, model_binding, tracker_hash, sources): + audit.require(type(row.get('complete')) is bool and type(row.get('strict_success')) is bool + and row['status'] in ('complete','interrupted','refused','unknown','unattempted'), + 'Unknown attempt status or nonboolean outcome.') + result = dict(id=row['id'], stage=stage_name, controller=row['controller'], case=row['case'], + policy_artifact_sha256=model_binding, + status=row['status'], complete=row.get('complete') is True, + strict_success=row.get('strict_success') is True, metrics=None, missing_reason=None) + relative = 'episodes/' + stage_name + '/' + row['id'] + if row['status'] in ('unknown', 'unattempted'): + audit.require(not result['complete'] and not result['strict_success'] + and (root / relative).exists() is (row['status'] == 'unknown'), + 'Unknown/unattempted outcome or directory presence differs.') + # Do not inspect an unknown prefix, which may not have sealed byte evidence. + result['missing_reason'] = 'No sealed episode for this scheduled attempt.' + return result + audit.require(row['path'] == relative and row['policy_artifact_sha256'] == model_binding, + 'Episode path or model binding differs.') + directory = audit.relative(root, relative) + manifest = load(root, relative + '/integrity.json', sources) + audit.require(sources[relative + '/integrity.json']['sha256'] == row['integrity_sha256'] + and manifest['schema'] == 'dvidia.camera-action-rollout/v2/integrity', 'Episode integrity differs from sealed stage.') + files = manifest['files']; local = {} + def bound(name): + raw = first.bound(directory, name, files, local) + sources[relative + '/' + name] = local[name] + return json.loads(raw) + receipt = bound('rollout.json'); started = bound('started.json') + mode = 'student' if row['controller'] == 'blind' else row['controller'] + case = {**row['case'], 'policy_artifact_sha256': model_binding} + audit.require(sources[relative + '/rollout.json']['sha256'] == row['receipt_sha256'] + and receipt['case'] == started['case'] == case and receipt['mode'] == started['mode'] == mode + and receipt['negative_control'] == started['negative_control'] == row['negative_control'] is None + and receipt['status'] == row['status'] and receipt['score'] == row['score'] + and receipt['strict_success'] is result['strict_success'] + and receipt['schema'] == started['schema'] == 'dvidia.camera-action-rollout/v2', + 'Episode receipt differs from stage or registration.') + audit.require(receipt['tracker_sha256'] == tracker_hash, 'Frozen tracker binding differs.') + config = started['config'] + audit.require(config['duration_s'] == 20. and config['seed'] == case['seed'] + and config['width'] == 320 and config['height'] == 240 and config['camera_every_controls'] == 2, + 'Started capture budget differs.') + teacher_files = sorted(name for name in files if name.startswith('teacher/') and name.endswith('.json')) + audit.require(receipt['teacher_queries'] == len(teacher_files) + and teacher_files == [f'teacher/{i:04}.json' for i in range(len(teacher_files))], + 'Recorded teacher query inventory differs.') + if mode == 'student': + audit.require(not teacher_files and receipt['no_privileged_features_or_teacher_intervention'] is True, + 'Learner contains teacher intervention.') + for name in ('native_episode.json', 'native_metadata.json', 'native.npz'): + if name in files: + first.bound(directory, name, files, local) + sources[relative + '/' + name] = local[name] + native, arrays, metadata, score = audit._native(directory, files, case, receipt) + steps = [] + if 'recording.json' in files: + recording = bound('recording.json'); steps = recording['steps'] + audit.require(recording['schema'] == 'dvidia.camera-action-rollout/v2' and recording['control_mode'] == mode + and recording['provenance']['case_id'] == row['case']['id'] + and recording['provenance']['source_group'] == row['case']['group_id'] + and recording['goal_position_m'] == row['case']['goal_position_m'] + and recording['tracker_sha256'] == receipt['tracker_sha256'], 'Recording identity differs.') + for index, step in enumerate(steps): + audit._close(step['before']['timestamp_seconds'], index * .02, 'Recorded decision clock differs.') + audit.require(step['index'] == index and step['command']['authority'] == mode + and step['command']['operator_correction'] is False + and step['command']['requested'] == step['command']['issued'], 'Unexpected intervention or control authority.') + audit.require(native is not None and len(native['actions']) > index, + 'Recorded command lacks native action evidence.') + audit.require({key:native['actions'][index][key] for key in ('joint_targets_rad','gripper_position_m')} + == step['command']['issued'], 'Issued command differs from native action evidence.') + if mode == 'student':audit.require(step['teacher_query'] is None, 'Learner contains a teacher query.') + if mode == 'teacher': + audit.require(len(steps) <= receipt['teacher_queries'] <= len(steps) + 1 and receipt['student_calls'] == 0, + 'Teacher query or student-call count differs.') + else: + audit.require(len(steps) <= receipt['student_calls'] <= len(steps) + 1, + 'Student-call count differs from completed prefix.') + audit.require(len(steps) == receipt['recorded_complete_intervals'] == row['recorded_complete_intervals'] + and result['complete'] == (receipt['status'] == 'complete' and len(steps) == 1000), 'Episode completeness differs.') + audit.require(result['strict_success'] == (result['complete'] and score.get('success') is True), + 'Strict success differs from complete independently scored evidence.') + if result['complete']: + audit.require(metadata is not None and metadata['rows'] == 20000, 'Complete trajectory lacks 20,000 native ticks.') + result.update(recorded_intervals=len(steps), native_ticks=metadata['rows'] if metadata else None, + native_score_valid=score['valid'], teacher_queries=receipt['teacher_queries']) + if result['complete'] and score['valid']: + result['metrics'] = measure(steps, arrays, score['native_recomputed']) + else: + result['missing_reason'] = 'No complete valid trajectory; prefix/refusal is retained without full-episode metric imputation.' + return result + + +def compare(row, teacher): + value = dict(teacher_id=teacher['id'], teacher_strict_success=teacher['strict_success'], + both_complete_metrics=row['metrics'] is not None and teacher['metrics'] is not None, + closure_time_difference_seconds=None, closure_xy_difference_m=None, + closes_before_teacher=None, rate_limited_interval_difference=None, + critical_contact_tick_difference=None) + if not value['both_complete_metrics']:return value + a, b = row['metrics'], teacher['metrics'] + value.update(rate_limited_interval_difference=a['rate_limited_intervals']-b['rate_limited_intervals'], + critical_contact_tick_difference=a['critical_contact_ticks']-b['critical_contact_ticks']) + x, y = a['first_closure_command'], b['first_closure_command'] + if x and y: + value['closure_time_difference_seconds'] = x['time_seconds']-y['time_seconds'] + value['closes_before_teacher'] = x['time_seconds'] < y['time_seconds'] + if x['geometry_available'] and y['geometry_available']: + value['closure_xy_difference_m'] = x['tcp_object_xy_distance_m']-y['tcp_object_xy_distance_m'] + return value + + +def distribution(values): + values = [float(x) for x in values if x is not None] + return dict(known=len(values), median=float(np.median(values)) if values else None, + p95=float(np.quantile(values, .95)) if values else None, + minimum=min(values) if values else None, maximum=max(values) if values else None) + + +def aggregate(rows): + metrics = [r['metrics'] for r in rows if r['metrics'] is not None] + def count(predicate): + values = [predicate(m) for m in metrics]; known = [v for v in values if v is not None] + return dict(events=sum(bool(v) for v in known), known=len(known), missing=len(rows)-len(known)) + result = dict(scheduled=len(rows), strict_successes=sum(r['strict_success'] for r in rows), + statuses=dict(sorted(Counter(r['status'] for r in rows).items())), complete_valid_metrics=len(metrics), + unavailable_metrics=len(rows)-len(metrics), + no_bilateral_contact=count(lambda m:m['bilateral_contact_ticks']==0), + no_strict_grasp=count(lambda m:not m['strict_grasp_seen']), + no_closure_command=count(lambda m:m['no_closure_command']), + closure_command_beyond_20mm_xy=count(lambda m:m['closure_command_beyond_20mm_xy']), + grasp_without_lift=dict(events=sum(m['strict_grasp_seen'] and not m['lift_seen'] for m in metrics), + known=sum(m['strict_grasp_seen'] for m in metrics), + not_applicable=sum(not m['strict_grasp_seen'] for m in metrics), missing=len(rows)-len(metrics)), + critical_contact_episodes=count(lambda m:m['critical_contact_episode']), + critical_contact_ticks_observed=sum(m['critical_contact_ticks'] for m in metrics), + rate_limited_intervals_observed=sum(m['rate_limited_intervals'] for m in metrics), + control_intervals_observed=sum(m['control_intervals'] for m in metrics), + closure_time_seconds=distribution([m['first_closure_command']['time_seconds'] if m['first_closure_command'] else None for m in metrics]), + closure_xy_distance_m=distribution([m['first_closure_command']['tcp_object_xy_distance_m'] if m['first_closure_command'] else None for m in metrics])) + comparisons = [r['teacher_comparison'] for r in rows if 'teacher_comparison' in r] + if comparisons: + earlier = [c['closes_before_teacher'] for c in comparisons if c['closes_before_teacher'] is not None] + result['matched_teacher'] = dict(paired_complete_metrics=sum(c['both_complete_metrics'] for c in comparisons), + closure_earlier=dict(events=sum(earlier), known=len(earlier), missing=len(rows)-len(earlier)), + closure_time_difference_seconds=distribution([c['closure_time_difference_seconds'] for c in comparisons]), + closure_xy_difference_m=distribution([c['closure_xy_difference_m'] for c in comparisons]), + rate_limited_interval_difference=distribution([c['rate_limited_interval_difference'] for c in comparisons])) + return result + + +def development_cases(protocol, stages): + expected_cases = {c['id']:c for c in protocol['cases']['development']} + audit.require(len(expected_cases)==len(protocol['cases']['development'])==16, + 'Expected all 16 unique registered development cases.') + for name in DEVELOPMENT: + rows = stages[name]['rows'] + audit.require(len(rows)==32 and {(r['case']['id'],r['controller']) for r in rows} + == {(key,controller) for key in expected_cases for controller in ('student','blind')} + and all(r['case']==expected_cases[r['case']['id']] for r in rows), 'Development coverage differs.') + return expected_cases + + +def summarize(study): + root = Path(study).absolute(); sources = {} + # No episode or outcome is read until BOTH development stage seals exist. + for name in DEVELOPMENT: + audit.require((root / 'stages' / (name+'.integrity.json')).is_file(), 'Both development stages must be sealed.') + protocol = load(root, 'protocol.json', sources); registration = load(root, 'registration.json', sources) + ph = sources['protocol.json']['sha256'] + audit.require(registration['protocol_sha256'] == ph and registration['source_sha256'] == protocol['source_sha256'], 'Registration differs.') + stages = {name:stage(root, name, protocol, ph, sources) for name in + ('demonstrations', 'train-initial', 'development-initial', 'train-corrected', 'development-corrected')} + expected_cases = development_cases(protocol, stages) + runtime = {} + for name, digest in protocol['source_sha256'].items(): + path = Path(audit.scoring.__file__).with_name(name) + actual = audit.identity(path); audit.require(actual['sha256']==digest, 'Frozen runtime file differs: '+name) + runtime[name]=actual + for candidate in ('initial','corrected'): + fit = stages['train-'+candidate] + audit.require(fit['status']=='complete', 'A referenced model fitting stage is incomplete.') + for variant, controller in (('full','student'),('blind','blind')): + path = 'models/'+candidate+'/'+variant+'/model.json' + model = load(root,path,sources) + audit.require(fit['models'][variant]['path']==path + and fit['models'][variant]['model_sha256']==sources[path]['sha256'] + and stages['development-'+candidate]['model_bindings'][controller]==sources[path]['sha256'] + and model['schema'] == 'dvidia.camera-action-mlp/v1' + and model['feature_contract'] == 'camera-encoder-absolute-causal160/v1' + and model['training']['stage'] == candidate + and model['training'] == fit['models'][variant]['training'] + and model['blind'] is (variant=='blind') + and model['training']['model_source_sha256']==protocol['source_sha256']['camera_action_model.py'], 'Model provenance differs.') + audit.require(stages['demonstrations']['model_bindings'] == {'teacher':audit.digest(protocol['source_sha256'])}, + 'Teacher source identity differs.') + tracker_hash=protocol['source_sha256']['piper_rgbd_tracker.py'] + teachers = {} + for row in stages['demonstrations']['rows']: + if row['case']['id'] in expected_cases: + audit.require(row['controller']=='teacher' and row['case']==expected_cases[row['case']['id']] + and row['case']['id'] not in teachers, 'Matched teacher is duplicated or differs.') + teachers[row['case']['id']] = episode(root,'demonstrations',row, + stages['demonstrations']['model_bindings']['teacher'],tracker_hash,sources) + audit.require(set(teachers)==set(expected_cases), 'A matching teacher is missing.') + rows=[] + for name in DEVELOPMENT: + for row in stages[name]['rows']: + result=episode(root,name,row,stages[name]['model_bindings'][row['controller']],tracker_hash,sources) + result['teacher_comparison']=compare(result,teachers[row['case']['id']]);rows.append(result) + cohorts={name+'/'+controller:aggregate([r for r in rows if r['stage']==name and r['controller']==controller]) + for name in DEVELOPMENT for controller in ('student','blind')} + helpers={name:audit.identity(Path(__file__).with_name(name)) for name in + ('development_failure_summary.py','diagnose_first_student.py','study_audit.py','independent_audit.py')} + return dict(schema=SCHEMA,posthoc=True,scope='All 64 registered learned development attempts and 16 exact matched teacher demonstrations. Descriptive numeric reconstruction; not causal inference, qualification or a full media/perception audit.', + audit_status='sealed_local_development_summary_pending_full_study_audit', + physics_executed=False,model_executed=False,training_executed=False,teacher_queried=False, + confirmation_read=False,selection_changed=False,qualification_changed=False, + definitions=dict(closure='First issued single-finger jaw target below 0.020 m, not measured closure or total aperture.', + far_closure='XY distal TCP-object separation above 0.020 m at that command; descriptive posthoc threshold, not a preregistered task gate.', + geometry='Saved native tick exactly at the decision boundary, before applying the new command; no t=0 native sample means geometry unavailable.', + earlier='Earlier than the matched teacher in episode time; a different valid path or timing is allowed.', + denominators='All scheduled rows retained. Full-episode metrics use only complete valid trajectories; prefix/refusal metrics are unavailable, never zero-imputed.', + hidden_inputs='No phase, teacher, native object or future observation is inferred as a learner input here; source/model/control-authority bindings do not attest arbitrary external code.', + depth='Synthetic camera transforms do not validate depth accuracy; renderer warnings remain a separate limitation, not a demonstrated cause.'), + cohorts=cohorts,teacher_cohort=aggregate(list(teachers.values())),attempts=rows,teachers=list(teachers.values()), + source_files=sources,source_inventory_sha256=audit.digest(sources),registered_runtime=runtime,helper_sources=helpers) + + +def markdown(value): + def event(item): + text=f"{item['events']}/{item['known']} known; {item['missing']} unavailable" + return text+(f"; {item['not_applicable']} without a grasp" if 'not_applicable' in item else '') + def number(x,scale=1):return 'Unavailable' if x is None else f'{x*scale:.2f}' + lines=['# Development grip-acquisition summary','',value['scope'],'', + '**Posthoc; pending full-study audit. No confirmation outcomes were read.**','', + '| Cohort | Strict success / scheduled | No strict grasp | Closure >20 mm XY | Rate-limited / observed intervals | Critical-contact attempts |', + '| --- | ---: | --- | --- | ---: | --- |'] + for name,c in [('Matched teachers',value['teacher_cohort']),*value['cohorts'].items()]: + lines.append(f"| {name} | {c['strict_successes']}/{c['scheduled']} | {event(c['no_strict_grasp'])} | {event(c['closure_command_beyond_20mm_xy'])} | {c['rate_limited_intervals_observed']}/{c['control_intervals_observed']} | {event(c['critical_contact_episodes'])} |") + lines+=['','| Learner cohort | First closure median (s) | XY distance median (mm) | Earlier than teacher | Grasp without lift |', '| --- | ---: | ---: | --- | --- |'] + for name,c in value['cohorts'].items(): + lines.append(f"| {name} | {number(c['closure_time_seconds']['median'])} | {number(c['closure_xy_distance_m']['median'],1000)} | {event(c['matched_teacher']['closure_earlier'])} | {event(c['grasp_without_lift'])} |") + lines+=['','A closure command is not an attained jaw position. The 20 mm thresholds are descriptive choices, not new acceptance gates. Native geometry uses the distal tool site, not the legacy encoder link6 feature. Missing metrics and no-command cases remain explicit in the JSON.', + '', 'These comparisons describe whether the first-case pattern recurs. They do not identify its cause, establish a population failure rate, tune a model, or replace registered development selection and frozen confirmation. Teacher phase labels are not used. Depth accuracy remains unvalidated.', + '', 'Source inventory SHA-256: `'+value['source_inventory_sha256']+'`. All exact case/model/stage/source bindings and conditional denominators are in `development-failure-summary.json`.',''] + return '\n'.join(lines) + + +def main(): + parser=argparse.ArgumentParser(description=__doc__);parser.add_argument('--study',required=True);parser.add_argument('--output',required=True) + args=parser.parse_args();output=Path(args.output).absolute() + audit.require(not output.exists() and not output.resolve().is_relative_to(Path(args.study).resolve()), 'Use a fresh output folder outside the study.') + result=summarize(args.study);output.mkdir(parents=True) + target=output/'development-failure-summary.json';target.write_text(json.dumps(result,indent=2,sort_keys=True,allow_nan=False)+'\n') + (output/'README.md').write_text(markdown(result)) + print(json.dumps(dict(output=str(output),attempts=len(result['attempts']),sha256=audit.identity(target)['sha256'],physics_executed=False))) + + +if __name__=='__main__':main() diff --git a/benchmarks/camera-action-v01/diagnose_first_student.py b/benchmarks/camera-action-v01/diagnose_first_student.py new file mode 100644 index 0000000..327a0f8 --- /dev/null +++ b/benchmarks/camera-action-v01/diagnose_first_student.py @@ -0,0 +1,311 @@ +"""Posthoc diagnosis of the first registered initial student and matching teacher. + +Only sealed completed episodes are read. No policy inference, native physics, +training, teacher queries, experiment selection or qualification is performed. +The development attempt is chosen by protocol order, never by its outcome. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import io +import json +from pathlib import Path + +import numpy as np + +SCHEMA = 'dvidia.camera-action-first-student-diagnosis/v2' + + +def identity(raw): + return dict(bytes=len(raw), sha256=sha256(raw).hexdigest()) + + +def read(path): + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128 * 1024 ** 2: + raise ValueError('Expected bounded ordinary evidence.') + return json.loads(path.read_bytes()) + + +def bound(root, name, inventory, used): + relative = Path(name) + if relative.is_absolute() or '..' in relative.parts or any(p.is_symlink() for p in (root / relative, *(root / relative).parents)): + raise ValueError('Only plain study-relative evidence is accepted.') + path = root / relative + if path.stat().st_size > 128 * 1024 ** 2: + raise ValueError('Evidence exceeds the bounded diagnostic format.') + raw = path.read_bytes() + actual = identity(raw) + if actual != inventory[name]: + raise ValueError('Sealed evidence changed: ' + name) + used[name] = actual + return raw + + +def first_time(mask, times): + indices = np.flatnonzero(mask) + return float(times[indices[0]]) if len(indices) else None + + +def intervals(mask, times): + indices = np.flatnonzero(mask) + if not len(indices): + return [] + groups = np.split(indices, np.flatnonzero(np.diff(indices) > 1) + 1) + return [dict(first_tick_seconds=float(times[g[0]]), last_tick_seconds=float(times[g[-1]]), + duration_seconds=len(g) * .001, ticks=len(g)) for g in groups] + + +def control_clock(steps): + times = np.asarray([step['before']['timestamp_seconds'] for step in steps], dtype=np.float64) + if (times.shape != (1000,) or not np.isfinite(times).all() + or not np.allclose(times, np.arange(1000) * .020, atol=1e-8, rtol=0)): + raise ValueError('Decision timestamps must follow the exact 50 Hz clock.') + return times + + +def stage_binding(root, name, attempt, protocol, protocol_hash): + """Bind the chosen local episode to its sealed, registered stage row.""" + path = root / 'stages' / (name + '.json') + receipt = read(path); receipt_identity = identity(path.read_bytes()) + seal_path = path.with_name(name + '.integrity.json') + started_path = path.with_name(name + '.started.json') + if read(seal_path) != dict(sha256=receipt_identity['sha256'], protocol_sha256=protocol_hash): + raise ValueError('Selected stage is not sealed: ' + name) + started = read(started_path) + if (receipt.get('schema') != 'dvidia.camera-action-study/v1/stage' + or receipt.get('stage') != name or receipt.get('status') != 'complete' + or receipt.get('protocol_sha256') != protocol_hash + or started.get('stage') != name or started.get('protocol_sha256') != protocol_hash + or started.get('source_sha256') != protocol['source_sha256'] + or started.get('model_bindings') != receipt.get('model_bindings')): + raise ValueError('Stage registration or started bindings differ: ' + name) + registered = protocol['rosters'][name] + rows = receipt['rows'] + if ([r['id'] for r in rows] != [r['id'] for r in registered] + or len({r['id'] for r in rows}) != len(rows) + or receipt.get('scheduled') != len(registered)): + raise ValueError('Stage does not contain the exact registered roster: ' + name) + row = next(r for r in rows if r['id'] == attempt['id']) + relative = 'episodes/' + name + '/' + attempt['id'] + if (any(row.get(key) != attempt.get(key) for key in ('case', 'controller', 'negative_control')) + or row.get('path') != relative or row.get('status') != 'complete' + or row.get('complete') is not True or row.get('recorded_complete_intervals') != 1000 + or row.get('policy_artifact_sha256') != receipt['model_bindings'][attempt['controller']]): + raise ValueError('Chosen stage row differs from the complete registered attempt.') + for filename, key in (('integrity.json', 'integrity_sha256'), ('rollout.json', 'receipt_sha256')): + episode_path = root / relative / filename + read(episode_path) # Reject nonordinary or unbounded files before hashing. + if identity(episode_path.read_bytes())['sha256'] != row[key]: + raise ValueError('Chosen episode differs from its sealed stage row: ' + filename) + rollout = read(root / relative / 'rollout.json') + if (rollout.get('case') != {**attempt['case'], 'policy_artifact_sha256': row['policy_artifact_sha256']} + or rollout.get('score') != row['score'] + or rollout.get('negative_control') != attempt['negative_control'] + or row['strict_success'] is not (rollout['score'].get('success') is True)): + raise ValueError('Chosen rollout identity or outcome differs from its sealed stage row.') + sources = {str(p.relative_to(root)): identity(p.read_bytes()) for p in (path, seal_path, started_path)} + return row, sources + + +def interpretation(student, teacher, divergence): + """Every observation depends on the measured case, including counterexamples.""" + acquired = student['bilateral_object_contact_ticks'] > 0 + observed = [f"Student strict success: {student['strict_success']}; teacher strict success: {teacher['strict_success']}."] + if not acquired: + observed.append('No bilateral student object contact was recorded; this case provides no acquired bilateral grip from which to assess retention.') + else: + observed.append(f"Student bilateral object contact occurred on {student['bilateral_object_contact_ticks']} native ticks; acquisition failure alone cannot explain this case.") + for name, result in (('Student', student), ('Teacher', teacher)): + event = result['closure_events']['jaw_target_first_below_20mm'] + if event is None: + observed.append(name + ' never commanded a single-finger jaw target below 20 mm.') + else: + observed.append(f"{name} first commanded a single-finger jaw target below 20 mm at {event['time_seconds']:.3f} s, " + f"with TCP-object XY distance {event['tcp_object_xy_distance_m'] * 1000:.2f} mm. This command does not imply attained finger position.") + observed.append(f"Rate-limited intervals: student {student['rate_limited_intervals']}/{student['complete_intervals']}, " + f"teacher {teacher['rate_limited_intervals']}/{teacher['complete_intervals']}.") + observed.append(f"Critical native contact ticks: student {student['native_score']['unexpected_contact_native_ticks']}, " + f"teacher {teacher['native_score']['unexpected_contact_native_ticks']}.") + onset = divergence['first_more_than_10mm_seconds']; missing = student['first_missing_estimate_seconds'] + precedes_missing = onset is not None and (missing is None or onset < missing) + if onset is None: + observed.append('The same-clock tool trajectories never separated by more than 10 mm.') + elif precedes_missing: + observed.append('Tool separation exceeded 10 mm before any missing student estimate; missing visual tracking alone cannot explain that onset.') + else: + observed.append('Missing student estimates began no later than 10 mm tool separation; this comparison does not establish divergence preceding missing vision.') + predicates = dict(student_failed=student['strict_success'] is False, no_bilateral_contact=not acquired, + student_more_rate_limited=student['rate_limited_intervals'] > teacher['rate_limited_intervals'], + divergence_precedes_missing=precedes_missing) + hypothesis = ('Small errors on teacher-state absolute targets may compound through learned feedback into a different measured state distribution and poor approach/closure timing. ' + 'The measured pattern is consistent with covariate shift, not a proven causal diagnosis.' if all(predicates.values()) + else 'The measured case does not meet the specific acquisition/rate-limit/early-divergence pattern; no mechanism is inferred automatically.') + return dict(observed=observed, hypothesis= hypothesis, hypothesis_predicates=predicates, + unresolved=['No counterfactual policy intervention was run.', + 'Normalization-unit excursions are unclipped descriptive distances, not training-support bounds; the frozen model clips standardized inputs to +/-10.', + 'One development case cannot establish failure frequency or the final corrected-model outcome.'], + constraint='Keep the registered initial/correction/confirmation protocol unchanged; use this finding only for posthoc reporting.') + + +def episode(root, relative, attempt, model): + directory = root / relative + manifest_path = directory / 'integrity.json' + manifest = read(manifest_path) + if manifest.get('schema') != 'dvidia.camera-action-rollout/v2/integrity': + raise ValueError('A sealed native v2 episode is required.') + inventory = manifest['files']; used = {'integrity.json': identity(manifest_path.read_bytes())} + read_bound = lambda name: json.loads(bound(directory, name, inventory, used)) + recording = read_bound('recording.json'); rollout = read_bound('rollout.json') + native = read_bound('native_episode.json'); metadata = read_bound('native_metadata.json') + raw_native = bound(directory, 'native.npz', inventory, used) + with np.load(io.BytesIO(raw_native), allow_pickle=False) as archive: + arrays = {name: archive[name] for name in archive.files} + steps = recording['steps'] + if (rollout.get('status') != 'complete' or rollout.get('recorded_complete_intervals') != 1000 + or len(steps) != 1000 or metadata.get('rows') != 20000 + or rollout['score'].get('valid') is not True + or any(rollout['case'].get(key) != value for key, value in attempt['case'].items())): + raise ValueError('Use only the matching sealed complete registered episode.') + controller = attempt['controller'] + if rollout['mode'] != controller or rollout['teacher_queries'] != (1000 if controller == 'teacher' else 0): + raise ValueError('Controller authority or query count differs.') + times = arrays['t_seconds'] + if (times.shape != (20000,) or not np.allclose(times, np.arange(1, 20001) * .001, atol=1e-8, rtol=0) + or any(not np.isfinite(value).all() for value in arrays.values())): + raise ValueError('Native numeric evidence must be finite with exact 1 ms cadence.') + issued = np.array([s['command']['issued']['joint_targets_rad'] for s in steps]) + jaw = np.array([s['command']['issued']['gripper_position_m'] for s in steps]) + measured = np.array([s['before']['signals']['joint_position_rad']['value'] for s in steps]) + velocity = np.array([s['before']['signals']['joint_velocity_rad_s']['value'] for s in steps]) + applied = np.array([s['applied']['ramped_targets']['joint_targets_rad'] for s in steps]) + control_times = control_clock(steps) + available = []; phases = []; previous_phase = None + for step in steps: + if step['command']['authority'] != controller: + raise ValueError('Episode has mixed authority.') + estimate = read_bound(step['perception']['path'])['estimate'] + available.append(estimate['status'] == 'available') + if controller == 'teacher': + phase = read_bound(step['teacher_query']['path'])['phase'] + if phase != previous_phase: + phases.append(dict(time_seconds=step['before']['timestamp_seconds'], phase=phase)) + previous_phase = phase + elif step['teacher_query'] is not None: + raise ValueError('A student episode cannot query the teacher.') + available = np.asarray(available) + both = np.all(arrays['grasp_contact_counts'] > 0, axis=1) + any_contact = np.any(arrays['grasp_contact_counts'] > 0, axis=1) + critical = arrays['unexpected_contact_count'] > 0 + score = rollout['score']['native_recomputed'] + max_lift = float(arrays['object_position_m'][:, 2].max() - native['initial_object_position_m'][2]) + if int(critical.sum()) != score['unexpected_contact_native_ticks'] or not np.isclose(max_lift, score['max_object_lift_m'], atol=1e-10, rtol=0): + raise ValueError('Native scalar metrics differ from saved independent score.') + velocity_indices = [model['feature_names'].index(f'lag0:joint_velocity_rad_s_{i}/3') for i in range(6)] + mean = np.asarray(model['feature_mean'])[velocity_indices] + scale = np.asarray(model['feature_scale'])[velocity_indices] + normalized_velocity = (velocity / 3. - mean) / scale + events = {} + for label, threshold in (('jaw_target_first_below_20mm', .020), ('jaw_target_first_below_5mm', .005)): + indices = np.flatnonzero(jaw < threshold) + if not len(indices): + events[label] = None + continue + index = int(indices[0]); time = float(control_times[index]); native_index = round(time * 1000) - 1 + if native_index < 0: + raise ValueError('This event has no preceding saved native tick.') + delta = arrays['tcp_position_m'][native_index] - arrays['object_position_m'][native_index] + events[label] = dict(time_seconds=time, requested_jaw_m=float(jaw[index]), + actual_finger_joint_positions_m=arrays['gripper_joint_position_m'][native_index].tolist(), + tcp_object_xy_distance_m=float(np.linalg.norm(delta[:2])), tcp_minus_object_z_m=float(delta[2]), + native_grasp_contacts=arrays['grasp_contact_counts'][native_index].tolist()) + pair_counts = native['metrics']['contact_pair_native_ticks'] + table = metadata['contact_names']['table_geom_id']; pads = metadata['contact_names']['pad_geom_ids'] + pad_table = {side: {pair: ticks for pair, ticks in pair_counts.items() + if table in [int(v) for v in pair.split(':')] and any(pad in [int(v) for v in pair.split(':')] for pad in ids)} + for side, ids in pads.items()} + result = dict(relative_episode=relative, case=attempt['case'], controller=controller, + complete_intervals=1000, teacher_queries=rollout['teacher_queries'], strict_success=rollout['score']['success'], + initial_visual_error_m=rollout['initial_anchor_error_m'], native_score=score, + tcp_path_length_m=float(np.linalg.norm(np.diff(arrays['tcp_position_m'], axis=0), axis=1).sum()), + tcp_path_scope='Euclidean travel between all saved 1 ms distal-site positions, not link6 feature positions.', + rate_limited_intervals=sum(step['applied']['rate_limited'] for step in steps), + issued_to_measured_max_absolute_joint_difference_rad=float(np.abs(issued - measured).max()), + issued_to_final_applied_joint_rms_difference_rad=float(np.sqrt(np.mean((issued - applied) ** 2))), + issued_joint_total_variation_rad=np.abs(np.diff(issued, axis=0)).sum(axis=0).tolist(), + issued_jaw_total_variation_m=float(np.abs(np.diff(jaw)).sum()), + bilateral_object_contact_ticks=int(both.sum()), bilateral_object_contact_intervals=intervals(both, times), + any_object_contact_intervals=intervals(any_contact, times), critical_contact_intervals=intervals(critical, times), + recorded_pad_table_contact_pair_ticks=pad_table, + pair_counter_scope='Simulator-recorded 1 ms geom-pair counters; sides may overlap. This analysis does not rerun a contact solver.', + first_missing_estimate_seconds=first_time(~available, control_times), + visual_available_decisions=int(available.sum()), closure_events=events, teacher_phase_starts=phases, + maximum_absolute_joint_velocity_rad_s=np.abs(velocity).max(axis=0).tolist(), + velocity_normalization=dict(maximum_absolute_units=float(np.abs(normalized_velocity).max()), + first_time_above_5_units_seconds=first_time(np.any(np.abs(normalized_velocity) > 5, axis=1), control_times), + decisions_above_5_units=int(np.any(np.abs(normalized_velocity) > 5, axis=1).sum()), + scope='Unclipped descriptive posthoc distance from training feature mean divided by stored feature scale. The frozen model clips standardized inputs to +/-10 before inference. Five units is not a registered gate or proof of being outside the full training support.'), + source_files=used) + return result, arrays + + +def diagnose(study): + root = Path(study).absolute() + protocol = read(root / 'protocol.json'); registration = read(root / 'registration.json') + protocol_identity = identity((root / 'protocol.json').read_bytes()) + if registration['protocol_sha256'] != protocol_identity['sha256']: + raise ValueError('Protocol binding differs.') + student_attempt = next(row for row in protocol['rosters']['development-initial'] if row['controller'] == 'student') + teacher_attempt = next(row for row in protocol['rosters']['demonstrations'] if row['case'] == student_attempt['case']) + train_path = root / 'stages/train-initial.json'; train = read(train_path) + train_identity = identity(train_path.read_bytes()) + if read(root / 'stages/train-initial.integrity.json') != dict(sha256=train_identity['sha256'], protocol_sha256=protocol_identity['sha256']): + raise ValueError('Initial fit is not sealed.') + model_path = root / train['models']['full']['path']; model_identity = identity(model_path.read_bytes()) + if train['models']['full']['model_sha256'] != model_identity['sha256']: + raise ValueError('Initial model changed.') + model = read(model_path) + student_relative = 'episodes/development-initial/' + student_attempt['id'] + teacher_relative = 'episodes/demonstrations/' + teacher_attempt['id'] + student_row, student_stage_sources = stage_binding(root, 'development-initial', student_attempt, protocol, protocol_identity['sha256']) + teacher_row, teacher_stage_sources = stage_binding(root, 'demonstrations', teacher_attempt, protocol, protocol_identity['sha256']) + student, student_arrays = episode(root, student_relative, student_attempt, model) + teacher, teacher_arrays = episode(root, teacher_relative, teacher_attempt, model) + if student_row['policy_artifact_sha256'] != model_identity['sha256']: + raise ValueError('The first student did not run the sealed initial full model.') + distance = np.linalg.norm(student_arrays['tcp_position_m'] - teacher_arrays['tcp_position_m'], axis=1) + divergence = dict(first_more_than_10mm_seconds=first_time(distance > .010, student_arrays['t_seconds']), + first_more_than_50mm_seconds=first_time(distance > .050, student_arrays['t_seconds']), + scope='Same simulated time, same registered start/goal; this is not phase-aligned prediction error.') + return dict(schema=SCHEMA, posthoc=True, scope='First registered initial development student versus exact matching synthetic teacher; not aggregate confirmation and not model tuning.', + audit_status='sealed_local_diagnostic_pending_full_study_audit', + audit_scope='Selected episode and stage byte bindings plus the numeric checks stated here; not a full-study audit, source attestation or hardware verification.', + physics_executed=False, model_executed=False, teacher_queried=False, qualification_changed=False, + selection='First protocol-ordered full-student attempt, regardless of outcome.', + source=dict(protocol=protocol_identity, initial_training_stage=train_identity, full_model=model_identity, + registration=identity((root / 'registration.json').read_bytes()), + initial_training_seal=identity((root / 'stages/train-initial.integrity.json').read_bytes()), + stage_files={**student_stage_sources, **teacher_stage_sources}, + analysis_source=identity(Path(__file__).read_bytes())), + validation_on_teacher_observations=model['training']['validation'], student=student, teacher=teacher, + same_clock_trajectory_divergence=divergence, interpretation=interpretation(student, teacher, divergence)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study', required=True); parser.add_argument('--output', required=True) + args = parser.parse_args(argv) + output = Path(args.output).absolute() + if output.exists() or output.resolve().is_relative_to(Path(args.study).resolve()): + parser.error('Choose a fresh proof file outside the study.') + result = diagnose(args.study) + with output.open('x', encoding='utf-8') as stream: + json.dump(result, stream, indent=2, sort_keys=True, allow_nan=False) + stream.write('\n') + print(json.dumps(dict(output=str(output), sha256=identity(output.read_bytes())['sha256'], + student_success=result['student']['strict_success'], teacher_success=result['teacher']['strict_success'], physics_executed=False))) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/diagnostic_phase_coverage.py b/benchmarks/camera-action-v01/diagnostic_phase_coverage.py new file mode 100644 index 0000000..15eeb92 --- /dev/null +++ b/benchmarks/camera-action-v01/diagnostic_phase_coverage.py @@ -0,0 +1,122 @@ +"""Read-only phase coverage characterization of the four audited diagnostics. + +Teacher phase is used only for this offline analysis. It is never exported as +a learner input. Motor decisions reuse a 25 Hz camera at 50 Hz, so decision +counts are correlated observations, not independent tracking trials. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from hashlib import sha256 +import json +from pathlib import Path + + +MANIPULATION_PHASES = ('close', 'lift', 'transfer', 'place', 'release') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128*1024**2: + raise ValueError('Expected an ordinary bounded JSON artifact.') + return json.loads(path.read_bytes()) + + +def hash_file(path): + return sha256(Path(path).read_bytes()).hexdigest() + + +def sidecar(root, reference, inventory): + name = reference['path']; relative = Path(name) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError('Invalid source path.') + path = root/relative + if path.is_symlink(): + raise ValueError('Source cannot be a symlink.') + raw = path.read_bytes() + identity = {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + if (inventory.get(name) != identity + or {key: reference[key] for key in ('bytes', 'sha256')} != identity): + raise ValueError('Teacher/perception sidecar identity changed.') + return json.loads(raw) + + +def analyze(study, audited): + root = Path(study) + evidence = read(audited) + if (evidence.get('schema') != 'dvidia.camera-action-diagnostic-audit/v1' + or evidence.get('passed') is not True or evidence.get('attempts_verified') != 4): + raise ValueError('A passing four-diagnostic independent audit is required.') + stage = read(root/'stages/diagnostic.json') + protocol = read(root/'protocol.json') + registration = read(root/'registration.json') + if hash_file(root/'protocol.json') != registration['protocol_sha256']: + raise ValueError('Registered protocol changed.') + expected = {row['id']: row for row in protocol['rosters']['diagnostic']} + if len(stage['rows']) != 4 or {row['id'] for row in stage['rows']} != set(expected): + raise ValueError('Expected the exact four diagnostic rows.') + audited_cases = {row['case_id'] for row in evidence['results']} + if audited_cases != {row['case']['id'] for row in stage['rows']}: + raise ValueError('Audit and diagnostic case identities differ.') + total = defaultdict(lambda: {'decisions': 0, 'available': 0}) + results = [] + for row in stage['rows']: + relative = Path(row['path']) + if relative.is_absolute() or '..' in relative.parts: + raise ValueError('Episode path is not study-relative.') + episode = root/relative + if (hash_file(episode/'integrity.json') != row['integrity_sha256'] + or hash_file(episode/'rollout.json') != row['receipt_sha256']): + raise ValueError('Episode binding changed after stage sealing.') + inventory = read(episode/'integrity.json')['files'] + raw = (episode/'recording.json').read_bytes() + if inventory['recording.json'] != {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()}: + raise ValueError('Recording bytes differ from the episode seal.') + recording = json.loads(raw) + counts = defaultdict(lambda: {'decisions': 0, 'available': 0}) + for step in recording['steps']: + query = sidecar(episode, step['teacher_query'], inventory) + perception = sidecar(episode, step['perception'], inventory) + if (query['snapshot_sha256'] != digest(step['before']) + or perception['snapshot_sha256'] != digest(step['before']) + or query['index'] != step['index'] or perception['index'] != step['index']): + raise ValueError('Phase and perception are not aligned to the same decision.') + phase = query.get('phase') + if type(phase) is not str or len(phase) > 80: + raise ValueError('Expected a bounded offline teacher phase label.') + available = int(perception['estimate']['status'] == 'available') + counts[phase]['decisions'] += 1; counts[phase]['available'] += available + total[phase]['decisions'] += 1; total[phase]['available'] += available + results.append(dict(case_id=row['case']['id'], source_group=row['case']['group_id'], + phases=dict(counts), integrity_sha256=row['integrity_sha256'])) + contact_count = sum(total[phase]['decisions'] for phase in MANIPULATION_PHASES) + contact_available = sum(total[phase]['available'] for phase in MANIPULATION_PHASES) + return dict(schema='dvidia.camera-action-phase-coverage/v1', + protocol_sha256=hash_file(root/'protocol.json'), diagnostic_audit_sha256=hash_file(audited), + analysis_source_sha256=hash_file(__file__), cases=4, starting_position_groups=2, + total_motor_decisions=sum(v['decisions'] for v in total.values()), phases=dict(total), + manipulation_phases=list(MANIPULATION_PHASES), manipulation_decisions=contact_count, + manipulation_available=contact_available, results=results, + evidence_scope='Offline teacher phase analysis only; phases are excluded from learner inputs. Repeated 50 Hz decisions share 25 Hz camera frames.', + physics_executed=False, estimator_executed=False, physical_robot_ready=False) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study'); parser.add_argument('--audit', required=True) + args = parser.parse_args(argv) + print(json.dumps(analyze(args.study, args.audit), indent=2, allow_nan=False)) + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/independent_audit.py b/benchmarks/camera-action-v01/independent_audit.py new file mode 100644 index 0000000..b4cdc58 --- /dev/null +++ b/benchmarks/camera-action-v01/independent_audit.py @@ -0,0 +1,276 @@ +"""Read-only diagnostic audit; imports no controller, renderer or environment. + +Recomputes numeric evidence and identities, not the simulator itself. Passing +this audit means the reported failures or successes agree with saved evidence; +it does not imply that the diagnostic or learned-skill gates passed. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import math +from pathlib import Path + +import numpy as np + +from dvidia_training import piper_feedback_score as scoring + + +SCHEMA = 'dvidia.camera-action-diagnostic-audit/v1' + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128*1024**2: + raise ValueError('Expected a bounded ordinary JSON artifact: '+path.name) + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON field.') + result[key] = value + return result + def invalid(value): + raise ValueError('Nonfinite JSON value.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + + +def identity(path): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ValueError('Artifact is not an ordinary file.') + h = sha256() + with path.open('rb') as stream: + for block in iter(lambda: stream.read(1024*1024), b''): + h.update(block) + return dict(bytes=path.stat().st_size, sha256=h.hexdigest()) + + +def relative(root, name): + path = Path(name) + if path.is_absolute() or '..' in path.parts or not path.parts or str(path) != name: + raise ValueError('Artifact paths must be canonical relative paths.') + result = root/path + if any(p.is_symlink() for p in (result, *result.parents) if str(p) not in ('/tmp', '/var')): + raise ValueError('Artifact path traverses a symlink.') + return result + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def inventory(root): + manifest = read(root/'integrity.json') + require(manifest.get('schema') == 'dvidia.camera-action-rollout/v2/integrity', 'Unexpected episode integrity schema.') + files = manifest['files'] + actual = {str(p.relative_to(root)) for p in root.rglob('*') if p.is_file() and p.name != 'integrity.json'} + require(set(files) == actual, 'Episode inventory does not match its seal.') + for name, expected in files.items(): + require(identity(relative(root, name)) == expected, 'Artifact identity changed: '+name) + return files + + +def _reference(root, ref, files): + require(type(ref) is dict and set(ref) == {'path', 'bytes', 'sha256'}, 'Malformed artifact reference.') + require(files.get(ref['path']) == {key: ref[key] for key in ('bytes', 'sha256')}, 'Artifact reference does not match sealed bytes.') + return read(relative(root, ref['path'])) + + +def _close(a, b, name, tolerance=1e-8): + require(type(a) in (int, float) and type(b) in (int, float) + and math.isfinite(a) and math.isfinite(b) and math.isclose(a, b, abs_tol=tolerance, rel_tol=0.), name) + + +def _transform(metadata): + raw = metadata.get('world_from_robot_base') + require(metadata.get('world_from_robot_base_sha256') == digest(raw), 'Fixed-base transform hash differs.') + value = np.asarray(raw, dtype=float) + if value.shape == (16,): + value = value.reshape(4, 4) + require(value.shape == (4, 4) and np.isfinite(value).all(), 'Actual world-from-base transform is absent.') + require(np.allclose(value[3], [0., 0., 0., 1.], atol=1e-8, rtol=0) + and np.allclose(value[:3, :3].T@value[:3, :3], np.eye(3), atol=1e-8, rtol=0) + and math.isclose(float(np.linalg.det(value[:3, :3])), 1., abs_tol=1e-8), 'Invalid fixed-base transform.') + return value + + +def audit_episode(root, expected_case): + root = Path(root) + files = inventory(root) + summary = read(root/'rollout.json') + require(summary.get('case') == expected_case and summary.get('mode') == 'teacher' + and summary.get('negative_control') is None, 'Diagnostic identity differs from registration.') + result = dict(case_id=expected_case['id'], files_verified=len(files), status=summary['status'], + strict_success=False, complete_intervals=0, native_ticks=0, camera_pairs=0, + initial_anchor_error_m=None, available_position_error_p95_m=None, + maximum_native_object_displacement_m=None) + if 'recording.json' not in files: + require(summary['status'] == 'refused' and summary['recorded_complete_intervals'] == 0 + and summary['strict_success'] is False, 'Setup failure claimed completed data or success.') + return result + recording = read(root/'recording.json') + require(recording.get('schema') == 'dvidia.camera-action-rollout/v2' + and recording.get('control_mode') == 'teacher', 'Diagnostic recording schema/control changed.') + require(recording['provenance']['source_group'] == expected_case['group_id'] + and recording['provenance']['case_id'] == expected_case['id'] + and recording['goal_position_m'] == expected_case['goal_position_m'], 'Recording group/goal identity changed.') + steps = recording['steps'] + require(len(steps) == summary['recorded_complete_intervals'], 'Recorded interval count differs.') + context = read(root/'context.json') + transform = _transform(context['metadata']) + native_episode = read(root/'native_episode.json') + native_metadata = read(root/'native_metadata.json') + with np.load(root/'native.npz', allow_pickle=False) as handle: + arrays = {name: handle[name] for name in handle.files} + require(all(value.dtype.kind in 'iuf' for value in arrays.values()), 'Native arrays are not numeric-only.') + score = scoring.score_feedback_episode(native_episode, arrays, native_metadata, expected_case['goal_position_m'][:2]) + require(score == summary['score'], 'Saved independent score differs from recomputation.') + require(len(arrays['native_tick']) == native_metadata['rows'], 'Native row count differs.') + require(np.array_equal(arrays['native_tick'], np.arange(1, native_metadata['rows']+1)), 'Native tick ordering differs.') + require(np.allclose(arrays['t_seconds'], np.arange(1, native_metadata['rows']+1)*.001, atol=1e-8, rtol=0), 'Native 1 ms cadence differs.') + initial_world = np.asarray(native_episode['trajectory'][0]['object_position_m'], dtype=float) + initial_base = transform[:3, :3].T@(initial_world-transform[:3, 3]) + native_position = np.asarray(arrays['object_position_m']) + prior = recording['initial']; estimates = []; visible_errors = []; cameras = {} + for index, snapshot in enumerate([prior]+[s['after'] for s in steps]): + _close(snapshot['timestamp_seconds'], index*.02, 'Encoder 50 Hz cadence differs.') + camera = snapshot['camera']; frame = camera['value'] + require(frame['index'] == index//2 and snapshot['vision_frame_index'] == index//2, 'Camera source index differs from 25 Hz cadence.') + _close(camera['captured_at_seconds'], (index//2)*.04, 'Camera capture time differs.') + _close(camera['age_seconds'], index*.02-(index//2)*.04, 'Camera age was reset or changed.') + require(camera['captured_at_seconds'] <= camera['available_at_seconds'] <= snapshot['timestamp_seconds']+1e-8, + 'Camera timestamps are noncausal.') + if frame['index'] in cameras: + require(cameras[frame['index']] == frame, 'A reused camera frame changed its bytes.') + cameras[frame['index']] = frame + for channel in ('rgb', 'depth'): + reference = frame[channel] + require(files.get(reference['path']) == {key: reference[key] for key in ('bytes', 'sha256')}, 'Frame source identity changed.') + require(digest(recording['calibration']) == recording['binding']['calibration_sha256'], 'Camera calibration binding differs.') + for index, step in enumerate(steps): + require(step['index'] == index and step['before'] == prior, 'Transition prefix continuity differs.') + require(step['command']['authority'] == 'teacher' and step['command']['operator_correction'] is False, + 'Diagnostic issued authority is not its teacher.') + perception = _reference(root, step['perception'], files) + query = _reference(root, step['teacher_query'], files) + for sidecar in (perception, query): + require(sidecar['index'] == index and sidecar['snapshot_sha256'] == digest(step['before']), 'Sidecar is bound to a different decision snapshot.') + require(query['action'] == step['command']['issued'] == step['command']['requested'] + and query['origin'] == 'privileged_simulator_teacher', 'Teacher query/issued targets differ.') + require(perception['frame'] == step['before']['camera']['value'] + and perception['calibration_sha256'] == recording['binding']['calibration_sha256'] + and perception['tracker_sha256'] == recording['tracker_sha256'], 'Perception frame/calibration/tracker binding differs.') + estimate = perception['estimate']; estimates.append(estimate) + camera = step['before']['camera'] + require(estimate['frame_index'] == camera['value']['index'], 'Estimate frame differs from camera.') + for field in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(estimate[field], camera[field], 'Estimate and source camera timestamps differ.') + _close(estimate['decision_at_seconds'], index*.02, 'Estimate decision clock differs.') + tick = (index+1)*20-1 + require(tick < native_metadata['rows'], 'Complete motor interval has no native endpoint.') + np.testing.assert_allclose(step['applied']['ramped_targets']['joint_targets_rad'], + arrays['arm_actuator_controls_rad'][tick], atol=1e-9, rtol=0) + _close(step['applied']['ramped_targets']['gripper_position_m'], float(arrays['applied_jaw_target_m'][tick]), 'Applied gripper endpoint differs from native data.') + if estimate['status'] == 'available': + position = np.asarray(estimate['object_position_estimate_m']) + captured_tick = round(estimate['captured_at_seconds']/.001) + world = initial_world if captured_tick == 0 else native_position[captured_tick-1] + truth = transform[:3, :3].T@(world-transform[:3, 3]) + visible_errors.append(float(np.linalg.norm(position-truth))) + for field in ('object_position_estimate_m', 'object_position_uncertainty_m'): + signal = estimate['signals'][field] + require(signal['status'] == 'available', 'Available estimate has unavailable signals.') + for stamp in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(signal[stamp], estimate[stamp], 'Estimate signal age/source differs.') + else: + for field in ('object_position_estimate_m', 'object_position_uncertainty_m'): + require(estimate[field] is None and estimate['signals'][field]['value'] is None, + 'Unavailable estimate forwards a position or uncertainty.') + prior = step['after'] + query_count = sum(name.startswith('teacher/') and name.endswith('.json') for name in files) + require(summary['teacher_queries'] == query_count and len(steps) <= query_count <= len(steps)+1 and summary['student_calls'] == 0, + 'Diagnostic teacher query count or student intervention differs.') + require(summary['camera_pairs'] == len(cameras), 'Unique camera pair count differs.') + expected_success = summary['status'] == 'complete' and len(steps) == 1000 and score.get('success') is True + require(summary['strict_success'] is expected_success, 'Diagnostic strict success differs from complete native scoring.') + initial_error = None + if estimates and estimates[0]['status'] == 'available': + initial_error = float(np.linalg.norm(np.asarray(estimates[0]['object_position_estimate_m'])-initial_base)) + _close(summary['initial_anchor_error_m'], initial_error, 'Initial camera anchor error differs from native reconstruction.') + else: + require(summary['initial_anchor_error_m'] is None, 'Unavailable anchor has a fabricated error.') + missing = [e['status'] != 'available' for e in estimates] + longest = streak = 0 + for value in missing: + streak = streak+1 if value else 0; longest = max(longest, streak) + expected_perception = dict(decisions=len(estimates), available=sum(not m for m in missing), + missing=sum(missing), longest_missing_seconds=longest*.02) + require(summary['perception'] == expected_perception, 'Perception coverage/missing-duration summary differs.') + result.update(strict_success=expected_success, complete_intervals=len(steps), native_ticks=native_metadata['rows'], + camera_pairs=len(cameras), initial_anchor_error_m=initial_error, perception=expected_perception, + available_position_error_p95_m=float(np.quantile(visible_errors, .95)) if visible_errors else None, + maximum_native_object_displacement_m=float(np.max(np.linalg.norm(native_position-initial_world, axis=1))) if len(native_position) else 0., + score=score) + return result + + +def audit(directory): + root = Path(directory) + errors, results = [], [] + try: + protocol, registration = read(root/'protocol.json'), read(root/'registration.json') + require(identity(root/'protocol.json')['sha256'] == registration['protocol_sha256'], 'Study protocol changed after registration.') + require(protocol['source_sha256'] == registration['source_sha256'], 'Registration/runtime source lists disagree.') + for name, expected in protocol['source_sha256'].items(): + require(identity(Path(scoring.__file__).with_name(name))['sha256'] == expected, 'Registered source changed: '+name) + roster = protocol['rosters']['diagnostic'] + require(len(roster) == 4 and len({r['id'] for r in roster}) == 4, 'Expected exactly four diagnostic attempts.') + stage = read(root/'stages/diagnostic.json') + seal = read(root/'stages/diagnostic.integrity.json') + require(seal == dict(sha256=identity(root/'stages/diagnostic.json')['sha256'], + protocol_sha256=identity(root/'protocol.json')['sha256']), 'Diagnostic stage seal changed.') + require({row['id'] for row in stage['rows']} == {row['id'] for row in roster} + and len(stage['rows']) == 4, 'Diagnostic denominator differs from its roster.') + teacher_binding = digest(protocol['source_sha256']) + require(stage.get('model_bindings') == {'teacher': teacher_binding}, 'Diagnostic teacher runtime binding differs.') + for attempt in roster: + try: + row = next(r for r in stage['rows'] if r['id'] == attempt['id']) + require(row['case'] == attempt['case'], 'Stage row is associated with a different case.') + bound_case = {**attempt['case'], 'policy_artifact_sha256': teacher_binding} + result = audit_episode(root/'episodes/diagnostic'/attempt['id'], bound_case) + require(row['strict_success'] is result['strict_success'], 'Stage strict success differs from raw evidence.') + results.append(result) + except (ValueError, OSError, KeyError, TypeError, AssertionError) as exc: + errors.append(dict(attempt=attempt['id'], reason=str(exc))) + except (ValueError, OSError, KeyError, TypeError) as exc: + errors.append(dict(attempt=None, reason=str(exc))) + return dict(schema=SCHEMA, passed=not errors and len(results) == 4, errors=errors, + attempts_verified=len(results), strict_successes=sum(r['strict_success'] for r in results), results=results, + audit_source_sha256=identity(Path(__file__))['sha256'], scorer_source_sha256=identity(Path(scoring.__file__))['sha256'], + physics_executed=False, estimator_executed=False, source_authenticated=False, physical_robot_ready=False, + scope='Independent byte/clock/control and numeric task reconstruction; synthetic depth accuracy and hardware transfer remain unverified.') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study') + args = parser.parse_args(argv) + result = audit(args.study) + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 if result['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/study_audit.py b/benchmarks/camera-action-v01/study_audit.py new file mode 100644 index 0000000..ab66301 --- /dev/null +++ b/benchmarks/camera-action-v01/study_audit.py @@ -0,0 +1,711 @@ +"""Independent saved-evidence audit across the camera/action study lifecycle. + +Imports numeric scoring and the diagnostic auditor's byte readers only. No +environment, runner, controller, tracker or study-report function is imported. +The audit does not initialize physics, infer poses or execute learned actions. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import importlib.util +import math +from pathlib import Path +import random + +import numpy as np + +# Several historical benchmarks have an independent_audit.py. Resolve our +# sibling by file identity rather than importing an unrelated cached module. +_helper_spec = importlib.util.spec_from_file_location('dvidia_camera_action_diagnostic_audit', + Path(__file__).with_name('independent_audit.py')) +_helper = importlib.util.module_from_spec(_helper_spec) +_helper_spec.loader.exec_module(_helper) +canonical, digest, read, identity, relative, require, inventory, _reference, _close, _transform, scoring = ( + getattr(_helper, name) for name in ('canonical', 'digest', 'read', 'identity', 'relative', + 'require', 'inventory', '_reference', '_close', '_transform', 'scoring')) + +SCHEMA = 'dvidia.camera-action-study-audit/v1' +STAGES = ('diagnostic', 'demonstrations', 'train-initial', 'development-initial', + 'corrections', 'train-corrected', 'development-corrected', 'confirmation', 'negative') +VISION = ('object_position_estimate_m', 'object_quaternion_estimate_wxyz', + 'object_position_uncertainty_m', 'object_orientation_uncertainty_rad') +ENCODERS = ('joint_position_rad', 'joint_velocity_rad_s', 'gripper_joint_position_m', + 'gripper_joint_velocity_m_s', 'tcp_position_estimate_m', 'tcp_quaternion_estimate_wxyz') + + +def _action(value, limits): + require(type(value) is dict and set(value) == {'joint_targets_rad', 'gripper_position_m'}, 'Unknown or missing action fields.') + q = np.asarray(value['joint_targets_rad'], dtype=float) + bounds = np.asarray(limits['joint_limits_rad'], dtype=float) + require(q.shape == (6,) and bounds.shape == (6, 2) and np.isfinite(q).all() + and np.all(q >= bounds[:, 0]) and np.all(q <= bounds[:, 1]), 'Action joint targets violate native bounds.') + jaw = value['gripper_position_m']; low, high = limits['gripper_position_limits_m'] + require(type(jaw) in (int, float) and math.isfinite(jaw) and low <= jaw <= high, 'Action gripper target violates native bounds.') + + +def command_check(step, query, mode, negative, rng, limits): + command = step['command'] + require(command.get('operator_correction') is False, 'Synthetic control cannot claim operator correction.') + for name in ('requested', 'issued'): + _action(command[name], limits) + expected_authority = ('teacher' if mode == 'teacher' else + ('teacher' if rng.random() < .5 else 'student') if mode == 'mixed' else mode) + require(command['authority'] == expected_authority, 'Command authority differs from registered controller or seeded mixture.') + expected_issued = deepcopy(command['requested']) + if negative == 'open_gripper': + expected_issued['gripper_position_m'] = .035 + require(command['issued'] == expected_issued, 'Issued command differs from its explicitly registered intervention.') + require(command['issue_reason'] == ('open_gripper_negative_control' if expected_issued != command['requested'] else 'unchanged'), + 'Command intervention reason is inconsistent.') + if mode in ('teacher', 'mixed'): + require(type(query) is dict and query.get('origin') == 'privileged_simulator_teacher', 'Missing or mislabeled teacher query.') + _action(query['action'], limits) + if expected_authority == 'teacher': + require(query['action'] == command['requested'], 'Executed teacher request differs from its separately sealed query.') + else: + require(query is None, 'A student or fixed rollout contains teacher intervention.') + + +def _native(root, files, case, summary): + required = {'native_episode.json', 'native_metadata.json', 'native.npz'} + if not required <= set(files): + require(not required.intersection(files) and summary.get('recorded_complete_intervals') == 0, + 'Partial native evidence cannot justify completed intervals.') + require(summary['score'].get('valid') is False and summary['score'].get('success') is False, + 'No-native-evidence attempt claims a valid native score.') + return None, None, None, dict(valid=False, success=False, reason='native_evidence_unavailable') + episode = read(root / 'native_episode.json'); metadata = read(root / 'native_metadata.json') + with np.load(root / 'native.npz', allow_pickle=False) as handle: + arrays = {key: handle[key] for key in handle.files} + require(set(arrays) == {name for name, width in scoring.FIELDS}, 'Unknown or missing native arrays.') + count = metadata.get('rows') + require(type(count) is int and 0 <= count <= 20000, 'Native trace exceeds the registered time budget.') + for name, width in scoring.FIELDS: + values = arrays[name] + require(values.dtype.kind in 'iuf' and values.shape == ((count,) if width == 1 else (count, width)) + and np.isfinite(values).all(), 'Native array shape or finite values differ: ' + name) + require(np.array_equal(arrays['native_tick'], np.arange(1, count + 1)), 'Native tick ordering differs.') + require(np.allclose(arrays['t_seconds'], np.arange(1, count + 1) * .001, atol=1e-8, rtol=0), 'Native 1 ms cadence differs.') + require(episode['config']['object_position_m'] == case['object_position_m'] + and episode['config']['target_position_m'] == case['goal_position_m'], 'Native scene differs from registered start or goal.') + score = scoring.score_feedback_episode(episode, arrays, metadata, case['goal_position_m'][:2]) + require(score == summary['score'], 'Saved native score differs from independent numeric reconstruction.') + ticks = int(np.count_nonzero(arrays['unexpected_contact_count'] > 0)) + require(summary.get('critical_contact_ticks') == ticks, 'Critical contact count differs from native ticks.') + return episode, arrays, metadata, score + + +def _snapshot(snapshot, index, files, cameras, native_state=None): + _close(snapshot['timestamp_seconds'], index * .02, 'Measured snapshot is not on the 50 Hz clock.') + require(set(snapshot['signals']) == set(VISION) | set(ENCODERS), 'Raw sensor channels differ from the registered whitelist.') + for name in VISION: + signal = snapshot['signals'][name] + require(signal['status'] == 'missing' and signal['value'] is None, 'Raw snapshot substitutes native object truth for perception.') + for name in ENCODERS: + signal = snapshot['signals'][name] + require(signal['status'] == 'available', 'Registered native encoder channel is unavailable.') + for stamp in ('captured_at_seconds', 'available_at_seconds'): + _close(signal[stamp], snapshot['timestamp_seconds'], 'Encoder signal timestamp differs.') + _close(signal['age_seconds'], 0., 'Encoder signal age differs.') + if native_state and name in native_state: + require(signal['value'] == native_state[name], 'Encoder signal differs from saved native measurement: ' + name) + camera = snapshot['camera']; frame = camera['value']; expected_index = index // 2 + require(frame['index'] == expected_index and snapshot['vision_frame_index'] == expected_index, 'Camera source index differs from 25 Hz cadence.') + _close(camera['captured_at_seconds'], expected_index * .04, 'Camera capture time differs.') + _close(camera['available_at_seconds'], expected_index * .04, 'Synthetic camera availability differs from its captured frame.') + _close(camera['age_seconds'], index * .02 - expected_index * .04, 'Camera age was reset or changed.') + if expected_index in cameras: + require(frame == cameras[expected_index], 'Reused camera frame changed its identity.') + cameras[expected_index] = frame + for channel in ('rgb', 'depth'): + reference = frame[channel] + require(files.get(reference['path']) == {key: reference[key] for key in ('bytes', 'sha256')}, 'Camera frame is not byte-bound to the recording.') + + +def _rotation(quaternion): + q = np.asarray(quaternion, dtype=float) + require(q.shape == (4,) and np.isfinite(q).all() and abs(float(q @ q)-1.) < 1e-7, + 'Encoder pose quaternion is not a finite unit quaternion.') + w, x, y, z = q + return np.asarray([[1-2*(y*y+z*z), 2*(x*y-z*w), 2*(x*z+y*w)], + [2*(x*y+z*w), 1-2*(x*x+z*z), 2*(y*z-x*w)], + [2*(x*z-y*w), 2*(y*z+x*w), 1-2*(x*x+y*y)]]) + + +def _encoder_pose(snapshot, native_state, world_from_base): + signals = snapshot['signals'] + require(signals['gripper_joint_velocity_m_s']['value'] == native_state['qvel'][6:8], + 'Gripper encoder velocity differs from saved native state.') + # Frozen capture names the link6 body origin "tcp_position_estimate_m". + # The task scorer uses its distal authored site at link6 [0,0,.12003]. + # Reconstruct this explicit site-to-body offset; do not execute kinematics. + world_rotation = _rotation(native_state['tcp_quaternion_wxyz']) + world_link6 = np.asarray(native_state['tcp_position_m'])-world_rotation @ np.asarray([0., 0., .12003]) + expected = world_from_base[:3, :3].T @ (world_link6-world_from_base[:3, 3]) + require(np.allclose(signals['tcp_position_estimate_m']['value'], expected, atol=1e-8, rtol=0), + 'Encoder tool position differs from saved native pose and base transform.') + expected_rotation = world_from_base[:3, :3].T @ world_rotation + require(np.allclose(_rotation(signals['tcp_quaternion_estimate_wxyz']['value']), expected_rotation, atol=1e-8, rtol=0), + 'Encoder tool orientation differs from saved native pose and base transform.') + + +def _perception(document, snapshot, recording, negative, index): + require(document['index'] == index and document['snapshot_sha256'] == digest(snapshot), 'Perception is bound to a different motor snapshot.') + _close(document['decision_at_seconds'], index * .02, 'Perception decision time differs.') + require(document['frame'] == snapshot['camera']['value'] + and document['calibration_sha256'] == recording['binding']['calibration_sha256'] + and document['tracker_sha256'] == recording['tracker_sha256'], 'Perception frame/calibration/tracker identity differs.') + expected_fault = negative if snapshot['camera']['value']['index'] == 0 else None + require(document.get('negative_control') == expected_fault, 'Perception intervention differs from its registered initial fault.') + calibration = deepcopy(recording['calibration']) + if expected_fault == 'altered_calibration': + calibration['base_from_camera'][0] *= 1.1 + require(document['input_calibration_sha256'] == digest(calibration), 'Actual estimator input calibration differs from the declared intervention.') + estimate = document['estimate']; camera = snapshot['camera'] + require(estimate['frame_index'] == camera['value']['index'], 'Estimate source frame differs from the camera.') + captured = -.2 if expected_fault == 'initial_stale' else camera['captured_at_seconds'] + available = -.2 if expected_fault == 'initial_stale' else camera['available_at_seconds'] + _close(estimate['captured_at_seconds'], captured, 'Estimate capture time differs from its declared source.') + _close(estimate['available_at_seconds'], available, 'Estimate availability differs from its declared source.') + _close(estimate['decision_at_seconds'], index * .02, 'Estimate decision time differs.') + _close(estimate['age_seconds'], max(0., index * .02 - captured), 'Estimate age differs from its source.') + if estimate['status'] == 'available': + require(available <= index * .02 + 1e-8 and index * .02 - captured <= .1 + 1e-8, 'Available pose is stale or from the future.') + for field in VISION: + signal = estimate['signals'][field] + expected = estimate[field] + if field in ('object_position_uncertainty_m', 'object_orientation_uncertainty_rad'): + expected = [expected] + require(signal['status'] == 'available' and signal['value'] == expected, 'Available pose and sensor values disagree.') + for stamp in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + _close(signal[stamp], estimate[stamp], 'Estimate signal age or source differs.') + else: + for field in VISION: + require(estimate[field] is None and estimate['signals'][field]['status'] == 'missing' + and estimate['signals'][field]['value'] is None, 'Unavailable perception forwards a live object value.') + return estimate + + +def audit_episode(root, attempt, binding, *, tracker_sha256=None): + root = Path(root); files = inventory(root); summary = read(root / 'rollout.json') + mode = 'student' if attempt['controller'] == 'blind' else attempt['controller'] + case = {**attempt['case'], 'policy_artifact_sha256': binding} + negative = attempt['negative_control'] + require(summary['case'] == case and summary['mode'] == mode and summary['negative_control'] == negative, + 'Episode controller, scene or intervention differs from registration.') + if tracker_sha256 is not None: + require(summary['tracker_sha256'] == tracker_sha256, 'Episode used a different tracker implementation.') + started = read(root / 'started.json') + require(started['case'] == case and started['mode'] == mode and started['negative_control'] == negative + and started['config']['duration_s'] == 20. and started['config']['seed'] == case['seed'] + and started['config']['width'] == 320 and started['config']['height'] == 240 + and started['config']['camera_every_controls'] == 2, 'Started rollout differs from its registered scene, controller or capture budget.') + episode, arrays, metadata, score = _native(root, files, case, summary) + result = dict(id=attempt['id'], case=attempt['case'], controller=attempt['controller'], negative_control=negative, + status=summary['status'], complete=False, strict_success=False, score=score, + initial_anchor_available=False, initial_anchor_binding_valid=False, initial_anchor_error_m=None, + recorded_complete_intervals=0, critical_contact_ticks=summary.get('critical_contact_ticks'), + provenance_ok=mode in ('student', 'fixed') and summary.get('teacher_queries') == 0, + files_verified=len(files), native_ticks=metadata['rows'] if metadata else 0, + native_score_recomputed=episode is not None, available_position_error_p95_m=None, + integrity_sha256=identity(root / 'integrity.json')['sha256'], receipt_sha256=identity(root / 'rollout.json')['sha256']) + require(summary['no_privileged_features_or_teacher_intervention'] is result['provenance_ok'], 'Reported no-teacher provenance differs from query evidence.') + if 'recording.json' not in files: + require(summary['status'] == 'refused' and summary['recorded_complete_intervals'] == 0 + and summary['strict_success'] is False, 'Setup failure claims completed task evidence.') + return result + recording = read(root / 'recording.json'); steps = recording['steps']; initial = recording['initial'] + require(recording['tracker_sha256'] == summary['tracker_sha256'], 'Recording and receipt tracker identities differ.') + require(recording['schema'] == 'dvidia.camera-action-rollout/v2' and recording['control_mode'] == mode, + 'Recording schema or authority differs.') + require(recording['provenance']['case_id'] == case['id'] and recording['provenance']['source_group'] == case['group_id'] + and recording['goal_position_m'] == case['goal_position_m'], 'Recording group, goal or case differs.') + require(len(steps) == summary['recorded_complete_intervals'] and len(steps) <= 1000, 'Completed interval count differs.') + if initial is None: + require(not steps and summary['status'] == 'refused' and not summary['initial_anchor_available'], 'Missing initial snapshot cannot support motion or a visual anchor.') + return result + context = read(root / 'context.json'); transform = _transform(context['metadata']) + require(context['case'] == case and context['initial'] == initial, 'Initial context differs from the sealed recording.') + require(digest(recording['calibration']) == recording['binding']['calibration_sha256'], 'Camera calibration binding differs.') + require(digest(recording['limits']) == recording['binding']['limits_sha256'], 'Recorded native action limits changed.') + cameras = {}; snapshots = [initial] + [step['after'] for step in steps] + for index, snapshot in enumerate(snapshots): + state = episode['trajectory'][index] if episode and len(episode['trajectory']) > index else None + _snapshot(snapshot, index, files, cameras, state) + if state is not None: + _encoder_pose(snapshot, state, transform) + require(summary['camera_pairs'] == len(cameras), 'Unique camera pair count differs.') + prior = initial; rng = random.Random(case['seed']) + for index, step in enumerate(steps): + require(step['index'] == index and step['before'] == prior, 'Transition sequence is discontinuous.') + _close(step['command']['issued_at_seconds'], index * .02, 'Command issue clock differs.') + query = _reference(root, step['teacher_query'], files) if step['teacher_query'] is not None else None + if query: + require(query['index'] == index and query['snapshot_sha256'] == digest(step['before']), 'Teacher query uses a different input snapshot.') + _close(query['computed_at_seconds'], index * .02, 'Teacher query timing differs.') + command_check(step, query, mode, negative, rng, recording['limits']) + perception = _reference(root, step['perception'], files) + _perception(perception, step['before'], recording, negative, index) + require(episode is not None and arrays is not None, 'Complete interval has no independently reconstructable native trace.') + native_action = episode['actions'][index] + require({key: native_action[key] for key in ('joint_targets_rad', 'gripper_position_m')} == step['command']['issued'], + 'Command label differs from actual issued native evidence.') + tick = (index + 1) * 20 - 1 + require(tick < metadata['rows'], 'Complete interval lacks its native endpoint.') + expected_controls = [*arrays['arm_actuator_controls_rad'][tick].tolist(), float(arrays['applied_jaw_target_m'][tick])] + require(step['applied']['native_position_controls'] == expected_controls, 'Actual native position control differs.') + require(step['applied']['ramped_targets'] == dict(joint_targets_rad=expected_controls[:6], + gripper_position_m=float(arrays['normal_rate_limited_jaw_target_m'][tick])), 'Previous applied references differ from native controls.') + prior = step['after'] + perception_files = sorted(name for name in files if name.startswith('perception/') and name.endswith('.json')) + teacher_files = sorted(name for name in files if name.startswith('teacher/') and name.endswith('.json')) + require(perception_files == [f'perception/{i:04}.json' for i in range(len(perception_files))] + and len(steps) <= len(perception_files) <= len(steps) + 1, 'Perception decision prefix is discontinuous.') + require(teacher_files == [f'teacher/{i:04}.json' for i in range(len(teacher_files))], 'Teacher query prefix is discontinuous.') + if mode in ('teacher', 'mixed'): + require(len(steps) <= len(teacher_files) <= len(steps) + 1, 'Teacher queries are missing or repeated.') + else: + require(not teacher_files, 'Student or fixed controller contains teacher-query evidence.') + require(summary['teacher_queries'] == len(teacher_files), 'Teacher query counter differs from sealed evidence.') + for index, filename in enumerate(teacher_files): + query = read(root / filename) + require(query['index'] == index and query['snapshot_sha256'] == digest(snapshots[index]) + and query['origin'] == 'privileged_simulator_teacher', 'Teacher query prefix has an invalid origin or source snapshot.') + _close(query['computed_at_seconds'], index * .02, 'Teacher query prefix clock differs.') + _action(query['action'], recording['limits']) + require(summary['student_calls'] == 0 if mode == 'teacher' else len(steps) <= summary['student_calls'] <= len(steps) + 1, + 'Student inference count differs from the completed decision prefix.') + require(summary['no_privileged_features_or_teacher_intervention'] is result['provenance_ok'], 'Reported no-teacher provenance differs from query evidence.') + estimates, visible_errors = [], [] + initial_world = np.asarray(episode['trajectory'][0]['object_position_m']) if episode and episode['trajectory'] else None + for index, filename in enumerate(perception_files): + document = read(root / filename) + estimate = _perception(document, snapshots[index], recording, negative, index) + estimates.append(estimate) + if estimate['status'] == 'available' and initial_world is not None: + tick = round(estimate['captured_at_seconds'] / .001) + truth_world = initial_world if tick == 0 else arrays['object_position_m'][tick - 1] + truth_base = transform[:3, :3].T @ (truth_world - transform[:3, 3]) + visible_errors.append(float(np.linalg.norm(np.asarray(estimate['object_position_estimate_m']) - truth_base))) + anchor_available = bool(estimates and estimates[0]['status'] == 'available') + anchor_bound = anchor_available and negative not in ('initial_stale', 'altered_calibration') + require(summary['initial_anchor_available'] is anchor_available and summary['initial_anchor_binding_valid'] is anchor_bound, + 'Initial visual anchor availability or binding differs.') + anchor_error = visible_errors[0] if anchor_available and visible_errors else None + if anchor_error is None: + require(summary['initial_anchor_error_m'] is None, 'Unavailable anchor claims native diagnostic accuracy.') + else: + _close(summary['initial_anchor_error_m'], anchor_error, 'Initial anchor error differs from native reconstruction.') + missing = [estimate['status'] != 'available' for estimate in estimates] + longest = streak = 0 + for absent in missing: + streak = streak + 1 if absent else 0; longest = max(longest, streak) + perception = dict(decisions=len(estimates), available=sum(not value for value in missing), + missing=sum(missing), longest_missing_seconds=longest * .02) + require(summary['perception'] == perception, 'Perception coverage or longest missing interval differs.') + complete = summary['status'] == 'complete' and len(steps) == 1000 + strict = complete and score.get('success') is True + require(summary['strict_success'] is strict, 'Reported strict success differs from complete native scoring.') + result.update(complete=complete, strict_success=strict, initial_anchor_available=anchor_available, + initial_anchor_binding_valid=anchor_bound, initial_anchor_error_m=anchor_error, + recorded_complete_intervals=len(steps), perception=perception, + available_position_error_p95_m=float(np.quantile(visible_errors, .95)) if visible_errors else None) + return result + + +def _stage(root, name, protocol_hash, inputs): + path = root / 'stages' / (name + '.json') + if not path.exists(): + return None + seal_path = path.with_name(name + '.integrity.json') + record = identity(path) + require(read(seal_path) == dict(sha256=record['sha256'], protocol_sha256=protocol_hash), 'Stage receipt seal differs: ' + name) + receipt = read(path) + require(receipt['stage'] == name and receipt['protocol_sha256'] == protocol_hash, 'Stage receipt names a different stage or protocol.') + started_path = path.with_name(name + '.started.json') + started = read(started_path) + require(started['stage'] == name and started['protocol_sha256'] == protocol_hash + and started.get('model_bindings') == receipt.get('model_bindings'), 'Started stage and completed receipt bindings differ.') + inputs[str(started_path.relative_to(root))] = identity(started_path) + inputs[str(path.relative_to(root))] = record + inputs[str(seal_path.relative_to(root))] = identity(seal_path) + return receipt + + +def validate_roster(protocol): + groups, points = {}, {} + for split, expected in (('train', 64), ('development', 16), ('confirmation', 64)): + cases = protocol['cases'][split] + require(len(cases) == expected and len({case['id'] for case in cases}) == expected, 'Case denominator or identity differs.') + by_group = {} + for case in cases: + require(case['split'] == split and case['goal_id'] in (0, 1), 'Case split or goal differs.') + require(groups.setdefault(case['group_id'], split) == split, 'Starting group leaks between splits.') + point = tuple(case['object_position_m']) + require(len(point) == 3 and protocol['start_domain']['x_m'][0] <= point[0] <= protocol['start_domain']['x_m'][1] + and protocol['start_domain']['y_m'][0] <= point[1] <= protocol['start_domain']['y_m'][1] + and point[2] == .082 and case['goal_position_m'] == protocol['goals'][case['goal_id']], 'Case position or destination differs from the registered domain.') + require(list(point[:2]) not in protocol['historical_start_exclusion'], 'Fresh start duplicates an explicitly excluded historical start.') + require(points.setdefault(point, split) == split, 'Exact starting position leaks between splits.') + by_group.setdefault(case['group_id'], []).append(case) + require(len(by_group) == expected // 2 and all({c['goal_id'] for c in row} == {0, 1} + and len({tuple(c['object_position_m']) for c in row}) == 1 for row in by_group.values()), + 'Both goals must share one starting-position group.') + require(len(points) == 72, 'Expected 72 unique starting positions.') + expected_counts = dict(diagnostic=4, demonstrations=80, corrections=32, + **{'development-initial': 32, 'development-corrected': 32}, confirmation=256, negative=16) + require({name: len(rows) for name, rows in protocol['rosters'].items()} == expected_counts, 'Registered attempt budgets differ.') + for name, roster in protocol['rosters'].items(): + require(len({r['id'] for r in roster}) == len(roster), 'Duplicate attempt identity.') + for attempt in roster: + split = attempt['case']['split'] + require(attempt['case'] in protocol['cases'][split], 'Attempt contains an unregistered scene.') + expected_split = ('train' if name == 'corrections' else 'confirmation' if name in ('confirmation', 'negative') else + 'development' if name == 'diagnostic' or name.startswith('development-') else None) + require(expected_split is None or split == expected_split, 'Attempt crossed its registered split.') + expected_controllers = ({'teacher': 4} if name == 'diagnostic' else {'teacher': 80} if name == 'demonstrations' + else {'mixed': 32} if name == 'corrections' else {'student': 16, 'blind': 16} if name.startswith('development-') + else {'student': 64, 'blind': 64, 'fixed': 64, 'teacher': 64} if name == 'confirmation' else {'student': 16}) + require({c: sum(a['controller'] == c for a in roster) for c in {a['controller'] for a in roster}} == expected_controllers, + 'Controller budgets differ from the approved experiment.') + require(all(a['negative_control'] is None for a in roster) if name != 'negative' else + {c: sum(a['negative_control'] == c for a in roster) for c in {a['negative_control'] for a in roster}} + == dict(initial_missing=4, initial_stale=4, altered_calibration=4, open_gripper=4), 'Negative fault budgets differ.') + pairs = [(a['case']['id'], a['controller'], a['negative_control']) for a in roster] + require(len(set(pairs)) == len(pairs), 'Same scene/controller/fault appears more than once in a stage.') + if name == 'corrections': + require(len({a['case']['group_id'] for a in roster}) == 32 + and all(a['case']['goal_id'] == i % 2 for i, a in enumerate(roster)), 'Corrections must cover each training group once with alternating destinations.') + if name == 'negative': + require(len({a['case']['id'] for a in roster}) == 4 + and all(sum(a['case']['id'] == case_id for a in roster) == 4 for case_id, _, _ in pairs), 'Every negative scene must receive all four faults.') + + +def data_quality(rows, protocol): + successful = [r for r in rows if r['case']['split'] == 'train' and r['strict_success'] + and r['initial_anchor_available'] and r['initial_anchor_binding_valid']] + expected = {case['group_id'] for case in protocol['cases']['train']} + covered = {row['case']['group_id'] for row in successful} + counts = {str(goal): sum(r['case']['goal_id'] == goal for r in successful) for goal in (0, 1)} + minimum = protocol['data_quality']['minimum_successful_episodes_per_goal'] + return dict(passed=covered == expected and all(n >= minimum for n in counts.values()), + covered_training_groups=len(covered), required_training_groups=len(expected), + missing_training_groups=sorted(expected - covered), successful_episodes_per_goal=counts, + minimum_successful_episodes_per_goal=minimum, + successful_with_available_bound_initial_camera=len(successful)) + + +def admitted(row): + return row.get('initial_anchor_available') is True and row.get('initial_anchor_binding_valid') is True and row.get('recorded_complete_intervals', 0) > 0 + + +def bootstrap(rows, comparator, acceptance, *, admitted_only=False): + groups = {} + for row in rows: + if row['controller'] in ('student', comparator): + key = (row['controller'], row['case']['goal_id']) + group = groups.setdefault(row['case']['group_id'], {}) + require(key not in group, 'Duplicate paired outcome.') + group[key] = row + require(len(groups) == 32 and all(set(group) == {(c, g) for c in ('student', comparator) for g in (0, 1)} for group in groups.values()), + 'Paired comparison lacks complete starting groups.') + retained = {key: group for key, group in groups.items() if not admitted_only or all(admitted(row) for row in group.values())} + differences = np.asarray([sum(int(group[('student', goal)]['strict_success']) - int(group[(comparator, goal)]['strict_success']) + for goal in (0, 1)) / 2 for key, group in sorted(retained.items())]) + n = len(differences); confidence = acceptance['confidence']; alpha = (1 - confidence) / 2 + if n: + rng = np.random.default_rng(acceptance['bootstrap_seed']) + sample = differences[rng.integers(0, n, size=(acceptance['bootstrap_resamples'], n))].mean(axis=1) + low, high = np.quantile(sample, [alpha, 1 - alpha]).tolist() + else: + low = high = None + return dict(comparison='student-minus-' + comparator, difference=float(differences.mean()) if n else None, + confidence=confidence, lower=low, upper=high, resamples=acceptance['bootstrap_resamples'], + seed=acceptance['bootstrap_seed'], groups=n, excluded_groups=32 - n, admitted_only=admitted_only, + pairing='both goals within exact starting-position group', positive=low is not None and low > 0.) + + +def negative_correct(row): + if row['negative_control'] == 'open_gripper': + return (row['complete'] is True and row['strict_success'] is False and row['score'].get('valid') is True + and row['score'].get('strict_v01', {}).get('trajectory_complete') is True) + return row['status'] == 'refused' and row['recorded_complete_intervals'] == 0 and row['strict_success'] is False + + +def final_gates(rows, negatives, protocol, *, stages_complete): + acceptance = protocol['acceptance']; controllers = {} + for controller in ('student', 'blind', 'fixed', 'teacher'): + selected = [row for row in rows if row['controller'] == controller] + contact = [row['critical_contact_ticks'] for row in selected] + controllers[controller] = dict(scheduled=len(selected), strict_successes=sum(r['strict_success'] for r in selected), + complete=sum(r['complete'] for r in selected), admitted_cases=sum(admitted(r) for r in selected), + initial_refusals=sum(r['status'] in ('refused', 'interrupted') and r['recorded_complete_intervals'] == 0 for r in selected), + critical_contact_episodes=sum(n > 0 for n in contact if n is not None), + critical_contact_ticks=sum(n for n in contact if n is not None), contact_counts_complete=all(n is not None for n in contact)) + complete = stages_complete and len(rows) == 256 and len(negatives) == 16 + comparisons = [bootstrap(rows, other, acceptance) for other in ('fixed', 'blind')] if complete else [] + common = [bootstrap(rows, other, acceptance, admitted_only=True) for other in ('fixed', 'blind')] if complete else [] + student, fixed = controllers['student'], controllers['fixed'] + gates = dict(all_registered_attempts_accounted=complete, + at_least_58_of_64=student['scheduled'] == acceptance['student_attempts'] and student['strict_successes'] >= acceptance['student_successes_minimum'], + positive_paired_comparisons=len(comparisons) == 2 and all(c['positive'] for c in comparisons), + positive_common_admitted_comparisons=len(common) == 2 and all(c['groups'] >= acceptance['minimum_common_admitted_groups'] and c['positive'] for c in common), + no_critical_contact_increase=student['contact_counts_complete'] and fixed['contact_counts_complete'] + and student['critical_contact_episodes'] <= fixed['critical_contact_episodes'], + all_negative_controls_correct=len(negatives) == 16 and all(negative_correct(row) for row in negatives), + no_privileged_features_or_teacher_intervention=all(row['provenance_ok'] is True for row in rows if row['controller'] in ('student', 'blind', 'fixed'))) + return dict(controllers=controllers, comparisons=comparisons, admitted_comparisons=common, gates=gates, + qualified=all(gates.values()), status='complete' if complete else 'incomplete') + + +def _model_hash(root, stage, controller, stages, inputs): + require(stage in ('initial', 'corrected') and stages.get('train-' + stage) is not None, 'Model has no sealed fitting stage.') + variant = 'blind' if controller == 'blind' else 'full' + path = f'models/{stage}/{variant}/model.json' + record = identity(root / path); inputs[path] = record + require(stages['train-' + stage]['models'][variant]['model_sha256'] == record['sha256'], 'Model differs from its sealed training receipt.') + return record['sha256'] + + +def _eligible(rows, *, queried=False): + return [row for row in rows if row.get('complete') is True and row.get('initial_anchor_available') is True + and row.get('initial_anchor_binding_valid') is True and (queried or row.get('strict_success') is True)] + + +def _export_source(root, row, label): + directory = relative(root, row['path']); recording = read(directory / 'recording.json'); steps = recording['steps'] + require(len(steps) == row['recorded_complete_intervals'], 'Training source sample count differs from completed intervals.') + require(recording['provenance']['source_group'] == row['case']['group_id'] + and recording['provenance']['case_id'] == row['case']['id'], 'Training source group differs from its registered scene.') + require(recording['control_mode'] == ('teacher' if label == 'issued' else 'mixed'), 'Training label origin differs from actual control mode.') + return dict(source_group=row['case']['group_id'], case_id=row['case']['id'], + recording_sha256=identity(directory / 'recording.json')['sha256'], + integrity_sha256=identity(directory / 'integrity.json')['sha256'], + calibration_sha256=recording['binding']['calibration_sha256'], tracker_sha256=recording['tracker_sha256'], + label_source=label, sample_count=len(steps), + perception_references_sha256=digest([s['perception']['sha256'] for s in steps]), + teacher_query_references_sha256=digest([s['teacher_query']['sha256'] for s in steps] if label == 'teacher_query' else []), + actual_authority_sha256=digest([s['command']['authority'] for s in steps])) + + +def _training(root, stages, protocol, inputs): + training_groups = {case['group_id'] for case in protocol['cases']['train']} + development_groups = {case['group_id'] for case in protocol['cases']['development']} + source_cache = {} + for stage in ('initial', 'corrected'): + receipt = stages.get('train-' + stage) + if receipt is None: + continue + require((stages.get('demonstrations') or {}).get('data_quality', {}).get('passed') is True, + 'Training proceeded without the demonstration quality gate.') + demos = stages['demonstrations']['rows'] + selected = _eligible([row for row in demos if row['case']['split'] == 'train']) + development = _eligible([row for row in demos if row['case']['split'] == 'development']) + tagged = [(row, 'issued') for row in selected] + if stage == 'corrected': + require(stages.get('corrections') is not None, 'Corrected fitting has no correction collection.') + corrections = _eligible(stages['corrections']['rows'], queried=True) + selected = selected + corrections + tagged += [(row, 'teacher_query') for row in corrections] + require(bool(selected) and bool(development), 'Fitting lacks eligible training or development recordings.') + expected_sources = [] + for row, label in tagged: + key = (row['path'], label) + if key not in source_cache: source_cache[key] = _export_source(root, row, label) + expected_sources.append(source_cache[key]) + require(len({s['recording_sha256'] for s in expected_sources}) == len(expected_sources), 'A training recording is duplicated.') + source_identities = [] + for controller, variant in (('student', 'full'), ('blind', 'blind')): + _model_hash(root, stage, controller, stages, inputs) + model = read(root / f'models/{stage}/{variant}/model.json'); training = model['training'] + receipt_entry = receipt['models'][variant] + expected_path = f'models/{stage}/{variant}/model.json' + require(receipt_entry['path'] == expected_path and receipt_entry['model_bytes'] == inputs[expected_path]['bytes'], 'Model receipt path or byte count differs.') + require(receipt_entry['training_episodes'] == [r['id'] for r in selected] + and receipt_entry['development_episodes'] == [r['id'] for r in development], 'Model fitting or validation episode roster differs from all eligible recordings.') + for name in ('receipt.json', 'started.json'): + path = f'models/{stage}/{variant}/{name}'; inputs[path] = identity(root / path) + require(read(root / f'models/{stage}/{variant}/receipt.json') == receipt_entry, 'Individual model receipt differs from sealed training stage.') + require(read(root / f'models/{stage}/{variant}/started.json') == dict(stage=stage, variant=variant, + protocol_sha256=identity(root / 'protocol.json')['sha256']), 'Model fitting start names a different stage or protocol.') + require(receipt_entry['training'] == training and receipt_entry['dataset_provenance'] == training['provenance'], 'Model metadata differs from its sealed fitting receipt.') + require(model['schema'] == 'dvidia.camera-action-mlp/v1' and model['blind'] is (variant == 'blind') + and model['observation_origin'] == 'rendered_rgbd_and_simulated_encoders' + and model['feature_contract'] == 'camera-encoder-absolute-causal160/v1' + and model['action_representation'] == 'absolute_native_targets' + and model['hidden_units'] == protocol['fitting']['hidden_units'], 'Learned model observation, architecture or action contract differs.') + require(training['stage'] == stage and training['model_source_sha256'] == protocol['source_sha256']['camera_action_model.py'], 'Training source/stage differs.') + for key in ('epochs', 'batch_size', 'learning_rate', 'seed'): + require(training[key] == protocol['fitting'][key], 'Training hyperparameter differs from registration.') + require(set(training['train_groups']) <= training_groups + and set(training['validation']['groups']) <= development_groups, 'Model fitting or validation crosses its split.') + require(training['train_groups'] == list(dict.fromkeys(r['case']['group_id'] for r in selected)) + and training['validation']['groups'] == list(dict.fromkeys(r['case']['group_id'] for r in development)), 'Model training or validation group roster differs.') + require(training['sample_count'] == sum(r['recorded_complete_intervals'] for r in selected) + and training['validation']['sample_count'] == sum(r['recorded_complete_intervals'] for r in development), 'Model training or validation sample count differs.') + provenance = training['provenance'] + require(training['provenance_sha256'] == digest(provenance) + and provenance['source_identity_sha256'] == digest(provenance['episodes']), 'Training provenance identity differs.') + require(provenance['blind'] is (variant == 'blind') + and provenance['export_source_sha256'] == protocol['source_sha256']['camera_action_export.py'] + and provenance['feature_contract_sha256'] == model['feature_contract_sha256'], 'Training export, feature contract or feedback masking identity differs.') + require(provenance['episodes'] == expected_sources, 'Training provenance omits, duplicates or changes eligible source recordings or supervision labels.') + source_identities.append(provenance['source_identity_sha256']) + require(source_identities[0] == source_identities[1], 'Full and blind models used different source demonstrations.') + + +def audit(directory): + root = Path(directory); errors = []; results = []; inputs = {}; stages = {}; stage_results = {}; qualification = None + protocol_hash = None + try: + require(not (root / 'worker.lock').exists(), 'Refusing to audit a running or interrupted study worker.') + protocol = read(root / 'protocol.json'); registration = read(root / 'registration.json') + protocol_hash = identity(root / 'protocol.json')['sha256'] + require(registration['protocol_sha256'] == protocol_hash and registration['source_sha256'] == protocol['source_sha256'], 'Registration identity differs.') + require(identity(root / 'fixed-model.json')['sha256'] == registration['fixed_model_file_sha256'], 'Pinned comparator identity differs.') + for name in ('protocol.json', 'registration.json', 'fixed-model.json'): + inputs[name] = identity(root / name) + for name, expected in protocol['source_sha256'].items(): + require(Path(name).name == name and identity(Path(scoring.__file__).with_name(name))['sha256'] == expected, 'Registered source bytes changed: ' + name) + validate_roster(protocol) + stages = {name: _stage(root, name, protocol_hash, inputs) for name in STAGES} + gap = blocked = False + for name in STAGES: + receipt = stages[name] + if receipt is None: + gap = True + continue + require(not gap and not blocked, 'Study continued after a missing or blocked prerequisite stage.') + blocked = receipt['status'] == 'blocked' + start = read(root / 'stages' / (name + '.started.json')) + require(start['source_sha256'] == protocol['source_sha256'], 'Stage started with different registered runtime bytes.') + frozen = read(root / 'freeze.json') if (root / 'freeze.json').exists() else None + if frozen: + inputs['freeze.json'] = identity(root / 'freeze.json') + require(frozen['protocol_sha256'] == protocol_hash and frozen['source_sha256'] == protocol['source_sha256'], 'Frozen protocol/source binding differs.') + for name, expected in frozen['artifacts'].items(): + record = identity(relative(root, name)); inputs[name] = record + require(record['sha256'] == expected, 'Frozen selected artifact changed.') + for stage in STAGES: + receipt = stages[stage] + if receipt is None or stage.startswith('train-'): + continue + roster = protocol['rosters'][stage]; rows = receipt['rows']; by_id = {row['id']: row for row in rows} + require(len(rows) == len(by_id) == len(roster) and set(by_id) == {a['id'] for a in roster}, 'Stage denominator or attempt IDs differ: ' + stage) + model_stage = ('initial' if stage in ('corrections', 'development-initial') else 'corrected' if stage == 'development-corrected' + else frozen['selected_stage'] if frozen and stage in ('confirmation', 'negative') else None) + expected_bindings = {} + for controller in {a['controller'] for a in roster}: + expected_bindings[controller] = (digest(protocol['source_sha256']) if controller == 'teacher' else + registration['fixed_model_file_sha256'] if controller == 'fixed' else + _model_hash(root, model_stage, controller, stages, inputs)) + require(receipt['model_bindings'] == expected_bindings, 'Stage model identities differ: ' + stage) + if stage in ('confirmation', 'negative'): + require(frozen is not None and receipt['freeze_sha256'] == inputs['freeze.json']['sha256'], 'Evaluation is not bound to the frozen model selection.') + audited = [] + for attempt in roster: + row = by_id[attempt['id']] + try: + require(all(row.get(key) == attempt[key] for key in ('case', 'controller', 'negative_control')), 'Stage row substitutes an unregistered scene or controller.') + path = f'episodes/{stage}/{attempt["id"]}' + if row['status'] in ('unknown', 'unattempted'): + require(row.get('strict_success') is False and row.get('complete') is False, 'Unknown/unattempted outcome claims success.') + require((root / path).exists() is (row['status'] == 'unknown'), 'Unknown/unattempted directory presence differs.') + result = dict(**attempt, status=row['status'], strict_success=False, complete=False, + initial_anchor_available=False, initial_anchor_binding_valid=False, initial_anchor_error_m=None, + recorded_complete_intervals=0, critical_contact_ticks=None, provenance_ok=False, + score=dict(valid=False, success=False), files_verified=0, native_score_recomputed=False) + else: + require(row['path'] == path, 'Sealed episode path differs from its attempt identity.') + require(row['policy_artifact_sha256'] == expected_bindings[attempt['controller']], 'Stage row model identity differs from its registered controller.') + result = audit_episode(root / path, attempt, expected_bindings[attempt['controller']], + tracker_sha256=protocol['source_sha256']['piper_rgbd_tracker.py']) + for key in ('status', 'strict_success', 'complete', 'initial_anchor_available', 'initial_anchor_binding_valid', + 'recorded_complete_intervals', 'critical_contact_ticks', 'provenance_ok', 'receipt_sha256', 'integrity_sha256', 'score'): + require(row.get(key) == result[key], 'Stage summary differs from independent evidence: ' + key) + if result['initial_anchor_error_m'] is None: + require(row.get('initial_anchor_error_m') is None, 'Stage summary invents an anchor error.') + else: + _close(row.get('initial_anchor_error_m'), result['initial_anchor_error_m'], 'Stage initial anchor error differs.') + for name in ('integrity.json', 'rollout.json', 'recording.json'): + if (root / path / name).exists(): + inputs[path + '/' + name] = identity(root / path / name) + result['stage'] = stage; audited.append(result); results.append(result) + except (ValueError, OSError, KeyError, TypeError, IndexError, AssertionError) as exc: + errors.append(dict(stage=stage, attempt=attempt['id'], reason=str(exc))) + stage_results[stage] = audited + if len(audited) != len(roster): + continue + require(receipt['strict_successes'] == sum(r['strict_success'] for r in audited), 'Stage success count differs.') + if stage == 'diagnostic': + threshold = protocol['diagnostic']['maximum_initial_camera_error_m'] + passed = all(r['strict_success'] and r['initial_anchor_available'] and r['initial_anchor_binding_valid'] + and type(r['initial_anchor_error_m']) in (int, float) and math.isfinite(r['initial_anchor_error_m']) + and 0 <= r['initial_anchor_error_m'] <= threshold for r in audited) + require(receipt['diagnostic_passed'] is passed, 'Diagnostic gate differs from reconstructed evidence.') + if stage == 'demonstrations': + require(receipt['data_quality'] == data_quality(audited, protocol), 'Demonstration quality gate differs from reconstructed evidence.') + _training(root, stages, protocol, inputs) + if frozen: + candidates = [] + for order, name in enumerate(('initial', 'corrected')): + rows = [row for row in stage_results['development-' + name] if row['controller'] == 'student'] + require(len(rows) == 16 and all(r['critical_contact_ticks'] is not None for r in rows), 'Development selection lacks complete outcomes.') + model = read(root / f'models/{name}/full/model.json') + candidates.append(dict(stage=name, strict_successes=sum(r['strict_success'] for r in rows), + critical_contact_ticks=sum(r['critical_contact_ticks'] for r in rows), + prediction_loss=model['training']['validation']['action_scale_mse'], order=order)) + winner = min(candidates, key=lambda c: (-c['strict_successes'], c['critical_contact_ticks'], c['prediction_loss'], c['order'])) + require(frozen['candidates'] == candidates and frozen['selected_stage'] == winner['stage'], 'Frozen selection differs from independent development comparison.') + if stages.get('confirmation') and stages.get('negative') and not errors: + qualification = final_gates(stage_results['confirmation'], stage_results['negative'], protocol, + stages_complete=all(stages[name]['status'] == 'complete' for name in ('confirmation', 'negative'))) + report = read(root / 'report.json'); inputs['report.json'] = identity(root / 'report.json') + for key, value in qualification.items(): + require(_equivalent(report[key], value), 'Final benchmark report differs from independent reconstruction: ' + key) + require(report['qualification'] == ('bounded_simulation_skill' if qualification['qualified'] else 'research_candidate') + and report['selected_stage'] == frozen['selected_stage'], 'Report qualification label or selected model differs.') + elif (root / 'report.json').exists(): + report = read(root / 'report.json'); inputs['report.json'] = identity(root / 'report.json') + blocked = [name for name in STAGES if stages[name] and stages[name]['status'] == 'blocked'] + require(report.get('qualified') is False and report.get('qualification') == 'research_candidate' + and report.get('status') == ('blocked' if blocked else 'incomplete'), 'Incomplete study claims qualification or an inconsistent blocker.') + except (ValueError, OSError, KeyError, TypeError, IndexError, AssertionError) as exc: + errors.append(dict(stage=None, attempt=None, reason=str(exc))) + return dict(schema=SCHEMA, passed=not errors, errors=errors, protocol_sha256=protocol_hash, + stages_verified=[name for name in STAGES if name in stage_results], + training_stages_verified=[name for name in STAGES if name.startswith('train-') and stages.get(name)], + attempts_verified=len(results), native_scores_recomputed=sum(r.get('native_score_recomputed') is True for r in results), + files_verified=sum(r['files_verified'] for r in results), results=results, qualification=qualification, + evidence_inventory=inputs, evidence_inventory_sha256=digest(inputs), + audit_source_sha256=identity(Path(__file__))['sha256'], + diagnostic_helper_source_sha256=identity(Path(__file__).with_name('independent_audit.py'))['sha256'], + scorer_source_sha256=identity(Path(scoring.__file__))['sha256'], + physics_executed=False, estimator_executed=False, model_executed=False, + source_authenticated=False, physical_robot_ready=False, + encoder_pose_reference='Capture estimates use the link6 body origin in robot_base. Native scoring uses the authored site at link6 local [0,0,0.12003] m; this audit explicitly reconstructs that fixed offset.', + scope='Independent byte, roster, clock, control-authority and numeric outcome reconstruction for all sealed completed stages. Missing stages remain unrun; no physical qualification.') + + +def _equivalent(left, right): + if isinstance(left, dict) and isinstance(right, dict): + return set(left) == set(right) and all(_equivalent(left[key], right[key]) for key in left) + if isinstance(left, list) and isinstance(right, list): + return len(left) == len(right) and all(_equivalent(a, b) for a, b in zip(left, right)) + if type(left) in (int, float) and type(right) in (int, float): + return math.isfinite(left) and math.isfinite(right) and math.isclose(left, right, abs_tol=1e-12, rel_tol=0) + return type(left) is type(right) and left == right + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('study') + args = parser.parse_args(argv) + result = audit(args.study) + print(canonical(result).decode()) + return 0 if result['passed'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/summarize_results.py b/benchmarks/camera-action-v01/summarize_results.py new file mode 100644 index 0000000..375c13e --- /dev/null +++ b/benchmarks/camera-action-v01/summarize_results.py @@ -0,0 +1,532 @@ +#!/usr/bin/env python3 +"""Read-only posthoc tables from sealed camera/action study receipts. + +This tool never imports a policy, simulator, trainer or the frozen study runtime. +It verifies evidence receipts and model bytes, then summarizes recorded native +scores and reconstructs terminal components numerically. It never reruns physics, +changes qualification gates or rehashes image archives. Missing evidence stays unknown. +""" +from __future__ import annotations + +import argparse +from collections import Counter +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import zipfile + +import numpy as np + +SCHEMA = 'dvidia.camera-action-posthoc-summary/v1' +MAX_JSON_BYTES = 32 * 1024 * 1024 +TIMING_FIELDS = ('estimator_wall_seconds', 'policy_inference_wall_seconds', + 'full_boundary_wall_seconds', 'render_encode_wall_seconds') +COMPONENTS = ( + ('grasp', ('native_recomputed', 'grasp_seen'), True, None), + ('lift', ('native_recomputed', 'lift_seen'), True, ('native_recomputed', 'grasp_seen')), + ('carry', ('native_recomputed', 'carry_complete'), True, ('native_recomputed', 'lift_seen')), + ('retention', ('native_recomputed', 'retention_lost'), False, ('native_recomputed', 'lift_seen')), + ('terminal_composite', ('terminal_only', 'success'), True, ('native_recomputed', 'carry_complete')), + ('release', None, True, ('native_recomputed', 'carry_complete')), + ('settling', None, True, ('native_recomputed', 'carry_complete')), + ('retreat', None, True, ('native_recomputed', 'carry_complete')), +) +NOTES = [ + 'Posthoc description only: no tuning, model inference, new episodes or qualification changes.', + 'All-attempt success denominators include scheduled refusals, interrupted attempts and missing receipts.', + 'Component counts use valid independent native score receipts. Missing or invalid evidence remains unassessed.', + 'Conditional component denominators require the preceding grasp, lift or supported carry predicate; they are not all-attempt success rates.', + 'Release, settling and retreat are separately reconstructed posthoc from sealed native arrays using the frozen scorer predicates. Final predicate and consecutive dwell are diagnostic localization, not new qualification gates.', + 'Latencies pool recorded boundary samples, never averages of episode percentiles. The simulator pauses for computation on a shared host; these are not physical sensor-to-actuator latencies.', + 'Camera acquisition and availability timestamps are synthetic. Depth and physical hardware accuracy remain unqualified.', + 'Renderer-warning counts describe boundaries carrying cumulative warning history, not new warning emissions or independent depth-error trials. An absent warning field is unreported, not evidence of warning-free rendering.', + 'Peak RSS is a process high-water mark including dataset export and any preceding fit, not an isolated model memory measurement.', + 'Evidence verification covers protocol/stage seals, referenced rollout/context receipts, native numeric evidence and model bytes. Image archives and full policy/scoring correctness remain the scope of the separately published independent audit.', +] + + +def digest(raw): + return sha256(raw).hexdigest() + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON key: '+key) + result[key] = value + return result + + +def _read(path, maximum=MAX_JSON_BYTES): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > maximum: + raise ValueError('Expected a bounded regular evidence file: '+path.name) + raw = path.read_bytes() + value = json.loads(raw, object_pairs_hook=_pairs, + parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Nonfinite JSON value'))) + return value, raw + + +def _relative(root, value): + if not isinstance(value, str): + raise ValueError('Evidence path must be relative.') + relative = Path(value) + if relative.is_absolute() or '..' in relative.parts or not relative.parts: + raise ValueError('Evidence path escapes the study.') + path = root/relative + if any(p.is_symlink() for p in (path, *path.parents) if p != root.parent): + raise ValueError('Evidence cannot traverse symlinks.') + if not path.resolve().is_relative_to(root.resolve()): + raise ValueError('Evidence path escapes the study.') + return path + + +def _number(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0. + + +def _integer(value): + return type(value) is int and value >= 0 + + +def _at(value, path): + for key in path or (): + if not isinstance(value, dict): + return None + value = value.get(key) + return value if path else None + + +def _quantile(values, q): + ordered = sorted(values) + if not ordered: + return None + point = (len(ordered)-1)*q + lower = int(point); upper = min(lower+1, len(ordered)-1) + return ordered[lower]+(ordered[upper]-ordered[lower])*(point-lower) + + +def distribution(values): + return dict(samples=len(values), p50=_quantile(values, .5), p95=_quantile(values, .95), + minimum=min(values) if values else None, maximum=max(values) if values else None) + + +def _verify_identity(path, expected): + value, raw = _read(path) + if expected != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Bound evidence identity differs: '+path.name) + return value + + +def native_terminal_components(arrays, metadata, episode): + """Exact terminal predicates from the frozen scorer, recomputed numerically. + + No grasp attachment, action, teacher phase or scored success flag is used. + Full 1 ms evidence is required; missing arrays produce an explicit refusal. + """ + config = episode.get('config', {}) + dt, timeout = config.get('timestep'), config.get('timeout_s') + required = config.get('goal_dwell_s') + retreat = config.get('retreat_clearance_m') + tolerance = config.get('goal_position_tolerance_m') + if (not _number(dt) or dt != .001 or not _number(timeout) or not .1 <= timeout <= 30. + or not math.isclose(timeout/dt, round(timeout/dt), abs_tol=1e-8, rel_tol=0) + or not _number(required) or not .1 <= required <= .6 + or not _number(retreat) or not .08 <= retreat <= .15 + or not _number(tolerance) or not .01 <= tolerance <= .025): + raise ValueError('Invalid native component task config.') + count = round(timeout/dt) + if (metadata.get('schema') != 'dvidia.piper-feedback-native/v1' + or metadata.get('native_timestep_s') != dt or metadata.get('rows') != count + or metadata.get('side_order') != ['left', 'right']): + raise ValueError('Native component metadata/cadence differs.') + widths = dict(t_seconds=1, native_tick=1, grasp_contact_counts=2, + gripper_joint_position_m=2, object_position_m=3, tcp_position_m=3, + object_linear_velocity_m_s=3, object_angular_velocity_rad_s=3, + table_object_contact_count=1) + values = {} + for name, width in widths.items(): + value = np.asarray(arrays.get(name)) + if (value.dtype.kind not in 'fiu' or value.shape != ((count,) if width == 1 else (count, width)) + or not np.isfinite(value).all()): + raise ValueError('Missing or malformed native component field: '+name) + values[name] = value + if (not np.array_equal(values['native_tick'], np.arange(1, count+1)) + or not np.allclose(values['t_seconds'], np.arange(1, count+1)*dt, atol=1e-8, rtol=0)): + raise ValueError('Native component time/tick evidence differs.') + for name in ('grasp_contact_counts', 'table_object_contact_count'): + if np.any(values[name] < 0) or np.any(values[name] != np.floor(values[name])): + raise ValueError('Native component contact count is not a nonnegative integer.') + goal = np.asarray(config.get('target_position_m')) + if goal.shape != (3,) or goal.dtype.kind not in 'fiu' or not np.isfinite(goal).all(): + raise ValueError('Native component destination is malformed.') + jaw = values['gripper_joint_position_m']; obj = values['object_position_m'] + released = (np.all(values['grasp_contact_counts'] == 0, axis=1) + & (jaw[:, 0] > .028) & (jaw[:, 1] < -.028)) + settled = ((np.linalg.norm(values['object_linear_velocity_m_s'], axis=1) < .035) + & (np.linalg.norm(values['object_angular_velocity_rad_s'], axis=1) < .3)) + clear = values['tcp_position_m'][:, 2] >= obj[:, 2]+retreat + supported = values['table_object_contact_count'] > 0 + at_destination = np.linalg.norm(obj-goal, axis=1) <= tolerance + predicates = dict(release=released, settling=settled, retreat=clear, + supported=supported, at_destination=at_destination, + joint_terminal=released & settled & clear & supported & at_destination) + result = {} + for name, mask in predicates.items(): + failures = np.flatnonzero(~mask) + terminal_ticks = count-int(failures[-1])-1 if len(failures) else count + dwell = terminal_ticks*dt + result[name] = dict(final_predicate=bool(mask[-1]), final_consecutive_dwell_seconds=dwell, + required_dwell_seconds=required, dwell_met=bool(dwell+1e-12 >= required)) + return dict(status='available', native_ticks=count, predicates=result, + scope='Posthoc native terminal-component diagnosis; does not replace the frozen task qualification gates.') + + +def _native_components(episode_root, files, receipt): + if receipt.get('score', {}).get('valid') is not True: + return dict(status='unassessed', reason='No valid independent native score receipt.') + required = ('native.npz', 'native_metadata.json', 'native_episode.json') + if any(name not in files for name in required): + return dict(status='unassessed', reason='Required native component evidence is missing.') + metadata = _verify_identity(episode_root/'native_metadata.json', files['native_metadata.json']) + episode = _verify_identity(episode_root/'native_episode.json', files['native_episode.json']) + path = episode_root/'native.npz' + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_JSON_BYTES: + raise ValueError('Native numeric artifact must be a bounded regular file.') + raw = path.read_bytes() + if files['native.npz'] != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Native numeric evidence differs from its bound inventory.') + with zipfile.ZipFile(io.BytesIO(raw)) as archive: + entries = archive.infolist() + if len(entries) > 64 or sum(entry.file_size for entry in entries) > 64*1024*1024: + raise ValueError('Native numeric expansion exceeds the bounded study artifact.') + names = ('t_seconds', 'native_tick', 'grasp_contact_counts', 'gripper_joint_position_m', + 'object_position_m', 'tcp_position_m', 'object_linear_velocity_m_s', + 'object_angular_velocity_rad_s', 'table_object_contact_count') + with np.load(io.BytesIO(raw), allow_pickle=False) as archive: + arrays = {name: archive[name] for name in names if name in archive.files} + result = native_terminal_components(arrays, metadata, episode) + terminal = receipt['score'].get('terminal_only', {}) + reconstructed = result['predicates']['joint_terminal'] + if (type(terminal.get('success')) is not bool or not _number(terminal.get('dwell_s')) + or terminal['success'] != reconstructed['dwell_met'] + or not math.isclose(terminal['dwell_s'], reconstructed['final_consecutive_dwell_seconds'], abs_tol=1e-8, rel_tol=0)): + raise ValueError('Posthoc joint terminal predicate differs from the sealed independent scorer.') + result['source_sha256'] = {name: files[name]['sha256'] for name in required} + return result + + +def _episode(root, row): + """Only read files already bound by the sealed stage and episode inventory.""" + if row.get('path') is None or row.get('receipt_sha256') is None: + if row.get('strict_success') is True or row.get('complete') is True or row.get('score', {}).get('valid') is True: + raise ValueError('Successful or valid outcomes require a bound rollout receipt.') + return None, None + episode_root = _relative(root, row['path']) + receipt, raw = _read(episode_root/'rollout.json') + if digest(raw) != row['receipt_sha256']: + raise ValueError('Rollout receipt differs from sealed stage.') + inventory, inventory_raw = _read(episode_root/'integrity.json') + if digest(inventory_raw) != row.get('integrity_sha256'): + raise ValueError('Episode inventory differs from sealed stage.') + files = inventory.get('files') + if not isinstance(files, dict): + raise ValueError('Episode inventory is missing.') + if files.get('rollout.json') != dict(bytes=len(raw), sha256=digest(raw)): + raise ValueError('Rollout receipt differs from episode inventory.') + if receipt.get('score') != row.get('score') or receipt.get('perception') != row.get('perception'): + raise ValueError('Stage score/perception differs from bound rollout.') + if receipt.get('status') != row.get('status'): + raise ValueError('Stage completion differs from bound rollout.') + complete = receipt.get('status') == 'complete' and receipt.get('recorded_complete_intervals') == 1000 + strict = bool(complete and receipt.get('score', {}).get('success') is True) + if row.get('complete') is not complete or row.get('strict_success') is not strict: + raise ValueError('Stage strict outcome differs from recorded completion and score.') + context = None + if 'context.json' in files: + context = _verify_identity(episode_root/'context.json', files['context.json']) + receipt['_posthoc_components'] = _native_components(episode_root, files, receipt) + return receipt, context + + +def component_counts(rows, expected_count, component): + name, path, success_value, prerequisite = component + known = passed = failed = eligible = conditional_known = conditional_failed = unknown_prerequisite = 0 + for row in rows: + score = row.get('score') or {} + if score.get('valid') is not True: + unknown_prerequisite += prerequisite is not None + continue + value = (_at(row.get('_posthoc_components', {}), ('predicates', name, 'dwell_met')) + if path is None else _at(score, path)) + admitted = True if prerequisite is None else _at(score, prerequisite) + if type(admitted) is not bool: + unknown_prerequisite += 1 + if admitted is True: + eligible += 1 + if type(value) is bool: + known += 1 + passes = value is success_value + passed += passes; failed += not passes + if admitted is True: + conditional_known += 1; conditional_failed += not passes + terminal_values = [_at(row.get('_posthoc_components', {}), ('predicates', name)) for row in rows] if path is None else [] + terminal_values = [value for value in terminal_values if isinstance(value, dict)] + final = [value['final_predicate'] for value in terminal_values if type(value.get('final_predicate')) is bool] + dwell = [value['final_consecutive_dwell_seconds'] for value in terminal_values if _number(value.get('final_consecutive_dwell_seconds'))] + return dict(metric=name, all_attempts=expected_count, assessed=known, passed=passed, failed=failed, + unassessed=expected_count-known, conditional_eligible=eligible, + conditional_assessed=conditional_known, conditional_failed=conditional_failed, + conditional_unassessed=eligible-conditional_known, + prerequisite_unknown=unknown_prerequisite+(expected_count-len(rows) if prerequisite else 0), + condition='all valid native score receipts' if prerequisite is None else '.'.join(prerequisite)+' == true', + availability='posthoc_reconstructed_native_predicate' if path is None else 'recorded_independent_native_predicate', + final_predicate=dict(assessed=len(final), true=sum(final), false=sum(not value for value in final)) if path is None else None, + final_consecutive_dwell_seconds=distribution(dwell) if path is None else None, + required_dwell_seconds=sorted({value['required_dwell_seconds'] for value in terminal_values + if _number(value.get('required_dwell_seconds'))}) if path is None else None) + + +def renderer_warnings(receipts, expected): + messages, affected_episodes, reported_episodes, reported_boundaries = Counter(), 0, 0, 0 + for receipt in receipts: + reported = affected = False + for boundary in receipt.get('timing', {}).get('boundaries', []): + if not isinstance(boundary, dict) or 'renderer_warnings' not in boundary: + continue + warnings = boundary['renderer_warnings'] + if (not isinstance(warnings, list) or len(warnings) > 64 + or any(not isinstance(w, str) or not w or len(w) > 4096 for w in warnings)): + raise ValueError('Malformed recorded renderer warnings.') + reported = True; reported_boundaries += 1 + affected |= bool(warnings) + messages.update(set(warnings)) + reported_episodes += reported; affected_episodes += affected + return dict(episodes_reported=reported_episodes, episodes_unreported=expected-reported_episodes, + episodes_with_warnings=affected_episodes, render_boundaries_reported=reported_boundaries, + warning_boundaries_by_message=dict(sorted(messages.items())), + scope='Saved boundaries carrying cumulative renderer-warning history, not new warning emissions or independent depth-error trials. Missing fields stay unreported.') + + +def _controller(stage_name, controller, roster, rows, episodes): + expected = len(roster) + values = [{**row, '_posthoc_components': (episodes.get(row['id'], (None, None))[0] or {}).get('_posthoc_components', {})} + for row in rows if row['controller'] == controller] + actual = [episodes.get(row['id'], (None, None))[0] for row in values] + bound = [value for value in actual if value is not None] + success = sum(row.get('strict_success') is True for row in values) + complete = sum(row.get('complete') is True for row in values) + native_valid = sum(row.get('score', {}).get('valid') is True for row in values) + perception_rows = [row['perception'] for row in values if isinstance(row.get('perception'), dict) + and all(_integer(row['perception'].get(name)) for name in ('decisions', 'available', 'missing'))] + if any(row['available']+row['missing'] != row['decisions'] for row in perception_rows): + raise ValueError('Perception counts disagree.') + decisions = sum(row['decisions'] for row in perception_rows) + available = sum(row['available'] for row in perception_rows) + missing_spans = [row['longest_missing_seconds'] for row in perception_rows if _number(row.get('longest_missing_seconds'))] + latencies = {} + for metric in TIMING_FIELDS: + samples = [boundary[metric] for value in bound for boundary in value.get('timing', {}).get('boundaries', []) + if isinstance(boundary, dict) and _number(boundary.get(metric))] + applicable = not (metric == 'policy_inference_wall_seconds' and controller == 'teacher') + result = distribution(samples if applicable else []) + result.update(applicable=applicable, episode_receipts=len(bound), unit='seconds') + if metric == 'full_boundary_wall_seconds': + result['wall_samples_over_20ms'] = sum(sample > .02 for sample in samples) + result['deadline_claim'] = False + latencies[metric] = result + durations = [value['timing']['wall_seconds'] for value in bound + if _number(value.get('timing', {}).get('wall_seconds'))] + critical_counts = [_at(row.get('score'), ('native_recomputed', 'unexpected_contact_native_ticks')) + for row in values if row.get('score', {}).get('valid') is True] + critical_counts = [value for value in critical_counts if _integer(value)] + return dict(stage=stage_name, controller=controller, scheduled=expected, + retained_rows=len(values), missing_rows=expected-len(values), complete=complete, + native_assessable=native_valid, strict_successes=success, + strict_success_rate_all_attempts=success/expected if expected else None, + strict_success_rate_complete=success/complete if complete else None, + not_confirmed_successful=expected-success, + critical_contacts=dict(assessed_attempts=len(critical_counts), + unassessed_attempts=expected-len(critical_counts), + attempts_with_contacts=sum(value > 0 for value in critical_counts), + affected_fraction_assessed=sum(value > 0 for value in critical_counts)/len(critical_counts) if critical_counts else None, + native_ticks=sum(critical_counts), native_timestep_seconds=.001, + scope='Independent native unexpected-contact ticks; repeated contact is counted at each 1 ms physics tick.'), + statuses=dict(Counter(row.get('status', 'unknown') for row in values)), + components={item[0]: component_counts(values, expected, item) for item in COMPONENTS}, + terminal_components_by_attempt=[dict(id=row['id'], **row['_posthoc_components']) for row in values], + perception=dict(episodes_reported=len(perception_rows), episodes_unknown=expected-len(perception_rows), + available_decisions=available, assessed_decisions=decisions, + available_fraction=available/decisions if decisions else None, + maximum_missing_seconds=max(missing_spans) if missing_spans else None), + renderer_warnings=renderer_warnings(bound, expected), + timing=latencies, episode_wall_seconds=distribution(durations)) + + +def summarize(study): + root = Path(study).resolve() + registration, _ = _read(root/'registration.json') + protocol, protocol_raw = _read(root/'protocol.json') + protocol_hash = digest(protocol_raw) + if registration.get('protocol_sha256') != protocol_hash: + raise ValueError('Protocol differs from registration.') + rosters = protocol.get('rosters') + if not isinstance(rosters, dict): + raise ValueError('Registered attempt rosters are required.') + result = dict(schema=SCHEMA, protocol_sha256=protocol_hash, posthoc=True, + physical_robot_ready=False, evidence_scope=NOTES[-1], notes=list(NOTES), + controllers=[], stages=[], training=[], hardware=[], source_receipts={}, qualification=None) + hardware = {} + stage_names = list(dict.fromkeys([*rosters, 'train-initial', 'train-corrected'])) + for name in stage_names: + path = root/'stages'/(name+'.json') + if not path.exists(): + result['stages'].append(dict(stage=name, status='pending_receipt', scheduled=len(rosters.get(name, [])))) + continue + stage, raw = _read(path) + seal, _ = _read(path.with_name(name+'.integrity.json')) + if seal != dict(sha256=digest(raw), protocol_sha256=protocol_hash): + raise ValueError('Stage seal differs: '+name) + if stage.get('stage') != name or stage.get('protocol_sha256') != protocol_hash: + raise ValueError('Stage protocol/name binding differs.') + result['source_receipts'][str(path.relative_to(root))] = digest(raw) + result['stages'].append(dict(stage=name, status=stage.get('status'), reason=stage.get('reason'), + scheduled=stage.get('scheduled'), strict_successes=stage.get('strict_successes'))) + if name.startswith('train-'): + for variant, record in stage.get('models', {}).items(): + model_path = _relative(root, record['path']) + _, model_raw = _read(model_path) + if digest(model_raw) != record['model_sha256'] or len(model_raw) != record['model_bytes']: + raise ValueError('Training model bytes differ from its receipt.') + result['training'].append(dict(stage=name, variant=variant, model_bytes=len(model_raw), + model_sha256=digest(model_raw), training_wall_seconds=record.get('training_wall_seconds'), + shared_export_wall_seconds=record.get('shared_export_wall_seconds'), + process_peak_rss_bytes=record.get('process_peak_rss_bytes'), memory_scope=record.get('memory_scope'), + training_episodes=len(record.get('training_episodes', [])))) + continue + roster = rosters.get(name, []) + expected = {row['id']: row for row in roster} + if len(expected) != len(roster) or stage.get('scheduled') != len(roster): + raise ValueError('Registered or sealed attempt counts disagree.') + rows = stage.get('rows', []) + if not isinstance(rows, list) or len({row['id'] for row in rows}) != len(rows): + raise ValueError('Stage attempt IDs are duplicated or invalid.') + if {row['id'] for row in rows} != set(expected): + raise ValueError('Stage attempt ID set differs from the registered roster; unattempted rows must be retained.') + episodes = {} + for row in rows: + registered = expected.get(row['id']) + if registered is None or any(row.get(key) != registered.get(key) + for key in ('controller', 'case', 'negative_control')): + raise ValueError('Attempt differs from the registered roster.') + receipt, context = _episode(root, row) + episodes[row['id']] = receipt, context + if context is not None: + runtime = context.get('metadata', {}).get('runtime') + if isinstance(runtime, dict): + key = json.dumps(runtime, sort_keys=True, allow_nan=False) + hardware.setdefault(key, dict(recorded_runtime=runtime, episode_receipts=0, + cpu_model=None, gpu_model=None, physical_sensor_model=None, + unreported_fields='CPU/GPU device identity and physical sensor identity were not captured in the runtime receipt.'))['episode_receipts'] += 1 + if stage.get('strict_successes') != sum(row.get('strict_success') is True for row in rows): + raise ValueError('Stage strict-success total disagrees with its rows.') + for controller in dict.fromkeys(row['controller'] for row in roster): + registered = [row for row in roster if row['controller'] == controller] + result['controllers'].append(_controller(name, controller, registered, rows, episodes)) + result['hardware'] = list(hardware.values()) + report_path = root/'report.json' + if report_path.exists(): + report, raw = _read(report_path) + if report.get('protocol_sha256') != protocol_hash: + raise ValueError('Study report protocol binding differs.') + result['qualification'] = dict(status=report.get('status'), qualified=report.get('qualified'), + qualification=report.get('qualification'), report_sha256=digest(raw), + source='existing study report; not newly awarded by this posthoc tool') + return result + + +def _cell(value): + return str(value).replace('|', '\\|').replace('\n', ' ') if value is not None else 'Not reported' + + +def _percent(value): + return 'Not reported' if value is None else f'{value*100:.1f}%' + + +def markdown(summary): + lines = ['# Camera-to-action study: posthoc evidence tables', '', + 'These tables summarize sealed evidence without running or tuning policies.', '', + '| Stage / controller | Strict success / scheduled | Complete | Native assessable | All-attempt success |', + '|---|---:|---:|---:|---:|'] + for row in summary['controllers']: + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {row['strict_successes']} / {row['scheduled']} | {row['complete']} | {row['native_assessable']} | {_percent(row['strict_success_rate_all_attempts'])} |") + lines += ['', 'Component failures are conditional on the stated prerequisite. Unknown counts are retained.', '', + '| Stage / controller | Component | Prerequisite | Failed / conditional assessed | Eligible but unassessed | All-attempt unassessed |', + '|---|---|---|---:|---:|---:|'] + for row in summary['controllers']: + for name, component in row['components'].items(): + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {_cell(name)} | {_cell(component['condition'])} | {component['conditional_failed']} / {component['conditional_assessed']} | {component['conditional_unassessed']} | {component['unassessed']} |") + lines += ['', '| Stage / controller | Contact-affected / native assessed | Unassessed | Critical 1 ms ticks |', + '|---|---:|---:|---:|'] + for row in summary['controllers']: + contact=row['critical_contacts'] + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {contact['attempts_with_contacts']} / {contact['assessed_attempts']} | {contact['unassessed_attempts']} | {contact['native_ticks']} |") + lines += ['', 'Posthoc terminal predicates are measured separately from the frozen task gate.', '', + '| Stage / controller | Predicate | False at final tick / assessed | Final dwell min / median | Required dwell |', + '|---|---|---:|---:|---:|'] + for row in summary['controllers']: + for name in ('release', 'settling', 'retreat'): + component=row['components'][name];final=component['final_predicate'];dwell=component['final_consecutive_dwell_seconds'] + observed='Not reported' if dwell['p50'] is None else f"{dwell['minimum']:.3f} / {dwell['p50']:.3f} s" + required=', '.join(f'{value:.3f} s' for value in component['required_dwell_seconds']) or 'Not reported' + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {name} | {final['false']} / {final['assessed']} | {observed} | {required} |") + lines += ['', '| Stage / controller | Visual decisions available | Maximum missing span | Inference p95 | Full boundary p95 |', + '|---|---:|---:|---:|---:|'] + for row in summary['controllers']: + perception=row['perception']; timing=row['timing'] + def ms(value):return 'Not reported' if value is None else f'{value*1000:.3f} ms' + span='Not reported' if perception['maximum_missing_seconds'] is None else f"{perception['maximum_missing_seconds']:.3f} s" + lines.append(f"| {_cell(row['stage'])} / {_cell(row['controller'])} | {perception['available_decisions']} / {perception['assessed_decisions']} | {span} | {ms(timing['policy_inference_wall_seconds']['p95'])} | {ms(timing['full_boundary_wall_seconds']['p95'])} |") + lines += ['', '| Fit | Model bytes | Fit/save seconds | Shared export seconds | Process peak RSS bytes |', + '|---|---:|---:|---:|---:|'] + for row in summary['training']: + lines.append('| '+' | '.join(_cell(v) for v in (row['stage']+' / '+row['variant'],row['model_bytes'], + row['training_wall_seconds'],row['shared_export_wall_seconds'],row['process_peak_rss_bytes']))+' |') + lines += ['', 'Recorded renderer warning history (boundaries carrying a message, not new emissions or independent accuracy trials):', ''] + for row in summary['controllers']: + warnings = row['renderer_warnings'] + lines.append('- '+_cell(row['stage']+' / '+row['controller'])+': '+ + f"{warnings['episodes_with_warnings']} affected / {warnings['episodes_reported']} reported episodes; " + f"{warnings['episodes_unreported']} unreported. "+_cell(json.dumps(warnings['warning_boundaries_by_message'], sort_keys=True))) + lines += ['', 'Recorded execution environments:', ''] + if summary['hardware']: + for row in summary['hardware']: + lines.append('- '+_cell(json.dumps(row['recorded_runtime'], sort_keys=True))+ + f"; {row['episode_receipts']} episode receipts. CPU/GPU device identities were not recorded.") + else: + lines.append('- No bound runtime receipt is available.') + lines += ['', *('- '+note for note in summary['notes']), ''] + return '\n'.join(lines) + + +def main(argv=None): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study',required=True);parser.add_argument('--output',required=True) + args=parser.parse_args(argv) + root=Path(args.study).resolve();output=Path(args.output).absolute() + if output.exists() or output.is_symlink() or output.resolve().is_relative_to(root): + parser.error('Choose a fresh output directory outside the study.') + summary=summarize(root) + output.mkdir(parents=True,exist_ok=False) + (output/'summary.json').write_text(json.dumps(summary,indent=2,sort_keys=True,allow_nan=False)+'\n') + (output/'summary.md').write_text(markdown(summary)) + print(json.dumps(dict(status='complete',output=str(output),controller_rows=len(summary['controllers']), + physical_robot_ready=False,posthoc=True))) + return 0 + + +if __name__=='__main__': + raise SystemExit(main()) diff --git a/benchmarks/camera-action-v01/test_development_failure_summary.py b/benchmarks/camera-action-v01/test_development_failure_summary.py new file mode 100644 index 0000000..a6831c8 --- /dev/null +++ b/benchmarks/camera-action-v01/test_development_failure_summary.py @@ -0,0 +1,224 @@ +"""Pure saved-evidence counterexamples: no physics or learned inference.""" +from copy import deepcopy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +spec = importlib.util.spec_from_file_location('development_failures', + Path(__file__).with_name('development_failure_summary.py')) +module = importlib.util.module_from_spec(spec); spec.loader.exec_module(module) + + +def trajectory(closure=60): + steps=[] + for i in range(1000): + action=dict(joint_targets_rad=[0.]*6,gripper_position_m=.01 if closure is not None and i>=closure else .035) + steps.append(dict(before=dict(timestamp_seconds=i*.02), after=dict(timestamp_seconds=(i+1)*.02), + command=dict(issued_at_seconds=i*.02,issued=action),applied=dict(interval_start_seconds=i*.02, + interval_end_seconds=(i+1)*.02,rate_limited=False,ramped_targets=deepcopy(action)))) + arrays=dict(arm_actuator_controls_rad=np.zeros((20000,6)),normal_rate_limited_jaw_target_m=np.repeat( + [s['command']['issued']['gripper_position_m'] for s in steps],20), + tcp_position_m=np.tile([.133,.0,.15],(20000,1)),object_position_m=np.zeros((20000,3)), + gripper_joint_position_m=np.tile([.0346,-.0346],(20000,1)), + unexpected_contact_count=np.zeros(20000),grasp_contact_counts=np.zeros((20000,2))) + score=dict(unexpected_contact_native_ticks=0,grasp_seen=False,lift_seen=False,max_object_lift_m=.002) + return steps,arrays,score + + +def measured(**changes): + value=dict(control_intervals=1000,rate_limited_intervals=977,critical_contact_ticks=100, + critical_contact_episode=True,bilateral_contact_ticks=0,strict_grasp_seen=False,lift_seen=False, + maximum_object_lift_m=.002,no_closure_command=False,closure_command_beyond_20mm_xy=True, + first_closure_command=dict(time_seconds=1.2,geometry_available=True,tcp_object_xy_distance_m=.133)) + value.update(changes);return value + + +def row(metrics=None,status='complete',success=False): + return dict(id='case0',metrics=metrics,status=status,strict_success=success) + + +class MeasurementTests(unittest.TestCase): + def test_closure_uses_preceding_native_tick_and_single_finger_command(self): + steps,arrays,score=trajectory();arrays['tcp_position_m'][1199]=[.133,.0,.15] + arrays['tcp_position_m'][1200]=[.001,.0,.0] + result=module.measure(steps,arrays,score) + event=result['first_closure_command'] + self.assertEqual(event['time_seconds'],1.2) + self.assertAlmostEqual(event['tcp_object_xy_distance_m'],.133) + self.assertEqual(event['issued_single_finger_target_m'],.01) + self.assertEqual(event['actual_finger_positions_m'],[.0346,-.0346]) + self.assertTrue(result['closure_command_beyond_20mm_xy']) + + def test_time_zero_geometry_and_no_closure_are_distinct(self): + result=module.measure(*trajectory(0)) + self.assertIsNone(result['closure_command_beyond_20mm_xy']) + self.assertFalse(result['first_closure_command']['geometry_available']) + self.assertFalse(result['no_closure_command']) + result=module.measure(*trajectory(None)) + self.assertTrue(result['no_closure_command']) + self.assertIsNone(result['first_closure_command']) + self.assertFalse(result['closure_command_beyond_20mm_xy']) + + def test_clock_flags_and_native_binding_tamper_refused(self): + for change in ('decision','end','flag','joint','jaw','critical'): + with self.subTest(change=change): + steps,arrays,score=trajectory() + if change=='decision':steps[60]['before']['timestamp_seconds']+=.001 + if change=='end':steps[60]['applied']['interval_end_seconds']+=.001 + if change=='flag':steps[60]['applied']['rate_limited']=True + if change=='joint':arrays['arm_actuator_controls_rad'][1219,0]=.5 + if change=='jaw':arrays['normal_rate_limited_jaw_target_m'][1219]=.02 + if change=='critical':arrays['unexpected_contact_count'][10]=1 + with self.assertRaises((ValueError,AssertionError)):module.measure(steps,arrays,score) + + +class DenominatorTests(unittest.TestCase): + def test_failures_refusals_and_conditional_grasp_denominators(self): + rows=[row(measured()),row(measured(strict_grasp_seen=True,bilateral_contact_ticks=40)), + row(measured(strict_grasp_seen=True,lift_seen=True),success=True),row(status='refused'),row(status='unknown')] + value=module.aggregate(rows) + self.assertEqual(value['scheduled'],5) + self.assertEqual(value['strict_successes'],1) + self.assertEqual(value['no_strict_grasp'],dict(events=1,known=3,missing=2)) + self.assertEqual(value['grasp_without_lift'],dict(events=1,known=2,not_applicable=1,missing=2)) + self.assertEqual(value['control_intervals_observed'],3000) + self.assertEqual(value['rate_limited_intervals_observed'],2931) + + def test_missing_pair_never_becomes_zero_difference(self): + a=row(measured());b=row(status='refused');c=module.compare(a,b) + self.assertFalse(c['both_complete_metrics']) + self.assertIsNone(c['closure_time_difference_seconds']) + a['teacher_comparison']=c + value=module.aggregate([a])['matched_teacher'] + self.assertEqual(value['closure_earlier'],dict(events=0,known=0,missing=1)) + self.assertIsNone(value['rate_limited_interval_difference']['median']) + + def test_no_closure_pair_is_not_claimed_late_or_early(self): + a=row(measured(first_closure_command=None,no_closure_command=True));b=row(measured()) + value=module.compare(a,b) + self.assertTrue(value['both_complete_metrics']) + self.assertIsNone(value['closes_before_teacher']) + self.assertEqual(value['rate_limited_interval_difference'],0) + + +class RegistrationTests(unittest.TestCase): + def test_summary_waits_for_both_seals_before_reading_evidence(self): + with tempfile.TemporaryDirectory() as tmp,patch.object(module,'load') as loader: + root=Path(tmp);(root/'stages').mkdir() + (root/'stages/development-initial.integrity.json').write_text('{}') + with self.assertRaisesRegex(ValueError,'Both development'):module.summarize(root) + loader.assert_not_called() + + def test_both_exact_cohorts_required_without_dropped_or_duplicate_cases(self): + cases=[dict(id=f'case{i}',seed=i) for i in range(16)] + protocol=dict(cases=dict(development=cases)) + stages={name:dict(rows=[dict(case=case,controller=c) for case in cases for c in ('student','blind')]) + for name in module.DEVELOPMENT} + self.assertEqual(len(module.development_cases(protocol,stages)),16) + for change in ('drop','duplicate','case','controller'): + value=deepcopy(stages);rows=value['development-corrected']['rows'] + if change=='drop':rows.pop() + if change=='duplicate':rows[1]=deepcopy(rows[0]) + if change=='case':rows[0]['case']['seed']=999 + if change=='controller':rows[0]['controller']='teacher' + with self.subTest(change=change),self.assertRaisesRegex(ValueError,'coverage'): + module.development_cases(protocol,value) + + def test_unknown_prefix_and_unattempted_keep_denominator_without_reads(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);attempt=dict(id='a',case={},controller='blind',status='unknown',complete=False,strict_success=False) + (root/'episodes/development-corrected/a').mkdir(parents=True) + with patch.object(module,'load') as loader: + value=module.episode(root,'development-corrected',attempt,'model','tracker',{}) + self.assertIsNone(value['metrics']);loader.assert_not_called() + attempt=dict(attempt,id='b',status='unattempted') + self.assertIsNone(module.episode(root,'development-corrected',attempt,'model','tracker',{})['metrics']) + attempt['strict_success']=True + with self.assertRaisesRegex(ValueError,'Unknown/unattempted'):module.episode(root,'development-corrected',attempt,'model','tracker',{}) + + def test_stage_seal_registration_roster_and_order(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);(root/'stages').mkdir();name='development-initial';ph='a'*64 + attempts=[dict(id=f'a{i}',case=dict(id=f'c{i}'),controller='student',negative_control=None) for i in range(2)] + protocol=dict(source_sha256={'test.py':'b'*64},rosters={name:attempts}) + receipt=dict(schema='dvidia.camera-action-study/v1/stage',stage=name,protocol_sha256=ph,status='complete', + rows=deepcopy(attempts),scheduled=2,model_bindings={'student':'c'*64}) + started=dict(schema='dvidia.camera-action-study/v1/stage-start',stage=name,protocol_sha256=ph, + source_sha256=protocol['source_sha256'],model_bindings=receipt['model_bindings']) + def save(): + path=root/'stages'/f'{name}.json';path.write_text(json.dumps(receipt)) + (path.with_name(name+'.integrity.json')).write_text(json.dumps(dict(sha256=module.audit.identity(path)['sha256'],protocol_sha256=ph))) + (path.with_name(name+'.started.json')).write_text(json.dumps(started)) + save();self.assertEqual(module.stage(root,name,protocol,ph,{})['scheduled'],2) + receipt['rows'].reverse();save() + with self.assertRaisesRegex(ValueError,'roster'):module.stage(root,name,protocol,ph,{}) + receipt['rows'].reverse();started['source_sha256']={};save() + with self.assertRaisesRegex(ValueError,'registration'):module.stage(root,name,protocol,ph,{}) + + def test_training_receipt_uses_distinct_schema_and_binds_start(self): + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp);(root/'stages').mkdir();name='train-corrected';ph='a'*64 + protocol=dict(source_sha256={'test.py':'b'*64}) + receipt=dict(schema='dvidia.camera-action-study/v1/training',stage=name,protocol_sha256=ph,status='complete',models={}) + started=dict(schema='dvidia.camera-action-study/v1/stage-start',stage=name,protocol_sha256=ph, + source_sha256=protocol['source_sha256'],model_bindings=None) + def save(): + path=root/'stages'/f'{name}.json';path.write_text(json.dumps(receipt)) + path.with_name(name+'.integrity.json').write_text(json.dumps(dict(sha256=module.audit.identity(path)['sha256'],protocol_sha256=ph))) + path.with_name(name+'.started.json').write_text(json.dumps(started)) + save();self.assertEqual(module.stage(root,name,protocol,ph,{})['models'],{}) + receipt['schema']='dvidia.camera-action-study/v1/stage';save() + with self.assertRaisesRegex(ValueError,'registration'):module.stage(root,name,protocol,ph,{}) + receipt['schema']='dvidia.camera-action-study/v1/training';started['schema']='wrong';save() + with self.assertRaisesRegex(ValueError,'registration'):module.stage(root,name,protocol,ph,{}) + + +class RefusalEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup);self.root=Path(self.temp.name).resolve() + self.name='development-corrected';self.model='b'*64;self.tracker='c'*64 + self.attempt=dict(id='corrected-case0-blind',case=dict(id='case0',group_id='group0',goal_position_m=[.35,.08,.08],seed=7), + controller='blind',negative_control=None,status='refused',complete=False,strict_success=False,recorded_complete_intervals=0, + policy_artifact_sha256=self.model,score=dict(valid=False,success=False,reason='native_evidence_unavailable')) + self.relative='episodes/'+self.name+'/'+self.attempt['id'];self.directory=self.root/self.relative;self.directory.mkdir(parents=True) + self.attempt['path']=self.relative;case={**self.attempt['case'],'policy_artifact_sha256':self.model} + self.receipt=dict(schema='dvidia.camera-action-rollout/v2',case=case,mode='student',negative_control=None, + status='refused',strict_success=False,score=self.attempt['score'],tracker_sha256=self.tracker,teacher_queries=0, + student_calls=0,recorded_complete_intervals=0,no_privileged_features_or_teacher_intervention=True) + self.started=dict(schema=self.receipt['schema'],case=case,mode='student',negative_control=None, + config=dict(duration_s=20.,seed=7,width=320,height=240,camera_every_controls=2)) + self.save() + + def save(self): + (self.directory/'started.json').write_text(json.dumps(self.started)) + (self.directory/'rollout.json').write_text(json.dumps(self.receipt)) + files={name:module.audit.identity(self.directory/name) for name in ('started.json','rollout.json')} + path=self.directory/'integrity.json';path.write_text(json.dumps(dict(schema='dvidia.camera-action-rollout/v2/integrity',files=files))) + self.attempt.update(integrity_sha256=module.audit.identity(path)['sha256'],receipt_sha256=files['rollout.json']['sha256']) + + def check(self):return module.episode(self.root,self.name,self.attempt,self.model,self.tracker,{}) + + def test_blind_is_student_control_and_refusal_is_missing_not_zero(self): + result=self.check();self.assertEqual(result['controller'],'blind') + self.assertIsNone(result['metrics']);self.assertFalse(result['native_score_valid']);self.assertEqual(result['teacher_queries'],0) + + def test_teacher_intervention_and_tracker_drift_refused(self): + self.receipt['teacher_queries']=1;self.save() + with self.assertRaisesRegex(ValueError,'teacher query'):self.check() + self.receipt['teacher_queries']=0;self.receipt['tracker_sha256']='d'*64;self.save() + with self.assertRaisesRegex(ValueError,'tracker'):self.check() + + def test_changed_bound_bytes_and_model_identity_refused(self): + self.receipt['student_calls']=1 + (self.directory/'rollout.json').write_text(json.dumps(self.receipt)) + with self.assertRaisesRegex(ValueError,'Sealed evidence changed'):self.check() + self.save();self.attempt['policy_artifact_sha256']='d'*64 + with self.assertRaisesRegex(ValueError,'model binding'):self.check() + + +if __name__=='__main__':unittest.main() diff --git a/benchmarks/camera-action-v01/test_diagnose_first_student.py b/benchmarks/camera-action-v01/test_diagnose_first_student.py new file mode 100644 index 0000000..c75e5e6 --- /dev/null +++ b/benchmarks/camera-action-v01/test_diagnose_first_student.py @@ -0,0 +1,135 @@ +"""Pure counterexamples for local diagnosis wording and stage membership.""" +from copy import deepcopy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('first_student_diagnosis', Path(__file__).with_name('diagnose_first_student.py')) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +def result(*, success=False, contacts=0, limited=977, missing=1.56): + return dict(strict_success=success, bilateral_object_contact_ticks=contacts, + complete_intervals=1000, rate_limited_intervals=limited, + native_score=dict(unexpected_contact_native_ticks=1427 if not success else 0), + first_missing_estimate_seconds=missing, + closure_events=dict(jaw_target_first_below_20mm=dict(time_seconds=1.2, tcp_object_xy_distance_m=.133))) + + +class InterpretationTests(unittest.TestCase): + def test_control_clock_rejects_wrong_event_alignment(self): + steps = [dict(before=dict(timestamp_seconds=i * .02)) for i in range(1000)] + self.assertEqual(module.control_clock(steps)[60], 1.2) + for index, value in ((60, 1.201), (0, .02), (999, float('nan'))): + changed = deepcopy(steps); changed[index]['before']['timestamp_seconds'] = value + with self.assertRaisesRegex(ValueError, '50 Hz clock'): module.control_clock(changed) + with self.assertRaisesRegex(ValueError, '50 Hz clock'): module.control_clock(steps[:-1]) + + def test_measured_failure_matches_pattern(self): + actual = module.interpretation(result(), result(success=True, contacts=100, limited=75), + dict(first_more_than_10mm_seconds=.409)) + self.assertTrue(all(actual['hypothesis_predicates'].values())) + self.assertIn('consistent with covariate shift', actual['hypothesis']) + self.assertTrue(any('commanded' in text for text in actual['observed'])) + + def test_success_and_late_divergence_cannot_repeat_failure_conclusions(self): + actual = module.interpretation(result(success=True, contacts=300, limited=0, missing=.1), result(), + dict(first_more_than_10mm_seconds=2.)) + text = ' '.join(actual['observed']) + self.assertNotIn('No bilateral', text) + self.assertNotIn('before any missing', text) + self.assertIn('acquisition failure alone cannot explain', text) + self.assertFalse(actual['hypothesis_predicates']['student_failed']) + self.assertFalse(actual['hypothesis_predicates']['divergence_precedes_missing']) + self.assertIn('no mechanism is inferred', actual['hypothesis']) + + def test_no_closure_and_no_divergence_are_reported(self): + student = result(); student['closure_events']['jaw_target_first_below_20mm'] = None + actual = module.interpretation(student, result(), dict(first_more_than_10mm_seconds=None)) + text = ' '.join(actual['observed']) + self.assertIn('never commanded', text) + self.assertIn('never separated', text) + self.assertIn('no mechanism is inferred', actual['hypothesis']) + + +class StageBindingTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name); (self.root / 'stages').mkdir() + self.name = 'development-initial'; self.ph = 'a' * 64; self.mh = 'b' * 64 + self.attempt = dict(id='initial-development-00-goal0-student', controller='student', + case=dict(id='development-00-goal0', split='development'), negative_control=None) + self.relative = 'episodes/' + self.name + '/' + self.attempt['id'] + self.episode = self.root / self.relative; self.episode.mkdir(parents=True) + self.rollout = dict(case={**self.attempt['case'], 'policy_artifact_sha256': self.mh}, + score=dict(success=False), negative_control=None) + self.write(self.episode / 'rollout.json', self.rollout) + self.write(self.episode / 'integrity.json', dict(files={})) + self.row = dict(**self.attempt, path=self.relative, status='complete', complete=True, + recorded_complete_intervals=1000, policy_artifact_sha256=self.mh, + integrity_sha256=module.identity((self.episode / 'integrity.json').read_bytes())['sha256'], + receipt_sha256=module.identity((self.episode / 'rollout.json').read_bytes())['sha256'], + strict_success=False, score=self.rollout['score']) + self.protocol = dict(source_sha256={'source.py': 'c' * 64}, rosters={self.name: [self.attempt]}) + self.receipt = dict(schema='dvidia.camera-action-study/v1/stage', stage=self.name, status='complete', + protocol_sha256=self.ph, model_bindings={'student': self.mh}, rows=[self.row], scheduled=1) + self.started = dict(stage=self.name, protocol_sha256=self.ph, + source_sha256=self.protocol['source_sha256'], model_bindings=self.receipt['model_bindings']) + self.seal() + + def write(self, path, value): + path.write_text(json.dumps(value)) + + def seal(self): + path = self.root / 'stages' / (self.name + '.json') + self.write(path, self.receipt) + self.write(path.with_name(self.name + '.integrity.json'), + dict(sha256=module.identity(path.read_bytes())['sha256'], protocol_sha256=self.ph)) + self.write(path.with_name(self.name + '.started.json'), self.started) + + def check(self): + return module.stage_binding(self.root, self.name, self.attempt, self.protocol, self.ph) + + def test_binds_stage_start_seal_row_and_episode(self): + row, sources = self.check() + self.assertEqual(row, self.row); self.assertEqual(len(sources), 3) + + def test_changed_episode_manifest_refused(self): + self.write(self.episode / 'integrity.json', dict(files={'changed': {}})) + with self.assertRaisesRegex(ValueError, 'sealed stage row'): self.check() + + def test_changed_episode_receipt_refused(self): + self.write(self.episode / 'rollout.json', dict(self.rollout, extra=True)) + with self.assertRaisesRegex(ValueError, 'sealed stage row'): self.check() + + def test_wrong_row_membership_duplicate_and_unsealed_stage_refused(self): + original = deepcopy(self.receipt) + for mutation in ('case', 'duplicate', 'model', 'status'): + with self.subTest(mutation=mutation): + self.receipt = deepcopy(original) + if mutation == 'case': self.receipt['rows'][0]['case']['id'] = 'other' + if mutation == 'duplicate': self.receipt['rows'].append(deepcopy(self.receipt['rows'][0])) + if mutation == 'model': self.receipt['rows'][0]['policy_artifact_sha256'] = 'd' * 64 + if mutation == 'status': self.receipt['status'] = 'running' + self.seal() + with self.assertRaises(ValueError): self.check() + + def test_stage_score_cannot_disagree_with_bound_rollout(self): + self.receipt['rows'][0]['score'] = dict(success=True) + self.seal() + with self.assertRaisesRegex(ValueError, 'outcome differs'): self.check() + + def test_broken_stage_seal_and_source_freeze_refused(self): + path = self.root / 'stages' / (self.name + '.json') + path.write_text(path.read_text() + ' ') + with self.assertRaisesRegex(ValueError, 'not sealed'): self.check() + self.started['source_sha256'] = {} + self.seal() + with self.assertRaisesRegex(ValueError, 'started bindings'): self.check() + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/camera-action-v01-report-draft.md b/docs/camera-action-v01-report-draft.md new file mode 100644 index 0000000..e2ded6d --- /dev/null +++ b/docs/camera-action-v01-report-draft.md @@ -0,0 +1,227 @@ +# Camera-to-action skill v0.1 — research report draft + +**Status: teacher data admitted; initial development learners each completed 0/16 strict placements.** +The registered correction round is running. Frozen confirmation is pending. +No qualified downloadable skill or physical robot readiness is claimed. + +## Question and bounded experiment + +Can a small local model learn a complete PiPER block-placement task from +synchronized RGB-D, measured robot state and explicit synthetic action labels? +The intended learned sequence is approach, grasp, lift, carry, release and +retreat. The study uses one simulated PiPER, its existing gripper, a known rigid +40 mm block and two fixed destinations. It does not recover motor actions from +ordinary human video. + +All starts, controller assignments, training settings and acceptance gates were +registered before collection. The runtime is frozen at +`c9a7c55531533f60e24651a99d509f721e195051`; the protocol SHA-256 is +`576851e321034a0f6d3cb4d0818b3a5ca97975fcc6f2c7fa30b07ffd203a2d76`. + +There are 72 fresh starting-position groups: 32 training, eight development and +32 confirmation. Both destinations remain together in their starting-position +group. Starts span x=0.287–0.311 m and y=−0.042–−0.018 m, with the configured +initial block height 0.082 m. This small interpolation domain is a deliberate +first proof, not general spatial manipulation. A 3 mm inset from the prior +fixed primitive's envelope buffers camera error; estimates are not clipped or +replaced with simulator truth. + +| Registered stage | Budget | Role | +| --- | ---: | --- | +| Initial teacher diagnostic | 4 trajectories | Verify actual motion, initial camera acquisition and complete teacher placement | +| Training teacher demonstrations | 64 episodes | Two goals for each of 32 training starts | +| Development teacher demonstrations | 16 episodes | Prediction diagnostics; excluded from fitting and normalization | +| Mixed-control corrections | 32 episodes | One training-only round; teacher chosen with probability 0.5 | +| Model development | 64 rollouts | Initial/corrected × full/blind × 16 development cases | +| Frozen confirmation | 256 attempts | 64 cases each for learner, matched blind model, fixed primitive and teacher | +| Negative controls | 16 attempts | Blank initial RGB, invalid old initial timestamps, malformed extrinsics and forced-open gripper | + +The maximum registered native workload is 452 attempts, with no replacement of +failed starts or hidden retry. A 20-second complete episode contains 501 +320×240 RGB-D pairs at 25 Hz, 1,000 motor intervals at 50 Hz, and 20,000 native +physics ticks at 1 ms. Camera capture/availability timestamps are simulated +clocks; wall-clock processing is recorded separately. They do not establish +physical camera-to-actuator latency. + +## Inputs, supervision and acceptance + +The CPU NumPy model has two 64-unit hidden layers and predicts six absolute +joint targets plus one jaw target. Its 269 features contain an explicitly +historical initial visual anchor, destination, elapsed time, encoders, jaw +measurements, encoder-derived link6 body pose, previous applied servo references and +five causal history samples at 0, 40, 80, 120 and 160 ms. Standardized +features are clipped to ±10 at inference. Current missing object estimates have zero numerical +fills, an unavailable mask and their source frame's original age. The historical +anchor is never presented as a new live observation. The frozen recording names +the link6 body-origin pose `tcp_position_estimate_m` and +`tcp_quaternion_estimate_wxyz`; these legacy field names do not identify a +calibrated gripper contact point. The native task scorer instead uses a distal +site offset by 0.12003 m along link6’s local z axis. The independent auditor +reconstructs this explicit offset when comparing the two coordinate records. + +Teacher phase, current teacher command, native object/contact truth and future +observations are excluded. Teacher commands are synthetic labels from a +privileged authored controller. In the mixed-control round, the one teacher +query per step is stored separately from the command actually executed; it is +not described as an operator correction or a proven recovery. The blind model +uses identical source episodes and initial context but has all live and +historical sensor feedback masked out. It retains past command history. + +Each fit uses seed 17, 50 epochs, batch size 512 and Adam learning rate 0.003. +Normalization uses training data only, including explicit missing fills and +masks. Candidate selection uses development task success, critical contacts, +then prediction error normalized by the same physical action scales across +candidates. The matched blind model is taken from the selected stage. + +Before training, every training start must have at least one successful usable +teacher demonstration, and each goal must have at least 28 successes. Final +qualification requires all of the following: + +- At least 58/64 strict learned placements on the scheduled unseen cases. +- Positive 97.5% paired bootstrap intervals against both fixed and blind + comparators, resampling whole starting-position groups with both goals paired. +- Positive comparisons also on at least 24 common admitted groups, preventing + initial comparator refusals from being the only basis for improvement. +- No increase in the frequency of critical-contact episodes versus the fixed + comparator; all negative controls behave as specified. +- Complete attempt accounting, independent native scoring, no privileged + learner inputs and no teacher intervention during evaluation. + +The teacher comparator has privileged simulation observations and is an upper +reference, not evidence of human-video learning. Installation alone cannot +award qualification. Failed candidates remain publishable research findings. + +The fault definitions are deliberately narrow: `initial_missing` zeros the +initial RGB image while the physical object remains in the scene; +`initial_stale` sets capture and availability times to −0.2 s at episode time +zero; `altered_calibration` scales one rotation-matrix element by 1.1; and +`open_gripper` overrides issued jaw targets to 0.035 m. The first three exercise +input-contract refusal. They do not establish detection of a physically absent +object, valid-but-old frames or small valid rigid calibration errors. + +## Completed diagnostic evidence + +![Native MuJoCo RGB frame at 10.00 simulated seconds, development start 00 and destination 0. The authored teacher controls the arm; the frozen tracker reports no object position in this frame. This is diagnostic evidence, not a learned-policy result.](https://research.dvidia.org/downloads/topics/physical-grounding/runs/2026-11-01-camera-to-action-v01/viewer/frames/development-00-goal0/0250.png) + +An independent reader verified **24,040 files**, exact camera/control/native +cadences, source and frame bindings, issued/query separation, camera-position +errors and independently recomputed task scores. All four teacher diagnostics +completed strict placement successfully. They represent **two development +starting positions, each with two goals**, not four independent starting groups. + +| Diagnostic | Initial camera error | Available-pose error p95 | Available decisions | Longest missing stretch | +| --- | ---: | ---: | ---: | ---: | +| Development 00, goal 0 | 0.196 mm | 0.314 mm | 620/1,000 | 7.24 s | +| Development 00, goal 1 | 0.196 mm | 0.504 mm | 618/1,000 | 7.28 s | +| Development 01, goal 0 | 0.249 mm | 0.816 mm | 602/1,000 | 7.96 s | +| Development 01, goal 1 | 0.249 mm | 0.816 mm | 604/1,000 | 7.92 s | + +Native object displacement reached 138.6–160.0 mm, confirming actual +manipulation rather than a static-camera or replay-only check. Errors above +are conditional on the tracker returning an estimate. Adjacent 50 Hz decisions +often reuse the same 25 Hz frame and are not independent tracking trials. + +The major diagnostic finding is **where visibility disappears**. An offline +analysis of sealed teacher-phase and perception sidecars found zero available +object estimates during close, lift, transfer, place and release: **0/749 motor +decisions** across the four trajectories. Approach and final verification were +fully visible. Teacher phase was used only to analyze this evidence, never as +a model input. + +Therefore sub-millimetre available-pose errors must not be described as tracking +throughout manipulation. During the contact-heavy sequence the proposed +learner must rely on measured robot state and its historical initial visual +context. This is a measured limitation of the frozen upright-block tracker. +The current experiment will test whether those remaining signals are enough; +it must not silently add camera views or substitute native object positions. + +[Inspect the real diagnostic camera frames and action traces](https://research.dvidia.org/downloads/topics/physical-grounding/runs/2026-11-01-camera-to-action-v01/viewer/index.html), [independent audit](https://research.dvidia.org/downloads/topics/physical-grounding/runs/2026-11-01-camera-to-action-v01/viewer/diagnostic-audit.json), [phase coverage analysis](https://research.dvidia.org/downloads/topics/physical-grounding/runs/2026-11-01-camera-to-action-v01/viewer/diagnostic-phase-coverage.json) and [registered protocol](https://research.dvidia.org/downloads/topics/physical-grounding/runs/2026-11-01-camera-to-action-v01/viewer/protocol.json). The compact viewer contains diagnostic evidence only. It preserves every motor decision and copies native RGB frames byte-for-byte at 2.5 Hz from the original 25 Hz capture. Its display sampling is distinct from the camera cadence used by the controller. Full raw recordings remain separate from this compact diagnostic subset. + +## Completed demonstration and data-quality evidence + +All **80/80 registered teacher demonstrations** completed strict placement: +64/64 training episodes and 16/16 development episodes. The development +recordings remain excluded from fitting and normalization. All 32 training +starting-position groups have usable successful demonstrations with an +available, bound initial camera estimate. Each training destination has +**32 successful usable episodes**, exceeding the registered minimum of 28; +the sealed teacher-data gate passed with no missing training group. + +| Teacher evidence | Strict successful / scheduled | Data use | +| --- | ---: | --- | +| Initial diagnostics | 4/4 | Initial camera and task checks; not additional fitting episodes | +| Training demonstrations | 64/64 | Issued-action supervision for 32 training starts × two goals | +| Development demonstrations | 16/16 | Held-out prediction and missing-observation diagnostics | + +An independent readback passed with **504,840 files verified and 84 native task +scores recomputed** across the four diagnostics plus 80 demonstrations. This +count includes the earlier four diagnostics; it is not 84 additional episodes. +The audit reports no errors and executes no physics, estimator or learned +model. Its protocol, sealed demonstration receipt and stage-integrity identities +match the original registration and saved bytes. + +The retained `demonstrations-audit.json` proof has SHA-256 +`3405d789c0b79066d142d476f1b45468463d41516350d7c80766881a6cecf753`; +its evidence-inventory SHA-256 is +`319074d85fcd84758d3cc464e6040f2032832179d354c4adcb86f300b743f811`. +This is an audit of the diagnostic and demonstration stages only: no fitting +stage or student rollout is covered by this proof. Successful teacher actions +establish usable synthetic supervision, not successful imitation or transfer. + +## Measured host context + +The retained `hardware-native.json` describes a shared local **Apple M5** host +(`Mac17,4`, arm64) with **10 physical / 10 logical CPU cores and 24 GiB of +installed memory**. The environment is Darwin 25.5.0, Python 3.12.9, NumPy 2.5.3 +and MuJoCo 3.15.0, with one native worker. Native physics and the policy use the +CPU; rendering uses the existing macOS CGL graphics path. No CUDA requirement +is introduced. The hardware-context file SHA-256 is +`fdc8007b27219c7c91dc600ae54a95f4a8abc64565db652002faa0f5a1e9f6d0`. + +Installed memory is host capacity, not the model's measured memory requirement. +This shared-host setup does not establish minimum hardware, dedicated-host +throughput, other-platform compatibility or physical sensor-to-actuator +latency. Inference distributions and full-loop wall-clock measurements remain +pending the completed study and independent readback. Initial fit costs below +are separate from the much longer rendering, recording and verified export. + +## Initial learning result: prediction is not execution + +The sealed initial fitting stage contains 64,000 training samples and 16,000 +held-out teacher-observation samples. The full learner fitted in 6.56 seconds +and occupies 521,715 bytes; the matched blind learner fitted in 5.99 seconds +and occupies 517,296 bytes. Shared byte-verified data export took a further +685.44 seconds. These are measured offline wall-clock durations on the shared +host, not video-to-skill completion times. Process peak RSS reached 643,891,200 +bytes after the full fit and 789,200,896 bytes after the blind fit; the second +value is a process high-water mark, not the blind model's isolated memory use. + +Both initial candidates completed all 16 registered development attempts and +made **zero strict placements**. The full learner's held-out action-scale MSE +on teacher observations was 0.0001133, yet its closed-loop success was 0/16. +The blind learner also scored 0/16. One-step prediction on a demonstrated +state does not establish execution after the model's own actions change it. + +These are intermediate development results, not frozen confirmation. The +single preregistered mixed-control round continues unchanged. No replacement +starts, new hyperparameters or extra correction round have been introduced. +Final independent full-study readback must bind these sealed receipts before +publication as completed benchmark evidence. + +**Pending:** corrected fitting and development, frozen confirmation, negative +controls, final inference and full-loop timings, and fresh-install execution. +The candidate selection and acceptance thresholds remain fixed. + +This study can establish a reproducible camera-and-action learning loop for a +bounded synthetic task. It cannot establish arbitrary video-to-skill transfer, +arbitrary-arm compatibility, real-time hardware operation, real-world depth or +contact calibration, shoe manipulation or physical qualification. A useful next +experiment after the fixed results is to measure a second camera or another +explicitly uncertain observation method during grasp and carry, where the +present tracker provides no object measurements. + +## Source and reuse + +[The frozen learning source](https://github.com/Dvidia-Inference/dvidia-training/commit/c9a7c55531533f60e24651a99d509f721e195051) and the [DVIDIA Training repository](https://github.com/Dvidia-Inference/dvidia-training) contain the capture, mask-aware exporter, CPU model and study runner. Model/data releases and the completed study report are pending; no download here is labeled a qualified skill. + +Original DVIDIA code and generated records use MIT; PiPER and simulator notices retain their upstream terms. The viewer includes actual native renderer output and no private human footage, credentials, model weights or redistributed robot meshes. Research writing follows this site's CC BY 4.0 license. diff --git a/docs/camera-action-v01.md b/docs/camera-action-v01.md new file mode 100644 index 0000000..6a673c1 --- /dev/null +++ b/docs/camera-action-v01.md @@ -0,0 +1,129 @@ +# Camera-to-action skill v0.1 + +This local research workflow pairs rendered RGB-D with explicitly synthetic +PiPER teacher commands, fits a small CPU policy and compares complete tasks on +unseen starts. A model or successful installation does not establish qualification. +The four diagnostic trajectories and teacher-data gate must pass before fitting. +Failed attempts remain in the evidence. Failed quality gates block dependent +stages rather than triggering retries; individual task failures do not stop a +registered comparison whose attempt accounting is complete. + +## Local setup + +Python 3.12, the pinned NumPy and MuJoCo dependencies, existing verified PiPER +assets and a working OpenGL context are required. The tested renderer is macOS +CGL. No hosted inference or CUDA GPU is required; other operating systems and +hardware minimums remain unqualified. Native stepping is CPU work; rendering +uses the graphics driver. After dependencies/assets are provisioned, the workflow +is offline. The study reserves 30 GiB of free disk and caps its evidence at 120 GiB. + +```sh +git clone --branch feat/piper-camera-action-v01 https://github.com/Dvidia-Inference/dvidia-training.git +cd dvidia-training +python3.12 -m venv .venv +source .venv/bin/activate +python -m pip install '.[arm]' +dvidia-robot-sim fetch piper --output /path/to/robot-assets +dvidia-train robot prepare --study runs/camera-action --fixed-model /path/to/piper-base-model.json +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage diagnostic +dvidia-train robot inspect --study runs/camera-action +dvidia-training-studio --directory runs/studio --port 8363 +``` + +The fetch command provisions the separately licensed, version-pinned PiPER assets. +For an exact reproduction, check out the reviewed source commit recorded in the +dataset card before installing, or install that release's wheel with its `arm` +extra. The branch above is the development entry point and may advance. +For reproduction, use `fixed-model.json` from the published study metadata as +`/path/to/piper-base-model.json`; it is the registered comparator, not a newly +trained model. Keep the original study untouched and prepare a fresh directory. + +Open `http://127.0.0.1:8363/robot`, choose the prepared study and assets folders, +check the inputs, then run the remaining registered stages. The UI and Terminal +read the same immutable stage receipts. The UI stops when a gate blocks progress. +It installs a model only after a completed benchmark; installation remains a +simulation-candidate action. Ordinary video without paired actions is not an +accepted motor-learning input. + +For manual execution after a passing diagnostic: + +```sh +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage demonstrations +dvidia-train robot train --study runs/camera-action --stage initial +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage development-initial +dvidia-train robot collect --study runs/camera-action --assets /path/to/robot-assets --stage corrections +dvidia-train robot train --study runs/camera-action --stage corrected +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage development-corrected +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage confirmation +dvidia-train robot evaluate --study runs/camera-action --assets /path/to/robot-assets --stage negative +dvidia-train robot package --study runs/camera-action --output runs/camera-action-summary +``` + +The CLI package contains summary evidence and any fitted model JSON; it explicitly +does not include raw recordings or the executable runtime. Published research +archives additionally carry the independently audited evidence available for the +reported stage. Existing destination directories are never overwritten. + +## What the learner sees + +The frozen RGB-D tracker receives current pixels, depth and calibration only. +The policy receives a timestamped initial visual position, goal, elapsed time, +encoders, encoder-derived link6 pose, previous applied servo references and 160 ms +of causal history. Missing live estimates retain availability masks and source +frame age; their numerical tensor fills are not measurements. Normalization uses +training data, including these explicit missing fills and masks. +History has five causal samples at 0, 40, 80, 120 and 160 ms; unavailable warmup +samples remain masked. Standardized features are clipped to ±10 at inference. +The matched blind learner keeps the initial visual anchor, destination, elapsed +time and previous applied commands, while masking current and historical sensor +feedback. It is a feedback ablation, not a model with no visual task context. + +The legacy `tcp_position_estimate_m` field represents the link6 body origin in +this pilot. The native task scorer uses a distal site 0.12003 m along link6's +local z axis. These are distinct points; the audit accounts for that fixed +transform rather than treating the feature as a calibrated tool-tip estimate. + +The policy never receives native object/contact truth, teacher phase, current +teacher target or future frames. Historical initial position is never copied into +the current-object channel. Native truth is used only by the teacher and offline +scorer. The first tracker only recognizes a known upright block: tilt and gripper +occlusion remain measured gaps, not invented observations. + +Teacher queries and issued commands are separate: one registered 50% mixed-control +round can request teacher labels on student-visited states. Those queries are +synthetic suggestions, not proven recoveries or operator corrections. Confirmation +has no teacher fallback. All candidates retain the native actuator and task bounds. +Fitting uses successful complete teacher episodes and complete mixed episodes +with a valid initial visual anchor. Failed mixed tasks are included. Interrupted +prefixes remain in the raw evidence and can be exported, but are excluded from +the registered fitting set; they do not silently become successful examples. + +## Registered comparison + +72 fresh initial-XY groups, split 32/8/32, share two known goals within each group. +Starts use a 3 mm inset from the original primitive's operating envelope to buffer +camera error, with no clipping or simulator-truth substitution. The fixed comparator +gets the same initial visual position. Comparisons also require at least 24 common +admitted groups so comparator refusals alone cannot establish an improvement. + +The acceptance gates are at least 58/64 strict placements, positive 97.5% grouped +paired intervals versus fixed and blind controls, no increase in critical-contact +frequency, all sixteen negative controls behaving correctly and no privileged +learner inputs or hidden intervention. All failures remain in scheduled denominators. +These are proposed acceptance thresholds; the study report records whether they +were actually met. Offline host timings are not physical sensor-to-actuator latency. + +## Candidate installation + +For a fitted model from a completed study: + +```sh +dvidia-arm check --model /path/to/model.json --robot piper +dvidia-arm install --model /path/to/model.json --robot piper --assets /path/to/robot-assets --output installed-camera-skill +dvidia-arm run installed-camera-skill --assets /path/to/robot-assets --output runs/installed-camera-attempt --start-x .30 --start-y -.03 --goal 0 +``` + +Only the exact local PiPER simulation interface is accepted. No physical driver, +arbitrary-arm transfer, shoe skill, human-video motor recovery or real-time guarantee +is introduced. Original code and generated evidence are MIT; upstream asset notices +remain in the repository and assets are provisioned separately. diff --git a/docs/camera-action-v02-proposal.md b/docs/camera-action-v02-proposal.md new file mode 100644 index 0000000..565947c --- /dev/null +++ b/docs/camera-action-v02-proposal.md @@ -0,0 +1,165 @@ +# Camera-to-action v0.2: proposed control-learning ablations + +**Proposal only, 2026-11-01. Nothing below is implemented, run, or registered.** +The v0.1 correction and confirmation results are pending at this proposal's +evidence cutoff. Finish, audit and report that study unchanged. This document +must not change its models, starts, budgets, selection or qualification gates. + +The first scheduled initial full-student development case diverged from its +matching teacher by 10 mm at 0.409 s, before its first missing object estimate +at 1.56 s. It commanded a single-finger target below 20 mm at 1.20 s while +133 mm horizontally from the block; that command did not mean its fingers had +already closed. It rate-limited 977/1,000 intervals versus the teacher's 75, +never acquired bilateral contact, and accumulated 1,427 critical-contact ticks. +These are observations from one sealed local diagnostic, not a full-study +audit, failure-frequency estimate, or causal proof. Covariate shift, target +scaling and command representation remain competing explanations. +The student renderer also warned that `ARB_clip_control` was unavailable for +`mjDEPTH_ZEROFAR` and that depth accuracy would be limited. Repeated warning records are not +accuracy measurements, and this is not evidence that the warning caused the +failure. Keep it visible as a depth-validation limitation; do not change the +frozen renderer or treat exact camera transforms as validated depth. + +## What prior work supports + +DAgger collects expert labels on states visited by the learner and aggregates +them with demonstrations. It explicitly permits mixtures of expert and learner +control; repeatedly visiting only expert states does not test the same +distribution. Its theoretical guarantees require assumptions and an iterative +procedure that our small neural experiment does not establish. v0.1's single +50% mixture round is already a useful first test; it is not yet a result. +[Ross, Gordon and Bagnell, AISTATS 2011](https://proceedings.mlr.press/v15/ross11a.html). + +A position target is a servo reference, not a measured pose or an instantaneous +movement. MuJoCo separates control/reference inputs from generated actuator +force. Its motor documentation also distinguishes reference slew limits from +setpoint bounds and torque limits. Our current experiment uses its existing +position servos and software limiter, **not** that documented DC-motor model. +[MuJoCo actuation](https://mujoco.readthedocs.io/en/stable/computation/index.html#actuation-model), +[DC Motor Model, §2.5.3–2.5.5](https://mujoco.readthedocs.io/en/3.7.0/_static/dcmotor.pdf). + +Action components can use different normalization and must retain identical +ordering, units and decoding in training and deployment. That motivates a +controlled normalization comparison, not a claim that one scale is universally +better. Residual-control research combines a conventional controller with a +learned correction; its RL results do not establish that our supervised +increment predictor will work. A previous-reference increment is also distinct +from a residual around a task-solving controller. +[robomimic action configuration](https://robomimic.github.io/docs/tutorials/action_configs.html), +[Johannink et al., Residual Reinforcement Learning for Robot Control](https://arxiv.org/abs/1812.03201). + +## Three contrasts, registered before any new rollout + +Use five explicitly named full-feedback candidates. **B0** is the common +control: original 64 training demonstrations plus the sealed 32 v0.1 mixed +episodes, absolute issued/query targets, and v0.1 output whitening. Fit it anew +under the v0.2 manifest; it is not automatically the selected v0.1 checkpoint. +Freeze feature normalization and the baseline output statistics from this +training set for the coverage comparison. Other changes below are separate; +do not combine winning choices in this experiment. + +| Contrast | Single question and intervention | Evidence against the proposed explanation | +| --- | --- | --- | +| **A versus B0: learner-state coverage** | Replace the 32 mixed episodes with 32 pure-student episodes on the exact same registered training cases, using the same sealed **initial** full checkpoint that generated v0.1's mixtures. Query the stateful teacher once per decision for labels; execute only the learner. Fit on original 64 + new 32, with identical sample and optimizer budgets. | More learner exposure without improved held-out completion weakens the claim that the 50% intervention rate was the principal bottleneck. If the confidence interval excludes the proposed practical improvement, reject that effect size for this bounded recipe. | +| **B versus B0: output scaling** | Same data and absolute targets; keep the training mean but remove empirical output standard-deviation division. Use the existing fixed scales `[3 rad × 6, 0.035 m]`. Keep feature processing unchanged. | Lower normalized training loss alone does not count. No closed-loop improvement under matched native-unit scoring rejects a practical benefit from this change at the stated budget. | +| **CΔ versus Cabs: reference-relative control** | Same B0 data; both learn the *same* feasible next-reference labels. Cabs predicts the absolute endpoint; CΔ predicts its increment from the preceding applied reference. Both use the same physical-scale endpoint loss and identical final feasibility projection. | If only the reference bounds improve metrics, both candidates should benefit similarly; CΔ must beat Cabs to support the representation hypothesis. Reduced command clipping alone is not evidence of better manipulation. | + +For the third contrast, let `r` be the preceding applied servo reference, +`a*` the teacher target, and `P(r, a*, 20 ms)` the exact existing limiter advanced +analytically through its 20 native ticks. Set `r* = P(r, a*, 20 ms)`. +Cabs predicts `r*`; CΔ predicts `(r* − r) / (rate_limit × 20 ms)` and decodes +relative to **r**, never to a future reference or measured joint position. +Use the same seeded hidden layers and analytically scale/offset the final heads +so their decoded outputs match on every registered reset context before fitting; +verify this in a numerical preflight, without rollout or hyperparameter search. +Report any differences on later training states explicitly. This matches initial +command bias, not the function classes or their subsequent feedback dynamics. +Before collection, require a pure numerical equivalence check that issuing +the projected endpoint reproduces all 20 teacher-reference microsteps; if it +does not, block this contrast and report the reason. Preserve original joint, +jaw, force and rate limits. Endpoint feasibility does not bound actual joint +velocity or guarantee safe contact. A residual around the fixed task primitive +is deferred: it would test a hybrid controller, not complete learned control. + +All candidates retain causal 160 ms history, honest missing masks, historical +initial anchor and the original input whitelist. Teacher phase, current query +target, native object/contact truth and future observations remain excluded. +Native truth may score attempts offline. New query labels stay separate from +executed actions. Failures and interrupted prefixes are retained; there are no +replacement scenes or fallback teacher actions. Incomplete or unequal usable +training-label counts block the affected comparison rather than silently +changing its budget. The stateful teacher's suggestions off its usual +trajectory are supervision, not proof of optimal recovery. + +## Bounded screen and falsifiable decision + +Proposed maximum: **32 new training rollouts**, five CPU NumPy 2×64 fits +(seed 17, 50 epochs, batch 512, Adam 0.003), and **80 learned evaluation +attempts**: eight fresh start groups × two goals × five candidates. Add the +teacher and fixed controller on the same 16 cases (**32 attempts**), and four +existing negative-control types once per candidate (**20 attempts**). +Total: **164 new attempts**, each at most 20 simulated seconds. Reuse sealed +training data; never train on any v0.1 development or confirmation recording. +Generate the eight new groups with a separately registered seed, reject exact +overlap with all old starts before running, and freeze the resulting coordinates, +case order, source hashes and resource limits. This small screen is intentionally +underpowered for a general success claim; it awards no qualified-skill label. + +Predeclare the three paired contrasts above. Primary outcome is strict complete +placement on all 16 scheduled cases, with aborted/unattempted cases counted as +failures and separately reported. A promising contrast requires at least +**two additional successes out of 16**, a positive lower bound of a **98⅓% +two-sided paired bootstrap interval** grouped by the eight starting positions +(50,000 resamples, seed 171701), no increase in either critical-contact episode +count or total critical-contact ticks, and every negative control behaving as +specified. The interval level allocates 5% across the three planned comparisons; +with eight groups it remains an approximate screen, not strong population +evidence. An upper bound below +12.5 percentage points rules out that proposed +practical improvement at this resolution; otherwise an unmet gate is +**inconclusive**, not proof of no effect. A lower prediction loss or a stationary +arm with fewer contacts cannot pass by itself. + +For contextual comparisons against the fixed controller, preserve the existing +common-admission check at the operating-envelope boundaries. Report both all +16 scheduled attempts and the subset where both goals of a starting group were +admitted by both controllers; require at least six of eight complete groups +before making a fixed-comparator claim. Refusals remain failures in the scheduled +denominator. This subset never replaces the three primary learned contrasts. + +Report bilateral-grasp acquisition, first-closure geometry, lift/retention, +strict success, critical contacts, issued-versus-applied target error, command +variation, perception missingness, standardized-feature clipping, training/export +time, process peak memory, inference and full-loop latency. Compare losses only +after decoding to the same native targets and fixed scales; report joint-radian +and jaw-metre errors separately. Teacher path separation is descriptive because +a different successful path is allowed. No confirmation-selected hyperparameters, +new fitting seeds, or posthoc combinations are added. Any promising candidate +needs a separately registered larger confirmation and matched blind comparison. + +## Keep the next cycle local and measure its real cost + +The sealed initial fits took **6.56 s full / 5.99 s blind**, while their shared +verified export took **685.44 s**. Each complete episode has **6,010 files**. +This shows that fitting the tiny MLP was not the expensive part of that measured +workflow; it does not by itself separate disk I/O from hashing, decoding and +validation. Profile those components before proposing a GPU remedy. Default +execution remains one local worker with the existing renderer requirements; +there is no cloud model, new hardware, or physical-arm command. Simulated time +is not a wall-clock duration estimate. Preflight storage for the fixed roster +and preserve the existing 30 GiB free-space reserve; stop with an explicit +incomplete result if that budget cannot be met. + +A later **separate operational benchmark** can compare the existing per-file +layout with compact immutable chunks/columnar sidecars on the exact same sealed +records. Measure bytes, file opens, hash/decode/validation time and peak memory; +require lossless reconstruction, identical per-record identities and outcomes, +and rejection of modified, reordered, truncated or mismatched records. Retain +raw media hashes, frame/calibration/model bindings and atomic sealing. No format +change or audit relaxation belongs in the current frozen study. + +Evidence: [v0.1 research report](camera-action-v01-report-draft.md) and +[reproducible first-case diagnosis](../benchmarks/camera-action-v01/diagnose_first_student.py). +The local diagnosis proof is retained separately until the full-study publication +binds the same episode bytes; this proposal does not claim that audit has passed. +All numerical thresholds and budgets above are proposed choices, not measured +v0.2 results or established optimum settings. diff --git a/publishing/CAMERA-ACTION-V01.md b/publishing/CAMERA-ACTION-V01.md new file mode 100644 index 0000000..25174b3 --- /dev/null +++ b/publishing/CAMERA-ACTION-V01.md @@ -0,0 +1,267 @@ +# Camera-to-action evidence publication + +This workflow publishes a stopped, sealed simulation study to +`Dvidia/piper-camera-action-v01`. It does not train a model, run physics, alter a +Space, or turn a failed study into a qualified skill. + +Build the wheel and a `git archive` tar.gz from the reviewed source commit. +The archive must retain Git's PAX commit comment. Both artifacts must contain +the exact registered runtime bytes, including unchanged native scorers. +The source-artifact limit is 1 GiB compressed, matching the uploader's per-file +limit. The complete reviewed repository at `937d3e3` measured 653,980,983 bytes +compressed and 779,972,592 regular-file bytes expanded; historical checked-in +benchmarks account for most of it. No source files are omitted to fit a smaller +archive. Independent expanded limits remain 2 GiB total, 256 MiB per member and +50,000 members, with path, secret and runtime-identity checks unchanged. +Provide the reviewed dataset card and the independently generated diagnostic +audit. The audit's scope is explicitly the four diagnostic episodes. +The optional `--phase-coverage` argument includes the diagnostic-only phase +coverage proof after verifying its protocol and audit bindings. + +After the study stops, independently audit every sealed stage with the numeric +scorer and saved evidence. This command does not import or execute the native +environment, tracker, teacher or learned policy. It verifies frame identities, +50 Hz command and 25 Hz camera clocks, 1 ms native traces, command authority, +teacher-query separation, model identities and split membership. It reconstructs +data-quality gates and, if confirmation exists, grouped bootstrap comparisons +and the final qualification gates. Missing stages remain unrun. + +```sh +PYTHONPATH=src:benchmarks/camera-action-v01 python \ + benchmarks/camera-action-v01/study_audit.py /absolute/stopped-study \ + > /absolute/full-study-audit.json +``` + +Pass `--study-audit /absolute/full-study-audit.json` to preparation. It must +pass, cover every currently sealed stage and match the protocol, model, stage +and episode identities. This proof supplements the diagnostic audit. An audit +passing means reported outcomes agree with saved evidence, including failures; +it does not mean the skill passed its qualification gates. +The reviewed source archive must include the exact full-study auditor and its +diagnostic byte-reader helper; use the final tooling commit while preserving +all registered runtime bytes. + +Add `--hardware /absolute/hardware-native.json` to include the measured host, +software versions, native worker count and graphics/timing scope as +`hardware.json`. The field whitelist permits public CPU/model/memory details +and rejects hostname, serial, account or other arbitrary fields. The entire +JSON must fit within 16 KiB. These details describe the measured run; they do +not establish minimum hardware requirements or physical sensor latency. + +Generate the dataset and individual model cards from the same stopped study, +posthoc summary and reviewed source commit. The renderer requires the full-study +audit for a completed study. Supply identical optional hardware and phase proofs +to rendering and preparation: + +```sh +PYTHONPATH=src python publishing/render_camera_action_card.py \ + --study /absolute/stopped-study --summary /absolute/summary.json \ + --study-audit /absolute/full-study-audit.json \ + --source-commit EXACT_40_CHARACTER_COMMIT --output /absolute/generated-cards +``` + +Pass the **generated output directory** to `--model-cards`, and its `README.md` +to `--card`. Preparation checks the exact file roster, model identities and +selection, generator/source bindings and receipt, and regenerates all card text +from sealed evidence before copying it. The generated directory retains the +original `summary.json` bytes: preparation verifies their receipt identity and +reconstructs their numerical content. This local input is not a separate public +payload artifact. The supplied source archive must contain the exact renderer, +summary helper and optional phase helper named by the receipt. This includes +`model-cards/{initial|corrected}/{full|blind}/README.md` and the bound +`model-cards-receipt.json` in the Hugging Face payload; model JSON remains under +`models/`. All trained comparison candidates have cards, even when not selected. +A blocked study with no fitted models has an empty model-card roster, never +invented model artifacts. Existing directories are not overwritten. Omitting +the optional flag preserves the historical dataset-card-only packaging path. +The generated dataset card puts the minimal selected-model installation first: +download its JSON and the released wheel, verify the model hash, provision the +PiPER assets, then check/install/run locally. The raw corpus and source archive +are required only for the separate full-study reproduction instructions. + +```sh +PYTHONPATH=src python publishing/prepare_camera_action_v01.py \ + --study /absolute/stopped-study \ + --output /absolute/new-publication-directory \ + --audit /absolute/diagnostic-audit.json \ + --study-audit /absolute/full-study-audit.json \ + --card /absolute/generated-cards/README.md \ + --model-cards /absolute/generated-cards \ + --wheel /absolute/dvidia_training.whl \ + --source-archive /absolute/reviewed-source.tar.gz \ + --source-commit EXACT_40_CHARACTER_COMMIT + +python publishing/huggingface/publish_camera_action_v01.py stage \ + /absolute/new-publication-directory --plan /absolute/reviewed-plan.json +``` + +Review the public roster and returned plan digest before publication. The +explicit `publish` command reads only the existing macOS Keychain entry +`dvidia-huggingface`, account `dvidia-publisher`. No credential argument, +environment variable, credential file, or login cache is required. + +```sh +python publishing/huggingface/publish_camera_action_v01.py publish \ + --plan /absolute/reviewed-plan.json --sha256 REVIEWED_PLAN_DIGEST \ + --cache /absolute/publication-cache + +python publishing/huggingface/publish_camera_action_v01.py verify \ + --plan /absolute/reviewed-plan.json --sha256 REVIEWED_PLAN_DIGEST \ + --cache /absolute/anonymous-readback-cache +``` + +Preparation streams one deterministic ZIP64 archive per sealed episode, +checks each file against the original integrity manifest, and decompresses +every member to verify its bytes. DEFLATE level 1 is the publication default. +On the same sealed teacher episode, one sequential shared-host comparison +measured 40.36 seconds / 81,058,092 bytes at level 1 versus 54.02 seconds / +75,301,793 bytes at level 6. That is a 25.3% shorter archive-and-verification +time for 7.64% more bytes in this example, not a general throughput guarantee. +It refuses a worker lock, unsealed episode, +unexpected study file, symbolic link, missing source binding, or detected +credential-shaped text. The 30 GiB disk reserve is preserved. Existing output +directories and files are never overwritten. + +The publisher freezes reviewed bytes into a private temporary directory before +credential lookup. On a supporting macOS filesystem, it uses the documented +`fclonefileat(2)` copy-on-write operation from a regular-file descriptor. The +copy has its own inode and attributes, is marked read-only, and is checked +against the reviewed bytes. Later writes to either file are private to that +file. Hardlinks are never used. Unsupported filesystems use bounded streaming +copies with free-space checks before each file and each chunk, preserving the +30 GiB reserve. Every path rejects symlinks and existing destinations. +The uploader uses a single commit bound to the empty dataset's exact +parent revision, and refuses existing nonempty or private repositories. +Anonymous verification downloads every published file at the exact resulting +commit and checks its byte identity. Each file uses a new owned temporary cache +under the explicit `--cache` directory, with its full byte budget plus metadata +headroom checked against the 30 GiB reserve before download. That temporary is +removed after verification, including on failure. Existing shared cache files +are never removed. The retained receipt records the pinned commit, reviewed +plan digest, inventory digest, byte count and largest single readback file. +The pinned remote file-size metadata must match every reviewed byte count +before any file download; missing or changed sizes fail closed. +Verification sets `HF_HUB_DISABLE_XET=1` before SDK import to use HTTP downloads +into the owned Hub cache, and refuses an already-imported SDK whose Xet +transport is still enabled. This avoids a separate shared Xet data cache. +Large files are streamed, not loaded into one large in-memory payload. + +To reconstruct the recorded study, verify `SHA256SUMS`, then extract +`study-metadata.zip` and every `episodes/**/*.zip` into one new directory. +The resulting layout contains the original protocol, stage receipts, camera +frames, encoders, issued controls, perception and teacher-query sidecars, +native traces, and each episode's original `integrity.json`. Raw files inside +an episode must not be edited or supplemented before inspection. + +The dataset includes separate model JSON files only when sealed training +receipts bind those exact files. A blocked data-quality gate therefore +publishes evidence and source without inventing a learned policy. The package +retains the study's qualification result; installation itself does not award +task qualification or physical-hardware readiness. + +## Optional precompression while the study runs + +One separate publication worker may precompress already sealed, complete +episodes while the native worker continues. It snapshots the registered stage +rosters, reads only fully sealed stage receipts, and includes completed task +failures as well as successes. It never reads a running stage's episode files, +retries a simulation, queries a policy, or changes study qualification. + +```sh +PYTHONPATH=publishing:publishing/huggingface python \ + publishing/episode_archive_cache.py \ + --study /absolute/running-study --cache /absolute/separate-episode-cache \ + --limit 84 +``` + +The limit is a finite maximum of new archives (1–452), with one sequential +worker guarded by its own lock. Before each archive, it reserves the full +remaining registered raw-data capacity, a conservative ZIP expansion allowance +and the existing 30 GiB disk reserve. It can refuse compression even when the +final expected compressed corpus would fit. Existing cache entries are never +overwritten; a partial entry or a lock left after a hard stop requires operator +inspection. The study and native worker remain untouched. + +After the study and cache worker stop, add +`--episode-cache /absolute/separate-episode-cache` to the normal preparation +command. Cache receipts provide no authority: final preparation independently +reads every original source file, checks the final inventory, decompresses and +hashes every ZIP member, and checks a private copy again. It uses the same +reviewed copy-on-write helper as publication, with independent inodes and a +bounded streaming fallback. Missing entries use ordinary fresh compression; +malformed or changed entries fail closed. Small refused or incomplete episodes +are retained by normal final packaging, rather than precompressed. No extra +files or relaxed acceptance rules are introduced into the public payload. +Archive and member comments, unknown extra fields, unsorted members, prefix or +trailing payloads, and unused compressed suffixes are refused. Only the exact +local ZIP64 size fields required by the original writer are accepted. DEFLATE +does not encode its compression level, so cached archives truthfully record +`compression_level: null`; final verification establishes original member +bytes and the declared ZIP layout, not an unverifiable producer setting. + +## Stopping, interruption and publication scope + +Studio and Terminal use the same immutable stage records. Studio owns one local +worker, checks the protocol and assets between stages, and stops at a blocked +or completed study. It does not retry a stage after an exception. A browser +disconnect does not cancel the server's worker. Closing or killing the server +is not a supported pause mechanism. + +A native attempt that can write its final receipt retains its completed prefix +as `interrupted` or `refused`; it is not counted as a strict success. If a process +ends before sealing an episode, a subsequent collection/evaluation inspection +retains that existing directory as `unknown` and never reruns its case. Remaining +registered cases are not replaced with easier starts. A sealed stage with an +unknown or unattempted case blocks downstream work. + +Fitting has a stricter interruption boundary. Once a model variant's directory +has been created, the frozen trainer refuses to fit that variant again. A fit +exception can therefore leave a model prefix without a sealed training stage. +In this state, the small `inspect` response may still list training as the next +stage; that is not permission or evidence that an automatic retry is possible. +Preserve the directory, fitting log and original stage-start record. Do not +delete them, fabricate a completed receipt or award a model qualification. + +An ordinary exception releases the study worker lock. A hard process stop may +leave `worker.lock`; Studio deliberately neither guesses whether it is stale +nor removes it. Any operator investigation must first establish that the +original worker has stopped and preserve the interrupted evidence. Do not +remove a lock to start a second worker against a live study. + +The raw-evidence publisher accepts only stopped, sealed `blocked` or `complete` +studies. It refuses an interrupted fitting prefix or an unsealed episode; those +cannot silently disappear from this dataset. If such an interruption happens, +publish its separate, explicitly incomplete evidence report without changing +the registered study or retrying the fit. This is an operational blocker, not a +negative task-success measurement or a completed benchmark. + +## Reviewed source snapshot + +This change is stacked on `feat/piper-rgbd-tracker-v01`, whose exact local base +for this study is `769d666ecd684406ae228daf722e41f0bf9ac17d`. The camera/action +branch is `feat/piper-camera-action-v01`; use the tracker branch as its pull +request base, not the older stable `main`. Recheck remote ancestry before +publication rather than merging unrelated changes into the frozen study. + +Build release artifacts only after the final tooling and report changes are +reviewed and committed. Use that exact commit for both the source archive and +wheel. The archive's PAX comment must identify the commit; the packager checks +that the registered runtime and independent auditor bytes agree. Keep the +existing development version; do not replace a stable release. + +```sh +git status --short +git merge-base HEAD origin/feat/piper-rgbd-tracker-v01 +git rev-list --left-right --count origin/feat/piper-rgbd-tracker-v01...HEAD +git rev-parse HEAD +git archive --format=tar.gz --prefix=dvidia-training/ \ + --output=/absolute/new-release/reviewed-source.tar.gz EXACT_REVIEWED_COMMIT +``` + +Extract that reviewed Git archive into a fresh build directory, then build the +wheel from its `dvidia-training` directory with the existing local Python build +environment (`python -m build --wheel --no-isolation --outdir +/absolute/new-release`). This keeps uncommitted files and generated run data out +of the source release. Re-run the fresh-environment installation checks against +the final wheel, and retain its digest with the publication plan. Neither the +source snapshot nor a successful install proves task qualification. diff --git a/publishing/episode_archive_cache.py b/publishing/episode_archive_cache.py new file mode 100644 index 0000000..c43f101 --- /dev/null +++ b/publishing/episode_archive_cache.py @@ -0,0 +1,291 @@ +"""Precompress sealed completed episodes without changing the running study. + +Cache receipts are advisory. Final packaging rereads original bytes and every +ZIP member against its own final inventory, then checks the private copy again. +No simulator, policy, credentials or network client is executed. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import os +from pathlib import Path +import shutil +import stat +import struct +import subprocess +import sys +import zipfile +import zlib + +import prepare_camera_action_v01 as evidence + +SCHEMA = 'dvidia.camera-action-archive-cache/v1' +STAGES = ('diagnostic', 'demonstrations', 'development-initial', 'corrections', + 'development-corrected', 'confirmation', 'negative') +MAX_ATTEMPTS = 452 + + +def key(inventory, prefix): + evidence.relative(prefix) + for name, record in inventory.items(): + evidence.relative(name) + evidence.checked_identity(record) + return sha256(evidence.canonical(dict(schema=SCHEMA, prefix=prefix, files=inventory, + zip_format='ZIP64-DEFLATE', level=evidence.ZIP_LEVEL, timestamp=[1980, 1, 1, 0, 0, 0], + file_mode=stat.S_IFREG | 0o644))).hexdigest() + + +def archive_headroom(inventory, prefix): + """Conservative DEFLATE/ZIP64 expansion, headers and metadata allowance.""" + key(inventory, prefix) + raw = sum(row['bytes'] for row in inventory.values()) + headers = sum(512 + 2 * len((prefix + '/' + name).encode('utf-8')) for name in inventory) + return raw + raw // 100 + headers + 1024 * 1024 + + +def _layout(path, members, central_offset): + """Account for every archive byte; permit only this writer's ZIP64 layout.""" + positions = {}; cursor = 0; central_parts = [] + with path.open('rb') as stream: + for item in members: + encoded = item.filename.encode('utf-8' if item.flag_bits == 0x800 else 'cp437') + extra = struct.pack(' expected['bytes'] or decoder.unused_data: + raise ValueError('Cached ZIP member expands beyond its budget or contains trailing compressed data.') + digest.update(raw); crc = zlib.crc32(raw, crc) + chunk = decoder.unconsumed_tail + if not chunk and len(raw) < 1024 * 1024: + break + if (not decoder.eof or crc != item.CRC + or dict(bytes=size, sha256=digest.hexdigest()) != expected): + raise ValueError('Cached ZIP member bytes differ from final evidence.') + + +def validate_archive(path, inventory, prefix): + """Verify every member against final source identities, never a cache receipt.""" + path = evidence.plain(path) + expected = {prefix + '/' + name: record for name, record in inventory.items()} + key(inventory, prefix) + raw_total = sum(row['bytes'] for row in expected.values()) + if not 2 <= len(expected) <= 9001 or raw_total > evidence.MAX_EPISODE: + raise ValueError('Cached episode exceeds the bounded file or byte budget.') + before = evidence.identity(path, maximum=evidence.MAX_EPISODE) + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + if (len(members) != len(expected) or [item.filename for item in members] != sorted(expected)): + raise ValueError('Cached ZIP member roster differs from final evidence.') + if archive.comment: + raise ValueError('Cached ZIP archive comments are not reviewed evidence.') + for item in members: + evidence.relative(item.filename) + record = expected[item.filename] + if (item.file_size != record['bytes'] or item.compress_type != zipfile.ZIP_DEFLATED + or item.flag_bits not in (0, 0x800) or item.is_dir() or item.create_system != 3 + or item.create_version != 45 or item.extract_version != 45 or item.reserved != 0 + or item.comment or item.extra or item.internal_attr != 0 or item.volume != 0 + or item.external_attr != (stat.S_IFREG | 0o644) << 16 + or item.date_time != (1980, 1, 1, 0, 0, 0)): + raise ValueError('Cached ZIP member type, size or format differs.') + positions = _layout(path, members, archive.start_dir) + with path.open('rb') as stream: + for item in members: + _member_identity(stream, positions[item.filename], item, expected[item.filename]) + if evidence.identity(path, maximum=evidence.MAX_EPISODE) != before: + raise ValueError('Cached ZIP changed during verification.') + return dict(**before, files=len(expected), unpacked_bytes=raw_total, + compression_level=None, compression_method='DEFLATE', + compression_level_scope='Not inferable from a verified DEFLATE stream; cache receipt is not trusted.', + inventory_sha256=sha256(evidence.canonical(expected)).hexdigest()) + + +def validate_sources(root, inventory): + """Recheck all original bytes even if cached archive members match a manifest.""" + if evidence.episode_inventory(root) != inventory: + raise ValueError('Final source inventory differs from the cache lookup inventory.') + for name, record in inventory.items(): + path = evidence.plain(Path(root) / name) + if evidence.identity(path, secret_scan=path.suffix in evidence.TEXT_SUFFIXES) != record: + raise ValueError('Original episode bytes differ from final evidence.') + + +def _private_copy(source, destination, record): + # Isolate the existing publication helper's historical flat imports from + # this process. Calling freeze_file performs no credential lookup or upload. + helper = Path(__file__).resolve().parent / 'huggingface' + script = """import json, sys +sys.path.insert(0, sys.argv[1]) +from publish_camera_action_v01 import freeze_file +method = freeze_file(sys.argv[2], sys.argv[3], {'bytes': int(sys.argv[4]), 'sha256': sys.argv[5]}) +print(json.dumps({'method': method})) +""" + result = subprocess.run([sys.executable, '-I', '-c', script, str(helper), str(source), + str(destination), str(record['bytes']), record['sha256']], capture_output=True, text=True, timeout=300) + if result.returncode != 0: + raise ValueError('Private cached-archive copy failed; source evidence remains unchanged.') + answer = json.loads(result.stdout) + if answer.get('method') not in ('apfs_clone', 'stream_copy'): + raise ValueError('Unknown private cached-archive copy method.') + + +def reuse(root, inventory, destination, *, prefix, cache): + """Return normal archive metadata on a fully checked hit, or None on a miss.""" + root, destination, cache = (evidence.plain(p).resolve() for p in (root, destination, cache)) + if (destination.exists() or destination.is_relative_to(root) or cache.is_relative_to(root) + or root.is_relative_to(cache) or destination.is_relative_to(cache)): + raise ValueError('Cache and fresh output must be outside the original episode.') + directory = evidence.plain(cache / key(inventory, prefix)) + if not directory.exists(): + return None + archive = evidence.plain(directory / 'episode.zip') + validate_sources(root, inventory) + checked = validate_archive(archive, inventory, prefix) + _private_copy(archive, destination, {k: checked[k] for k in ('bytes', 'sha256')}) + copied = validate_archive(destination, inventory, prefix) + if copied != checked: + raise ValueError('Private cached archive differs from the verified cache bytes.') + return copied + + +def candidates(study): + """Take a finite snapshot of registered, sealed stage records only.""" + study = evidence.plain(study) + protocol = evidence.read(study / 'protocol.json') + protocol_hash = evidence.identity(study / 'protocol.json')['sha256'] + if evidence.read(study / 'registration.json')['protocol_sha256'] != protocol_hash: + raise ValueError('Study protocol differs from registration.') + rosters = protocol['rosters'] + if set(rosters) != set(STAGES) or sum(len(rows) for rows in rosters.values()) > MAX_ATTEMPTS: + raise ValueError('Only the bounded registered camera/action stage roster is accepted.') + result = []; sealed_raw_bytes = 0 + for stage in STAGES: + path = study / 'stages' / (stage + '.json') + seal = path.with_name(stage + '.integrity.json') + if not path.exists() or not seal.exists(): + continue # an active stage has not sealed; do not inspect its episodes + receipt = evidence.read(path) + if evidence.read(seal) != dict(sha256=evidence.identity(path)['sha256'], protocol_sha256=protocol_hash): + raise ValueError('Sealed stage identity changed.') + if (receipt.get('schema') != 'dvidia.camera-action-study/v1/stage' + or receipt.get('stage') != stage or receipt.get('protocol_sha256') != protocol_hash + or receipt.get('status') not in ('complete', 'blocked') + or receipt.get('scheduled') != len(rosters[stage])): + raise ValueError('Sealed stage registration differs from the protocol.') + roster = {row['id']: row for row in rosters[stage]} + rows = receipt['rows'] + if len(roster) != len(rosters[stage]) or len(rows) != len(roster) or {r['id'] for r in rows} != set(roster): + raise ValueError('Sealed stage differs from its complete registered roster.') + for row in rows: + registered = roster[row['id']] + if any(row.get(k) != registered.get(k) for k in ('case', 'controller', 'negative_control')): + raise ValueError('Episode identity differs from its registered scene/controller.') + if row.get('status') in ('unknown', 'unattempted'): + continue + prefix = f'episodes/{stage}/{row["id"]}' + evidence.relative(prefix) + if row.get('path') != prefix: + raise ValueError('Episode path differs from the registered attempt.') + episode = evidence.plain(study / prefix) + if (evidence.identity(episode / 'integrity.json')['sha256'] != row['integrity_sha256'] + or evidence.identity(episode / 'rollout.json')['sha256'] != row['receipt_sha256']): + raise ValueError('Episode receipt differs from its sealed stage.') + inventory = evidence.episode_inventory(episode) + sealed_raw_bytes += sum(item['bytes'] for item in inventory.values()) + rollout = evidence.read(episode / 'rollout.json') + if rollout['status'] != row['status']: + raise ValueError('Episode completion differs from its sealed stage.') + if (rollout['status'] == 'complete' and rollout.get('recorded_complete_intervals') == 1000 + and row.get('complete') is True and row.get('recorded_complete_intervals') == 1000): + result.append(dict(root=episode, prefix=prefix, inventory=inventory)) + maximum = protocol.get('storage', {}).get('maximum_study_bytes') + if type(maximum) is not int or maximum != evidence.MAX_TOTAL or sealed_raw_bytes > maximum: + raise ValueError('Study storage cap differs from the registered 120 GiB budget.') + return result, maximum - sealed_raw_bytes + + +def precompress(study, cache, *, limit=1): + study, cache = (evidence.plain(p).resolve() for p in (study, cache)) + if type(limit) is not int or not 1 <= limit <= MAX_ATTEMPTS: + raise ValueError('Choose a finite episode limit between 1 and 452.') + if cache == study or cache.is_relative_to(study) or study.is_relative_to(cache): + raise ValueError('Use a separate cache outside the study tree.') + cache.mkdir(parents=True, exist_ok=True) + lock = cache / 'worker.lock' + descriptor = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + created = []; existing = 0 + try: + with os.fdopen(descriptor, 'w') as stream: + json.dump(dict(pid=os.getpid(), operation='publication_archive_cache'), stream) + roster, remaining = candidates(study) + for item in roster: + if len(created) >= limit: + break + directory = evidence.plain(cache / key(item['inventory'], item['prefix'])) + if directory.exists(): + existing += 1 + continue # no trust claim; final packaging validates the ZIP itself + archive_bytes_reserved = archive_headroom(item['inventory'], item['prefix']) + # The original native worker may continue writing. Reserving its + # entire remaining registered cap is intentionally conservative. + if shutil.disk_usage(cache).free - remaining - archive_bytes_reserved < evidence.DISK_RESERVE: + raise ValueError('Cache would consume future registered recording space or the 30 GiB reserve.') + directory.mkdir(exist_ok=False) + receipt = evidence.archive(item['root'], item['inventory'], directory / 'episode.zip', prefix=item['prefix']) + evidence.write(directory / 'receipt.json', dict(schema=SCHEMA, prefix=item['prefix'], + cache_key=directory.name, archive=receipt, advisory_only=True)) + created.append(dict(prefix=item['prefix'], cache_key=directory.name, archive_bytes=receipt['bytes'])) + return dict(schema=SCHEMA, status='complete', created=created, existing_entries_skipped=existing, + eligible_episodes=len(roster), remaining_registered_raw_bytes_reserved=remaining, + physics_executed=False, qualification_changed=False) + finally: + lock.unlink(missing_ok=True) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study', required=True); parser.add_argument('--cache', required=True) + parser.add_argument('--limit', type=int, default=1) + args = parser.parse_args(argv) + print(json.dumps(precompress(args.study, args.cache, limit=args.limit), indent=2)) + + +if __name__ == '__main__': + main() diff --git a/publishing/huggingface/camera_action_hardware.py b/publishing/huggingface/camera_action_hardware.py new file mode 100644 index 0000000..511dfd9 --- /dev/null +++ b/publishing/huggingface/camera_action_hardware.py @@ -0,0 +1,32 @@ +"""Small public hardware context; hostnames, serials and arbitrary fields excluded.""" +import re + +MAX_BYTES = 16 * 1024 +FIELDS = {'scope', 'system', 'release', 'machine', 'python', 'versions', 'hardware', 'native_workers', 'gpu_use'} +HARDWARE = {'hw.model', 'hw.memsize', 'hw.physicalcpu', 'hw.logicalcpu', 'machdep.cpu.brand_string'} + + +def validate_hardware(value): + if type(value) is not dict or set(value) != FIELDS: + raise ValueError('Hardware context must use the public field whitelist; no hostname, serial or account fields.') + for key in ('scope', 'system', 'release', 'machine', 'python', 'gpu_use'): + text=value[key] + if type(text) is not str or not 1 <= len(text) <= (1024 if key in ('scope','gpu_use') else 80) or any(ord(c)<32 for c in text): + raise ValueError('Hardware context contains invalid or unbounded text.') + if type(value['native_workers']) is not int or not 1 <= value['native_workers'] <= 1024: + raise ValueError('Hardware context requires a bounded native worker count.') + versions=value['versions']; hardware=value['hardware'] + if type(versions) is not dict or set(versions) != {'numpy','mujoco'} or type(hardware) is not dict or set(hardware) != HARDWARE: + raise ValueError('Hardware/software details must use the public field whitelist.') + for version in [value['python'],*versions.values()]: + if type(version) is not str or not re.fullmatch(r'[0-9][0-9A-Za-z.+_-]{0,79}',version): + raise ValueError('Software version must be a plain bounded version string.') + for key in ('hw.memsize','hw.physicalcpu','hw.logicalcpu'): + number=hardware[key] + if type(number) is not str or not re.fullmatch(r'[1-9][0-9]{0,18}',number) or int(number)>(2**60 if key=='hw.memsize' else 4096): + raise ValueError('Hardware memory and CPU counts must be positive bounded decimal strings.') + for key in ('hw.model','machdep.cpu.brand_string'): + text=hardware[key] + if type(text) is not str or not 1<=len(text)<=120 or any(ord(c)<32 for c in text) or any(c in text for c in ('/','\\','@')): + raise ValueError('Hardware product name must be a bounded public model identifier.') + return value diff --git a/publishing/huggingface/camera_action_model_cards.py b/publishing/huggingface/camera_action_model_cards.py new file mode 100644 index 0000000..eaf5bdb --- /dev/null +++ b/publishing/huggingface/camera_action_model_cards.py @@ -0,0 +1,94 @@ +"""Pure, shared model-card receipt contract; no files, credentials or network.""" +from __future__ import annotations + +import re + +SCHEMA = 'dvidia.camera-action-cards/v1' +RECEIPT_FILE = 'model-cards-receipt.json' +MAX_CARD_BYTES = 256 * 1024 +MAX_RECEIPT_BYTES = 2 * 1024 * 1024 +MODEL = re.compile(r'models/(initial|corrected)/(full|blind)/model\.json\Z') +SOURCE_REQUIRED = {'summary.json', 'generator.py', 'summarize_results.py'} +SOURCE_OPTIONAL = {'hardware.json', 'full-study-audit.json', 'diagnostic-phase-coverage.json', + 'independent-audit.json', 'diagnostic_phase_coverage.py'} +FIELDS = {'schema', 'source_commit', 'protocol_sha256', 'report_sha256', 'status', + 'qualification', 'qualified', 'full_study_audited', 'physical_robot_ready', + 'summary_reconstructed', 'physics_executed', 'model_executed', + 'source_inputs', 'study_bindings', 'files', 'model_cards'} + + +def file_identity(value, maximum=1024 ** 3): + if (type(value) is not dict or set(value) != {'bytes', 'sha256'} + or type(value['bytes']) is not int or not 0 <= value['bytes'] <= maximum + or type(value['sha256']) is not str or not re.fullmatch('[a-f0-9]{64}', value['sha256'])): + raise ValueError('Malformed model-card file identity.') + return value + + +def card_name(model): + match = MODEL.fullmatch(model) + if match is None: + raise ValueError('Unsupported model-card artifact path.') + return f'model-cards/{match[1]}/{match[2]}/README.md' + + +def validate_receipt(receipt, *, source_commit, protocol_sha256, report_sha256, report, models): + """Bind the exact card/model roster and qualification to supplied sealed facts.""" + if (type(receipt) is not dict or set(receipt) != FIELDS or receipt['schema'] != SCHEMA + or not re.fullmatch('[a-f0-9]{40}', source_commit or '') + or receipt['source_commit'] != source_commit + or receipt['protocol_sha256'] != protocol_sha256 or receipt['report_sha256'] != report_sha256 + or any(receipt[key] != report[key] for key in ('status', 'qualification', 'qualified')) + or type(receipt['qualified']) is not bool or type(receipt['full_study_audited']) is not bool + or any(receipt[key] is not value for key, value in ( + ('physical_robot_ready', False), ('summary_reconstructed', True), + ('physics_executed', False), ('model_executed', False)))): + raise ValueError('Model-card receipt identity or qualification differs from sealed evidence.') + if len(models) > 4: + raise ValueError('Model-card roster exceeds the registered four candidates.') + expected = [] + for model, identity in sorted(models.items()): + file_identity(identity) + match = MODEL.fullmatch(model) + card = card_name(model) + expected.append(dict(card=card, model=model, model_sha256=identity['sha256'], + selected=match[1] == report.get('selected_stage'))) + stages = {MODEL.fullmatch(name)[1] for name in models} + if (any({name for name in models if name.startswith('models/'+stage+'/')} != + {f'models/{stage}/{variant}/model.json' for variant in ('full', 'blind')} for stage in stages) + or report.get('selected_stage') is not None and report['selected_stage'] not in stages): + raise ValueError('Model-card roster lacks its matched or selected model pair.') + records = receipt['model_cards'] + if (type(records) is not list or any(type(row) is not dict or set(row) != + {'card', 'model', 'model_sha256', 'selected'} or type(row['selected']) is not bool for row in records) + or sorted(records, key=lambda row: row['model']) != expected): + raise ValueError('Model cards differ from the exact sealed model roster or selection.') + if type(receipt['files']) is not dict or set(receipt['files']) != {'README.md'} | {r['card'] for r in expected}: + raise ValueError('Generated model-card file roster differs.') + for value in receipt['files'].values(): + file_identity(value, MAX_CARD_BYTES) + sources = receipt['source_inputs'] + if (type(sources) is not dict or not SOURCE_REQUIRED <= set(sources) + or set(sources) - SOURCE_REQUIRED - SOURCE_OPTIONAL + or receipt['full_study_audited'] is not ('full-study-audit.json' in sources)): + raise ValueError('Model-card source input roster differs.') + phase = {'diagnostic-phase-coverage.json', 'diagnostic_phase_coverage.py'} + if set(sources) & phase and not phase | {'independent-audit.json'} <= set(sources): + raise ValueError('Model-card phase provenance is incomplete.') + for value in sources.values(): + file_identity(value) + bindings = receipt['study_bindings'] + if type(bindings) is not dict or not 3 <= len(bindings) <= 80: + raise ValueError('Model-card study binding roster is invalid.') + for name, value in bindings.items(): + if not (name in {'protocol.json', 'registration.json', 'report.json', 'freeze.json'} + or re.fullmatch(r'stages/(diagnostic|demonstrations|train-initial|development-initial|corrections|train-corrected|development-corrected|confirmation|negative)(\.integrity|\.started)?\.json', name) + or re.fullmatch(r'models/(initial|corrected)/(full|blind)/(model|receipt|started)\.json', name)): + raise ValueError('Unexpected model-card study binding path.') + file_identity(value) + if (bindings.get('protocol.json', {}).get('sha256') != protocol_sha256 + or bindings.get('report.json', {}).get('sha256') != report_sha256 + or 'registration.json' not in bindings + or any(bindings.get(name) != identity for name, identity in models.items())): + raise ValueError('Model-card study bindings differ from public model or report identities.') + return [row['card'] for row in expected] diff --git a/publishing/huggingface/publish_camera_action_v01.py b/publishing/huggingface/publish_camera_action_v01.py new file mode 100644 index 0000000..d8b0782 --- /dev/null +++ b/publishing/huggingface/publish_camera_action_v01.py @@ -0,0 +1,424 @@ +"""Publish one reviewed, byte-bound camera/action evidence dataset. + +Stage and verify never read credentials. Publish copies the reviewed payload +into a private temporary directory before Keychain access, then makes one +parent-bound commit to the named empty public dataset. No Space is modified. +""" +from __future__ import annotations + +import argparse +import ctypes +import errno +from hashlib import sha256 +import json +import os +from pathlib import Path +import re +import shutil +import stat +import sys +import tempfile + +from publish import PublicationError, client +from publish_task_environments_v02 import encoded, plain, public_client, relative, scan +from camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware +from camera_action_model_cards import RECEIPT_FILE, MAX_RECEIPT_BYTES, validate_receipt + +REPO = 'Dvidia/piper-camera-action-v01' +SCHEMA = 'dvidia.camera-action-publication/v1' +EVIDENCE_SCHEMA = 'dvidia.camera-action-evidence/v1' +MAX_JSON = 32 * 1024 ** 2 +MAX_FILE = 1024 ** 3 +MAX_TOTAL = 120 * 1024 ** 3 +RESERVE = 30 * 1024 ** 3 +OID = re.compile(r'(?:[a-f0-9]{40}|[a-f0-9]{64})\Z') +REQUIRED = {'README.md', 'LICENSE', 'manifest.json', 'SHA256SUMS', 'report.json', + 'protocol.json', 'registration.json', 'independent-audit.json', 'study-metadata.zip'} + + +def identity(path, *, maximum=MAX_FILE): + path = plain(path) + with path.open('rb') as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= maximum: + raise PublicationError('Publication file is missing, nonregular or oversized.') + h, count, tail = sha256(), 0, b'' + for raw in iter(lambda: stream.read(1024 * 1024), b''): + count += len(raw) + if count > maximum: + raise PublicationError('Publication payload grew beyond its bound.') + h.update(raw) + scan(tail + raw) + tail = raw[-256:] + after = os.fstat(stream.fileno()) + if count != before.st_size or any(getattr(before, key) != getattr(after, key) + for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise PublicationError('Publication bytes changed during inspection.') + return dict(bytes=count, sha256=h.hexdigest()) + + +def read(path): + path = plain(path) + identity(path, maximum=MAX_JSON) + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise PublicationError('Duplicate JSON field in reviewed publication artifact.') + result[key] = value + return result + def invalid(_value): + raise PublicationError('Nonfinite reviewed publication artifact.') + try: + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + except (UnicodeError, ValueError, TypeError) as exc: + raise PublicationError('Malformed reviewed publication artifact.') from exc + + +def allowed(name): + relative(name) + return (name in REQUIRED or name in ('diagnostic-phase-coverage.json', 'full-study-audit.json', 'hardware.json', RECEIPT_FILE) + or re.fullmatch(r'episodes/[a-z-]+/[a-z0-9-]+\.zip', name) is not None + or re.fullmatch(r'models/(initial|corrected)/(full|blind)/model\.json', name) is not None + or re.fullmatch(r'model-cards/(initial|corrected)/(full|blind)/README\.md', name) is not None + or re.fullmatch(r'source/[A-Za-z0-9_.+-]+\.(whl|tar\.gz)', name) is not None) + + +def _record(value): + if (not isinstance(value, dict) or set(value) != {'bytes', 'sha256'} + or type(value['bytes']) is not int or not 0 <= value['bytes'] <= MAX_FILE + or not isinstance(value['sha256'], str) or not re.fullmatch('[a-f0-9]{64}', value['sha256'])): + raise PublicationError('Malformed reviewed file identity.') + return value + + +def files(root): + root = plain(root) + manifest = read(root / 'manifest.json') + if (manifest.get('schema') != EVIDENCE_SCHEMA or manifest.get('repo_id') != REPO + or manifest.get('repo_type') != 'dataset' or manifest.get('physical_robot_ready') is not False + or manifest.get('original_footage_included') is not False + or manifest.get('status') not in ('blocked', 'complete', 'incomplete') + or manifest.get('pipeline_status', manifest.get('status')) not in ('blocked', 'complete') + or type(manifest.get('qualified')) is not bool): + raise PublicationError('Expected the reviewed simulation-only camera/action evidence manifest.') + expected = manifest.get('files') + if not isinstance(expected, dict) or not REQUIRED - {'manifest.json', 'SHA256SUMS'} <= set(expected): + raise PublicationError('Required evidence files are missing.') + if any(not allowed(name) or name in ('manifest.json', 'SHA256SUMS') for name in expected): + raise PublicationError('Unrecognized public evidence path.') + if manifest.get('hardware_context_file') != ('hardware.json' if 'hardware.json' in expected else None): + raise PublicationError('Hardware context file differs from its manifest binding.') + if 'hardware.json' in expected: + identity(root / 'hardware.json', maximum=MAX_HARDWARE_BYTES) + try: + validate_hardware(read(root / 'hardware.json')) + except ValueError as exc: + raise PublicationError('Hardware context contains unknown, private or unbounded fields.') from exc + observed = [] + for path in sorted(root.rglob('*')): + plain(path) + if not (path.is_file() or path.is_dir()): + raise PublicationError('Public evidence contains a special file.') + if path.is_file(): + observed.append(path.relative_to(root).as_posix()) + if not 10 <= len(observed) <= 1024 or set(observed) != set(expected) | {'manifest.json', 'SHA256SUMS'}: + raise PublicationError('Public file roster differs from the reviewed manifest.') + # Match the writer's POSIX-name order, including model-cards/ versus + # model-cards-receipt.json (Path component ordering differs here). + observed.sort() + source_files = [name for name in expected if name.startswith('source/')] + if len(source_files) != 2 or sum(name.endswith('.whl') for name in source_files) != 1 or sum(name.endswith('.tar.gz') for name in source_files) != 1: + raise PublicationError('Exact source archive and installable wheel are required.') + episodes = manifest.get('episodes') + if not isinstance(episodes, list) or len(episodes) != manifest.get('episode_count'): + raise PublicationError('Episode shard roster is malformed.') + if {r.get('archive') for r in episodes} != {name for name in expected if name.startswith('episodes/')}: + raise PublicationError('Episode archives differ from the declared roster.') + if set(manifest.get('trained_model_files', [])) != {name for name in expected if name.startswith('models/')}: + raise PublicationError('Model artifacts differ from the actual training roster.') + report = read(root / 'report.json') + if (report.get('qualification') != manifest.get('qualification') or report.get('qualified') is not manifest['qualified'] + or report.get('physical_robot_ready') is not False or report.get('status') != manifest['status']): + raise PublicationError('Public report and qualification manifest disagree.') + card_files = {name for name in expected if name.startswith('model-cards/')} + declared_cards = manifest.get('model_card_files', []) + card_receipt = manifest.get('model_card_receipt_file') + if card_receipt is None: + if declared_cards or card_files or RECEIPT_FILE in expected: + raise PublicationError('Model cards require their explicit receipt and manifest bindings.') + else: + if card_receipt != RECEIPT_FILE or RECEIPT_FILE not in expected: + raise PublicationError('Model-card receipt path differs from its manifest binding.') + identity(root / RECEIPT_FILE, maximum=MAX_RECEIPT_BYTES) + receipt = read(root / RECEIPT_FILE) + try: + required_cards = validate_receipt(receipt, source_commit=manifest.get('source_commit'), + protocol_sha256=expected['protocol.json']['sha256'], report_sha256=expected['report.json']['sha256'], + report=report, models={name: expected[name] for name in manifest['trained_model_files']}) + except (ValueError, KeyError, TypeError) as exc: + raise PublicationError('Model-card receipt differs from the public models or report.') from exc + if (declared_cards != required_cards or card_files != set(required_cards) + or any(receipt['files'][name] != expected[name] for name in ['README.md', *required_cards]) + or any(receipt['study_bindings'][name] != expected[name] + for name in ('protocol.json', 'registration.json', 'report.json')) + or any(receipt['source_inputs'].get(name) != expected.get(name) + for name in ('hardware.json', 'full-study-audit.json', 'diagnostic-phase-coverage.json')) + or ('independent-audit.json' in receipt['source_inputs'] and + receipt['source_inputs']['independent-audit.json'] != expected.get('independent-audit.json')) + or receipt['full_study_audited'] is not ('full-study-audit.json' in expected)): + raise PublicationError('Model-card files or provenance differ from the publication manifest.') + if manifest['qualified']: + if 'full-study-audit.json' not in expected: + raise PublicationError('A qualified skill requires the full-study independent audit.') + proof = read(root / 'full-study-audit.json') + if (proof.get('schema') != 'dvidia.camera-action-study-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or (proof.get('qualification') or {}).get('qualified') is not True): + raise PublicationError('Full-study audit does not support the qualified-skill label.') + inventory = [] + for name in observed: + record = identity(root / name) + if name in expected and record != _record(expected[name]): + raise PublicationError('Payload bytes differ from the reviewed evidence manifest.') + inventory.append(dict(path=name, **record, + representation='lfs' if name.endswith(('.zip', '.gz', '.whl')) else 'git')) + if sum(row['bytes'] for row in inventory) > MAX_TOTAL: + raise PublicationError('Public evidence exceeds the bounded dataset size.') + sums = ''.join(f'{row["sha256"]} {row["path"]}\n' for row in inventory if row['path'] != 'SHA256SUMS') + if (root / 'SHA256SUMS').read_text() != sums: + raise PublicationError('Published checksum list does not match all payload bytes.') + return inventory + + +def stage(root, plan): + root, plan = plain(root), plain(plan) + value = dict(schema=SCHEMA, repo_id=REPO, repo_type='dataset', private=False, + root=str(root), files=files(root)) + with plan.open('xb') as stream: + stream.write(encoded(value)) + return dict(plan_sha256=identity(plan, maximum=MAX_JSON)['sha256'], files=len(value['files']), + bytes=sum(r['bytes'] for r in value['files'])) + + +def checked(plan, expected_digest): + if identity(plan, maximum=MAX_JSON)['sha256'] != expected_digest: + raise PublicationError('Reviewed publication plan digest changed.') + value = read(plan) + if (set(value) != {'schema', 'repo_id', 'repo_type', 'private', 'root', 'files'} + or value['schema'] != SCHEMA or value['repo_id'] != REPO or value['repo_type'] != 'dataset' + or value['private'] is not False or type(value['root']) is not str or type(value['files']) is not list): + raise PublicationError('Reviewed publication destination or scope changed.') + if files(value['root']) != value['files']: + raise PublicationError('Reviewed publication inventory changed.') + return value + + +def _remote(info): + if (getattr(info, 'private', None) is not False or not OID.fullmatch(getattr(info, 'sha', '') or '') + or getattr(info, 'siblings', None) is None): + raise PublicationError('Expected an exact initialized public dataset head.') + + +def _space_guard(directory, additional=0): + if shutil.disk_usage(directory).free - additional < RESERVE: + raise PublicationError('Freezing the publication would consume the protected disk reserve.') + + +def _clone_from_fd(source_fd, directory_fd, name): + """Darwin clonefile(2): new inode/attributes; subsequent writes are private. + + Source and destination directory are already open without following links. + No hardlink fallback is permitted. Local SDK clonefile.h defines the + CLONE_NOOWNERCOPY flag as 0x0002; source is a regular-file descriptor. + """ + if sys.platform != 'darwin': + return False + library = ctypes.CDLL(None, use_errno=True) + try: + clone = library.fclonefileat + except AttributeError: + return False + clone.argtypes = (ctypes.c_int, ctypes.c_int, ctypes.c_char_p, ctypes.c_uint32) + clone.restype = ctypes.c_int + if clone(source_fd, directory_fd, os.fsencode(name), 0x0002) == 0: + return True + number = ctypes.get_errno() + if number in {errno.ENOTSUP, errno.EOPNOTSUPP, errno.EXDEV, errno.ENOSYS}: + return False + raise OSError(number, 'Private copy-on-write freeze failed.') + + +def freeze_file(source, target, expected): + """Create and verify one private snapshot, with bounded streaming fallback.""" + source, target = plain(source), plain(target) + _record(expected) + if target.exists(): + raise PublicationError('Private freeze destination already exists.') + target.parent.mkdir(parents=True, exist_ok=True) + plain(target.parent) + _space_guard(target.parent, 1024 * 1024) # metadata headroom; no full-data reservation for a clone + created = False + source_fd = directory_fd = None + try: + source_fd = os.open(source, os.O_RDONLY | os.O_NOFOLLOW) + directory_fd = os.open(target.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + before = os.fstat(source_fd) + if not stat.S_ISREG(before.st_mode) or before.st_size != expected['bytes']: + raise PublicationError('Freeze source is not the reviewed ordinary file.') + cloned = _clone_from_fd(source_fd, directory_fd, target.name) + if cloned: + created = True + else: + _space_guard(target.parent, expected['bytes']) + fd = os.open(target.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600, dir_fd=directory_fd) + created = True + with os.fdopen(fd, 'wb') as destination: + copied = 0 + while True: + chunk = os.read(source_fd, 1024 * 1024) + if not chunk: break + copied += len(chunk) + if copied > expected['bytes']: + raise PublicationError('Freeze source grew beyond its reviewed byte count.') + _space_guard(target.parent, len(chunk)) + destination.write(chunk) + destination.flush() + _space_guard(target.parent) + if copied != expected['bytes']: + raise PublicationError('Freeze source length changed while copying.') + os.fsync(destination.fileno()) + after = os.fstat(source_fd) + if any(getattr(before, k) != getattr(after, k) for k in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise PublicationError('Source changed while freezing reviewed publication bytes.') + frozen_fd = os.open(target.name, os.O_RDONLY | os.O_NOFOLLOW, dir_fd=directory_fd) + try: + frozen_stat = os.fstat(frozen_fd) + if (not stat.S_ISREG(frozen_stat.st_mode) or frozen_stat.st_nlink != 1 + or (frozen_stat.st_dev, frozen_stat.st_ino) == (before.st_dev, before.st_ino)): + raise PublicationError('Freeze must create an independent ordinary inode, never a hardlink.') + os.fchmod(frozen_fd, 0o400) + finally: + os.close(frozen_fd) + _space_guard(target.parent) + if identity(target) != expected: + raise PublicationError('Payload changed while freezing reviewed publication bytes.') + return 'apfs_clone' if cloned else 'stream_copy' + except BaseException: + if created: + os.unlink(target.name, dir_fd=directory_fd) + raise + finally: + if source_fd is not None: os.close(source_fd) + if directory_fd is not None: os.close(directory_fd) + + +def publish(plan, expected_digest, cache): + value = checked(plan, expected_digest) + cache = plain(cache) + cache.mkdir(parents=True, exist_ok=True) + _space_guard(cache, 1024 * 1024) + with tempfile.TemporaryDirectory(prefix='camera-action-frozen-', dir=cache) as temporary: + frozen = Path(temporary) + freeze_methods = dict(apfs_clone=0, stream_copy=0) + for row in value['files']: + source = plain(Path(value['root']) / row['path']) + target = frozen / relative(row['path']) + method = freeze_file(source, target, {key: row[key] for key in ('bytes', 'sha256')}) + freeze_methods[method] += 1 + # Originals are never reopened after the credential boundary. The + # private immutable copy bounds memory while preserving reviewed bytes. + api, token = client(cache) + from huggingface_hub import CommitOperationAdd + from huggingface_hub.errors import RepositoryNotFoundError + try: + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + except RepositoryNotFoundError: + api.create_repo(REPO, repo_type='dataset', private=False, exist_ok=False) + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + _remote(info) + if len(info.siblings) != 1 or {item.rfilename for item in info.siblings} != {'.gitattributes'}: + raise PublicationError('Dataset already contains content; verify it instead of overwriting it.') + operations = [CommitOperationAdd(path_in_repo=row['path'], path_or_fileobj=str(frozen / row['path'])) + for row in value['files']] + commit = api.create_commit(REPO, repo_type='dataset', operations=operations, + parent_commit=info.sha, commit_message='Publish camera-to-action v0.1 reproducible simulation evidence') + del token, api + return dict(repo_id=REPO, commit=commit.oid, files_added=len(operations), freeze_methods=freeze_methods) + + +def _http_readback(): + # Hub constants are initialized at import time. Fail closed if a caller + # pre-imported an SDK that would route data into a separate Xet cache. + os.environ.update(HF_HUB_DISABLE_XET='1', HF_DEBUG='0', HF_HUB_DISABLE_TELEMETRY='1', + HF_HUB_DISABLE_IMPLICIT_TOKEN='1', HF_HUB_DISABLE_PROGRESS_BARS='1', HF_HUB_DISABLE_UPDATE_CHECK='1') + from huggingface_hub import constants + if getattr(constants, 'HF_HUB_DISABLE_XET', None) is not True: + raise PublicationError('Start verification in a fresh process with HF_HUB_DISABLE_XET=1 before SDK import.') + + +def download(name, revision, cache): + _http_readback() + from huggingface_hub import hf_hub_download + return Path(hf_hub_download(REPO, name, repo_type='dataset', revision=revision, + token=False, endpoint='https://huggingface.co', cache_dir=str(cache), force_download=True)).resolve() + + +def verify(plan, expected_digest, cache): + value = checked(plan, expected_digest) + cache = plain(cache) + cache.mkdir(parents=True, exist_ok=True) + _http_readback() + api = public_client() + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True, token=False) + _remote(info) + expected = {row['path'] for row in value['files']} + if len(info.siblings) != len(expected) + 1 or {item.rfilename for item in info.siblings} != expected | {'.gitattributes'}: + raise PublicationError('Public dataset roster differs from the reviewed plan.') + remote_files = {item.rfilename: item for item in info.siblings} + for row in value['files']: + size = getattr(remote_files[row['path']], 'size', None) + if type(size) is not int or size != row['bytes']: + raise PublicationError('Revision-pinned remote size differs from the reviewed per-file download budget.') + for row in value['files']: + _space_guard(cache, row['bytes'] + 1024 * 1024) + with tempfile.TemporaryDirectory(prefix='camera-action-readback-', dir=cache) as temporary: + owned = Path(temporary) + path = download(row['path'], info.sha, owned) + if not path.resolve().is_relative_to(owned.resolve()): + raise PublicationError('Anonymous readback escaped its owned temporary cache.') + if identity(path) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Anonymous public bytes differ from reviewed evidence.') + _space_guard(cache) + return dict(repo_id=REPO, commit=info.sha, anonymous_files_verified=len(value['files']), + anonymous_bytes_verified=sum(r['bytes'] for r in value['files']), + maximum_readback_file_bytes=max(r['bytes'] for r in value['files']), + reviewed_plan_sha256=expected_digest, + readback_inventory_sha256=sha256(encoded(value['files'])).hexdigest(), + cache_policy='One file per owned temporary cache, removed after full pinned-revision byte verification; shared caches untouched.') + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + create = commands.add_parser('stage') + create.add_argument('root') + create.add_argument('--plan', required=True) + for name in ('publish', 'verify'): + command = commands.add_parser(name) + command.add_argument('--plan', required=True) + command.add_argument('--sha256', required=True) + command.add_argument('--cache', required=True) + args = parser.parse_args(argv) + try: + result = stage(args.root, args.plan) if args.command == 'stage' else globals()[args.command](args.plan, args.sha256, args.cache) + print(json.dumps(result, indent=2)) + return 0 + except PublicationError as exc: + print(str(exc), file=sys.stderr) + except Exception: + print('Publication operation failed; no credential or remote response details displayed.', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/huggingface/test_camera_action_hardware.py b/publishing/huggingface/test_camera_action_hardware.py new file mode 100644 index 0000000..5c0a123 --- /dev/null +++ b/publishing/huggingface/test_camera_action_hardware.py @@ -0,0 +1,33 @@ +from copy import deepcopy +import unittest + +from camera_action_hardware import validate_hardware + + +def fixture(): + return dict(scope='Shared host; offline simulator timings only.', system='Darwin', release='25.5.0', + machine='arm64', python='3.12.9', versions={'numpy':'2.5.3','mujoco':'3.15.0'}, + hardware={'hw.model':'Mac17,4','hw.memsize':'25769803776','hw.physicalcpu':'10', + 'hw.logicalcpu':'10','machdep.cpu.brand_string':'Apple M5'},native_workers=1, + gpu_use='Existing macOS CGL rendering; CPU policy and physics.') + + +class HardwareContextTests(unittest.TestCase): + def test_public_hardware_is_accepted_and_private_fields_rejected(self): + validate_hardware(fixture()) + for key in ('hostname','serial','account','machine_uuid'): + for nested in (False,True): + value=fixture(); target=value['hardware'] if nested else value;target[key]='private fixture' + with self.subTest(key=key,nested=nested),self.assertRaisesRegex(ValueError,'whitelist'): + validate_hardware(value) + + def test_text_worker_counts_and_numeric_hardware_are_bounded(self): + for key,invalid in (('scope','x'*1025),('python','/private/version'),('native_workers',True),('native_workers',0)): + value=fixture();value[key]=invalid + with self.subTest(key=key),self.assertRaises(ValueError):validate_hardware(value) + for key,invalid in (('hw.memsize','0'),('hw.physicalcpu','4097'),('hw.logicalcpu',10),('hw.model','/Users/private')): + value=fixture();value['hardware'][key]=invalid + with self.subTest(key=key),self.assertRaises(ValueError):validate_hardware(value) + + +if __name__=='__main__':unittest.main() diff --git a/publishing/huggingface/test_publish_camera_action_v01.py b/publishing/huggingface/test_publish_camera_action_v01.py new file mode 100644 index 0000000..c5b9244 --- /dev/null +++ b/publishing/huggingface/test_publish_camera_action_v01.py @@ -0,0 +1,281 @@ +from copy import deepcopy +from hashlib import sha256 +import json +import os +from pathlib import Path +import sys +import stat +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import Mock, patch + +import publish_camera_action_v01 as publisher + + +class CameraPublicationTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.dataset = self.root / 'dataset' + self.dataset.mkdir() + names = (publisher.REQUIRED - {'manifest.json', 'SHA256SUMS'}) | { + 'source/fixture.whl', 'source/fixture.tar.gz'} + for name in names: + path = self.dataset / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'Synthetic test evidence only.') + (self.dataset / 'report.json').write_text(json.dumps(dict( + qualification='research_candidate', qualified=False, physical_robot_ready=False, status='blocked'))) + files = {name: publisher.identity(self.dataset / name) for name in sorted(names)} + manifest = dict(schema=publisher.EVIDENCE_SCHEMA, repo_id=publisher.REPO, repo_type='dataset', + status='blocked', qualification='research_candidate', qualified=False, + original_footage_included=False, physical_robot_ready=False, + files=files, episodes=[], episode_count=0, trained_model_files=[]) + (self.dataset / 'manifest.json').write_text(json.dumps(manifest)) + files['manifest.json'] = publisher.identity(self.dataset / 'manifest.json') + (self.dataset / 'SHA256SUMS').write_text(''.join(f'{row["sha256"]} {name}\n' for name, row in sorted(files.items()))) + self.plan = self.root / 'plan.json' + self.receipt = publisher.stage(self.dataset, self.plan) + class Operation: + def __init__(self, path_in_repo, path_or_fileobj): + self.path_in_repo, self.path_or_fileobj = path_in_repo, path_or_fileobj + class NotFound(Exception): + pass + patches = patch.dict(sys.modules, {'huggingface_hub': SimpleNamespace(CommitOperationAdd=Operation, + constants=SimpleNamespace(HF_HUB_DISABLE_XET=True)), + 'huggingface_hub.errors': SimpleNamespace(RepositoryNotFoundError=NotFound)}) + patches.start() + self.addCleanup(patches.stop) + + def info(self, names=('.gitattributes',), *, private=False, sha='a' * 40): + return SimpleNamespace(private=private, sha=sha, + siblings=[SimpleNamespace(rfilename=name,size=(self.dataset/name).stat().st_size if (self.dataset/name).is_file() else 0) for name in names]) + + def downloaded(self,name,revision,cache,*,wrong=False): + self.assertEqual(revision,'a'*40) + target=Path(cache)/'synthetic-download'/name + target.parent.mkdir(parents=True,exist_ok=True) + target.write_bytes(b'not the published bytes' if wrong else (self.dataset/name).read_bytes()) + return target + + def test_stage_check_and_full_public_readback_never_access_credentials(self): + value = publisher.checked(self.plan, self.receipt['plan_sha256']) + api = Mock() + api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) + with patch.object(publisher, 'client', side_effect=AssertionError('Unexpected credential access')), \ + patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', side_effect=self.downloaded) as download: + proof = publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(proof['anonymous_files_verified'], len(value['files'])) + self.assertEqual(proof['anonymous_bytes_verified'],sum(r['bytes'] for r in value['files'])) + self.assertEqual(proof['reviewed_plan_sha256'],self.receipt['plan_sha256']) + self.assertEqual(len({call.args[2] for call in download.call_args_list}),len(value['files'])) + self.assertTrue(all(not Path(call.args[2]).exists() for call in download.call_args_list)) + self.assertEqual(list((self.root/'cache').iterdir()),[]) + api.repo_info.assert_called_once_with(publisher.REPO, repo_type='dataset', files_metadata=True, token=False) + + def test_mutated_payload_and_changed_destination_fail_before_keychain(self): + (self.dataset / 'README.md').write_bytes(b'changed') + with patch.object(publisher, 'client') as client: + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + client.assert_not_called() + value = json.loads(self.plan.read_bytes()) + value['repo_id'] = 'Dvidia/robot-simulation-lab' + self.plan.write_bytes(publisher.encoded(value)) + with self.assertRaisesRegex(publisher.PublicationError, 'destination'): + publisher.checked(self.plan, sha256(self.plan.read_bytes()).hexdigest()) + + def test_existing_private_or_nonempty_dataset_cannot_be_overwritten(self): + for info in (self.info(['README.md']), self.info(private=True), self.info(sha='main')): + api = Mock() + api.repo_info.return_value = info + with self.subTest(info=info), patch.object(publisher, 'client', return_value=(api, 'fixture')): + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + api.create_commit.assert_not_called() + + def test_payload_is_frozen_before_credential_boundary_and_uses_parent_cas(self): + api = Mock() + api.repo_info.return_value = self.info() + uploaded = {} + def commit(repo, **kwargs): + self.assertEqual(repo, publisher.REPO) + self.assertEqual(kwargs['parent_commit'], 'a' * 40) + for operation in kwargs['operations']: + uploaded[operation.path_in_repo] = Path(operation.path_or_fileobj).read_bytes() + return SimpleNamespace(oid='b' * 40) + api.create_commit.side_effect = commit + original = (self.dataset / 'README.md').read_bytes() + def credential_boundary(_cache): + (self.dataset / 'README.md').write_bytes(b'late mutation must not be uploaded') + return api, 'fixture' + with patch.object(publisher, 'client', side_effect=credential_boundary): + result = publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(uploaded['README.md'], original) + self.assertEqual(result['commit'], 'b' * 40) + self.assertEqual(list((self.root / 'cache').glob('camera-action-frozen-*')), []) + + def test_extra_file_symlink_and_false_qualification_are_refused(self): + target = self.dataset / 'secret.env' + target.write_text('not a credential') + with self.assertRaises(publisher.PublicationError): + publisher.files(self.dataset) + target.unlink() + readme = self.dataset / 'README.md' + readme.unlink() + readme.symlink_to(self.dataset / 'LICENSE') + with self.assertRaises(publisher.PublicationError): + publisher.files(self.dataset) + readme.unlink() + readme.write_bytes(b'Synthetic test evidence only.') + report = json.loads((self.dataset / 'report.json').read_bytes()) + report['qualified'] = True + (self.dataset / 'report.json').write_text(json.dumps(report)) + with self.assertRaisesRegex(publisher.PublicationError, 'qualification'): + publisher.files(self.dataset) + + def test_public_readback_detects_changed_remote_bytes(self): + value = publisher.checked(self.plan, self.receipt['plan_sha256']) + api = Mock() + api.repo_info.return_value = self.info([r['path'] for r in value['files']] + ['.gitattributes']) + with patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', side_effect=lambda *args:self.downloaded(*args,wrong=True)): + with self.assertRaisesRegex(publisher.PublicationError, 'Anonymous public bytes'): + publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(list((self.root/'cache').iterdir()),[]) + + def test_readback_protects_reserve_and_never_deletes_shared_cache_files(self): + value=publisher.checked(self.plan,self.receipt['plan_sha256']);api=Mock() + api.repo_info.return_value=self.info([r['path'] for r in value['files']]+['.gitattributes']) + cache=self.root/'cache';cache.mkdir();shared=cache/'user-shared-cache';shared.write_bytes(b'keep existing user data') + with patch.object(publisher,'public_client',return_value=api), \ + patch.object(publisher.shutil,'disk_usage',return_value=SimpleNamespace(free=publisher.RESERVE)), \ + patch.object(publisher,'download') as download: + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.verify(self.plan,self.receipt['plan_sha256'],cache) + download.assert_not_called() + with patch.object(publisher,'public_client',return_value=api),patch.object(publisher,'download',return_value=shared): + with self.assertRaisesRegex(publisher.PublicationError,'escaped its owned'): + publisher.verify(self.plan,self.receipt['plan_sha256'],cache) + self.assertEqual(shared.read_bytes(),b'keep existing user data') + self.assertEqual(list(cache.iterdir()),[shared]) + + def test_download_is_anonymous_and_revision_pinned_in_explicit_private_cache(self): + cache=self.root/'private-readback';cache.mkdir();blob=cache/'blob';blob.write_bytes(b'fixture') + download=Mock(return_value=str(blob)) + with patch.dict(sys.modules,{'huggingface_hub':SimpleNamespace(hf_hub_download=download, + constants=SimpleNamespace(HF_HUB_DISABLE_XET=True))}): + self.assertEqual(publisher.download('source/fixture.whl','a'*40,cache),blob.resolve()) + download.assert_called_once_with(publisher.REPO,'source/fixture.whl',repo_type='dataset',revision='a'*40, + token=False,endpoint='https://huggingface.co',cache_dir=str(cache),force_download=True) + + def test_missing_or_changed_remote_size_refuses_before_any_download(self): + value=publisher.checked(self.plan,self.receipt['plan_sha256']) + for size in (None,True,-1,10**12): + api=Mock();info=self.info([r['path'] for r in value['files']]+['.gitattributes']) + info.siblings[0].size=size;api.repo_info.return_value=info + with self.subTest(size=size),patch.object(publisher,'public_client',return_value=api), \ + patch.object(publisher,'download') as download: + with self.assertRaisesRegex(publisher.PublicationError,'remote size differs'): + publisher.verify(self.plan,self.receipt['plan_sha256'],self.root/'cache') + download.assert_not_called() + + def test_preimported_xet_enabled_sdk_refuses_before_remote_calls(self): + with patch.dict(sys.modules,{'huggingface_hub':SimpleNamespace(constants=SimpleNamespace(HF_HUB_DISABLE_XET=False))}), \ + patch.object(publisher,'public_client') as client: + with self.assertRaisesRegex(publisher.PublicationError,'fresh process'): + publisher.verify(self.plan,self.receipt['plan_sha256'],self.root/'cache') + client.assert_not_called() + + def test_qualified_label_requires_full_study_audit_even_when_report_matches_manifest(self): + for name in ('manifest.json','report.json'): + path=self.dataset/name; value=json.loads(path.read_bytes()) + value.update(qualified=True,qualification='bounded_simulation_skill',status='complete') + path.write_text(json.dumps(value)) + with self.assertRaisesRegex(publisher.PublicationError,'requires the full-study'): + publisher.files(self.dataset) + + def test_private_hardware_fields_are_rejected_before_upload(self): + from test_camera_action_hardware import fixture + value=fixture();value['hardware']['serial']='private fixture' + target=self.dataset/'hardware.json';target.write_text(json.dumps(value)) + manifest=json.loads((self.dataset/'manifest.json').read_bytes()) + manifest['hardware_context_file']='hardware.json';manifest['files']['hardware.json']=publisher.identity(target) + (self.dataset/'manifest.json').write_text(json.dumps(manifest)) + with self.assertRaisesRegex(publisher.PublicationError,'private or unbounded'): + publisher.files(self.dataset) + + def test_private_freeze_has_independent_inode_attributes_and_mutations(self): + source=self.root/'source';source.write_bytes(b'reviewed bytes'*1000);source.chmod(0o640) + target=self.root/'private'/'frozen';expected=publisher.identity(source) + method=publisher.freeze_file(source,target,expected) + self.assertIn(method,('apfs_clone','stream_copy')) + self.assertNotEqual(source.stat().st_ino,target.stat().st_ino) + self.assertEqual(target.stat().st_nlink,1) + self.assertEqual(stat.S_IMODE(target.stat().st_mode),0o400) + self.assertEqual(stat.S_IMODE(source.stat().st_mode),0o640) + source.write_bytes(b'original mutated after private freeze') + self.assertEqual(publisher.identity(target),expected) + target.chmod(0o600);target.write_bytes(b'private mutation') + self.assertEqual(source.read_bytes(),b'original mutated after private freeze') + + def test_freeze_rejects_destination_collision_and_symlinks_without_overwrite(self): + source=self.root/'source';source.write_bytes(b'reviewed bytes');expected=publisher.identity(source) + target=self.root/'frozen';target.write_bytes(b'existing destination') + with self.assertRaisesRegex(publisher.PublicationError,'already exists'): + publisher.freeze_file(source,target,expected) + self.assertEqual(target.read_bytes(),b'existing destination') + target.unlink();target.symlink_to(source) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source,target,expected) + target.unlink();source_link=self.root/'source-link';source_link.symlink_to(source) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source_link,target,expected) + directory=self.root/'linked-parent';directory.symlink_to(self.root,target_is_directory=True) + with self.assertRaises(publisher.PublicationError):publisher.freeze_file(source,directory/'frozen',expected) + + def test_stream_fallback_is_guarded_per_file_and_during_copy(self): + source=self.root/'source';source.write_bytes(b'x'*(2*1024*1024));expected=publisher.identity(source) + target=self.root/'private'/'frozen' + def usage(free):return SimpleNamespace(free=publisher.RESERVE+free) + with patch.object(publisher,'_clone_from_fd',return_value=False), \ + patch.object(publisher.shutil,'disk_usage',return_value=usage(1536*1024)): + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.freeze_file(source,target,expected) + self.assertFalse(target.exists()) + with patch.object(publisher,'_clone_from_fd',return_value=False), \ + patch.object(publisher.shutil,'disk_usage',side_effect=[usage(5*1024*1024)]*3+[usage(-1)]): + with self.assertRaisesRegex(publisher.PublicationError,'protected disk reserve'): + publisher.freeze_file(source,target,expected) + self.assertFalse(target.exists()) + with patch.object(publisher,'_clone_from_fd',return_value=False): + self.assertEqual(publisher.freeze_file(source,target,expected),'stream_copy') + source.write_bytes(b'late change');self.assertEqual(publisher.identity(target),expected) + + def test_growing_stream_source_refuses_before_writing_excess_and_cleans_target(self): + source=self.root/'source';source.write_bytes(b'x'*(2*1024*1024));expected=publisher.identity(source) + target=self.root/'private'/'frozen';read=os.read;calls=[] + def growing(fd,count): + chunk=read(fd,count);calls.append(len(chunk)) + if len(calls)==1: + with source.open('ab') as stream:stream.write(b'g'*(3*1024*1024)) + return chunk + with patch.object(publisher,'_clone_from_fd',return_value=False),patch.object(publisher.os,'read',side_effect=growing): + with self.assertRaisesRegex(publisher.PublicationError,'grew beyond'): + publisher.freeze_file(source,target,expected) + self.assertEqual(calls,[1024*1024]*3) + self.assertFalse(target.exists()) + + def test_atomic_clone_collision_does_not_delete_the_existing_destination(self): + source=self.root/'source';source.write_bytes(b'reviewed');target=self.root/'private'/'frozen' + def collision(*_args): + target.write_bytes(b'racing destination') + raise FileExistsError('Destination already created.') + with patch.object(publisher,'_clone_from_fd',side_effect=collision): + with self.assertRaises(FileExistsError):publisher.freeze_file(source,target,publisher.identity(source)) + self.assertEqual(target.read_bytes(),b'racing destination') + + +if __name__ == '__main__': + unittest.main() diff --git a/publishing/prepare_camera_action_v01.py b/publishing/prepare_camera_action_v01.py new file mode 100644 index 0000000..edf32e5 --- /dev/null +++ b/publishing/prepare_camera_action_v01.py @@ -0,0 +1,661 @@ +"""Build deterministic raw-evidence shards from a stopped, sealed local study. + +This is an offline packaging command. It performs no simulation, training, +credential lookup or upload, and never changes study artifacts. A diagnostic +audit remains explicitly diagnostic-only when a larger study is packaged. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +import os +from pathlib import Path, PurePosixPath +import re +import shutil +import stat +import tarfile +import zipfile + +from huggingface.camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware + +SCHEMA = 'dvidia.camera-action-evidence/v1' +MAX_FILE = 256 * 1024 ** 2 +# The complete reviewed repository includes historical benchmark evidence. +# Keep this aligned with the uploader's per-file cap; expanded/member limits +# remain independent below (2 GiB total, 256 MiB per member). +MAX_SOURCE = 1024 ** 3 +MAX_EPISODE = 1024 ** 3 +MAX_TOTAL = 120 * 1024 ** 3 +DISK_RESERVE = 30 * 1024 ** 3 +ZIP_LEVEL = 1 +TEXT_SUFFIXES = {'.py', '.js', '.ts', '.json', '.html', '.md', '.txt', '.toml', '.yaml', '.yml'} +SECRET = re.compile(rb'(?:hf_|gh[pousr]_|apikey_)[A-Za-z0-9_]{24,}') + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def plain(path): + path = Path(path).absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (path, *path.parents)): + raise ValueError('Evidence paths cannot traverse symbolic links.') + return path + + +def relative(name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name + or str(PurePosixPath(name)) != name or PurePosixPath(name).is_absolute() + or any(p in ('', '.', '..') or p.startswith('.') for p in PurePosixPath(name).parts) + or any(ord(c) < 32 for c in name)): + raise ValueError('Expected a plain relative evidence path.') + return PurePosixPath(name) + + +def scan(raw): + if SECRET.search(raw): + raise ValueError('Credential-shaped bytes found in proposed public evidence.') + + +def identity(path, *, maximum=MAX_FILE, secret_scan=False): + path = plain(path) + with path.open('rb') as stream: + before = os.fstat(stream.fileno()) + if not stat.S_ISREG(before.st_mode) or not 0 <= before.st_size <= maximum: + raise ValueError('Missing, nonregular or oversized evidence file.') + digest, size, tail = sha256(), 0, b'' + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + size += len(chunk) + if size > maximum: + raise ValueError('Evidence exceeded its byte limit while reading.') + digest.update(chunk) + if secret_scan: + scan(tail + chunk) + tail = chunk[-256:] + after = os.fstat(stream.fileno()) + if any(getattr(before, key) != getattr(after, key) for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns')): + raise ValueError('Evidence changed while reading.') + if size != before.st_size: + raise ValueError('Evidence byte count changed.') + return dict(bytes=size, sha256=digest.hexdigest()) + + +def read(path): + path = plain(path) + if path.stat().st_size > MAX_FILE: + raise ValueError('JSON evidence exceeds the bounded format.') + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON evidence field.') + result[key] = value + return result + def invalid(_value): + raise ValueError('Nonfinite JSON evidence.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=invalid) + + +def write(path, value): + path = plain(path) + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('xb') as stream: + stream.write(canonical(value) + b'\n') + + +def files(root): + root = plain(root) + paths = [] + for path in sorted(root.rglob('*')): + plain(path) + if not (path.is_dir() or path.is_file()): + raise ValueError('Special files are not public evidence.') + if path.is_file(): + relative(path.relative_to(root).as_posix()) + paths.append(path) + return paths + + +def checked_identity(value): + if (not isinstance(value, dict) or set(value) != {'bytes', 'sha256'} + or type(value['bytes']) is not int or not 0 <= value['bytes'] <= MAX_FILE + or not isinstance(value['sha256'], str) or not re.fullmatch('[a-f0-9]{64}', value['sha256'])): + raise ValueError('Malformed sealed evidence identity.') + return value + + +def episode_inventory(root): + root = plain(root) + manifest = read(root / 'integrity.json') + if manifest.get('schema') != 'dvidia.camera-action-rollout/v2/integrity': + raise ValueError('Only sealed v2 native rollout evidence can be archived.') + expected = manifest.get('files') + if not isinstance(expected, dict) or not 2 <= len(expected) <= 9000: + raise ValueError('Episode file inventory is outside the bounded format.') + for name, record in expected.items(): + relative(name) + checked_identity(record) + observed = {p.relative_to(root).as_posix() for p in files(root)} + if observed != set(expected) | {'integrity.json'}: + raise ValueError('Episode contains unsealed, missing or extra files.') + result = dict(expected, **{'integrity.json': identity(root / 'integrity.json')}) + if sum(row['bytes'] for row in result.values()) > MAX_EPISODE: + raise ValueError('Episode exceeds its raw byte budget.') + return result + + +def archive(root, inventory, destination, *, prefix='', compresslevel=ZIP_LEVEL): + """Stream byte-bound files into a deterministic ZIP, then read every member.""" + root, destination = plain(root), plain(destination) + if compresslevel not in (1, 6): + raise ValueError('Use a reviewed ZIP compression level: 1 or 6.') + if destination.exists(): + raise ValueError('Archive exists; existing publication artifacts are immutable.') + if prefix: + relative(prefix) + destination.parent.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(destination.parent).free - sum(r['bytes'] for r in inventory.values()) < DISK_RESERVE: + raise ValueError('Archive preparation would consume the protected disk reserve.') + expected = {} + with zipfile.ZipFile(destination, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=compresslevel, allowZip64=True) as output: + for name, record in sorted(inventory.items()): + relative(name) + checked_identity(record) + path = plain(root / name) + member = (prefix + '/' if prefix else '') + name + relative(member) + info = zipfile.ZipInfo(member, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.create_system = 3 + info.external_attr = (stat.S_IFREG | 0o644) << 16 + info._compresslevel = compresslevel + h, size, tail = sha256(), 0, b'' + with path.open('rb') as source, output.open(info, 'w', force_zip64=True) as target: + before = os.fstat(source.fileno()) + if not stat.S_ISREG(before.st_mode) or before.st_size != record['bytes']: + raise ValueError('Source file identity changed before archiving.') + for chunk in iter(lambda: source.read(1024 * 1024), b''): + size += len(chunk) + if size > record['bytes']: + raise ValueError('Source file grew while archiving.') + if path.suffix in TEXT_SUFFIXES: + scan(tail + chunk) + tail = chunk[-256:] + h.update(chunk) + target.write(chunk) + after = os.fstat(source.fileno()) + if (dict(bytes=size, sha256=h.hexdigest()) != record or any(getattr(before, key) != getattr(after, key) + for key in ('st_dev', 'st_ino', 'st_size', 'st_mtime_ns', 'st_ctime_ns'))): + raise ValueError('Sealed source bytes changed while archiving.') + expected[member] = record + with zipfile.ZipFile(destination) as verification: + if len(verification.infolist()) != len(expected) or set(verification.namelist()) != set(expected): + raise ValueError('Archive member roster differs from sealed evidence.') + for member, record in expected.items(): + h, size = sha256(), 0 + with verification.open(member) as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + size += len(chunk) + h.update(chunk) + if dict(bytes=size, sha256=h.hexdigest()) != record: + raise ValueError('Archive decompression differs from sealed evidence.') + return dict(**identity(destination, maximum=MAX_EPISODE), files=len(expected), + unpacked_bytes=sum(r['bytes'] for r in expected.values()), + compression_level=compresslevel, + inventory_sha256=sha256(canonical(expected)).hexdigest()) + + +def _runtime_members(path, sources, *, wheel): + """Check the actual wheel/source archive contains each registered runtime byte.""" + identities = {} + total = 0 + def source_name(name): + if (not isinstance(name, str) or not name or '\\' in name or ':' in name + or PurePosixPath(name).is_absolute() or str(PurePosixPath(name)) != name + or any(part in ('', '.', '..', '.git', '.env', '.venv', 'node_modules') + or part.startswith('.env.') for part in PurePosixPath(name).parts) + or any(ord(c) < 32 for c in name)): + raise ValueError('Unsafe or private source artifact path.') + if wheel: + with zipfile.ZipFile(path) as archive_file: + entries = archive_file.infolist() + if len(entries) > 50000 or len({e.filename for e in entries}) != len(entries): + raise ValueError('Invalid source wheel member roster.') + for entry in entries: + if entry.is_dir(): + source_name(entry.filename.rstrip('/')) + continue + source_name(entry.filename) + if stat.S_ISLNK(entry.external_attr >> 16) or entry.file_size > MAX_FILE: + raise ValueError('Source wheel contains an unsupported member.') + total += entry.file_size + if total > 2 * MAX_EPISODE: + raise ValueError('Expanded source wheel exceeds its bounded size.') + name = PurePosixPath(entry.filename).name + raw = archive_file.read(entry) if PurePosixPath(entry.filename).suffix in TEXT_SUFFIXES or name in sources else None + if raw is not None: + scan(raw) + if entry.filename.startswith('dvidia_training/') and name in sources: + if name in identities: + raise ValueError('Duplicate registered runtime member.') + identities[name] = sha256(raw).hexdigest() + else: + with tarfile.open(path, mode='r:*') as archive_file: + seen = set() + for entry in archive_file: + source_name(entry.name.rstrip('/')) + if entry.name in seen or len(seen) >= 50000: + raise ValueError('Invalid source archive member roster.') + seen.add(entry.name) + if entry.isdir(): + continue + if not entry.isfile() or entry.size > MAX_FILE: + raise ValueError('Source archive contains links or special members.') + total += entry.size + if total > 2 * MAX_EPISODE: + raise ValueError('Expanded source archive exceeds its bounded size.') + name = PurePosixPath(entry.name).name + raw = None + if PurePosixPath(entry.name).suffix in TEXT_SUFFIXES or name in sources: + with archive_file.extractfile(entry) as stream: + raw = stream.read(MAX_FILE + 1) + scan(raw) + if '/src/dvidia_training/' in '/' + entry.name and name in sources: + if name in identities: + raise ValueError('Duplicate registered runtime member.') + identities[name] = sha256(raw).hexdigest() + if identities != sources: + raise ValueError('Wheel/source archive does not contain the exact registered runtime.') + return identities + + +def _copy(source, destination, *, maximum=MAX_FILE): + expected = identity(source, maximum=maximum, secret_scan=Path(source).suffix in TEXT_SUFFIXES) + destination = plain(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + with plain(source).open('rb') as read_stream, destination.open('xb') as write_stream: + shutil.copyfileobj(read_stream, write_stream, 1024 * 1024) + if identity(destination, maximum=maximum) != expected: + raise ValueError('Copied publication bytes differ from reviewed source.') + return expected + + +def verify_study_audit(study, proof_path, receipts, state): + """Bind an optional independent audit to every currently sealed stage.""" + proof = read(proof_path) + if (proof.get('schema') != 'dvidia.camera-action-study-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or any(proof.get(k) is not False for k in + ('physics_executed', 'estimator_executed', 'model_executed', 'source_authenticated', 'physical_robot_ready')) + or proof.get('protocol_sha256') != identity(study / 'protocol.json')['sha256']): + raise ValueError('Full-study audit does not pass or is bound to a different protocol.') + inventory = proof['evidence_inventory'] + if sha256(canonical(inventory)).hexdigest() != proof['evidence_inventory_sha256']: + raise ValueError('Full-study audit inventory digest differs.') + for name, expected in inventory.items(): + if identity(study / relative(name)) != expected: + raise ValueError('Full-study audit evidence changed after auditing.') + expected_rows = {(name, row['id']): row for name, receipt in receipts.items() for row in receipt.get('rows', [])} + observed = {(row['stage'], row['id']): row for row in proof['results']} + expected_stages = {name for name, receipt in receipts.items() if 'rows' in receipt} + if (len(observed) != len(proof['results']) or set(observed) != set(expected_rows) + or proof['attempts_verified'] != len(expected_rows) or set(proof['stages_verified']) != expected_stages): + raise ValueError('Full-study audit omits or invents a sealed stage or attempt.') + if set(proof.get('training_stages_verified', [])) != {name for name in receipts if name.startswith('train-')}: + raise ValueError('Full-study audit omits or invents a sealed fitting stage.') + required = {'protocol.json', 'registration.json', 'fixed-model.json'} + for name in ('report.json', 'freeze.json'): + if (study / name).exists(): required.add(name) + for stage, receipt in receipts.items(): + required.update('stages/' + stage + suffix for suffix in ('.json', '.integrity.json', '.started.json')) + for model in receipt.get('models', {}).values(): required.add(model['path']) + for key, row in expected_rows.items(): + result = observed[key] + for name in ('case', 'controller', 'negative_control', 'status', 'strict_success', 'complete'): + if result.get(name) != row.get(name): + raise ValueError('Full-study audit outcome or registered scene differs.') + if row.get('path'): + for name in ('integrity.json', 'rollout.json', 'recording.json'): + path = str(relative(row['path'])) + '/' + name + if (study / path).exists(): required.add(path) + if not required <= set(inventory): + raise ValueError('Full-study audit lacks essential source evidence identities.') + qualification = proof.get('qualification') + if qualification is not None and qualification.get('qualified') is not state['qualified']: + raise ValueError('Full-study audit and publication qualification differ.') + if state['qualified'] and qualification is None: + raise ValueError('Qualified publication requires independently reconstructed qualification.') + helpers = Path(__file__).resolve().parents[1] / 'benchmarks/camera-action-v01' + if proof.get('scorer_source_sha256') != read(study / 'protocol.json')['source_sha256']['piper_feedback_score.py']: + raise ValueError('Independent audit used a different native scorer.') + for path, field in ((helpers / 'study_audit.py', 'audit_source_sha256'), + (helpers / 'independent_audit.py', 'diagnostic_helper_source_sha256')): + if proof.get(field) != identity(path)['sha256']: + raise ValueError('Independent audit implementation differs from the reviewed proof.') + return proof + + +def verify_audit_source_archive(source_archive, proof): + """The published source snapshot must actually include this audit code.""" + expected = {'benchmarks/camera-action-v01/study_audit.py': proof['audit_source_sha256'], + 'benchmarks/camera-action-v01/independent_audit.py': proof['diagnostic_helper_source_sha256']} + with tarfile.open(source_archive, 'r:*') as archive: + members = archive.getmembers() + for name, expected_hash in expected.items(): + matches = [m for m in members if m.name == name or m.name.endswith('/' + name)] + if len(matches) != 1 or not matches[0].isfile() or matches[0].size > MAX_FILE: + raise ValueError('Reviewed source archive must contain each independent audit implementation exactly once.') + with archive.extractfile(matches[0]) as stream: + digest = sha256() + for chunk in iter(lambda: stream.read(1024*1024), b''): digest.update(chunk) + if digest.hexdigest() != expected_hash: + raise ValueError('Reviewed source archive differs from the independent audit implementation.') + + +def verify_card_source_archive(source_archive, receipt): + """Bind the generated text's implementations to the actual reviewed archive.""" + sources = receipt['source_inputs'] + expected = {'publishing/render_camera_action_card.py': sources['generator.py'], + 'benchmarks/camera-action-v01/summarize_results.py': sources['summarize_results.py']} + if 'diagnostic_phase_coverage.py' in sources: + expected['benchmarks/camera-action-v01/diagnostic_phase_coverage.py'] = sources['diagnostic_phase_coverage.py'] + with tarfile.open(source_archive, 'r:*') as archive: + members = archive.getmembers() + for name, record in expected.items(): + matches = [m for m in members if m.name == name or m.name.endswith('/' + name)] + if len(matches) != 1 or not matches[0].isfile() or matches[0].size != record['bytes'] or matches[0].size > MAX_FILE: + raise ValueError('Reviewed source archive must contain each card implementation exactly once with its bound size.') + with archive.extractfile(matches[0]) as stream: + digest = sha256() + for chunk in iter(lambda: stream.read(1024*1024), b''): digest.update(chunk) + if digest.hexdigest() != record['sha256']: + raise ValueError('Reviewed source archive differs from the bound card implementation.') + + +def checked_model_cards(directory, *, study, card, source_commit, state, protocol, receipts, + metadata, models, audit, study_audit=None, hardware=None, phase_coverage=None): + """Regenerate reviewed card text from sealed facts before admitting any file.""" + import render_camera_action_card as renderer + from huggingface.camera_action_model_cards import ( + MAX_CARD_BYTES, MAX_RECEIPT_BYTES, RECEIPT_FILE, validate_receipt) + from dvidia_training.camera_action_model import validate_model + directory = plain(directory) + receipt_path = directory / 'card-receipt.json' + receipt_identity = identity(receipt_path, maximum=MAX_RECEIPT_BYTES, secret_scan=True) + receipt = read(receipt_path) + model_identities = {name: metadata[name] for name in models} + names = validate_receipt(receipt, source_commit=source_commit, + protocol_sha256=metadata['protocol.json']['sha256'], report_sha256=metadata['report.json']['sha256'], + report=state['report'], models=model_identities) + expected_files = set(names) | {'README.md', 'card-receipt.json', 'summary.json'} + if {p.relative_to(directory).as_posix() for p in files(directory)} != expected_files: + raise ValueError('Generated card directory contains missing or arbitrary files.') + expected_directories = {str(parent) for name in names for parent in Path(name).parents if str(parent) != '.'} + if {p.relative_to(directory).as_posix() for p in directory.rglob('*') if p.is_dir()} != expected_directories: + raise ValueError('Generated card directory contains arbitrary empty directories.') + bindings = {name: metadata[name] for name in ('protocol.json', 'registration.json', 'report.json')} + for stage in receipts: + for suffix in ('.json', '.integrity.json', '.started.json'): + name = 'stages/' + stage + suffix + bindings[name] = metadata[name] + if state['report'].get('selected_stage') is not None: + bindings['freeze.json'] = metadata['freeze.json'] + records = [] + for name in models: + _, stage, variant, _ = name.split('/') + training = receipts['train-' + stage]['models'][variant] + model = validate_model(read(study / name)) + for filename in ('model.json', 'receipt.json', 'started.json'): + path = f'models/{stage}/{variant}/{filename}' + bindings[path] = metadata[path] + if (training['path'] != name or training['model_sha256'] != metadata[name]['sha256'] + or training['model_bytes'] != metadata[name]['bytes'] + or read(study / f'models/{stage}/{variant}/receipt.json') != training + or model['training'] != training.get('training') or model['training']['stage'] != stage + or model['blind'] is not (variant == 'blind')): + raise ValueError('Generated card model differs from sealed fitting evidence.') + records.append(dict(stage=stage, variant=variant, path=name, identity=metadata[name], + selected=state['report'].get('selected_stage') == stage, training=training, model=model)) + records.sort(key=lambda r: (('initial', 'corrected').index(r['stage']), ('full', 'blind').index(r['variant']))) + if receipt['study_bindings'] != bindings: + raise ValueError('Generated card study bindings differ from current sealed evidence.') + sources = {'summary.json': identity(directory / 'summary.json', secret_scan=True), + 'generator.py': identity(Path(renderer.__file__)), + 'summarize_results.py': identity(renderer.SUMMARY_SOURCE)} + for name, path in (('hardware.json', hardware), ('full-study-audit.json', study_audit), + ('diagnostic-phase-coverage.json', phase_coverage)): + if path is not None: + sources[name] = identity(path) + if 'independent-audit.json' in receipt['source_inputs']: + sources['independent-audit.json'] = identity(audit) + if phase_coverage is not None: + sources['diagnostic_phase_coverage.py'] = identity(renderer.PHASE_SOURCE) + if receipt['source_inputs'] != sources: + raise ValueError('Generated card generator or auxiliary source bindings changed.') + summary = renderer.recompute_summary(study) + if read(directory / 'summary.json') != summary: + raise ValueError('Retained card summary differs from fresh sealed-evidence reconstruction.') + data = dict(study=study, source_commit=source_commit, state=state, protocol=protocol, + report=state['report'], receipts=receipts, summary=summary, models=records, + hardware=read(hardware) if hardware is not None else None, + audit=read(study_audit) if study_audit is not None else None, + phase=read(phase_coverage) if phase_coverage is not None else None, study_bindings=bindings) + texts = {'README.md': renderer.dataset_card(data)} + texts.update({f"model-cards/{r['stage']}/{r['variant']}/README.md": renderer.model_card(data, r) for r in records}) + verified = {} + for name, text in texts.items(): + actual = identity(directory / name, maximum=MAX_CARD_BYTES, secret_scan=True) + raw = text.encode() + if actual != receipt['files'][name] or actual != dict(bytes=len(raw), sha256=sha256(raw).hexdigest()): + raise ValueError('Generated card bytes differ from fresh sealed-evidence rendering.') + if name != 'README.md': + verified[name] = actual + if identity(card, maximum=MAX_CARD_BYTES, secret_scan=True) != receipt['files']['README.md']: + raise ValueError('Selected dataset README differs from the generated card receipt.') + verified[RECEIPT_FILE] = receipt_identity + return dict(directory=directory, files=verified, cards=names, receipt_file=RECEIPT_FILE, receipt=receipt) + + +def prepare(study, output, *, audit, card, wheel, source_archive, source_commit, phase_coverage=None, study_audit=None, hardware=None, episode_cache=None, model_cards=None): + from dvidia_training import camera_action_study as workflow + study, output = plain(study), plain(output) + if output.exists() or output == study or output.is_relative_to(study): + raise ValueError('Use a fresh publication directory outside the study.') + if (study / 'worker.lock').exists(): + raise ValueError('A running or interrupted study worker must be resolved before packaging.') + if episode_cache is not None: + episode_cache = plain(episode_cache).resolve() + if any(a == b or a.is_relative_to(b) or b.is_relative_to(a) + for a, b in ((episode_cache, study.resolve()), (episode_cache, output.resolve()))): + raise ValueError('Episode cache must be separate from study and publication output trees.') + if (episode_cache / 'worker.lock').exists(): + raise ValueError('Wait for the episode cache worker to finish before final packaging.') + if not re.fullmatch('[a-f0-9]{40}', source_commit): + raise ValueError('An exact source commit is required.') + state = workflow.inspect(study) + if state['status'] not in ('blocked', 'complete'): + raise ValueError('Publish only a stopped blocked or completed study, not a partial active pipeline.') + if state['qualified'] and study_audit is None: + raise ValueError('A qualified-skill publication requires a full-study independent audit.') + if hardware is not None: + identity(hardware, maximum=MAX_HARDWARE_BYTES, secret_scan=True) + validate_hardware(read(hardware)) + protocol = read(study / 'protocol.json') + proof = read(audit) + if (proof.get('schema') != 'dvidia.camera-action-diagnostic-audit/v1' or proof.get('passed') is not True + or proof.get('errors') != [] or proof.get('attempts_verified') != 4 + or proof.get('physics_executed') is not False or proof.get('estimator_executed') is not False): + raise ValueError('The explicit passed four-episode diagnostic audit is required.') + diagnostic = read(study / 'stages/diagnostic.json') + if {r['case_id'] for r in proof['results']} != {r['case']['id'] for r in diagnostic['rows']}: + raise ValueError('Diagnostic audit cases differ from this study.') + for result in proof['results']: + row = next(r for r in diagnostic['rows'] if r['case']['id'] == result['case_id']) + if row['strict_success'] is not result['strict_success']: + raise ValueError('Diagnostic audit and study outcomes differ.') + if phase_coverage is not None: + phase_proof = read(phase_coverage) + if (phase_proof.get('schema') != 'dvidia.camera-action-phase-coverage/v1' + or phase_proof.get('protocol_sha256') != identity(study / 'protocol.json')['sha256'] + or phase_proof.get('diagnostic_audit_sha256') != identity(audit)['sha256']): + raise ValueError('Phase coverage proof is not bound to this protocol and diagnostic audit.') + _runtime_members(wheel, protocol['source_sha256'], wheel=True) + _runtime_members(source_archive, protocol['source_sha256'], wheel=False) + with tarfile.open(source_archive, 'r:*') as source_snapshot: + if source_snapshot.pax_headers.get('comment') != source_commit: + raise ValueError('Source archive must carry the exact git-archive commit in its PAX comment.') + metadata = {} + allowed_root = {'protocol.json', 'registration.json', 'fixed-model.json', 'freeze.json', 'report.json'} + episode_rows = [] + models = [] + for path in files(study): + relative_path = path.relative_to(study).as_posix() + parts = PurePosixPath(relative_path).parts + if parts[0] == 'episodes': + continue + allowed = (len(parts) == 1 and relative_path in allowed_root + or len(parts) == 2 and parts[0] == 'stages' and path.suffix == '.json' + or len(parts) == 4 and parts[0] == 'models' and parts[1] in ('initial', 'corrected') + and parts[2] in ('full', 'blind') and parts[3] in ('model.json', 'receipt.json', 'started.json')) + if not allowed: + raise ValueError('Unexpected study artifact; review it before publication.') + metadata[relative_path] = identity(path, secret_scan=True) + if parts[0] == 'models' and parts[-1] == 'model.json': + training = workflow._require(study, 'train-' + parts[1]) + if training['models'][parts[2]]['model_sha256'] != metadata[relative_path]['sha256']: + raise ValueError('Model does not match completed training evidence.') + models.append(relative_path) + all_expected = {}; receipts = {} + for stage in workflow.STAGES: + receipt = workflow._stage(study, stage) + if receipt is None: + continue + receipts[stage] = receipt + for row in receipt.get('rows', []): + if 'path' not in row: + continue + path = relative(row['path']) + if len(path.parts) != 3 or path.parts[0] != 'episodes' or path.parts[1] != stage or path.parts[2] != row['id']: + raise ValueError('Episode path differs from the sealed stage identity.') + directory = study / path + if row.get('receipt_sha256') != identity(directory / 'rollout.json')['sha256'] or row.get('integrity_sha256') != identity(directory / 'integrity.json')['sha256']: + raise ValueError('Episode bytes differ from the sealed stage receipt.') + inventory = episode_inventory(directory) + all_expected[str(path)] = inventory + episode_rows.append(dict(stage=stage, id=row['id'], group_id=row['case']['group_id'], + split=row['case']['split'], controller=row['controller'], negative_control=row['negative_control'], + status=row['status'], strict_success=row['strict_success'], source_directory=str(path), + integrity_sha256=row['integrity_sha256'], receipt_sha256=row['receipt_sha256'])) + observed_episode_dirs = {p.parent.relative_to(study).as_posix() for p in (study / 'episodes').glob('*/*/started.json')} + if observed_episode_dirs != set(all_expected): + raise ValueError('Unsealed, unregistered or missing episode attempts cannot be silently omitted.') + if study_audit is not None: + full_proof = verify_study_audit(study, study_audit, receipts, state) + verify_audit_source_archive(source_archive, full_proof) + raw_bytes = sum(v['bytes'] for inventory in all_expected.values() for v in inventory.values()) + if raw_bytes > MAX_TOTAL: + raise ValueError('Study exceeds the registered raw evidence budget.') + checked_cards = checked_model_cards(model_cards, study=study, card=card, source_commit=source_commit, + state=state, protocol=protocol, receipts=receipts, metadata=metadata, models=models, + audit=audit, study_audit=study_audit, hardware=hardware, phase_coverage=phase_coverage) if model_cards is not None else None + if checked_cards is not None: + verify_card_source_archive(source_archive, checked_cards['receipt']) + output.mkdir(parents=True) + inventory = {} + repo = Path(__file__).resolve().parents[1] + for source, destination in ((card, 'README.md'), (repo / 'LICENSE', 'LICENSE'), + (audit, 'independent-audit.json'), (study / 'protocol.json', 'protocol.json'), + (study / 'registration.json', 'registration.json')): + inventory[destination] = _copy(source, output / destination) + if phase_coverage is not None: + inventory['diagnostic-phase-coverage.json'] = _copy(phase_coverage, output / 'diagnostic-phase-coverage.json') + if study_audit is not None: + inventory['full-study-audit.json'] = _copy(study_audit, output / 'full-study-audit.json') + if hardware is not None: + inventory['hardware.json'] = _copy(hardware, output / 'hardware.json', maximum=MAX_HARDWARE_BYTES) + write(output / 'report.json', state['report']) + inventory['report.json'] = identity(output / 'report.json') + for source, label in ((wheel, 'wheel'), (source_archive, 'source')): + source = plain(source) + if source.name.startswith('.') or len(PurePosixPath(source.name).parts) != 1: + raise ValueError('Unexpected source artifact filename.') + name = 'source/' + source.name + inventory[name] = _copy(source, output / name, maximum=MAX_SOURCE) + metadata_archive = archive(study, metadata, output / 'study-metadata.zip') + inventory['study-metadata.zip'] = {k: metadata_archive[k] for k in ('bytes', 'sha256')} + for row in episode_rows: + name = row['source_directory'] + '.zip' + sealed = None + if episode_cache is not None: + from episode_archive_cache import reuse + sealed = reuse(study / row['source_directory'], all_expected[row['source_directory']], + output / name, prefix=row['source_directory'], cache=episode_cache) + if sealed is None: + sealed = archive(study / row['source_directory'], all_expected[row['source_directory']], + output / name, prefix=row['source_directory']) + row['archive'] = name + row['archive_receipt'] = sealed + inventory[name] = {k: sealed[k] for k in ('bytes', 'sha256')} + print(json.dumps(dict(stage='archive_verified', episode=row['id'], bytes=sealed['bytes'])), flush=True) + for name in models: + inventory[name] = _copy(study / name, output / name) + if checked_cards is not None: + for name, expected in checked_cards['files'].items(): + source = checked_cards['directory'] / ('card-receipt.json' if name == checked_cards['receipt_file'] else name) + actual = _copy(source, output / name) + if actual != expected: + raise ValueError('Generated card changed during publication preparation.') + inventory[name] = actual + value = dict(schema=SCHEMA, repo_id='Dvidia/piper-camera-action-v01', repo_type='dataset', + source_commit=source_commit, registered_runtime_sha256=protocol['source_sha256'], + status=state['report']['status'], pipeline_status=state['status'], + qualification=state['qualification'], qualified=state['qualified'], + physical_robot_ready=False, original_footage_included=False, + recorded_evidence_origin='Synthetic MuJoCo RGB-D, encoders, controls and native scorer traces.', + independent_audit_scope=('All sealed stages are independently byte-, authority-, clock- and numerically audited in full-study-audit.json. Missing stages remain unrun; no physics, estimator or policy was re-executed.' if study_audit is not None else + 'Four diagnostic episodes only. Other shards are checked against sealed manifests; this is not a full-study independent audit.'), + raw_evidence_bytes=raw_bytes, episode_count=len(episode_rows), episodes=episode_rows, + trained_model_files=models, files=inventory, + model_card_files=checked_cards['cards'] if checked_cards is not None else [], + model_card_receipt_file=checked_cards['receipt_file'] if checked_cards is not None else None, + hardware_context_file='hardware.json' if hardware is not None else None, + reconstruction='Extract study-metadata.zip and every episodes/**/*.zip into one new directory. Compare SHA256SUMS first. Each recording retains its original integrity.json.', + archive_format=f'ZIP64, sorted members, 1980 timestamp, regular 0644 files; all members read back byte-for-byte. Fresh archives use deterministic DEFLATE level {ZIP_LEVEL}. Reused cache archives have verified DEFLATE streams but their compression level is not inferable and is recorded as null.') + write(output / 'manifest.json', value) + inventory_with_manifest = dict(inventory, **{'manifest.json': identity(output / 'manifest.json')}) + sums = ''.join(f'{record["sha256"]} {name}\n' for name, record in sorted(inventory_with_manifest.items())) + with (output / 'SHA256SUMS').open('x') as stream: + stream.write(sums) + return dict(status='complete', output=str(output), episode_count=len(episode_rows), raw_evidence_bytes=raw_bytes, + public_bytes=sum(v['bytes'] for v in inventory_with_manifest.values()), trained_models=len(models), + qualification=state['qualification'], manifest_sha256=identity(output / 'manifest.json')['sha256']) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study', required=True) + parser.add_argument('--output', required=True) + parser.add_argument('--audit', required=True) + parser.add_argument('--card', required=True) + parser.add_argument('--wheel', required=True) + parser.add_argument('--source-archive', required=True) + parser.add_argument('--source-commit', required=True) + parser.add_argument('--phase-coverage', help='Optional diagnostic-only phase coverage proof bound to the same protocol and audit.') + parser.add_argument('--study-audit', help='Optional independent audit of all sealed study stages, byte-bound to this stopped study.') + parser.add_argument('--hardware', help='Optional bounded public hardware/software and timing-scope JSON; no hostname or serial fields.') + parser.add_argument('--episode-cache', help='Optional separately stored ZIP cache; source bytes and every cached member are independently verified.') + parser.add_argument('--model-cards', help='Generated card output directory containing README.md, card-receipt.json and the exact model-cards tree.') + args = parser.parse_args(argv) + try: + print(json.dumps(prepare(args.study, args.output, audit=args.audit, card=args.card, + wheel=args.wheel, source_archive=args.source_archive, source_commit=args.source_commit, + phase_coverage=args.phase_coverage, study_audit=args.study_audit, hardware=args.hardware, + episode_cache=args.episode_cache, model_cards=args.model_cards), indent=2)) + return 0 + except (ValueError, OSError, KeyError, TypeError, zipfile.BadZipFile, tarfile.TarError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc)))) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/render_camera_action_card.py b/publishing/render_camera_action_card.py new file mode 100644 index 0000000..ff721da --- /dev/null +++ b/publishing/render_camera_action_card.py @@ -0,0 +1,561 @@ +"""Render publication cards from a stopped study, never from hand-entered metrics. + +No simulation, training, credentials or network calls. A supplied posthoc summary +is recomputed from sealed numeric receipts; a completed study additionally needs +the independent full-study audit. Blocked studies remain explicit candidates. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +from pathlib import Path +import re + +import prepare_camera_action_v01 as evidence +from huggingface.camera_action_hardware import MAX_BYTES as MAX_HARDWARE_BYTES, validate_hardware + +SCHEMA = 'dvidia.camera-action-cards/v1' +REPO = Path(__file__).resolve().parents[1] +SUMMARY_SOURCE = REPO / 'benchmarks/camera-action-v01/summarize_results.py' +PHASE_SOURCE = REPO / 'benchmarks/camera-action-v01/diagnostic_phase_coverage.py' +SOURCE_URL = 'https://github.com/Dvidia-Inference/dvidia-training' +STAGES = ('diagnostic', 'demonstrations', 'train-initial', 'development-initial', + 'corrections', 'train-corrected', 'development-corrected', 'confirmation', 'negative') +BUDGETS = {'diagnostic': 4, 'demonstrations': 80, 'development-initial': 32, + 'corrections': 32, 'development-corrected': 32, 'confirmation': 256, 'negative': 16} + + +def recompute_summary(study): + spec = importlib.util.spec_from_file_location('camera_action_card_posthoc', SUMMARY_SOURCE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.summarize(study) + + +def recompute_phase(study, diagnostic_audit): + spec = importlib.util.spec_from_file_location('camera_action_card_phase', PHASE_SOURCE) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.analyze(study, diagnostic_audit) + + +def _cell(value): + if value is None: + return 'Not reported' + return str(value).replace('&', '&').replace('<', '<').replace('>', '>').replace('|', '\\|').replace('`', '`').replace('\n', ' ').replace('\r', ' ') + + +def _number(value, digits=3): + return 'Not reported' if value is None else f'{value:.{digits}f}' + + +def _table(headers, rows): + return ['| ' + ' | '.join(headers) + ' |', '|' + '|'.join('---' for _ in headers) + '|', + *('| ' + ' | '.join(_cell(cell) for cell in row) + ' |' for row in rows), ''] + + +def _inputs(study, *, summary, source_commit, hardware=None, study_audit=None, phase_coverage=None, diagnostic_audit=None): + from dvidia_training import camera_action_study as workflow + study = evidence.plain(study) + if (study / 'worker.lock').exists(): + raise ValueError('Resolve the active or interrupted study worker before rendering publication cards.') + if not re.fullmatch('[a-f0-9]{40}', source_commit): + raise ValueError('An exact reviewed source commit is required.') + state = workflow.inspect(study) + if state['status'] not in ('complete', 'blocked'): + raise ValueError('Cards require a completed study or an explicitly blocked study.') + protocol = evidence.read(study / 'protocol.json') + if {name: len(rows) for name, rows in protocol['rosters'].items()} != BUDGETS: + raise ValueError('This card template requires the exact registered v0.1 stage budgets.') + if (protocol.get('capture') != dict(duration_s=20., width=320, height=240, camera_hz=25, + control_hz=50, physics_timestep_s=.001, camera_pairs_per_complete_episode=501) + or protocol.get('fitting') != dict(hidden_units=[64, 64], epochs=50, batch_size=512, + learning_rate=.003, seed=17, normalization='training-only', corrections_rounds=1, + teacher_mix_probability=.5) + or protocol.get('start_domain', {}).get('x_m') != [.287, .311] + or protocol.get('start_domain', {}).get('y_m') != [-.042, -.018] + or {key: len(value) for key, value in protocol.get('cases', {}).items()} != + dict(train=64, development=16, confirmation=64)): + raise ValueError('Capture, fitting or start-domain registration differs from this v0.1 methodology.') + report = evidence.read(study / 'report.json') + if report != state['report'] or report.get('physical_robot_ready') is not False: + raise ValueError('Saved report differs from sealed study inspection.') + if state['status'] == 'complete' and report['status'] != 'complete': + raise ValueError('Completed study lacks a complete result report.') + if state['status'] == 'blocked' and (report.get('status') != 'blocked' or report.get('qualified') is not False): + raise ValueError('Blocked studies must remain unqualified research candidates.') + if study_audit is None and state['status'] != 'blocked': + raise ValueError('Completed cards require the independent full-study audit.') + receipts = {name: receipt for name in STAGES if (receipt := workflow._stage(study, name)) is not None} + proof = evidence.verify_study_audit(study, study_audit, receipts, state) if study_audit is not None else None + document = evidence.read(summary) + if document != recompute_summary(study): + raise ValueError('Posthoc summary differs from fresh read-only reconstruction of sealed receipts.') + if document.get('schema') != 'dvidia.camera-action-posthoc-summary/v1' or document.get('physical_robot_ready') is not False: + raise ValueError('Expected the simulation-only posthoc summary format.') + expected_qualification = dict(status=report['status'], qualified=report['qualified'], + qualification=report['qualification'], report_sha256=evidence.identity(study / 'report.json')['sha256'], + source='existing study report; not newly awarded by this posthoc tool') + if document.get('qualification') != expected_qualification: + raise ValueError('Posthoc qualification is not bound to the final report.') + bindings = {'protocol.json': evidence.identity(study / 'protocol.json'), + 'registration.json': evidence.identity(study / 'registration.json'), + 'report.json': evidence.identity(study / 'report.json')} + for name, receipt in receipts.items(): + for suffix in ('.json', '.integrity.json', '.started.json'): + path = 'stages/' + name + suffix + bindings[path] = evidence.identity(study / path) + if document['source_receipts'].get('stages/' + name + '.json') != bindings['stages/' + name + '.json']['sha256']: + raise ValueError('Posthoc stage identity differs from the sealed receipt.') + if set(document['source_receipts']) != {'stages/' + name + '.json' for name in receipts}: + raise ValueError('Posthoc stage roster differs from current sealed evidence.') + selected = report.get('selected_stage') + if selected is not None: + frozen = evidence.read(study / 'freeze.json') + if selected not in ('initial', 'corrected') or frozen.get('selected_stage') != selected: + raise ValueError('Selected model stage differs from frozen selection.') + bindings['freeze.json'] = evidence.identity(study / 'freeze.json') + models = [] + for stage in ('initial', 'corrected'): + receipt = receipts.get('train-' + stage) + if receipt is None: + continue + if set(receipt['models']) != {'full', 'blind'}: + raise ValueError('Each sealed fit must retain the matched full and blind models.') + for variant in ('full', 'blind'): + record = receipt['models'][variant] + path = f'models/{stage}/{variant}/model.json' + if record['path'] != path: + raise ValueError('Model receipt path differs from its fitting stage.') + actual = evidence.identity(study / path) + if actual != dict(bytes=record['model_bytes'], sha256=record['model_sha256']): + raise ValueError('Model bytes differ from the sealed fitting receipt.') + from dvidia_training.camera_action_model import validate_model + model = validate_model(evidence.read(study / path)) + if (model.get('schema') != 'dvidia.camera-action-mlp/v1' or model.get('blind') is not (variant == 'blind') + or model.get('physical_robot_ready') is not False or model.get('hidden_units') != [64, 64] + or model['training'] != record.get('training') or model['training']['stage'] != stage): + raise ValueError('Model schema or architecture differs from this card template.') + for filename in ('model.json', 'receipt.json', 'started.json'): + name = f'models/{stage}/{variant}/{filename}' + bindings[name] = evidence.identity(study / name) + if evidence.read(study / f'models/{stage}/{variant}/receipt.json') != record: + raise ValueError('Individual model receipt differs from sealed fitting stage.') + models.append(dict(stage=stage, variant=variant, path=path, identity=actual, + selected=selected == stage, training=record, model=model)) + if selected and {row['variant'] for row in models if row['selected']} != {'full', 'blind'}: + raise ValueError('Selected full/blind pair is missing.') + expected_model_files = {name for name in bindings if name.startswith('models/')} + model_root = study / 'models' + if model_root.exists(): + actual_model_files = {path.relative_to(study).as_posix() for path in evidence.files(model_root)} + expected_directories = {str(Path(name).parent) for name in expected_model_files} + expected_directories |= {str(Path(name).parent) for name in expected_directories} + actual_directories = {path.relative_to(study).as_posix() for path in model_root.rglob('*') if path.is_dir()} + if actual_model_files != expected_model_files or actual_directories != expected_directories: + raise ValueError('Unsealed or orphan model prefixes cannot disappear from a publication card.') + hardware_doc = None + if hardware is not None: + evidence.identity(hardware, maximum=MAX_HARDWARE_BYTES, secret_scan=True) + hardware_doc = validate_hardware(evidence.read(hardware)) + declared_runtime = {name: hardware_doc[name] for name in ('system', 'machine', 'python')} + declared_runtime.update(hardware_doc['versions']) + if any(any(row['recorded_runtime'].get(name) != value for name, value in declared_runtime.items()) + for row in document['hardware']): + raise ValueError('Public hardware/software context differs from the recorded episode runtime.') + phase = None + if phase_coverage is not None: + if diagnostic_audit is None: + raise ValueError('Phase coverage requires its independently audited diagnostic source.') + phase = evidence.read(phase_coverage) + if (phase.get('schema') != 'dvidia.camera-action-phase-coverage/v1' + or phase.get('protocol_sha256') != bindings['protocol.json']['sha256'] + or phase.get('diagnostic_audit_sha256') != evidence.identity(diagnostic_audit)['sha256'] + or phase != recompute_phase(study, diagnostic_audit)): + raise ValueError('Diagnostic phase coverage differs from this protocol.') + elif diagnostic_audit is not None: + raise ValueError('A diagnostic audit is used only with its supplied phase coverage.') + return dict(study=study, source_commit=source_commit, state=state, protocol=protocol, + report=report, receipts=receipts, summary=document, models=models, hardware=hardware_doc, + audit=proof, phase=phase, study_bindings=bindings) + + +def _status(data): + report = data['report'] + if report['qualified']: + return 'Bounded simulation skill: all registered gates passed. Physical robot ready: **no**.' + if report['status'] == 'blocked': + return '**Research candidate — blocked.** The study stopped at a declared blocker; missing stages are unrun, not successful. Physical robot ready: **no**.' + return '**Research candidate — qualification gates failed.** A completed benchmark does not establish an installable qualified skill. Physical robot ready: **no**.' + + +def _results(data): + report, summary = data['report'], data['summary'] + lines = ['## Registered experiment and results', '', + '72 fresh starting-position groups are split 32 training / 8 development / 32 confirmation. ' + 'Both destinations and all related recordings stay within their group. ' + 'The tiny interpolation domain is x = 0.287–0.311 m, y = −0.042–−0.018 m, z = 0.082 m. ' + 'Demonstrations comprise 64 training and 16 development episodes. No failed scene is replaced.', ''] + rows = [] + for name in STAGES: + receipt = data['receipts'].get(name) + budget = BUDGETS.get(name, '2 models; 50 epochs each') + rows.append((name, budget, receipt['status'] if receipt else 'Unrun', + receipt.get('strict_successes') if receipt and name in BUDGETS else None)) + lines += _table(['Stage', 'Registered budget', 'Status', 'Strict successful attempts'], rows) + lines += ['All registered rollout budgets total **452 attempts**. Fitting does not add native attempts. ' + 'The 32 correction episodes use training groups only and seeded 50% teacher control. ' + 'Development evaluates two stages × two model variants × 16 cases = 64 rollouts. ' + 'Frozen confirmation is 64 cases × four controllers = 256 attempts. Negatives add 16 attempts.', ''] + lines += ['Before fitting, all four diagnostics must strictly succeed with a valid initial visual anchor ' + 'within 10 mm of native diagnostic truth. Teacher data must cover every training start-group ' + 'at least once and provide at least 28 successful usable episodes per destination.', ''] + quality = data['receipts'].get('demonstrations', {}).get('data_quality') + if quality is not None: + lines += ['Recorded teacher-data gate:', '', '```json', json.dumps(quality, indent=2, sort_keys=True), '```', ''] + rows = [(row['stage'] + ' / ' + row['controller'], f"{row['strict_successes']} / {row['scheduled']}", + row['complete'], row['native_assessable'], json.dumps(row['statuses'], sort_keys=True)) + for row in summary['controllers']] + lines += _table(['Stage / controller', 'Strict success / scheduled', 'Complete', 'Native assessable', 'Retained statuses'], rows) + lines += ['Contact-affected counts use the independent 1 ms native scorer; unassessed attempts stay explicit.', ''] + lines += _table(['Stage / controller', 'Contact-affected / assessed', 'Unassessed', 'Critical native ticks'], + [(r['stage'] + ' / ' + r['controller'], + f"{r['critical_contacts']['attempts_with_contacts']} / {r['critical_contacts']['assessed_attempts']}", + r['critical_contacts']['unassessed_attempts'], r['critical_contacts']['native_ticks']) + for r in summary['controllers']]) + lines += ['Component cells are **failed / conditionally assessed; unassessed overall / eligible / unknown prerequisite**. ' + 'Lift requires observed grasp, carry/retention require observed lift, and release/settling/retreat ' + 'require supported carry. These conditional diagnostics do not replace all-attempt task success.', ''] + component_rows = [] + for r in summary['controllers']: + cells = [r['stage'] + ' / ' + r['controller']] + for name in ('grasp', 'lift', 'carry', 'retention', 'release', 'settling', 'retreat'): + c = r['components'][name] + cells.append(f"{c['conditional_failed']} / {c['conditional_assessed']}; " + f"{c['unassessed']} / {c['conditional_unassessed']} / {c['prerequisite_unknown']}") + component_rows.append(cells) + lines += _table(['Stage / controller', 'Grasp', 'Lift', 'Carry', 'Retention', 'Release', 'Settling', 'Retreat'], component_rows) + lines += ['These posthoc tables retain unknown prerequisites and all-attempt unassessed counts; ' + 'an absent prerequisite does not become an observed component failure.', ''] + lines += ['### Qualification gates', '', + 'Required: ≥58/64 strict learned placements; positive 97.5% paired bootstrap intervals versus ' + 'both fixed and blind controllers, grouping both destinations by starting position; positive intervals ' + 'also on at least 24 common admitted groups; no increase in critical-contact episode frequency; ' + 'all 16 negative controls correct; no privileged learner inputs or teacher evaluation fallback.', ''] + lines += _table(['Gate', 'Result'], [(name, 'PASS' if passed else 'FAIL') for name, passed in sorted(report['gates'].items())]) if report['gates'] else ['Gates are **unassessed** because the study is blocked.', ''] + for label, key in (('All scheduled starting groups', 'comparisons'), ('Common admitted starting groups', 'admitted_comparisons')): + if report.get(key): + lines += [label + ' (verbatim structured result):', '', '```json', + json.dumps(report[key], indent=2, sort_keys=True, allow_nan=False), '```', ''] + for name, receipt in data['receipts'].items(): + if receipt.get('status') == 'blocked': + lines += [f"Blocked at **{name}**: {_cell(receipt.get('reason'))}.", ''] + return lines + + +def _observations(data): + lines = ['## Perception, speed and hardware', '', + 'RGB-D is captured at 320×240 and 25 Hz; encoders/commands at 50 Hz; independent native checks ' + 'at 1 ms. A complete 20-second episode has 501 camera pairs and 1,000 motor intervals. ' + 'Missing estimates remain missing with masks and frame age; the initial anchor is explicitly historical. ' + 'The known-upright-block tracker can abstain when the gripper occludes or tilts the block.', ''] + rows = [] + for row in data['summary']['controllers']: + p, t = row['perception'], row['timing'] + inference = t['policy_inference_wall_seconds'] + loop = t['full_boundary_wall_seconds'] + rows.append((row['stage'] + ' / ' + row['controller'], + f"{p['available_decisions']} / {p['assessed_decisions']}", p['episodes_unknown'], + _number(p['maximum_missing_seconds']), + _number(inference['p50'] * 1000 if inference['p50'] is not None else None) if inference['applicable'] else 'Not applicable', + _number(inference['p95'] * 1000 if inference['p95'] is not None else None) if inference['applicable'] else 'Not applicable', + _number(loop['p95'] * 1000 if loop['p95'] is not None else None))) + lines += _table(['Stage / controller', 'Available / assessed visual decisions', 'Episodes unreported', + 'Longest missing (s)', 'Inference p50 (ms)', 'Inference p95 (ms)', 'Full boundary p95 (ms)'], rows) + lines += ['Latencies pool individual recorded boundaries. A teacher zero is not a learned-policy inference measurement. ' + 'Synthetic camera time, source-frame age, simulation time and wall-clock computation are distinct. ' + 'The simulator pauses for computation on a shared host: these numbers do **not** establish physical ' + 'sensor-to-actuator latency, a hard deadline or hardware control frequency.', ''] + warning_rows = [] + for row in data['summary']['controllers']: + warnings = row.get('renderer_warnings') + if warnings: + warning_rows.append((row['stage']+' / '+row['controller'], + f"{warnings['episodes_with_warnings']} / {warnings['episodes_reported']}", + warnings['episodes_unreported'], json.dumps(warnings['warning_boundaries_by_message'], sort_keys=True))) + if warning_rows: + lines += ['Recorded graphics warnings are retained below. Counts refer to render boundaries ' + 'carrying cumulative warning history, not new warning emissions or independent depth-error trials. ' + 'An absent field remains unreported; no physical ' + 'depth accuracy is inferred from a warning-free frame.', ''] + lines += _table(['Stage / controller', 'Affected / reported episodes', 'Unreported episodes', + 'Warning messages and boundary counts'], warning_rows) + phase = data['phase'] + if phase: + lines += [f"**Diagnostic manipulation blind spot:** {phase['manipulation_available']} / " + f"{phase['manipulation_decisions']} visual estimates were available during " + + ', '.join(_cell(name) for name in phase['manipulation_phases']) + + f" across {phase['cases']} teacher diagnostic episodes. " + 'This is a diagnostic subset, not student confirmation. Teacher phase is used only in this ' + 'offline readback and is excluded from learner features; repeated 50 Hz decisions share 25 Hz camera frames.', ''] + rows = [(row['stage'] + ' / ' + row['variant'], row['model_bytes'], _number(row['training_wall_seconds']), + _number(row['shared_export_wall_seconds']), row['process_peak_rss_bytes'], row['training_episodes']) + for row in data['summary']['training']] + lines += _table(['Fit', 'Model bytes', 'Fit/save wall seconds', 'Shared export seconds', 'Process peak RSS bytes', 'Training episodes'], rows) + lines += ['Process peak RSS includes shared dataset export and any preceding fit; it is not isolated model memory. ' + 'Training uses CPU NumPy. Native rendering needs a working graphics driver/OpenGL context; no CUDA requirement ' + 'or minimum-hardware qualification is inferred from this host.', ''] + if data['hardware']: + lines += ['Public host context supplied with this release (`hardware.json`); software identifiers ' + 'are cross-checked against bound episode runtime receipts when present:', '', '```json', + json.dumps(data['hardware'], indent=2, sort_keys=True), '```', ''] + else: + lines += ['CPU model, memory capacity and GPU identity were not supplied in a bound public hardware context; ' + 'those details remain **not reported**.', ''] + for row in data['summary']['hardware']: + lines += [f"Recorded runtime in {row['episode_receipts']} episode receipts:", '', '```json', + json.dumps(row['recorded_runtime'], indent=2, sort_keys=True), '```', ''] + return lines + + +def dataset_card(data): + protocol_hash = data['study_bindings']['protocol.json']['sha256'] + lines = ['---', 'license: mit', 'task_categories:', '- robotics', 'language:', '- en', + 'tags:', '- robotics', '- simulation', '- behavior-cloning', '- synthetic-data', '- piper', '---', '', + '# DVIDIA · Camera-to-action v0.1', '', _status(data), '', + 'A local, reproducible PiPER block-placement experiment: learn seven absolute motor targets ' + 'from synchronized synthetic RGB-D, encoders and demonstrated actions; evaluate the complete ' + 'approach, grasp, lift, carry, release, settling and retreat task on unseen starting positions.', '', + f"[Reviewed source]({SOURCE_URL}/tree/{data['source_commit']}) · " + '[Research](https://research.dvidia.org) · [DVIDIA](https://dvidia.org)', '', + f"Protocol SHA-256: `{protocol_hash}`. Source commit: `{data['source_commit']}`.", '', + '## Dataset origin and intended use', '', + 'All footage is rendered in MuJoCo. Motor labels come from an authored teacher with privileged ' + 'simulator state. No human video, human teleoperation, private footage or physical robot measurements ' + 'are claimed. Initial demonstrations label issued targets; the correction round labels separately ' + 'recorded teacher suggestions, while the tape also preserves the commands actually executed.', '', + 'Use this evidence to study bounded camera/encoder imitation and its failures. ' + 'It does not demonstrate arbitrary-video action recovery, general world modeling, shoes, ' + 'arbitrary-arm transfer or physical hardware qualification.', '', + '## Model contract and selection', '', + 'A CPU NumPy MLP with two 64-unit tanh hidden layers predicts six joint targets and one gripper target. ' + 'Each fitting stage uses 50 epochs, batch size 512, Adam learning rate 0.003 and seed 17. ' + 'Training-only normalization includes explicit missing fills and masks. The model uses 160 ms ' + 'of causal history, current encoders, encoder-derived link6 pose, previous applied servo references, ' + 'goal, elapsed time and a historical initial visual position. It never receives teacher phase, ' + '`reference.action`, the current teacher target, native object/contact truth, future frames or ' + 'native object orientation.', '', + 'The matched blind model masks all live/historical sensor features, retaining the same initial ' + 'visual context, goal, time and causal prior-command history. It sees the same training episodes. ' + 'The legacy tcp_* feature fields are link6 body-origin estimates, not a calibrated tool point; ' + 'the native scorer uses a distal site at link6 local [0, 0, 0.12003] m.', '', + 'Selection uses full-learner development success, then critical-contact ticks, then common ' + 'physical-action-scale prediction loss, then initial before corrected. The blind comparator ' + 'comes from that same selected stage. Confirmation permits no teacher intervention.', ''] + models = data['models'] + lines += _table(['Stage', 'Variant', 'Selected', 'Model artifact', 'Model card', 'Bytes', 'SHA-256'], + [(r['stage'], r['variant'], r['selected'], r['path'], + f"[Read card](model-cards/{r['stage']}/{r['variant']}/README.md)", + r['identity']['bytes'], r['identity']['sha256']) for r in models]) + if not models: + lines += ['No completed model fit is sealed before the blocker. No downloadable skill is claimed.', ''] + lines += _results(data) + _observations(data) + lines += ['## Limitations and independent verification', '', + 'Negative controls are deliberately narrow: initial_missing blanks the initial RGB while leaving ' + 'the object in the simulator; initial_stale sets capture/availability to −0.2 s; altered_calibration ' + 'scales one rotation element by 1.1; open_gripper forces the jaw open. Passing these checks does ' + 'not establish detection of physical absence, all stale-frame cases or small valid rigid calibration errors.', '', + 'Depth, contact, friction, actuator fidelity, sensor delivery and the sim-to-real gap remain ' + 'unqualified. A passing integrity audit certifies consistency of saved evidence, not task success. ' + 'The posthoc summary does not tune or award qualification.', '', + ('The passed full-study audit covers every currently sealed stage, roster, action authority, clocks ' + 'and independent native score reconstruction. No physics, tracker or policy was re-executed.' if data['audit'] else + '**No full-study independent audit was supplied.** This blocked publication is explicitly unaudited beyond ' + 'the sealed receipts and read-only posthoc reconstruction; it cannot award a qualified-skill label.'), '', + 'See `report.json`, `protocol.json`, `full-study-audit.json` when supplied, and the raw per-episode ' + 'integrity inventories. The table generator rechecks a local posthoc summary against sealed receipts; ' + 'When packaged with the model cards, `model-cards-receipt.json` binds their generated bytes to the same models and report. ' + 'All failed, refused, interrupted and unattempted outcomes remain in ' + 'their registered denominators. Unassessed component outcomes are not guessed from overall failure.', ''] + selected = next((record for record in models if record['stage'] == data['report'].get('selected_stage') + and record['variant'] == 'full'), None) + if selected is not None: + lines += ['## Try the selected simulation candidate', '', + f"Download the [selected full-feedback model]({selected['path']}) as `model.json` and the " + '[released wheel](https://huggingface.co/datasets/Dvidia/piper-camera-action-v01/tree/main/source). Only these two files plus the separately provisioned PiPER assets ' + 'are needed to try the candidate. The raw episode corpus and source archive are not needed ' + 'for this installation. The [reviewed source](' + SOURCE_URL + '/tree/' + data['source_commit'] + + ') remains available for inspection.', '', + 'Installation does not change the model\'s qualification. This simulation candidate cannot ' + 'command physical hardware. Use Python 3.12 and a working OpenGL renderer; the tested ' + 'renderer is macOS CGL. Other platforms remain unqualified. Replace the wheel path below ' + 'with the actual downloaded filename, retaining the `[arm]` extra.', '', + '```sh', 'python3.12 -m venv .venv', 'source .venv/bin/activate', + "python -m pip install '/path/to/downloaded-wheel.whl[arm]'", + f"printf '%s model.json\\n' '{selected['identity']['sha256']}' | shasum -a256 -c -", + 'dvidia-robot-sim fetch piper --output /path/to/robot-assets', + 'dvidia-arm check --model model.json --robot piper', + 'dvidia-arm install --model model.json --robot piper --assets /path/to/robot-assets --output /path/to/new-installed-skill', + 'dvidia-arm run /path/to/new-installed-skill --assets /path/to/robot-assets --output /path/to/new-attempt --start-x .30 --start-y -.03 --goal 0', + '```', '', 'Use fresh output directories; the saved attempt preserves its independent task outcome. ' + 'A successful install is not a successful placement.', ''] + lines += ['## Reproduce the complete study', '', + 'The release contains source/wheel artifacts, `study-metadata.zip`, and one deterministic ZIP ' + 'per recorded attempt under `episodes/`. Verify the manifest and checksums before extracting. ' + 'Archives preserve each original recording integrity file; the compact diagnostic viewer is ' + 'not the full raw archive.', '', + '```sh', 'shasum -a256 -c SHA256SUMS', + 'python -m zipfile -e study-metadata.zip restored-study', '```', '', + 'Extract every episode archive listed in `manifest.json` into that same fresh `restored-study` ' + 'directory; each archive already contains its `episodes///` prefix. ' + 'Install the released wheel with its `arm` extra, or extract the reviewed source archive and run:', '', + '```sh', 'python3.12 -m venv .venv', 'source .venv/bin/activate', + "python -m pip install '.[arm]'", 'dvidia-train robot inspect --study /path/to/restored-study', + 'dvidia-training-studio --directory /path/to/new-local-runs --port 8363', '```', '', + 'The UI at `http://127.0.0.1:8363/robot` and Terminal share the same run record. ' + 'Reproduction uses a **fresh** study, the published fixed comparator and separately provisioned ' + 'verified PiPER assets; original study attempts are immutable. Provision the pinned assets separately:', '', + '```sh', 'dvidia-robot-sim fetch piper --output /path/to/robot-assets', '```', '', + '```sh', 'dvidia-train robot prepare --study /path/to/new-study --fixed-model /path/to/restored-study/fixed-model.json', + 'dvidia-train robot collect --study /path/to/new-study --assets /path/to/robot-assets --stage diagnostic', + 'dvidia-train robot inspect --study /path/to/new-study', '```', '', + f"[Complete collect/train/evaluate/package commands]({SOURCE_URL}/blob/{data['source_commit']}/docs/camera-action-v01.md). " + 'After dependencies/assets are installed, inference and training are local. The exact PiPER ' + 'simulation adapter is the only accepted embodiment. Native stepping is CPU work; rendering ' + 'needs OpenGL. The tested renderer is macOS CGL. Preserve 30 GiB free disk; raw evidence is ' + 'bounded at 120 GiB. Other platforms and hardware minimums remain unqualified.', ''] + lines += ['## License and attribution', '', + 'Original DVIDIA code, generated dataset evidence and learned parameter JSON are MIT. ' + 'Upstream robot models, MuJoCo and other third-party components retain their own licenses ' + 'and notices; assets are provisioned separately. See `LICENSE` and the source third-party notices. ' + 'Individual contributor attribution is intentionally unset.', ''] + return '\n'.join(lines) + + +def model_card(data, record): + variant = record['variant'] + selected = record['selected'] + lines = ['---', 'license: mit', 'tags:', '- robotics', '- simulation', '- behavior-cloning', '- piper', '---', '', + f"# DVIDIA · PiPER camera-to-action · {record['stage']} / {variant}", '', + (_status(data) if selected and variant == 'full' else + '**Research candidate — comparison artifact.** This model has no separately awarded skill qualification. ' + 'The study gates concern the selected full-feedback model. Physical robot ready: **no**.'), '', + ('This is the selected full-feedback model.' if variant == 'full' else + 'This is the matched blind baseline, not the deployable full-feedback skill.') if selected else + 'This development candidate was not selected for frozen confirmation.', '', + f"Model file: [model.json](../../../{record['path']}) ({record['identity']['bytes']} bytes). SHA-256: `{record['identity']['sha256']}`.", '', + f"Reviewed source: [{data['source_commit']}]({SOURCE_URL}/tree/{data['source_commit']}). " + f"Protocol: `{data['study_bindings']['protocol.json']['sha256']}`.", '', + '## Inputs, output and training', '', + 'Data-only JSON, schema `dvidia.camera-action-mlp/v1`; no pickle or remote code. ' + 'CPU NumPy, two 64-unit tanh hidden layers, seven absolute targets (six joints and jaw). ' + 'Native joint/rate limits and independent task checks remain in the simulation runtime.', '', + ('Full feedback uses historical initial visual position, goal, time, current encoder/link6 measurements, ' + 'previous applied commands and 160 ms of causal sensor history with explicit masks and age.' if variant == 'full' else + 'All live and historical sensor-feedback tensor channels are masked. Initial visual context, goal, time ' + 'and causal previous-applied-command history remain, exactly matching the full model context.'), '', + 'Labels are authored privileged-simulator teacher actions; they are not recovered human actions. ' + 'The correction pass uses separately recorded teacher queries at mixed-controller visited states. ' + 'Executed commands and suggestions retain separate authority. No current teacher command, ' + 'reference.action, teacher phase, native object/contact truth or future frame is a learner input.', '', + 'Fitting budget: seed 17, 50 epochs, batch 512, Adam 0.003, training-only normalization including ' + 'missing zeros/masks. The full and blind models use identical source recordings.', '', + '## Measured fitting cost', ''] + training = record['training'] + lines += _table(['Measurement', 'Value'], [(key, training.get(key)) for key in + ('training_wall_seconds', 'shared_export_wall_seconds', 'process_peak_rss_bytes', 'memory_scope')]) + lines += [f"Training recordings: {len(training['training_episodes'])}; development recordings: {len(training['development_episodes'])}.", '', + 'Model provenance:', '', '```json', json.dumps({key: record['model']['training'].get(key) for key in + ('stage', 'sample_count', 'train_groups', 'dataset_sha256', 'provenance_sha256', 'model_source_sha256')}, + indent=2, sort_keys=True), '```', '', + '## Evaluation and use', '', + 'See the [dataset card](../../../README.md) and `report.json` at the release root ' + 'for all four confirmation controllers, failed gates, grouped intervals, negative controls and ' + 'perception/latency denominators. A model file or successful installation is not evidence of skill qualification.', ''] + matching = [r for r in data['summary']['controllers'] if r['controller'] == ('student' if variant == 'full' else 'blind') + and (r['stage'] == 'development-' + record['stage'] or selected and r['stage'] == 'confirmation')] + lines += _table(['Evaluation', 'Strict success / scheduled', 'Complete', 'Native assessable'], + [(r['stage'], f"{r['strict_successes']} / {r['scheduled']}", r['complete'], r['native_assessable']) for r in matching]) + lines += ['Only one simulated PiPER, known rigid upright block and two known destinations were studied. ' + 'Object estimates can disappear throughout manipulation. Link6 pose is not the calibrated distal tool point. ' + 'No shoe skill, human-video motor recovery, arbitrary-arm transfer, physical robot readiness or real-time guarantee. ' + 'Teacher latency zeroes are inapplicable; process RSS includes export/preceding fits.', '', + 'Original learned parameters and DVIDIA code are MIT; upstream dependencies retain their notices. ' + 'This artifact cannot command physical hardware through the provided installer.', ''] + return '\n'.join(lines) + + +def render(study, output, *, summary, source_commit, hardware=None, study_audit=None, phase_coverage=None, diagnostic_audit=None): + study, output = evidence.plain(study), evidence.plain(output) + if output.exists() or output == study or output.is_relative_to(study): + raise ValueError('Use a fresh cards directory outside the study.') + data = _inputs(study, summary=summary, source_commit=source_commit, hardware=hardware, + study_audit=study_audit, phase_coverage=phase_coverage, diagnostic_audit=diagnostic_audit) + documents = {'README.md': dataset_card(data)} + for record in data['models']: + documents[f"model-cards/{record['stage']}/{record['variant']}/README.md"] = model_card(data, record) + for text in documents.values(): + evidence.scan(text.encode()) + sources = {'summary.json': evidence.identity(summary), 'generator.py': evidence.identity(Path(__file__)), + 'summarize_results.py': evidence.identity(SUMMARY_SOURCE)} + for name, path in (('hardware.json', hardware), ('full-study-audit.json', study_audit), + ('diagnostic-phase-coverage.json', phase_coverage), ('independent-audit.json', diagnostic_audit)): + if path is not None: + sources[name] = evidence.identity(path) + if phase_coverage is not None: + sources['diagnostic_phase_coverage.py'] = evidence.identity(PHASE_SOURCE) + files = {} + output.mkdir(parents=True, exist_ok=False) + # Retain the exact input bytes so packaging can verify this claimed source + # identity as well as independently reconstructing its numeric content. + with Path(summary).open('rb') as source, (output / 'summary.json').open('xb') as target: + copied = 0 + for chunk in iter(lambda: source.read(1024*1024), b''): + if copied + len(chunk) > sources['summary.json']['bytes']: + raise ValueError('Summary grew while retaining the generated card source.') + target.write(chunk) + copied += len(chunk) + if copied != sources['summary.json']['bytes']: + raise ValueError('Summary shrank while retaining the generated card source.') + if evidence.identity(output / 'summary.json') != sources['summary.json']: + raise ValueError('Summary changed while retaining the generated card source.') + for name, text in documents.items(): + path = output / name + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('x', encoding='utf-8', newline='\n') as stream: + stream.write(text) + files[name] = evidence.identity(path) + receipt = dict(schema=SCHEMA, source_commit=source_commit, + protocol_sha256=data['study_bindings']['protocol.json']['sha256'], + report_sha256=data['study_bindings']['report.json']['sha256'], status=data['report']['status'], + qualification=data['report']['qualification'], qualified=data['report']['qualified'], + full_study_audited=data['audit'] is not None, physical_robot_ready=False, + summary_reconstructed=True, physics_executed=False, model_executed=False, + source_inputs=sources, study_bindings=data['study_bindings'], files=files, + model_cards=[dict(card=f"model-cards/{r['stage']}/{r['variant']}/README.md", model=r['path'], + model_sha256=r['identity']['sha256'], selected=r['selected']) for r in data['models']]) + evidence.write(output / 'card-receipt.json', receipt) + return dict(status='complete', output=str(output), cards=len(files), qualification=data['report']['qualification'], + receipt_sha256=evidence.identity(output / 'card-receipt.json')['sha256']) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + for name in ('study', 'output', 'summary', 'source-commit'): + parser.add_argument('--' + name, required=True) + for name in ('hardware', 'study-audit', 'phase-coverage', 'diagnostic-audit'): + parser.add_argument('--' + name) + args = parser.parse_args(argv) + try: + print(json.dumps(render(args.study, args.output, summary=args.summary, + source_commit=args.source_commit, hardware=args.hardware, study_audit=args.study_audit, + phase_coverage=args.phase_coverage, diagnostic_audit=args.diagnostic_audit), indent=2)) + return 0 + except (ValueError, OSError, KeyError, TypeError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc)))) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/test_camera_action_model_cards.py b/publishing/test_camera_action_model_cards.py new file mode 100644 index 0000000..1b16155 --- /dev/null +++ b/publishing/test_camera_action_model_cards.py @@ -0,0 +1,287 @@ +"""Card bytes and publication bindings, with synthetic facts and no native work.""" +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import prepare_camera_action_v01 as package +import render_camera_action_card as renderer +import publish_camera_action_v01 as publisher +from huggingface.camera_action_model_cards import validate_receipt, RECEIPT_FILE + + +class ModelCardBundleTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name).resolve(); self.study = self.root/'study' + self.cards = self.root/'cards'; self.commit = 'a'*40 + self.summary = dict(controllers=[], training=[], hardware=[]) + self.report = dict(status='blocked', qualification='research_candidate', qualified=False, + physical_robot_ready=False, selected_stage=None, gates={}) + self.state = dict(report=self.report, status='blocked', qualified=False, qualification='research_candidate') + self.protocol = {'fixture': 'synthetic pure publication test'} + for name, value in [('protocol.json', self.protocol), ('registration.json', {}), ('report.json', self.report)]: + package.write(self.study/name, value) + self.receipts = {}; self.models = []; self.records = [] + fitted = {} + for variant in ('full', 'blind'): + name = f'models/initial/{variant}/model.json' + model = dict(blind=variant == 'blind', training=dict(stage='initial', sample_count=1)) + package.write(self.study/name, model) + identity = package.identity(self.study/name) + record = dict(path=name, model_sha256=identity['sha256'], model_bytes=identity['bytes'], + training=model['training'], training_episodes=['synthetic'], development_episodes=[]) + package.write(self.study/f'models/initial/{variant}/receipt.json', record) + package.write(self.study/f'models/initial/{variant}/started.json', {'synthetic': True}) + fitted[variant] = record; self.models.append(name) + self.records.append(dict(stage='initial', variant=variant, path=name, + identity=identity, selected=False, training=record, model=model)) + self.receipts['train-initial'] = dict(status='complete', models=fitted) + for suffix in ('.json', '.integrity.json', '.started.json'): + package.write(self.study/('stages/train-initial'+suffix), self.receipts['train-initial']) + self.metadata = {p.relative_to(self.study).as_posix(): package.identity(p) for p in package.files(self.study)} + self.bindings = dict(self.metadata) + self.data = dict(study=self.study, source_commit=self.commit, state=self.state, + protocol=self.protocol, report=self.report, receipts=self.receipts, summary=self.summary, + models=self.records, hardware=None, audit=None, phase=None, study_bindings=self.bindings) + self.make_cards() + + def make_cards(self): + texts = {'README.md': renderer.dataset_card(self.data)} + texts.update({f"model-cards/{r['stage']}/{r['variant']}/README.md": renderer.model_card(self.data, r) + for r in self.records}) + self.cards.mkdir(exist_ok=True) + (self.cards/'summary.json').write_bytes(package.canonical(self.summary)) + for name, text in texts.items(): + path = self.cards/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_text(text) + self.receipt = dict(schema=renderer.SCHEMA, source_commit=self.commit, + protocol_sha256=self.metadata['protocol.json']['sha256'], report_sha256=self.metadata['report.json']['sha256'], + status=self.report['status'], qualification=self.report['qualification'], qualified=False, + full_study_audited=False, physical_robot_ready=False, summary_reconstructed=True, + physics_executed=False, model_executed=False, + source_inputs={'summary.json': package.identity(self.cards/'summary.json'), + 'generator.py': package.identity(Path(renderer.__file__)), + 'summarize_results.py': package.identity(renderer.SUMMARY_SOURCE)}, + study_bindings=self.bindings, files={name: package.identity(self.cards/name) for name in texts}, + model_cards=[dict(card=f"model-cards/{r['stage']}/{r['variant']}/README.md", model=r['path'], + model_sha256=r['identity']['sha256'], selected=r['selected']) for r in self.records]) + self.save_receipt() + + def save_receipt(self): + (self.cards/'card-receipt.json').write_bytes(package.canonical(self.receipt)+b'\n') + + def check(self, *, audit='unused'): + # Model schema is independently tested by the learner suite. Here the + # mock admits visibly synthetic weights to isolate publication binding. + with patch('dvidia_training.camera_action_model.validate_model', side_effect=lambda value:value), \ + patch.object(renderer, 'recompute_summary', return_value=self.summary): + return package.checked_model_cards(self.cards, study=self.study, card=self.cards/'README.md', + source_commit=self.commit, state=self.state, protocol=self.protocol, + receipts=self.receipts, metadata=self.metadata, models=list(reversed(self.models)), audit=audit) + + def test_exact_generated_pair_and_correct_relative_download_links(self): + result = self.check() + self.assertEqual(len(result['cards']), 2) + self.assertEqual(set(result['files']), set(result['cards'])|{RECEIPT_FILE}) + for variant in ('full', 'blind'): + text = (self.cards/f'model-cards/initial/{variant}/README.md').read_text() + self.assertIn(f'[model.json](../../../models/initial/{variant}/model.json)', text) + self.assertIn('[Read card](model-cards/initial/full/README.md)', (self.cards/'README.md').read_text()) + + def test_text_tamper_is_refused_even_after_receipt_rehash(self): + name = 'model-cards/initial/full/README.md' + (self.cards/name).write_text('Fabricated perfect hardware success.') + self.receipt['files'][name] = package.identity(self.cards/name); self.save_receipt() + with self.assertRaisesRegex(ValueError, 'fresh sealed-evidence rendering'): self.check() + + def test_selected_model_hash_report_protocol_commit_and_roster_tampering_refuse(self): + original = deepcopy(self.receipt) + for field in ('selected', 'model_sha256', 'report_sha256', 'protocol_sha256', 'source_commit', 'duplicate'): + with self.subTest(field=field): + self.receipt = deepcopy(original) + if field == 'selected': self.receipt['model_cards'][0][field] = True + elif field == 'model_sha256': self.receipt['model_cards'][0][field] = '0'*64 + elif field == 'duplicate': self.receipt['model_cards'].append(deepcopy(self.receipt['model_cards'][0])) + else: self.receipt[field] = '0'*(40 if field=='source_commit' else 64) + self.save_receipt() + with self.assertRaises(ValueError): self.check() + + def test_extra_file_empty_directory_symlink_and_secret_are_refused(self): + extra = self.cards/'extra.md'; extra.write_text('arbitrary') + with self.assertRaisesRegex(ValueError, 'arbitrary files'): self.check() + extra.unlink(); extra.mkdir() + with self.assertRaisesRegex(ValueError, 'arbitrary empty'): self.check() + extra.rmdir(); extra.symlink_to(self.cards/'README.md') + with self.assertRaisesRegex(ValueError, 'symbolic'): self.check() + extra.unlink() + path=self.cards/'model-cards/initial/full/README.md'; path.write_text('hf_'+'A'*32) + with self.assertRaisesRegex(ValueError, 'Credential-shaped'): self.check() + + def test_changed_generator_binding_cannot_reuse_stale_cards(self): + self.receipt['source_inputs']['generator.py']['sha256'] = '0'*64; self.save_receipt() + with self.assertRaisesRegex(ValueError, 'generator or auxiliary'): self.check() + + def test_summary_input_identity_and_reconstructed_values_are_both_bound(self): + original = deepcopy(self.receipt) + self.receipt['source_inputs']['summary.json']['sha256'] = '0'*64; self.save_receipt() + with self.assertRaisesRegex(ValueError, 'generator or auxiliary'): self.check() + self.receipt = original + (self.cards/'summary.json').write_bytes(package.canonical({**self.summary, 'invented_latency': 0.00001})) + self.receipt['source_inputs']['summary.json'] = package.identity(self.cards/'summary.json'); self.save_receipt() + with self.assertRaisesRegex(ValueError, 'Retained card summary'): self.check() + + def test_standalone_diagnostic_audit_is_supported_and_bound_to_actual_input(self): + audit = self.root/'audit.json'; package.write(audit, {'fixture': 'diagnostic audit'}) + self.receipt['source_inputs']['independent-audit.json'] = package.identity(audit); self.save_receipt() + self.check(audit=audit) + audit.write_bytes(package.canonical({'fixture': 'different audit'})) + with self.assertRaisesRegex(ValueError, 'generator or auxiliary'): self.check(audit=audit) + + def test_phase_helper_cannot_be_claimed_without_coverage_and_audit(self): + self.receipt['source_inputs']['diagnostic_phase_coverage.py'] = package.identity(renderer.PHASE_SOURCE) + self.save_receipt() + with self.assertRaisesRegex(ValueError, 'phase provenance'): self.check() + + def test_source_archive_must_contain_exact_bound_card_helpers(self): + import io + import tarfile + source = self.root/'source.tar.gz' + helpers = {'publishing/render_camera_action_card.py': Path(renderer.__file__).read_bytes(), + 'benchmarks/camera-action-v01/summarize_results.py': renderer.SUMMARY_SOURCE.read_bytes()} + def archive(values): + with tarfile.open(source, 'w:gz') as output: + for name, value in values.items(): + info = tarfile.TarInfo('fixture/'+name); info.size = len(value) + output.addfile(info, io.BytesIO(value)) + archive(helpers); package.verify_card_source_archive(source, self.receipt) + name = 'publishing/render_camera_action_card.py' + archive({**helpers, name: b'x'*len(helpers[name])}) + with self.assertRaisesRegex(ValueError, 'differs from the bound card'): package.verify_card_source_archive(source, self.receipt) + archive({name: helpers[name]}) + with self.assertRaisesRegex(ValueError, 'each card implementation'): package.verify_card_source_archive(source, self.receipt) + + def test_preparation_copies_generated_pair_receipt_and_models_into_public_payload(self): + import io + import tarfile + import zipfile + from dvidia_training import camera_action_study as workflow + runtime=b'"""Synthetic runtime fixture; never executed."""\n' + self.protocol['source_sha256']={'camera_action_study.py':sha256(runtime).hexdigest()} + (self.study/'protocol.json').write_bytes(package.canonical(self.protocol)) + rows=[] + for index in range(4): + name=f'diagnostic-{index}-teacher';prefix='episodes/diagnostic/'+name + package.write(self.study/prefix/'started.json',{'fixture':True}) + package.write(self.study/prefix/'rollout.json',{'status':'complete','fixture':True}) + package.write(self.study/prefix/'integrity.json',dict(schema='dvidia.camera-action-rollout/v2/integrity', + files={n:package.identity(self.study/prefix/n) for n in ('started.json','rollout.json')})) + rows.append(dict(id=name,case=dict(id=name,group_id=f'dev-{index}',split='development'), + path=prefix,controller='teacher',negative_control=None,status='complete',strict_success=False, + receipt_sha256=package.identity(self.study/prefix/'rollout.json')['sha256'], + integrity_sha256=package.identity(self.study/prefix/'integrity.json')['sha256'])) + self.receipts['diagnostic']=dict(status='blocked',rows=rows) + for suffix in ('.json','.integrity.json','.started.json'): + package.write(self.study/('stages/diagnostic'+suffix),self.receipts['diagnostic']) + self.metadata={p.relative_to(self.study).as_posix():package.identity(p) for p in package.files(self.study) + if not p.is_relative_to(self.study/'episodes')} + self.bindings=dict(self.metadata);self.data['study_bindings']=self.bindings + self.make_cards() + audit=self.root/'audit.json' + package.write(audit,dict(schema='dvidia.camera-action-diagnostic-audit/v1',passed=True,errors=[], + attempts_verified=4,physics_executed=False,estimator_executed=False, + results=[dict(case_id=r['case']['id'],strict_success=False) for r in rows])) + wheel=self.root/'fixture.whl';source=self.root/'source.tar.gz' + with zipfile.ZipFile(wheel,'w') as archive: + archive.writestr('dvidia_training/camera_action_study.py',runtime) + with tarfile.open(source,'w:gz',format=tarfile.PAX_FORMAT,pax_headers={'comment':self.commit}) as archive: + info=tarfile.TarInfo('fixture/src/dvidia_training/camera_action_study.py');info.size=len(runtime) + archive.addfile(info,io.BytesIO(runtime)) + for name, path in [('publishing/render_camera_action_card.py', Path(renderer.__file__)), + ('benchmarks/camera-action-v01/summarize_results.py', renderer.SUMMARY_SOURCE)]: + value=path.read_bytes();info=tarfile.TarInfo('fixture/'+name);info.size=len(value) + archive.addfile(info,io.BytesIO(value)) + output=self.root/'prepared' + with patch.object(workflow,'inspect',return_value=self.state), \ + patch.object(workflow,'_stage',side_effect=lambda _root,name:self.receipts.get(name)), \ + patch.object(workflow,'_require',side_effect=lambda _root,name:self.receipts[name]), \ + patch.object(renderer,'recompute_summary',return_value=self.summary), \ + patch('dvidia_training.camera_action_model.validate_model',side_effect=lambda value:value): + result=package.prepare(self.study,output,audit=audit,card=self.cards/'README.md',wheel=wheel, + source_archive=source,source_commit=self.commit,model_cards=self.cards) + self.assertEqual(result['trained_models'],2) + manifest=package.read(output/'manifest.json') + self.assertEqual(len(manifest['model_card_files']),2) + self.assertEqual(manifest['model_card_receipt_file'],RECEIPT_FILE) + for name in manifest['model_card_files']: + self.assertEqual((output/name).read_bytes(),(self.cards/name).read_bytes()) + self.assertEqual((output/RECEIPT_FILE).read_bytes(),(self.cards/'card-receipt.json').read_bytes()) + self.assertEqual(len(publisher.files(output)),len(manifest['files'])+2) + + def test_blocked_without_models_has_empty_cards_and_a_bound_receipt(self): + import shutil + shutil.rmtree(self.study/'models'); shutil.rmtree(self.study/'stages'); shutil.rmtree(self.cards) + self.models=[]; self.records=[]; self.receipts={} + self.metadata={p.name:package.identity(p) for p in package.files(self.study)}; self.bindings=dict(self.metadata) + self.data.update(models=[],receipts={},study_bindings=self.bindings) + self.make_cards(); result=self.check() + self.assertEqual(result['cards'],[]);self.assertEqual(set(result['files']),{RECEIPT_FILE}) + + def make_publication(self): + root=self.root/'publication';root.mkdir() + for name in publisher.REQUIRED-{'manifest.json','SHA256SUMS'}: + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(b'Synthetic publication fixture.') + (root/'README.md').write_bytes((self.cards/'README.md').read_bytes()) + for name in ('protocol.json','registration.json','report.json',*self.models): + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes((self.study/name).read_bytes()) + for name in self.receipt['files']: + if name=='README.md':continue + path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes((self.cards/name).read_bytes()) + (root/RECEIPT_FILE).write_bytes((self.cards/'card-receipt.json').read_bytes()) + for name in ('source/fixture.whl','source/fixture.tar.gz'): + path=root/name;path.parent.mkdir(exist_ok=True);path.write_bytes(b'fixture') + self.public=root + self.manifest=dict(schema=publisher.EVIDENCE_SCHEMA,repo_id=publisher.REPO,repo_type='dataset', + status='blocked',qualification='research_candidate',qualified=False,original_footage_included=False, + physical_robot_ready=False,source_commit=self.commit,episodes=[],episode_count=0, + trained_model_files=self.models,model_card_receipt_file=RECEIPT_FILE, + model_card_files=sorted(r['card'] for r in self.receipt['model_cards'])) + self.seal_publication();return root + + def seal_publication(self): + inventory={p.relative_to(self.public).as_posix():publisher.identity(p) for p in self.public.rglob('*') + if p.is_file() and p.name not in ('manifest.json','SHA256SUMS')} + self.manifest['files']=inventory + (self.public/'manifest.json').write_bytes(package.canonical(self.manifest)) + inventory=dict(inventory,**{'manifest.json':publisher.identity(self.public/'manifest.json')}) + (self.public/'SHA256SUMS').write_text(''.join(f'{r["sha256"]} {n}\n' for n,r in sorted(inventory.items()))) + + def test_publisher_accepts_only_exact_receipted_cards_without_credentials(self): + public=self.make_publication() + with patch.object(publisher,'client',side_effect=AssertionError('credential access')): + result=publisher.files(public) + self.assertIn(RECEIPT_FILE,{r['path'] for r in result}) + self.manifest['model_card_files']=[];self.seal_publication() + with self.assertRaisesRegex(publisher.PublicationError,'files or provenance'):publisher.files(public) + + def test_publisher_rejects_changed_card_even_with_resealed_payload_manifest(self): + public=self.make_publication() + (public/'model-cards/initial/full/README.md').write_text('Changed after review.') + self.seal_publication() + with self.assertRaisesRegex(publisher.PublicationError,'files or provenance'):publisher.files(public) + + def test_publisher_refuses_missing_receipt_selection_and_arbitrary_paths(self): + public=self.make_publication() + self.manifest['model_card_receipt_file']=None;self.seal_publication() + with self.assertRaisesRegex(publisher.PublicationError,'explicit receipt'):publisher.files(public) + self.manifest['model_card_receipt_file']=RECEIPT_FILE + value=deepcopy(self.receipt);value['model_cards'][0]['selected']=True + (public/RECEIPT_FILE).write_bytes(package.canonical(value));self.seal_publication() + with self.assertRaisesRegex(publisher.PublicationError,'receipt differs'):publisher.files(public) + self.assertFalse(publisher.allowed('model-cards/initial/full/extra.py')) + self.assertFalse(publisher.allowed('model-cards/foreign/full/README.md')) + + +if __name__=='__main__':unittest.main() diff --git a/publishing/test_episode_archive_cache.py b/publishing/test_episode_archive_cache.py new file mode 100644 index 0000000..7bbb5a0 --- /dev/null +++ b/publishing/test_episode_archive_cache.py @@ -0,0 +1,281 @@ +from copy import deepcopy +import json +from pathlib import Path +import shutil +import stat +import struct +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch +import zipfile + +import episode_archive_cache as cache +import prepare_camera_action_v01 as evidence + + +class EpisodeArchiveCacheTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.study = self.root / 'study'; self.study.mkdir() + self.cache = self.root / 'cache' + self.prefix = 'episodes/diagnostic/case-teacher' + self.episode = self.study / self.prefix + self.make_episode(self.episode) + self.inventory = evidence.episode_inventory(self.episode) + self.row = dict(id='case-teacher', case={'id': 'case', 'split': 'development'}, + controller='teacher', negative_control=None, path=self.prefix, status='complete', + complete=True, recorded_complete_intervals=1000, strict_success=False, + receipt_sha256=evidence.identity(self.episode / 'rollout.json')['sha256'], + integrity_sha256=evidence.identity(self.episode / 'integrity.json')['sha256']) + self.protocol = dict(rosters={name: [] for name in cache.STAGES}, + storage={'maximum_study_bytes': evidence.MAX_TOTAL}) + self.protocol['rosters']['diagnostic'] = [{k: self.row[k] for k in ('id', 'case', 'controller', 'negative_control')}] + evidence.write(self.study / 'protocol.json', self.protocol) + self.protocol_hash = evidence.identity(self.study / 'protocol.json')['sha256'] + evidence.write(self.study / 'registration.json', {'protocol_sha256': self.protocol_hash}) + self.seal_stage() + + def make_episode(self, root, status='complete'): + evidence.write(root / 'started.json', {'synthetic_test_fixture': True}) + evidence.write(root / 'rollout.json', dict(status=status, + recorded_complete_intervals=1000 if status == 'complete' else 0)) + evidence.write(root / 'integrity.json', dict(schema='dvidia.camera-action-rollout/v2/integrity', + files={p.name: evidence.identity(p) for p in root.iterdir()})) + + def seal_stage(self): + path = self.study / 'stages/diagnostic.json' + path.parent.mkdir(exist_ok=True) + path.write_bytes(evidence.canonical(dict(schema='dvidia.camera-action-study/v1/stage', + stage='diagnostic', status='complete', scheduled=1, + protocol_sha256=self.protocol_hash, rows=[self.row]))) + path.with_name('diagnostic.integrity.json').write_bytes(evidence.canonical(dict( + sha256=evidence.identity(path)['sha256'], protocol_sha256=self.protocol_hash))) + + def seed_cache(self): + directory = self.cache / cache.key(self.inventory, self.prefix) + directory.mkdir(parents=True) + evidence.archive(self.episode, self.inventory, directory / 'episode.zip', prefix=self.prefix) + return directory + + @staticmethod + def private_copy(source, destination, record): + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open('xb') as target: + target.write(source.read_bytes()) + + def test_hit_checks_source_and_every_member_without_trusting_receipt(self): + directory = self.seed_cache() + (directory / 'receipt.json').write_text('invalid, explicitly not trusted') + original = cache.validate_archive(directory / 'episode.zip', self.inventory, self.prefix) + destination = self.root / 'output/episode.zip' + with patch.object(cache, '_private_copy', side_effect=self.private_copy): + result = cache.reuse(self.episode, self.inventory, destination, prefix=self.prefix, cache=self.cache) + self.assertEqual(result, original) + self.assertEqual(destination.read_bytes(), (directory / 'episode.zip').read_bytes()) + self.assertNotEqual(destination.stat().st_ino, (directory / 'episode.zip').stat().st_ino) + (directory / 'episode.zip').write_bytes(b'changed source') + self.assertEqual(evidence.identity(destination)['sha256'], original['sha256']) + + def test_miss_does_not_create_destination_or_copy(self): + with patch.object(cache, '_private_copy') as copy: + self.assertIsNone(cache.reuse(self.episode, self.inventory, self.root / 'missing.zip', + prefix=self.prefix, cache=self.cache)) + copy.assert_not_called() + self.assertFalse((self.root / 'missing.zip').exists()) + + def test_original_mutation_refuses_even_when_cached_members_are_valid(self): + self.seed_cache() + (self.episode / 'started.json').write_text('{}') + with patch.object(cache, '_private_copy') as copy, self.assertRaisesRegex(ValueError, 'Original episode'): + cache.reuse(self.episode, self.inventory, self.root / 'wrong.zip', prefix=self.prefix, cache=self.cache) + copy.assert_not_called() + + def test_zip_member_tampering_roster_types_and_path_escape_refuse(self): + source = self.seed_cache() / 'episode.zip' + with zipfile.ZipFile(source) as archive: + members = [(item, archive.read(item)) for item in archive.infolist()] + for kind in ('bytes', 'extra', 'duplicate', 'symlink', 'path', 'timestamp', 'stored'): + with self.subTest(kind=kind): + target = self.root / (kind + '.zip') + with zipfile.ZipFile(target, 'x') as out: + for index, (item, raw) in enumerate(members): + item = deepcopy(item) + if index == 0: + if kind == 'bytes': raw = b'X' * len(raw) + if kind == 'symlink': item.external_attr = (stat.S_IFLNK | 0o777) << 16 + if kind == 'path': item.filename = '../escape' + if kind == 'timestamp': item.date_time = (2026, 1, 1, 0, 0, 0) + if kind == 'stored': item.compress_type = zipfile.ZIP_STORED + out.writestr(item, raw) + if kind == 'extra': out.writestr('extra', b'no') + if kind == 'duplicate': + import warnings + with warnings.catch_warnings(): + warnings.simplefilter('ignore', UserWarning) + out.writestr(members[0][0], members[0][1]) + with self.assertRaises(ValueError): + cache.validate_archive(target, self.inventory, self.prefix) + + def test_valid_zip_comments_extra_fields_order_prefix_and_trailing_bytes_refuse(self): + source = self.seed_cache() / 'episode.zip' + original = source.read_bytes() + with zipfile.ZipFile(source) as archive: + members = [(item, archive.read(item)) for item in archive.infolist()] + for kind in ('archive_comment', 'member_comment', 'extra', 'reverse', 'prefix', 'trailing', 'local_extra'): + with self.subTest(kind=kind): + target = self.root / (kind + '.zip') + if kind in ('prefix', 'trailing', 'local_extra'): + raw = bytearray(original) + if kind == 'prefix': raw = b'unreviewed prefix' + raw + if kind == 'trailing': raw += b'unreviewed trailing payload' + if kind == 'local_extra': + offset = 30 + len(members[0][0].filename.encode()) + raw[offset:offset+2] = struct.pack('|`\nnext'), '<script>\\|` next') + data = cards._inputs(self.root, summary=self.summary, source_commit='a' * 40) + data['receipts']['diagnostic']['reason'] = '|\nnew row' + text = cards.dataset_card(data) + self.assertNotIn(' '); -} -const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); -const source = path.resolve(sourceArg), outfile = path.resolve(outputArg); -if (source === outfile || /piper_feedback_viewer\.bundle\.js$/.test(outfile)) { - throw new Error('The v0.3 build requires a distinct output path.'); -} -const vendorRoot = existsSync(path.join(root, 'viewer/node_modules/esbuild')) - ? root - : path.resolve(root, '../dvidia-training'); -const require = createRequire(path.join(vendorRoot, 'viewer/package.json')); -for (const [name, version] of [['esbuild', '0.25.10'], ['three', '0.180.0']]) { - const packagePath = path.join(vendorRoot, 'viewer/node_modules', name, 'package.json'); - const installed = JSON.parse(await readFile(packagePath, 'utf8')).version; - if (installed !== version) throw new Error(`Expected pinned ${name} ${version}; found ${installed}.`); -} -const {build} = require('esbuild'); -await build({ - entryPoints: [source], outfile, bundle: true, minify: true, format: 'iife', - target: ['es2020'], platform: 'browser', - alias: { - three: path.join(vendorRoot, 'viewer/node_modules/three/build/three.module.js'), - 'three/addons/controls/OrbitControls.js': path.join(vendorRoot, 'viewer/node_modules/three/examples/jsm/controls/OrbitControls.js'), - }, - nodePaths: [path.join(vendorRoot, 'viewer/node_modules')], legalComments: 'none', charset: 'utf8', - banner: {js: '/* DVIDIA PiPER grip clock replay v0.3. MIT. Bundled Three.js r180, MIT; https://github.com/mrdoob/three.js/tree/r180 */'}, -}); -const output = await readFile(outfile); -if (output.byteLength > 8 * 1024 * 1024) throw new Error('Compiled viewer exceeds the 8 MiB bundle limit.'); -console.log(`Built ${outfile} (${output.byteLength} bytes).`); +// Keep the checkout entry point; wheels carry the actual pinned builder. +import '../src/dvidia_training/piper_grip_clock_build.mjs'; diff --git a/src/dvidia_training/arm_pack.py b/src/dvidia_training/arm_pack.py index 72362af..4f8069f 100644 --- a/src/dvidia_training/arm_pack.py +++ b/src/dvidia_training/arm_pack.py @@ -6,6 +6,36 @@ from .skillspace import ADAPTER_ID, ARM_PROFILE, load_skillspace from .installer import runtime_contract, RUNTIME_SOURCE_FILES +# Exact closure for the documented simlab.studio and simlab.arm_qualify entry +# points. Unrelated installed products and user-local package files are excluded. +BUNDLE_SOURCE_FILES = ( + '__init__.py', 'arm_distill.py', 'arm_env.py', 'arm_pack.py', + 'arm_policy.py', 'arm_qualify.py', 'arm_runner.py', 'capsule_installer.py', + 'cli.py', 'env.py', 'installer.py', 'pack.py', 'policies.py', + 'skill_capsule.py', 'skillspace.py', 'studio.py', 'viewer.py', +) +BUNDLE_RESOURCE_FILES = ('studio.html', 'place_cup.skill.json', 'requirements.lock.txt', 'LICENSE') +BUNDLE_OPTIONAL_FILES = ('arm_benchmark_summary.json', 'placement_candidate.skill.json') + + +def _bundle_files(root): + root = Path(root) + files = {} + for name in (*BUNDLE_SOURCE_FILES, *BUNDLE_RESOURCE_FILES): + path = root/name + if path.is_symlink() or not path.is_file(): + raise ValueError('Required arm bundle dependency is absent or not an ordinary file: '+name) + files['simlab/'+name] = path.read_bytes() + # Include separately generated releases only when supplied by the package; + # do not fabricate a benchmark or placement candidate to complete an archive. + for name in BUNDLE_OPTIONAL_FILES: + resource = root/name + if resource.is_symlink() or resource.exists(): + if resource.is_symlink() or not resource.is_file(): + raise ValueError('Optional arm bundle dependency must be an ordinary file: '+name) + files['simlab/'+name] = resource.read_bytes() + return files + def export_arm_pack(directory): directory=Path(directory) @@ -17,10 +47,7 @@ def export_arm_pack(directory): source=(directory/'source.json').read_bytes() if sha256(source).hexdigest()!=payload['skill']['source_sha256']: raise ValueError('Skill source hash does not match its exact run input.') - files={} - for path in Path(__file__).parent.iterdir(): - if path.is_file() and (path.suffix in ('.py','.html','.json','.txt') or path.name=='LICENSE'): - files['simlab/'+path.name]=path.read_bytes() + files=_bundle_files(Path(__file__).parent) for name in ('run.json','source.json','grounding.json'): if (directory/name).exists(): files['evidence/'+name]=(directory/name).read_bytes() diff --git a/src/dvidia_training/arm_skill_install.py b/src/dvidia_training/arm_skill_install.py index 2bbc3a3..6dfc730 100644 --- a/src/dvidia_training/arm_skill_install.py +++ b/src/dvidia_training/arm_skill_install.py @@ -54,6 +54,10 @@ def check_model(model_path, robot, *, interface=INTERFACE): raise ValueError('This learned policy requires PiPER. Other arms need their own trained ' 'policy and task benchmark; matching joint counts do not transfer it.') raw = _read(model_path, MAX_MODEL_BYTES) + decoded = _decode(raw) + if type(decoded) is dict and decoded.get('schema') == 'dvidia.camera-action-mlp/v1': + from .camera_action_skill import check + return check(model_path, robot, interface=interface) model = validate_model(raw) return { 'compatible': True, 'runner': RUNNER, 'robot': robot, 'interface_id': interface, @@ -83,6 +87,9 @@ def _record(check, binding): def install_model(model, robot, assets, output, *, interface=INTERFACE): """Verify model/assets, then reserve a fresh installation without stepping.""" check = check_model(model, robot, interface=interface) + if check['runner'] == 'piper-camera-action-v1': + from .camera_action_skill import install + return install(model, robot, assets, output, interface=interface) raw = _read(model, MAX_MODEL_BYTES) if sha256(raw).hexdigest() != check['model_sha256']: raise ValueError('Model changed during compatibility check.') @@ -105,6 +112,9 @@ def inspect_installation(directory, assets): if root.is_symlink() or not root.is_dir(): raise ValueError('Installation must be a regular directory.') stored = _decode(_read(root/'installation.json', 256*1024)) + if type(stored) is dict and stored.get('schema') == 'dvidia.camera-action-installation/v1': + from .camera_action_skill import inspect + return inspect(root, assets) if type(stored) is not dict or stored.get('schema') != SCHEMA: raise ValueError('Unknown arm skill installation.') compatibility = stored.get('compatibility') @@ -122,6 +132,9 @@ def inspect_installation(directory, assets): def run_installation(directory, assets, output, *, start_x=.30, start_y=-.03, goal=0): """Revalidate, then dispatch only to the built-in bounded simulation runner.""" record = inspect_installation(directory, assets) + if record['compatibility']['runner'] == 'piper-camera-action-v1': + from .camera_action_skill import run + return run(directory, assets, output, start_x=start_x, start_y=start_y, goal=goal) from .piper_learned_skill import run_learned_skill result = run_learned_skill(Path(directory)/'model.json', assets, output, start_x=start_x, start_y=start_y, goal=goal) diff --git a/src/dvidia_training/camera_action_capture.py b/src/dvidia_training/camera_action_capture.py new file mode 100644 index 0000000..da50936 --- /dev/null +++ b/src/dvidia_training/camera_action_capture.py @@ -0,0 +1,560 @@ +"""Causal camera/action rollout v2 for the bounded simulated PiPER task. + +Rendered observations, issued commands, privileged teacher suggestions, and +native scoring are distinct artifacts. This is an offline simulation recorder; +it makes no real-time delivery or physical-hardware qualification claim. +""" +from __future__ import annotations + +from copy import deepcopy +from dataclasses import asdict, dataclass +from hashlib import sha256 +import io +import json +import math +from pathlib import Path +import random +import time + +from . import action_demonstration as tape +from . import piper_action_capture as base +from . import piper_rgbd_tracker as tracker + +SCHEMA = 'dvidia.camera-action-rollout/v2' +PERCEPTION_SCHEMA = 'dvidia.camera-action-perception/v1' +TEACHER_SCHEMA = 'dvidia.camera-action-teacher-query/v1' +MAX_ROLLOUT_BYTES = 64 * 1024 * 1024 +MODES = ('teacher', 'student', 'mixed', 'fixed') +NEGATIVES = (None, 'initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper') +SCOPE = ('Offline native simulation with authored teacher demonstrations and ' + 'current-frame RGB-D estimates. No hardware commands, physical delivery ' + 'latency, human-video motor recovery or general object tracking.') + + +@dataclass(frozen=True) +class RolloutConfig: + duration_s: float = 20. + seed: int = 0 + mode: str = 'authored_task' + width: int = 320 + height: int = 240 + camera_every_controls: int = 2 + interrupt_after: int | None = None + + def __post_init__(self): + value = self.duration_s + if (type(value) not in (int, float) or not math.isfinite(value) or not .1 <= value <= 20. + or not math.isclose(value / tape.DT, round(value / tape.DT), abs_tol=1e-9, rel_tol=0)): + raise ValueError('Duration requires complete 20 ms intervals in [0.1,20] seconds.') + if type(self.seed) is not int or not 0 <= self.seed <= 999999: + raise ValueError('Seed must be an integer in [0,999999].') + if (self.mode != 'authored_task' or type(self.width) is not int or type(self.height) is not int + or self.width != 320 or self.height != 240 + or type(self.camera_every_controls) is not int or self.camera_every_controls != 2): + raise ValueError('The registered camera contract is 320x240 RGB-D at 25 Hz.') + count = round(value / tape.DT) + if self.interrupt_after is not None and (type(self.interrupt_after) is not int + or not 0 <= self.interrupt_after < count): + raise ValueError('Interruption must precede a scheduled interval.') + pairs = count // self.camera_every_controls + 1 + if (self.width * self.height * 7 + 4096) * pairs > tape.MAX_FRAME_TOTAL: + raise ValueError('Unique camera frames exceed the existing 512 MiB frame budget.') + + +def _case(case): + if type(case) is not dict: + raise ValueError('A registered case dictionary is required.') + result = deepcopy(case) + for name in ('id', 'group_id'): + if not isinstance(case.get(name), str) or not case[name] or len(case[name]) > 200: + raise ValueError('Case requires bounded nonempty '+name) + if type(case.get('seed')) is not int or not 0 <= case['seed'] <= 999999: + raise ValueError('Case requires a bounded seed.') + from .piper_task import PiperTaskConfig + config = PiperTaskConfig(object_position_m=case.get('object_position_m'), + target_position_m=case.get('goal_position_m')) + result['object_position_m'] = list(config.object_position_m) + result['goal_position_m'] = list(config.target_position_m) + return result + + +def _identity(path): + path = Path(path) + return dict(bytes=path.stat().st_size, sha256=sha256(path.read_bytes()).hexdigest()) + + +def _artifact(root, relative, document): + identity = base._json(root / relative, document) + identity['path'] = relative + return identity + + +def _sample_at(sample, seconds): + result = deepcopy(sample) + if result['status'] == 'available': + result['age_seconds'] = max(0., seconds - result['captured_at_seconds']) + return result + + +def _estimate_at(estimate, seconds): + """Reuse one captured frame, retaining its original acquisition timestamp.""" + result = deepcopy(estimate) + result['decision_at_seconds'] = seconds + if result.get('captured_at_seconds') is not None: + result['age_seconds'] = max(0., seconds - result['captured_at_seconds']) + for name, sample in result.get('signals', {}).items(): + result['signals'][name] = _sample_at(sample, seconds) + if result.get('status') == 'available' and result['age_seconds'] > .1 + 1e-12: + result['status'], result['reason'] = 'stale', 'source_frame_too_old' + for name in tracker.SIGNALS: + result[name] = None + result['signals'][name] = base._missing() + return result + + +class _Session(base._NativeSession): + def __init__(self, assets, config, case): + super().__init__(assets, config) + self.case = case + + def _task_configuration(self, task): + return task.PiperTaskConfig(timeout_s=self.config.duration_s, + object_position_m=self.case['object_position_m'], + target_position_m=self.case['goal_position_m']) + + def _environment(self, task): + from .piper_feedback_env import FeedbackPiperTaskEnv + return FeedbackPiperTaskEnv(self.assets, self.task_config) + + def open(self): + header, targets, metadata = super().open() + rotation = self.env.data.xmat[self.base].reshape(3, 3) + transform = self.np.eye(4) + transform[:3, :3] = rotation + transform[:3, 3] = self.env.data.xpos[self.base] + if (not self.np.isfinite(transform).all() + or not self.np.allclose(rotation.T @ rotation, self.np.eye(3), atol=1e-8, rtol=0) + or not math.isclose(float(self.np.linalg.det(rotation)), 1., abs_tol=1e-8, rel_tol=0)): + raise ValueError('Native robot base transform is not finite and rigid.') + metadata['world_from_robot_base'] = transform.reshape(16).tolist() + metadata['world_from_robot_base_sha256'] = tape.digest(metadata['world_from_robot_base']) + metadata['base_transform_scope'] = 'Fixed native world/base diagnostic transform; excluded from policy inputs.' + return header, targets, metadata + + def measured(self, previous_camera): + """50 Hz encoders/FK and the explicitly old 25 Hz camera frame.""" + seconds = float(self.env.elapsed) + if not math.isclose(float(self.env.data.time)-self.native_origin, seconds, + abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise RuntimeError('Native and declared clocks differ.') + rotation = self.env.data.xmat[self.base].reshape(3, 3) + tool_rotation = rotation.T @ self.env.data.xmat[self.tool].reshape(3, 3) + quaternion = self.np.empty(4) + self.mj.mju_mat2Quat(quaternion, tool_rotation.reshape(9)) + values = dict(joint_position_rad=self.env.data.qpos[:6].tolist(), + joint_velocity_rad_s=self.env.data.qvel[:6].tolist(), + gripper_joint_position_m=self.env.data.qpos[6:8].tolist(), + gripper_joint_velocity_m_s=self.env.data.qvel[6:8].tolist(), + tcp_position_estimate_m=(rotation.T @ (self.env.data.xpos[self.tool]-self.env.data.xpos[self.base])).tolist(), + tcp_quaternion_estimate_wxyz=quaternion.tolist()) + signals = {name: base._missing() for name in tape.SIGNALS} + signals.update({name: base._sample(value, seconds) for name, value in values.items()}) + snapshot = dict(timestamp_seconds=seconds, signals=signals, + camera=_sample_at(previous_camera, seconds), vision_frame_index=previous_camera['value']['index']) + diagnostic = dict(scope='Privileged native diagnostics; excluded from policy inputs.', + elapsed_seconds=seconds, native_data_time_seconds=float(self.env.data.time), + native_info=deepcopy(self.info), policy_observation=deepcopy(self.observation)) + return snapshot, {}, diagnostic, dict(render_encode_wall_seconds=0.) + + def native_evidence(self): + return self.env.native_arrays(), self.env.native_metadata() + + def anchor_error(self, estimate): + if estimate.get('status') != 'available': + return None + position = self.env.data.xmat[self.base].reshape(3, 3).T @ ( + self.env.data.xpos[self.env.block]-self.env.data.xpos[self.base]) + return float(self.np.linalg.norm(position-estimate['object_position_estimate_m'])) + + +def _initial_negative(snapshot, rgb, depth, calibration, negative): + """Faults affect only the perception input; no hidden object pose changes.""" + local = deepcopy(calibration) + captured = snapshot['camera']['captured_at_seconds'] + available = snapshot['camera']['available_at_seconds'] + if negative == 'initial_missing': + rgb = rgb.copy(); rgb[:] = 0 + elif negative == 'initial_stale': + captured = available = -.2 + elif negative == 'altered_calibration': + local['base_from_camera'][0] *= 1.1 + return rgb, depth, local, captured, available + + +def _fresh_estimate(session, snapshot, calibration, negative=None): + rgb, depth = session.latest_rgbd + rgb, depth, actual, captured, available = _initial_negative(snapshot, rgb, depth, calibration, negative) + began = time.perf_counter() + estimate = tracker.estimate_pose(rgb, depth, actual, + frame_index=snapshot['vision_frame_index'], captured_at_seconds=captured, + available_at_seconds=available, decision_at_seconds=snapshot['timestamp_seconds']) + return estimate, dict(estimator_wall_seconds=time.perf_counter()-began, + input_calibration_sha256=tape.digest(actual), negative_control=negative) + + +def _native_episode(session, states, actions): + info = deepcopy(session.info) + return dict(schema='dvidia.piper-task-episode/v1', config=asdict(session.task_config), + initial_object_position_m=deepcopy(states[0]['object_position_m']) if states else None, + trajectory=states, actions=actions, valid=bool(info.get('valid')), + success=bool(info.get('success')), reason=info.get('reason'), + metrics={**info, 'contact_pair_native_ticks': deepcopy(session.env.contact_pair_native_ticks)}, + physical_robot_ready=False, simulated_seconds=float(session.env.elapsed)) + + +def _score(root, goal): + import numpy as np + from .piper_feedback_score import score_feedback_episode + if not (root / 'native_episode.json').exists(): + return dict(valid=False, success=False, reason='native_not_initialized') + episode = json.loads((root / 'native_episode.json').read_text()) + metadata = json.loads((root / 'native_metadata.json').read_text()) + with np.load(root / 'native.npz', allow_pickle=False) as data: + arrays = {name: data[name] for name in data.files} + return score_feedback_episode(episode, arrays, metadata, goal[:2]) + + +def run_rollout(assets, output, *, case, mode, policy=None, negative_control=None, + duration_s=20., interrupt_after=None): + """Run a new attempt; failures seal the exact complete prefix, never retry. + + ``policy(context)`` receives only ``snapshot``, ``estimate``, + ``previous_applied`` and ``goal_position_m``. It returns the existing seven + target action dictionary. Teacher queries follow the actual trajectory and + are called once per boundary only in teacher/mixed modes. + """ + case = _case(case) + if mode not in MODES or negative_control not in NEGATIVES: + raise ValueError('Unknown controller mode or negative control.') + if mode != 'teacher' and not callable(policy): + raise ValueError('Student, mixed and fixed modes require a policy callback.') + config = RolloutConfig(duration_s=duration_s, seed=case['seed'], interrupt_after=interrupt_after) + root = Path(output).absolute() + if any(p.is_symlink() and str(p) not in ('/var', '/tmp') for p in (root, *root.parents)): + raise ValueError('Output cannot traverse symlinks.') + root.mkdir(parents=True, exist_ok=False) + tracker_hash = sha256(Path(tracker.__file__).read_bytes()).hexdigest() + base._json(root / 'started.json', dict(schema=SCHEMA, status='started', case=case, + mode=mode, config=asdict(config), negative_control=negative_control, scope=SCOPE)) + session = _Session(assets, config, case) + started = time.perf_counter() + steps, states, actions, timings, estimates = [], [], [], [], [] + header = initial = metadata = initial_targets = None + error = None; reason = 'capture_budget_complete'; started_intervals = 0 + frame_bytes = 0; camera_pairs = 0; anchor_available = False; anchor_error = None + native_status = {}; native_saved = False; rng = random.Random(case['seed']) + teacher_calls = student_calls = 0 + try: + header, initial_targets, metadata = session.open() + metadata = deepcopy(metadata) + metadata.update(tracker_implemented=True, + policy_observation_origin='teacher: privileged native state; student/fixed: whitelist of measured signals and RGB-D estimates', + rendered_camera_cadence_seconds=.04, measured_control_cadence_seconds=.02) + header = deepcopy(header); header['schema'] = SCHEMA + header['provenance']['kind'] = 'simulation_'+mode+'_controller' + header['provenance']['source_group'] = case['group_id'] + header['provenance']['case_id'] = case['id'] + header['goal_position_m'] = deepcopy(case['goal_position_m']) + header['tracker_sha256'] = tracker_hash + header['camera_cadence_seconds'] = .04 + header['control_mode'] = mode + snapshot, frames, diagnostic, timing = session.sample(0) + initial = deepcopy(snapshot) + current_estimate, estimate_timing = _fresh_estimate(session, snapshot, header['calibration'], negative_control) + timing.update(estimate_timing); timings.append(timing) + anchor_available = current_estimate['status'] == 'available' + anchor_error = session.anchor_error(current_estimate) + for path, body in frames.items(): + frame_bytes += len(body); base._record(root / path, body) + camera_pairs += 1 + if frame_bytes > tape.MAX_FRAME_TOTAL: + raise ValueError('Actual unique frame bytes exceed the 512 MiB limit.') + base._json(root / 'context.json', dict(header=header, initial=snapshot, + initial_applied_targets=initial_targets, metadata=metadata, case=case)) + base._json(root / 'native/0000.json', diagnostic) + states.append(deepcopy(diagnostic['policy_observation'])) + previous_applied = deepcopy(initial_targets) + for index in range(round(duration_s / tape.DT)): + if config.interrupt_after is not None and index >= config.interrupt_after: + raise KeyboardInterrupt('Declared engineering interruption.') + boundary = time.perf_counter() + seconds = snapshot['timestamp_seconds'] + estimate = _estimate_at(current_estimate, seconds) + perception = dict(schema=PERCEPTION_SCHEMA, index=index, + decision_at_seconds=seconds, snapshot_sha256=tape.digest(snapshot), + frame=deepcopy(snapshot['camera']['value']), + calibration_sha256=header['binding']['calibration_sha256'], + tracker_sha256=tracker_hash, estimate=estimate, + input_calibration_sha256=estimate_timing['input_calibration_sha256'], + negative_control=estimate_timing['negative_control']) + perception_ref = _artifact(root, f'perception/{index:04}.json', perception) + estimates.append(estimate) + if index == 0 and not anchor_available and mode != 'teacher': + reason = 'initial_visual_anchor_unavailable' + break + teacher_action = None; teacher_ref = None + if mode in ('teacher', 'mixed'): + teacher_calls += 1; teacher_action, teacher_decision = session.decide() + tape._action(teacher_action, header, 'queried teacher targets') + teacher_ref = _artifact(root, f'teacher/{index:04}.json', dict( + schema=TEACHER_SCHEMA, index=index, snapshot_sha256=tape.digest(snapshot), + computed_at_seconds=seconds, action=teacher_action, + origin='privileged_simulator_teacher', phase=teacher_decision.get('phase'))) + if mode == 'teacher': + requested, authority = deepcopy(teacher_action), 'teacher' + else: + context = dict(snapshot=deepcopy(snapshot), estimate=deepcopy(estimate), + previous_applied=deepcopy(previous_applied), goal_position_m=deepcopy(case['goal_position_m'])) + inference = time.perf_counter(); student_calls += 1; student_action = policy(context) + inference_wall = time.perf_counter()-inference + tape._action(student_action, header, 'policy targets') + if mode == 'mixed' and rng.random() < .5: + requested, authority = deepcopy(teacher_action), 'teacher' + else: + requested, authority = deepcopy(student_action), ('fixed' if mode == 'fixed' else 'student') + issued = deepcopy(requested) + if negative_control == 'open_gripper': + issued['gripper_position_m'] = .035 + tape._action(issued, header, 'issued targets') + command = dict(issued_at_seconds=seconds, requested=requested, issued=issued, + authority=authority, operator_correction=False, + issue_reason='open_gripper_negative_control' if issued != requested else 'unchanged') + base._json(root / f'decisions/{index:04}.json', dict(index=index, command=command, + perception=perception_ref, teacher_query=teacher_ref)) + started_intervals += 1 + applied = session.advance(issued) + next_index = index + 1 + if next_index % config.camera_every_controls == 0: + after, frames, diagnostic, timing = session.sample(next_index // config.camera_every_controls) + for path, body in frames.items(): + frame_bytes += len(body); base._record(root / path, body) + camera_pairs += 1 + if frame_bytes > tape.MAX_FRAME_TOTAL: + raise ValueError('Actual unique frame bytes exceed the 512 MiB limit.') + current_estimate, estimate_timing = _fresh_estimate(session, after, header['calibration']) + timing.update(estimate_timing) + else: + after, frames, diagnostic, timing = session.measured(snapshot['camera']) + step = dict(index=index, before=snapshot, command=command, applied=applied, + after=after, perception=perception_ref, teacher_query=teacher_ref) + base._json(root / f'transitions/{index:04}.json', step) + base._json(root / f'native/{next_index:04}.json', diagnostic) + steps.append(step) + action_row = dict(t_seconds=seconds, **issued, phase=authority) + states[-1].update(issued_joint_targets_rad=issued['joint_targets_rad'], + issued_gripper_position_m=issued['gripper_position_m'], command=action_row) + actions.append(action_row); states.append(deepcopy(diagnostic['policy_observation'])) + timing['full_boundary_wall_seconds'] = time.perf_counter()-boundary + timing['policy_inference_wall_seconds'] = 0. if mode == 'teacher' else inference_wall + timings.append(timing) + snapshot = after; previous_applied = deepcopy(applied['ramped_targets']) + except (Exception, KeyboardInterrupt) as exc: + reason = 'producer_interrupted' if isinstance(exc, KeyboardInterrupt) else 'producer_error' + error = dict(type=type(exc).__name__, message=str(exc)[:500]) + finally: + try: + native_status = session.status() + if header is not None: + import numpy as np + arrays, native_metadata = session.native_evidence() + buffer = io.BytesIO(); np.savez_compressed(buffer, **arrays) + base._record(root / 'native.npz', buffer.getvalue()) + base._json(root / 'native_metadata.json', native_metadata) + base._json(root / 'native_episode.json', _native_episode(session, states, actions)) + native_saved = True + except Exception as exc: + if error is None: + error = dict(type=type(exc).__name__, message=str(exc)[:500]); reason = 'native_evidence_error' + try: + session.close() + except Exception as exc: + if error is None: + error = dict(type=type(exc).__name__, message=str(exc)[:500]); reason = 'renderer_close_error' + complete = error is None and len(steps) == round(duration_s / tape.DT) + status = 'complete' if complete else 'interrupted' if steps else 'refused' + if header is not None: + recording = {**header, 'initial': initial, 'initial_applied_targets': initial_targets, + 'steps': steps, 'outcome': dict(status=status, reason=reason, + end_seconds=steps[-1]['after']['timestamp_seconds'] if steps else 0., + task_result='unverified' if complete else 'unknown')} + base._json(root / 'recording.json', recording) + score = _score(root, case['goal_position_m']) if native_saved else dict(valid=False, success=False, reason='native_evidence_unavailable') + missing = [estimate['status'] != 'available' for estimate in estimates] + longest = streak = 0 + for absent in missing: + streak = streak + 1 if absent else 0; longest = max(longest, streak) + critical_ticks = (native_status.get('native_info', {}).get('unexpected_contact_native_ticks') + if native_status.get('native_info') else None) + report = dict(schema=SCHEMA, status=status, reason=reason, error=error, case=case, mode=mode, + scope=SCOPE, physical_robot_ready=False, negative_control=negative_control, + requested_intervals=round(duration_s / tape.DT), started_intervals=started_intervals, + recorded_complete_intervals=len(steps), uncommitted_native_intervals=started_intervals-len(steps), + camera_pairs=camera_pairs, frame_bytes=frame_bytes, + initial_anchor_available=anchor_available, initial_anchor_error_m=anchor_error, + initial_anchor_binding_valid=bool(header is not None and anchor_available and negative_control not in ('initial_stale','altered_calibration')), + teacher_queries=teacher_calls, student_calls=student_calls, + no_privileged_features_or_teacher_intervention=bool(mode in ('student', 'fixed') and teacher_calls == 0), + policy_boundary_scope='Whitelisted deep-copied callback inputs; no process sandbox or external callback code attestation.', + score=score, strict_success=bool(complete and score.get('success')), + critical_contacts=critical_ticks, critical_contact_ticks=critical_ticks, + native_status=native_status, tracker_sha256=tracker_hash, + perception=dict(decisions=len(estimates), available=sum(not value for value in missing), + missing=sum(missing), longest_missing_seconds=longest*tape.DT), + timing=dict(wall_seconds=time.perf_counter()-started, boundaries=timings, + scope='Offline paused-simulator acquisition and computation. Synthetic capture/availability times; no physical transport or end-to-end real-time latency.')) + base._json(root / 'rollout.json', report) + files = {str(path.relative_to(root)): _identity(path) for path in sorted(root.rglob('*')) if path.is_file()} + base._json(root / 'integrity.json', dict(schema=SCHEMA+'/integrity', files=files)) + return report + + +def _read(root, relative, maximum=MAX_ROLLOUT_BYTES): + return json.loads(tape._ordinary_bytes(root / str(tape._relative(relative)), maximum)) + + +def inspect_rollout(directory): + """Read-only byte, source-frame, authority and independent-score checks.""" + root = Path(directory) + manifest = _read(root, 'integrity.json') + if manifest.get('schema') != SCHEMA+'/integrity' or type(manifest.get('files')) is not dict: + raise ValueError('Unsealed or unknown rollout manifest.') + observed = {str(p.relative_to(root)) for p in root.rglob('*') if p.is_file() and p.name != 'integrity.json'} + if observed != set(manifest['files']): + raise ValueError('Rollout inventory differs.') + for name, expected in manifest['files'].items(): + body = tape._ordinary_bytes(root / str(tape._relative(name)), MAX_ROLLOUT_BYTES) + if expected != dict(bytes=len(body), sha256=sha256(body).hexdigest()): + raise ValueError('Rollout byte identity differs: '+name) + report = _read(root, 'rollout.json') + if report.get('schema') != SCHEMA or report.get('status') not in ('complete', 'interrupted', 'refused'): + raise ValueError('Unknown rollout receipt.') + if 'recording.json' in manifest['files']: + recording = _read(root, 'recording.json') + if recording.get('schema') != SCHEMA or len(recording.get('steps', [])) != report['recorded_complete_intervals']: + raise ValueError('Recording schema or completeness differs.') + previous = recording['initial'] + native_episode = _read(root, 'native_episode.json') if 'native_episode.json' in manifest['files'] else None + native_arrays = None + if native_episode is not None and 'actions' in native_episode: + import numpy as np + with np.load(root / 'native.npz', allow_pickle=False) as archive: + native_arrays = {name: archive[name] for name in ('arm_actuator_controls_rad', + 'applied_jaw_target_m', 'normal_rate_limited_jaw_target_m')} + tape._snapshot(previous, recording['calibration']) + if tape.digest(recording['calibration']) != recording['binding']['calibration_sha256']: + raise ValueError('Bound calibration differs.') + for index, step in enumerate(recording['steps']): + if step['index'] != index or step['before'] != previous: + raise ValueError('Recording transition prefix differs.') + seconds = index*tape.DT + if not math.isclose(step['before']['timestamp_seconds'], seconds, abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise ValueError('Decision cadence differs.') + if not math.isclose(step['after']['timestamp_seconds'], seconds+tape.DT, abs_tol=tape.CLOCK_TOL, rel_tol=0): + raise ValueError('Applied interval is incomplete.') + tape._snapshot(step['after'], recording['calibration'], previous) + if any(step['before']['signals'][name]['status'] != 'missing' for name in tracker.SIGNALS): + raise ValueError('Raw sensor snapshots cannot substitute tracker or native object truth.') + frame = step['before']['camera']['value'] + if frame['index'] != index // 2: + raise ValueError('Camera cadence differs from 25 Hz.') + for channel in ('rgb', 'depth'): + reference = frame[channel] + if manifest['files'].get(reference['path']) != {key: reference[key] for key in ('bytes', 'sha256')}: + raise ValueError('Camera source byte binding differs.') + for field in ('requested', 'issued'): + tape._action(step['command'][field], recording, field) + expected_issued = deepcopy(step['command']['requested']) + if report['negative_control'] == 'open_gripper': + expected_issued['gripper_position_m'] = .035 + if step['command']['issued'] != expected_issued: + raise ValueError('Actual issued command differs from declared intervention.') + if native_episode is not None and 'actions' in native_episode: + actual = native_episode['actions'][index] + if ({name: actual[name] for name in ('joint_targets_rad', 'gripper_position_m')} + != step['command']['issued']): + raise ValueError('Issued command differs from native action evidence.') + native_state = native_episode['trajectory'][index] + for signal in ('joint_position_rad', 'joint_velocity_rad_s', 'gripper_joint_position_m'): + if step['before']['signals'][signal]['value'] != native_state[signal]: + raise ValueError('Measured inputs differ from native encoder evidence.') + boundary = (index+1)*20-1 + controls = [*native_arrays['arm_actuator_controls_rad'][boundary].tolist(), + float(native_arrays['applied_jaw_target_m'][boundary])] + ramped = dict(joint_targets_rad=controls[:6], + gripper_position_m=float(native_arrays['normal_rate_limited_jaw_target_m'][boundary])) + if step['applied']['native_position_controls'] != controls or step['applied']['ramped_targets'] != ramped: + raise ValueError('Previous-applied learning inputs differ from native servo references.') + for field in ('perception', 'teacher_query'): + reference = step[field] + if reference is None: + if field == 'perception' or report['mode'] in ('teacher', 'mixed'): + raise ValueError('Required sidecar missing.') + continue + identity = manifest['files'].get(reference['path']) + if identity != {key: reference[key] for key in ('bytes', 'sha256')}: + raise ValueError('Sidecar binding differs.') + sidecar = _read(root, reference['path']) + if sidecar['index'] != index or sidecar['snapshot_sha256'] != tape.digest(step['before']): + raise ValueError('Sidecar source snapshot differs.') + if field == 'perception': + if (sidecar['tracker_sha256'] != recording['tracker_sha256'] + or sidecar['calibration_sha256'] != recording['binding']['calibration_sha256'] + or sidecar['frame'] != step['before']['camera']['value']): + raise ValueError('Perception frame/tracker/calibration binding differs.') + if sidecar['estimate']['decision_at_seconds'] != step['before']['timestamp_seconds']: + raise ValueError('Perception decision time differs.') + estimate = sidecar['estimate'] + if estimate['frame_index'] != frame['index']: + raise ValueError('Estimated source frame index differs.') + if estimate['status'] == 'available': + camera = step['before']['camera'] + if (estimate['captured_at_seconds'] != camera['captured_at_seconds'] + or estimate['available_at_seconds'] != camera['available_at_seconds'] + or estimate['age_seconds'] != camera['age_seconds']): + raise ValueError('Estimated source frame acquisition timing differs.') + for name in tracker.SIGNALS: + signal = estimate['signals'][name] + if (signal['status'] != 'available' or any(signal[key] != estimate[key] for key in + ('captured_at_seconds', 'available_at_seconds', 'age_seconds'))): + raise ValueError('Estimated source signal timing differs.') + expected = estimate[name] + if name.endswith('_uncertainty_m') or name.endswith('_uncertainty_rad'): + expected = [expected] + if signal['value'] != expected: + raise ValueError('Estimated source signal value differs.') + elif any(estimate[name] is not None or estimate['signals'][name]['value'] is not None + for name in tracker.SIGNALS): + raise ValueError('Missing estimates must not carry substituted values.') + elif step['command']['authority'] == 'teacher' and sidecar['action'] != step['command']['requested']: + raise ValueError('Teacher authority differs from its query.') + authority = step['command']['authority'] + allowed = {'teacher': {'teacher'}, 'student': {'student'}, 'mixed': {'teacher', 'student'}, 'fixed': {'fixed'}}[report['mode']] + if authority not in allowed or step['command']['operator_correction'] is not False: + raise ValueError('Invalid control authority.') + previous = step['after'] + if report['status'] == 'complete' and len(recording['steps']) != report['requested_intervals']: + raise ValueError('Complete receipt omits scheduled intervals.') + recomputed = _score(root, report['case']['goal_position_m']) if 'native_episode.json' in manifest['files'] else dict(valid=False, success=False, reason='native_evidence_unavailable') + if recomputed != report['score']: + raise ValueError('Independent task score differs.') + if report['strict_success'] != bool(report['status'] == 'complete' and recomputed.get('success')): + raise ValueError('Reported strict success differs from complete native scoring.') + return dict(schema=SCHEMA+'/inspection', status=report['status'], files_verified=len(manifest['files']), + source_authenticated=False, physics_executed=False, physical_robot_ready=False, + score=recomputed, strict_success=report['strict_success'], + recorded_complete_intervals=report['recorded_complete_intervals']) + + +def load_rollout(directory): + inspect_rollout(directory) + return _read(Path(directory), 'recording.json') diff --git a/src/dvidia_training/camera_action_export.py b/src/dvidia_training/camera_action_export.py new file mode 100644 index 0000000..2ce7208 --- /dev/null +++ b/src/dvidia_training/camera_action_export.py @@ -0,0 +1,128 @@ +"""Mask-aware v2 exports, separate from the intentionally strict v1 exporter. + +Only pre-action measurements and separately sealed perception enter inputs. +Teacher queries are explicit supervision on mixed trajectories; they are never +relabeled as commands that the producer actually executed. +""" +from __future__ import annotations + +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path + +import numpy as np + +from . import camera_action_model as model + + +def _document(root, reference): + from . import action_demonstration as tape + body = tape._ordinary_bytes(root / str(tape._relative(reference['path'])), 2*1024*1024) + if reference['bytes'] != len(body) or reference['sha256'] != sha256(body).hexdigest(): + raise ValueError('Learning sidecar bytes no longer match their sealed reference.') + return model.core._decode(body) + + +def export_episode(directory, *, blind=False, label_source='issued'): + """Read and verify one sealed episode; no rendering, physics or relabeling. + + ``issued`` uses teacher-only producer commands. ``teacher_query`` requires + mixed control and labels each pre-action snapshot using its one separate + teacher query, whether the teacher or student actually controlled that step. + Interrupted complete prefixes remain exportable; a fresh t=0 anchor is + always required. Source selection is the experiment orchestrator's job. + """ + from . import camera_action_capture as capture + root = Path(directory) + recording = capture.load_rollout(root) + recording_body = (root/'recording.json').read_bytes() + if recording_body.strip() != model.canonical(recording): + raise ValueError('Recording changed after rollout verification.') + if label_source not in ('issued', 'teacher_query'): + raise ValueError('Choose issued or teacher_query labels explicitly.') + expected = 'teacher' if label_source == 'issued' else 'mixed' + if recording.get('control_mode') != expected: + raise ValueError('Issued supervision requires teacher control; queried correction supervision requires mixed control.') + if not recording['steps']: + raise ValueError('No complete pre-action intervals are available.') + encoder = model.CameraActionFeatures(blind=blind) + previous_applied = recording['initial_applied_targets'] + features, targets, perception_hashes, query_hashes, authority = [], [], [], [], [] + for step in recording['steps']: + perception = _document(root, step['perception']) + context = dict(snapshot=step['before'], estimate=perception['estimate'], + previous_applied=previous_applied, goal_position_m=recording['goal_position_m']) + features.append(encoder(context)) + perception_hashes.append(step['perception']['sha256']) + if label_source == 'issued': + if step['command']['authority'] != 'teacher': + raise ValueError('Issued imitation labels must actually come from the teacher.') + target = step['command']['issued'] + else: + reference = step.get('teacher_query') + if reference is None: + raise ValueError('A separately sealed teacher query is required for every correction label.') + query = _document(root, reference) + if query.get('origin') != 'privileged_simulator_teacher': + raise ValueError('Unknown teacher-query supervision origin.') + target = query['action'] + query_hashes.append(reference['sha256']) + targets.append(model.action_vector(target)) + authority.append(step['command']['authority']) + previous_applied = step['applied']['ramped_targets'] + source_group = recording['provenance']['source_group'] + source = dict(source_group=source_group, case_id=recording['provenance']['case_id'], + recording_sha256=sha256(recording_body).hexdigest(), + integrity_sha256=sha256((root/'integrity.json').read_bytes()).hexdigest(), + calibration_sha256=recording['binding']['calibration_sha256'], + tracker_sha256=recording['tracker_sha256'], label_source=label_source, + sample_count=len(features), perception_references_sha256=model.digest(perception_hashes), + teacher_query_references_sha256=model.digest(query_hashes), + actual_authority_sha256=model.digest(authority)) + sources = [source] + provenance = dict(feature_contract_sha256=model.FEATURE_CONTRACT_SHA256, blind=blind, + episodes=sources, source_identity_sha256=model.digest(sources), + export_source_sha256=sha256(Path(__file__).read_bytes()).hexdigest(), + label_definition='absolute issued teacher targets' if label_source == 'issued' + else 'separate teacher query suggestions; actual control authority retained independently') + return dict(schema=model.DATASET_SCHEMA, features=np.asarray(features, dtype=np.float32), + targets=np.asarray(targets, dtype=np.float64), groups=[source_group]*len(features), provenance=provenance) + + +def combine_datasets(datasets): + """Concatenate already exported episodes once; retain whole-start group IDs.""" + if type(datasets) not in (list, tuple) or not datasets: + raise ValueError('At least one exported dataset is required.') + blind = datasets[0]['provenance']['blind'] + features, targets, groups, sources, identities = [], [], [], [], set() + for dataset in datasets: + x, y, g, p, _ = model._dataset(dataset, blind=blind) + if (p.get('blind') is not blind or p.get('feature_contract_sha256') != model.FEATURE_CONTRACT_SHA256 + or p.get('source_identity_sha256') != model.digest(p.get('episodes'))): + raise ValueError('Combined exports must have the same exact feature contract and feedback masking.') + for episode in p['episodes']: + identity = episode['recording_sha256'] + if identity in identities: + raise ValueError('An episode cannot be duplicated or relabeled within one training dataset.') + identities.add(identity) + sources.append(episode) + features.append(x); targets.append(y); groups.extend(g) + if len(groups) > 1_000_000: + raise ValueError('Combined training data exceeds its bounded sample budget.') + provenance = dict(feature_contract_sha256=model.FEATURE_CONTRACT_SHA256, blind=blind, + episodes=sources, source_identity_sha256=model.digest(sources), + export_source_sha256=sha256(Path(__file__).read_bytes()).hexdigest(), + label_definition='per-episode explicit issued teacher targets or separately sealed teacher query suggestions') + return dict(schema=model.DATASET_SCHEMA, features=np.concatenate(features), targets=np.concatenate(targets), + groups=groups, provenance=provenance) + + +def blind_dataset(dataset): + """Mask a verified full export without rereading its potentially large frames.""" + x, y, groups, provenance, _ = model._dataset(dataset, blind=False) + if provenance.get('blind') is not False: + raise ValueError('Blind masking expects a full-feedback exported dataset.') + x = x.copy(); x[:, model.LIVE_FEATURE_START:] = 0. + provenance = deepcopy(provenance); provenance['blind'] = True + return dict(schema=model.DATASET_SCHEMA, features=x, targets=y.copy(), + groups=groups, provenance=provenance) diff --git a/src/dvidia_training/camera_action_model.py b/src/dvidia_training/camera_action_model.py new file mode 100644 index 0000000..d7b96eb --- /dev/null +++ b/src/dvidia_training/camera_action_model.py @@ -0,0 +1,463 @@ +"""Small causal RGB-D/encoder policy, trained on explicit native action labels. + +The network never receives a simulator, teacher, task phase, current target, +contact truth or a future snapshot. Its visual anchor is historical context, +not a held-forward current measurement. Missing live estimates stay missing. +""" +from __future__ import annotations + +from collections import deque +from copy import deepcopy +from hashlib import sha256 +import math +from pathlib import Path +import time + +import numpy as np + +from . import piper_imitation as core +from . import piper_rgbd_tracker as tracker + + +SCHEMA = 'dvidia.camera-action-mlp/v1' +DATASET_SCHEMA = 'dvidia.camera-action-learning/v1' +FEATURE_CONTRACT = 'camera-encoder-absolute-causal160/v1' +SCOPE = ('Simulation-only camera/encoder behavioral cloning of a privileged synthetic ' + 'teacher; known upright block, no human-video motor recovery or physical qualification.') +LAGS = (0., .04, .08, .12, .16) +SENSOR_SPECS = ( + ('joint_position_rad', 6, 3.), ('joint_velocity_rad_s', 6, 3.), + ('gripper_joint_position_m', 2, .035), ('gripper_joint_velocity_m_s', 2, .035), + ('tcp_position_estimate_m', 3, .5), ('tcp_quaternion_estimate_wxyz', 4, 1.), +) +CONTEXT_NAMES = tuple(f'initial_visual_{a}_m/.5' for a in 'xyz') + ( + 'initial_visual_captured_at_s/20', 'initial_visual_uncertainty_m/.02', +) + tuple(f'goal_{a}_m/.5' for a in 'xyz') + ('elapsed_s/20',) +COMMAND_NAMES = tuple(f'previous_applied_joint_{i}_rad/3' for i in range(1, 7)) + ( + 'previous_applied_jaw_m/.035', 'history_present', 'history_age_s/.16',) +SENSOR_NAMES = tuple(name for field, size, scale in SENSOR_SPECS + for name in tuple(f'{field}_{i}/{scale:g}' for i in range(size)) + + (field+'_available', field+'_age_s/.1')) + ( + 'visual_x_m/.5', 'visual_y_m/.5', 'visual_z_m/.5', + 'visual_uncertainty_m/.02', 'visual_available', 'visual_age_s/.1', + 'history_present', 'history_age_s/.16',) +FEATURE_NAMES = CONTEXT_NAMES + tuple(f'lag{round(lag*1000)}:{name}' + for lag in LAGS for name in COMMAND_NAMES) + tuple(f'lag{round(lag*1000)}:{name}' + for lag in LAGS for name in SENSOR_NAMES) +FEATURE_COUNT = len(FEATURE_NAMES) +LIVE_FEATURE_START = len(CONTEXT_NAMES) + len(LAGS)*len(COMMAND_NAMES) +ACTION_SCALE_NATIVE = np.asarray([3.]*6 + [core.JAW_MAX_M], dtype=np.float64) +MODEL_FIELDS = {'schema', 'scope', 'physical_robot_ready', 'observation_origin', + 'supervision_origin', 'feature_contract', 'feature_names', 'feature_contract_sha256', + 'blind', 'hidden_units', 'joint_limits_rad', 'jaw_limit_m', 'action_representation', + 'feature_mean', 'feature_scale', 'action_mean', 'action_scale', 'weights', 'biases', 'training'} +TRAINING_FIELDS = {'algorithm', 'stage', 'seed', 'epochs', 'batch_size', 'learning_rate', + 'sample_count', 'train_groups', 'dataset_sha256', 'provenance', 'provenance_sha256', + 'model_source_sha256', 'initial_loss', 'final_loss', 'validation'} + + +def canonical(document): + return core.canonical(document) + + +def digest(document): + return sha256(canonical(document)).hexdigest() + + +FEATURE_CONTRACT_SHA256 = digest({'contract': FEATURE_CONTRACT, 'names': FEATURE_NAMES, + 'lags': LAGS, 'history_seconds': .16, 'camera_maximum_age_seconds': .1, + 'blind_live_start': LIVE_FEATURE_START, + 'previous_applied': 'preceding interval final native servo references; never measured q', + 'initial_anchor': 'fresh t=0 visual xyz and heuristic uncertainty; historical thereafter'}) + + +def _hash(value, label): + if type(value) is not str or len(value) != 64 or any(c not in '0123456789abcdef' for c in value): + raise ValueError(label+' must be a lowercase SHA-256 digest.') + return value + + +def action_vector(action): + """Validate seven *absolute* issued/query targets, without interpreting authority.""" + if type(action) is not dict or set(action) != {'joint_targets_rad', 'gripper_position_m'}: + raise ValueError('An action must contain only six joint targets and one jaw target.') + q = core._array(action['joint_targets_rad'], (6,), 'joint targets', 10.) + limits = np.asarray(core.JOINT_LIMITS) + if np.any(q < limits[:, 0]) or np.any(q > limits[:, 1]): + raise ValueError('Action exceeds the original native joint bounds.') + jaw = core._number(action['gripper_position_m'], 'jaw target', 0., core.JAW_MAX_M) + return np.concatenate((q, [jaw])) + + +def _time(value, name): + return core._number(value, name, 0., 20.) + + +def _sample(sample, size, now, name): + if type(sample) is not dict or sample.get('status') not in ('available', 'missing', 'stale', 'refused'): + raise ValueError(name+' has an invalid availability state.') + if sample['status'] != 'available': + if sample.get('value') is not None: + raise ValueError(name+' must not forward-fill unavailable values.') + return np.zeros(size), 0., 0. + captured = _time(sample.get('captured_at_seconds'), name+' capture time') + available = _time(sample.get('available_at_seconds'), name+' availability time') + age = _time(sample.get('age_seconds'), name+' age') + if (captured > available+1e-10 or available > now+1e-10 + or abs(age-(now-captured)) > 1e-8 or age > .1+1e-10): + raise ValueError(name+' is future, stale or incorrectly age-stamped.') + return core._array(sample.get('value'), (size,), name, 1e4), 1., age + + +def _visual(estimate, now, snapshot): + if type(estimate) is not dict or estimate.get('schema') != tracker.SCHEMA: + raise ValueError('A frozen tracker result is required.') + if estimate.get('status') not in ('available', 'missing', 'stale', 'refused'): + raise ValueError('Invalid tracker status.') + decision = _time(estimate.get('decision_at_seconds'), 'tracker decision time') + if abs(decision-now) > 1e-8: + raise ValueError('Tracker decision time must match this motor boundary.') + camera = snapshot.get('camera') + if type(camera) is not dict or camera.get('status') != 'available' or type(camera.get('value')) is not dict: + raise ValueError('Visual estimates require their available source camera frame.') + index = camera['value'].get('index') + if (type(index) is not int or index < 0 or estimate.get('frame_index') != index + or snapshot.get('vision_frame_index') != index): + raise ValueError('Tracker frame index differs from the source camera snapshot.') + captured = _time(camera.get('captured_at_seconds'), 'camera capture time') + available = _time(camera.get('available_at_seconds'), 'camera availability time') + camera_age = _time(camera.get('age_seconds'), 'camera age') + if (captured > available+1e-10 or available > now+1e-10 + or abs(camera_age-(now-captured)) > 1e-8): + raise ValueError('Source camera timestamps are noncausal or incorrectly aged.') + if any(estimate.get(field) != camera.get(field) + for field in ('captured_at_seconds', 'available_at_seconds', 'age_seconds')): + raise ValueError('Tracker source timestamps differ from the camera snapshot.') + signals = estimate.get('signals') + if type(signals) is not dict: + raise ValueError('Tracker signals are required.') + p, present, age = _sample(signals.get('object_position_estimate_m'), 3, now, 'visual position') + u, uncertainty_present, uncertainty_age = _sample(signals.get('object_position_uncertainty_m'), 1, now, 'visual uncertainty') + if present != uncertainty_present or abs(age-uncertainty_age) > 1e-8: + raise ValueError('Visual position and uncertainty must share availability and age.') + if bool(present) != (estimate['status'] == 'available'): + raise ValueError('Tracker status and visual signal availability disagree.') + if present: + if u[0] < 0. or u[0] > 1.: + raise ValueError('Visual uncertainty must be a nonnegative bounded scale.') + top = core._array(estimate.get('object_position_estimate_m'), (3,), 'tracker position', 1e4) + if not np.array_equal(p, top) or estimate.get('object_position_uncertainty_m') != float(u[0]): + raise ValueError('Tracker position/uncertainty receipt disagrees with signals.') + for name in ('captured_at_seconds', 'available_at_seconds', 'age_seconds'): + if (estimate.get(name) != signals['object_position_estimate_m'].get(name) + or estimate.get(name) != signals['object_position_uncertainty_m'].get(name)): + raise ValueError('Tracker top-level and signal times disagree.') + else: + if estimate.get('object_position_estimate_m') is not None or estimate.get('object_position_uncertainty_m') is not None: + raise ValueError('Unavailable visual estimates must be null.') + # An abstention has no pose age, but its source image still has a known + # acquisition age. Preserve that age next to the zero position and mask. + return p, float(u[0]), present, camera_age + + +class CameraActionFeatures: + """Fresh per episode. Five causal slots span 160 ms; warmup slots are absent. + + The blind comparator retains the same initial camera anchor and previous + applied command history. It receives no live sensor values, masks or ages. + Its command history is the preceding native reference, not measured joints. + """ + def __init__(self, *, blind=False): + if type(blind) is not bool: + raise ValueError('blind must be boolean.') + self.blind = blind + self.history = deque() + self.initial = self.goal = None + self.last_time = None + + def features(self, context): + if type(context) is not dict or type(context.get('snapshot')) is not dict: + raise ValueError('Camera-action context and pre-action snapshot are required.') + snapshot = context['snapshot'] + now = _time(snapshot.get('timestamp_seconds'), 'snapshot time') + if self.last_time is None and now != 0.: + raise ValueError('Camera-action episodes must start at t=0.') + if self.last_time is not None and not math.isclose(now-self.last_time, .02, abs_tol=1e-8, rel_tol=0.): + raise ValueError('Camera-action observations require increasing 20 ms boundaries.') + goal = core._array(context.get('goal_position_m'), (3,), 'goal position', 1.) + if self.goal is not None and not np.array_equal(goal, self.goal): + raise ValueError('The destination cannot change during an episode.') + command = action_vector(context.get('previous_applied'))/ACTION_SCALE_NATIVE + visual = None + if self.initial is None or not self.blind: + visual = _visual(context.get('estimate'), now, snapshot) + initial = self.initial + if initial is None: + p, uncertainty, present, age = visual + if not present or age > 1e-10: + raise ValueError('A fresh initial visual anchor is required before the first student action.') + initial = np.concatenate((p/.5, [0., uncertainty/.02])) + if self.blind: + sensed = np.zeros(len(SENSOR_NAMES)-2) + else: + signals = snapshot.get('signals') + if type(signals) is not dict: + raise ValueError('Measured encoder/FK signals are required.') + values = [] + for name, size, scale in SENSOR_SPECS: + value, present, age = _sample(signals.get(name), size, now, name) + if present and name == 'tcp_quaternion_estimate_wxyz': + value = core._quaternion(value, name) + values.extend(np.clip(value/scale, -10., 10.)) + values.extend((present, age/.1)) + p, uncertainty, present, age = visual + values.extend(np.clip(p/.5, -10., 10.)) + values.extend((uncertainty/.02, present, age/.1)) + sensed = np.asarray(values) + pending = [row for row in self.history if row[0] >= now-.16-1e-8] + pending.append((now, command.copy(), sensed.copy())) + selected = [] + for lag in LAGS: + eligible = [row for row in pending if row[0] <= now-lag+1e-8] + selected.append(eligible[-1] if eligible else None) + command_slots, sensed_slots = [], [] + for row in selected: + if row is None: + command_slots.extend(np.zeros(len(COMMAND_NAMES))) + sensed_slots.extend(np.zeros(len(SENSOR_NAMES))) + else: + age = max(0., now-row[0])/.16 + command_slots.extend([*row[1], 1., age]) + sensed_slots.extend(np.zeros(len(SENSOR_NAMES)) if self.blind else [*row[2], 1., age]) + result = np.concatenate((initial, goal/.5, [now/20.], command_slots, sensed_slots)) + if result.shape != (FEATURE_COUNT,) or not np.isfinite(result).all() or np.any(np.abs(result) > 10.+1e-8): + raise ValueError('Camera-action features exceed the bounded finite contract.') + self.history = deque(pending) + self.initial, self.goal, self.last_time = initial.copy(), goal.copy(), now + return result.astype(np.float32) + + __call__ = features + + +def _dataset(dataset, *, blind): + if type(dataset) is not dict or set(dataset) != {'schema', 'features', 'targets', 'groups', 'provenance'} or dataset['schema'] != DATASET_SCHEMA: + raise ValueError('Use a camera-action mask-aware dataset with provenance.') + groups = dataset['groups'] + if type(groups) not in (list, tuple) or not 1 <= len(groups) <= 1_000_000: + raise ValueError('Dataset requires 1 to 1000000 grouped samples.') + groups = list(groups) + if any(type(group) is not str for group in groups): + raise ValueError('Every training sample requires a string source-group ID.') + core._case_ids(list(dict.fromkeys(groups))) + x = core._array(dataset['features'], (len(groups), FEATURE_COUNT), 'features', 10.+1e-7) + y = core._array(dataset['targets'], (len(groups), 7), 'absolute action targets', 10.) + limits = np.asarray(core.JOINT_LIMITS) + if (np.any(y[:, :6] < limits[:, 0]) or np.any(y[:, :6] > limits[:, 1]) + or np.any(y[:, 6] < 0.) or np.any(y[:, 6] > core.JAW_MAX_M)): + raise ValueError('Training targets exceed original native bounds.') + if blind and np.any(x[:, LIVE_FEATURE_START:] != 0.): + raise ValueError('Blind datasets must zero all live/history feedback features.') + provenance = dataset['provenance'] + if type(provenance) is not dict or not provenance: + raise ValueError('Dataset provenance is required.') + provenance = core._decode(canonical(provenance)) + h = sha256(canonical({'schema': DATASET_SCHEMA, 'features': list(x.shape), + 'targets': list(y.shape), 'groups': groups, 'provenance': provenance})) + for value in (x, y): + h.update(np.asarray(value, dtype=' + +Simulation skill training · DVIDIA + + + + +
Camera → action

Teach an arm, then test it.

Train a compact block-placement policy from paired camera recordings and robot actions. Compare it on unseen starts before installing a simulation candidate.

Local files · Offline computation · One job at a time

+

01Choose paired demonstrations

Open an existing prepared camera-to-action study. It keeps the demonstration groups, fixed experiment budget and benchmark results together.

+ + +

Paths refer to this computer. Plain videos alone do not contain the demonstrated motor targets required by this pilot. The study collects paired recordings from its explicitly labeled simulator teacher.

+
Choose both folders to begin.
+
First time?

Prepare the registered study in Terminal with the existing fixed comparator model. Then enter the study and verified PiPER asset folders above.

dvidia-train robot prepare --study /path/to/new-study --fixed-model /path/to/fixed-model.json
+
+

02Train and benchmark

The same workflow as Terminal: verify demonstrations, train the model, collect one bounded correction round, and compare it with fixed and feedback-blind controls.

+
Check the study before starting.
+
Connecting to the local studio…
+

Completed stages are preserved. A failed prerequisite stops the workflow; it does not retry or replace difficult scenes. Graphics support is required for native RGB-D rendering.

+
+

03Review the evidence

The checked study and its benchmark results will appear here.

+ +
+ diff --git a/src/dvidia_training/camera_action_studio.py b/src/dvidia_training/camera_action_studio.py new file mode 100644 index 0000000..1665b16 --- /dev/null +++ b/src/dvidia_training/camera_action_studio.py @@ -0,0 +1,264 @@ +"""Local Training Studio bridge for registered camera/action studies. + +It shares the footage server's one worker and lock. A browser never supplies +Python code, arbitrary commands or hardware endpoints. Stage order comes from +the same verified study inspection used by the Terminal workflow. +""" +from __future__ import annotations + +from hashlib import sha256 +import json +from pathlib import Path +import time +import uuid + +SCHEMA = 'dvidia.camera-action-studio/v1' + + +def _worker_lock(study): + """Read only the study's small lock record; never infer liveness or unlink it.""" + path = Path(study)/'worker.lock' + if not path.exists() and not path.is_symlink(): + return dict(present=False) + result = dict(present=True) + try: + if not path.is_symlink() and path.is_file() and path.stat().st_size <= 1024: + value = json.loads(path.read_text()) + if type(value) is dict and isinstance(value.get('operation'), str): + result['operation'] = value['operation'][:80] + except (OSError, ValueError): + pass + return result + + +def _local_path(value, label): + if not isinstance(value, str) or not value.strip() or len(value) > 8192 or any(ord(c) < 32 for c in value): + raise ValueError('Supply a local '+label+' folder.') + if '://' in value: + raise ValueError('Use local folders; the robot study does not download inputs.') + path = Path(value).expanduser().absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (path, *path.parents)): + raise ValueError('The '+label+' folder cannot traverse symlinks.') + if not path.is_dir(): + raise ValueError('The '+label+' folder does not exist on this server.') + return path.resolve() + + +def check_study(study, assets): + from . import camera_action_study, robot_profiles + root = _local_path(study, 'prepared study') + asset_root = _local_path(assets, 'verified robot assets') + state = camera_action_study.inspect(root) + camera_action_study._protocol(root, runtime=True) + metadata = robot_profiles.verify_assets('piper', asset_root) + binding = dict(study=str(root), assets=str(asset_root), protocol_sha256=state['protocol_sha256'], + assets_sha256=sha256(json.dumps(metadata, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest()) + stage = state.get('report', {}).get('selected_stage') + model = root/'models'/str(stage)/'full'/'model.json' if stage in ('initial', 'corrected') else None + install_available = bool(state['status'] == 'complete' and model is not None and model.is_file() and not model.is_symlink()) + worker_lock = _worker_lock(root) + return dict(schema=SCHEMA, binding=binding, inspection=state, install_available=install_available, + selected_model=str(model) if install_available else None, study_locked=worker_lock['present'], + study_worker_lock=worker_lock, physical_robot_ready=False) + + +def _write_progress(output, status, message, **extra): + output = Path(output) + value = dict(schema=SCHEMA, status=status, message=message, **extra) + target = output/'robot-progress.json' + temporary = output/'robot-progress.tmp' + temporary.write_text(json.dumps(value, allow_nan=False)) + temporary.replace(target) + + +def _record(output, value): + with (Path(output)/'robot-receipt.json').open('x') as stream: + json.dump(value, stream, allow_nan=False) + + +def run_worker(kind, study, assets, output, expected_binding=None, installation=None): + """Worker-only orchestration. Each registered stage is attempted at most once.""" + from . import camera_action_study + steps = []; started = time.perf_counter() + try: + _write_progress(output, 'checking', 'Verifying the registered study and robot assets.') + checked = check_study(study, assets) + if expected_binding is not None and checked['binding'] != expected_binding: + raise ValueError('The checked study, protocol or assets changed. Check the inputs again.') + if kind != 'check' and _worker_lock(study)['present']: + raise ValueError('This study has an existing worker lock. Wait for its Terminal run, then check the inputs again.') + if kind == 'check': + status = 'blocked' if checked['inspection']['status'] == 'blocked' else 'checked' + elif kind == 'run': + for _ in range(len(camera_action_study.STAGES)+1): + state = checked['inspection'] + if state['status'] in ('blocked', 'complete'): + break + actions = state.get('next_actions', []) + if len(actions) != 1: + raise ValueError('The study did not declare exactly one next stage.') + action = actions[0] + command, stage = action['command'], action['stage'] + if command not in ('collect', 'train', 'evaluate'): + raise ValueError('Unknown registered study stage.') + key = (command, stage) + if key in steps: + raise ValueError('The study requested a repeated stage; no automatic retry is allowed.') + _write_progress(output, 'running', 'Running '+command+' / '+stage+'.', command=command, stage=stage) + steps.append(key) + if command == 'train': + camera_action_study.train(study, stage=stage) + else: + getattr(camera_action_study, command)(study, assets, stage=stage) + checked = check_study(study, assets) + if checked['binding'] != expected_binding: + raise ValueError('The bound study protocol or assets changed during execution.') + else: + raise ValueError('Registered study stage limit exceeded.') + status = checked['inspection']['status'] + elif kind == 'install': + if not checked['install_available']: + raise ValueError('Complete the benchmark before installing its simulation candidate.') + from .arm_skill_install import install_model, inspect_installation + destination = Path(installation) + if destination.exists() or destination.is_relative_to(Path(study)): + raise ValueError('Installation must use a fresh directory outside the study.') + install_model(checked['selected_model'], 'piper', assets, destination) + installation_record = inspect_installation(destination, assets) + checked['installation'] = dict(path=str(destination), record=installation_record) + status = 'installed' + else: + raise ValueError('Unknown robot studio action.') + result = dict(**checked, status=status, executed_stages=[list(step) for step in steps], + elapsed_seconds=time.perf_counter()-started) + _write_progress(output, status, 'Simulation study '+status+'.') + _record(output, result) + return result + except Exception as exc: + result = dict(schema=SCHEMA, status='failed', error=str(exc)[:2000], + executed_stages=[list(step) for step in steps], elapsed_seconds=time.perf_counter()-started, + install_available=False, physical_robot_ready=False) + _write_progress(output, 'failed', str(exc)[:600]) + _record(output, result) + raise + + +class CameraActionStudio: + """One robot job, protected by the parent server's shared lock and pool.""" + def __init__(self, server, *, worker=None, checker=None): + self.server = server + self.worker = worker or run_worker + self.checker = checker or check_study + self.job = None; self.checked = None + self.external_lock_seen = False + + def refresh(self): + job = self.job + if job is None or job['status'] != 'running' or not job['future'].done(): + return + try: + value = job['future'].result() + if type(value) is not dict or value.get('status') not in ('checked', 'blocked', 'complete', 'installed'): + raise ValueError('The robot worker did not return a verified study state.') + verified = self.checker(job['study'], job['assets']) + if verified['binding'] != value['binding']: + raise ValueError('The study or assets changed after the worker completed.') + if value['status'] == 'blocked' or verified['inspection']['status'] == 'blocked': + self.checked = None + value = {**verified, 'status': 'blocked', 'install_available': False} + else: + self.checked = verified + value.update(verified) + if job['kind'] == 'check': + self.external_lock_seen = _worker_lock(job['study'])['present'] + job.update(status=value['status'], summary=value, error=None, finished=time.monotonic()) + except Exception as exc: + self.checked = None + job.update(status='failed', summary=None, error=str(exc)[:2000], finished=time.monotonic()) + + def busy(self): + self.refresh() + return self.job is not None and self.job['status'] == 'running' + + def submit(self, kind, study, assets): + with self.server.lock: + self.server.refresh(); self.refresh() + if self.busy() or (self.server.job is not None and self.server.job['status'] == 'running'): + raise RuntimeError('One CPU job is already running. Wait for its outcome.') + try: + study = str(_local_path(study, 'prepared study')); assets = str(_local_path(assets, 'robot assets')) + binding = None + if kind in ('run', 'install'): + if self.checked is None: + raise ValueError('Check the paired demonstrations and assets before continuing.') + verified = self.checker(study, assets) + if self.external_lock_seen or _worker_lock(study)['present']: + raise ValueError('Study running in Terminal or awaiting a fresh check. Recheck after its worker lock clears.') + if verified['binding'] != self.checked['binding']: + raise ValueError('The checked protocol or assets changed. Check the inputs again.') + if kind == 'run' and verified['inspection']['status'] != 'ready': + raise ValueError('This study has no runnable next stage.') + if kind == 'install' and not verified['install_available']: + raise ValueError('No completed simulation candidate is available to install.') + binding = verified['binding'] + elif kind == 'check': + self.checked = None + self.external_lock_seen = False + else: + raise ValueError('Unknown robot studio action.') + except (OSError, ValueError, RuntimeError): + self.checked = None + self.job = None + raise + identity = uuid.uuid4().hex + output = self.server.directory/'robot-jobs'/identity + output.mkdir(parents=True, exist_ok=False) + installation = str(self.server.directory/'installations'/identity) if kind == 'install' else None + future = self.server.pool.submit(self.worker, kind, study, assets, str(output), binding, installation) + self.job = dict(id=identity, kind=kind, output=output, study=study, assets=assets, + status='running', future=future, summary=None, error=None, + started=time.monotonic(), finished=None) + return dict(status='running', job_id=identity, kind=kind) + + def state(self): + with self.server.lock: + self.server.refresh(); self.refresh() + if self.checked is not None and not (self.job and self.job['status'] == 'running'): + try: + protocol = Path(self.checked['binding']['study'])/'protocol.json' + if protocol.is_symlink() or sha256(protocol.read_bytes()).hexdigest() != self.checked['binding']['protocol_sha256']: + raise ValueError('The checked protocol changed. Check the inputs again.') + except (OSError, ValueError) as exc: + self.checked = None + if self.job is not None: + self.job.update(status='failed', summary=None, error=str(exc)[:2000], finished=time.monotonic()) + footage_busy = self.server.job is not None and self.server.job['status'] == 'running' + result = dict(schema=SCHEMA, status='idle', busy=footage_busy, job=None, + summary=None, error=None, checked=False, can_run=False, can_install=False, + study_locked=False, recheck_required=False, study_worker_lock=dict(present=False), + physical_robot_ready=False) + if self.job is not None: + job = self.job + progress = None + path = job['output']/'robot-progress.json' + if path.is_file() and path.stat().st_size <= 8192: + try: + progress = json.loads(path.read_text()) + except (ValueError, OSError): + pass + result.update(status=job['status'], busy=footage_busy or job['status']=='running', + summary=job['summary'], error=job['error'], job=dict(id=job['id'],kind=job['kind'], + elapsed_seconds=(job['finished'] or time.monotonic())-job['started'], progress=progress, + study=job['study'], assets=job['assets'])) + if self.checked is not None: + worker_lock = _worker_lock(self.checked['binding']['study']) + if worker_lock['present'] and not (self.job and self.job['status']=='running'): + self.external_lock_seen = True + result.update(checked=True, + study_locked=worker_lock['present'], study_worker_lock=worker_lock, + recheck_required=self.external_lock_seen, + can_run=not result['busy'] and not self.external_lock_seen and not worker_lock['present'] + and self.checked['inspection']['status']=='ready', + can_install=not result['busy'] and not self.external_lock_seen and not worker_lock['present'] + and self.checked['install_available']) + return result diff --git a/src/dvidia_training/camera_action_study.py b/src/dvidia_training/camera_action_study.py new file mode 100644 index 0000000..b12816d --- /dev/null +++ b/src/dvidia_training/camera_action_study.py @@ -0,0 +1,861 @@ +"""Registered local camera-to-action study; failures never become qualifications. + +Every native attempt has a literal registered identity. Repeating a stage may +finish its previously unattempted roster, but never overwrites or retries an +existing attempt. Study inspection imports no renderer and issues no commands. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +from hashlib import sha256 +from functools import wraps +import json +import math +import os +from pathlib import Path +import platform +import shutil +import tempfile +import time +import zipfile + +import numpy as np + +SCHEMA = 'dvidia.camera-action-study/v1' +SEED = 311401 +BOOTSTRAP_SEED = 311402 +GIB = 1024 ** 3 +STUDY_MAX_BYTES = 120 * GIB +DISK_RESERVE_BYTES = 30 * GIB +ATTEMPT_HEADROOM_BYTES = 320 * 1024 ** 2 +GOALS = ((.35, .08, .08), (.40, -.08, .08)) +SOURCES = ('camera_action_study.py', 'camera_action_capture.py', + 'camera_action_model.py', 'camera_action_export.py', + 'piper_rgbd_tracker.py', 'piper_action_capture.py', + 'action_demonstration.py', 'piper_task.py', 'reference_task.py', + 'reference_sim.py', 'piper_imitation_score.py', 'piper_motor_primitive.py', + 'piper_feedback_env.py', 'piper_feedback_score.py', + 'piper_imitation.py', 'robot_profiles.py', 'robot_profiles.lock.json', + 'episode_tape.py', 'embodiment_calibration.py') +STAGES = ('diagnostic', 'demonstrations', 'train-initial', + 'development-initial', 'corrections', 'train-corrected', + 'development-corrected', 'confirmation', 'negative') +LIMITATIONS = [ + 'Synthetic robot demonstrations from an authored teacher; not human-video motor recovery.', + 'One simulated PiPER, a known rigid block, two known goals and bounded starting-position interpolation.', + 'The frozen visual estimator can abstain under gripper occlusion or object tilt; historical anchors are explicitly historical.', + 'Synthetic camera time and host computation are recorded separately; no physical sensor delivery or real-time hardware claim.', + 'Contact, friction, depth accuracy and actuator models remain unqualified on physical hardware.', +] + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(',', ':'), allow_nan=False).encode() + + +def digest(value): + return sha256(canonical(value)).hexdigest() + + +def file_hash(path): + h = sha256() + with Path(path).open('rb') as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + h.update(chunk) + return h.hexdigest() + + +def read(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 128 * 1024 ** 2: + raise ValueError('Expected a bounded ordinary JSON artifact.') + def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON field.') + result[key] = value + return result + def reject(value): + raise ValueError('Nonfinite JSON number.') + return json.loads(path.read_bytes(), object_pairs_hook=pairs, parse_constant=reject) + + +def write(path, value): + _create(Path(path), canonical(value) + b'\n') + + +def _create(path, body): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=path.parent, prefix='.camera-action-', delete=False) as stream: + temporary = Path(stream.name) + stream.write(body) + stream.flush() + os.fsync(stream.fileno()) + os.link(temporary, path) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def _root(root): + root = Path(root).absolute() + if any(p.is_symlink() and str(p) not in ('/tmp', '/var') for p in (root, *root.parents)): + raise ValueError('Study paths cannot traverse symbolic links.') + return root + + +def runtime_hashes(): + return {name: file_hash(Path(__file__).with_name(name)) for name in SOURCES} + + +def _historical_starts(): + # Previously published original imitation starts; random fresh positions + # use micrometre precision and are explicitly checked against this roster. + return {(x, y) for x in (.284, .294, .304, .314) for y in (-.045, -.035, -.025, -.015)} | { + (x, y) for x in (.289, .309) for y in (-.04, -.02)} | { + (.287, -.032), (.291, -.018), (.299, -.041), (.303, -.028), (.311, -.038), (.307, -.022)} + + +def registered_cases(seed=SEED): + if type(seed) is not int or seed != SEED: + raise ValueError('This v0.1 study uses the registered seed 311401.') + rng = np.random.default_rng(seed) + seen = _historical_starts() + points = [] + while len(points) < 72: + point = tuple(np.round(rng.uniform([.287, -.042], [.311, -.018]), 6).tolist()) + if point not in seen: + seen.add(point) + points.append(point) + splits = {} + offset = 0 + for split, count in (('train', 32), ('development', 8), ('confirmation', 32)): + splits[split] = [] + for index, (x, y) in enumerate(points[offset:offset + count]): + group = f'{split}-{index:02}' + for goal, position in enumerate(GOALS): + splits[split].append(dict(id=f'{group}-goal{goal}', group_id=group, + split=split, goal_id=goal, seed=seed + 2 * (offset + index) + goal, + object_position_m=[x, y, .082], goal_position_m=list(position))) + offset += count + return splits + + +def _attempt(case, controller, *, prefix='', negative=None): + return dict(id=f'{prefix}{case["id"]}-{controller}' + (f'-{negative}' if negative else ''), + case=deepcopy(case), controller=controller, negative_control=negative) + + +def prepare(root, *, fixed_model, seed=SEED): + """Register every scene/controller before capture and pin the comparator.""" + from .piper_motor_primitive import validate_model + root = _root(root) + if root.exists(): + raise ValueError('Use a fresh study directory; registered runs are immutable.') + model = validate_model(read(fixed_model)) + sources = runtime_hashes() + cases = registered_cases(seed) + dev_evaluation = cases['development'] + rosters = { + 'diagnostic': [_attempt(c, 'teacher', prefix='diagnostic-') for c in dev_evaluation[:4]], + 'demonstrations': [_attempt(c, 'teacher') for c in cases['train'] + cases['development']], + 'corrections': [_attempt(cases['train'][2 * i + i % 2], 'mixed') for i in range(32)], + } + for stage in ('initial', 'corrected'): + rosters[f'development-{stage}'] = [_attempt(c, controller, prefix=stage + '-') + for c in dev_evaluation for controller in ('student', 'blind')] + rosters['confirmation'] = [_attempt(c, controller) for c in cases['confirmation'] + for controller in ('student', 'blind', 'fixed', 'teacher')] + rosters['negative'] = [_attempt(c, 'student', prefix='negative-', negative=negative) + for c in cases['confirmation'][:4] + for negative in ('initial_missing', 'initial_stale', 'altered_calibration', 'open_gripper')] + protocol = dict(schema=SCHEMA, seed=seed, cases=cases, rosters=rosters, + goals=[list(g) for g in GOALS], source_sha256=sources, fixed_model_sha256=digest(model), + historical_start_exclusion=[list(p) for p in sorted(_historical_starts())], + start_domain=dict(x_m=[.287, .311], y_m=[-.042, -.018], + reason='Three millimetre inset from the original fixed primitive envelope buffers observed camera estimation error; estimates are never clipped or replaced by native truth.'), + grouping='Both goals, all cameras, controls and corrections stay in their exact starting-position group.', + capture=dict(duration_s=20., width=320, height=240, camera_hz=25, control_hz=50, + physics_timestep_s=.001, camera_pairs_per_complete_episode=501), + fitting=dict(hidden_units=[64, 64], epochs=50, batch_size=512, learning_rate=.003, seed=17, + normalization='training-only', corrections_rounds=1, teacher_mix_probability=.5), + data_quality=dict(minimum_successful_episodes_per_goal=28, all_training_groups_covered=True), + diagnostic=dict(all_four_strict_successful=True, maximum_initial_camera_error_m=.010, + error_source='Privileged native diagnostic only; never a learner feature.'), + selection='full learner development strict success descending, critical contacts ascending, common physical-scale prediction loss ascending, then initial before corrected; matched blind from selected stage', + acceptance=dict(student_successes_minimum=58, student_attempts=64, confidence=.975, + bootstrap_resamples=10000, bootstrap_seed=BOOTSTRAP_SEED, bootstrap_unit='starting-position group; both goals paired', + comparisons=['student-minus-fixed', 'student-minus-blind'], no_critical_contact_increase=True, + minimum_common_admitted_groups=24, + common_admission='Both goals must have an available bound initial camera and at least one completed control interval for both compared controllers. Require a positive paired interval on this subset as well; initial comparator refusal alone cannot establish improvement.', + all_negative_controls_correct=True, no_privileged_features_or_teacher_intervention=True), + storage=dict(maximum_study_bytes=STUDY_MAX_BYTES, disk_reserve_bytes=DISK_RESERVE_BYTES, + next_attempt_headroom_bytes=ATTEMPT_HEADROOM_BYTES), + physical_robot_ready=False, limitations=LIMITATIONS) + root.mkdir(parents=True) + write(root / 'fixed-model.json', model) + write(root / 'protocol.json', protocol) + write(root / 'registration.json', dict(schema=SCHEMA + '/registration', + protocol_sha256=file_hash(root / 'protocol.json'), fixed_model_file_sha256=file_hash(root / 'fixed-model.json'), + source_sha256=sources, registered_before_any_capture=True)) + return inspect(root) + + +def _protocol(root, *, runtime=False): + root = _root(root) + registration = read(root / 'registration.json') + if registration.get('schema') != SCHEMA + '/registration': + raise ValueError('Unknown registration schema.') + if file_hash(root / 'protocol.json') != registration['protocol_sha256']: + raise ValueError('Registered protocol bytes changed.') + if file_hash(root / 'fixed-model.json') != registration['fixed_model_file_sha256']: + raise ValueError('Pinned fixed comparator changed.') + protocol = read(root / 'protocol.json') + if protocol.get('schema') != SCHEMA or protocol.get('source_sha256') != registration['source_sha256']: + raise ValueError('Invalid or inconsistent protocol.') + if runtime and runtime_hashes() != registration['source_sha256']: + raise ValueError('Runtime changed after registration; preserve this study and register a new one.') + return protocol + + +def _tree_bytes(root): + size = 0 + for directory, subdirs, files in os.walk(root): + if any((Path(directory) / name).is_symlink() for name in subdirs + files): + raise ValueError('Study artifacts cannot contain symbolic links.') + size += sum((Path(directory) / name).stat().st_size for name in files) + return size + + +def storage_preflight(root, *, known_study_bytes=None): + root = _root(root) + size = _tree_bytes(root) if known_study_bytes is None else known_study_bytes + free = shutil.disk_usage(root).free + if size + ATTEMPT_HEADROOM_BYTES > STUDY_MAX_BYTES: + raise ValueError('Study storage budget would exceed 120 GiB; existing evidence retained.') + if free - ATTEMPT_HEADROOM_BYTES < DISK_RESERVE_BYTES: + raise ValueError('Next attempt would consume the protected 30 GiB disk reserve.') + return dict(study_bytes=size, free_bytes=free, maximum_study_bytes=STUDY_MAX_BYTES, + disk_reserve_bytes=DISK_RESERVE_BYTES) + + +def _single_worker(function): + @wraps(function) + def wrapped(root, *args, **kwargs): + _protocol(root, runtime=True) + lock = _root(root) / 'worker.lock' + try: + descriptor = os.open(lock, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError as exc: + raise ValueError('This study already has an active or interrupted worker; no concurrent execution or automatic lock removal.') from exc + try: + with os.fdopen(descriptor, 'wb') as stream: + stream.write(canonical(dict(pid=os.getpid(), operation=function.__name__))) + stream.flush() + os.fsync(stream.fileno()) + return function(root, *args, **kwargs) + finally: + lock.unlink(missing_ok=True) + return wrapped + + +def _stage(root, name): + path = Path(root) / 'stages' / (name + '.json') + if not path.exists(): + return None + seal = read(path.with_name(name + '.integrity.json')) + if seal != dict(sha256=file_hash(path), protocol_sha256=file_hash(Path(root) / 'protocol.json')): + raise ValueError('Stage receipt identity changed.') + return read(path) + + +def _seal_stage(root, name, receipt): + path = Path(root) / 'stages' / (name + '.json') + write(path, receipt) + write(path.with_name(name + '.integrity.json'), dict(sha256=file_hash(path), + protocol_sha256=file_hash(Path(root) / 'protocol.json'))) + + +def _require(root, stage, status='complete'): + result = _stage(root, stage) + if result is None or result.get('status') != status: + raise ValueError(f'Stage {stage} must be {status} before continuing.') + return result + + +def _begin(root, name, protocol, *, model_bindings=None): + target = Path(root) / 'stages' / (name + '.json') + if target.exists(): + raise ValueError('Stage already sealed; existing attempts cannot be retried.') + started = target.with_name(name + '.started.json') + if not started.exists(): + write(started, dict(schema=SCHEMA + '/stage-start', stage=name, + protocol_sha256=file_hash(Path(root) / 'protocol.json'), source_sha256=protocol['source_sha256'], + model_bindings=model_bindings)) + elif read(started)['protocol_sha256'] != file_hash(Path(root) / 'protocol.json'): + raise ValueError('Stage start is bound to a different protocol.') + elif read(started).get('model_bindings') != model_bindings: + raise ValueError('Model identity differs from the started stage.') + + +def _critical(summary): + # The capture receipt reports the native per-millisecond count explicitly. + value = summary.get('critical_contact_ticks') + if value is None: + value = summary.get('native_status', {}).get('native_info', {}).get('metrics', {}).get('critical_contact_ticks') + if value is None: + value = summary.get('score', {}).get('critical_contact_ticks') + return value if type(value) is int and value >= 0 else None + + +def _row(root, attempt, summary, directory, *, model_sha256=None): + expected_mode = 'student' if attempt['controller'] == 'blind' else attempt['controller'] + expected_case = {**attempt['case'], 'policy_artifact_sha256': model_sha256} if model_sha256 else attempt['case'] + if (summary.get('case') != expected_case or summary.get('mode') != expected_mode + or summary.get('negative_control') != attempt['negative_control']): + raise ValueError('Rollout identity differs from its registered case and control.') + score = summary.get('score') or {} + complete = summary.get('status') == 'complete' and summary.get('recorded_complete_intervals') == 1000 + return dict(id=attempt['id'], case=attempt['case'], controller=attempt['controller'], + negative_control=attempt['negative_control'], path=str(directory.relative_to(root)), + policy_artifact_sha256=model_sha256, + status=summary.get('status', 'unknown'), complete=complete, strict_success=complete and score.get('success') is True, + score=score, critical_contact_ticks=_critical(summary), + initial_anchor_available=summary.get('initial_anchor_available') is True, + initial_anchor_binding_valid=summary.get('initial_anchor_binding_valid') is True, + initial_anchor_error_m=summary.get('initial_anchor_error_m'), + recorded_complete_intervals=summary.get('recorded_complete_intervals', 0), + provenance_ok=summary.get('no_privileged_features_or_teacher_intervention', + summary.get('provenance', {}).get('learner_input_whitelist_verified', False)) is True, + perception=summary.get('perception'), timing=_timing_summary(summary.get('timing') or {}), + receipt_sha256=file_hash(directory / 'rollout.json') if (directory / 'rollout.json').exists() else None, + integrity_sha256=file_hash(directory / 'integrity.json') if (directory / 'integrity.json').exists() else None) + + +def _timing_summary(timing): + result = {name: timing.get(name) for name in ('wall_seconds', 'scope')} + boundaries = timing.get('boundaries') or [] + for metric in ('estimator_wall_seconds', 'policy_inference_wall_seconds', 'full_boundary_wall_seconds', + 'render_encode_wall_seconds'): + values = [row[metric] for row in boundaries if type(row.get(metric)) in (int, float) + and math.isfinite(row[metric]) and row[metric] >= 0.] + result[metric] = (dict(samples=len(values), p50=float(np.quantile(values, .5)), + p95=float(np.quantile(values, .95)), maximum=max(values)) if values else None) + return result + + +def _peak_rss(): + try: + import resource + return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) * (1 if platform.system() == 'Darwin' else 1024) + except ImportError: + return None + + +class _FixedPolicy: + """Adapt the pinned open-loop comparator using only allowed camera context.""" + def __init__(self, model): + from .piper_motor_primitive import MotorPrimitivePolicy + self.policy = MotorPrimitivePolicy(model) + self.anchor = None + + def __call__(self, context): + snapshot = context['snapshot'] + signals = snapshot['signals'] + if self.anchor is None: + estimate = context['estimate'] + candidate = estimate.get('object_position_estimate_m') + if candidate is None: + candidate = estimate.get('signals', {}).get('object_position_estimate_m') + if isinstance(candidate, dict): + candidate = candidate.get('value') if candidate.get('status') == 'available' else None + if candidate is None: + raise ValueError('Fixed comparator requires its initial camera position.') + self.anchor = list(candidate) + return self.policy(dict(t_seconds=snapshot['timestamp_seconds'], + joint_position_rad=signals['joint_position_rad']['value'], + object_position_m=self.anchor, target_position_m=context['goal_position_m'])) + + +def _model_path(root, stage, controller): + return Path(root) / 'models' / stage / ('blind' if controller == 'blind' else 'full') / 'model.json' + + +def _model_document(root, stage, controller): + receipt = _require(root, 'train-' + stage) + variant = 'blind' if controller == 'blind' else 'full' + path = _model_path(root, stage, controller) + if receipt['models'][variant]['model_sha256'] != file_hash(path): + raise ValueError('Model bytes differ from the sealed training receipt.') + return read(path) + + +def _model_bindings(root, name, model_stage): + protocol = _protocol(root) + result = {} + for controller in {a['controller'] for a in protocol['rosters'][name]}: + if controller == 'teacher': + result[controller] = digest(protocol['source_sha256']) + elif controller == 'fixed': + result[controller] = file_hash(Path(root) / 'fixed-model.json') + else: + _model_document(root, model_stage, controller) + result[controller] = file_hash(_model_path(root, model_stage, controller)) + return result + + +def _policy(root, controller, model_stage): + if controller == 'teacher': + return None + if controller == 'fixed': + return _FixedPolicy(read(Path(root) / 'fixed-model.json')) + from .camera_action_model import ModelPolicy + return ModelPolicy(_model_document(root, model_stage, controller)) + + +def _run_stage(root, assets, name, *, model_stage=None): + from .camera_action_capture import run_rollout, inspect_rollout + root = _root(root) + protocol = _protocol(root, runtime=True) + model_bindings = _model_bindings(root, name, model_stage) + _begin(root, name, protocol, model_bindings=model_bindings) + rows = [] + blocked = None + study_bytes = _tree_bytes(root) + for attempt in protocol['rosters'][name]: + directory = root / 'episodes' / name / attempt['id'] + if blocked is not None: + rows.append(dict(**attempt, status='unattempted', complete=False, strict_success=False, + critical_contact_ticks=None, reason=blocked)) + continue + if directory.exists(): + if not (directory / 'rollout.json').exists() or not (directory / 'integrity.json').exists(): + rows.append(dict(**attempt, status='unknown', complete=False, strict_success=False, + critical_contact_ticks=None, path=str(directory.relative_to(root)), reason='Unsealed prior attempt retained; never retried.')) + continue + inspect_rollout(directory) + summary = read(directory / 'rollout.json') + else: + try: + storage_preflight(root, known_study_bytes=study_bytes) + except ValueError as exc: + blocked = str(exc) + rows.append(dict(**attempt, status='unattempted', complete=False, strict_success=False, + critical_contact_ticks=None, reason=blocked)) + continue + controller = attempt['controller'] + policy = _policy(root, controller, model_stage) if controller != 'teacher' else None + print(json.dumps(dict(stage=name, attempt=attempt['id'], status='starting')), flush=True) + bound_case = {**attempt['case'], 'policy_artifact_sha256': model_bindings[controller]} + summary = run_rollout(assets, directory, case=bound_case, + mode='student' if controller == 'blind' else controller, policy=policy, + negative_control=attempt['negative_control'], duration_s=20.) + inspect_rollout(directory) + study_bytes += _tree_bytes(directory) + rows.append(_row(root, attempt, summary, directory, model_sha256=model_bindings[attempt['controller']])) + print(json.dumps(dict(stage=name, attempt=attempt['id'], status=rows[-1]['status'], + strict_success=rows[-1]['strict_success'])), flush=True) + receipt = dict(schema=SCHEMA + '/stage', stage=name, + protocol_sha256=file_hash(root / 'protocol.json'), rows=rows, model_bindings=model_bindings, + scheduled=len(protocol['rosters'][name]), strict_successes=sum(r['strict_success'] for r in rows), + status='blocked' if blocked or any(r['status'] in ('unknown', 'unattempted') for r in rows) else 'complete', + reason=blocked, physical_robot_ready=False) + return receipt + + +def data_quality(rows, protocol): + training = [r for r in rows if r['case']['split'] == 'train'] + successes = [r for r in training if r.get('strict_success') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid')] + required = {c['group_id'] for c in protocol['cases']['train']} + covered = {r['case']['group_id'] for r in successes} + goals = {str(g): sum(r['case']['goal_id'] == g for r in successes) for g in range(2)} + return dict(passed=covered == required and all(n >= 28 for n in goals.values()), + covered_training_groups=len(covered), required_training_groups=32, + missing_training_groups=sorted(required - covered), successful_episodes_per_goal=goals, + minimum_successful_episodes_per_goal=28, + successful_with_available_bound_initial_camera=len(successes)) + + +@_single_worker +def collect(root, assets, *, stage='diagnostic'): + root = _root(root) + if stage not in ('diagnostic', 'demonstrations', 'corrections'): + raise ValueError('Unknown collection stage.') + if stage != 'diagnostic': + _require(root, 'diagnostic') + if stage == 'corrections': + _require(root, 'train-initial') + _require(root, 'development-initial') + receipt = _run_stage(root, assets, stage, model_stage='initial' if stage == 'corrections' else None) + if stage == 'diagnostic': + passed = all(r.get('strict_success') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid') + and type(r.get('initial_anchor_error_m')) in (int, float) + and math.isfinite(r['initial_anchor_error_m']) + and 0. <= r['initial_anchor_error_m'] <= .010 for r in receipt['rows']) + receipt['diagnostic_passed'] = passed + if not passed: + receipt.update(status='blocked', reason='Diagnostic did not demonstrate complete teacher success and a bound initial visual anchor within 10 mm of native diagnostic truth in all four cases.') + elif stage == 'demonstrations': + quality = data_quality(receipt['rows'], _protocol(root)) + receipt['data_quality'] = quality + if not quality['passed']: + receipt.update(status='blocked', reason='Teacher demonstration quality gate failed; learning and confirmation are blocked.') + _seal_stage(root, stage, receipt) + if receipt['status'] == 'blocked' and not (root / 'report.json').exists(): + write(root / 'report.json', report_study(root)) + return receipt + + +def _dataset(root, rows, *, blind, label_source): + from .camera_action_export import export_episode, combine_datasets + selected = [r for r in rows if r.get('complete') and r.get('initial_anchor_available') + and r.get('initial_anchor_binding_valid') + and (label_source == 'teacher_query' or r.get('strict_success'))] + if not selected: + raise ValueError('No complete, camera-bound episodes are eligible for learning.') + return combine_datasets([export_episode(Path(root) / r['path'], blind=blind, + label_source=label_source) for r in selected]), [r['id'] for r in selected] + + +@_single_worker +def train(root, *, stage='initial'): + from .camera_action_export import combine_datasets, blind_dataset + from .camera_action_model import train as fit + root = _root(root) + if stage not in ('initial', 'corrected'): + raise ValueError('Unknown model fitting stage.') + protocol = _protocol(root, runtime=True) + demos = _require(root, 'demonstrations') + corrections = _require(root, 'corrections') if stage == 'corrected' else None + name = 'train-' + stage + _begin(root, name, protocol) + training_rows = [r for r in demos['rows'] if r['case']['split'] == 'train'] + dev_rows = [r for r in demos['rows'] if r['case']['split'] == 'development'] + export_began = time.perf_counter() + full_dataset, selected = _dataset(root, training_rows, blind=False, label_source='issued') + if corrections: + correction_data, correction_ids = _dataset(root, corrections['rows'], blind=False, label_source='teacher_query') + full_dataset = combine_datasets([full_dataset, correction_data]) + selected += correction_ids + full_development, development_ids = _dataset(root, dev_rows, blind=False, label_source='issued') + export_seconds = time.perf_counter() - export_began + models = {} + for variant in ('full', 'blind'): + target = root / 'models' / stage / variant + if target.exists(): + raise ValueError('Model fitting attempt already exists; never silently refit it.') + target.mkdir(parents=True) + write(target / 'started.json', dict(stage=stage, variant=variant, + protocol_sha256=file_hash(root / 'protocol.json'))) + began = time.perf_counter() + dataset = blind_dataset(full_dataset) if variant == 'blind' else full_dataset + development = blind_dataset(full_development) if variant == 'blind' else full_development + model = fit(dataset, blind=variant == 'blind', stage=stage, seed=17, epochs=50, + batch_size=512, learning_rate=.003, validation_dataset=development) + write(target / 'model.json', model) + models[variant] = dict(path=str((target / 'model.json').relative_to(root)), + model_sha256=file_hash(target / 'model.json'), model_bytes=(target / 'model.json').stat().st_size, + training_wall_seconds=time.perf_counter() - began, training_episodes=selected, + shared_export_wall_seconds=export_seconds, process_peak_rss_bytes=_peak_rss(), + memory_scope='Process high-water RSS including shared dataset export and any preceding fit.', + development_episodes=development_ids, training=model.get('training'), + dataset_provenance=dataset.get('provenance'), physical_robot_ready=False) + write(target / 'receipt.json', models[variant]) + receipt = dict(schema=SCHEMA + '/training', stage=name, status='complete', models=models, + protocol_sha256=file_hash(root / 'protocol.json'), physical_robot_ready=False) + _seal_stage(root, name, receipt) + return receipt + + +def _loss(model): + training = model.get('training') or {} + validation = training.get('validation') or {} + value = validation.get('action_scale_mse') + if type(value) in (int, float) and math.isfinite(value): + return float(value) + raise ValueError('Model must report a finite development prediction loss for registered selection.') + + +def freeze(root): + root = _root(root) + protocol = _protocol(root, runtime=True) + existing = root / 'freeze.json' + if existing.exists(): + _verify_freeze(root) + return read(existing) + candidates = [] + for index, stage in enumerate(('initial', 'corrected')): + receipt = _require(root, 'development-' + stage) + if receipt.get('model_bindings') != _model_bindings(root, 'development-' + stage, stage): + raise ValueError('Development outcomes do not match the trained model identities.') + rows = [r for r in receipt['rows'] if r['controller'] == 'student'] + if len(rows) != 16 or any(r.get('critical_contact_ticks') is None for r in rows): + raise ValueError('Development comparison requires all 16 learner outcomes and contact counts.') + candidates.append(dict(stage=stage, strict_successes=sum(r['strict_success'] for r in rows), + critical_contact_ticks=sum(r['critical_contact_ticks'] for r in rows), + prediction_loss=_loss(_model_document(root, stage, 'student')), order=index)) + chosen = min(candidates, key=lambda c: (-c['strict_successes'], c['critical_contact_ticks'], c['prediction_loss'], c['order'])) + identities = {} + for stage in ('initial', 'corrected'): + for controller in ('student', 'blind'): + _model_document(root, stage, controller) + path = _model_path(root, stage, controller) + identities[str(path.relative_to(root))] = file_hash(path) + for stage in STAGES[:7]: + path = root / 'stages' / (stage + '.json') + identities[str(path.relative_to(root))] = file_hash(path) + result = dict(schema=SCHEMA + '/freeze', selected_stage=chosen['stage'], candidates=candidates, + protocol_sha256=file_hash(root / 'protocol.json'), source_sha256=protocol['source_sha256'], + artifacts=identities, fixed_model_sha256=file_hash(root / 'fixed-model.json'), + frozen_before_confirmation=True, physical_robot_ready=False) + write(existing, result) + return result + + +def _verify_freeze(root, *, runtime=True): + frozen = read(Path(root) / 'freeze.json') + protocol = _protocol(root, runtime=runtime) + if frozen.get('schema') != SCHEMA + '/freeze' or frozen['source_sha256'] != protocol['source_sha256']: + raise ValueError('Frozen runtime identity changed.') + if frozen['protocol_sha256'] != file_hash(Path(root) / 'protocol.json'): + raise ValueError('Frozen protocol identity changed.') + if frozen['fixed_model_sha256'] != file_hash(Path(root) / 'fixed-model.json'): + raise ValueError('Frozen comparator identity changed.') + for name, expected in frozen['artifacts'].items(): + path = Path(name) + if path.is_absolute() or '..' in path.parts or file_hash(Path(root) / path) != expected: + raise ValueError('Frozen artifact changed.') + return frozen + + +@_single_worker +def evaluate(root, assets, *, stage='development-initial'): + root = _root(root) + if stage in ('development-initial', 'development-corrected'): + model_stage = stage.split('-', 1)[1] + _require(root, 'train-' + model_stage) + elif stage in ('confirmation', 'negative'): + frozen = freeze(root) + model_stage = frozen['selected_stage'] + if stage == 'negative': + _require(root, 'confirmation') + else: + raise ValueError('Unknown evaluation stage.') + receipt = _run_stage(root, assets, stage, model_stage=model_stage) + if stage in ('confirmation', 'negative'): + receipt['freeze_sha256'] = file_hash(root / 'freeze.json') + _seal_stage(root, stage, receipt) + if stage == 'negative': + report = report_study(root) + write(root / 'report.json', report) + return receipt + + +def _admitted(row): + return (row.get('initial_anchor_available') is True and row.get('initial_anchor_binding_valid') is True + and type(row.get('recorded_complete_intervals')) is int and row['recorded_complete_intervals'] > 0) + + +def paired_bootstrap(rows, comparator, *, admitted_only=False): + grouped = {} + for row in rows: + if row['controller'] not in ('student', comparator): + continue + group = row['case']['group_id'] + key = (row['controller'], row['case']['goal_id']) + if key in grouped.setdefault(group, {}): + raise ValueError('Duplicate paired outcome.') + grouped[group][key] = row + keys = {(c, g) for c in ('student', comparator) for g in (0, 1)} + if len(grouped) != 32 or any(set(g) != keys for g in grouped.values()): + raise ValueError('Paired interval requires exactly 32 complete starting-position groups.') + retained = {key: value for key, value in grouped.items() + if not admitted_only or all(_admitted(row) for row in value.values())} + differences = np.asarray([sum(float(g[('student', goal)].get('strict_success') is True) + - float(g[(comparator, goal)].get('strict_success') is True) + for goal in (0, 1)) / 2. for _, g in sorted(retained.items())]) + rng = np.random.default_rng(BOOTSTRAP_SEED) + count = len(differences) + sampled = differences[rng.integers(0, count, size=(10000, count))].mean(axis=1) if count else None + bounds = np.quantile(sampled, [.0125, .9875]).tolist() if count else [None, None] + return dict(comparison='student-minus-' + comparator, difference=float(differences.mean()) if count else None, + confidence=.975, lower=bounds[0], upper=bounds[1], resamples=10000, seed=BOOTSTRAP_SEED, + groups=count, excluded_groups=32-count, admitted_only=admitted_only, + pairing='both goals within exact starting-position group', positive=bounds[0] is not None and bounds[0] > 0.) + + +def _negative_correct(row): + if row.get('negative_control') == 'open_gripper': + score = row.get('score') or {} + return (row.get('complete') is True and row.get('strict_success') is False + and score.get('valid') is True + and (score.get('strict_v01') or {}).get('trajectory_complete') is True) + return (row.get('status') == 'refused' and row.get('recorded_complete_intervals') == 0 + and row.get('strict_success') is False) + + +def report_study(root): + root = _root(root) + protocol = _protocol(root) + confirmation = _stage(root, 'confirmation') + negatives = _stage(root, 'negative') + report = dict(schema=SCHEMA + '/report', physical_robot_ready=False, qualified=False, + qualification='research_candidate', protocol_sha256=file_hash(root / 'protocol.json'), + limitations=LIMITATIONS, status='incomplete', controllers={}, comparisons=[], admitted_comparisons=[], gates={}) + if confirmation is None or negatives is None: + blocked = [name for name in STAGES if (_stage(root, name) or {}).get('status') == 'blocked'] + report.update(status='blocked' if blocked else 'incomplete', blocked_stages=blocked) + return report + frozen = _verify_freeze(root, runtime=False) + if any(stage.get('freeze_sha256') != file_hash(root / 'freeze.json') for stage in (confirmation, negatives)): + raise ValueError('Evaluation stage is not bound to the frozen selection.') + rows = confirmation['rows'] + for stage, receipt in (('confirmation', confirmation), ('negative', negatives)): + expected = {a['id']: a for a in protocol['rosters'][stage]} + for row in receipt['rows']: + original = expected.get(row['id']) + if original is None or any(row.get(key) != original[key] for key in ('case', 'controller', 'negative_control')): + raise ValueError('Reported outcome identity differs from the registered roster.') + complete_roster = (len(rows) == 256 and {r['id'] for r in rows} == {r['id'] for r in protocol['rosters']['confirmation']} + and len(negatives['rows']) == 16 + and {r['id'] for r in negatives['rows']} == {r['id'] for r in protocol['rosters']['negative']} + and confirmation['status'] == 'complete' and negatives['status'] == 'complete') + for controller in ('student', 'blind', 'fixed', 'teacher'): + selected = [r for r in rows if r['controller'] == controller] + contacts = [r.get('critical_contact_ticks') for r in selected] + report['controllers'][controller] = dict(scheduled=len(selected), + strict_successes=sum(r.get('strict_success') is True for r in selected), + complete=sum(r.get('complete') is True for r in selected), + admitted_cases=sum(_admitted(r) for r in selected), + initial_refusals=sum(r.get('status') in ('refused', 'interrupted') + and r.get('recorded_complete_intervals') == 0 for r in selected), + critical_contact_episodes=sum(v > 0 for v in contacts if v is not None), + critical_contact_ticks=sum(v for v in contacts if v is not None), + contact_counts_complete=all(v is not None for v in contacts)) + if complete_roster: + report['comparisons'] = [paired_bootstrap(rows, comparator) for comparator in ('fixed', 'blind')] + report['admitted_comparisons'] = [paired_bootstrap(rows, comparator, admitted_only=True) for comparator in ('fixed', 'blind')] + student = report['controllers']['student'] + fixed = report['controllers']['fixed'] + report['gates'] = dict( + all_registered_attempts_accounted=complete_roster, + at_least_58_of_64=student['scheduled'] == 64 and student['strict_successes'] >= 58, + positive_paired_comparisons=len(report['comparisons']) == 2 and all(c['positive'] for c in report['comparisons']), + positive_common_admitted_comparisons=len(report['admitted_comparisons']) == 2 + and all(c['groups'] >= 24 and c['positive'] for c in report['admitted_comparisons']), + no_critical_contact_increase=student['contact_counts_complete'] and fixed['contact_counts_complete'] + and student['critical_contact_episodes'] <= fixed['critical_contact_episodes'], + all_negative_controls_correct=len(negatives['rows']) == 16 and all(_negative_correct(r) for r in negatives['rows']), + no_privileged_features_or_teacher_intervention=all(r.get('provenance_ok') is True for r in rows if r['controller'] in ('student', 'blind', 'fixed'))) + report['qualified'] = all(report['gates'].values()) + report['qualification'] = 'bounded_simulation_skill' if report['qualified'] else 'research_candidate' + report['status'] = 'complete' if complete_roster else 'incomplete' + report['selected_stage'] = frozen['selected_stage'] + return report + + +def inspect(root): + """Read-only UI/CLI state; does not import native simulation or run models.""" + root = _root(root) + protocol = _protocol(root) + stages = {name: _stage(root, name) for name in STAGES} + blocked = [name for name, value in stages.items() if value and value.get('status') == 'blocked'] + current = next((name for name, value in stages.items() if value is None), None) + report = report_study(root) + command = ('collect' if current in ('diagnostic', 'demonstrations', 'corrections') else + 'train' if current and current.startswith('train-') else 'evaluate') + actions = [] if blocked or current is None else [dict(command=command, + stage=current.removeprefix('train-') if command == 'train' else current, + requires_assets=command != 'train')] + return dict(schema=SCHEMA + '/inspection', status='blocked' if blocked else ('complete' if current is None else 'ready'), + study=str(root), protocol_sha256=file_hash(root / 'protocol.json'), + stage_summaries={name: None if value is None else {key: value.get(key) for key in + ('status', 'scheduled', 'strict_successes', 'reason', 'diagnostic_passed', 'data_quality')} for name, value in stages.items()}, + next_stage=None if blocked else current, next_actions=actions, blocked_stages=blocked, report=report, + qualification=report['qualification'], qualified=report['qualified'], + registered_attempts={name: len(rows) for name, rows in protocol['rosters'].items()}, + scope='Simulation-only camera-to-action research; no physical commands or human-video motor recovery.', + physical_robot_ready=False, physics_executed=False, robot_commands_sent=False) + + +def package(root, output): + """Create an evidence package, including blockers; installer bundles are separate.""" + root, output = _root(root), _root(output) + if output.exists() or output.is_relative_to(root): + raise ValueError('Choose a fresh package directory outside the study.') + state = inspect(root) + output.mkdir(parents=True) + files = [root / 'protocol.json', root / 'registration.json', root / 'fixed-model.json'] + files += sorted((root / 'stages').glob('*.json')) if (root / 'stages').exists() else [] + files += sorted((root / 'models').rglob('*.json')) if (root / 'models').exists() else [] + if (root / 'freeze.json').exists(): + files.append(root / 'freeze.json') + inventory = {} + for source in files: + relative = source.relative_to(root) + raw = source.read_bytes() + _create(output / 'evidence' / relative, raw) + inventory[str(Path('evidence') / relative)] = dict(bytes=len(raw), sha256=sha256(raw).hexdigest()) + write(output / 'report.json', state['report']) + inventory['report.json'] = dict(bytes=(output / 'report.json').stat().st_size, sha256=file_hash(output / 'report.json')) + write(output / 'manifest.json', dict(schema=SCHEMA + '/evidence-package', files=inventory, + qualification=state['report']['qualification'], qualified=state['report']['qualified'], + raw_episode_data_included=False, model_runtime_installation_included=False, + source_study_protocol_sha256=file_hash(root / 'protocol.json'), physical_robot_ready=False)) + with zipfile.ZipFile(output / 'camera-action-evidence.zip', 'x', zipfile.ZIP_DEFLATED) as archive: + for name in [*inventory, 'manifest.json']: + archive.write(output / name, name) + return dict(status='complete', output=str(output), qualification=state['report']['qualification'], + qualified=state['report']['qualified'], evidence_sha256=file_hash(output / 'camera-action-evidence.zip'), + physical_robot_ready=False) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + create = sub.add_parser('prepare') + create.add_argument('--study', required=True) + create.add_argument('--fixed-model', required=True) + create.add_argument('--seed', type=int, default=SEED) + for name, choices, default in ( + ('collect', ('diagnostic', 'demonstrations', 'corrections'), 'diagnostic'), + ('train', ('initial', 'corrected'), 'initial'), + ('evaluate', ('development-initial', 'development-corrected', 'confirmation', 'negative'), 'development-initial')): + cmd = sub.add_parser(name) + cmd.add_argument('--study', required=True) + cmd.add_argument('--stage', choices=choices, default=default) + if name != 'train': + cmd.add_argument('--assets', required=True) + check = sub.add_parser('inspect') + check.add_argument('--study', required=True) + packed = sub.add_parser('package') + packed.add_argument('--study', required=True) + packed.add_argument('--output', required=True) + args = parser.parse_args(argv) + try: + if args.command == 'prepare': + result = prepare(args.study, fixed_model=args.fixed_model, seed=args.seed) + elif args.command in ('collect', 'evaluate'): + result = globals()[args.command](args.study, args.assets, stage=args.stage) + elif args.command == 'train': + result = train(args.study, stage=args.stage) + elif args.command == 'package': + result = package(args.study, args.output) + else: + result = inspect(args.study) + print(json.dumps(result, indent=2, allow_nan=False)) + return 0 if result.get('status') not in ('blocked', 'refused', 'incomplete') else 1 + except (ValueError, OSError, RuntimeError, TypeError) as exc: + print(json.dumps(dict(status='refused', reason=str(exc), physical_robot_ready=False), allow_nan=False)) + return 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/dvidia_training/footage_pipeline.py b/src/dvidia_training/footage_pipeline.py index f2b1cd5..265a2f9 100644 --- a/src/dvidia_training/footage_pipeline.py +++ b/src/dvidia_training/footage_pipeline.py @@ -337,6 +337,9 @@ def inspect_run(path): def main(): + if len(sys.argv) > 1 and sys.argv[1] == 'robot': + from .camera_action_study import main as robot_main + raise SystemExit(robot_main(sys.argv[2:])) parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) for name in ('prepare', 'run'): diff --git a/src/dvidia_training/piper_action_capture.py b/src/dvidia_training/piper_action_capture.py index 775fd31..f5d429c 100644 --- a/src/dvidia_training/piper_action_capture.py +++ b/src/dvidia_training/piper_action_capture.py @@ -132,12 +132,18 @@ def __init__(self, assets, config): self.assets=assets;self.config=config;self.renderer=None;self.env=None self.physics_attempted=False;self.initialized=False;self.renderer_warnings=[] + def _task_configuration(self, task): + return task.PiperTaskConfig(timeout_s=self.config.duration_s) + + def _environment(self, task): + return task.PiperTaskEnv(self.assets,self.task_config) + def open(self): from . import piper_task as task, reference_sim as sim, robot_profiles self.mj,self.np=sim._native_imports();self.sim=sim - self.task_config=task.PiperTaskConfig(timeout_s=self.config.duration_s) + self.task_config=self._task_configuration(task) self.assets_metadata=robot_profiles.verify_assets('piper',task._asset_root(self.assets)) - self.env=task.PiperTaskEnv(self.assets,self.task_config);self.physics_attempted=True + self.env=self._environment(task);self.physics_attempted=True self.observation,self.info=self.env.reset(self.config.seed);self.initialized=True self.native_origin=float(self.env.data.time) self.base=self.mj.mj_name2id(self.env.model,self.mj.mjtObj.mjOBJ_BODY,'base_link') @@ -219,6 +225,9 @@ def sample(self,index): or depth.shape!=(self.config.height,self.config.width) or depth.dtype!=self.np.float32 or not self.np.isfinite(depth).all() or not (depth>0).all()):raise RuntimeError('Renderer did not supply finite aligned RGB and metric depth arrays.') if native_time!=float(self.env.data.time) or seconds!=float(self.env.elapsed):raise RuntimeError('Physics advanced during paired RGB-D capture.') + # Only the latest pair is retained for optional observation-sidecar + # consumers. Existing v1 capture files and behavior are unchanged. + self.latest_rgbd=(rgb,depth) depth_buffer=io.BytesIO();self.np.save(depth_buffer,depth.astype(' '); +} +const sourceRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..'); +const source = path.resolve(sourceArg), outfile = path.resolve(outputArg); +if (source === outfile || /piper_feedback_viewer\.bundle\.js$/.test(outfile)) { + throw new Error('The v0.3 build requires a distinct output path.'); +} +// A wheel provides this builder, but not npm dependencies. Explicit root wins; +// otherwise recognize a provisioned checkout or a provisioned current directory. +const candidates = process.env.DVIDIA_VIEWER_ROOT + ? [path.resolve(process.env.DVIDIA_VIEWER_ROOT)] + : [path.join(process.cwd(), 'viewer'), path.join(sourceRoot, 'viewer'), + path.resolve(sourceRoot, '../dvidia-training/viewer')]; +const vendorRoot = candidates.find(root=>existsSync(path.join(root, 'node_modules/esbuild/package.json'))); +if (!vendorRoot) throw new Error('Pinned viewer dependencies are missing. Run npm ci --ignore-scripts --prefix viewer in a source checkout and set DVIDIA_VIEWER_ROOT to that absolute viewer directory.'); +const require = createRequire(path.join(vendorRoot, 'package.json')); +for (const [name, version] of [['esbuild', '0.25.10'], ['three', '0.180.0']]) { + const packagePath = path.join(vendorRoot, 'node_modules', name, 'package.json'); + const installed = JSON.parse(await readFile(packagePath, 'utf8')).version; + if (installed !== version) throw new Error(`Expected pinned ${name} ${version}; found ${installed}.`); +} +const {build} = require('esbuild'); +await build({ + entryPoints: [source], outfile, bundle: true, minify: true, format: 'iife', + target: ['es2020'], platform: 'browser', + alias: { + three: path.join(vendorRoot, 'node_modules/three/build/three.module.js'), + 'three/addons/controls/OrbitControls.js': path.join(vendorRoot, 'node_modules/three/examples/jsm/controls/OrbitControls.js'), + }, + nodePaths: [path.join(vendorRoot, 'node_modules')], legalComments: 'none', charset: 'utf8', + banner: {js: '/* DVIDIA PiPER grip clock replay v0.3. MIT. Bundled Three.js r180, MIT; https://github.com/mrdoob/three.js/tree/r180 */'}, +}); +const output = await readFile(outfile); +if (output.byteLength > 8 * 1024 * 1024) throw new Error('Compiled viewer exceeds the 8 MiB bundle limit.'); +console.log(`Built ${outfile} (${output.byteLength} bytes).`); diff --git a/src/dvidia_training/piper_grip_clock_viewer.py b/src/dvidia_training/piper_grip_clock_viewer.py index 2a1be80..3c1c660 100644 --- a/src/dvidia_training/piper_grip_clock_viewer.py +++ b/src/dvidia_training/piper_grip_clock_viewer.py @@ -385,6 +385,14 @@ def _write(root, name, raw): stream.write(raw) +def _build_script(): + """The actual builder is package data, available after wheel installation.""" + path = Path(__file__).with_name('piper_grip_clock_build.mjs') + if path.is_symlink() or not path.is_file(): + raise ValueError('The installed grip-clock viewer build resource is missing. Reinstall the complete wheel.') + return path + + def _finalize(root, manifest, motion, scene, scientific_sources, source, residuals): source_root = Path(__file__).parent template = _read(source_root/'piper_grip_clock_viewer.js', MAX_FILE) @@ -399,7 +407,7 @@ def _finalize(root, manifest, motion, scene, scientific_sources, source, residua with tempfile.TemporaryDirectory(prefix='.dvidia-clock-build-', dir=root.parent) as temporary: entry = Path(temporary)/'clock.js' entry.write_bytes(generated) - build_script = Path(__file__).resolve().parents[2]/'scripts'/'build-piper-grip-clock-viewer.mjs' + build_script = _build_script() subprocess.run(['node', str(build_script), str(entry), str(root/'piper_grip_clock_viewer.bundle.js')], check=True) bundle = _read(root/'piper_grip_clock_viewer.bundle.js', MAX_FILE) html = _read(source_root/'piper_grip_clock_viewer.html', MAX_FILE) @@ -475,7 +483,7 @@ def export_viewer(study, output, assets, dataset_revision): geometry_exporter_sha256=sha256(Path(__file__).with_name('reference_replay.py').read_bytes()).hexdigest(), browser_source_sha256=sha256(Path(__file__).with_name('piper_grip_clock_viewer.js').read_bytes()).hexdigest(), html_source_sha256=sha256(Path(__file__).with_name('piper_grip_clock_viewer.html').read_bytes()).hexdigest(), - bundle_builder_sha256=sha256((Path(__file__).resolve().parents[2]/'scripts'/'build-piper-grip-clock-viewer.mjs').read_bytes()).hexdigest(), + bundle_builder_sha256=sha256(_build_script().read_bytes()).hexdigest(), base_model_raw_sha256=frozen['base_model_raw_sha256'], jaw_model_raw_sha256=frozen['jaw_model_raw_sha256'], selected_sha256=frozen['selected_sha256'], blind_sha256=frozen['blind_sha256'], jaw_mode='coordinated_reference', physical_robot_ready=False) diff --git a/src/dvidia_training/training_studio.html b/src/dvidia_training/training_studio.html index 82194bd..c3bb1f9 100644 --- a/src/dvidia_training/training_studio.html +++ b/src/dvidia_training/training_studio.html @@ -10,7 +10,7 @@ @@ -60,7 +60,7 @@ function visualMetrics(report,parent){const test=report&&report.test;if(!test)return;for(const [key,label] of [['learned_rgb_mse','Learned prediction error'],['persistence_rgb_mse','Last-frame baseline'],['train_mean_rgb_mse','Training-mean baseline']])if(typeof test[key]==='number'&&Number.isFinite(test[key]))metric(parent,label,Number(test[key].toPrecision(5)));} function showResult(summary){clearResult();$('result-content').hidden=false;$('result-empty').hidden=true;const metrics=$('result-metrics');metrics.replaceChildren();const counts=summary.counts||{};metric(metrics,'Held-out episodes',counts.test);metric(metrics,'Source groups',counts.groups);metric(metrics,'Elapsed seconds',typeof summary.elapsed_seconds==='number'?Math.round(summary.elapsed_seconds*100)/100:summary.elapsed_seconds);const cap=capability(summary);const readiness=$('readiness');readiness.replaceChildren();for(const statement of ['Visual representation: '+(cap.visual_trained?'trained':'not confirmed')+'.','Movement candidate: '+(cap.movement_candidate_trained?'trained; requires task qualification':'not trained')+'.','Acquired robot skill: no. Physical robot readiness: no.']){const p=document.createElement('p');text(p,statement);readiness.append(p);}const vm=$('visual-metrics');vm.replaceChildren();const dl=document.createElement('dl');dl.className='metrics';visualMetrics(summary.visual,dl);if(dl.children.length){const title=document.createElement('p');text(title,'Held-out next-frame prediction');const note=document.createElement('p');note.className='caption';text(note,'Normalized pixel mean squared error; lower is better. This measures visual prediction, not task completion.');vm.append(title,dl,note);}const list=$('limitations');list.replaceChildren();for(const message of summary.limitations||[]){const li=document.createElement('li');text(li,String(message));list.append(li);}text($('record'),JSON.stringify(summary,null,2));$('download').hidden=false;text($('train-hint'),'Completed locally. Inspect the held-out measurements below.');} function schedule(){clearTimeout(pollTimer);pollTimer=setTimeout(poll,800);} - async function poll(){try{const state=await request('/api/state');connected=true;$('online').dataset.allowed=state.allow_online?'true':'';text($('runtime'),state.allow_online?'CPU · local training\nHTTPS intake available':'CPU · offline server');text($('online-help'),state.allow_online?'HTTPS intake is optional. Leave unchecked to use local data without network access.':'This server is offline. Start with --allow-online only when HTTPS intake is needed.');busy=state.status==='running';const job=state.job;if(job){const elapsed=Number(job.elapsed_seconds)||0;text($('job-state'),state.status==='running'?((job.progress&&job.progress.message)||'Running locally…')+' '+Math.floor(elapsed)+' s elapsed.':state.status==='prepared'?'Footage check complete.':state.status==='complete'?'Training complete. The result has been verified.':'The job failed.');if(state.status==='prepared'&&lastJob!==job.id){showCheck(state.summary||{});checkedBinding=visualReady(state.summary||{})?binding():null;lastJob=job.id;text($('train-hint'),checkedBinding?'Ready for local visual training.':'Visual readiness was not established. Resolve the dataset check before training.');}else if(state.status==='complete'&&lastJob!==job.id){const result=await request('/api/result');showResult(result);lastJob=job.id;}else if(state.status==='failed'){checkedBinding=null;error(state.error||'The job failed. No result is available.');}}controls();if(busy)schedule();}catch(exc){connected=false;busy=false;checkedBinding=null;error(exc.message);controls();}} + async function poll(){try{const state=await request('/api/state');connected=true;$('online').dataset.allowed=state.allow_online?'true':'';text($('runtime'),state.allow_online?'CPU · local training\nHTTPS intake available':'CPU · offline server');text($('online-help'),state.allow_online?'HTTPS intake is optional. Leave unchecked to use local data without network access.':'This server is offline. Start with --allow-online only when HTTPS intake is needed.');busy=state.status==='running'||state.robot_job_running===true;const job=state.job;if(job){const elapsed=Number(job.elapsed_seconds)||0;text($('job-state'),state.status==='running'?((job.progress&&job.progress.message)||'Running locally…')+' '+Math.floor(elapsed)+' s elapsed.':state.status==='prepared'?'Footage check complete.':state.status==='complete'?'Training complete. The result has been verified.':'The job failed.');if(state.status==='prepared'&&lastJob!==job.id){showCheck(state.summary||{});checkedBinding=visualReady(state.summary||{})?binding():null;lastJob=job.id;text($('train-hint'),checkedBinding?'Ready for local visual training.':'Visual readiness was not established. Resolve the dataset check before training.');}else if(state.status==='complete'&&lastJob!==job.id){const result=await request('/api/result');showResult(result);lastJob=job.id;}else if(state.status==='failed'){checkedBinding=null;error(state.error||'The job failed. No result is available.');}}controls();if(busy)schedule();}catch(exc){connected=false;busy=false;checkedBinding=null;error(exc.message);controls();}} async function start(kind){$('error').hidden=true;clearResult();if(kind==='prepare')clearCheck();const seed=Number($('seed').value);if(!Number.isInteger(seed)||seed<0||seed>999999){error('Use an integer seed between 0 and 999999.');return;}busy=true;controls();text($('job-state'),kind==='prepare'?'Starting footage check…':'Starting local training…');try{await post('/api/'+kind,{source:$('source').value.trim(),online:$('online').checked,seed});await poll();}catch(exc){busy=false;checkedBinding=null;error(exc.message);controls();}} async function upload(file){if(!file)return;$('error').hidden=true;clearCheck();if(file.size>24*1024*1024){error('ZIP upload limit is 24 MiB. Enter the local folder path for larger footage.');return;}busy=true;controls();text($('job-state'),'Saving ZIP in the local workspace…');try{const value=await request('/api/upload',{method:'POST',headers:{'Content-Type':'application/zip'},body:file});$('source').value=value.source;$('online').checked=false;text($('job-state'),'ZIP saved locally. Check its footage before training.');}catch(exc){error(exc.message);}busy=false;$('upload').value='';controls();} $('prepare').addEventListener('click',()=>start('prepare'));$('train-button').addEventListener('click',()=>start('train'));for(const id of ['source','online','seed'])$(id).addEventListener('input',clearCheck);$('upload').addEventListener('change',event=>upload(event.target.files[0]));const drop=$('drop');for(const name of ['dragenter','dragover'])drop.addEventListener(name,event=>{event.preventDefault();if(!busy)drop.classList.add('dragging');});for(const name of ['dragleave','drop'])drop.addEventListener(name,event=>{event.preventDefault();drop.classList.remove('dragging');});drop.addEventListener('drop',event=>{if(!busy){const files=event.dataTransfer.files;if(files.length!==1){error('Drop one Skillspace ZIP.');return;}upload(files[0]);}});poll(); diff --git a/src/dvidia_training/training_studio.py b/src/dvidia_training/training_studio.py index 50721b3..4f06eb1 100644 --- a/src/dvidia_training/training_studio.py +++ b/src/dvidia_training/training_studio.py @@ -106,7 +106,8 @@ def validate_zip(raw): class TrainingStudioServer(ThreadingHTTPServer): daemon_threads = True - def __init__(self, address, directory, offline=True, *, runner=None, preparer=None, inspector=None): + def __init__(self, address, directory, offline=True, *, runner=None, preparer=None, inspector=None, + robot_worker=None, robot_checker=None): if address[0] != '127.0.0.1': raise ValueError('The training studio must bind to 127.0.0.1.') if type(offline) is not bool: @@ -119,8 +120,10 @@ def __init__(self, address, directory, offline=True, *, runner=None, preparer=No self.lock = threading.RLock() self.job = None self.prepared = None - self.pool = (ThreadPoolExecutor(max_workers=1) if runner is not None or preparer is not None + self.pool = (ThreadPoolExecutor(max_workers=1) if runner is not None or preparer is not None or robot_worker is not None else ProcessPoolExecutor(max_workers=1, mp_context=multiprocessing.get_context('spawn'))) + from .camera_action_studio import CameraActionStudio + self.robot = CameraActionStudio(self, worker=robot_worker, checker=robot_checker) try: super().__init__(address, TrainingStudioHandler) except Exception: @@ -204,6 +207,7 @@ def state(self): result = {'status': 'idle', 'allow_online': not self.offline, 'scope': 'visual learning; movement readiness is separate', 'robot_skill_acquired': False, 'physical_robot_ready': False, + 'robot_job_running': self.robot.busy(), 'job': None, 'summary': None, 'error': None} if self.job is not None: job = self.job @@ -228,7 +232,7 @@ def state(self): def submit(self, kind, source, online, seed): with self.lock: self.refresh() - if self.job is not None and self.job['status'] == 'running': + if (self.job is not None and self.job['status'] == 'running') or self.robot.busy(): raise RuntimeError('One CPU job is already running. Wait for its outcome.') offline = not online actual_source = source @@ -338,6 +342,10 @@ def do_GET(self): self.respond(Path(__file__).with_name('training_studio.html').read_bytes(), mime='text/html; charset=utf-8') elif self.path == '/coverage': self.respond(Path(__file__).with_name('coverage_studio.html').read_bytes(), mime='text/html; charset=utf-8') + elif self.path == '/robot': + self.respond(Path(__file__).with_name('camera_action_studio.html').read_bytes(), mime='text/html; charset=utf-8') + elif self.path == '/api/robot/state': + self.respond(self.server.robot.state()) elif self.path == '/api/coverage/example': from .coverage import example_plan self.respond(example_plan()) @@ -369,6 +377,11 @@ def do_POST(self): source, online, seed = self.train_request() kind = 'prepare' if self.path == '/api/prepare' else 'train' self.respond(self.server.submit(kind, source, online, seed), 202) + elif self.path in ('/api/robot/check', '/api/robot/run', '/api/robot/install'): + body = _json(self.body('application/json', JSON_LIMIT)) + if type(body) is not dict or set(body) != {'study', 'assets'}: + raise ValueError('Supply local prepared study and robot assets folders.') + self.respond(self.server.robot.submit(self.path.rsplit('/', 1)[-1], body['study'], body['assets']), 202) elif self.path == '/api/coverage/inspect': from .coverage import MAX_PLAN_BYTES, summarize_plan, validate_plan plan = validate_plan(self.body('application/json', MAX_PLAN_BYTES)) @@ -378,7 +391,7 @@ def do_POST(self): validate_zip(raw) with self.server.lock: self.server.refresh() - if self.server.job is not None and self.server.job['status'] == 'running': + if (self.server.job is not None and self.server.job['status'] == 'running') or self.server.robot.busy(): raise RuntimeError('One CPU job is already running. Wait before replacing its source.') self.server.clear_completed() root = self.server.directory / 'uploads' @@ -398,11 +411,12 @@ def do_POST(self): def configure(directory=Path('runs/training-studio'), *, port=8270, offline=True, - runner=None, preparer=None, inspector=None): + runner=None, preparer=None, inspector=None, robot_worker=None, robot_checker=None): if type(port) is not int or not 0 <= port <= 65535: raise ValueError('Use a valid local port.') return TrainingStudioServer(('127.0.0.1', port), directory, offline, - runner=runner, preparer=preparer, inspector=inspector) + runner=runner, preparer=preparer, inspector=inspector, + robot_worker=robot_worker, robot_checker=robot_checker) def run_server(directory=Path('runs/training-studio'), *, port=8270, offline=True): diff --git a/tests/test_arm_pack_dependencies.py b/tests/test_arm_pack_dependencies.py new file mode 100644 index 0000000..2dd763c --- /dev/null +++ b/tests/test_arm_pack_dependencies.py @@ -0,0 +1,86 @@ +"""Adapter archives retain only their executable closure; no physics is run.""" +import ast +import json +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest +from zipfile import ZipFile + +from dvidia_training import arm_pack +from dvidia_training.capsule_installer import EXTRA_RUNTIME_FILES +from dvidia_training.installer import RUNTIME_SOURCE_FILES + + +class ArmPackDependenciesTests(unittest.TestCase): + def test_all_relative_imports_and_recorded_runtime_are_included(self): + root = Path(arm_pack.__file__).parent + included = set(arm_pack.BUNDLE_SOURCE_FILES) + self.assertTrue(set(RUNTIME_SOURCE_FILES).issubset(included)) + self.assertTrue(set(EXTRA_RUNTIME_FILES).issubset(included)) + self.assertTrue({'studio.py', 'arm_qualify.py'}.issubset(included)) + for filename in included: + for node in ast.walk(ast.parse((root/filename).read_text())): + if isinstance(node, ast.ImportFrom) and node.level: + self.assertEqual(node.level, 1, filename) + names = [node.module] if node.module else [entry.name for entry in node.names] + for name in names: + self.assertIn(name.split('.')[0]+'.py', included, filename) + + def test_unrelated_installed_modules_do_not_expand_archive(self): + with TemporaryDirectory() as directory: + root = Path(directory) + required = (*arm_pack.BUNDLE_SOURCE_FILES, *arm_pack.BUNDLE_RESOURCE_FILES) + for name in required: + (root/name).write_text(name) + for index in range(150): + (root/f'unrelated_{index}.py').write_text('') + files = arm_pack._bundle_files(root) + self.assertEqual(set(files), {'simlab/'+name for name in required}) + self.assertLess(len(files)+6, 128) + (root/'arm_benchmark_summary.json').write_text('{}') + self.assertEqual(arm_pack._bundle_files(root)['simlab/arm_benchmark_summary.json'], b'{}') + (root/'studio.html').unlink() + with self.assertRaisesRegex(ValueError, 'dependency is absent'): + arm_pack._bundle_files(root) + + def test_symlink_dependency_is_refused(self): + with TemporaryDirectory() as directory: + root = Path(directory) + for name in (*arm_pack.BUNDLE_SOURCE_FILES, *arm_pack.BUNDLE_RESOURCE_FILES): + (root/name).write_text(name) + (root/'studio.html').unlink() + (root/'studio.html').symlink_to(root/'LICENSE') + with self.assertRaisesRegex(ValueError, 'not an ordinary file'): + arm_pack._bundle_files(root) + + def test_export_round_trip_retains_exact_runtime_with_small_archive(self): + raw = (Path(__file__).with_name('fixtures')/'arm_place_skillspace.json').read_bytes() + skill = arm_pack.load_skillspace(raw) + with TemporaryDirectory() as directory: + root = Path(directory) + payload = {'task': 'ArmPickPlace-v0', 'scope': 'simulation-only', + 'runtime': arm_pack.runtime_contract(), 'skill': skill.to_dict(), + 'summary': {'successes': 0, 'episodes': 0}, 'episodes': [], + 'limitations': ['Synthetic packaging unit test; no physics executed.']} + (root/'run.json').write_text(json.dumps(payload)) + (root/'source.json').write_bytes(raw) + manifest = arm_pack.export_arm_pack(root) + self.assertEqual(arm_pack.verify_arm_pack(root/'skill-adapter.zip'), manifest) + self.assertEqual(manifest['runtime'], payload['runtime']) + with ZipFile(root/'skill-adapter.zip') as archive: + self.assertLess(len(archive.namelist()), 40) + self.assertIn('simlab/studio.html', archive.namelist()) + self.assertNotIn('simlab/footage_pipeline.py', archive.namelist()) + + def test_existing_archive_entry_budget_is_preserved(self): + with TemporaryDirectory() as directory: + path = Path(directory)/'oversized.zip' + with ZipFile(path, 'w') as archive: + for index in range(129): + archive.writestr(str(index), b'') + with self.assertRaisesRegex(ValueError, 'resource budget exceeded'): + arm_pack.verify_arm_pack(path) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_audit.py b/tests/test_camera_action_audit.py new file mode 100644 index 0000000..8909789 --- /dev/null +++ b/tests/test_camera_action_audit.py @@ -0,0 +1,155 @@ +from copy import deepcopy +import importlib.util +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from tests.test_camera_action_model import context + + +SCRIPT = Path(__file__).parents[1]/'benchmarks/camera-action-v01/independent_audit.py' +SPEC = importlib.util.spec_from_file_location('camera_action_independent_audit', SCRIPT) +audit = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(audit) + + +def put(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(audit.canonical(document)+b'\n') + + +def seal(root): + files = {str(p.relative_to(root)): audit.identity(p) for p in root.rglob('*') + if p.is_file() and p.name != 'integrity.json'} + put(root/'integrity.json', {'schema': 'dvidia.camera-action-rollout/v2/integrity', 'files': files}) + + +def episode(root, case): + root.mkdir(parents=True) + before = context()['snapshot']; after = context(.02)['snapshot'] + for name in ('rgb', 'depth'): + path = root/('frames/'+name); path.parent.mkdir(exist_ok=True) + path.write_bytes(b'fixture bytes '+name.encode()) + reference = {'path': str(path.relative_to(root)), **audit.identity(path)} + before['camera']['value'][name] = reference + after['camera']['value'][name] = deepcopy(reference) + calibration = {'fixture': True} + estimate = context()['estimate'] + action = context()['previous_applied'] + perception = dict(index=0, snapshot_sha256=audit.digest(before), frame=before['camera']['value'], + calibration_sha256=audit.digest(calibration), tracker_sha256='a'*64, estimate=estimate) + query = dict(index=0, snapshot_sha256=audit.digest(before), action=action, origin='privileged_simulator_teacher') + put(root/'perception/0000.json', perception); put(root/'teacher/0000.json', query) + step = dict(index=0, before=before, after=after, + command=dict(authority='teacher', operator_correction=False, requested=action, issued=action), + perception=dict(path='perception/0000.json', **audit.identity(root/'perception/0000.json')), + teacher_query=dict(path='teacher/0000.json', **audit.identity(root/'teacher/0000.json')), + applied={'ramped_targets': action}) + recording = dict(schema='dvidia.camera-action-rollout/v2', control_mode='teacher', initial=before, + provenance={'source_group': case['group_id'], 'case_id': case['id']}, + goal_position_m=case['goal_position_m'], calibration=calibration, + binding={'calibration_sha256': audit.digest(calibration)}, tracker_sha256='a'*64, steps=[step]) + transform = np.eye(4).reshape(-1).tolist() + put(root/'context.json', {'metadata': {'world_from_robot_base': transform, + 'world_from_robot_base_sha256': audit.digest(transform)}}) + put(root/'recording.json', recording) + put(root/'native_episode.json', {'trajectory': [{'object_position_m': [.3, 0., .08]}]}) + put(root/'native_metadata.json', {'rows': 20}) + arrays = dict(native_tick=np.arange(1, 21), t_seconds=np.arange(1, 21)*.001, + object_position_m=np.tile([.3, 0., .08], (20, 1)), + arm_actuator_controls_rad=np.tile(action['joint_targets_rad'], (20, 1)), + applied_jaw_target_m=np.full(20, .02)) + with (root/'native.npz').open('wb') as stream: + np.savez_compressed(stream, **arrays) + summary = dict(case=case, mode='teacher', negative_control=None, status='interrupted', + recorded_complete_intervals=1, strict_success=False, score={'success': False, 'reason': 'fixture'}, + teacher_queries=1, student_calls=0, camera_pairs=1, initial_anchor_error_m=0., + perception={'decisions': 1, 'available': 1, 'missing': 0, 'longest_missing_seconds': 0.}) + put(root/'rollout.json', summary) + seal(root) + return summary + + +class AuditTests(unittest.TestCase): + def case(self, index=0): + return {'id': f'dev-{index}', 'group_id': f'dev-{index}', 'goal_position_m': [.35, .08, .08]} + + def test_partial_failure_is_audited_without_becoming_success(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + with patch.object(audit.scoring, 'score_feedback_episode', return_value=summary['score']): + result = audit.audit_episode(root, case) + self.assertFalse(result['strict_success']) + self.assertEqual(result['complete_intervals'], 1) + self.assertEqual(result['native_ticks'], 20) + self.assertEqual(result['initial_anchor_error_m'], 0.) + self.assertEqual(result['available_position_error_p95_m'], 0.) + + def test_anchor_error_recomputed_and_cannot_be_claimed(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + summary['initial_anchor_error_m'] = .1 + put(root/'rollout.json', summary); seal(root) + with patch.object(audit.scoring, 'score_feedback_episode', return_value=summary['score']): + with self.assertRaisesRegex(ValueError, 'anchor error'): + audit.audit_episode(root, case) + + def test_changed_bytes_and_nonrigid_transform_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder)/'episode'; case = self.case(); summary = episode(root, case) + (root/'frames/rgb').write_bytes(b'changed') + with self.assertRaisesRegex(ValueError, 'identity changed'): + audit.audit_episode(root, case) + seal(root) + context_doc = audit.read(root/'context.json') + context_doc['metadata']['world_from_robot_base'][0] = 2. + context_doc['metadata']['world_from_robot_base_sha256'] = audit.digest(context_doc['metadata']['world_from_robot_base']) + put(root/'context.json', context_doc); seal(root) + with self.assertRaisesRegex(ValueError, 'fixed-base transform'): + audit.audit_episode(root, case) + + def test_failed_setup_can_pass_consistency_audit(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); case = self.case() + put(root/'rollout.json', {'case': case, 'mode': 'teacher', 'negative_control': None, + 'status': 'refused', 'recorded_complete_intervals': 0, 'strict_success': False}) + seal(root) + result = audit.audit_episode(root, case) + self.assertEqual(result['native_ticks'], 0) + self.assertFalse(result['strict_success']) + + def test_four_attempt_denominator_and_source_bytes(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + sources = {'piper_feedback_score.py': audit.identity(Path(audit.scoring.__file__))['sha256']} + roster, rows = [], [] + for index in range(4): + case = self.case(index); attempt = {'id': f'diagnostic-{index}', 'case': case} + directory = root/'episodes/diagnostic'/attempt['id'] + episode(directory, {**case, 'policy_artifact_sha256': audit.digest(sources)}) + roster.append(attempt); rows.append({**attempt, 'strict_success': False}) + protocol = {'source_sha256': sources, 'rosters': {'diagnostic': roster}} + put(root/'protocol.json', protocol) + protocol_sha = audit.identity(root/'protocol.json')['sha256'] + put(root/'registration.json', {'source_sha256': sources, 'protocol_sha256': protocol_sha}) + put(root/'stages/diagnostic.json', {'rows': rows, 'model_bindings': {'teacher': audit.digest(sources)}}) + put(root/'stages/diagnostic.integrity.json', { + 'sha256': audit.identity(root/'stages/diagnostic.json')['sha256'], 'protocol_sha256': protocol_sha}) + with patch.object(audit.scoring, 'score_feedback_episode', return_value={'success': False, 'reason': 'fixture'}): + result = audit.audit(root) + self.assertTrue(result['passed'], result['errors']) + self.assertEqual(result['attempts_verified'], 4) + self.assertEqual(result['strict_successes'], 0) + self.assertFalse(result['physics_executed']) + rows.pop(); put(root/'stages/diagnostic.json', {'rows': rows, 'model_bindings': {'teacher': audit.digest(sources)}}) + put(root/'stages/diagnostic.integrity.json', { + 'sha256': audit.identity(root/'stages/diagnostic.json')['sha256'], 'protocol_sha256': protocol_sha}) + self.assertFalse(audit.audit(root)['passed']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_capture.py b/tests/test_camera_action_capture.py new file mode 100644 index 0000000..36ff819 --- /dev/null +++ b/tests/test_camera_action_capture.py @@ -0,0 +1,288 @@ +"""Data-only rollout tests. No renderer, native physics or hardware commands.""" +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import action_demonstration as tape +from dvidia_training import camera_action_capture as capture +from dvidia_training import piper_rgbd_tracker as tracker +from tests.test_action_demonstration import fixture + +CASE = dict(id='test-case', group_id='test-start', seed=4, + object_position_m=[.30, -.03, .082], goal_position_m=[.35, .08, .08]) +ACTION = dict(joint_targets_rad=[.01, .5, -.5, 0., 0., 0.], gripper_position_m=.024) +STUDENT = dict(joint_targets_rad=[.03, .5, -.5, 0., 0., 0.], gripper_position_m=.021) + + +class FakeSession: + instances = [] + fail_at = None + + def __init__(self, assets, config, case): + self.config, self.case = config, case + self.position = 0; self.teacher_calls = 0; self.closed = False + self.demo, self.assets = fixture(round(config.duration_s / .02)) + self.header = {key: deepcopy(self.demo[key]) for key in tape.HEADER} + self.initialized = False; self.last_camera = None + self.latest_rgbd = (np.zeros((240, 320, 3), dtype=np.uint8), np.ones((240, 320), dtype=np.float32)) + self.instances.append(self) + + def open(self): + if self.fail_at == 'open': + raise RuntimeError('Authored setup failure') + self.initialized = True + return self.header, deepcopy(self.demo['initial_applied_targets']), {'authored_test_double': True} + + def _snapshot(self): + snapshot = deepcopy(self.demo['initial'] if self.position == 0 else self.demo['steps'][self.position-1]['after']) + for name in tracker.SIGNALS: + snapshot['signals'][name] = capture.base._missing() + return snapshot + + def sample(self, index): + if self.fail_at == 'sample' and self.position == 4: + raise RuntimeError('Authored camera failure') + snapshot = self._snapshot() + camera = snapshot['camera']; old_index = camera['value']['index'] + frames = {} + for channel in ('rgb', 'depth'): + old_name = camera['value'][channel]['path'] + name = old_name.replace(f'{old_index:04}', f'{index:04}') + body = self.assets[old_name] + camera['value'][channel] = dict(path=name, bytes=len(body), sha256=sha256(body).hexdigest()) + frames[name] = body + camera['value']['index'] = snapshot['vision_frame_index'] = index + self.last_camera = deepcopy(camera) + return snapshot, frames, {'policy_observation': {'test': True}}, {'render_encode_wall_seconds': 0.} + + def measured(self, previous_camera): + snapshot = self._snapshot() + snapshot['camera'] = capture._sample_at(previous_camera, self.position*.02) + snapshot['vision_frame_index'] = previous_camera['value']['index'] + return snapshot, {}, {'policy_observation': {'test': True}}, {'render_encode_wall_seconds': 0.} + + def decide(self): + self.teacher_calls += 1 + return deepcopy(ACTION), {'phase': 'fake_teacher'} + + def advance(self, action): + before = self.position*.02 + self.position += 1 + if self.fail_at == 'advance' and self.position == 3: + raise RuntimeError('Authored partial interval failure') + ramped = deepcopy(action); ramped['joint_targets_rad'][0] += .001 + return dict(interval_start_seconds=before, interval_end_seconds=self.position*.02, + ramped_targets=ramped, native_position_controls=[*ramped['joint_targets_rad'],ramped['gripper_position_m']], + rate_limited=True, scope='final_native_tick_only') + + def status(self): + return dict(physics_attempted=False, native_initialized=False, + actual_native_elapsed_seconds=self.position*.02, native_info={'unexpected_contact_native_ticks': 0}, + authored_test_double=True) + + def native_evidence(self): + return {'test_only': np.zeros((self.position*20,))}, {'test_only': True} + + def anchor_error(self, estimate): + return .001 if estimate['status'] == 'available' else None + + def close(self): + self.closed = True + + +def fake_estimate(session, snapshot, calibration, negative=None): + seconds = snapshot['timestamp_seconds'] + status = 'missing' if negative in ('initial_missing', 'initial_stale', 'altered_calibration') else 'available' + estimate = tracker._empty(snapshot['vision_frame_index'], seconds, seconds, seconds) + estimate['status'] = status; estimate['age_seconds'] = 0. + if status == 'available': + values = dict(object_position_estimate_m=[.30, -.03, .08], + object_quaternion_estimate_wxyz=[1., 0., 0., 0.], object_position_uncertainty_m=[.001], + object_orientation_uncertainty_rad=[.01]) + for name, value in values.items(): + estimate[name] = value[0] if 'uncertainty' in name else value + estimate['signals'][name] = capture.base._sample(value, seconds) + return estimate, dict(estimator_wall_seconds=0., input_calibration_sha256=tape.digest(calibration), negative_control=negative) + + +class CameraCaptureTests(unittest.TestCase): + def setUp(self): + FakeSession.instances = []; FakeSession.fail_at = None + self.temp = TemporaryDirectory(); self.addCleanup(self.temp.cleanup) + for target, value in (('_Session', FakeSession), ('_fresh_estimate', fake_estimate), + ('_native_episode', lambda *args: {'test_only': True}), + ('_score', lambda *args: dict(valid=False, success=False, reason='authored_fixture_not_physics'))): + self.enterContext(patch.object(capture, target, value)) + + def run_case(self, name='run', **kwargs): + root = Path(self.temp.name) / name + report = capture.run_rollout('unused', root, case=CASE, mode=kwargs.pop('mode', 'teacher'), + duration_s=kwargs.pop('duration_s', .2), **kwargs) + return root, report + + def test_25hz_camera_50hz_measurements_and_unique_frame_budget(self): + self.assertEqual(capture.RolloutConfig().duration_s, 20.) + self.assertEqual(20*25+1, 501) + root, report = self.run_case() + self.assertEqual(report['status'], 'complete'); self.assertEqual(report['camera_pairs'], 6) + recording = capture.load_rollout(root) + self.assertEqual(len(recording['steps']), 10) + for i, step in enumerate(recording['steps']): + self.assertAlmostEqual(step['before']['timestamp_seconds'], i*.02) + camera = step['before']['camera'] + self.assertEqual(camera['value']['index'], i//2) + self.assertAlmostEqual(camera['captured_at_seconds'], (i//2)*.04) + self.assertAlmostEqual(camera['age_seconds'], (i % 2)*.02) + for name in tracker.SIGNALS: + self.assertEqual(step['before']['signals'][name]['status'], 'missing') + self.assertEqual(len(list((root / 'frames').iterdir())), 12) + self.assertFalse(report['strict_success']) + + def test_queries_once_per_boundary_actual_mixed_authority_separate(self): + observed = [] + def student(context): + self.assertEqual(set(context), {'snapshot', 'estimate', 'previous_applied', 'goal_position_m'}) + observed.append(deepcopy(context)); return deepcopy(STUDENT) + root, report = self.run_case(mode='mixed', policy=student, duration_s=.4) + self.assertEqual(report['teacher_queries'], 20); self.assertEqual(report['student_calls'], 20) + self.assertEqual(FakeSession.instances[-1].teacher_calls, 20) + recording = capture.load_rollout(root) + authorities = {step['command']['authority'] for step in recording['steps']} + self.assertEqual(authorities, {'student', 'teacher'}) + for step in recording['steps']: + query = json.loads((root / step['teacher_query']['path']).read_text()) + self.assertEqual(query['action'], ACTION) + expected = ACTION if step['command']['authority'] == 'teacher' else STUDENT + self.assertEqual(step['command']['issued'], expected) + self.assertNotIn('reference', step) + self.assertEqual(observed[1]['previous_applied'], recording['steps'][0]['applied']['ramped_targets']) + other, _ = self.run_case('other', mode='mixed', policy=student, duration_s=.4) + self.assertEqual([s['command']['authority'] for s in recording['steps']], + [s['command']['authority'] for s in capture.load_rollout(other)['steps']]) + + def test_student_and_fixed_have_no_teacher_queries(self): + for mode in ('student', 'fixed'): + root, report = self.run_case(mode, mode=mode, policy=lambda context: deepcopy(STUDENT)) + self.assertEqual(report['teacher_queries'], 0) + self.assertFalse((root / 'teacher').exists()) + self.assertEqual(FakeSession.instances[-1].teacher_calls, 0) + self.assertTrue(all(s['teacher_query'] is None for s in capture.load_rollout(root)['steps'])) + + def test_initial_missing_refuses_without_any_native_action_or_teacher_query(self): + for control in ('initial_missing', 'initial_stale', 'altered_calibration'): + root, report = self.run_case(control, mode='mixed', policy=lambda _: deepcopy(STUDENT), negative_control=control) + self.assertEqual(report['status'], 'refused'); self.assertEqual(report['started_intervals'], 0) + self.assertEqual(report['teacher_queries'], 0); self.assertEqual(report['student_calls'], 0) + self.assertEqual(FakeSession.instances[-1].position, 0) + self.assertFalse(report['initial_anchor_available']) + self.assertTrue((root / 'perception/0000.json').exists()) + capture.inspect_rollout(root) + + def test_teacher_diagnostic_retains_missing_and_continues(self): + root, report = self.run_case(negative_control='initial_missing') + self.assertEqual(report['status'], 'complete'); self.assertFalse(report['initial_anchor_available']) + first = json.loads((root / 'perception/0000.json').read_text()) + second = json.loads((root / 'perception/0001.json').read_text()) + third = json.loads((root / 'perception/0002.json').read_text()) + self.assertEqual(first['estimate']['status'], 'missing') + self.assertEqual(second['negative_control'], 'initial_missing') + self.assertEqual(third['estimate']['status'], 'available') + + def test_open_gripper_records_requested_and_issued_truthfully(self): + root, report = self.run_case(mode='student', policy=lambda _: deepcopy(STUDENT), negative_control='open_gripper') + for step in capture.load_rollout(root)['steps']: + self.assertEqual(step['command']['requested']['gripper_position_m'], STUDENT['gripper_position_m']) + self.assertEqual(step['command']['issued']['gripper_position_m'], .035) + self.assertEqual(step['applied']['native_position_controls'][-1], .035) + self.assertEqual(step['command']['issue_reason'], 'open_gripper_negative_control') + + def test_interrupt_failure_and_uncommitted_interval_are_retained(self): + root, report = self.run_case(interrupt_after=3) + self.assertEqual(report['status'], 'interrupted'); self.assertEqual(report['recorded_complete_intervals'], 3) + self.assertEqual(report['uncommitted_native_intervals'], 0) + capture.inspect_rollout(root) + FakeSession.fail_at = 'advance' + root, report = self.run_case('failed') + self.assertEqual(report['status'], 'interrupted'); self.assertEqual(report['recorded_complete_intervals'], 2) + self.assertEqual(report['started_intervals'], 3); self.assertEqual(report['uncommitted_native_intervals'], 1) + self.assertTrue((root / 'decisions/0002.json').exists()) + self.assertFalse((root / 'transitions/0002.json').exists()) + capture.inspect_rollout(root) + + def test_camera_failure_retains_prior_prefix_and_unique_written_frames(self): + FakeSession.fail_at = 'sample' + root, report = self.run_case() + self.assertEqual(report['recorded_complete_intervals'], 3) + self.assertEqual(report['uncommitted_native_intervals'], 1) + self.assertEqual(report['camera_pairs'], 2) + self.assertTrue(FakeSession.instances[-1].closed) + capture.inspect_rollout(root) + + def test_setup_failure_is_sealed_and_output_cannot_be_reused(self): + FakeSession.fail_at = 'open' + root, report = self.run_case() + self.assertEqual(report['status'], 'refused') + capture.inspect_rollout(root) + with self.assertRaises(FileExistsError): + self.run_case() + + def test_tampered_sidecar_and_rebound_snapshot_are_rejected(self): + root, report = self.run_case() + path = root / 'perception/0000.json' + document = json.loads(path.read_text()); document['snapshot_sha256'] = '0'*64 + body = tape.canonical(document)+b'\n'; path.write_bytes(body) + with self.assertRaisesRegex(ValueError, 'byte identity'): + capture.inspect_rollout(root) + manifest = json.loads((root / 'integrity.json').read_text()) + identity = dict(bytes=len(body), sha256=sha256(body).hexdigest()) + manifest['files']['perception/0000.json'] = identity + (root / 'integrity.json').write_bytes(tape.canonical(manifest)+b'\n') + with self.assertRaisesRegex(ValueError, 'Sidecar binding'): + capture.inspect_rollout(root) + + def test_reused_estimate_has_historical_time_and_no_missing_forward_fill(self): + snap = dict(timestamp_seconds=0.,vision_frame_index=0) + estimate, _ = fake_estimate(None, snap, {}) + aged = capture._estimate_at(estimate, .02) + self.assertEqual(aged['captured_at_seconds'], 0.) + self.assertEqual(aged['age_seconds'], .02) + self.assertEqual(aged['signals']['object_position_estimate_m']['age_seconds'], .02) + stale = capture._estimate_at(estimate, .12) + self.assertEqual(stale['status'], 'stale') + self.assertIsNone(stale['object_position_estimate_m']) + self.assertIsNone(stale['signals']['object_position_estimate_m']['value']) + self.assertEqual(estimate['age_seconds'], 0.) + + def test_registered_configuration_rejects_silent_cadence_and_scope_changes(self): + for changes in ({'width': 160}, {'camera_every_controls': 1}, {'duration_s': .11}, + {'duration_s': True}, {'duration_s': float('nan')}, {'duration_s': 21.}, + {'seed': True}, {'interrupt_after': -1}, {'interrupt_after': 1000}): + with self.subTest(changes=changes), self.assertRaises(ValueError): + capture.RolloutConfig(**changes) + with self.assertRaises(ValueError): + capture.run_rollout('unused',Path(self.temp.name)/'bad',case={**CASE,'object_position_m':[.3,-.03,.3]},mode='teacher') + + def test_sealed_capture_exports_actual_commands_and_separate_mixed_queries(self): + from dvidia_training.camera_action_export import export_episode + root, _ = self.run_case() + exported = export_episode(root) + self.assertEqual(exported['features'].shape, (10, 269)) + np.testing.assert_array_equal(exported['targets'], np.tile( + [*ACTION['joint_targets_rad'], ACTION['gripper_position_m']], (10, 1))) + root, _ = self.run_case('mixed-export', mode='mixed', policy=lambda _: deepcopy(STUDENT)) + exported = export_episode(root, label_source='teacher_query') + np.testing.assert_array_equal(exported['targets'], np.tile( + [*ACTION['joint_targets_rad'], ACTION['gripper_position_m']], (10, 1))) + with self.assertRaisesRegex(ValueError, 'requires teacher control'): + export_episode(root, label_source='issued') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_export.py b/tests/test_camera_action_export.py new file mode 100644 index 0000000..f55d761 --- /dev/null +++ b/tests/test_camera_action_export.py @@ -0,0 +1,145 @@ +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import camera_action_export as export +from dvidia_training import camera_action_model as m +from tests.test_camera_action_model import context, missing_visual + + +def fixture(root, *, mode='teacher', group='train-01', anchor_missing=False): + root.mkdir(exist_ok=True) + recording = dict(control_mode=mode, goal_position_m=[.35, .08, .08], + initial_applied_targets=context()['previous_applied'], + provenance={'source_group': group, 'case_id': group+'-goal0'}, + binding={'calibration_sha256': 'a'*64}, tracker_sha256='b'*64, steps=[]) + for index in range(3): + c = context(index*.02) + if index == 0 and anchor_missing: + c = missing_visual(c) + elif index == 1: + c = missing_visual(c) + before = c['snapshot'] + def save(relative, document): + path = root/relative + path.parent.mkdir(exist_ok=True) + body = m.canonical(document)+b'\n'; path.write_bytes(body) + return dict(path=relative, bytes=len(body), sha256=sha256(body).hexdigest()) + perception = save(f'perception/{index:04}.json', {'estimate': c['estimate']}) + query_action = dict(joint_targets_rad=[.8, 1.4, -1.3, 0., 0., 0.], gripper_position_m=.01) + issued_action = dict(joint_targets_rad=[.2, 1.4, -1.3, 0., 0., 0.], gripper_position_m=.03) + query = save(f'teacher/{index:04}.json', {'action': query_action, 'origin': 'privileged_simulator_teacher'}) + step = dict(index=index, before=before, perception=perception, teacher_query=query, + command={'issued': issued_action, 'authority': 'teacher' if mode == 'teacher' else 'student'}, + applied={'ramped_targets': c['previous_applied']}, + reference={'action': query_action}, native_truth={'object_position': [9, 9, 9]}) + recording['steps'].append(step) + (root/'recording.json').write_bytes(m.canonical(recording)+b'\n') + (root/'integrity.json').write_text('{"fixture":true}\n') + return recording + + +class ExportTests(unittest.TestCase): + def test_export_uses_issued_labels_and_explicit_missing_masks(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + dataset = export.export_episode(folder) + self.assertEqual(dataset['features'].shape, (3, m.FEATURE_COUNT)) + self.assertEqual(dataset['targets'][:, 0].tolist(), [.2]*3) + self.assertEqual(dataset['groups'], ['train-01']*3) + i = m.FEATURE_NAMES.index('lag0:visual_available') + self.assertEqual(dataset['features'][:, i].tolist(), [1., 0., 1.]) + self.assertEqual(dataset['provenance']['episodes'][0]['label_source'], 'issued') + + def test_queries_are_labels_not_claimed_execution(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder), mode='mixed') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + dataset = export.export_episode(folder, label_source='teacher_query') + with self.assertRaisesRegex(ValueError, 'teacher control'): + export.export_episode(folder) + self.assertEqual(dataset['targets'][:, 0].tolist(), [.8]*3) + self.assertEqual(recording['steps'][0]['command']['issued']['joint_targets_rad'][0], .2) + self.assertEqual(recording['steps'][0]['command']['authority'], 'student') + self.assertIn('separate teacher query', dataset['provenance']['label_definition']) + + def test_full_blind_same_source_identity_and_different_feature_masks(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + full = export.export_episode(folder) + blind = export.export_episode(folder, blind=True) + self.assertEqual(full['provenance']['source_identity_sha256'], blind['provenance']['source_identity_sha256']) + np.testing.assert_array_equal(full['features'][:, :m.LIVE_FEATURE_START], blind['features'][:, :m.LIVE_FEATURE_START]) + np.testing.assert_array_equal(full['targets'], blind['targets']) + self.assertTrue(np.all(blind['features'][:, m.LIVE_FEATURE_START:] == 0.)) + masked = export.blind_dataset(full) + np.testing.assert_array_equal(masked['features'], blind['features']) + self.assertEqual(masked['provenance'], blind['provenance']) + self.assertTrue(np.any(full['features'][:, m.LIVE_FEATURE_START:] != 0.)) + with self.assertRaisesRegex(ValueError, 'full-feedback'): + export.blind_dataset(blind) + + def test_current_labels_truth_reference_and_future_do_not_change_inputs(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + recording = fixture(root) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + before = export.export_episode(root) + changed = deepcopy(recording) + for row in changed['steps']: + row['command']['issued']['joint_targets_rad'][0] = .5 + row['native_truth'] = {'pose': [100]*9} + row['reference'] = {'action': [99]*7} + row['after'] = {'timestamp_seconds': 20., 'object': [99]*3} + (root/'recording.json').write_bytes(m.canonical(changed)) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=changed): + after = export.export_episode(root) + np.testing.assert_array_equal(before['features'], after['features']) + self.assertFalse(np.array_equal(before['targets'], after['targets'])) + + def test_tampered_sidecar_or_changed_recording_rejected(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder); recording = fixture(root) + (root/'perception/0001.json').write_text('{}') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'sidecar bytes'): + export.export_episode(root) + (root/'recording.json').write_text('{}') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'changed after'): + export.export_episode(root) + + def test_initial_missing_prevents_learning_pair_export(self): + with tempfile.TemporaryDirectory() as folder: + recording = fixture(Path(folder), anchor_missing=True) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=recording): + with self.assertRaisesRegex(ValueError, 'initial visual anchor'): + export.export_episode(folder) + + def test_combine_rejects_duplicates_and_mixed_feature_contracts(self): + with tempfile.TemporaryDirectory() as folder: + a, b = Path(folder)/'a', Path(folder)/'b' + ra, rb = fixture(a, group='train-a'), fixture(b, group='train-b') + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=ra): + left = export.export_episode(a) + blind = export.export_episode(a, blind=True) + with patch('dvidia_training.camera_action_capture.load_rollout', return_value=rb): + right = export.export_episode(b) + result = export.combine_datasets([left, right]) + self.assertEqual(result['groups'], ['train-a']*3+['train-b']*3) + self.assertEqual(len(result['provenance']['episodes']), 2) + with self.assertRaisesRegex(ValueError, 'duplicated'): + export.combine_datasets([left, left]) + with self.assertRaisesRegex(ValueError, 'feedback masking'): + export.combine_datasets([left, blind]) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_model.py b/tests/test_camera_action_model.py new file mode 100644 index 0000000..84a14da --- /dev/null +++ b/tests/test_camera_action_model.py @@ -0,0 +1,239 @@ +from copy import deepcopy +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import camera_action_model as m + + +def sample(value, now, captured=None): + captured = now if captured is None else captured + return dict(status='available', captured_at_seconds=captured, + available_at_seconds=captured, age_seconds=now-captured, value=value) + + +def context(now=0., x=.3, y=0.): + signals = {} + values = ([.2, 1.4, -1.3, 0., 0., 0.], [0.]*6, [.02, -.02], [0., 0.], + [.3, 0., .2], [1., 0., 0., 0.]) + for (name, _, _), value in zip(m.SENSOR_SPECS, values): + signals[name] = sample(value, now) + captured = round(int(round(now/.02))//2*.04, 8) + position = [x, y, .08] + estimate = dict(schema=m.tracker.SCHEMA, status='available', decision_at_seconds=now, + frame_index=int(round(captured/.04)), + captured_at_seconds=captured, available_at_seconds=captured, age_seconds=now-captured, + object_position_estimate_m=position, object_position_uncertainty_m=.002, + signals={'object_position_estimate_m': sample(position, now, captured), + 'object_position_uncertainty_m': sample([.002], now, captured)}) + return dict(snapshot={'timestamp_seconds': now, 'signals': signals, + 'camera': sample({'index': int(round(captured/.04))}, now, captured), + 'vision_frame_index': int(round(captured/.04))}, estimate=estimate, + previous_applied={'joint_targets_rad': values[0], 'gripper_position_m': .02}, + goal_position_m=[.35, .08, .08]) + + +def missing_visual(value): + value = deepcopy(value) + value['estimate']['status'] = 'missing' + for key in ('object_position_estimate_m', 'object_position_uncertainty_m'): + value['estimate'][key] = None + value['estimate']['signals'][key] = dict(status='missing', captured_at_seconds=None, + available_at_seconds=None, age_seconds=None, value=None) + return value + + +def dataset(blind=False, prefix='train'): + rows, targets, groups = [], [], [] + for group in range(4): + encoder = m.CameraActionFeatures(blind=blind) + for index in range(16): + c = context(index*.02, x=.28+group*.015, y=.01*group) + c['snapshot']['signals']['joint_position_rad']['value'][0] = .1+index*.01 + rows.append(encoder(c)) + targets.append([.1+group*.03+index*.001, 1.42, -1.28, 0., 0., 0., .015+group*.001]) + groups.append(prefix+'-'+str(group)) + return dict(schema=m.DATASET_SCHEMA, features=np.asarray(rows), targets=np.asarray(targets), + groups=groups, provenance={'test_fixture': True, 'origin': prefix}) + + +class FeatureTests(unittest.TestCase): + def test_dimensions_and_source_whitelist(self): + clean = context() + poisoned = deepcopy(clean) + poisoned.update(reference={'action': {'jaw': 999}}, teacher_phase='release', + current_issued=[999]*7, after={'object_position': [9]*3}, native_truth={'contact': 999}) + poisoned['snapshot'].update(reference={'action': [99]*7}, future={'time': .02}) + poisoned['snapshot']['signals']['object_position_estimate_m'] = sample([8, 8, 8], 0.) + poisoned['estimate'].update(object_quaternion_estimate_wxyz=[0, 0, 0, 0], + diagnostics={'native_truth': [99]*3}) + a = m.CameraActionFeatures()(clean) + b = m.CameraActionFeatures()(poisoned) + self.assertEqual(m.FEATURE_COUNT, 269) + self.assertEqual(a.shape, (269,)) + np.testing.assert_array_equal(a, b) + self.assertFalse(any('reference' in name or 'object_quaternion' in name for name in m.FEATURE_NAMES)) + + def test_missing_live_estimate_stays_missing_and_anchor_remains_historical(self): + encoder = m.CameraActionFeatures() + first = encoder(context()) + live = encoder(missing_visual(context(.02, x=.8))) + np.testing.assert_array_equal(first[:5], live[:5]) + index = m.FEATURE_NAMES.index('lag0:visual_available') + self.assertEqual(first[index], 1.) + self.assertEqual(live[index], 0.) + self.assertEqual(live[m.FEATURE_NAMES.index('lag0:visual_x_m/.5')], 0.) + self.assertAlmostEqual(float(live[m.FEATURE_NAMES.index('lag0:visual_age_s/.1')]), .2) + self.assertEqual(live[m.FEATURE_NAMES.index('lag40:history_present')], 0.) + + def test_camera_age_is_original_and_no_future_history(self): + encoder = m.CameraActionFeatures() + encoder(context()) + result = encoder(context(.02)) + self.assertAlmostEqual(float(result[m.FEATURE_NAMES.index('lag0:visual_age_s/.1')]), .2) + encoder(context(.04)); encoder(context(.06)) + result = encoder(context(.08)) + # Duplicate names refer to command and sensor lag presence; both are causal. + i = m.LIVE_FEATURE_START+len(m.SENSOR_NAMES)*2 + self.assertEqual(result[i+len(m.SENSOR_NAMES)-2], 1.) + self.assertAlmostEqual(float(result[i+len(m.SENSOR_NAMES)-1]), .5) + + def test_initial_missing_or_future_or_stale_refused(self): + with self.assertRaisesRegex(ValueError, 'initial visual anchor'): + m.CameraActionFeatures()(missing_visual(context())) + bad = context() + for key in ('object_position_estimate_m', 'object_position_uncertainty_m'): + bad['estimate']['signals'][key]['captured_at_seconds'] = .04 + bad['estimate']['signals'][key]['available_at_seconds'] = .04 + with self.assertRaisesRegex(ValueError, 'future'): + m.CameraActionFeatures()(bad) + bad = context(); bad['estimate']['decision_at_seconds'] = .02 + with self.assertRaisesRegex(ValueError, 'decision time'): + m.CameraActionFeatures()(bad) + + def test_blind_live_values_masks_and_ages_cannot_change_features(self): + a, b = m.CameraActionFeatures(blind=True), m.CameraActionFeatures(blind=True) + np.testing.assert_array_equal(a(context()), b(context())) + altered = missing_visual(context(.02)) + altered['snapshot']['signals'] = {'untrusted_native_truth': [100]*9} + left, right = a(context(.02)), b(altered) + np.testing.assert_array_equal(left, right) + self.assertTrue(np.all(left[m.LIVE_FEATURE_START:] == 0.)) + altered = context(.04) + altered['previous_applied']['joint_targets_rad'][0] = .3 + self.assertFalse(np.array_equal(a(context(.04)), b(altered))) + + def test_bounds_goal_and_timestamp_failures_do_not_advance_history(self): + encoder = m.CameraActionFeatures(); encoder(context()) + bad = context(.02); bad['previous_applied']['joint_targets_rad'][1] = -1. + with self.assertRaisesRegex(ValueError, 'native joint bounds'): + encoder(bad) + bad = context(.02); bad['goal_position_m'][0] = .4 + with self.assertRaisesRegex(ValueError, 'destination'): + encoder(bad) + with self.assertRaisesRegex(ValueError, '20 ms'): + encoder(context(.04)) + encoder(context(.02)) + self.assertAlmostEqual(encoder.last_time, .02) + + def test_unavailable_nonnull_and_inconsistent_receipts_refused(self): + bad = context(); bad['estimate']['signals']['object_position_estimate_m']['status'] = 'missing' + with self.assertRaisesRegex(ValueError, 'forward-fill'): + m.CameraActionFeatures()(bad) + bad = context(); bad['estimate']['object_position_estimate_m'] = [.8, 0., .08] + with self.assertRaisesRegex(ValueError, 'disagrees'): + m.CameraActionFeatures()(bad) + + def test_camera_frame_binding_and_uncertainty_capture_cannot_be_rebound(self): + bad = context(); bad['estimate']['frame_index'] = 99 + with self.assertRaisesRegex(ValueError, 'frame index'): + m.CameraActionFeatures()(bad) + encoder = m.CameraActionFeatures(); encoder(context()) + bad = context(.02) + bad['estimate']['signals']['object_position_uncertainty_m']['available_at_seconds'] = .01 + with self.assertRaisesRegex(ValueError, 'signal times'): + encoder(bad) + bad = context(.02); bad['estimate']['captured_at_seconds'] = .01 + with self.assertRaisesRegex(ValueError, 'source timestamps'): + encoder(bad) + + +class ModelTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.data = dataset() + cls.model = m.train(cls.data, validation_dataset=dataset(prefix='development')) + + def test_learning_absolute_targets_and_recipe(self): + model = self.model + self.assertEqual(model['hidden_units'], [64, 64]) + self.assertEqual(model['action_representation'], 'absolute_native_targets') + self.assertLess(model['training']['final_loss'], model['training']['initial_loss']/10.) + self.assertLess(model['training']['validation']['normalized_mse'], .1) + validation = model['training']['validation'] + expected = float(np.mean((np.asarray(validation['native_rmse'])/m.ACTION_SCALE_NATIVE)**2)) + self.assertAlmostEqual(validation['action_scale_mse'], expected, places=12) + with self.assertRaisesRegex(ValueError, 'registered recipe'): + m.train(self.data, epochs=51) + + def test_training_only_normalization_and_split_isolation(self): + altered = dataset(prefix='different-development') + altered['features'] = np.clip(altered['features']+2., -10., 10.) + alternative = m.train(self.data, validation_dataset=altered) + for key in ('weights', 'biases', 'feature_mean', 'feature_scale', 'action_mean', 'action_scale'): + self.assertEqual(self.model[key], alternative[key]) + with self.assertRaisesRegex(ValueError, 'disjoint'): + m.train(self.data, validation_dataset=self.data) + + def test_policy_has_no_teacher_and_does_not_read_truth(self): + clean, poisoned = context(), context() + poisoned['reference'] = {'action': [100]*7} + poisoned['current_teacher_query'] = {'phase': 'release'} + with patch('dvidia_training.piper_task.PiperTaskPolicy', side_effect=AssertionError('No teacher')): + a, b = m.ModelPolicy(self.model), m.ModelPolicy(self.model) + self.assertEqual(a(clean), b(poisoned)) + self.assertEqual(a.diagnostics()['teacher_fallback_calls'], 0) + self.assertEqual(a.diagnostics()['student_inference_calls'], 1) + + def test_strict_json_roundtrip_and_provenance(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder)/'model.json' + saved = m.save_model(self.model, path) + self.assertEqual(saved, m.digest(m.load_model(path))) + self.assertEqual(m.ModelPolicy(self.model)(context()), m.ModelPolicy(m.load_model(path))(context())) + with self.assertRaises(FileExistsError): + m.save_model(self.model, path) + for key, value in (('schema', 'arbitrary'), ('physical_robot_ready', True), ('jaw_limit_m', .2)): + bad = deepcopy(self.model); bad[key] = value + with self.assertRaises(ValueError): + m.validate_model(bad) + bad = deepcopy(self.model); bad['training']['provenance']['test_fixture'] = False + with self.assertRaisesRegex(ValueError, 'provenance hash'): + m.validate_model(bad) + with self.assertRaisesRegex(ValueError, 'Duplicate'): + m.validate_model('{"schema":"x","schema":"x"}') + + def test_native_action_bounds_and_blind_mask_training(self): + extreme = deepcopy(self.model) + extreme['weights'] = [np.zeros_like(w).tolist() for w in extreme['weights']] + extreme['biases'][-1] = [64., -64., 64., -64., 64., -64., 64.] + action = m.ModelPolicy(extreme)(context()) + m.action_vector(action) + blind = m.train(dataset(blind=True), blind=True) + self.assertTrue(blind['blind']) + with self.assertRaisesRegex(ValueError, 'zero all'): + m.train(self.data, blind=True) + + def test_dataset_hash_and_fixed_training_are_reproducible(self): + again = m.train(self.data, validation_dataset=dataset(prefix='development')) + self.assertEqual(self.model, again) + changed = deepcopy(self.data); changed['targets'][0, 0] += .01 + self.assertNotEqual(self.model['training']['dataset_sha256'], m.train(changed)['training']['dataset_sha256']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_skill.py b/tests/test_camera_action_skill.py new file mode 100644 index 0000000..7ee1e82 --- /dev/null +++ b/tests/test_camera_action_skill.py @@ -0,0 +1,112 @@ +"""Data-only camera policy installation cannot grant success or change arms.""" +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from dvidia_training import arm_skill_install as dispatch +from dvidia_training import camera_action_model as model +from dvidia_training import camera_action_skill as skill + + +class CameraSkillTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + data = dict(schema=model.DATASET_SCHEMA, features=[[0.]*model.FEATURE_COUNT]*16, + targets=[[0.]*6+[.02]]*16, groups=['training-fixture']*16, + provenance={'kind': 'unit-test-only'}) + cls.document = model.train(data) + cls.raw = model.canonical(cls.document)+b'\n' + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root/'model.json' + self.source.write_bytes(self.raw) + self.output = self.root/'installed' + self.binding = {'test_fixture': True} + self.native = patch.object(skill.arm_interfaces, 'build', return_value=(None, None, self.binding)).start() + self.addCleanup(patch.stopall) + + def test_dispatch_checks_camera_schema_without_motion_or_qualification(self): + result = dispatch.check_model(self.source, 'piper') + self.assertEqual(result['runner'], skill.RUNNER) + self.assertFalse(result['task_qualified_by_installation']) + self.assertFalse(result['physical_robot_ready']) + self.native.assert_not_called() + + def test_other_arms_interfaces_and_invalid_model_fail_before_motion(self): + for robot, interface in [('so101', skill.INTERFACE), ('ur5e-2f85', skill.INTERFACE), ('piper', 'ros2')]: + with self.assertRaises(ValueError): + dispatch.install_model(self.source, robot, 'unused', self.output, interface=interface) + self.native.assert_not_called() + bad = deepcopy(self.document); bad['physical_robot_ready'] = True + self.source.write_bytes(model.canonical(bad)) + with self.assertRaises(ValueError): + dispatch.check_model(self.source, 'piper') + + def test_exact_copy_inspection_and_fresh_install(self): + result = dispatch.install_model(self.source, 'piper', 'unused', self.output) + self.assertEqual((self.output/'model.json').read_bytes(), self.raw) + self.assertEqual(dispatch.inspect_installation(self.output, 'unused'), result) + self.assertEqual(result['status'], 'installed_simulation_candidate') + with self.assertRaises(ValueError): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + + def test_tampered_receipt_model_and_runtime_are_rejected(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + path = self.output/'installation.json'; original = path.read_bytes() + receipt = json.loads(original) + for key, value in [('status', 'qualified'), ('runtime_sha256', {}), ('native_binding', {})]: + bad = deepcopy(receipt); bad[key] = value + path.write_bytes(skill.canonical(bad)) + with self.assertRaises(ValueError): + dispatch.inspect_installation(self.output, 'unused') + path.write_bytes(original) + with patch.object(skill, 'runtime', return_value={}): + with self.assertRaisesRegex(ValueError, 'Runtime|runtime'): + dispatch.inspect_installation(self.output, 'unused') + (self.output/'model.json').write_bytes(self.raw+b' ') + with self.assertRaises(ValueError): + dispatch.inspect_installation(self.output, 'unused') + + def test_run_dispatch_preserves_failure_and_native_rollout_inventory(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + out = self.root/'attempt' + def run(assets, output, **kwargs): + self.assertEqual(kwargs['mode'], 'student') + self.assertNotIn('teacher', kwargs) + self.assertEqual(kwargs['case']['object_position_m'], [.30, -.03, .082]) + self.assertEqual(kwargs['case']['policy_artifact_sha256'], sha256(self.raw).hexdigest()) + output.mkdir() + (output/'integrity.json').write_text('{}') + (output/'rollout.json').write_text('{"fixture":true}') + return {'strict_success': False, 'status': 'complete', 'reason': 'failed_task'} + with patch('dvidia_training.camera_action_capture.run_rollout', side_effect=run): + result = dispatch.run_installation(self.output, 'unused', out) + self.assertFalse(result['success']) + self.assertEqual(sorted(p.name for p in out.iterdir()), ['integrity.json', 'rollout.json']) + self.assertTrue((self.root/'attempt-installation.json').is_file()) + link = json.loads((self.root/'attempt-installation.json').read_bytes()) + self.assertEqual(link['rollout_sha256'], sha256((out/'rollout.json').read_bytes()).hexdigest()) + self.assertEqual(link['integrity_sha256'], sha256((out/'integrity.json').read_bytes()).hexdigest()) + with patch('dvidia_training.camera_action_capture.run_rollout') as runner: + with self.assertRaisesRegex(ValueError, 'fresh execution'): + dispatch.run_installation(self.output, 'unused', out) + runner.assert_not_called() + + def test_out_of_envelope_refuses_before_execution(self): + dispatch.install_model(self.source, 'piper', 'unused', self.output) + with patch('dvidia_training.camera_action_capture.run_rollout') as runner: + for kwargs in ({'start_x': .284}, {'goal': True}, {'start_y': float('nan')}): + with self.assertRaises(ValueError): + dispatch.run_installation(self.output, 'unused', self.root/'attempt', **kwargs) + runner.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_camera_action_studio.py b/tests/test_camera_action_studio.py new file mode 100644 index 0000000..206d50f --- /dev/null +++ b/tests/test_camera_action_studio.py @@ -0,0 +1,221 @@ +"""Real loopback API tests with data-only injected workers; no native physics.""" +from hashlib import sha256 +import http.client +import json +from pathlib import Path +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +from dvidia_training import camera_action_studio as robot +from dvidia_training.training_studio import configure +from tests.test_training_studio import artifact_run, artifact_inspect, prepare_data + + +def fake_check(study, assets): + root = Path(study) + state = (root/'state.txt').read_text() + return dict(schema=robot.SCHEMA, + binding=dict(study=str(root.resolve()), assets=str(Path(assets).resolve()), + protocol_sha256=sha256((root/'protocol.json').read_bytes()).hexdigest(), assets_sha256='a'*64), + inspection=dict(status=state, next_actions=[] if state!='ready' else [dict(command='collect',stage='diagnostic',requires_assets=True)], + stage_summaries={}, report=dict(qualified=False, qualification='research_candidate'), + blocked_stages=['diagnostic'] if state=='blocked' else []), + install_available=state=='complete', selected_model=str(root/'model.json') if state=='complete' else None, + physical_robot_ready=False) + + +def fake_worker(kind, study, assets, output, expected_binding=None, installation=None): + result = fake_check(study, assets) + if expected_binding is not None and result['binding'] != expected_binding: + raise ValueError('Inputs changed') + if kind == 'run': + (Path(study)/'state.txt').write_text('complete') + result = fake_check(study, assets) + status = 'checked' if kind=='check' else 'complete' if kind=='run' else 'installed' + if result['inspection']['status']=='blocked': status='blocked' + result['status'] = status + if kind=='install': + destination = Path(installation); destination.mkdir(parents=True) + (destination/'test-only.txt').write_text('No real model installation') + result['installation'] = dict(path=str(destination),record={'test_only':True}) + return result + + +class CameraStudioHTTPTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory(); self.root=Path(self.temp.name) + self.study=self.root/'study'; self.study.mkdir(); (self.study/'protocol.json').write_text('{"test":true}') + (self.study/'state.txt').write_text('ready') + self.assets=self.root/'assets'; self.assets.mkdir() + self.events=[]; self.servers=[] + self.server=self.make_server() + + def make_server(self, worker=fake_worker, runner=artifact_run): + server=configure(self.root/('server'+str(len(self.servers))),port=0, + runner=runner,preparer=prepare_data,inspector=artifact_inspect, + robot_worker=worker,robot_checker=fake_check) + thread=threading.Thread(target=server.serve_forever,kwargs={'poll_interval':.01},daemon=True) + thread.start();self.servers.append((server,thread));return server + + def tearDown(self): + for event in self.events:event.set() + for server,thread in self.servers: + server.shutdown();server.server_close();thread.join(timeout=2) + self.temp.cleanup() + + def request(self,path,body=None,*,method='POST',headers=None,server=None): + server=server or self.server;port=server.server_address[1] + connection=http.client.HTTPConnection('127.0.0.1',port,timeout=3) + h={'Content-Type':'application/json','Origin':f'http://127.0.0.1:{port}'};h.update(headers or {}) + connection.request(method,path,body=None if body is None else json.dumps(body),headers=h) + response=connection.getresponse();raw=response.read();code=response.status;mime=response.getheader('Content-Type','');connection.close() + return code,json.loads(raw) if mime.startswith('application/json') else raw + + def payload(self):return dict(study=str(self.study),assets=str(self.assets)) + + def done(self,server=None): + for _ in range(200): + status,state=self.request('/api/robot/state',method='GET',server=server) + self.assertEqual(status,200) + if state['status']!='running':return state + time.sleep(.01) + self.fail('Injected worker did not finish') + + def checked(self,server=None): + status,body=self.request('/api/robot/check',self.payload(),server=server) + self.assertEqual(status,202,body);return self.done(server) + + def test_simple_ui_and_empty_state_do_not_claim_robot_readiness(self): + code,html=self.request('/robot',method='GET') + self.assertEqual(code,200);self.assertIn(b'Teach an arm, then test it.',html) + self.assertIn(b'id="install" disabled',html);self.assertIn(b'Plain videos alone',html) + self.assertNotIn(b'innerHTML',html) + code,footage=self.request('/',method='GET');self.assertIn(b'href="/robot"',footage) + state=self.done();self.assertFalse(state['can_install']);self.assertFalse(state['can_run']) + self.assertFalse(state['physical_robot_ready']) + + def test_existing_host_origin_and_cross_site_boundary_applies_to_robot_routes(self): + for route,method,body in (('/robot','GET',None),('/api/robot/state','GET',None),('/api/robot/check','POST',self.payload())): + for headers in ({'Host':'evil.example'},{'Origin':'https://evil.example'},{'Sec-Fetch-Site':'cross-site'}): + code,_=self.request(route,body,method=method,headers=headers) + self.assertEqual(code,400) + self.assertIsNone(self.server.robot.job) + + def test_check_run_install_uses_same_worker_and_installs_outside_study(self): + state=self.checked();self.assertTrue(state['checked']);self.assertTrue(state['can_run']);self.assertFalse(state['can_install']) + code,_=self.request('/api/robot/run',self.payload());self.assertEqual(code,202) + state=self.done();self.assertEqual(state['status'],'complete');self.assertTrue(state['can_install']) + code,_=self.request('/api/robot/install',self.payload());self.assertEqual(code,202) + state=self.done();self.assertEqual(state['status'],'installed') + installation=Path(state['summary']['installation']['path']) + self.assertTrue(installation.is_relative_to(self.server.directory/'installations')) + self.assertFalse(installation.is_relative_to(self.study));self.assertTrue(installation.is_dir()) + + def test_train_before_check_and_plain_video_input_are_refused(self): + self.assertEqual(self.request('/api/robot/run',self.payload())[0],400) + video=self.root/'video.mp4';video.write_bytes(b'not a study') + self.assertEqual(self.request('/api/robot/check',{**self.payload(),'study':str(video)})[0],400) + self.assertEqual(self.request('/api/robot/check',{**self.payload(),'study':'https://example.com/video'})[0],400) + + def test_one_shared_job_rejects_footage_during_robot_and_robot_during_footage(self): + event=threading.Event();self.events.append(event) + def slow_robot(*args):event.wait(3);return fake_worker(*args) + server=self.make_server(worker=slow_robot) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],202) + self.assertEqual(self.request('/api/train',{'source':'/fake'},server=server)[0],409) + code,state=self.request('/api/state',method='GET',server=server) + self.assertTrue(state['robot_job_running']) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],409) + event.set();self.done(server) + event2=threading.Event();self.events.append(event2) + def slow_footage(*args,**kwargs):event2.wait(3);return artifact_run(*args,**kwargs) + server=self.make_server(runner=slow_footage) + self.assertEqual(self.request('/api/train',{'source':'/fake'},server=server)[0],202) + self.assertEqual(self.request('/api/robot/check',self.payload(),server=server)[0],409) + self.assertTrue(self.request('/api/robot/state',method='GET',server=server)[1]['busy']) + event2.set() + + def test_blocker_clears_stale_install_capability_and_never_retries(self): + (self.study/'state.txt').write_text('complete') + self.assertTrue(self.checked()['can_install']) + (self.study/'state.txt').write_text('blocked') + state=self.checked();self.assertEqual(state['status'],'blocked') + self.assertFalse(state['checked']);self.assertFalse(state['can_install']);self.assertFalse(state['can_run']) + self.assertEqual(self.request('/api/robot/install',self.payload())[0],400) + + def test_protocol_change_invalidates_check_and_result(self): + (self.study/'state.txt').write_text('complete') + self.assertTrue(self.checked()['can_install']) + (self.study/'protocol.json').write_text('{"test":"changed"}') + state=self.done();self.assertEqual(state['status'],'failed');self.assertFalse(state['can_install']) + self.assertIsNone(state['summary']);self.assertIn('protocol changed',state['error']) + self.assertEqual(self.request('/api/robot/install',self.payload())[0],400) + + def test_terminal_worker_lock_allows_check_but_requires_recheck_before_execution(self): + lock=self.study/'worker.lock' + lock.write_text(json.dumps(dict(pid=999999,operation='collect'))) + state=self.checked() + self.assertTrue(state['checked']);self.assertTrue(state['study_locked']) + self.assertEqual(state['study_worker_lock']['operation'],'collect') + self.assertFalse(state['can_run']);self.assertFalse(state['can_install']) + self.assertTrue(lock.exists()) + # The test owns this fake lock; the implementation never removes it. + lock.unlink() + state=self.done();self.assertFalse(state['study_locked']) + self.assertTrue(state['recheck_required']);self.assertFalse(state['can_run']) + self.assertTrue(self.checked()['can_run']) + + def test_new_terminal_lock_after_check_blocks_direct_run_request(self): + self.assertTrue(self.checked()['can_run']) + lock=self.study/'worker.lock';lock.write_text('{"operation":"collect"}') + code,result=self.request('/api/robot/run',self.payload()) + self.assertEqual(code,400);self.assertIn('Terminal',result['error']);self.assertTrue(lock.exists()) + + def test_worker_error_preserves_error_without_stale_success(self): + def fail(*args):raise ValueError('Fake renderer unavailable') + server=self.make_server(worker=fail) + state=self.checked(server) + self.assertEqual(state['status'],'failed');self.assertIn('renderer unavailable',state['error']) + self.assertFalse(state['can_install']);self.assertIsNone(state['summary']) + + +class CameraWorkerTests(unittest.TestCase): + def setUp(self): + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.root=Path(self.temp.name);self.output=self.root/'job';self.output.mkdir() + self.binding=dict(study='test-study',assets='test-assets',protocol_sha256='a'*64,assets_sha256='b'*64) + + def value(self,status,action=None): + return dict(binding=self.binding,inspection=dict(status=status,next_actions=[] if action is None else [action]),install_available=False) + + def test_worker_dispatches_registered_terminal_stages_without_shell(self): + from dvidia_training import camera_action_study as study + checks=[self.value('ready',dict(command='collect',stage='diagnostic',requires_assets=True)), + self.value('ready',dict(command='train',stage='initial',requires_assets=False)), + self.value('ready',dict(command='evaluate',stage='development-initial',requires_assets=True)),self.value('complete')] + with patch.object(robot,'check_study',side_effect=checks),patch.object(study,'collect') as collect,patch.object(study,'train') as train,patch.object(study,'evaluate') as evaluate: + result=robot.run_worker('run','test-study','test-assets',self.output,self.binding) + collect.assert_called_once_with('test-study','test-assets',stage='diagnostic') + train.assert_called_once_with('test-study',stage='initial') + evaluate.assert_called_once_with('test-study','test-assets',stage='development-initial') + self.assertEqual(result['status'],'complete');self.assertEqual(len(result['executed_stages']),3) + self.assertTrue((self.output/'robot-receipt.json').is_file()) + + def test_worker_stops_at_blocker_without_retry(self): + from dvidia_training import camera_action_study as study + with patch.object(robot,'check_study',side_effect=[self.value('ready',dict(command='collect',stage='diagnostic',requires_assets=True)),self.value('blocked')]),patch.object(study,'collect') as collect: + result=robot.run_worker('run','test-study','test-assets',self.output,self.binding) + self.assertEqual(result['status'],'blocked');collect.assert_called_once() + + def test_worker_rejects_changed_binding_before_stage_and_preserves_failure(self): + with patch.object(robot,'check_study',return_value=self.value('ready')): + with self.assertRaisesRegex(ValueError,'changed'): + robot.run_worker('run','test-study','test-assets',self.output,{**self.binding,'protocol_sha256':'c'*64}) + receipt=json.loads((self.output/'robot-receipt.json').read_text()) + self.assertEqual(receipt['status'],'failed');self.assertEqual(receipt['executed_stages'],[]) + + +if __name__=='__main__':unittest.main() diff --git a/tests/test_camera_action_studio_render.py b/tests/test_camera_action_studio_render.py new file mode 100644 index 0000000..a6306d7 --- /dev/null +++ b/tests/test_camera_action_studio_render.py @@ -0,0 +1,90 @@ +"""Run the Studio result renderer in a minimal DOM; no server or physics.""" +import json +from pathlib import Path +import shutil +import subprocess +import unittest + + +HTML = Path(__file__).resolve().parents[1] / 'src/dvidia_training/camera_action_studio.html' +NODE = shutil.which('node') +HARNESS = r""" +const fs=require('fs'),vm=require('vm'),input=JSON.parse(fs.readFileSync(0,'utf8')); +const script=input.html.match(/