Golden-Set Verification

Byte-exact, per-emitter, annotation, and cross-library release gates.

Warning

Pre-implementation. This page describes proposed contracts. Class signatures, parameter types, config field names, and behavior are subject to change before code lands. Once implementation exists, content here will be regenerated from docstrings or sourced from running tests.

Golden-set verification

The strictest layer. Runs nightly. Gates releases. Compares fully-resolved outputs against pinned baselines stored in GCS (gs://rfgen-ci-golden/).

test_baseline_byte_exact.py

@pytest.mark.golden
def test_baseline_xs_byte_exact():
    """Reproduce the v0 baseline exactly."""
    out = Path("./golden-current")
    subprocess.run([
        "rfgen", "generate",
        "+preset=narrowband_classifier_baseline_xs",
        "run.seed=42", "run.num_samples=10000",
        f"storage.path={out}",
    ], check=True)
    # Sync baseline
    subprocess.run(["gsutil", "-m", "rsync", "-r",
                    "gs://rfgen-ci-golden/v0/baseline_xs", "./golden-baseline"], check=True)
    # Strict diff
    result = subprocess.run([
        "python", "scripts/diff_zarr.py",
        "./golden-baseline", str(out),
        "--strict-iq", "--strict-labels", "--strict-attrs",
    ], capture_output=True, text=True)
    assert result.returncode == 0, result.stdout

test_per_emitter_golden.py

Each emitter family has a “known-good” golden IQ recording in tests/fixtures/. The test re-generates and compares.

@pytest.mark.golden
@pytest.mark.parametrize("family,class_label", [
    ("comms", "qpsk"),
    ("comms", "16qam"),
    ("comms", "ofdm-256"),
    ("radar", "pulsed.barker.b13"),
    ("radar", "fmcw.up"),
    ("iot", "lora.css.sf7.bw125.cr-4-5"),
    ("iot", "ble.gfsk.1m.adv"),
    ("iot", "zigbee.oqpsk.ch11"),
    ("aviation", "adsb.df17.airborne_position"),
    # ... full coverage
])
def test_emitter_golden(family, class_label):
    emitter = get_emitter(family)
    rng = _make_generator(seed=42).torch_rng
    signal = emitter.generate(
        class_label=class_label, sample_rate=10e6, duration_s=1e-3,
        f_offset_hz=0.0, rng=rng,
    )
    fixture = _load_golden(f"emitters/{family}/{class_label}.npy")
    assert np.allclose(signal.iq.numpy(), fixture, atol=1e-6)

test_paes_calibration.py

PAES tolerance bands must stay tight enough that the metric is meaningful, but not so tight that legitimate paraphrase fails.

@pytest.mark.golden
@pytest.mark.annotator
def test_paes_tolerance_bands_stable():
    """Run PAES on a known {good caption, ground truth} pair; expect score ≥ 0.95."""
    record, gt = _load_golden_paes_pair()
    paes = compute_paes(record, gt, model="gemini-3.1-flash-lite")
    assert paes >= 0.95, f"Tolerance bands too tight; PAES={paes}"

@pytest.mark.golden
@pytest.mark.annotator
def test_paes_catches_known_hallucination():
    """Run PAES on a hallucinated caption; expect score ≤ 0.5."""
    record, hallu = _load_golden_paes_hallucination_pair()
    paes = compute_paes(record, hallu, model="gemini-3.1-flash-lite")
    assert paes <= 0.5

Cross-library round-trip

@pytest.mark.golden
def test_torchsig_compat_golden():
    """Round-trip a known TorchSig WidebandSig53 sample through our HDF5 writer."""
    ts_sample = _load_torchsig_golden_sample("wbs53_sample_0042.h5")
    rg_record = HDF5Store().to_internal(ts_sample)
    rg_recovered = HDF5Store().to_torchsig(rg_record)
    assert np.array_equal(ts_sample.iq, rg_recovered.iq)
    assert ts_sample.metadata.lower_freq == rg_recovered.metadata.lower_freq

See Also