From f4e357672bf00b216671ca78d9fab1f42aa4c417 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:23:17 -0700 Subject: [PATCH 1/7] Add harp.data.timecode for decoding the Harp Time Code Decode a recorded Harp Time Code, as specified in harp-tech/protocol TimeCode.md, into anchors mapping a non-Harp system's sample or local time to the whole Harp second that starts there. Packets are four headerless 8N1 bytes, and anchors sit on each packet's first falling edge, which the specification fixes as the start of the encoded second. --- docs/api/data.md | 3 + docs/examples/time-code.md | 14 + docs/examples/time_code.py | 28 ++ mkdocs.yml | 1 + src/packages/harp-data/README.md | 27 ++ .../src/harp/data/timecode/__init__.py | 13 + .../src/harp/data/timecode/_decode.py | 274 +++++++++++++ tests/data/test_timecode.py | 373 ++++++++++++++++++ 8 files changed, 733 insertions(+) create mode 100644 docs/examples/time-code.md create mode 100644 docs/examples/time_code.py create mode 100644 src/packages/harp-data/src/harp/data/timecode/__init__.py create mode 100644 src/packages/harp-data/src/harp/data/timecode/_decode.py create mode 100644 tests/data/test_timecode.py diff --git a/docs/api/data.md b/docs/api/data.md index ab15e37..9f6f167 100644 --- a/docs/api/data.md +++ b/docs/api/data.md @@ -11,3 +11,6 @@ ::: harp.data.to_file ::: harp.data.to_buffer ::: harp.data.REFERENCE_EPOCH +::: harp.data.timecode.decode_timecode_from_samples +::: harp.data.timecode.decode_timecode_from_transitions +::: harp.data.timecode.DEFAULT_BAUD_RATE diff --git a/docs/examples/time-code.md b/docs/examples/time-code.md new file mode 100644 index 0000000..76f45d2 --- /dev/null +++ b/docs/examples/time-code.md @@ -0,0 +1,14 @@ +# Align to the Harp Time Code + +Devices that are not part of the Harp bus run on their own clock, so their timestamps drift with respect to Harp time. The [Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) exists so they can be aligned afterwards: a Harp device emits the current whole Harp second as a slow serial signal (typically 1 kbps) on a digital output, the non-Harp system records it on a spare digital or analog input, and `harp.data.timecode` decodes that recording back into Harp seconds. + +Each packet starts exactly on the whole second it carries, so every decoded packet is an anchor: the sample (or local time) of its first falling edge, against the Harp second that begins there. + +!!! note + The decoded table is a set of anchors, local time → whole Harp second. How timestamps are placed between them is up to you: interpolating between neighbouring anchors absorbs the drift between the two clocks, whereas a global fit trades that away for noise rejection. + + +```python +[](./time_code.py) +``` + diff --git a/docs/examples/time_code.py b/docs/examples/time_code.py new file mode 100644 index 0000000..5b54cea --- /dev/null +++ b/docs/examples/time_code.py @@ -0,0 +1,28 @@ +import numpy as np +from harp.data.timecode import decode_timecode_from_samples, decode_timecode_from_transitions + +# A non-Harp acquisition system recorded the Harp Time Code on one of its digital +# lines. `samples` is that line, sampled at the system's own rate. +samples = np.load("timecode_line.npy") # digital states; analog input needs a `threshold` +sample_rate = 30_000.0 + +# Decode it: one row per whole Harp second, keyed on the sample where that second +# starts, the axis this system timestamps the rest of its data on too. +anchors = decode_timecode_from_samples(samples, sample_rate, baud_rate=1000.0) +print(anchors.head()) +# Time +# Sample +# 36000 3806874.0 +# 66003 3806875.0 + +# Any of the system's timestamps (spikes, video frames, stimulus onsets) can now be +# placed on the Harp axis. Interpolating between neighbouring anchors absorbs the +# drift between the two clocks. +spike_samples = np.load("spike_samples.npy") +harp_times = np.interp(spike_samples, anchors.index, anchors["Time"]) + +# Event-based systems report line transitions instead of a sampled waveform: a local +# time and the level the line took. Anchors then carry local seconds. +transitions = np.load("line_transitions.npy") +anchors = decode_timecode_from_transitions(transitions[:, 0], transitions[:, 1]) +harp_times = np.interp(spike_samples / sample_rate, anchors.index, anchors["Time"]) diff --git a/mkdocs.yml b/mkdocs.yml index 1045a5f..fe7564b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,6 +78,7 @@ nav: - Read a dataset folder: examples/dataset.md - Read a single register file: examples/register-file.md - Registers from a schema: examples/registers-from-schema.md + - Align to the Harp Time Code: examples/time-code.md - Guides: - Choose a device module: articles/device-modules.md - Migrate from harp-python: articles/harp-python-migration.md diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index faf537d..ee6f195 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -86,6 +86,33 @@ _data, timestamps, _msg, payload = AnalogData.parse_bulk(raw) df = data.payload_to_dataframe(payload) ``` +## Align a non-Harp device with the Harp Time Code + +Devices outside the Harp bus keep their own clock. A Harp device emitting the +[Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) sends the current +whole Harp second as a slow serial signal (typically 1 kbps) on a digital output, +so those devices can record it and be aligned post-hoc. `harp.data.timecode` +turns such a recording back into a table of anchors, keyed on whichever axis the +device timestamps its own data on (the sample, or the local time in seconds, of +each packet's first falling edge) against the Harp second (`"Time"`) that starts +there: + +```python +import numpy as np +from harp.data import timecode + +anchors = timecode.decode_timecode_from_samples(line, sample_rate=30_000.0) # Sample -> Time +harp_times = np.interp(spike_samples, anchors.index, anchors["Time"]) + +# event-based systems report transitions instead, so anchors carry local seconds +anchors = timecode.decode_timecode_from_transitions(edge_times, edge_states) # LocalTime -> Time +harp_times = np.interp(spike_times, anchors.index, anchors["Time"]) +``` + +Packets that fail their start/stop bit check, or whose seconds do not add up +against the local clock, are dropped, so a glitched packet costs one anchor, not +the alignment around it. + ## Write data back out `to_file` and `to_buffer` are the inverse of the readers, encoding values as Harp frames. Useful for round-tripping data or generating test corpora: diff --git a/src/packages/harp-data/src/harp/data/timecode/__init__.py b/src/packages/harp-data/src/harp/data/timecode/__init__.py new file mode 100644 index 0000000..7be859a --- /dev/null +++ b/src/packages/harp-data/src/harp/data/timecode/__init__.py @@ -0,0 +1,13 @@ +"""Aligning a non-Harp device's local timestamps to Harp time with the Harp Time Code.""" + +from ._decode import ( + DEFAULT_BAUD_RATE, + decode_timecode_from_samples, + decode_timecode_from_transitions, +) + +__all__ = [ + "decode_timecode_from_samples", + "decode_timecode_from_transitions", + "DEFAULT_BAUD_RATE", +] diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py new file mode 100644 index 0000000..225def2 --- /dev/null +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -0,0 +1,274 @@ +"""Decode a recorded Harp Time Code back into Harp seconds. + +The [Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) is a serial signal +carrying the current Harp time in whole seconds, emitted on a digital output at a low baud +rate (typically 1 kbps) so that non-Harp acquisition systems can record it alongside data +timestamped on their own clock. The functions here decode such a recording back into Harp +seconds, paired with the local time each of them was received at. + +The wire format is 8N1 serial: the line idles high and every byte is sent as one low start +bit, eight data bits (least significant first) and one high stop bit. Each packet is four +back-to-back bytes holding the whole Harp second as a little-endian ``uint32``, with no +header or checksum, and its first falling edge marks the start of that second. + +Two ways in, depending on what the recording system gives you: + +- a uniformly sampled waveform (digital or analog) → :func:`decode_timecode_from_samples` +- a list of line transitions ``(time, 0/1)`` → :func:`decode_timecode_from_transitions` + +Each returns a table of anchors keyed on the axis that system reports its own timestamps +on (a sample number or local seconds) against the whole Harp second that starts there. +How those timestamps are then interpolated onto the Harp axis is left to the caller. +""" + +import warnings +from typing import Any + +import numpy as np +import pandas as pd +from numpy.typing import ArrayLike, NDArray + +#: Baud rate used by existing Harp Time Code emitters, in bits per second. +DEFAULT_BAUD_RATE = 1000.0 + +#: Bits per byte on the wire: one start bit, eight data bits, one stop bit. +_BITS_PER_BYTE = 10 + +#: Bytes in a Harp Time Code packet, holding the whole second as a ``U32``. +_PACKET_BYTES = 4 + +#: How far, as a fraction of a bit period, a start bit may sit from its predicted +#: position and still be used to re-synchronize the frame that follows it. +_RESYNC_WINDOW = 0.3 + + +def decode_timecode_from_transitions( + timestamps: ArrayLike, + states: ArrayLike, + *, + baud_rate: float = DEFAULT_BAUD_RATE, + max_drift: float | None = 0.1, +) -> pd.DataFrame: + """Decode Harp seconds from a list of time code line transitions. + + ``timestamps`` are the local times, in seconds, of each transition of the time code + line and ``states`` the level the line took at each of them (0 low, 1 high); this is + the shape most event-based acquisition systems report. The returned DataFrame maps the + local time of each decoded packet's first falling edge (the index, named + ``"LocalTime"``) to the whole Harp second that starts there (``"Time"``, the name the + readers also give the Harp time axis). + + ``baud_rate`` must match the emitter; bits are read at the center of each bit period + and every frame re-synchronizes on its own start bit, so a mismatch of a few percent + is tolerated. Packets whose start/stop bits do not check out are dropped silently, so + a partially recorded or glitched packet costs one anchor, not the decoding of the ones + around it. ``max_drift`` additionally drops packets whose seconds do not advance + consistently with the local clock, allowing for at most that fractional rate mismatch + between the two (pass ``None`` to keep everything). + """ + times, levels = _as_transitions(timestamps, states) + local, harp = _decode(times, levels, baud_rate=baud_rate) + local, harp = _drop_inconsistent(local, harp, max_drift) + return _anchor_frame(pd.Index(local, name="LocalTime", dtype=np.float64), harp) + + +def decode_timecode_from_samples( + samples: ArrayLike, + sample_rate: float, + *, + threshold: float | None = None, + baud_rate: float = DEFAULT_BAUD_RATE, + max_drift: float | None = 0.1, +) -> pd.DataFrame: + """Decode Harp seconds from a uniformly sampled recording of the time code line. + + ``samples`` is the recorded waveform and ``sample_rate`` its sampling frequency in Hz; + sample ``i`` is taken to occur at local time ``i / sample_rate``. Boolean and integer + input is treated as the digital line state (nonzero is high); pass ``threshold`` to + binarize an analog recording as ``samples >= threshold``. + + The returned DataFrame maps the first sample of each decoded packet's start bit (the + index, named ``"Sample"``) to the whole Harp second that starts there (``"Time"``). + See :func:`decode_timecode_from_transitions` for ``baud_rate`` and ``max_drift``. + About five samples per bit are needed for reliable decoding, so a 1 kbps signal wants + a sample rate of at least ~5 kHz. + """ + if sample_rate <= 0: + raise ValueError(f"sample_rate must be positive, got {sample_rate}.") + if baud_rate <= 0: + raise ValueError(f"baud_rate must be positive, got {baud_rate}.") + if sample_rate < 2 * baud_rate: + raise ValueError( + f"A {baud_rate} bps signal cannot be decoded from samples taken at " + f"{sample_rate} Hz; at least 2 samples per bit are required (5 recommended)." + ) + + level = _as_levels(samples, threshold) + edges = np.flatnonzero(np.diff(level)) + 1 + times = edges / sample_rate + local, harp = _decode(times, level[edges], baud_rate=baud_rate) + local, harp = _drop_inconsistent(local, harp, max_drift) + sample = np.rint(local * sample_rate).astype(np.int64) + return _anchor_frame(pd.Index(sample, name="Sample"), harp) + + +def _as_transitions( + timestamps: ArrayLike, states: ArrayLike +) -> tuple[NDArray[np.float64], NDArray[np.bool_]]: + """Validate a transition list into sorted times and boolean levels.""" + times = np.asarray(timestamps, dtype=np.float64) + levels = np.asarray(states) + if times.ndim != 1 or levels.ndim != 1: + raise ValueError("timestamps and states must be one-dimensional.") + if times.size != levels.size: + raise ValueError( + f"timestamps and states must have the same length, got {times.size} and {levels.size}." + ) + if np.any(np.diff(times) < 0): + raise ValueError("timestamps must be sorted in ascending order.") + return times, levels.astype(np.bool_) + + +def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_]: + """Binarize a sampled waveform into digital line levels.""" + values = np.asarray(samples) + if values.ndim != 1: + raise ValueError("samples must be one-dimensional.") + if threshold is not None: + return values >= threshold + if values.dtype == np.bool_ or np.issubdtype(values.dtype, np.integer): + return values != 0 + raise ValueError( + f"Pass a threshold to binarize samples of dtype {values.dtype}; only boolean " + "and integer samples are taken as digital line states." + ) + + +def _level_at( + times: NDArray[np.float64], levels: NDArray[np.bool_], at: NDArray[np.floating[Any]] +) -> NDArray[np.bool_]: + """The line level at each time in ``at``, from the transitions ``(times, levels)``.""" + index = np.searchsorted(times, at, side="right") - 1 + # transitions alternate, so before the first one the line held the opposite level + return np.where(index < 0, ~levels[0], levels[np.maximum(index, 0)]) + + +def _resync(falling: NDArray[np.float64], expected: float, bit: float) -> float: + """Snap a predicted start-bit time to a nearby falling edge, as a UART would.""" + best, window = expected, _RESYNC_WINDOW * bit + index = int(np.searchsorted(falling, expected)) + for candidate in (index - 1, index): + if 0 <= candidate < falling.size: + distance = abs(falling[candidate] - expected) + if distance <= window: + best, window = float(falling[candidate]), distance + return best + + +def _decode_packet( + times: NDArray[np.float64], + levels: NDArray[np.bool_], + falling: NDArray[np.float64], + origin: float, + bit: float, +) -> tuple[int, float] | None: + """Decode one time code packet whose first start bit falls at ``origin``. + + Returns the whole second it carries and the time its last stop bit ends, or ``None`` + if any of its frames is not delimited by a low start bit and a high stop bit. + """ + payload = bytearray(_PACKET_BYTES) + offsets = (np.arange(_BITS_PER_BYTE) + 0.5) * bit + start = origin + for frame in range(_PACKET_BYTES): + if frame: + # predict from the previous start bit, not from the packet, so that a small + # baud rate error stays a fraction of a bit instead of accumulating + start = _resync(falling, start + _BITS_PER_BYTE * bit, bit) + bits = _level_at(times, levels, start + offsets) + if bits[0] or not bits[-1]: # start bit must be low, stop bit high + return None + payload[frame] = np.packbits(bits[1:-1], bitorder="little")[0] + return int.from_bytes(payload, "little"), start + _BITS_PER_BYTE * bit + + +def _decode( + times: NDArray[np.float64], + levels: NDArray[np.bool_], + *, + baud_rate: float, +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + """Decode every packet in a transition list into (first falling edge, Harp second).""" + if baud_rate <= 0: + raise ValueError(f"baud_rate must be positive, got {baud_rate}.") + + bit = 1.0 / baud_rate + falling = times[~levels] # a packet can only start on a falling edge + local: list[float] = [] + harp: list[float] = [] + decoded_until = -np.inf + for origin in falling: + if origin < decoded_until: + continue # inside a packet we already decoded + packet = _decode_packet(times, levels, falling, float(origin), bit) + if packet is None: + continue + seconds, decoded_until = packet + local.append(float(origin)) + harp.append(float(seconds)) + return np.array(local, dtype=np.float64), np.array(harp, dtype=np.float64) + + +def _drop_inconsistent( + local: NDArray[np.float64], harp: NDArray[np.float64], max_drift: float | None +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + """Drop anchors whose Harp seconds disagree with the elapsed local time.""" + if max_drift is None or local.size < 2: + return local, harp + if max_drift < 0: + raise ValueError(f"max_drift must be non-negative, got {max_drift}.") + + def agrees(earlier: int, later: int) -> bool: + """Whether two anchors are one plausible stretch of the same clock apart.""" + elapsed_harp = harp[later] - harp[earlier] + elapsed_local = local[later] - local[earlier] + return bool( + elapsed_harp >= 1 + and abs(elapsed_harp - elapsed_local) <= max_drift * abs(elapsed_local) + ) + + # grow the longest chain of mutually consistent anchors out of the first agreeing + # pair, so that one corrupt value costs one anchor instead of every anchor after it + keep = np.zeros(local.size, dtype=np.bool_) + seed = next((i for i in range(local.size - 1) if agrees(i, i + 1)), None) + if seed is not None: + keep[seed] = keep[seed + 1] = True + last = seed + 1 + for candidate in range(seed + 2, local.size): + if agrees(last, candidate): + keep[candidate] = True + last = candidate + first = seed + for candidate in range(seed - 1, -1, -1): + if agrees(candidate, first): + keep[candidate] = True + first = candidate + + dropped = int(np.count_nonzero(~keep)) + if dropped: + warnings.warn( + f"Dropped {dropped} time code packet(s) whose Harp time is inconsistent with " + "the local clock. Pass max_drift=None to keep them.", + stacklevel=3, + ) + return local[keep], harp[keep] + + +def _anchor_frame(anchors: pd.Index, harp: NDArray[np.float64]) -> pd.DataFrame: + """Assemble the decoded packets into an ``anchors`` → Harp time table. + + ``anchors`` locate each packet on the local device's own axis, whichever one it + reports timestamps on. "Time" is the Harp time axis throughout ``harp.data``; here + it is the column, because the index is the axis being translated from. + """ + return pd.DataFrame({"Time": harp}, index=anchors) diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py new file mode 100644 index 0000000..6be8d0e --- /dev/null +++ b/tests/data/test_timecode.py @@ -0,0 +1,373 @@ +from contextlib import nullcontext + +import numpy as np +import pytest +from harp.data.timecode import decode_timecode_from_samples, decode_timecode_from_transitions + +BAUD_RATE = 1000.0 + + +def _packet_bits(value): + """The 8N1 bit stream of one time code packet: start, 8 data bits (LSB first), stop.""" + bits = [] + for byte in int(value).to_bytes(4, "little"): + bits.append(0) + bits.extend((byte >> position) & 1 for position in range(8)) + bits.append(1) + return bits + + +def transitions( + values, + *, + baud_rate=BAUD_RATE, + emitted_baud_rate=None, + period=1.0, + first=0.5, + jitter=0.0, + seed=0, +): + """An idle-high time code line carrying one packet per ``period``. + + Returns the transition times, the level taken at each of them, and the local time + of the first falling edge of every packet. ``emitted_baud_rate`` sets the rate the + signal is actually generated at, which may differ from the nominal ``baud_rate``. + """ + bit = 1.0 / (emitted_baud_rate or baud_rate) + rng = np.random.default_rng(seed) + times, states, starts = [], [], [] + level = 1 + for packet, value in enumerate(values): + start = first + packet * period + starts.append(start) + for position, expected in enumerate(_packet_bits(value)): + if expected != level: + offset = rng.uniform(-jitter, jitter) * bit if jitter else 0.0 + times.append(start + position * bit + offset) + states.append(expected) + level = expected + return np.array(times), np.array(states), np.array(starts) + + +def render(times, states, sample_rate, *, duration=None): + """Sample an idle-high transition list onto a uniform grid of digital states.""" + duration = duration if duration is not None else times[-1] + 0.5 + grid = np.arange(int(round(duration * sample_rate))) / sample_rate + index = np.searchsorted(times, grid, side="right") - 1 + return np.where(index < 0, ~states[0].astype(bool), states[np.maximum(index, 0)]).astype(bool) + + +def packet_duration(*, baud_rate=BAUD_RATE): + return len(_packet_bits(0)) / baud_rate + + +# ---------------------------------------------------------------- decoding + + +def test_decodes_a_single_packet_on_its_first_falling_edge(): + times, states, starts = transitions([3806874]) + clock = decode_timecode_from_transitions(times, states) + + assert clock.index.name == "LocalTime" + assert clock.columns.tolist() == ["Time"] + assert clock["Time"].tolist() == [3806874] + assert clock.index.to_numpy() == pytest.approx(starts) + + +def test_decodes_consecutive_seconds(): + values = [3806874, 3806875, 3806876, 3806877] + times, states, starts = transitions(values) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) + + +@pytest.mark.parametrize("value", [0, 1, 0xFF, 0xFF00, 0xFF00FF00, 0xFFFFFFFE, 3806874]) +def test_round_trips_every_byte_pattern(value): + """Bytes of all zeros or all ones leave long runs without a transition.""" + times, states, _ = transitions([value, value + 1]) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == [value, value + 1] + + +@pytest.mark.parametrize("sample_rate", [5_000.0, 30_000.0]) +def test_sampled_recording_matches_the_transition_list(sample_rate): + values = [3806874, 3806875, 3806876] + times, states, starts = transitions(values) + samples = render(times, states, sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock["Time"].tolist() == values + assert clock.index.name == "Sample" + assert clock.index.dtype == np.int64 + assert clock.index.tolist() == [round(anchor * sample_rate) for anchor in starts] + + +def test_sample_index_points_at_the_anchor_sample(): + sample_rate = 10_000.0 + times, states, starts = transitions([42, 43], first=0.5) + samples = render(times, states, sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock.columns.tolist() == ["Time"] + # the anchor sample is the first sample of the packet's start bit + for sample, start in zip(clock.index, starts): + assert sample == round(start * sample_rate) + assert not samples[sample] + assert samples[sample - 1] + + +def test_analog_recording_is_binarized_by_threshold(): + sample_rate = 10_000.0 + values = [7, 8] + times, states, _ = transitions(values) + analog = np.where(render(times, states, sample_rate), 3.3, 0.1) + 0.02 + + clock = decode_timecode_from_samples(analog, sample_rate, threshold=1.5) + + assert clock["Time"].tolist() == values + + +def test_float_samples_without_threshold_raises(): + with pytest.raises(ValueError, match="threshold"): + decode_timecode_from_samples(np.zeros(100, dtype=np.float64), 10_000.0) + + +def test_sample_rate_below_two_samples_per_bit_raises(): + with pytest.raises(ValueError, match="cannot be decoded"): + decode_timecode_from_samples(np.zeros(100, dtype=bool), 1500.0, baud_rate=BAUD_RATE) + + +def test_recording_without_any_transition_yields_no_anchors(): + clock = decode_timecode_from_samples(np.ones(10_000, dtype=bool), 10_000.0) + + assert clock.empty + assert clock.columns.tolist() == ["Time"] + assert clock.index.name == "Sample" + + +def test_empty_transition_list_yields_no_anchors(): + clock = decode_timecode_from_transitions([], []) + + assert clock.empty + assert clock.columns.tolist() == ["Time"] + assert clock["Time"].dtype == np.float64 + + +def test_wrong_baud_rate_yields_no_anchors(): + times, states, _ = transitions([100, 101, 102]) + + clock = decode_timecode_from_transitions(times, states, baud_rate=9600.0) + + assert clock.empty + assert clock.index.name == "LocalTime" + + +# ---------------------------------------------------------------- robustness + + +def test_glitches_between_packets_are_ignored(): + values = [500, 501] + times, states, starts = transitions(values) + # a 100 us spike in the idle line, well away from any packet + times = np.concatenate([times, [1.4, 1.4001]]) + states = np.concatenate([states, [0, 1]]) + order = np.argsort(times) + + clock = decode_timecode_from_transitions(times[order], states[order]) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) + + +def test_packet_truncated_by_the_end_of_the_recording_is_dropped(): + values = [500, 501, 502] + times, states, starts = transitions(values) + keep = times < starts[-1] + 0.5 * packet_duration() + + clock = decode_timecode_from_transitions(times[keep], states[keep]) + + assert clock["Time"].tolist() == values[:-1] + + +def test_recording_starting_mid_packet_keeps_the_later_packets(): + values = [500, 501, 502] + times, states, starts = transitions(values) + keep = times > starts[0] + 0.5 * packet_duration() + + clock = decode_timecode_from_transitions(times[keep], states[keep]) + + assert clock["Time"].tolist() == values[1:] + + +def test_tolerates_jitter_on_every_edge(): + values = [3806874, 3806875, 3806876] + times, states, starts = transitions(values, jitter=0.2) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts, abs=0.3 / BAUD_RATE) + + +def test_tolerates_a_baud_rate_error_that_would_slip_a_whole_bit(): + """Per-frame re-synchronization keeps a 3% rate error from walking off the bits.""" + values = [3806874, 3806875] + times, states, _ = transitions(values, emitted_baud_rate=BAUD_RATE * 1.03) + + clock = decode_timecode_from_transitions(times, states, baud_rate=BAUD_RATE) + + assert clock["Time"].tolist() == values + + +def test_corrupt_packet_is_dropped_with_a_warning(): + values = [3806874, 3806875, 12345, 3806877] + times, states, starts = transitions(values) + + with pytest.warns(UserWarning, match="inconsistent with the local clock"): + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == [3806874, 3806875, 3806877] + assert clock.index.to_numpy() == pytest.approx(np.delete(starts, 2)) + + +def test_corrupt_first_packet_is_dropped_with_a_warning(): + values = [12345, 3806875, 3806876, 3806877] + times, states, _ = transitions(values) + + with pytest.warns(UserWarning): + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values[1:] + + +def test_max_drift_none_keeps_every_decoded_packet(): + values = [3806874, 3806875, 12345, 3806877] + times, states, _ = transitions(values) + + clock = decode_timecode_from_transitions(times, states, max_drift=None) + + assert clock["Time"].tolist() == values + + +def test_missing_packets_do_not_break_the_chain(): + values = [3806874, 3806875, 3806876, 3806877] + times, states, starts = transitions(values) + dropped = (times < starts[1]) | (times >= starts[2]) + + clock = decode_timecode_from_transitions(times[dropped], states[dropped]) + + assert clock["Time"].tolist() == [3806874, 3806876, 3806877] + + +def test_invalid_transition_lists_raise(): + times, states, _ = transitions([1, 2]) + with pytest.raises(ValueError, match="same length"): + decode_timecode_from_transitions(times, states[:-1]) + with pytest.raises(ValueError, match="sorted"): + decode_timecode_from_transitions(times[::-1], states) + with pytest.raises(ValueError, match="one-dimensional"): + decode_timecode_from_transitions(np.zeros((2, 2)), np.zeros((2, 2))) + with pytest.raises(ValueError, match="baud_rate"): + decode_timecode_from_transitions(times, states, baud_rate=0.0) + + +def test_a_long_drifting_sampled_recording_decodes_end_to_end(): + """A minute of a 30 kHz recording: local clock drift, jitter and one lost packet.""" + rate = 1.001 # Harp seconds per local second + sample_rate = 30_000.0 + values = [3806874 + second for second in range(60)] + times, states, starts = transitions( + values, period=1.0 / rate, baud_rate=BAUD_RATE * rate, jitter=0.15 + ) + lost = (times >= starts[30]) & (times < starts[30] + 1.5 * packet_duration()) + samples = render(times[~lost], states[~lost], sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate, baud_rate=BAUD_RATE * rate) + + assert clock["Time"].tolist() == [value for value in values if value != values[30]] + # anchors land on the emitted start bits, within the jitter and one sample period + assert clock.index.to_numpy() / sample_rate == pytest.approx(np.delete(starts, 30), abs=5e-4) + + +# ---------------------------------------------------------------- recorded signals + +# Transitions acquired by an Open Ephys system from a Behavior board emitting the +# time code at 1 kbps, taken from https://github.com/harp-tech/harp-python/pull/38. +# fmt: off +RECORDED = [ + { + # two valid packets, followed by one cut short by the end of the recording + "timestamps": np.array([ + 0. , 0.96106667, 0.96306667, 0.96406667, 0.96506667, + 0.96706667, 0.96906667, 0.97106667, 0.97306667, 0.97506667, + 0.97606667, 0.97706667, 0.98006667, 0.98106667, 0.98306667, + 0.98406667, 0.98506667, 0.98806667, 0.99006667, 0.99106667, + 1.00006667, 1.96116667, 1.96216667, 1.96416667, 1.96516667, + 1.96716667, 1.96916667, 1.97116667, 1.97316667, 1.97516667, + 1.97616667, 1.97716667, 1.98016667, 1.98116667, 1.98316667, + 1.98416667, 1.98516667, 1.98816667, 1.99016667, 1.99116667, + 2.00016667, 2.96126667, 2.96426667, 2.96726667, 2.96926667, + 2.97126667, 2.97326667, 2.97526667, 2.97626667, 2.97726667, + 2.98026667, 2.98126667, 2.98326667, 2.98426667, 2.98526667, + 2.98826667, 2.99026667, 2.99126667]), + "expected_start_times": np.array([0.96106667, 1.96116667]), + "expected_harp_times": [3806874, 3806875], + # the cut-short packet fails its framing check, so nothing reaches the drift check + "expected_warning": False, + }, + { + # four valid packets and one corrupt one + "timestamps": np.array([ + 0.14036667, 1.10146667, 1.10246667, 1.10346667, 1.10446667, + 1.11146667, 1.11246667, 1.11646667, 1.11746667, 1.11846667, + 1.11946667, 1.12146667, 1.12246667, 1.12546667, 1.12746667, + 1.12846667, 1.13046667, 1.13146667, 1.14046667, 2.10156667, + 2.10356667, 2.11156667, 2.11256667, 2.11656667, 2.11756667, + 2.11856667, 2.11956667, 2.12156667, 2.12256667, 2.12556667, + 2.12756667, 2.12856667, 2.13056667, 2.13156667, 2.14056667, + 3.10163333, 3.10263333, 3.11163333, 3.11263333, 3.11663333, + 3.11763333, 3.11863333, 3.11963333, 3.12163333, 3.12263333, + 3.12563333, 3.12763333, 3.12863333, 3.13063333, 3.13163333, + 3.14063333, 4.10173333, 4.11073333, 4.11173333, 4.11673333, + 4.11873333, 4.11973333, 4.12173333, 4.12273333, 4.12573333, + 4.12773333, 4.12873333, 4.13073333, 4.13173333, 4.14073333, + 5.1018 , 5.1028 , 5.1038 , 5.1108 , 5.1118 , + 5.1168 , 5.1188 , 5.1198 , 5.1218 , 5.1228 , + 5.1258 , 5.1278 , 5.1288 , 5.1308 , 5.1318 , + 5.1368 , 5.1388 , 5.1398 , 5.14183333, 5.1428 , + 5.14583333, 5.14783333, 5.14883333, 5.15083333, 5.15183333, + 5.16083333, 6.1059 ]), + "expected_start_times": np.array([1.10146667, 2.10156667, 3.10163333, 4.10173333]), + "expected_harp_times": [2600957, 2600958, 2600959, 2600960], + # the corrupt packet decodes cleanly but its seconds do not add up + "expected_warning": True, + }, +] +# fmt: on + + +def _decode_recorded(recorded, **kwargs): + times = recorded["timestamps"] + states = np.resize([1, 0], times.size) # transitions alternate, starting from high + expected = ( + pytest.warns(UserWarning, match="inconsistent with the local clock") + if recorded["expected_warning"] + else nullcontext() + ) + with expected: + return decode_timecode_from_transitions(times, states, **kwargs) + + +@pytest.mark.parametrize("recorded", RECORDED) +def test_decodes_a_recorded_time_code(recorded): + clock = _decode_recorded(recorded) + + assert clock["Time"].tolist() == recorded["expected_harp_times"] + assert clock.index.to_numpy() == pytest.approx(recorded["expected_start_times"]) From c276da22bfdcc4dc8567cf5ee9cdbd5e6ce6eff6 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:52:24 -0800 Subject: [PATCH 2/7] Trim time code docstrings --- .../src/harp/data/timecode/__init__.py | 2 - .../src/harp/data/timecode/_decode.py | 98 +++++-------------- 2 files changed, 24 insertions(+), 76 deletions(-) diff --git a/src/packages/harp-data/src/harp/data/timecode/__init__.py b/src/packages/harp-data/src/harp/data/timecode/__init__.py index 7be859a..8e4e83d 100644 --- a/src/packages/harp-data/src/harp/data/timecode/__init__.py +++ b/src/packages/harp-data/src/harp/data/timecode/__init__.py @@ -1,5 +1,3 @@ -"""Aligning a non-Harp device's local timestamps to Harp time with the Harp Time Code.""" - from ._decode import ( DEFAULT_BAUD_RATE, decode_timecode_from_samples, diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index 225def2..609ff5e 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -1,26 +1,3 @@ -"""Decode a recorded Harp Time Code back into Harp seconds. - -The [Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) is a serial signal -carrying the current Harp time in whole seconds, emitted on a digital output at a low baud -rate (typically 1 kbps) so that non-Harp acquisition systems can record it alongside data -timestamped on their own clock. The functions here decode such a recording back into Harp -seconds, paired with the local time each of them was received at. - -The wire format is 8N1 serial: the line idles high and every byte is sent as one low start -bit, eight data bits (least significant first) and one high stop bit. Each packet is four -back-to-back bytes holding the whole Harp second as a little-endian ``uint32``, with no -header or checksum, and its first falling edge marks the start of that second. - -Two ways in, depending on what the recording system gives you: - -- a uniformly sampled waveform (digital or analog) → :func:`decode_timecode_from_samples` -- a list of line transitions ``(time, 0/1)`` → :func:`decode_timecode_from_transitions` - -Each returns a table of anchors keyed on the axis that system reports its own timestamps -on (a sample number or local seconds) against the whole Harp second that starts there. -How those timestamps are then interpolated onto the Harp axis is left to the caller. -""" - import warnings from typing import Any @@ -28,18 +5,12 @@ import pandas as pd from numpy.typing import ArrayLike, NDArray -#: Baud rate used by existing Harp Time Code emitters, in bits per second. +#: Default Harp Time Code baud rate, in bits per second. DEFAULT_BAUD_RATE = 1000.0 -#: Bits per byte on the wire: one start bit, eight data bits, one stop bit. -_BITS_PER_BYTE = 10 - -#: Bytes in a Harp Time Code packet, holding the whole second as a ``U32``. -_PACKET_BYTES = 4 - -#: How far, as a fraction of a bit period, a start bit may sit from its predicted -#: position and still be used to re-synchronize the frame that follows it. -_RESYNC_WINDOW = 0.3 +_BITS_PER_BYTE = 10 # 8N1: start bit, eight data bits, stop bit +_PACKET_BYTES = 4 # whole second as a little-endian U32 +_RESYNC_WINDOW = 0.3 # max start-bit offset, in bits, to re-synchronize a frame on def decode_timecode_from_transitions( @@ -49,22 +20,16 @@ def decode_timecode_from_transitions( baud_rate: float = DEFAULT_BAUD_RATE, max_drift: float | None = 0.1, ) -> pd.DataFrame: - """Decode Harp seconds from a list of time code line transitions. + """Decode a Harp Time Code from line transitions. - ``timestamps`` are the local times, in seconds, of each transition of the time code - line and ``states`` the level the line took at each of them (0 low, 1 high); this is - the shape most event-based acquisition systems report. The returned DataFrame maps the - local time of each decoded packet's first falling edge (the index, named - ``"LocalTime"``) to the whole Harp second that starts there (``"Time"``, the name the - readers also give the Harp time axis). + See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). + ``timestamps`` are the local times, in seconds, of each transition and ``states`` the + level the line took (0 low, 1 high). Returns the whole Harp second of each packet + (``"Time"``) indexed by the local time of its first falling edge (``"LocalTime"``). - ``baud_rate`` must match the emitter; bits are read at the center of each bit period - and every frame re-synchronizes on its own start bit, so a mismatch of a few percent - is tolerated. Packets whose start/stop bits do not check out are dropped silently, so - a partially recorded or glitched packet costs one anchor, not the decoding of the ones - around it. ``max_drift`` additionally drops packets whose seconds do not advance - consistently with the local clock, allowing for at most that fractional rate mismatch - between the two (pass ``None`` to keep everything). + Packets with bad framing are dropped. ``max_drift`` also drops packets whose seconds + disagree with the elapsed local time by more than that fraction, with a warning; + pass ``None`` to keep them. """ times, levels = _as_transitions(timestamps, states) local, harp = _decode(times, levels, baud_rate=baud_rate) @@ -80,18 +45,13 @@ def decode_timecode_from_samples( baud_rate: float = DEFAULT_BAUD_RATE, max_drift: float | None = 0.1, ) -> pd.DataFrame: - """Decode Harp seconds from a uniformly sampled recording of the time code line. - - ``samples`` is the recorded waveform and ``sample_rate`` its sampling frequency in Hz; - sample ``i`` is taken to occur at local time ``i / sample_rate``. Boolean and integer - input is treated as the digital line state (nonzero is high); pass ``threshold`` to - binarize an analog recording as ``samples >= threshold``. + """Decode a Harp Time Code from a sampled line. - The returned DataFrame maps the first sample of each decoded packet's start bit (the - index, named ``"Sample"``) to the whole Harp second that starts there (``"Time"``). - See :func:`decode_timecode_from_transitions` for ``baud_rate`` and ``max_drift``. - About five samples per bit are needed for reliable decoding, so a 1 kbps signal wants - a sample rate of at least ~5 kHz. + See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). + Sample ``i`` is taken at local time ``i / sample_rate``. Boolean and integer samples + are digital states; pass ``threshold`` to binarize analog ones. Returns the whole Harp + second of each packet (``"Time"``) indexed by the sample of its first falling edge + (``"Sample"``). See :func:`decode_timecode_from_transitions` for ``max_drift``. """ if sample_rate <= 0: raise ValueError(f"sample_rate must be positive, got {sample_rate}.") @@ -115,7 +75,7 @@ def decode_timecode_from_samples( def _as_transitions( timestamps: ArrayLike, states: ArrayLike ) -> tuple[NDArray[np.float64], NDArray[np.bool_]]: - """Validate a transition list into sorted times and boolean levels.""" + """Validate transitions into sorted times and boolean levels.""" times = np.asarray(timestamps, dtype=np.float64) levels = np.asarray(states) if times.ndim != 1 or levels.ndim != 1: @@ -147,14 +107,14 @@ def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_] def _level_at( times: NDArray[np.float64], levels: NDArray[np.bool_], at: NDArray[np.floating[Any]] ) -> NDArray[np.bool_]: - """The line level at each time in ``at``, from the transitions ``(times, levels)``.""" + """Line level at each time in ``at``.""" index = np.searchsorted(times, at, side="right") - 1 # transitions alternate, so before the first one the line held the opposite level return np.where(index < 0, ~levels[0], levels[np.maximum(index, 0)]) def _resync(falling: NDArray[np.float64], expected: float, bit: float) -> float: - """Snap a predicted start-bit time to a nearby falling edge, as a UART would.""" + """Snap a predicted start-bit time to a nearby falling edge.""" best, window = expected, _RESYNC_WINDOW * bit index = int(np.searchsorted(falling, expected)) for candidate in (index - 1, index): @@ -172,11 +132,7 @@ def _decode_packet( origin: float, bit: float, ) -> tuple[int, float] | None: - """Decode one time code packet whose first start bit falls at ``origin``. - - Returns the whole second it carries and the time its last stop bit ends, or ``None`` - if any of its frames is not delimited by a low start bit and a high stop bit. - """ + """Decode the packet starting at ``origin`` into (seconds, end time), or ``None``.""" payload = bytearray(_PACKET_BYTES) offsets = (np.arange(_BITS_PER_BYTE) + 0.5) * bit start = origin @@ -198,7 +154,7 @@ def _decode( *, baud_rate: float, ) -> tuple[NDArray[np.float64], NDArray[np.float64]]: - """Decode every packet in a transition list into (first falling edge, Harp second).""" + """Decode every packet into (first falling edge, Harp second) arrays.""" if baud_rate <= 0: raise ValueError(f"baud_rate must be positive, got {baud_rate}.") @@ -229,7 +185,6 @@ def _drop_inconsistent( raise ValueError(f"max_drift must be non-negative, got {max_drift}.") def agrees(earlier: int, later: int) -> bool: - """Whether two anchors are one plausible stretch of the same clock apart.""" elapsed_harp = harp[later] - harp[earlier] elapsed_local = local[later] - local[earlier] return bool( @@ -265,10 +220,5 @@ def agrees(earlier: int, later: int) -> bool: def _anchor_frame(anchors: pd.Index, harp: NDArray[np.float64]) -> pd.DataFrame: - """Assemble the decoded packets into an ``anchors`` → Harp time table. - - ``anchors`` locate each packet on the local device's own axis, whichever one it - reports timestamps on. "Time" is the Harp time axis throughout ``harp.data``; here - it is the column, because the index is the axis being translated from. - """ + """Build the anchor → Harp time table.""" return pd.DataFrame({"Time": harp}, index=anchors) From 57d2b5344bc8c278e44a9d6bbc5e5274c731ab25 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:54:24 -0800 Subject: [PATCH 3/7] Detect the threshold of sampled time code recordings --- docs/examples/time_code.py | 2 +- .../src/harp/data/timecode/_decode.py | 37 ++++++++++---- tests/data/test_timecode.py | 48 +++++++++++++++++-- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/docs/examples/time_code.py b/docs/examples/time_code.py index 5b54cea..a6ff611 100644 --- a/docs/examples/time_code.py +++ b/docs/examples/time_code.py @@ -3,7 +3,7 @@ # A non-Harp acquisition system recorded the Harp Time Code on one of its digital # lines. `samples` is that line, sampled at the system's own rate. -samples = np.load("timecode_line.npy") # digital states; analog input needs a `threshold` +samples = np.load("timecode_line.npy") # digital or analog; the threshold is detected sample_rate = 30_000.0 # Decode it: one row per whole Harp second, keyed on the sample where that second diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index 609ff5e..d6f0b79 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -48,8 +48,9 @@ def decode_timecode_from_samples( """Decode a Harp Time Code from a sampled line. See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). - Sample ``i`` is taken at local time ``i / sample_rate``. Boolean and integer samples - are digital states; pass ``threshold`` to binarize analog ones. Returns the whole Harp + Sample ``i`` is taken at local time ``i / sample_rate``. Samples are binarized as + ``samples >= threshold``; by default the threshold is detected from the data as the + midpoint between the high and low levels. Returns the whole Harp second of each packet (``"Time"``) indexed by the sample of its first falling edge (``"Sample"``). See :func:`decode_timecode_from_transitions` for ``max_drift``. """ @@ -94,14 +95,30 @@ def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_] values = np.asarray(samples) if values.ndim != 1: raise ValueError("samples must be one-dimensional.") - if threshold is not None: - return values >= threshold - if values.dtype == np.bool_ or np.issubdtype(values.dtype, np.integer): - return values != 0 - raise ValueError( - f"Pass a threshold to binarize samples of dtype {values.dtype}; only boolean " - "and integer samples are taken as digital line states." - ) + if threshold is None: + if values.dtype == np.bool_: + return values + threshold = _detect_threshold(values) + return values >= threshold + + +def _detect_threshold(values: NDArray[Any]) -> float: + """Find the midpoint between the low and high levels by iterative (isodata) clustering.""" + values = values.astype(np.float64, copy=False) + if values.size == 0: + return 0.0 + # start from extreme percentiles rather than min/max, so rare glitches cannot seed a cluster + low, high = (float(level) for level in np.percentile(values, [0.01, 99.99])) + threshold = (low + high) / 2 + if low == high: + return threshold + for _ in range(100): + below = values < threshold + updated = (values[below].mean() + values[~below].mean()) / 2 + if abs(updated - threshold) <= 1e-9 * (high - low): + break + threshold = updated + return float(threshold) def _level_at( diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py index 6be8d0e..3d43685 100644 --- a/tests/data/test_timecode.py +++ b/tests/data/test_timecode.py @@ -134,9 +134,51 @@ def test_analog_recording_is_binarized_by_threshold(): assert clock["Time"].tolist() == values -def test_float_samples_without_threshold_raises(): - with pytest.raises(ValueError, match="threshold"): - decode_timecode_from_samples(np.zeros(100, dtype=np.float64), 10_000.0) +@pytest.mark.parametrize("value", [3806874, 0xFFFFFFF0]) +def test_analog_threshold_is_detected_from_a_noisy_mostly_idle_line(value): + """Bytes of all ones leave the line low only for start bits, about 0.5% of the time.""" + sample_rate = 10_000.0 + values = [value + second for second in range(5)] + times, states, _ = transitions(values) + level = render(times, states, sample_rate) + noise = np.random.default_rng(0).normal(0.0, 0.15, level.size) + analog = np.where(level, 3.3, 0.1) - 1.0 + noise + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + +@pytest.mark.parametrize( + "high, low", [(1, 0), (8, 0), (12_000, -3_000)], ids=["digital", "ttl-word", "int16-analog"] +) +def test_integer_threshold_is_detected_from_the_levels(high, low): + sample_rate = 10_000.0 + values = [3806874, 3806875] + times, states, _ = transitions(values) + samples = np.where(render(times, states, sample_rate), high, low).astype(np.int16) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock["Time"].tolist() == values + + +def test_detected_threshold_ignores_rare_glitches(): + sample_rate = 10_000.0 + values = [3806874 + second for second in range(5)] + times, states, _ = transitions(values) + analog = np.where(render(times, states, sample_rate), 3.3, 0.1) + analog[[100, 20_000, 40_000]] = [100.0, -100.0, 100.0] + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + +def test_constant_recording_with_detected_threshold_yields_no_anchors(): + clock = decode_timecode_from_samples(np.full(10_000, 2.5), 10_000.0) + + assert clock.empty def test_sample_rate_below_two_samples_per_bit_raises(): From 2bddf781407e8a05faa9b7d2f0fd1d799bf77bbc Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 13:58:54 -0800 Subject: [PATCH 4/7] Detect the time code threshold from a subsample --- .../src/harp/data/timecode/_decode.py | 8 +++++++- tests/data/test_timecode.py | 20 ++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index d6f0b79..9a009cc 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -11,6 +11,7 @@ _BITS_PER_BYTE = 10 # 8N1: start bit, eight data bits, stop bit _PACKET_BYTES = 4 # whole second as a little-endian U32 _RESYNC_WINDOW = 0.3 # max start-bit offset, in bits, to re-synchronize a frame on +_THRESHOLD_SUBSAMPLE = 1_000_000 # samples used to detect the threshold def decode_timecode_from_transitions( @@ -104,9 +105,14 @@ def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_] def _detect_threshold(values: NDArray[Any]) -> float: """Find the midpoint between the low and high levels by iterative (isodata) clustering.""" - values = values.astype(np.float64, copy=False) if values.size == 0: return 0.0 + if values.size > _THRESHOLD_SUBSAMPLE: + # a fixed-seed random subsample is plenty to place the cut, and keeps a long + # recording from being copied; a strided one could alias with the bit period + picks = np.random.default_rng(0).integers(0, values.size, _THRESHOLD_SUBSAMPLE) + values = values[picks] + values = values.astype(np.float64, copy=False) # start from extreme percentiles rather than min/max, so rare glitches cannot seed a cluster low, high = (float(level) for level in np.percentile(values, [0.01, 99.99])) threshold = (low + high) / 2 diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py index 3d43685..bffc852 100644 --- a/tests/data/test_timecode.py +++ b/tests/data/test_timecode.py @@ -2,7 +2,11 @@ import numpy as np import pytest -from harp.data.timecode import decode_timecode_from_samples, decode_timecode_from_transitions +from harp.data.timecode import ( + _decode, + decode_timecode_from_samples, + decode_timecode_from_transitions, +) BAUD_RATE = 1000.0 @@ -175,6 +179,20 @@ def test_detected_threshold_ignores_rare_glitches(): assert clock["Time"].tolist() == values +def test_threshold_is_detected_from_a_subsample_of_long_recordings(monkeypatch): + monkeypatch.setattr(_decode, "_THRESHOLD_SUBSAMPLE", 5_000) + sample_rate = 10_000.0 + values = [0xFFFFFFF0 + second for second in range(5)] + times, states, _ = transitions(values) + level = render(times, states, sample_rate) + noise = np.random.default_rng(0).normal(0.0, 0.15, level.size) + analog = np.where(level, 3.3, 0.1) + noise + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + def test_constant_recording_with_detected_threshold_yields_no_anchors(): clock = decode_timecode_from_samples(np.full(10_000, 2.5), 10_000.0) From 5dcf99e5a88f185eb8a4e4234464ac3181afc3cc Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:21:05 -0800 Subject: [PATCH 5/7] Return every framed time code packet and fit the anchors in the docs --- docs/examples/time-code.md | 2 +- docs/examples/time_code.py | 15 +++-- src/packages/harp-data/README.md | 12 ++-- .../src/harp/data/timecode/_decode.py | 60 ++----------------- tests/data/test_timecode.py | 60 +++++-------------- 5 files changed, 37 insertions(+), 112 deletions(-) diff --git a/docs/examples/time-code.md b/docs/examples/time-code.md index 76f45d2..3d3d9b3 100644 --- a/docs/examples/time-code.md +++ b/docs/examples/time-code.md @@ -5,7 +5,7 @@ Devices that are not part of the Harp bus run on their own clock, so their times Each packet starts exactly on the whole second it carries, so every decoded packet is an anchor: the sample (or local time) of its first falling edge, against the Harp second that begins there. !!! note - The decoded table is a set of anchors, local time → whole Harp second. How timestamps are placed between them is up to you: interpolating between neighbouring anchors absorbs the drift between the two clocks, whereas a global fit trades that away for noise rejection. + Every packet that frames cleanly is returned as decoded, including the rare one carrying a corrupt value, and a reset of the Harp clock mid-recording shows up as a jump in the anchors. The example below drops anchors that do not line up with the rest before fitting the offset and drift between the two clocks. ```python diff --git a/docs/examples/time_code.py b/docs/examples/time_code.py index a6ff611..c1bf941 100644 --- a/docs/examples/time_code.py +++ b/docs/examples/time_code.py @@ -15,14 +15,19 @@ # 36000 3806874.0 # 66003 3806875.0 -# Any of the system's timestamps (spikes, video frames, stimulus onsets) can now be -# placed on the Harp axis. Interpolating between neighbouring anchors absorbs the -# drift between the two clocks. +# Every cleanly framed packet is returned as decoded. Harp time minus local time stays +# nearly constant, so the rare corrupt value, or a reset of the Harp clock, stands out. +offset = anchors["Time"] - anchors.index / sample_rate +anchors = anchors[(offset - offset.median()).abs() < 1.0] + +# A line through the anchors gives the offset and drift between the two clocks, and +# extends past the first and last anchors. Any of the system's timestamps (spikes, +# video frames, stimulus onsets) can then be placed on the Harp axis. +slope, intercept = np.polyfit(anchors.index, anchors["Time"], 1) spike_samples = np.load("spike_samples.npy") -harp_times = np.interp(spike_samples, anchors.index, anchors["Time"]) +harp_times = intercept + slope * spike_samples # Event-based systems report line transitions instead of a sampled waveform: a local # time and the level the line took. Anchors then carry local seconds. transitions = np.load("line_transitions.npy") anchors = decode_timecode_from_transitions(transitions[:, 0], transitions[:, 1]) -harp_times = np.interp(spike_samples / sample_rate, anchors.index, anchors["Time"]) diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index ee6f195..8212c48 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -102,16 +102,18 @@ import numpy as np from harp.data import timecode anchors = timecode.decode_timecode_from_samples(line, sample_rate=30_000.0) # Sample -> Time -harp_times = np.interp(spike_samples, anchors.index, anchors["Time"]) +slope, intercept = np.polyfit(anchors.index, anchors["Time"], 1) +harp_times = intercept + slope * spike_samples # event-based systems report transitions instead, so anchors carry local seconds anchors = timecode.decode_timecode_from_transitions(edge_times, edge_states) # LocalTime -> Time -harp_times = np.interp(spike_times, anchors.index, anchors["Time"]) ``` -Packets that fail their start/stop bit check, or whose seconds do not add up -against the local clock, are dropped, so a glitched packet costs one anchor, not -the alignment around it. +Packets that fail their start/stop bit check are dropped. Every other packet is +returned as decoded, so a rare corrupt value or a reset of the Harp clock shows up +as an anchor that does not line up with the rest; the +[example](https://harp-tech.org/python/examples/time-code/) filters those out +before fitting. ## Write data back out diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index 9a009cc..df516b3 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -1,4 +1,3 @@ -import warnings from typing import Any import numpy as np @@ -19,7 +18,6 @@ def decode_timecode_from_transitions( states: ArrayLike, *, baud_rate: float = DEFAULT_BAUD_RATE, - max_drift: float | None = 0.1, ) -> pd.DataFrame: """Decode a Harp Time Code from line transitions. @@ -28,13 +26,11 @@ def decode_timecode_from_transitions( level the line took (0 low, 1 high). Returns the whole Harp second of each packet (``"Time"``) indexed by the local time of its first falling edge (``"LocalTime"``). - Packets with bad framing are dropped. ``max_drift`` also drops packets whose seconds - disagree with the elapsed local time by more than that fraction, with a warning; - pass ``None`` to keep them. + Packets with bad framing are dropped; every other packet is returned as decoded, so + a clock reset or a corrupt value shows up as an anchor that does not line up. """ times, levels = _as_transitions(timestamps, states) local, harp = _decode(times, levels, baud_rate=baud_rate) - local, harp = _drop_inconsistent(local, harp, max_drift) return _anchor_frame(pd.Index(local, name="LocalTime", dtype=np.float64), harp) @@ -44,16 +40,15 @@ def decode_timecode_from_samples( *, threshold: float | None = None, baud_rate: float = DEFAULT_BAUD_RATE, - max_drift: float | None = 0.1, ) -> pd.DataFrame: """Decode a Harp Time Code from a sampled line. See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). Sample ``i`` is taken at local time ``i / sample_rate``. Samples are binarized as ``samples >= threshold``; by default the threshold is detected from the data as the - midpoint between the high and low levels. Returns the whole Harp - second of each packet (``"Time"``) indexed by the sample of its first falling edge - (``"Sample"``). See :func:`decode_timecode_from_transitions` for ``max_drift``. + midpoint between the high and low levels. Returns the whole Harp second of each packet + (``"Time"``) indexed by the sample of its first falling edge (``"Sample"``), filtered + as in :func:`decode_timecode_from_transitions`. """ if sample_rate <= 0: raise ValueError(f"sample_rate must be positive, got {sample_rate}.") @@ -69,7 +64,6 @@ def decode_timecode_from_samples( edges = np.flatnonzero(np.diff(level)) + 1 times = edges / sample_rate local, harp = _decode(times, level[edges], baud_rate=baud_rate) - local, harp = _drop_inconsistent(local, harp, max_drift) sample = np.rint(local * sample_rate).astype(np.int64) return _anchor_frame(pd.Index(sample, name="Sample"), harp) @@ -198,50 +192,6 @@ def _decode( return np.array(local, dtype=np.float64), np.array(harp, dtype=np.float64) -def _drop_inconsistent( - local: NDArray[np.float64], harp: NDArray[np.float64], max_drift: float | None -) -> tuple[NDArray[np.float64], NDArray[np.float64]]: - """Drop anchors whose Harp seconds disagree with the elapsed local time.""" - if max_drift is None or local.size < 2: - return local, harp - if max_drift < 0: - raise ValueError(f"max_drift must be non-negative, got {max_drift}.") - - def agrees(earlier: int, later: int) -> bool: - elapsed_harp = harp[later] - harp[earlier] - elapsed_local = local[later] - local[earlier] - return bool( - elapsed_harp >= 1 - and abs(elapsed_harp - elapsed_local) <= max_drift * abs(elapsed_local) - ) - - # grow the longest chain of mutually consistent anchors out of the first agreeing - # pair, so that one corrupt value costs one anchor instead of every anchor after it - keep = np.zeros(local.size, dtype=np.bool_) - seed = next((i for i in range(local.size - 1) if agrees(i, i + 1)), None) - if seed is not None: - keep[seed] = keep[seed + 1] = True - last = seed + 1 - for candidate in range(seed + 2, local.size): - if agrees(last, candidate): - keep[candidate] = True - last = candidate - first = seed - for candidate in range(seed - 1, -1, -1): - if agrees(candidate, first): - keep[candidate] = True - first = candidate - - dropped = int(np.count_nonzero(~keep)) - if dropped: - warnings.warn( - f"Dropped {dropped} time code packet(s) whose Harp time is inconsistent with " - "the local clock. Pass max_drift=None to keep them.", - stacklevel=3, - ) - return local[keep], harp[keep] - - def _anchor_frame(anchors: pd.Index, harp: NDArray[np.float64]) -> pd.DataFrame: """Build the anchor → Harp time table.""" return pd.DataFrame({"Time": harp}, index=anchors) diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py index bffc852..0c7b191 100644 --- a/tests/data/test_timecode.py +++ b/tests/data/test_timecode.py @@ -1,5 +1,3 @@ -from contextlib import nullcontext - import numpy as np import pytest from harp.data.timecode import ( @@ -286,37 +284,18 @@ def test_tolerates_a_baud_rate_error_that_would_slip_a_whole_bit(): assert clock["Time"].tolist() == values -def test_corrupt_packet_is_dropped_with_a_warning(): - values = [3806874, 3806875, 12345, 3806877] +def test_corrupt_and_jumping_packets_are_returned_as_decoded(): + """Framing is the only filter, so a corrupt value or a clock reset is left to the caller.""" + values = [3806874, 12345, 3806876, 100, 101] times, states, starts = transitions(values) - with pytest.warns(UserWarning, match="inconsistent with the local clock"): - clock = decode_timecode_from_transitions(times, states) - - assert clock["Time"].tolist() == [3806874, 3806875, 3806877] - assert clock.index.to_numpy() == pytest.approx(np.delete(starts, 2)) - - -def test_corrupt_first_packet_is_dropped_with_a_warning(): - values = [12345, 3806875, 3806876, 3806877] - times, states, _ = transitions(values) - - with pytest.warns(UserWarning): - clock = decode_timecode_from_transitions(times, states) - - assert clock["Time"].tolist() == values[1:] - - -def test_max_drift_none_keeps_every_decoded_packet(): - values = [3806874, 3806875, 12345, 3806877] - times, states, _ = transitions(values) - - clock = decode_timecode_from_transitions(times, states, max_drift=None) + clock = decode_timecode_from_transitions(times, states) assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) -def test_missing_packets_do_not_break_the_chain(): +def test_missing_packets_leave_a_gap(): values = [3806874, 3806875, 3806876, 3806877] times, states, starts = transitions(values) dropped = (times < starts[1]) | (times >= starts[2]) @@ -379,11 +358,9 @@ def test_a_long_drifting_sampled_recording_decodes_end_to_end(): 2.98826667, 2.99026667, 2.99126667]), "expected_start_times": np.array([0.96106667, 1.96116667]), "expected_harp_times": [3806874, 3806875], - # the cut-short packet fails its framing check, so nothing reaches the drift check - "expected_warning": False, }, { - # four valid packets and one corrupt one + # four valid packets and one that frames cleanly but carries a corrupt value "timestamps": np.array([ 0.14036667, 1.10146667, 1.10246667, 1.10346667, 1.10446667, 1.11146667, 1.11246667, 1.11646667, 1.11746667, 1.11846667, @@ -404,30 +381,21 @@ def test_a_long_drifting_sampled_recording_decodes_end_to_end(): 5.1368 , 5.1388 , 5.1398 , 5.14183333, 5.1428 , 5.14583333, 5.14783333, 5.14883333, 5.15083333, 5.15183333, 5.16083333, 6.1059 ]), - "expected_start_times": np.array([1.10146667, 2.10156667, 3.10163333, 4.10173333]), - "expected_harp_times": [2600957, 2600958, 2600959, 2600960], - # the corrupt packet decodes cleanly but its seconds do not add up - "expected_warning": True, + "expected_start_times": np.array( + [1.10146667, 2.10156667, 3.10163333, 4.10173333, 5.1018] + ), + "expected_harp_times": [2600957, 2600958, 2600959, 2600960, 2955390977], }, ] # fmt: on -def _decode_recorded(recorded, **kwargs): +@pytest.mark.parametrize("recorded", RECORDED) +def test_decodes_a_recorded_time_code(recorded): times = recorded["timestamps"] states = np.resize([1, 0], times.size) # transitions alternate, starting from high - expected = ( - pytest.warns(UserWarning, match="inconsistent with the local clock") - if recorded["expected_warning"] - else nullcontext() - ) - with expected: - return decode_timecode_from_transitions(times, states, **kwargs) - -@pytest.mark.parametrize("recorded", RECORDED) -def test_decodes_a_recorded_time_code(recorded): - clock = _decode_recorded(recorded) + clock = decode_timecode_from_transitions(times, states) assert clock["Time"].tolist() == recorded["expected_harp_times"] assert clock.index.to_numpy() == pytest.approx(recorded["expected_start_times"]) From 232d0276af40f7e8727c69dbbeea4d2cc3154fa9 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:23:30 -0800 Subject: [PATCH 6/7] Explain why a time code recording yields no packets --- .../src/harp/data/timecode/_decode.py | 33 +++++++++++++++- tests/data/test_timecode.py | 38 ++++++++++++++++++- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index df516b3..711fb26 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -1,3 +1,4 @@ +import warnings from typing import Any import numpy as np @@ -174,8 +175,15 @@ def _decode( """Decode every packet into (first falling edge, Harp second) arrays.""" if baud_rate <= 0: raise ValueError(f"baud_rate must be positive, got {baud_rate}.") + local, harp = _decode_packets(times, levels, 1.0 / baud_rate) + if local.size == 0 and np.any(~levels): + warnings.warn(_explain_no_packets(times, levels, baud_rate), stacklevel=3) + return local, harp - bit = 1.0 / baud_rate + +def _decode_packets( + times: NDArray[np.float64], levels: NDArray[np.bool_], bit: float +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: falling = times[~levels] # a packet can only start on a falling edge local: list[float] = [] harp: list[float] = [] @@ -192,6 +200,29 @@ def _decode( return np.array(local, dtype=np.float64), np.array(harp, dtype=np.float64) +def _explain_no_packets( + times: NDArray[np.float64], levels: NDArray[np.bool_], baud_rate: float +) -> str: + """Guess why a line with falling edges yielded no packet.""" + message = f"No Harp Time Code packet decoded from {np.count_nonzero(~levels)} falling edges." + if _decode_packets(times, ~levels, 1.0 / baud_rate)[0].size: + return f"{message} The line looks inverted (idle low); invert it before decoding." + pulses = np.diff(times) + pulses = pulses[pulses > 0] + if pulses.size == 0: + return message + # most pulses inside a packet are one bit long, so the short end of the + # distribution estimates the bit period actually on the line + apparent = 1.0 / float(np.percentile(pulses, 10)) + if abs(apparent / baud_rate - 1) <= 0.2: + return f"{message} Its pulses match {baud_rate:g} bps, but no packet frames cleanly." + rate = f"{apparent:,.0f}" if apparent >= 10 else f"{apparent:.2g}" + message += f" Its pulses look like ~{rate} bps, not the {baud_rate:g} bps of baud_rate." + if apparent < baud_rate / 100: + message += " If the timestamps are sample numbers, divide them by the sample rate." + return message + + def _anchor_frame(anchors: pd.Index, harp: NDArray[np.float64]) -> pd.DataFrame: """Build the anchor → Harp time table.""" return pd.DataFrame({"Time": harp}, index=anchors) diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py index 0c7b191..c9ebc78 100644 --- a/tests/data/test_timecode.py +++ b/tests/data/test_timecode.py @@ -218,15 +218,49 @@ def test_empty_transition_list_yields_no_anchors(): assert clock["Time"].dtype == np.float64 -def test_wrong_baud_rate_yields_no_anchors(): +def test_wrong_baud_rate_warns_with_the_rate_on_the_line(): times, states, _ = transitions([100, 101, 102]) - clock = decode_timecode_from_transitions(times, states, baud_rate=9600.0) + with pytest.warns(UserWarning, match=r"look like ~1,000 bps, not the 9600 bps"): + clock = decode_timecode_from_transitions(times, states, baud_rate=9600.0) assert clock.empty assert clock.index.name == "LocalTime" +def test_sample_numbers_passed_as_seconds_warn(): + sample_rate = 30_000.0 + times, states, _ = transitions([100, 101, 102]) + + with pytest.warns(UserWarning, match="divide them by the sample rate"): + clock = decode_timecode_from_transitions(np.rint(times * sample_rate), states) + + assert clock.empty + + +def test_inverted_line_warns(): + sample_rate = 10_000.0 + times, states, _ = transitions([100, 101, 102]) + samples = ~render(times, states, sample_rate) + + with pytest.warns(UserWarning, match="looks inverted"): + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock.empty + + +def test_line_with_bit_sized_pulses_but_no_packet_warns(): + # one bit low, two bits high: every tenth bit, where a stop bit belongs, is low + falling = 0.5 + np.arange(30) * 3e-3 + times = np.column_stack([falling, falling + 1e-3]).ravel() + states = np.resize([0, 1], times.size) + + with pytest.warns(UserWarning, match="match 1000 bps, but no packet frames cleanly"): + clock = decode_timecode_from_transitions(times, states) + + assert clock.empty + + # ---------------------------------------------------------------- robustness From c45b014e92258ea8c17ede7e5ce6552d5aac446b Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:39:39 -0800 Subject: [PATCH 7/7] Simplify the time code decoder --- docs/api/data.md | 1 - .../src/harp/data/timecode/__init__.py | 2 - .../src/harp/data/timecode/_decode.py | 41 ++++--------------- 3 files changed, 9 insertions(+), 35 deletions(-) diff --git a/docs/api/data.md b/docs/api/data.md index 9f6f167..81add74 100644 --- a/docs/api/data.md +++ b/docs/api/data.md @@ -13,4 +13,3 @@ ::: harp.data.REFERENCE_EPOCH ::: harp.data.timecode.decode_timecode_from_samples ::: harp.data.timecode.decode_timecode_from_transitions -::: harp.data.timecode.DEFAULT_BAUD_RATE diff --git a/src/packages/harp-data/src/harp/data/timecode/__init__.py b/src/packages/harp-data/src/harp/data/timecode/__init__.py index 8e4e83d..1b1e631 100644 --- a/src/packages/harp-data/src/harp/data/timecode/__init__.py +++ b/src/packages/harp-data/src/harp/data/timecode/__init__.py @@ -1,5 +1,4 @@ from ._decode import ( - DEFAULT_BAUD_RATE, decode_timecode_from_samples, decode_timecode_from_transitions, ) @@ -7,5 +6,4 @@ __all__ = [ "decode_timecode_from_samples", "decode_timecode_from_transitions", - "DEFAULT_BAUD_RATE", ] diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py index 711fb26..4a74d44 100644 --- a/src/packages/harp-data/src/harp/data/timecode/_decode.py +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -5,9 +5,6 @@ import pandas as pd from numpy.typing import ArrayLike, NDArray -#: Default Harp Time Code baud rate, in bits per second. -DEFAULT_BAUD_RATE = 1000.0 - _BITS_PER_BYTE = 10 # 8N1: start bit, eight data bits, stop bit _PACKET_BYTES = 4 # whole second as a little-endian U32 _RESYNC_WINDOW = 0.3 # max start-bit offset, in bits, to re-synchronize a frame on @@ -18,7 +15,7 @@ def decode_timecode_from_transitions( timestamps: ArrayLike, states: ArrayLike, *, - baud_rate: float = DEFAULT_BAUD_RATE, + baud_rate: float = 1000.0, ) -> pd.DataFrame: """Decode a Harp Time Code from line transitions. @@ -32,7 +29,7 @@ def decode_timecode_from_transitions( """ times, levels = _as_transitions(timestamps, states) local, harp = _decode(times, levels, baud_rate=baud_rate) - return _anchor_frame(pd.Index(local, name="LocalTime", dtype=np.float64), harp) + return pd.DataFrame({"Time": harp}, index=pd.Index(local, name="LocalTime")) def decode_timecode_from_samples( @@ -40,7 +37,7 @@ def decode_timecode_from_samples( sample_rate: float, *, threshold: float | None = None, - baud_rate: float = DEFAULT_BAUD_RATE, + baud_rate: float = 1000.0, ) -> pd.DataFrame: """Decode a Harp Time Code from a sampled line. @@ -53,8 +50,6 @@ def decode_timecode_from_samples( """ if sample_rate <= 0: raise ValueError(f"sample_rate must be positive, got {sample_rate}.") - if baud_rate <= 0: - raise ValueError(f"baud_rate must be positive, got {baud_rate}.") if sample_rate < 2 * baud_rate: raise ValueError( f"A {baud_rate} bps signal cannot be decoded from samples taken at " @@ -66,7 +61,7 @@ def decode_timecode_from_samples( times = edges / sample_rate local, harp = _decode(times, level[edges], baud_rate=baud_rate) sample = np.rint(local * sample_rate).astype(np.int64) - return _anchor_frame(pd.Index(sample, name="Sample"), harp) + return pd.DataFrame({"Time": harp}, index=pd.Index(sample, name="Sample")) def _as_transitions( @@ -99,27 +94,14 @@ def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_] def _detect_threshold(values: NDArray[Any]) -> float: - """Find the midpoint between the low and high levels by iterative (isodata) clustering.""" + """Midpoint between the low and high levels of the line.""" if values.size == 0: return 0.0 if values.size > _THRESHOLD_SUBSAMPLE: - # a fixed-seed random subsample is plenty to place the cut, and keeps a long - # recording from being copied; a strided one could alias with the bit period - picks = np.random.default_rng(0).integers(0, values.size, _THRESHOLD_SUBSAMPLE) - values = values[picks] - values = values.astype(np.float64, copy=False) - # start from extreme percentiles rather than min/max, so rare glitches cannot seed a cluster - low, high = (float(level) for level in np.percentile(values, [0.01, 99.99])) - threshold = (low + high) / 2 - if low == high: - return threshold - for _ in range(100): - below = values < threshold - updated = (values[below].mean() + values[~below].mean()) / 2 - if abs(updated - threshold) <= 1e-9 * (high - low): - break - threshold = updated - return float(threshold) + # a fixed-seed random subsample is plenty, and a strided one could alias with the bits + values = values[np.random.default_rng(0).integers(0, values.size, _THRESHOLD_SUBSAMPLE)] + # extreme percentiles rather than min/max, so that rare glitches are ignored + return float(np.percentile(values, [0.01, 99.99]).mean()) def _level_at( @@ -221,8 +203,3 @@ def _explain_no_packets( if apparent < baud_rate / 100: message += " If the timestamps are sample numbers, divide them by the sample rate." return message - - -def _anchor_frame(anchors: pd.Index, harp: NDArray[np.float64]) -> pd.DataFrame: - """Build the anchor → Harp time table.""" - return pd.DataFrame({"Time": harp}, index=anchors)