diff --git a/docs/api/data.md b/docs/api/data.md index ab15e37..81add74 100644 --- a/docs/api/data.md +++ b/docs/api/data.md @@ -11,3 +11,5 @@ ::: harp.data.to_file ::: harp.data.to_buffer ::: harp.data.REFERENCE_EPOCH +::: harp.data.timecode.decode_timecode_from_samples +::: harp.data.timecode.decode_timecode_from_transitions diff --git a/docs/examples/time-code.md b/docs/examples/time-code.md new file mode 100644 index 0000000..3d3d9b3 --- /dev/null +++ b/docs/examples/time-code.md @@ -0,0 +1,14 @@ +# Align to the Harp Time Code + +Devices that are not part of the Harp bus run on their own clock, so their timestamps drift with respect to Harp time. The [Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) exists so they can be aligned afterwards: a Harp device emits the current whole Harp second as a slow serial signal (typically 1 kbps) on a digital output, the non-Harp system records it on a spare digital or analog input, and `harp.data.timecode` decodes that recording back into Harp seconds. + +Each packet starts exactly on the whole second it carries, so every decoded packet is an anchor: the sample (or local time) of its first falling edge, against the Harp second that begins there. + +!!! note + Every packet that frames cleanly is returned as decoded, including the rare one carrying a corrupt value, and a reset of the Harp clock mid-recording shows up as a jump in the anchors. The example below drops anchors that do not line up with the rest before fitting the offset and drift between the two clocks. + + +```python +[](./time_code.py) +``` + diff --git a/docs/examples/time_code.py b/docs/examples/time_code.py new file mode 100644 index 0000000..c1bf941 --- /dev/null +++ b/docs/examples/time_code.py @@ -0,0 +1,33 @@ +import numpy as np +from harp.data.timecode import decode_timecode_from_samples, decode_timecode_from_transitions + +# A non-Harp acquisition system recorded the Harp Time Code on one of its digital +# lines. `samples` is that line, sampled at the system's own rate. +samples = np.load("timecode_line.npy") # digital or analog; the threshold is detected +sample_rate = 30_000.0 + +# Decode it: one row per whole Harp second, keyed on the sample where that second +# starts, the axis this system timestamps the rest of its data on too. +anchors = decode_timecode_from_samples(samples, sample_rate, baud_rate=1000.0) +print(anchors.head()) +# Time +# Sample +# 36000 3806874.0 +# 66003 3806875.0 + +# Every cleanly framed packet is returned as decoded. Harp time minus local time stays +# nearly constant, so the rare corrupt value, or a reset of the Harp clock, stands out. +offset = anchors["Time"] - anchors.index / sample_rate +anchors = anchors[(offset - offset.median()).abs() < 1.0] + +# A line through the anchors gives the offset and drift between the two clocks, and +# extends past the first and last anchors. Any of the system's timestamps (spikes, +# video frames, stimulus onsets) can then be placed on the Harp axis. +slope, intercept = np.polyfit(anchors.index, anchors["Time"], 1) +spike_samples = np.load("spike_samples.npy") +harp_times = intercept + slope * spike_samples + +# Event-based systems report line transitions instead of a sampled waveform: a local +# time and the level the line took. Anchors then carry local seconds. +transitions = np.load("line_transitions.npy") +anchors = decode_timecode_from_transitions(transitions[:, 0], transitions[:, 1]) diff --git a/mkdocs.yml b/mkdocs.yml index 1045a5f..fe7564b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -78,6 +78,7 @@ nav: - Read a dataset folder: examples/dataset.md - Read a single register file: examples/register-file.md - Registers from a schema: examples/registers-from-schema.md + - Align to the Harp Time Code: examples/time-code.md - Guides: - Choose a device module: articles/device-modules.md - Migrate from harp-python: articles/harp-python-migration.md diff --git a/src/packages/harp-data/README.md b/src/packages/harp-data/README.md index faf537d..8212c48 100644 --- a/src/packages/harp-data/README.md +++ b/src/packages/harp-data/README.md @@ -86,6 +86,35 @@ _data, timestamps, _msg, payload = AnalogData.parse_bulk(raw) df = data.payload_to_dataframe(payload) ``` +## Align a non-Harp device with the Harp Time Code + +Devices outside the Harp bus keep their own clock. A Harp device emitting the +[Harp Time Code](https://harp-tech.org/protocol/TimeCode.html) sends the current +whole Harp second as a slow serial signal (typically 1 kbps) on a digital output, +so those devices can record it and be aligned post-hoc. `harp.data.timecode` +turns such a recording back into a table of anchors, keyed on whichever axis the +device timestamps its own data on (the sample, or the local time in seconds, of +each packet's first falling edge) against the Harp second (`"Time"`) that starts +there: + +```python +import numpy as np +from harp.data import timecode + +anchors = timecode.decode_timecode_from_samples(line, sample_rate=30_000.0) # Sample -> Time +slope, intercept = np.polyfit(anchors.index, anchors["Time"], 1) +harp_times = intercept + slope * spike_samples + +# event-based systems report transitions instead, so anchors carry local seconds +anchors = timecode.decode_timecode_from_transitions(edge_times, edge_states) # LocalTime -> Time +``` + +Packets that fail their start/stop bit check are dropped. Every other packet is +returned as decoded, so a rare corrupt value or a reset of the Harp clock shows up +as an anchor that does not line up with the rest; the +[example](https://harp-tech.org/python/examples/time-code/) filters those out +before fitting. + ## Write data back out `to_file` and `to_buffer` are the inverse of the readers, encoding values as Harp frames. Useful for round-tripping data or generating test corpora: diff --git a/src/packages/harp-data/src/harp/data/timecode/__init__.py b/src/packages/harp-data/src/harp/data/timecode/__init__.py new file mode 100644 index 0000000..1b1e631 --- /dev/null +++ b/src/packages/harp-data/src/harp/data/timecode/__init__.py @@ -0,0 +1,9 @@ +from ._decode import ( + decode_timecode_from_samples, + decode_timecode_from_transitions, +) + +__all__ = [ + "decode_timecode_from_samples", + "decode_timecode_from_transitions", +] diff --git a/src/packages/harp-data/src/harp/data/timecode/_decode.py b/src/packages/harp-data/src/harp/data/timecode/_decode.py new file mode 100644 index 0000000..4a74d44 --- /dev/null +++ b/src/packages/harp-data/src/harp/data/timecode/_decode.py @@ -0,0 +1,205 @@ +import warnings +from typing import Any + +import numpy as np +import pandas as pd +from numpy.typing import ArrayLike, NDArray + +_BITS_PER_BYTE = 10 # 8N1: start bit, eight data bits, stop bit +_PACKET_BYTES = 4 # whole second as a little-endian U32 +_RESYNC_WINDOW = 0.3 # max start-bit offset, in bits, to re-synchronize a frame on +_THRESHOLD_SUBSAMPLE = 1_000_000 # samples used to detect the threshold + + +def decode_timecode_from_transitions( + timestamps: ArrayLike, + states: ArrayLike, + *, + baud_rate: float = 1000.0, +) -> pd.DataFrame: + """Decode a Harp Time Code from line transitions. + + See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). + ``timestamps`` are the local times, in seconds, of each transition and ``states`` the + level the line took (0 low, 1 high). Returns the whole Harp second of each packet + (``"Time"``) indexed by the local time of its first falling edge (``"LocalTime"``). + + Packets with bad framing are dropped; every other packet is returned as decoded, so + a clock reset or a corrupt value shows up as an anchor that does not line up. + """ + times, levels = _as_transitions(timestamps, states) + local, harp = _decode(times, levels, baud_rate=baud_rate) + return pd.DataFrame({"Time": harp}, index=pd.Index(local, name="LocalTime")) + + +def decode_timecode_from_samples( + samples: ArrayLike, + sample_rate: float, + *, + threshold: float | None = None, + baud_rate: float = 1000.0, +) -> pd.DataFrame: + """Decode a Harp Time Code from a sampled line. + + See the [Harp Time Code specification](https://harp-tech.org/protocol/TimeCode.html). + Sample ``i`` is taken at local time ``i / sample_rate``. Samples are binarized as + ``samples >= threshold``; by default the threshold is detected from the data as the + midpoint between the high and low levels. Returns the whole Harp second of each packet + (``"Time"``) indexed by the sample of its first falling edge (``"Sample"``), filtered + as in :func:`decode_timecode_from_transitions`. + """ + if sample_rate <= 0: + raise ValueError(f"sample_rate must be positive, got {sample_rate}.") + if sample_rate < 2 * baud_rate: + raise ValueError( + f"A {baud_rate} bps signal cannot be decoded from samples taken at " + f"{sample_rate} Hz; at least 2 samples per bit are required (5 recommended)." + ) + + level = _as_levels(samples, threshold) + edges = np.flatnonzero(np.diff(level)) + 1 + times = edges / sample_rate + local, harp = _decode(times, level[edges], baud_rate=baud_rate) + sample = np.rint(local * sample_rate).astype(np.int64) + return pd.DataFrame({"Time": harp}, index=pd.Index(sample, name="Sample")) + + +def _as_transitions( + timestamps: ArrayLike, states: ArrayLike +) -> tuple[NDArray[np.float64], NDArray[np.bool_]]: + """Validate transitions into sorted times and boolean levels.""" + times = np.asarray(timestamps, dtype=np.float64) + levels = np.asarray(states) + if times.ndim != 1 or levels.ndim != 1: + raise ValueError("timestamps and states must be one-dimensional.") + if times.size != levels.size: + raise ValueError( + f"timestamps and states must have the same length, got {times.size} and {levels.size}." + ) + if np.any(np.diff(times) < 0): + raise ValueError("timestamps must be sorted in ascending order.") + return times, levels.astype(np.bool_) + + +def _as_levels(samples: ArrayLike, threshold: float | None) -> NDArray[np.bool_]: + """Binarize a sampled waveform into digital line levels.""" + values = np.asarray(samples) + if values.ndim != 1: + raise ValueError("samples must be one-dimensional.") + if threshold is None: + if values.dtype == np.bool_: + return values + threshold = _detect_threshold(values) + return values >= threshold + + +def _detect_threshold(values: NDArray[Any]) -> float: + """Midpoint between the low and high levels of the line.""" + if values.size == 0: + return 0.0 + if values.size > _THRESHOLD_SUBSAMPLE: + # a fixed-seed random subsample is plenty, and a strided one could alias with the bits + values = values[np.random.default_rng(0).integers(0, values.size, _THRESHOLD_SUBSAMPLE)] + # extreme percentiles rather than min/max, so that rare glitches are ignored + return float(np.percentile(values, [0.01, 99.99]).mean()) + + +def _level_at( + times: NDArray[np.float64], levels: NDArray[np.bool_], at: NDArray[np.floating[Any]] +) -> NDArray[np.bool_]: + """Line level at each time in ``at``.""" + index = np.searchsorted(times, at, side="right") - 1 + # transitions alternate, so before the first one the line held the opposite level + return np.where(index < 0, ~levels[0], levels[np.maximum(index, 0)]) + + +def _resync(falling: NDArray[np.float64], expected: float, bit: float) -> float: + """Snap a predicted start-bit time to a nearby falling edge.""" + best, window = expected, _RESYNC_WINDOW * bit + index = int(np.searchsorted(falling, expected)) + for candidate in (index - 1, index): + if 0 <= candidate < falling.size: + distance = abs(falling[candidate] - expected) + if distance <= window: + best, window = float(falling[candidate]), distance + return best + + +def _decode_packet( + times: NDArray[np.float64], + levels: NDArray[np.bool_], + falling: NDArray[np.float64], + origin: float, + bit: float, +) -> tuple[int, float] | None: + """Decode the packet starting at ``origin`` into (seconds, end time), or ``None``.""" + payload = bytearray(_PACKET_BYTES) + offsets = (np.arange(_BITS_PER_BYTE) + 0.5) * bit + start = origin + for frame in range(_PACKET_BYTES): + if frame: + # predict from the previous start bit, not from the packet, so that a small + # baud rate error stays a fraction of a bit instead of accumulating + start = _resync(falling, start + _BITS_PER_BYTE * bit, bit) + bits = _level_at(times, levels, start + offsets) + if bits[0] or not bits[-1]: # start bit must be low, stop bit high + return None + payload[frame] = np.packbits(bits[1:-1], bitorder="little")[0] + return int.from_bytes(payload, "little"), start + _BITS_PER_BYTE * bit + + +def _decode( + times: NDArray[np.float64], + levels: NDArray[np.bool_], + *, + baud_rate: float, +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + """Decode every packet into (first falling edge, Harp second) arrays.""" + if baud_rate <= 0: + raise ValueError(f"baud_rate must be positive, got {baud_rate}.") + local, harp = _decode_packets(times, levels, 1.0 / baud_rate) + if local.size == 0 and np.any(~levels): + warnings.warn(_explain_no_packets(times, levels, baud_rate), stacklevel=3) + return local, harp + + +def _decode_packets( + times: NDArray[np.float64], levels: NDArray[np.bool_], bit: float +) -> tuple[NDArray[np.float64], NDArray[np.float64]]: + falling = times[~levels] # a packet can only start on a falling edge + local: list[float] = [] + harp: list[float] = [] + decoded_until = -np.inf + for origin in falling: + if origin < decoded_until: + continue # inside a packet we already decoded + packet = _decode_packet(times, levels, falling, float(origin), bit) + if packet is None: + continue + seconds, decoded_until = packet + local.append(float(origin)) + harp.append(float(seconds)) + return np.array(local, dtype=np.float64), np.array(harp, dtype=np.float64) + + +def _explain_no_packets( + times: NDArray[np.float64], levels: NDArray[np.bool_], baud_rate: float +) -> str: + """Guess why a line with falling edges yielded no packet.""" + message = f"No Harp Time Code packet decoded from {np.count_nonzero(~levels)} falling edges." + if _decode_packets(times, ~levels, 1.0 / baud_rate)[0].size: + return f"{message} The line looks inverted (idle low); invert it before decoding." + pulses = np.diff(times) + pulses = pulses[pulses > 0] + if pulses.size == 0: + return message + # most pulses inside a packet are one bit long, so the short end of the + # distribution estimates the bit period actually on the line + apparent = 1.0 / float(np.percentile(pulses, 10)) + if abs(apparent / baud_rate - 1) <= 0.2: + return f"{message} Its pulses match {baud_rate:g} bps, but no packet frames cleanly." + rate = f"{apparent:,.0f}" if apparent >= 10 else f"{apparent:.2g}" + message += f" Its pulses look like ~{rate} bps, not the {baud_rate:g} bps of baud_rate." + if apparent < baud_rate / 100: + message += " If the timestamps are sample numbers, divide them by the sample rate." + return message diff --git a/tests/data/test_timecode.py b/tests/data/test_timecode.py new file mode 100644 index 0000000..c9ebc78 --- /dev/null +++ b/tests/data/test_timecode.py @@ -0,0 +1,435 @@ +import numpy as np +import pytest +from harp.data.timecode import ( + _decode, + decode_timecode_from_samples, + decode_timecode_from_transitions, +) + +BAUD_RATE = 1000.0 + + +def _packet_bits(value): + """The 8N1 bit stream of one time code packet: start, 8 data bits (LSB first), stop.""" + bits = [] + for byte in int(value).to_bytes(4, "little"): + bits.append(0) + bits.extend((byte >> position) & 1 for position in range(8)) + bits.append(1) + return bits + + +def transitions( + values, + *, + baud_rate=BAUD_RATE, + emitted_baud_rate=None, + period=1.0, + first=0.5, + jitter=0.0, + seed=0, +): + """An idle-high time code line carrying one packet per ``period``. + + Returns the transition times, the level taken at each of them, and the local time + of the first falling edge of every packet. ``emitted_baud_rate`` sets the rate the + signal is actually generated at, which may differ from the nominal ``baud_rate``. + """ + bit = 1.0 / (emitted_baud_rate or baud_rate) + rng = np.random.default_rng(seed) + times, states, starts = [], [], [] + level = 1 + for packet, value in enumerate(values): + start = first + packet * period + starts.append(start) + for position, expected in enumerate(_packet_bits(value)): + if expected != level: + offset = rng.uniform(-jitter, jitter) * bit if jitter else 0.0 + times.append(start + position * bit + offset) + states.append(expected) + level = expected + return np.array(times), np.array(states), np.array(starts) + + +def render(times, states, sample_rate, *, duration=None): + """Sample an idle-high transition list onto a uniform grid of digital states.""" + duration = duration if duration is not None else times[-1] + 0.5 + grid = np.arange(int(round(duration * sample_rate))) / sample_rate + index = np.searchsorted(times, grid, side="right") - 1 + return np.where(index < 0, ~states[0].astype(bool), states[np.maximum(index, 0)]).astype(bool) + + +def packet_duration(*, baud_rate=BAUD_RATE): + return len(_packet_bits(0)) / baud_rate + + +# ---------------------------------------------------------------- decoding + + +def test_decodes_a_single_packet_on_its_first_falling_edge(): + times, states, starts = transitions([3806874]) + clock = decode_timecode_from_transitions(times, states) + + assert clock.index.name == "LocalTime" + assert clock.columns.tolist() == ["Time"] + assert clock["Time"].tolist() == [3806874] + assert clock.index.to_numpy() == pytest.approx(starts) + + +def test_decodes_consecutive_seconds(): + values = [3806874, 3806875, 3806876, 3806877] + times, states, starts = transitions(values) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) + + +@pytest.mark.parametrize("value", [0, 1, 0xFF, 0xFF00, 0xFF00FF00, 0xFFFFFFFE, 3806874]) +def test_round_trips_every_byte_pattern(value): + """Bytes of all zeros or all ones leave long runs without a transition.""" + times, states, _ = transitions([value, value + 1]) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == [value, value + 1] + + +@pytest.mark.parametrize("sample_rate", [5_000.0, 30_000.0]) +def test_sampled_recording_matches_the_transition_list(sample_rate): + values = [3806874, 3806875, 3806876] + times, states, starts = transitions(values) + samples = render(times, states, sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock["Time"].tolist() == values + assert clock.index.name == "Sample" + assert clock.index.dtype == np.int64 + assert clock.index.tolist() == [round(anchor * sample_rate) for anchor in starts] + + +def test_sample_index_points_at_the_anchor_sample(): + sample_rate = 10_000.0 + times, states, starts = transitions([42, 43], first=0.5) + samples = render(times, states, sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock.columns.tolist() == ["Time"] + # the anchor sample is the first sample of the packet's start bit + for sample, start in zip(clock.index, starts): + assert sample == round(start * sample_rate) + assert not samples[sample] + assert samples[sample - 1] + + +def test_analog_recording_is_binarized_by_threshold(): + sample_rate = 10_000.0 + values = [7, 8] + times, states, _ = transitions(values) + analog = np.where(render(times, states, sample_rate), 3.3, 0.1) + 0.02 + + clock = decode_timecode_from_samples(analog, sample_rate, threshold=1.5) + + assert clock["Time"].tolist() == values + + +@pytest.mark.parametrize("value", [3806874, 0xFFFFFFF0]) +def test_analog_threshold_is_detected_from_a_noisy_mostly_idle_line(value): + """Bytes of all ones leave the line low only for start bits, about 0.5% of the time.""" + sample_rate = 10_000.0 + values = [value + second for second in range(5)] + times, states, _ = transitions(values) + level = render(times, states, sample_rate) + noise = np.random.default_rng(0).normal(0.0, 0.15, level.size) + analog = np.where(level, 3.3, 0.1) - 1.0 + noise + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + +@pytest.mark.parametrize( + "high, low", [(1, 0), (8, 0), (12_000, -3_000)], ids=["digital", "ttl-word", "int16-analog"] +) +def test_integer_threshold_is_detected_from_the_levels(high, low): + sample_rate = 10_000.0 + values = [3806874, 3806875] + times, states, _ = transitions(values) + samples = np.where(render(times, states, sample_rate), high, low).astype(np.int16) + + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock["Time"].tolist() == values + + +def test_detected_threshold_ignores_rare_glitches(): + sample_rate = 10_000.0 + values = [3806874 + second for second in range(5)] + times, states, _ = transitions(values) + analog = np.where(render(times, states, sample_rate), 3.3, 0.1) + analog[[100, 20_000, 40_000]] = [100.0, -100.0, 100.0] + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + +def test_threshold_is_detected_from_a_subsample_of_long_recordings(monkeypatch): + monkeypatch.setattr(_decode, "_THRESHOLD_SUBSAMPLE", 5_000) + sample_rate = 10_000.0 + values = [0xFFFFFFF0 + second for second in range(5)] + times, states, _ = transitions(values) + level = render(times, states, sample_rate) + noise = np.random.default_rng(0).normal(0.0, 0.15, level.size) + analog = np.where(level, 3.3, 0.1) + noise + + clock = decode_timecode_from_samples(analog, sample_rate) + + assert clock["Time"].tolist() == values + + +def test_constant_recording_with_detected_threshold_yields_no_anchors(): + clock = decode_timecode_from_samples(np.full(10_000, 2.5), 10_000.0) + + assert clock.empty + + +def test_sample_rate_below_two_samples_per_bit_raises(): + with pytest.raises(ValueError, match="cannot be decoded"): + decode_timecode_from_samples(np.zeros(100, dtype=bool), 1500.0, baud_rate=BAUD_RATE) + + +def test_recording_without_any_transition_yields_no_anchors(): + clock = decode_timecode_from_samples(np.ones(10_000, dtype=bool), 10_000.0) + + assert clock.empty + assert clock.columns.tolist() == ["Time"] + assert clock.index.name == "Sample" + + +def test_empty_transition_list_yields_no_anchors(): + clock = decode_timecode_from_transitions([], []) + + assert clock.empty + assert clock.columns.tolist() == ["Time"] + assert clock["Time"].dtype == np.float64 + + +def test_wrong_baud_rate_warns_with_the_rate_on_the_line(): + times, states, _ = transitions([100, 101, 102]) + + with pytest.warns(UserWarning, match=r"look like ~1,000 bps, not the 9600 bps"): + clock = decode_timecode_from_transitions(times, states, baud_rate=9600.0) + + assert clock.empty + assert clock.index.name == "LocalTime" + + +def test_sample_numbers_passed_as_seconds_warn(): + sample_rate = 30_000.0 + times, states, _ = transitions([100, 101, 102]) + + with pytest.warns(UserWarning, match="divide them by the sample rate"): + clock = decode_timecode_from_transitions(np.rint(times * sample_rate), states) + + assert clock.empty + + +def test_inverted_line_warns(): + sample_rate = 10_000.0 + times, states, _ = transitions([100, 101, 102]) + samples = ~render(times, states, sample_rate) + + with pytest.warns(UserWarning, match="looks inverted"): + clock = decode_timecode_from_samples(samples, sample_rate) + + assert clock.empty + + +def test_line_with_bit_sized_pulses_but_no_packet_warns(): + # one bit low, two bits high: every tenth bit, where a stop bit belongs, is low + falling = 0.5 + np.arange(30) * 3e-3 + times = np.column_stack([falling, falling + 1e-3]).ravel() + states = np.resize([0, 1], times.size) + + with pytest.warns(UserWarning, match="match 1000 bps, but no packet frames cleanly"): + clock = decode_timecode_from_transitions(times, states) + + assert clock.empty + + +# ---------------------------------------------------------------- robustness + + +def test_glitches_between_packets_are_ignored(): + values = [500, 501] + times, states, starts = transitions(values) + # a 100 us spike in the idle line, well away from any packet + times = np.concatenate([times, [1.4, 1.4001]]) + states = np.concatenate([states, [0, 1]]) + order = np.argsort(times) + + clock = decode_timecode_from_transitions(times[order], states[order]) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) + + +def test_packet_truncated_by_the_end_of_the_recording_is_dropped(): + values = [500, 501, 502] + times, states, starts = transitions(values) + keep = times < starts[-1] + 0.5 * packet_duration() + + clock = decode_timecode_from_transitions(times[keep], states[keep]) + + assert clock["Time"].tolist() == values[:-1] + + +def test_recording_starting_mid_packet_keeps_the_later_packets(): + values = [500, 501, 502] + times, states, starts = transitions(values) + keep = times > starts[0] + 0.5 * packet_duration() + + clock = decode_timecode_from_transitions(times[keep], states[keep]) + + assert clock["Time"].tolist() == values[1:] + + +def test_tolerates_jitter_on_every_edge(): + values = [3806874, 3806875, 3806876] + times, states, starts = transitions(values, jitter=0.2) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts, abs=0.3 / BAUD_RATE) + + +def test_tolerates_a_baud_rate_error_that_would_slip_a_whole_bit(): + """Per-frame re-synchronization keeps a 3% rate error from walking off the bits.""" + values = [3806874, 3806875] + times, states, _ = transitions(values, emitted_baud_rate=BAUD_RATE * 1.03) + + clock = decode_timecode_from_transitions(times, states, baud_rate=BAUD_RATE) + + assert clock["Time"].tolist() == values + + +def test_corrupt_and_jumping_packets_are_returned_as_decoded(): + """Framing is the only filter, so a corrupt value or a clock reset is left to the caller.""" + values = [3806874, 12345, 3806876, 100, 101] + times, states, starts = transitions(values) + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == values + assert clock.index.to_numpy() == pytest.approx(starts) + + +def test_missing_packets_leave_a_gap(): + values = [3806874, 3806875, 3806876, 3806877] + times, states, starts = transitions(values) + dropped = (times < starts[1]) | (times >= starts[2]) + + clock = decode_timecode_from_transitions(times[dropped], states[dropped]) + + assert clock["Time"].tolist() == [3806874, 3806876, 3806877] + + +def test_invalid_transition_lists_raise(): + times, states, _ = transitions([1, 2]) + with pytest.raises(ValueError, match="same length"): + decode_timecode_from_transitions(times, states[:-1]) + with pytest.raises(ValueError, match="sorted"): + decode_timecode_from_transitions(times[::-1], states) + with pytest.raises(ValueError, match="one-dimensional"): + decode_timecode_from_transitions(np.zeros((2, 2)), np.zeros((2, 2))) + with pytest.raises(ValueError, match="baud_rate"): + decode_timecode_from_transitions(times, states, baud_rate=0.0) + + +def test_a_long_drifting_sampled_recording_decodes_end_to_end(): + """A minute of a 30 kHz recording: local clock drift, jitter and one lost packet.""" + rate = 1.001 # Harp seconds per local second + sample_rate = 30_000.0 + values = [3806874 + second for second in range(60)] + times, states, starts = transitions( + values, period=1.0 / rate, baud_rate=BAUD_RATE * rate, jitter=0.15 + ) + lost = (times >= starts[30]) & (times < starts[30] + 1.5 * packet_duration()) + samples = render(times[~lost], states[~lost], sample_rate) + + clock = decode_timecode_from_samples(samples, sample_rate, baud_rate=BAUD_RATE * rate) + + assert clock["Time"].tolist() == [value for value in values if value != values[30]] + # anchors land on the emitted start bits, within the jitter and one sample period + assert clock.index.to_numpy() / sample_rate == pytest.approx(np.delete(starts, 30), abs=5e-4) + + +# ---------------------------------------------------------------- recorded signals + +# Transitions acquired by an Open Ephys system from a Behavior board emitting the +# time code at 1 kbps, taken from https://github.com/harp-tech/harp-python/pull/38. +# fmt: off +RECORDED = [ + { + # two valid packets, followed by one cut short by the end of the recording + "timestamps": np.array([ + 0. , 0.96106667, 0.96306667, 0.96406667, 0.96506667, + 0.96706667, 0.96906667, 0.97106667, 0.97306667, 0.97506667, + 0.97606667, 0.97706667, 0.98006667, 0.98106667, 0.98306667, + 0.98406667, 0.98506667, 0.98806667, 0.99006667, 0.99106667, + 1.00006667, 1.96116667, 1.96216667, 1.96416667, 1.96516667, + 1.96716667, 1.96916667, 1.97116667, 1.97316667, 1.97516667, + 1.97616667, 1.97716667, 1.98016667, 1.98116667, 1.98316667, + 1.98416667, 1.98516667, 1.98816667, 1.99016667, 1.99116667, + 2.00016667, 2.96126667, 2.96426667, 2.96726667, 2.96926667, + 2.97126667, 2.97326667, 2.97526667, 2.97626667, 2.97726667, + 2.98026667, 2.98126667, 2.98326667, 2.98426667, 2.98526667, + 2.98826667, 2.99026667, 2.99126667]), + "expected_start_times": np.array([0.96106667, 1.96116667]), + "expected_harp_times": [3806874, 3806875], + }, + { + # four valid packets and one that frames cleanly but carries a corrupt value + "timestamps": np.array([ + 0.14036667, 1.10146667, 1.10246667, 1.10346667, 1.10446667, + 1.11146667, 1.11246667, 1.11646667, 1.11746667, 1.11846667, + 1.11946667, 1.12146667, 1.12246667, 1.12546667, 1.12746667, + 1.12846667, 1.13046667, 1.13146667, 1.14046667, 2.10156667, + 2.10356667, 2.11156667, 2.11256667, 2.11656667, 2.11756667, + 2.11856667, 2.11956667, 2.12156667, 2.12256667, 2.12556667, + 2.12756667, 2.12856667, 2.13056667, 2.13156667, 2.14056667, + 3.10163333, 3.10263333, 3.11163333, 3.11263333, 3.11663333, + 3.11763333, 3.11863333, 3.11963333, 3.12163333, 3.12263333, + 3.12563333, 3.12763333, 3.12863333, 3.13063333, 3.13163333, + 3.14063333, 4.10173333, 4.11073333, 4.11173333, 4.11673333, + 4.11873333, 4.11973333, 4.12173333, 4.12273333, 4.12573333, + 4.12773333, 4.12873333, 4.13073333, 4.13173333, 4.14073333, + 5.1018 , 5.1028 , 5.1038 , 5.1108 , 5.1118 , + 5.1168 , 5.1188 , 5.1198 , 5.1218 , 5.1228 , + 5.1258 , 5.1278 , 5.1288 , 5.1308 , 5.1318 , + 5.1368 , 5.1388 , 5.1398 , 5.14183333, 5.1428 , + 5.14583333, 5.14783333, 5.14883333, 5.15083333, 5.15183333, + 5.16083333, 6.1059 ]), + "expected_start_times": np.array( + [1.10146667, 2.10156667, 3.10163333, 4.10173333, 5.1018] + ), + "expected_harp_times": [2600957, 2600958, 2600959, 2600960, 2955390977], + }, +] +# fmt: on + + +@pytest.mark.parametrize("recorded", RECORDED) +def test_decodes_a_recorded_time_code(recorded): + times = recorded["timestamps"] + states = np.resize([1, 0], times.size) # transitions alternate, starting from high + + clock = decode_timecode_from_transitions(times, states) + + assert clock["Time"].tolist() == recorded["expected_harp_times"] + assert clock.index.to_numpy() == pytest.approx(recorded["expected_start_times"])