Source code for gpype.backend.pipeline

import copy
import logging
import os
import platform
import sys
import tempfile
import threading
import time
from time import perf_counter
from typing import Optional, Union

import ioiocore as ioc

from .. import _installer
from ..common import config_keys
from ..common._private import (
    attestation,
    device_probe,
    licence,
    naming,
    pipelines,
    remote_attestation,
    timebase,
)
from ..common._private.channels import stamp_clock_complaint
from ..common._private.entitlement import (
    MARK,
    Entitlement,
    Permission,
    device_required,
    discard_supplied,
    entitlement_of,
    lifted,
    meet,
    release_entitlement,
    remote_required,
    set_entitlement,
    take_supplied,
)
from ..common.constants import Constants
from ..common.launch_config import ENV_EDGE_ID, LaunchConfig
from ..common.result import Result
from .core._private.controllable import (
    action_names,
    ensure_json,
    has_control,
    merged_update,
    resolve_action,
    resolve_control,
)
from .core._private.link import CONTEXT_TIMEOUT_S
from .core._private.placement import position_gate
from .core._private.timeline import (
    TimelineManager,
    release_timeline,
    timeline_of,
)
from .core.node import Node
from .core.o_port import OPort

log = logging.getLogger(__name__)

#: Where a node's document id lives. It is a *configuration key*, not a
#: property -- there is no ``id`` property anywhere in ioiocore -- so it
#: is read out of the node's configuration dict.
_ID_KEY = ioc.Portable.Configuration.Keys.ID


def _node_id_of(node) -> Optional[str]:
    """The document id of *node*.

    Deliberately the id and not the name. Names are **not unique**: a
    node defaults its name to its class name
    (``ioiocore/processing_element.py``), and ``chain_params`` refuses to
    use names for exactly this reason, in writing -- "two unnamed chains
    of the same class both take the class name, verified, so deriving
    from the name would pair two sinks on the same stream and the
    broker's last-writer-wins would cross them silently". Addressing a
    control at a name would pick whichever of them came first, and
    succeed.

    Args:
        node: The node to read.

    Returns:
        The id, or None when the node carries no configuration. Read
        defensively so a stand-in without one cannot break a report that
        is meant to describe the whole pipeline.
    """
    config = getattr(node, "config", None)
    if not isinstance(config, dict):
        return None
    return config.get(_ID_KEY)


def _fallback_log_directory() -> Optional[str]:
    """Where to log when ioiocore may have no default.

    ioiocore knows Windows, Darwin and Linux and raises
    ``RuntimeError("Unsupported OS")`` for any other
    ``platform.system()`` -- Android and iOS included (PEP 738, PEP 730).
    A durable directory there lies in the app's container, which only
    the host app knows; the temp directory is the app's own on both, and
    the one Python checks is writable.

    Returns:
        None where ioiocore has a default. Otherwise a directory under
        the temp directory, or "" -- memory-only logging -- when none is
        writable.
    """
    if platform.system() in ("Windows", "Darwin", "Linux"):
        return None
    try:
        return os.path.join(tempfile.gettempdir(), "gtec", "gPype")
    except OSError:
        return ""


[docs] class Pipeline(ioc.Pipeline): """Brain-Computer Interface pipeline for real-time data processing. Extends ioiocore Pipeline for BCI applications with automatic logging to platform-specific directories. Manages node lifecycle, data flow connections, and real-time execution of interconnected processing nodes. """
[docs] def __init__(self): """Initialize Pipeline with platform-specific logging directory.""" # Determine platform-specific log directory if sys.platform == "win32": log_dir = os.path.join(os.getenv("APPDATA", ""), "gtec", "gPype") elif sys.platform == "darwin": app_support = os.path.expanduser("~/Library/Application Support") log_dir = os.path.join(app_support, "gtec", "gPype") else: log_dir = _fallback_log_directory() # Initialize parent pipeline with logging directory super().__init__(directory=log_dir) self._init_gpype_state()
#: Names ``repr`` shows in a line before it elides the middle. _REPR_NAMES = 6
[docs] def __repr__(self) -> str: """One line: the mode, the nodes, and the state. ``Pipeline(batch, 3 nodes: eeg -> bp -> out, stopped)`` for a pipeline that is one line; otherwise the sources and the sinks, ``Pipeline(realtime, 5 nodes, sources [a, b], sinks [c], running)``. A node is counted as its author wrote it: the chain the pipeline builds around it is that node. Never raises. """ try: return self._repr_text() except Exception: # pragma: no cover - a repr must not raise return object.__repr__(self)
def _repr_text(self) -> str: """Build the text ``__repr__`` returns. Returns: The one-line description. """ elements = list(getattr(self._imp, "_nodes", None) or []) try: mode = self.execution_mode() except ValueError: mode = "mixed modes" if self._sources() else "no source" count = len(elements) nodes = f"{count} node{'' if count == 1 else 's'}" path = self._linear_path(elements) if path: nodes += ": " + " -> ".join(self._repr_names(path)) else: sources = [e for e in elements if self._repr_kind(e) == "source"] sinks = [e for e in elements if self._repr_kind(e) == "sink"] if sources: nodes += f", sources [{', '.join(self._repr_names(sources))}]" if sinks: nodes += f", sinks [{', '.join(self._repr_names(sinks))}]" if self.get_condition() == ioc.Constants.Conditions.ERROR: state = "failed" else: state = str(self.get_state()).lower() return f"Pipeline({mode}, {nodes}, {state})" def _linear_path(self, elements: list) -> Optional[list]: """The pipeline in order, where it is a single line of nodes. Args: elements: Every element in the pipeline. Returns: The elements from source to sink, or None when the pipeline branches, merges, has a loop, or holds a node that was never connected. """ if len(elements) < 2 or len(elements) != len(self._elements): return None downstream = { key: {id(e): e for e in targets} for key, targets in self._cycle_adjacency().items() } upstream: dict = {} for targets in downstream.values(): if len(targets) > 1: return None for key in targets: upstream[key] = upstream.get(key, 0) + 1 if any(n > 1 for n in upstream.values()): return None heads = [e for e in elements if id(e) not in upstream] if len(heads) != 1: return None path = [heads[0]] while id(path[-1]) in downstream and len(path) <= len(elements): path.extend(downstream[id(path[-1])].values()) return path if len(path) == len(elements) else None @staticmethod def _repr_kind(element) -> Optional[str]: """``source`` for outputs only, ``sink`` for inputs only.""" if isinstance(element, ioc.Chain): if isinstance(element, ioc.OChain): return "source" if isinstance(element, ioc.IChain): return "sink" return None inputs = isinstance(element, ioc.INode) outputs = isinstance(element, ioc.ONode) if outputs and not inputs: return "source" if inputs and not outputs: return "sink" return None @classmethod def _repr_names(cls, elements: list) -> list: """The elements' names, the middle elided past ``_REPR_NAMES``.""" names = [ str(getattr(e, "name", None) or type(e).__name__) for e in elements ] if len(names) <= cls._REPR_NAMES: return names half = cls._REPR_NAMES // 2 return names[:half] + ["..."] + names[-half:] @property def event_margin_ms(self) -> float: """How late an event may arrive and still be placed, in ms. One number for the whole pipeline -- see TimelineManager.DEFAULT_EVENT_MARGIN_MS for what it buys and what it costs. A property rather than a constructor argument because ``deserialize`` builds its instance with ``object.__new__`` and never runs ``__init__``, so a constructor-only setting would be absent on exactly the pipelines a document produced. Kept here rather than on the timeline, even though the timeline is where placement reads it: ``start()`` releases the timeline and builds a fresh one, deliberately, so that a run cannot inherit the previous one's master election or epoch. A margin written to the timeline before ``start()`` went with it -- measured, every value from 0 to 100 ms produced the same 10-frame hold, because placement was reading the default off an object that had just been replaced. Returns: The margin in milliseconds. """ stored = getattr(self, "_event_margin_ms", None) if stored is not None: return stored if self._oscar_active(): return 0.0 return TimelineManager.DEFAULT_EVENT_MARGIN_MS def _oscar_active(self) -> bool: """Whether an enabled OSCAR node is in this pipeline. The margin exists to give a late observation somewhere to land, and it pays for that by holding frames. OSCAR already delays the signal path by far more than the margin ever buys -- 70 samples at 250 Hz against the 10 the default asks for -- and an event arriving inside that delay is placed on its own position regardless. Holding frames on top of it would add latency and buy nothing, so the margin defaults to zero here. An explicit setting still wins: this only decides the default. Imported inside the method: this module is imported from ``gpype/__init__`` and the node package reaches back into it. Returns: True when a node in this pipeline is removing artifacts. """ from .core._private.oscar import Oscar for element in getattr(self, "_elements", []): candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if isinstance(node, Oscar) and node.enabled: return True return False @event_margin_ms.setter def event_margin_ms(self, value: Optional[float]) -> None: """Set the margin for every placing node in this pipeline. Args: value: Milliseconds, 0 to opt out, or None for the default. """ self._event_margin_ms = ( None if value is None else max(0.0, float(value)) ) # Pushed through as well, so a change lands on a pipeline that is # already running rather than only on its next start(). timeline_of(self).event_margin_ms = self.event_margin_ms def _init_gpype_state(self) -> None: """Establish the state that belongs to this class, not ioiocore. Called from ``__init__`` *and* from ``deserialize``, which builds its instance with ``object.__new__`` and therefore never runs ``__init__`` at all. One method rather than two lists, because two lists drift: this one had, and every attribute added to ``__init__`` since was simply absent on every deserialised pipeline. Measured on one: ``.failure`` and ``.close()`` both raised ``AttributeError``, it was missing from the registry a GUI finds pipelines through, and it carried **zero** error handlers where a constructed pipeline carries one -- so a deserialised pipeline that died recorded nothing and told nobody. That is the path a remote runtime uses for every pipeline it is given, which is where it mattered. """ #: Allowed late arrival for asynchronous observations, in #: milliseconds, or None for the default. Set here as well as #: in __init__ because deserialize() runs only this. self._event_margin_ms = None #: Elements seen by connect(), so the pipeline can find the nodes #: that need its master timeline. ioiocore adds nodes implicitly #: and exposes no enumeration, but every participant passes #: through connect() at least once. self._elements: list = [] #: The failing log entry, once a run has failed. Recorded #: without anyone asking: a pipeline that has died is a thing #: every caller may need to know about, and requiring a #: registration step means the one script that forgot is the one #: that fails silently. self._failure = None # Registered here rather than by the caller, for the same reason. self.add_error_handler(self._record_failure) # So a GUI can find this pipeline without being handed it: # MainApp and Pipeline are built independently and never # introduced, and a windowed application may never show the # console the failure is printed to. pipelines.register(self) #: EDGE-side responders answering the server's challenges (C10). self._responders: list = [] #: A broker this pipeline bound *early*, before its nodes #: started, only so a remote attestation could happen at all. #: Held for the run and released in stop(), which balances the #: extra acquire() against the Links' own. self._attest_broker = None #: Live probes, by stream id. Not part of the pipeline and not in #: _elements: a probe is a callable on a port, deliberately not #: a node -- see backend/core/_private/probe.py for why. self._probes: dict = {} #: The chain built around each world-facing core, by object id. #: What makes `chain_of` answer for a core the chain holds but #: does not run -- a source under SERVER residency. See _wrap. self._wrappers: dict = {} #: The residencies the chains were built under. A pipeline runs #: under the residency it was built for: built as a server and #: started as standalone, it would keep its broker and Links while #: the licence gate saw development (D-ENT-100). self._built_residencies: set = set() #: The clock-sync health this run's edge start converged on, as #: its source streams' contexts carry it; None when the run did #: not sync (STANDALONE, SERVER, batch, or no source here). self._clock_health: Optional[dict] = None #: The pins serialize() writes when given none: those it was last #: given, or those of the document this pipeline was rebuilt #: from, as it carried them (D-CORE-109). self._requirements = None def _register(self, element: Union[Node, dict]) -> None: """Remember an element taking part in this pipeline. Args: element: Node, chain, or ``node["port"]`` specification. """ if isinstance(element, dict): element = element.get("node") if element is not None and not any( e is element for e in self._elements ): self._elements.append(element) def _wrap(self, element: Union[Node, dict]): """Put a chain around a world-facing node that has none. ``connect()`` is the only place a node enters a pipeline, so it is where assembly can stop being the node's own business. Handed a bare source, sink or widget core, this returns the chain that carries it -- Link, Sync, recording stage and all -- and handed anything else it returns what it was given. A chain is returned unchanged, so a hand-written one is left as its author built it. See ``wrapping.wrap``. The same core handed in twice gets the same chain, held in a map keyed on the node. A second chain would be a second Link on the same stream id, which is the kind of fault that shows up as a silent half-delivery rather than as an error. Args: element: Node, chain, or ``node["port"]`` specification. Returns: The element to connect, which may be the one passed in. A port specification comes back as a new dict naming the chain, leaving the caller's own dict untouched. """ from .core._private import wrapping if isinstance(element, dict): node = element.get("node") wrapped = self._wrap(node) if wrapped is node: return element return {**element, "node": wrapped} if element is None or isinstance(element, ioc.Chain): return element # Keyed on the node, not on `_elements`: `add_node` puts a node # into the pipeline without registering it there -- deliberately, as # `_validate_has_signal` records -- so a pipeline that adds a core # and then connects it would otherwise get a second chain for the # same core. Which is a second Link on one stream id, and that # fails as a silent half-delivery rather than as an error. # # The chain holds the core, so the identity cannot be reused # while the entry is live. chain = self._wrappers.get(id(element)) if chain is None: chain = wrapping.wrap(element) if chain is not element: self._wrappers[id(element)] = chain self._built_residencies.add(LaunchConfig.get().residency) return chain
[docs] def add_node(self, node) -> None: """Add a node to the pipeline, wrapping it if it needs a chain. ``connect`` is the ordinary way in and does the same thing; this is the other one, and it has to agree. A core added here and connected afterwards would otherwise reach the pipeline twice -- once bare and once inside its chain -- which starts the core's acquisition thread twice and writes it into the document twice. Unlike ``connect`` this does not register the node in ``_elements``, which is unchanged and deliberate: ``_elements`` holds what was *connected*, and ``_validate_has_signal`` says so. Args: node: Node or chain to add. """ super().add_node(self._wrap(node))
def _timeline_consumers(self) -> list: """Return every node in this pipeline that wants the timeline. Any node exposing ``attach_timeline`` is offered it, so sinks that stamp outgoing data can reach it as well as Sync nodes. Returns: Nodes to bind, including those nested inside chains. """ found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if callable( getattr(node, "attach_timeline", None) ) and not any(n is node for n in found): found.append(node) return found
[docs] def chain_of(self, node): """Return the chain carrying *node*, or None. The question a caller asks now that assembly is not the node's own business: a public node is not a chain, and the chain this pipeline built around it is found here. A hand-written chain answers for itself. Accepts a node this pipeline does not carry and answers None rather than raising: a test or a tool may reasonably ask about a node it has not connected yet, and an exception would make the question harder to ask than the reach-in it replaces. Args: node: A node handed to ``connect``, or one nested inside a chain that was. Returns: The chain, the node itself where it is already one, or None when this pipeline does not carry it. """ for element in self._elements: if element is node: return element internal = getattr(element, "internal_nodes", None) or [] if any(n is node for n in internal): return element # A chain the pipeline built holds its core whether or not it # run it, and under SERVER residency it does not: the scope # is not drawn and the device is not opened there. Walking # ``internal_nodes`` alone therefore answered None for a core # this pipeline is very much carrying, which is the one residency # where a caller most needs the answer. chain = self._wrappers.get(id(node)) if chain is not None and any(e is chain for e in self._elements): return chain return None
def _sources(self) -> list: """Return every source node in this pipeline. A source is identified by declaring a time base, which only Source subclasses do. Returns: Source nodes, including those nested inside chains. """ found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if getattr(node, "TIME_BASE", None) is not None and not any( n is node for n in found ): found.append(node) return found def _placement_consumers(self) -> list: """Return every node that places async inputs onto the grid. A separate sweep from the timeline one because the method has a separate name: a node that already defines ``attach_timeline`` would shadow the placing mixin's version, so placement asks for the timeline under its own name. Returns: Nodes to inform, including those nested inside chains. """ found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if hasattr(node, "attach_placement_timeline") and not any( n is node for n in found ): found.append(node) return found def _flattened_nodes(self) -> list: """Return every node this pipeline runs in, chains flattened. Returns: The nodes, each once, nested chains included. """ found = [] seen = set() def visit(element) -> None: internal = getattr(element, "internal_nodes", None) if internal: for node in internal: visit(node) elif id(element) not in seen: seen.add(id(element)) found.append(element) for element in self._elements: visit(element) return found def _attach_position_gate(self) -> None: """Tell every node which of its outputs carry positions. Decided from the pipeline here, before any node runs, because a source's first frames leave its ``Sync`` before the node that places them has set up; see ``placement.position_gate``. A pipeline with no node that may place carries nothing, so its nodes run exactly as before carried positions existed. """ nodes = self._flattened_nodes() candidates, carrying = position_gate(nodes) for node in nodes: attach = getattr(node, "attach_position_gate", None) if callable(attach): attach(carrying.get(id(node), ()), id(node) in candidates) def _record_failure(self, entry) -> None: """Remember the entry that ended the run. Called on ioiocore's monitoring thread, so it does as little as possible: storing a reference cannot fail, and anything that could would be happening on the only thread that reports failures at all. Args: entry: The failing log entry. """ self._failure = entry @property def failure(self): """The entry that ended the run, or None if it has not failed. ioiocore sets ERROR, stops the pipeline and only then calls the handler that records this, so a poll can see ERROR first. Until the handler has run -- or for good, if a raising ``stop()`` keeps it from running -- the entry is read from ioiocore's log instead. A failure therefore implies ERROR, but not yet STOPPED. Returns: A log entry carrying the message, the node and the source location. """ if self._failure is not None: return self._failure if self.get_condition() != ioc.Constants.Conditions.ERROR: return None return self._consumed_error() def _consumed_error(self): """The ERROR entry ioiocore's monitor stopped the run on. The monitor takes the logger's latest ERROR and marks it seen by its timestamp. The latest moves on if stopping logs another, so the entry is found by that mark, newest first. It falls back to the latest error when the marked entry has left the logger's 100-entry buffer, or when ioiocore keeps no mark. And an error logged after the monitor's read, within one clock tick of the entry it read, shares the mark and is reported instead; a tick is 15.6 ms on Windows under Python 3.10-3.12. Returns: A log entry, or None if no error was logged. """ error = ioc.Constants.LogTypes.ERROR latest = self.get_last_error() logger = getattr(self._imp, "_logger", None) marks = getattr(getattr(logger, "_imp", None), "_last_timestamp", None) mark = marks.get(error) if isinstance(marks, dict) else None if latest is None or mark is None or latest["timestamp"] == mark: return latest for entry in reversed(logger.get_by_type(error)): if entry["timestamp"] == mark: return entry return latest
[docs] def raise_if_failed(self) -> None: """Raise if the run has failed, naming the node and the cause. Overridden to name :attr:`failure`. ioiocore's names its latest error, a different entry once stopping has logged another. Raises: RuntimeError: If the pipeline is in an error condition. """ if self.get_condition() != ioc.Constants.Conditions.ERROR: return entry = self.failure error = RuntimeError("The pipeline failed.") if entry is not None: source = entry["source"] or {} parts = [str(entry["message"])] if source.get("instance"): parts.append(f"in node '{source['instance']}'") if source.get("summary"): parts.append(f"at {source['summary']}") error = RuntimeError("The pipeline failed: " + " ".join(parts)) cause = self._first_failure([node for node, _ in self._walk_nodes()]) if cause is not None: raise error from cause raise error
def _links(self) -> list: """Return every Link in this pipeline, including nested ones. Same traversal as the other sweeps, and nested for the same reason: a Link lives inside the chain of whatever node spans the residency boundary. Returns: Link nodes. """ from .core._private.link import Link found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if isinstance(node, Link) and not any( n is node for n in found ): found.append(node) return found def _first_device_handle(self): """Return a handle that can attest, or None. One challenge carries one attestation, so an edge holding several amplifiers answers with the first that has a handle. Sufficient under the rule that ships -- the run is attested if *any* source attested -- and worth revisiting only if the §5.4 matrix ever needs a per-device remote verdict. Returns: An amplifier handle, or None if this pipeline has none. """ for node in self._sources(): handle = getattr(node, "_device", None) if handle is not None: return handle return None def _provenance_consumers(self) -> list: """Return every node that wants this run's provenance record. Any node exposing ``attach_provenance`` is offered it -- every :class:`~gpype.Source` does, which is how it comes to merge its own ``INPUT`` in and publish the whole thing under ``Constants.Keys.PROVENANCE`` (LQ-P2). Same shape as the timeline and entitlement sweeps. Returns: Nodes to inform, including those nested inside chains. """ found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if callable( getattr(node, "attach_provenance", None) ) and not any(n is node for n in found): found.append(node) return found def _entitlement_consumers(self) -> list: """Return every node that wants to know the run's entitlement. Any node exposing ``attach_entitlement`` is offered it, which is how a sink learns whether its output must carry the mark. Same shape as the timeline sweep, and it reaches inside chains for the same reason: every sink is a chain wrapping a core. Returns: Nodes to inform, including those nested inside chains. """ found = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) for node in candidates: if callable( getattr(node, "attach_entitlement", None) ) and not any(n is node for n in found): found.append(node) return found def _validate(self) -> None: """Reject pipelines that cannot mean anything, before they run. Validity is not entitlement and not configuration: these are pipelines whose sources cannot be reconciled at all. Checked here, ahead of everything else, so the error names the real problem rather than surfacing later as a licence message or as data that merely looks wrong. Raises: ValueError: If sources with different time bases are mixed, if no source produces a continuously sampled stream, if the pipeline contains an illegal feedback loop, or if an input that inherits its timing is not connected. """ self._validate_time_bases() self._validate_has_signal() self._validate_no_illegal_cycles() self._validate_batch_only_nodes() self._validate_fittable_state() self._validate_inputs_connected() self._warn_unknown_chain_keys() self._warn_untapped_sources() self._warn_edge_named_by_nothing() self._warn_sources_every_edge_sends() self._warn_edge_ids_differing_in_case() def _wrapped_cores(self) -> list: """The world-facing node each chain of this pipeline carries. Returns: The cores, in element order; an element that is not one of the pipeline's chains contributes nothing. """ from .core._private import wrapping cores = [wrapping.core_of(element) for element in self._elements] return [core for core in cores if core is not None] def _warn_sources_every_edge_sends(self) -> None: """Say so when this edge opens a source the document leaves to all. A source that names no edge is built by every edge running the document, which is what a document written before edge ids says and still means. In a document that assigns other nodes to edges, several edges are the likely deployment, and then each one opens that source and sends its stream: two senders on one stream id. The broker keeps the first to write and refuses the rest (D-NODE-52), so the server's stream is one sender's -- but which edge's is a race, and every other edge's device was opened for nothing. Warned rather than refused: a document that assigns only its sinks and is run by one edge is a legitimate deployment, and an edge cannot count the edges that run beside it. Said on the edge, where the source is opened; the refusal is said on the server. """ from .core._private import assembly if LaunchConfig.get().residency != Constants.Residency.EDGE: return cores = self._wrapped_cores() named = {assembly.edge_id_of(core) for core in cores} - {None} if not named: return unassigned = sorted( core.name for core in cores if assembly.kind_of(core) == assembly.SOURCE and assembly.edge_id_of(core) is None ) if not unassigned: return one = len(unassigned) == 1 names = ", ".join(repr(name) for name in unassigned) ids = ", ".join(repr(edge_id) for edge_id in sorted(named)) self.log( f"{names} {'names' if one else 'name'} no edge, so every edge " f"that runs this document opens {'it' if one else 'them'} " f"and sends {'its stream' if one else 'their streams'}, while " f"the document assigns other nodes to {ids}. Run by two edges " f"that is two senders on one stream: the server keeps the " f"edge that connects first and refuses the other. Give each " f"source an edge_id.", type=Constants.LogTypes.WARNING, ) def _warn_edge_ids_differing_in_case(self) -> None: """Say so when two edge ids in play differ only in case. Ids are matched exactly (D-NODE-49), so ``'Phone'`` and ``'phone'`` are two edges, and a node assigned the one is built by no process running as the other. When this edge's id is named by some other node, :meth:`_warn_edge_named_by_nothing` stays quiet, and the node that missed is simply not built anywhere -- no writer, no file, no message. Surrounding whitespace, the other near miss, is refused wherever an id is written. Asked on an edge, with its own id among the spellings, and on a server, which sees the whole document; standalone ignores ids. """ from .core._private import assembly launch = LaunchConfig.get() if launch.residency == Constants.Residency.STANDALONE: return spellings: dict = {} for core in self._wrapped_cores(): edge_id = assembly.edge_id_of(core) if edge_id is not None: owners = spellings.setdefault(edge_id.casefold(), {}) owners.setdefault(edge_id, []).append(repr(core.name)) if launch.edge_id is not None: owners = spellings.setdefault(launch.edge_id.casefold(), {}) owners.setdefault(launch.edge_id, []).append("this edge") for owners in spellings.values(): if len(owners) < 2: continue described = ", ".join( f"{edge_id!r} ({', '.join(owners[edge_id])})" for edge_id in sorted(owners) ) self.log( f"edge ids {described} differ only in case. Ids are " f"matched exactly, so these are different edges: a node " f"assigned one of them is built by no edge running as " f"another. Spell each edge one way.", type=Constants.LogTypes.WARNING, ) def _warn_edge_named_by_nothing(self) -> None: """Say so when this edge's id is one the document never uses. An edge launched with a mistyped ``--edge-id``, or with none against a document that assigns every node, builds only the nodes that name no edge -- possibly none at all. It then starts, reports healthy and sends nothing, and the only symptom is a stream that never arrives at the server. Warned rather than refused: an edge that runs just the unassigned nodes -- a viewer that draws the scopes every edge draws -- is a legitimate deployment, and the same document serves it. """ from .core._private import assembly, wrapping launch = LaunchConfig.get() if launch.residency != Constants.Residency.EDGE: return named = set() for element in self._elements: edge_id = assembly.edge_id_of(wrapping.core_of(element)) if edge_id is not None: named.add(edge_id) if not named or launch.edge_id in named: return this = ( f"edge {launch.edge_id!r}" if launch.edge_id is not None else "an edge with no edge id" ) ids = ", ".join(repr(edge_id) for edge_id in sorted(named)) self.log( f"this process is {this}, and no node in this pipeline is " f"assigned to it: the document assigns nodes to {ids}. It " f"builds only the nodes that name no edge. If this process " f"holds a device, check its --edge-id (or {ENV_EDGE_ID}).", type=Constants.LogTypes.WARNING, ) def _warn_untapped_sources(self) -> None: """Name any source this run's ``save_as``/``load_from`` misses. Both are inserted by ``raw.source_stage``, which every chain the pipeline builds calls. A source inside a chain written by hand that does not call it has no such point at all, so ``save_as`` contributes no file for it and ``load_from`` leaves its live core in place. Both halves matter, and the second one more than it looks. Under ``save_as``, a manifest that simply does not mention a stream is indistinguishable from a manifest of a pipeline that never had it. Under ``load_from``, that source goes on acquiring live data beside the replayed streams -- and since a replay core declares WALL_CLOCK (D-TIME-28), ``_validate_time_bases`` no longer refuses the mixture the way it refuses a file reader beside a live source. Nothing else would say so. Warned rather than refused: an unrecorded source is not a reason to take away a working pipeline. But it must be said, because the promise of both parameters is *every* source, and the run that discovers otherwise is the one being replayed. """ config = LaunchConfig.get() if not config.save_as and not config.load_from: return from .sources.base.raw import _RawReplayCore, _RawTap staged = set() for element in self._elements: internal = getattr(element, "internal_nodes", None) or [] if any( isinstance(node, (_RawTap, _RawReplayCore)) for node in internal ): staged.update(id(node) for node in internal) missing = sorted( naming.public_name(type(node).__name__) for node in self._sources() if id(node) not in staged ) if not missing: return names = ", ".join(missing) if config.save_as: self.log( f"save_as does not cover {names}: a source that is not a " f"chain has no point to tap, so this run's manifest will " f"not mention it and a replay of it will have no such " f"stream. Everything else in this pipeline is recorded.", type=Constants.LogTypes.WARNING, ) else: self.log( f"load_from does not cover {names}: a source that is not " f"a chain keeps its own core, so it is acquiring live " f"data beside the replayed streams. Nothing refuses that " f"mixture -- a replay core is WALL_CLOCK, like a live " f"source -- so the two have no instant in common beyond " f"the one this run happens to give them.", type=Constants.LogTypes.WARNING, ) def _validate_batch_only_nodes(self) -> None: """Refuse a node that only means something offline. ``Apply`` and anything else declaring ``BATCH_ONLY`` runs a plain callable, and D-BATCH-56 keeps that off the realtime path: an arbitrary function has no bound on its execution time against a one-frame budget, and purity is not enforceable there. Refused here rather than on the first cycle, and the difference is not cosmetic. By cycle time a source may already hold a device and a writer may already have created a file, so a refusal then leaves work to undo; and ``start()`` would have returned normally, which is the failure ``_await_setup`` exists to stop happening. Raises: ValueError: If the pipeline is not in batch mode and contains a node that requires one. """ batch_only = [ node for node, _ in self._walk_nodes() if getattr(type(node), "BATCH_ONLY", False) ] if not batch_only: return # Asked after the nodes, so a pipeline with none of them never # pays for this -- and so a pipeline with no source at all # still fails with "no source" rather than with a mode error. mode = self.execution_mode() if mode == Constants.ExecutionMode.BATCH: return names = ", ".join( sorted(naming.public_name(type(n).__name__) for n in batch_only) ) raise ValueError( f"{names} runs only in a batch pipeline, and this one is " f"'{mode}'. A plain function has no bound on how long it " f"takes, and one frame is 4 to 16 ms -- so it is offered " f"offline only. Either drive this with a batch source -- " f"CsvReader(..., mode='batch') and run() -- or write a node " f"class with a step() for the realtime path." ) def _refuse_batch_without_run(self) -> None: """Refuse a batch pipeline that start() was called on directly. run() drives a batch source's cycles on the caller's thread; start() alone never does, so the pipeline reported RUNNING and Healthy, logged nothing, and processed nothing. The mirror of run()'s refusal of a realtime pipeline. Kept out of :meth:`_validate`, which judges the pipeline: this judges how it is being driven, and run() reaches it through start(). Raises: ValueError: If the pipeline is batch and run() is not driving it. """ # A pipeline with no source of its own -- a server, or a Link # receiver -- has no mode to judge, and one whose sources # disagree is not this check's to refuse: start() never # consulted the mode for either, and still does not. if getattr(self, "_batch_driven", False) or not self._sources(): return try: mode = self.execution_mode() except ValueError: return if mode != Constants.ExecutionMode.BATCH: return raise ValueError( "start() runs a realtime pipeline, and this one is 'batch': " "its source is read once, as fast as possible, and only " "run() drives it. Call run(), which returns the result -- or, " "for a recording reader, leave mode at its default to replay " "the file against the clock with start() and stop()." ) def _validate_fittable_state(self) -> None: """Refuse a REALTIME start with a fittable node that has no state. Implements D-BATCH-33: a fittable node fits only in a batch run (:meth:`fit`), so one with nothing loaded is a deployment mistake, not a training run in progress. Refused here for the same reason as :meth:`_validate_batch_only_nodes`: by cycle time a source may already hold a device. A batch run is exempt -- that is exactly how a first fit happens -- and a batch run that is not fitting and finds no state fails later, from inside the node's own ``process``. Raises: ValueError: If the pipeline is not batch and holds a fittable node with no state, naming it. """ unfit = [node for node in self._fittable_nodes() if node.state is None] if not unfit: return mode = self.execution_mode() if mode == Constants.ExecutionMode.BATCH: return names = ", ".join( sorted( f"{naming.public_name(type(n).__name__)} ('{n.name}')" for n in unfit ) ) raise ValueError( f"{names} has no fitted state, and this pipeline is " f"'{mode}'. Fit it first -- Pipeline.fit() against a batch " f"source -- or deserialize a document whose artifacts " f"section already carries one for it." ) def _validate_inputs_connected(self) -> None: """Refuse an input that inherits its timing and is not connected. An input port declared ``INHERITED`` -- ``Collector``'s, ``Router``'s, ``Equation``'s, ``Trigger``'s trigger input -- takes its timing from what connects to it. Unconnected, it has none, and ioiocore refuses it at start with ``INHERITED timing not allowed on INode input ports during start()``, which names neither the node nor the port. The rule is ioiocore's; this names what it refuses (D-BATCH-74). Walks the pipeline's own nodes, which ``add_node`` reaches and ``_elements`` does not, and every chain's internal nodes. Raises: ValueError: If an input port's timing is still inherited, naming the port and its node. """ from .core._private import wrapping ports_key = Constants.Keys.INPUT_PORTS name_key = OPort.Configuration.Keys.NAME loose: list = [] seen: set = set() def visit(node, core) -> None: internal = getattr(node, "internal_nodes", None) if internal is not None: for inner in internal: visit(inner, core) return config = getattr(node, "config", None) or {} for port in config.get(ports_key) or []: name = port.get(name_key) try: timing = node.get_input_port(name).timing except Exception: continue if timing != Constants.Timing.INHERITED: continue if (id(node), name) in seen: continue seen.add((id(node), name)) # The node the author wrote: the core a chain of ours # carries, or the node itself. owner = core if core is not None else node loose.append(f"'{name}' of {naming.node_label(owner)}") for element in list(getattr(self._imp, "_nodes", None) or []): visit(element, wrapping.core_of(element)) if not loose: return one = len(loose) == 1 raise ValueError( f"{'input' if one else 'inputs'} {', '.join(loose)} " f"{'is' if one else 'are'} not connected. An input like that " f"takes its timing from what connects to it, so unconnected " f"it has none and the pipeline cannot start. Connect it, or " f"leave its node out of the pipeline." ) def _warn_unknown_chain_keys(self) -> None: """Warn about configuration keys a chain does not recognise. ``Node.__init__`` already does this for nodes, and a chain never reaches it: a chain is not a Node, and a stray keyword lands on *its* configuration rather than on the core's. That is the case that costs debugging time. ``Generator(amplitude=100.0)`` instead of ``signal_amplitude`` is accepted, stored, and survives re-serialisation, while the pipeline reports Running and Healthy and emits exact zeros -- three debugging cycles in one session, recorded in devdoc/archive/TODO.md. Warned, not refused, on the owner's instruction: refusing would stop every stored document carrying a stray key from loading at all, and a key nothing reads is not a reason to refuse to run. The rule is shared -- ``common.config_keys`` -- with the node check and with the runtime's answer to "is this document valid", so all three say the same thing about the same document rather than agreeing until one of them changes. """ for element in self._elements: # A node, not a chain: Node.__init__ has already looked, and # warning twice about one key reads as two problems. if getattr(element, "internal_nodes", None) is None: continue config = getattr(element, "config", None) unknown = config_keys.unrecognised_keys( type(element), config if isinstance(config, dict) else {} ) if not unknown: continue message = config_keys.describe(type(element), unknown) logger = getattr(element, "log", None) if callable(logger): logger(message, type=Constants.LogTypes.WARNING) else: # pragma: no cover - a chain that cannot log yet self.log(message, type=Constants.LogTypes.WARNING) def _validate_time_bases(self) -> None: """Reject a pipeline mixing replayed and live sources. Raises: ValueError: If sources with different time bases are mixed. """ bases = {} for node in self._sources(): bases.setdefault(node.TIME_BASE, []).append(type(node).__name__) if len(bases) > 1: recorded = ", ".join( sorted(bases.get(Constants.TimeBase.RECORDED, [])) ) live = ", ".join( sorted(bases.get(Constants.TimeBase.WALL_CLOCK, [])) ) raise ValueError( f"a file source cannot be combined with live sources: " f"{recorded} replays stored data while {live} " f"advance{'s' if len(bases.get(Constants.TimeBase.WALL_CLOCK, [])) == 1 else ''} " # noqa: E501 f"with the host clock, so there is no single instant " f"their samples both refer to." ) def _validate_has_signal(self) -> None: """Reject a pipeline built only from event sources. Every position in a pipeline comes from the master timeline, and the master timeline comes from a continuously sampled source: it is what defines the sample grid that everything else is placed on. An event has no position of its own -- a keypress is a host timestamp, nothing more -- so with no continuous source there is no grid to place it on, and placement is where an event acquires a position at all. Such a pipeline does not fail. It starts, reports healthy, cycles once at start and then never again, and writes nothing. That is the worst possible way for it to behave, because the author has no symptom to work from: measured on a Marker feeding a Router, one cycle and no values, with every node reporting Healthy. Refusing at start() turns that into a sentence naming the cause. Raises: ValueError: If sources exist but none of them is continuous. """ if not self._sources(): # Nothing was connected, or everything reached the pipeline # through add_node(), which does not register. Not this # check's business either way. return timing_key = OPort.Configuration.Keys.TIMING ports_key = ioc.ONode.Configuration.Keys.OUTPUT_PORTS # Walked per element rather than over _sources(), so the message # can name what the author wrote: the node a chain carries, not # the chain -- "SourceChain" is no help to somebody who wrote # gp.Marker(). from .core._private import wrapping event_only = [] for element in self._elements: candidates = [element] internal = getattr(element, "internal_nodes", None) if internal: candidates.extend(internal) # The core too, run here or not. An edge that owns only a # Marker runs no continuous source of its own -- the # amplifier belongs to another edge -- and the grid its # markers are placed on is the server's, fed by that other # edge. The question is whether the *document* has a signal, # and a core this process does not run is still in it. core = wrapping.core_of(element) if core is not None and not any(c is core for c in candidates): candidates.append(core) cores = [ node for node in candidates if getattr(node, "TIME_BASE", None) is not None ] if not cores: continue for core in cores: ports = core.config.get(ports_key) or [] timings = [port.get(timing_key) for port in ports] # Anything not declared ASYNC counts as continuous. The # test is deliberately this way round: a false refusal # breaks a working pipeline, while a false acceptance # only leaves the behaviour that was there before. if any(t != Constants.Timing.ASYNC for t in timings): return owner = wrapping.core_of(element) or element event_only.append(naming.public_name(type(owner).__name__)) named = sorted(set(event_only)) names = ", ".join(named) emit = "emits" if len(named) == 1 else "emit" raise ValueError( f"this pipeline has no continuously sampled source: " f"{names} {emit} events, and an event has no position until " f"something places it on a sample grid. The grid comes from " f"a continuous source, so there is nothing here to place " f"events on and the pipeline would run without producing " f"any output. Add a data source -- an amplifier, Generator " f"or CsvReader -- alongside the event source." ) def _validate_no_illegal_cycles(self) -> None: """Reject a pipeline containing an illegal feedback loop. Runs here for a pipeline assembled by :meth:`deserialize`, which never passes through :meth:`connect` at all. For an ordinarily built pipeline it is mostly a second look at rules already applied per-edge -- see :meth:`connect` -- with one rule that only this call can apply: a loop must be fed from outside itself, which is a property of the finished pipeline and cannot be judged while edges are still arriving. Raises: ValueError: If the pipeline contains a loop refused by :meth:`_describe_component_violation`. """ violation = self._cycle_violation() if violation is not None: raise ValueError(violation) def _cycle_owner_map(self) -> dict: """Map every declared input port to the element that owns it. Rebuilt on every call rather than cached, because the answer changes on every :meth:`connect` and nothing here tracks that invalidation. The pipelines this runs against are, at most, dozens of nodes -- nothing next to actually running a pipeline. Returns: ``id(port) -> element``, for every declared input port of every registered element that exposes one. An element that cannot be introspected (a malformed config, a port that will not resolve) is skipped rather than raised on -- this check must never be the reason an otherwise-working connect() fails. Every skip is logged, though: an edge this map misses is an edge the loop check cannot see, so a swallowed error degrades the detector towards "no loops found" -- exactly the silent acceptance it exists to prevent. The reason has to stay recoverable from a debug log instead of vanishing. """ ip_key = ioc.INode.Configuration.Keys.INPUT_PORTS name_key = ioc.IPort.Configuration.Keys.NAME owners: dict = {} for element in self._elements: getter = getattr(element, "get_input_port", None) if getter is None: # Not a failure: every gpype source is an OChain, which # has no input ports at all, so no edge can end at it. continue try: specs = element.config.get(ip_key) or [] except Exception as error: log.debug( "cycle check: cannot read the input ports of %s " "(%s): %s -- edges into it stay invisible to the " "loop check", getattr(element, "name", "?"), type(element).__name__, error, ) continue for spec in specs: name = spec.get(name_key) try: port = getter(name) except Exception as error: log.debug( "cycle check: cannot resolve input port %r of " "%s: %s -- edges into that port stay invisible " "to the loop check", name, getattr(element, "name", "?"), error, ) continue owners[id(port)] = element return owners def _cycle_adjacency(self) -> dict: """Return the directed connections built by connect() so far. Returns: ``id(element) -> [downstream elements]``, one entry per outgoing edge -- an element with a fan-out of two appears twice, pointing at each of its consumers. Elements that cannot be introspected are skipped and logged, for the reason given in :meth:`_cycle_owner_map`. """ op_key = ioc.ONode.Configuration.Keys.OUTPUT_PORTS name_key = ioc.IPort.Configuration.Keys.NAME owners = self._cycle_owner_map() adjacency: dict = {} for element in self._elements: getter = getattr(element, "get_output_port", None) if getter is None: # Not a failure: a sink or a scope is an IChain, which # has no output ports, so no edge can start at it. continue try: specs = element.config.get(op_key) or [] except Exception as error: log.debug( "cycle check: cannot read the output ports of %s " "(%s): %s -- edges out of it stay invisible to the " "loop check", getattr(element, "name", "?"), type(element).__name__, error, ) continue for spec in specs: name = spec.get(name_key) try: port = getter(name) except Exception as error: log.debug( "cycle check: cannot resolve output port %r of " "%s: %s -- edges out of that port stay " "invisible to the loop check", name, getattr(element, "name", "?"), error, ) continue for connected in list(getattr(port, "_connected_ports", ())): downstream = owners.get(id(connected)) if downstream is not None: adjacency.setdefault(id(element), []).append( downstream ) return adjacency def _cycle_components(self, adjacency: dict) -> list: """Return every strongly connected component holding a cycle. Classifying per component rather than per cycle is the whole point of this detector, and the reason it stopped enumerating cycles. Enumeration cannot be made correct here: a depth-first search that slices a path out whenever it meets a node already on its stack skips edges into finished nodes, so it reports at most one cycle per branch and never sees a second cycle sharing nodes with the first. Measured on ``A -> B -> C -> A`` with B a declaring :class:`~gpype.Delay`, then adding ``A -> C``: the legal loop was reported, the illegal ``A -> C -> A`` -- no breaker in it, the exact deadlock this check exists to prevent -- was not, and connect() accepted it. A component contains *every* cycle through its nodes, so a property established for the component holds for all of them, with nothing left to enumerate. Tarjan's algorithm, written iteratively because recursion depth here is the pipeline's depth and a pipeline may be deeper than the interpreter's limit; linear in nodes plus edges. Args: adjacency: The connections from :meth:`_cycle_adjacency`. Returns: One list of elements per component that contains at least one cycle -- either more than one member, or a single member with an edge to itself. A component with one member and no self-edge is just an ordinary node and is left out. Ordered by each component's earliest member in ``_elements``, so which loop gets reported first does not wander between runs. """ index: dict = {} low: dict = {} on_stack: dict = {} pending: list = [] components: list = [] counter = 0 for root in self._elements: if id(root) in index: continue index[id(root)] = low[id(root)] = counter counter += 1 pending.append(root) on_stack[id(root)] = True # Each frame is a node and its not-yet-followed successors; # the iterator is what makes resuming a frame cheap. work = [(root, iter(adjacency.get(id(root), ())))] while work: node, successors = work[-1] descended = False for downstream in successors: did = id(downstream) if did not in index: index[did] = low[did] = counter counter += 1 pending.append(downstream) on_stack[did] = True work.append((downstream, iter(adjacency.get(did, ())))) descended = True break if on_stack.get(did): low[id(node)] = min(low[id(node)], index[did]) if descended: continue work.pop() if work: parent = work[-1][0] low[id(parent)] = min(low[id(parent)], low[id(node)]) if low[id(node)] == index[id(node)]: members = [] while True: member = pending.pop() on_stack[id(member)] = False members.append(member) if member is node: break components.append(members) cyclic = [] for members in components: if len(members) > 1: cyclic.append(members) continue node = members[0] if any(d is node for d in adjacency.get(id(node), ())): cyclic.append(members) last = len(self._elements) order = {id(e): i for i, e in enumerate(self._elements)} cyclic.sort(key=lambda ms: min(order.get(id(m), last) for m in ms)) return cyclic def _first_cycle(self, nodes: set, adjacency: dict) -> list: """Return the first cycle lying inside nodes, or an empty list. A depth-first search restricted to ``nodes``, colouring an element grey while it sits on the current path and black once fully explored; an edge into a grey element closes a cycle. This doubles as the acyclicity test the rules below need -- an empty result means ``nodes`` induces no cycle at all -- and it is linear in nodes plus edges, so running it on every connect() costs nothing worth measuring. The path holds *nodes*, not edges. An earlier version of this detector kept an edge stack and named the wrong entry node for a self-loop (``Equation -> Equation`` was reported as ``Generator -> Equation -> Generator``), because an edge stack records where a node was reached *from*, not the node reached. Args: nodes: Ids of the elements the search may enter; an edge to anything else is treated as if it did not exist. adjacency: The connections from :meth:`_cycle_adjacency`. Returns: The cycle's elements in traversal order, the element where it closes first, or an empty list if there is no cycle. """ grey, black = 1, 2 colour: dict = {} for root in self._elements: if id(root) not in nodes or colour.get(id(root)): continue path: list = [root] colour[id(root)] = grey work = [iter(adjacency.get(id(root), ()))] while work: descended = False for downstream in work[-1]: did = id(downstream) if did not in nodes or colour.get(did) == black: continue if colour.get(did) == grey: start = next( i for i, n in enumerate(path) if id(n) == did ) return path[start:] colour[did] = grey path.append(downstream) work.append(iter(adjacency.get(did, ()))) descended = True break if descended: continue colour[id(path.pop())] = black work.pop() return [] def _cycle_through(self, node, nodes: set, adjacency: dict) -> list: """Return one cycle that starts and ends at node. Only ever used to name a loop in a refusal, so the path a message prints begins at the node that message blames. Runs on the refusal path alone -- an accepted pipeline never calls it. Args: node: Element the returned path must start at. nodes: Ids of the elements the search may enter. adjacency: The connections from :meth:`_cycle_adjacency`. Returns: The cycle's elements, ``node`` first, or an empty list if no cycle through ``node`` lies inside ``nodes`` -- which cannot happen for a member of a cyclic component, since every node of one lies on a cycle with every other. """ path: list = [node] seen = {id(node)} def walk(current) -> bool: for downstream in adjacency.get(id(current), ()): did = id(downstream) if did == id(node): return True if did not in nodes or did in seen: continue seen.add(did) path.append(downstream) if walk(downstream): return True path.pop() return False return path if walk(node) else [] def _async_input_of(self, node) -> Optional[str]: """Return the name of node's first connected asynchronous input. Args: node: Candidate node inside a feedback loop. Returns: The port name, or None if every connected input port is synchronous. An input declared ASYNC but left unconnected does not count -- nothing arrives on it, so it plays no part in this loop's timing. A port that cannot be introspected is skipped and logged, for the reason given in :meth:`_cycle_owner_map`. """ ip_key = ioc.INode.Configuration.Keys.INPUT_PORTS name_key = ioc.IPort.Configuration.Keys.NAME getter = getattr(node, "get_input_port", None) if getter is None: return None try: specs = node.config.get(ip_key) or [] except Exception as error: log.debug( "cycle check: cannot read the input ports of %s (%s): " "%s -- treated as fully synchronous, so an async input " "on it would not be refused", getattr(node, "name", "?"), type(node).__name__, error, ) return None for spec in specs: name = spec.get(name_key) try: port = getter(name) except Exception as error: log.debug( "cycle check: cannot resolve input port %r of %s: " "%s -- treated as synchronous, so an async timing " "on it would not be refused", name, getattr(node, "name", "?"), error, ) continue if not port.is_connected(): continue if port.timing == Constants.Timing.ASYNC: return name return None @staticmethod def _cycle_path_text(members: list) -> str: """Render a loop as the round trip its author wrote. Args: members: The loop's elements, entry node first. Returns: ``"A -> B -> A"``, closing back on the entry node so the reader can see it is a loop and not a chain. """ return " -> ".join(m.name for m in members + [members[0]]) def _describe_component_violation( self, members: list, adjacency: dict, require_feed: bool = True ) -> Optional[str]: """Classify one cyclic component and refuse what cannot run. Three shapes are refused, all measured rather than merely reasoned about: 1. Some cycle in the component has no breaker in it -- no :class:`~gpype.Delay` that declares its own context, which is the only form that can emit a frame before it has seen one. Nothing in that cycle ever fires, since every node is waiting on every other, so the pipeline reports Healthy, produces nothing, and grows its input queues without bound. A plain ``gp.Delay(num_samples=...)`` does not count: it derives its context from its input, so it reports ``BREAKS_CYCLES`` False. Tested by deleting the breakers and asking whether anything cyclic survives, which is exact: every cycle passes through a breaker if and only if removing the breakers leaves the rest acyclic. A component that shares nodes between a broken and an unbroken loop is the case that makes this worth doing -- see :meth:`_cycle_components`. 2. No edge enters the component from outside it. The loop is then its own only producer as well as its own only consumer: it can never start, and if it somehow did, recursion would run until the process stack overflows (0xC00000FD on Windows, not a catchable RecursionError). Unlike the other two, this is a property of the *finished* pipeline rather than of any one edge: closing a loop before wiring the feed that drives it is an ordinary way to write the same pipeline. So it is checked at start() and skipped while edges are still being added -- see ``require_feed``. 3. Some node in the component has a connected asynchronous input. Such a node fires as soon as its synchronous inputs have data, so with the loop edge as its only synchronous input it becomes its own clock: measured at roughly 43000 traversals/s on a 250 Hz pipeline, 48% of one core, reporting Healthy throughout. A legal loop is therefore the complement: every cycle in it passes through a breaker, something outside feeds it, and every connected input of every node in it is synchronous. Args: members: The component's elements, in no useful order. adjacency: The connections from :meth:`_cycle_adjacency`. require_feed: Whether to apply rule 2. False while the pipeline is still being assembled, where the feed may simply not be connected yet. Returns: The refusal text naming the offending loop, or None if this component is legal. """ member_ids = {id(m) for m in members} breakers = { id(m) for m in members if getattr(m, "BREAKS_CYCLES", False) } unbroken = self._first_cycle(member_ids - breakers, adjacency) if unbroken: path = self._cycle_path_text(unbroken) return ( f"the connection closes a feedback loop: {path}. g.Pype " f"executes on data arrival, and a node fires only once " f"every one of its inputs has a frame, so a loop with " f"nothing in it never fires at all: the pipeline would " f"report Healthy, produce nothing, and grow its input " f"queues without bound. Insert a gp.Delay that " f"declares its own context -- gp.Delay(num_samples=1, " f"channel_count=..., sampling_rate=...) -- into this " f"loop: it emits one declared initial frame before it " f"has seen input, which is what lets the first cycle " f"happen, and makes the loop delay exactly num_samples " f"samples. A plain gp.Delay(num_samples=...) does not " f"count, because it derives its context from its input " f"and so cannot exist before the loop it would break. " f"A breaker elsewhere does not cover this loop either, " f"not even one in a loop sharing nodes with it: every " f"loop needs a breaker of its own." ) fed_from_outside = True if require_feed: fed_from_outside = False for source_id, downstream in adjacency.items(): if source_id in member_ids: continue if any(id(d) in member_ids for d in downstream): fed_from_outside = True break if not fed_from_outside: cycle = self._first_cycle(member_ids, adjacency) or members path = self._cycle_path_text(cycle) return ( f"the feedback loop {path} has no input from outside " f"the loop. Nothing outside it feeds any of its nodes, " f"so the loop is both its own only producer and its " f"own only consumer: it can never start, and if it ever " f"did it would recurse on one thread until the process " f"stack overflows. Feed one node in the loop from a " f"source outside it." ) # Walked in _elements order rather than in the order Tarjan # popped the component, so the node a message blames is the # first one the author wrote, not an implementation detail. for node in (e for e in self._elements if id(e) in member_ids): port_name = self._async_input_of(node) if port_name is None: continue cycle = self._cycle_through(node, member_ids, adjacency) or members path = self._cycle_path_text(cycle) return ( f"the feedback loop {path} is closed at " f"{node.name}, whose input port '{port_name}' " f"is asynchronous. A node with an asynchronous " f"input fires as soon as its synchronous inputs " f"have data, so with the loop edge as the only " f"synchronous input the loop becomes its own " f"clock: measured at 43000 traversals per second " f"on a 250 Hz pipeline, reporting Healthy " f"throughout. Every input port of every node in a " f"loop must be synchronous." ) return None def _cycle_violation(self, require_feed: bool = True) -> Optional[str]: """Return the message for the first illegal loop, or None. Args: require_feed: Whether a loop must already be fed from outside itself. False while the pipeline is still being assembled -- see :meth:`_describe_component_violation`. Returns: Refusal text naming the offending loop, or None if every cycle in the pipeline built so far is legal (including none at all). """ adjacency = self._cycle_adjacency() for members in self._cycle_components(adjacency): message = self._describe_component_violation( members, adjacency, require_feed=require_feed ) if message is not None: return message return None def _reset_run_state(self) -> None: """Let every node set itself up again on the next run. ioiocore gates ``setup()`` on a per-node counter and resets that counter only when the previous run ended in ERROR. After an ordinary ``stop()`` it therefore never re-runs, and a second run is not a second run at all: measured on a Generator to CsvWriter pipeline, run 2 created no file and wrote no rows while ``get_condition()`` reported Healthy and the node counters climbed. Every sink behaves that way, because each one skips its work on the handle its own ``stop()`` set to None. So this is not new behaviour -- restart-after-error already re-runs setup(). It removes the inconsistency between the two restart paths, in the direction that works. Two things make it safe, and both were established by measurement rather than reasoning: It resets **ioiocore's own node list**, not ``self._elements``. ``_elements`` holds what passed through ``connect()``, so a node added by ``add_node()`` alone is invisible to it -- and a *partial* reset does not fail loudly, because a cleared input context is silently backfilled from an un-reset upstream port. Run 2 would set itself up against run 1's context and look plausible. It stays inside the caller's ``not RUNNING`` guard. ioiocore deliberately makes ``start()`` on a running pipeline a no-op; resetting there would re-run ``setup()`` mid-recording, reopen the *same* file path and truncate what had been captured, and destroy and recreate an LSL outlet under every connected consumer. """ imp = getattr(self, "_imp", None) for node in list(getattr(imp, "_nodes", None) or []): reset = getattr( getattr(node, "_imp", None), "reset_run_state", None ) if callable(reset): try: reset() except Exception: # A node that cannot reset must not prevent the others # from doing so, nor block the start. pass def _elect_master(self, timeline) -> None: """Settle the master-timeline election before any data flows. Candidacy is a construction-time property, so the election does not have to wait for anyone to produce a block -- and it must not. A source that claims on its first block wins by starting fastest, and a synthetic source always starts faster than hardware that has to open a device: measured against a g.HIamp, a 512 Hz Generator took the timeline every time and the amplifier's stronger claim arrived after the election had already frozen. Claims are offered strongest first, so the timeline's own displacement rule never has to run and the outcome depends only on which sources are present. Args: timeline: The pipeline's timeline manager. """ candidates = [] for node in self._sources(): declare = getattr(node, "master_candidacy", None) candidacy = declare() if callable(declare) else None if candidacy is None: continue rate, priority = candidacy candidates.append((int(priority), float(rate), node)) # Strongest first: highest priority, then the better clock. The # node is not part of the sort key -- nodes are not orderable, and # ties are genuinely interchangeable. candidates.sort(key=lambda entry: (entry[0], entry[1]), reverse=True) for priority, rate, node in candidates: timeline.claim_master(node, rate, priority) def _resolve_entitlement(self) -> Entitlement: """Resolve both gates for this run and latch the outcome. Runs after the sources exist and before any node starts, which is the only window that works: a handle exists at construction, but a sink writes its header at start() -- so resolving later would let a Device-tier run stamp an evaluation mark, or the reverse, silently. Latches for the run, not for the process: ``stop()`` releases the verdict, so a second ``start()`` resolves again. That is the intended split. Re-resolving *mid-run* would let a network blip or a radio drop downgrade a recording in progress, and a sink has already written its header by then -- whereas a restart is a new recording writing new files, so a device unplugged in between should be reflected in what the next one claims. Returns: The combined verdict. """ current = entitlement_of(self) if current is not None: return current self._licence_unverified = None residency = LaunchConfig.get().residency built = getattr(self, "_built_residencies", set()) if built and built != {residency}: verdict = Entitlement( Permission.ABORT, f"this pipeline was built for {', '.join(sorted(built))} " f"residency and is starting as {residency}; a pipeline " f"runs under the residency it was built for", ) set_entitlement(self, verdict) return verdict # A verdict supplied from outside wins, and is taken exactly # once. An embedder that established entitlement by means this # process cannot repeat -- a runtime that challenged an admin # machine for a signed licence attestation and verified it -- # has a conclusion this resolution could not reach on its own: # licence.evaluate() asks about *this* machine, and a container # holds no licence. See D-ENT-61. The mark the data itself carries # is not the supplier's to lift (D-ENT-97), so it still applies. supplied = take_supplied(self) # A supplied verdict carries a conclusion, not the evidence: it # never stands in for a device that answered (D-ENT-98). self._entitlement_supplied = supplied is not None if supplied is not None: data_permission, data_reason = self._data_mark() if data_permission < supplied.permission: supplied = Entitlement( data_permission, data_reason, licence=supplied.licence, attestation=supplied.attestation, attestation_reason=getattr( supplied, "attestation_reason", "" ), ) set_entitlement(self, supplied) return supplied # One query for both questions, so they cannot disagree. frozen = licence.is_frozen_deployment() state = licence.query_licence() licence_permission, licence_reason = licence.evaluate( frozen, residency, state ) # A deployment whose licence cannot be determined runs only if a # g.tec device attests (D-ENT-100); the detail, or None. self._licence_unverified = licence.unverified(frozen, residency, state) # Ask each amplifier to open its handle first. Every source # opens its device inside its own start(), which runs after this # -- so without this loop every handle below is None and the # challenge list is empty by construction. Measured 2026-09-06 # on a BCI Core-8: zero challenges issued during a real start(), # while the same handle challenged by hand answered # verified=True. See AmplifierSource.open_for_attestation, and # _begin_deferred_device_check's docstring, which already # describes the behaviour this restores. # # Sources that do not implement the hook are unchanged: they # contribute no handle, exactly as before. # Guarded here as well as in each implementation. A device that # will not open is a connection problem, and the source's own # start() reports it with its own error type and message. Letting # it escape *here* would turn an unreachable amplifier into an # entitlement failure -- a confusing error, raised from the wrong # gate, about the wrong thing. for node in self._sources(): opener = getattr(node, "open_for_attestation", None) if callable(opener): try: opener() except Exception as error: # noqa: BLE001 self.log( f"could not open a device before the entitlement " f"gate, so this run is unattested: {error}", type=ioc.Constants.LogTypes.WARNING, ) results = [] for node in self._sources(): handle = getattr(node, "_device", None) if handle is not None: results.append(attestation.attest_device(handle)) results.extend(self._remote_attestation_results()) attest_permission, attest_reason = attestation.evaluate(results) # A device that answered here is held by this run, so it cannot # answer the presence probe's challenge too; it has answered # this one (D-ENT-98). for result in results: if getattr(result, "verified", False): device_probe.vouch(getattr(result, "serial", "")) # A held licence lifts the device gate's mark, and only its mark # (D-ENT-96): a licensed run is not marked for want of an # amplifier. device_permission = lifted(attest_permission, licence_permission) device_reason = attest_reason # The owner: never "without our devices AND without a license". A # frozen standalone application's device check enforces it after # start(); anywhere else nothing would, so it is refused here. if ( self._licence_unverified is not None and attest_permission is not Permission.FULL and not device_required(frozen=frozen, residency=residency) ): device_permission = Permission.ABORT device_reason = ( f"this deployment's licence could not be verified " f"({self._licence_unverified}) and no g.tec device " f"attested; a deployment runs with a licence or with an " f"attested device. {licence.remedy(self._licence_unverified)}" ) # A third, independent input: the mark the data itself carries. # meet() takes the strictest of the three, so this can only # tighten FULL to LIMITED -- an ABORT from either gate above # stays an ABORT. data_permission, data_reason = self._data_mark() permission = meet( (licence_permission, device_permission, data_permission) ) # The reason is the restricting input's, and an unrestricted run # has none: a lifted device gate still has its own reason, which # says the output is marked. reason = "" if permission is not Permission.FULL: for gate, why in ( (licence_permission, licence_reason), (device_permission, device_reason), (data_permission, data_reason), ): if gate is permission and why: reason = why break verdict = Entitlement( permission, reason, licence=licence_permission, attestation=attest_permission, attestation_reason=attest_reason, ) set_entitlement(self, verdict) return verdict def _data_mark(self) -> tuple: """The mark this run's data carries, whatever the licence says. A loaded artifact fitted on marked data carries it into every run that deploys it (D-BATCH-92), and a recording a marked run wrote carries it into every run that replays it (D-ENT-97): the licence that lifts this run's own mark does not license what an unlicensed run produced. A source carries a recording's mark only as its ``mark`` property, read from the file where it is run, and only the mark's own text counts, so an unrelated attribute of that name marks nothing. Returns: ``(Permission.LIMITED, reason)`` if the data is marked, else ``(Permission.FULL, "")``. """ fitted = sorted( f"'{node.name}'" for node in self._fittable_nodes() if getattr(node, "_gpype_marked", False) ) if fitted: return Permission.LIMITED, ( f"{', '.join(fitted)} carries an artifact fitted on marked " f"data, so this run carries the mark forward" ) replayed = sorted( f"'{node.name}'" for node in self._sources() if getattr(node, "mark", None) == MARK ) if replayed: return Permission.LIMITED, ( f"{', '.join(replayed)} replays a recording a marked run " f"wrote, so this run carries the mark forward" ) return Permission.FULL, "" def _remote_attestation_results(self) -> list: """Challenge the far end of every Link (C10, SERVER side). Called from inside the entitlement phase, which is before any node starts, because that is the only placement that works: a sink writes its header at ``start()``, so a verdict arriving later could not change what the artifact claims. That forces an awkward step. The broker is bound *here*, ahead of the Links that normally bind it, because the edge cannot connect to a port nothing is listening on -- and the extra reference is held for the run and released in ``stop()``. Returns: One AttestationResult per connected edge. Empty when this is not a server or the switch is off, so the gate sees exactly what it saw before. """ if not remote_required(): return [] config = LaunchConfig.get() if config.residency != Constants.Residency.SERVER: return [] from .core._private import threads if not threads.available(): # A page connects only once start() has returned, so there is # no edge to challenge here (D-CORE-129). return [ attestation.AttestationResult( "", False, "no edge can connect before a server without threads " "has started, so none was challenged", ) ] from .core._private.ws.broker import WsBroker broker = WsBroker.get_instance() broker.acquire(config.endpoint) self._attest_broker = broker if not broker.wait_for_client(remote_attestation.JOIN_TIMEOUT_S): return [ attestation.AttestationResult( "", False, f"no edge connected to {config.endpoint} within " f"{remote_attestation.JOIN_TIMEOUT_S:g}s", ) ] channels = broker.attest_channels() if not channels: return [ attestation.AttestationResult( "", False, "no edge to challenge" ) ] return remote_attestation.challenge_all(channels) def _start_attestation_responders(self) -> None: """Answer the server's challenges for this run (C10, EDGE side). After ``super().start()``, because a Link only creates its WsClient when it starts -- and unlike the server, the edge does not gate its own start on the handshake: its handle is local, so C9 already challenged it directly. The edge is only answering on the server's behalf. """ if not remote_required(): return if LaunchConfig.get().residency != Constants.Residency.EDGE: return handle = self._first_device_handle() for link in self._links(): client = getattr(link, "_ws", None) factory = getattr(client, "attest_channel", None) if factory is None: continue responder = remote_attestation.Responder(factory(), handle) responder.start() self._responders.append(responder) def _stop_attestation(self) -> None: """Stop the responders and give back an early broker reference. Never raises: this runs on the teardown path, where losing the original error is worse than failing to tidy up. """ responders, self._responders = self._responders, [] for responder in responders: try: responder.stop() except Exception: pass broker, self._attest_broker = self._attest_broker, None if broker is not None: try: broker.release() except Exception: pass def _walk_nodes(self) -> list[tuple]: """Every node this pipeline owns, chain internals included. One walk, shared by :meth:`get_nodes` and node addressing, so that a node which can be reported is exactly a node which can be addressed. Two walks would agree until one of them changed. Returns: One ``(node, is_internal)`` pair per node, in registration order, each chain followed by its own internal nodes. """ walked: list[tuple] = [] for element in self._elements: walked.append((element, False)) internal = getattr(element, "internal_nodes", None) if internal: walked.extend((node, True) for node in internal) return walked def _resolve_node(self, node_id: Union[str, Node]): """The node in *this* pipeline carrying *node_id*. Resolved over this pipeline's own elements rather than through ``PortableImp.get_by_id``, which is a **process-global** registry: it would hand back a node belonging to a different pipeline in the same process, so a control plane holding two sessions could drive the wrong one and get an ordinary success back. A document id names the node the author wrote. Where this process runs it, that node is returned; where it does not -- a source under server residency -- the chain carrying it is, which is what the id addressed before the pipeline built the chains. The node object itself resolves the same way, by identity: a script holds its nodes, and had to look each id up by name through :meth:`get_nodes` first. Never by name -- names are not unique (see :func:`_node_id_of`). Args: node_id: The node's document id, or the node itself. Returns: The node object. Raises: KeyError: If no node in this pipeline carries that id, or the node given is not one of this pipeline's. """ from .core._private import wrapping if not isinstance(node_id, str): for node, _ in self._walk_nodes(): if node is node_id: return node for element in self._elements: if wrapping.core_of(element) is node_id: return element raise KeyError( f"{naming.node_label(node_id)} is not a node of this " f"pipeline" ) for node, _ in self._walk_nodes(): if _node_id_of(node) == node_id: return node for element in self._elements: if _node_id_of(wrapping.core_of(element)) == node_id: return element raise KeyError(f"no node with id '{node_id}' in this pipeline") @staticmethod def _control_names(node_id: Union[str, Node]) -> tuple: """How control messages name the node they were addressed at. Args: node_id: A document id, or a node. Returns: (subject, prefix): ``node 'x'`` and ``x`` for an id, as before nodes were accepted; the node's label for both for a node, which already reads ``Knob 'knob'``. """ if isinstance(node_id, str): return f"node '{node_id}'", node_id label = naming.node_label(node_id) return label, label
[docs] def get_nodes(self) -> list[dict]: """Report every node in this pipeline and its current state. Intended for anything driving a pipeline from outside -- a control plane, a monitoring view -- which otherwise has to reach into private implementation attributes to answer "what is in here, and is it healthy". Chain internals are included, since a chain is a container and the node that actually failed is inside it. Returns: One entry per node, each with its ``name``, ``class``, whether it is an ``internal`` node of a chain, its ``state`` and cycle ``counter`` when the node exposes them, and its control surface: * ``id`` -- the document id to address a control at, or None for a node carrying no configuration. * ``actions`` -- the declared action names, sorted, empty when the node declares none. * ``controllable`` -- whether the node has a usable control channel, meaning it declares *both* halves. Two fields rather than one, because a node may well have actions and no state, or state and no actions. """ from .core._private import wrapping report: list[dict] = [] for node, is_internal in self._walk_nodes(): # A chain the pipeline built is reported as the node it # carries -- its class, document id and controls -- as a # public chain was in 4.0: "SourceChain" and the chain's own # id are in no document. The node keeps its own row among the # internals, so a caller that reads the internals still sees # the node that did the work. owner = wrapping.core_of(node) or node entry = { "name": getattr(node, "name", ""), "class": type(owner).__name__, "internal": is_internal, "id": _node_id_of(owner), "actions": action_names(owner), "controllable": has_control(owner), } for key, attr in ( ("state", "get_state"), ("counter", "get_counter"), ): getter = getattr(node, attr, None) if callable(getter): try: entry[key] = getter() except Exception: # A node that cannot report its own state must # not stop the caller from seeing the rest. entry[key] = None report.append(entry) return report
#: Duty above which a node is reported as short of headroom. Chosen #: below 1.0 rather than at it: a pipeline that is exactly keeping up #: has no margin for the next scope repaint or garbage collection, #: and the useful warning is the one that arrives before frames are #: actually late. LOAD_WARNING_DUTY: float = 0.70
[docs] def get_load(self) -> dict: """Report how much of the real-time budget the pipeline is using. Each node times its own ``step()``; this divides that by the wall time since the previous call, which is the only denominator that is right for every driving mode. A rate-derived one -- frame size over sampling rate -- is wrong for any source paced per sample: ``FixedRateSource`` wakes once per sample whatever the frame size, so dividing by the frame period would report several times more headroom than exists. Reading resets the counters, so two calls a second apart describe that second. The first call after ``start()`` covers everything since the nodes were built, including setup, and is not representative. Returns: dict: ``headroom`` (1 - duty, so 1.0 is idle and 0.0 is saturated), ``duty``, ``elapsed_s``, ``busiest`` naming the node with the highest duty, ``short_of_headroom`` naming every node over :attr:`LOAD_WARNING_DUTY`, and ``nodes``, one row per node with its ``name``, ``class``, ``duty``, ``worst_cycle_ms`` and ``cycles``. """ now = perf_counter() previous = getattr(self, "_load_read_at", None) self._load_read_at = now elapsed = (now - previous) if previous else 0.0 rows: list[dict] = [] total = 0.0 for node, _is_internal in self._walk_nodes(): take = getattr(node, "take_load", None) if not callable(take): continue busy, worst, cycles = take() total += busy rows.append( { # The label the author would recognise. A chain's # work happens in its private core, so the raw name # here is "_GeneratorCore" -- which reads as an # internal leak in a report whose whole job is to # say which node to go and look at. The unmapped # class name stays in "class" for diagnosis. "name": naming.node_label(node), "class": type(node).__name__, "duty": (busy / elapsed) if elapsed else None, "worst_cycle_ms": worst * 1000.0, "cycles": cycles, } ) duty = (total / elapsed) if elapsed else None busiest = None if rows and elapsed: # Self-time, because the push to downstream happens after the # step handler returns -- so the busiest node is the one to # look at, not merely the one furthest downstream. busiest = max(rows, key=lambda r: r["duty"] or 0.0)["name"] # The threshold applied here rather than left to the caller, # because a number without a verdict is a number nobody acts # on: "duty 0.72" reads as fine until you know what fine was. # A list rather than a bool, since the useful answer to "am I # short of headroom" is which node to look at. short = [ row["name"] for row in rows if row["duty"] is not None and row["duty"] > self.LOAD_WARNING_DUTY ] return { "headroom": (1.0 - duty) if duty is not None else None, "duty": duty, "elapsed_s": elapsed, "busiest": busiest, "short_of_headroom": short, "nodes": rows, }
[docs] def get_control(self, node_id: Union[str, Node]) -> dict: """Read the full control state of one node. Args: node_id: The node's document id, as reported by :meth:`get_nodes`, or the node itself. Returns: The node's whole control state, as a copy -- mutating it cannot reach the node. Raises: KeyError: If no node in this pipeline carries that id, or the node given is not one of this pipeline's. AttributeError: If the node declares no control channel. TypeError: If the node's answer is not a JSON-representable dict. """ node = self._resolve_node(node_id) subject, prefix = self._control_names(node_id) if not has_control(node): raise AttributeError(f"{subject} declares no control channel.") getter, _ = resolve_control(node) state = ensure_json(getter(), False, f"{prefix}.get_control") # Copied on the way out. A node returning its own live dict would # otherwise let a caller mutate its state by hand -- no merge, no # unknown-key check, none of the node's own validation -- which is # the whole surface this method exists to be. return dict(state)
[docs] def set_control(self, node_id: Union[str, Node], arg: dict) -> dict: """Apply a partial control update to one node. The update **merges**: *arg* names only what changes, so a one-field change stays one field and two clients need no get-before-set to avoid overwriting each other. Any key the node's own ``get_control`` does not report is refused **before the node sees it** -- the getter is the key set. Args: node_id: The node's document id, or the node itself. arg: The keys to change. Returns: The resulting full control state, so a set is also a get. A copy, as in :meth:`get_control`. Raises: KeyError: If no node carries that id, the node given is not one of this pipeline's, or *arg* holds a key the node does not accept -- that message names both the offending keys and the accepted ones. TypeError: If *arg* is not a dict, or the node's answer is not a JSON-representable dict. AttributeError: If the node declares no control channel. ValueError: Whatever the node raises when the resulting combination is not usable. Propagated deliberately: only the node knows its own cross-field rules. """ node = self._resolve_node(node_id) subject, prefix = self._control_names(node_id) if not has_control(node): raise AttributeError(f"{subject} declares no control channel.") getter, setter = resolve_control(node) current = ensure_json(getter(), False, f"{prefix}.get_control") state = ensure_json( setter(merged_update(current, arg)), False, f"{prefix}.set_control", ) return dict(state) # copied on the way out -- see get_control
[docs] def invoke_action( self, node_id: Union[str, Node], name: str, arg: dict ) -> Optional[dict]: """Run one declared action on one node. Args: node_id: The node's document id, or the node itself. name: The action name, as reported by :meth:`get_nodes`. arg: The action's argument; may be empty. Returns: Whatever the action reports, or None when it reports nothing. Raises: KeyError: If no node in this pipeline carries that id, or the node given is not one of this pipeline's. AttributeError: If the node declares no action of that name. An ordinary method is not an action, so this is also what an attempt to call anything else by name gets -- which is what keeps the rest of the class unreachable. TypeError: If *arg* is not a dict, or the result is not a JSON-representable dict or None. ValueError: Whatever the action itself raises. """ if not isinstance(arg, dict): raise TypeError( f"action argument must be a dict, " f"got {type(arg).__name__}." ) node = self._resolve_node(node_id) subject, prefix = self._control_names(node_id) method = resolve_action(node, name) if method is None: raise AttributeError( f"{subject} declares no action '{name}'; " f"declared actions are {action_names(node)}" ) result = ensure_json(method(arg), True, f"{prefix}.{name}") return None if result is None else dict(result)
[docs] def attach_probe( self, node_id: Union[str, Node], port: str = Constants.Defaults.PORT_OUT, max_rate: float = None, ) -> str: """Publish what one port emits, live, for a subscriber to watch. A debugging instrument. The port keeps working exactly as it did: a probe copies what goes past and never consumes, delays or modifies it. What arrives is a **subsample**, not the signal. The probe decimates to ``max_rate`` and applies no anti-aliasing filter, because a filter would make this a measurement instrument and the number read off a probe must never be mistaken for one. The published context says so -- it carries the delivered rate, the decimation factor and ``antialiased: false``. The returned stream id is also the *permission* to watch: it carries random bytes, because the broker's ``subscribe`` capability is granted per connection rather than per stream, so anyone holding a session's reader token could otherwise watch any probe whose id they guessed. Hand it only to whoever asked. Costs nothing while nobody is subscribed: the probe checks before it copies. Args: node_id: Document id of the node to probe, as :meth:`get_nodes` reports it, or the node itself. port: Name of its output port. max_rate: Samples per second to deliver, at most. Defaults to :data:`probe.DEFAULT_MAX_RATE_HZ`. Returns: str: The stream id to subscribe to. Raises: KeyError: If no node in this pipeline carries that id, or the node given is not one of this pipeline's. ValueError: If the node has no such output port, or if this process has no running broker to publish on. RuntimeError: Where no thread can be started (Pyodide). """ from .core._private import probe as probe_module from .core._private.ws.broker import WsBroker node = self._resolve_node(node_id) subject, _ = self._control_names(node_id) # The probe's record and stream id carry the document id even when # the node itself was given, so list_probes() stays JSON for a # remote client (D-CORE-126). if not isinstance(node_id, str): node_id = _node_id_of(node_id) or naming.node_label(node_id) # A source the pipeline wrapped is watched where downstream sees # it, at its chain's boundary: after the Sync, with the arrival # stamp stripped. The node's own port carries the raw frame. node = self._wrappers.get(id(node), node) getter = getattr(node, "get_output_port", None) if getter is None: raise ValueError( f"{subject} has no output ports, so there is nothing on " f"it to watch. A probe reads what a node emits; a sink " f"emits nothing." ) try: port_imp = getter(port) except Exception as error: raise ValueError( f"{subject} has no output port '{port}': {error}" ) from error from .core._private import threads if not threads.available(): # Before anything is published: a probe's sender is a thread. raise RuntimeError( "attach_probe needs a thread for the probe's sender, and " "none can be started here (Pyodide). A page reads a " "widget's stream through its Link instead." ) broker = WsBroker.get_instance() if not broker.is_running: raise ValueError( "no data socket is bound in this process, so there is " "nowhere to publish a probe. A probe is delivered over " "the same broker the pipeline's own streams use, and a " "standalone run binds none -- nothing is listening and " "nothing is serving. A server-residency run has one; so " "does any process that acquired a broker itself." ) verdict = entitlement_of(self) stream_id = probe_module.new_stream_id(node_id, port) instrument = probe_module._Probe( stream_id=stream_id, node_id=node_id, port_name=port, port_imp=port_imp, broker=broker, max_rate=( probe_module.DEFAULT_MAX_RATE_HZ if max_rate is None else float(max_rate) ), marked=bool(verdict is not None and verdict.marked), ) instrument.start() self._probes[stream_id] = instrument self.log( f"probing '{node_id}' port '{port}' as {stream_id}", type=Constants.LogTypes.INFO, ) return stream_id
[docs] def detach_probe(self, stream_id: str) -> bool: """Stop and remove one probe. Args: stream_id: What :meth:`attach_probe` returned. Returns: bool: True if a probe was removed, False if there was none. """ instrument = self._probes.pop(stream_id, None) if instrument is None: return False instrument.stop() self.log(f"stopped probing {stream_id}", type=Constants.LogTypes.INFO) return True
[docs] def list_probes(self) -> list[dict]: """Report every probe attached to this pipeline. Returns: list[dict]: One entry per probe, each naming the node and port it watches, its decimation, how many samples it has sent, how many frames it dropped, and how many connections are currently watching it. """ return [probe.status() for probe in list(self._probes.values())]
def _detach_all_probes(self) -> None: """Stop every probe. Called when the run ends.""" for stream_id in list(self._probes): try: self.detach_probe(stream_id) except Exception: # pragma: no cover - teardown self._probes.pop(stream_id, None)
[docs] def connect(self, source: Union[Node, dict], target: Union[Node, dict]): """Connect two nodes to establish data flow in the pipeline. Nodes are automatically added to pipeline if not already present. Args: source (Union[Node, dict]): Source node or port specification. Use dict for specific ports (e.g., node["port_name"]). target (Union[Node, dict]): Target node or port specification. Use dict for specific ports (e.g., node["port_name"]). Raises: ValueError: If this edge would make a feedback loop illegal -- see :meth:`_describe_component_violation`. The edge is undone before raising, so the pipeline is left exactly as it was before this call. """ # Before registering either: a world-facing node reaches the # pipeline inside the chain that gives it a Link and a Sync, and it # is the chain that has to be the element -- ``_elements`` is # what every residency, timeline and entitlement walk reads. source = self._wrap(source) target = self._wrap(target) self._register(source) self._register(target) super().connect(source, target) # Checked after the edge exists, not before: the real port connections # -- which a bare node, a dict spec, and a chain proxying to its # first/last internal node all normalise to differently -- only # exists once ioiocore has resolved it. # # The check looks at the whole pipeline rather than at this edge, # and disconnecting this edge is still the right rollback. Every # prior connect() left the pipeline with no illegal loop in it # (a pipeline rebuilt by deserialize() is covered at start() # instead -- see _validate_no_illegal_cycles), so a component # that violates a rule now has to involve this edge: either it # closed the loop, or it connected the port that turned an # existing legal loop illegal -- an ASYNC input, say. Removing it # restores the pipeline that passed. # # What the message names is the loop, which is not always this # edge. That is deliberate: the loop is what the author has to # change. # # require_feed is off here: "something outside feeds this loop" # is true of a finished pipeline, not of every intermediate one. # Closing the loop and then wiring the source that drives it is # an ordinary way to write the same pipeline, so enforcing it # per-edge would refuse a pipeline that is about to be legal. # start() applies it -- see _validate_no_illegal_cycles. violation = self._cycle_violation(require_feed=False) if violation is not None: self._imp.disconnect(source, target) raise ValueError(violation)
#: How long an edge's start waits for its clock to converge on the #: server's reference, in seconds. As long as a receiving Link waits #: for a context, so an edge started before its server keeps working. CLOCK_SYNC_TIMEOUT_S = CONTEXT_TIMEOUT_S @staticmethod def _running_pipelines(exclude=None) -> list: """Return the pipelines in this process that are running. Args: exclude: A pipeline to leave out, usually the caller. Returns: The running pipelines. """ running = ioc.Constants.States.RUNNING return [ p for p in pipelines.live() if p is not exclude and p.get_state() == running ]
[docs] @staticmethod def rezero_reference() -> int: """Zero this process's reference clock, in a new epoch. The server's half of the time base (D-TIME-20, D-TIME-67). Edges sync their clocks to the reference this process answers clock probes with, and every stamp this process reads is in it. Zeroing it when a pipeline is loaded keeps stamps small and starts a new epoch, which an edge synced to the previous one is told about. Under SERVER residency :meth:`deserialize` calls this, which is how the g.Pype runtime's load zeroes it. A process that loads pipelines any other way calls it itself. Returns: The new epoch. Raises: RuntimeError: While any pipeline in this process is running: its stamps would move by the whole reference at once. """ running = Pipeline._running_pipelines() if running: raise RuntimeError( f"Cannot zero the reference clock while {len(running)} " f"pipeline(s) in this process are running: every stamp " f"they hold would jump by the reference's age. Stop them " f"first." ) return timebase.current().rezero()
def _sync_edge_clock(self) -> None: """Relate this edge's clock to the server's reference, once. The edge's start gate (D-TIME-65, D-TIME-68). Before any source stamps: the offset is stepped in, so the first stamp is already in the server's base, and it then stays constant until stop(). Nothing re-syncs during the run. Every start syncs afresh, which is what picks up a reference the server zeroed at a new load. Only on an edge that builds a source here: a sink or a widget writes no stamp. Not under STANDALONE or SERVER residency, and not in a batch run. Raises: ClockSyncError: If the clock does not converge within :attr:`CLOCK_SYNC_TIMEOUT_S`, naming the endpoint. """ # A start() on a running pipeline must not sync again: stepping # the offset now would move every later stamp of this run. if self.get_state() == ioc.Constants.States.RUNNING: return self._clock_health = None launch = LaunchConfig.get() if launch.residency != Constants.Residency.EDGE: return links = [ link for link in self._links() if link.residency == Constants.Residency.EDGE and link.sender == Constants.Residency.EDGE ] for link in links: link.attach_clock_sync(None) if getattr(self, "_batch_driven", False) or not self._sources(): return from .core._private import clock_sync from .core._private.ws.clock_client import ClockConnection base = timebase.current() # A new run holds no stamp, unless another pipeline here runs. if not self._running_pipelines(): base.reopen() connection = ClockConnection(launch.endpoint) try: health = clock_sync.synchronise( connection, self.CLOCK_SYNC_TIMEOUT_S, clock=base.clock ) finally: connection.close() # What the stamps are corrected by, which is what the context # must report: not the estimate, if another pipeline's stood. health = clock_sync.apply_offset(health, base) self._clock_health = clock_sync.context_entry( health, launch.edge_id or clock_sync.EDGE_TOKEN ) for link in links: link.attach_clock_sync(self._clock_health) precise = health.uncertainty <= clock_sync.CONVERGED_UNCERTAINTY_S self.log( f"Clock synced to {launch.endpoint}: this edge is " f"{health.offset * 1e3:.3f} ms ahead of the server's reference " f"(epoch {health.epoch}), known to " f"{health.uncertainty * 1e3:.3f} ms; constant for this run." + ( "" if precise else f" Above the " f"{clock_sync.CONVERGED_UNCERTAINTY_S * 1e3:g} ms target: " f"the network's round trip is too long for better, and " f"events may be misplaced against other edges' by up to " f"that much." ), type=( Constants.LogTypes.INFO if precise else Constants.LogTypes.WARNING ), )
[docs] def start(self): """Start the pipeline and begin real-time data processing. Initiates execution of all nodes according to their configured connections and timing. Runs continuously until stop() is called. This method is non-blocking. On an edge that builds a source, first relates its clock to the server's reference, and raises if that does not converge within :attr:`CLOCK_SYNC_TIMEOUT_S` (``ClockSyncError``). """ # Startup runs in ordered phases, and the order is load-bearing. # Note setup() does not run here -- ioiocore defers it to each # node's first cycle -- so a phase may only use what is known at # construction. # # 1. validate -- reject pipelines that cannot mean anything, so # the user is told the real problem # 2. timeline -- bind it, so election has somewhere to happen # 3. start -- nodes run, setup happens, election resolves # # Phases for tier resolution belong between 2 and 3, before any # sink can write. # # A run that failed is still being stopped when ``failure`` is # first set, with the state RUNNING, and records the failure only # after that. A start() in either window would be skipped or # would inherit that record, so it waits the run out first. self._await_failing_stop() self._validate() self._refuse_batch_without_run() # Before anything reads the stamp clock -- the raw recording's # origin just below included -- because the offset can be # stepped in only before the first reading (D-TIME-46). self._sync_edge_clock() # Anchor a replay to *this* run. The clock every replay core # schedules against is process-global, so a pipeline started a # second time would keep the first run's origin -- every # recorded offset would already be in the past, and the whole # recording would be re-emitted as fast as the loop could go, # which reproduces nothing and looks like a working replay. # # Here rather than in the cores: they anchor lazily on their # first block, so the reset has to happen while none of them is # running, and start() is the only such moment. launch = LaunchConfig.get() if launch.load_from: from .sources.base.raw import ReplayClock ReplayClock.reset() if launch.save_as: # The same argument on the recording side. The run object # lives as long as the pipeline, so without this a pipeline # started a second time records block times relative to the # *first* run's origin -- and a replay of that recording # sleeps out the gap between the two runs before emitting # anything. See RawRun.begin. from .sources.base.raw import RawRun run = RawRun.current(launch.save_as) run.begin() # Say once, per run, when the clock the samples are stamped # from is too coarse to mean what the documentation says. # # Every source stamps its own samples (D-NODE-24), so this is # not specific to a pipeline shape and there is nothing to inspect # to decide whether it matters. It is silent on CPython 3.13 # and newer and on every Unix, which is why it can afford to be # unconditional here. complaint = stamp_clock_complaint() if complaint: self.log(complaint, type=Constants.LogTypes.WARNING) # Bind the master timeline before anything runs. It is owned per # pipeline rather than per process: a process routinely builds # several pipelines, and a shared timeline would let one inherit # another's master and epoch. # # A fresh one is established for every run, not merely released # in stop(): ioiocore stops the pipeline from inside its monitor # thread when a node errors, which calls its own stop() and never # this override, and start() then auto-resets instead of # refusing. A rolled-back start() failure leaves the same residue # without even setting the error condition. Reusing that timeline # would keep the previous run's master election, epoch and rate # anchor while the sources restart at sample zero, placing every # event a whole run into the past. if self.get_state() != ioc.Constants.States.RUNNING: release_timeline(self) release_entitlement(self) self._reset_run_state() # A fresh run must not report the previous run's failure. self._failure = None timeline = timeline_of(self) # After timeline_of, because the release above discards the # object this was last written to. timeline.event_margin_ms = self.event_margin_ms for node in self._timeline_consumers(): node.attach_timeline(timeline) for node in self._placement_consumers(): node.attach_placement_timeline(timeline) self._attach_position_gate() self._elect_master(timeline) # Resolve both gates before any node starts. A sink writes its # header at start(), so a verdict reached later would let a run # stamp the wrong mark -- silently, in the artifact. verdict = self._resolve_entitlement() if not verdict.allowed: # Give back the broker reference the remote handshake may # have taken. A refused start must not leave a port bound # until somebody remembers to call close(). self._stop_attestation() # Nor an amplifier held. The gate opened handles to challenge # them, and no node ran, so stop() does not reach a source: # a BCI Core stayed connected until close(). A GDS source's # constructor handle goes too; the next start reopens it. self._release_devices() raise PermissionError(verdict.reason) # Hand the verdict to whatever produces an artifact, before # anything starts. A sink that learns it later has already # written its header. for node in self._entitlement_consumers(): node.attach_entitlement(verdict) if verdict.marked: self.log( f"{MARK}: {verdict.reason}", type=ioc.Constants.LogTypes.INFO, ) # Stamped once per run, in both modes, before any node's setup() # can run -- a source merges it, plus its own INPUT, into the # context it publishes (LQ-P2). Built from the pipeline as it is # right now: a fittable node already holding a state (a realtime # deployment) is named in the record; one that will only fit # once this run starts is not yet (D-BATCH-100). from ..common._private import provenance as provenance_module record = provenance_module.build(self) for node in self._provenance_consumers(): node.attach_provenance(record) super().start() # Only now, once the pipeline is actually running: the case where # a device must be *reachable* and none attested as a source. See # _begin_deferred_device_check for why it is after start() and # not before. self._begin_deferred_device_check(verdict) # Also only now: the edge's Links have their WsClients, so it can # start answering. See _start_attestation_responders for why the # edge does not wait for this and the server does. self._start_attestation_responders() # Last, because it is the only phase that waits: give the first # cycles a moment to report a setup failure, so a pipeline that was # never viable fails here rather than going quiet. Skipped for a # batch run, which run() drives on the caller's thread -- a setup # failure there is raised rather than logged, and nothing has # cycled yet at this point, so the wait would buy nothing and # charge every offline run the whole grace period. if not getattr(self, "_batch_driven", False): self._await_setup() # And, on a server, name later what never arrived: a stream # waits without bound, so a wait here would be a guess. self._begin_stream_report() # Anchor the load window *here*, after everything start() waits # for, so the first get_load() describes the run and not the # starting of it. # # It used to be anchored immediately after super().start(), on # the stated grounds that "setup ran before this point and its # cost is not a steady-state duty". The first half was wrong: # ioiocore runs setup() on a node's first *cycle*, not in # start(), which is the whole reason _await_setup() exists # below it. So the grace period spent waiting for setup was # folded into the first duty -- exactly the thing the comment # said must not happen. # # Invisible on a fast host, where the wait is a few # milliseconds. On a slow one it is most of SETUP_GRACE_S: # measured on macOS CI, a get_load() after sleeping 0.2 s # reported 0.38-0.40 s, consistently, on three Python versions. self._load_read_at = perf_counter()
#: How long start() waits for the first cycles to report a setup #: failure, in seconds. ioiocore runs setup() on a node's first #: cycle rather than in start(), so without a wait there is nothing #: to look at yet. Bounded because a node fed only by a sparse event #: stream sets up when its first event arrives, which may be never. SETUP_GRACE_S = 0.5 #: Poll interval while waiting, so a healthy pipeline pays only the #: time it actually needs -- one frame period, typically. SETUP_POLL_S = 0.005 def _setup_failures(self) -> list: """Return every node whose ``setup()`` raised. Asked of the node rather than inferred from its condition: a ``step()`` that raises on cycle 1 leaves the same condition behind, and that is a run that died rather than a pipeline that could not be built. Only the second is start()'s business. Returns: The nodes that could not set up, chain internals included. """ found = [] for node, _ in self._walk_nodes(): failed = getattr(node, "setup_failed", None) if callable(failed) and failed(): found.append(node) return found def _await_setup(self) -> None: """Let the first cycles run, and refuse a pipeline that cannot. ``setup()`` is where a node says what it cannot do -- a cutoff above Nyquist, a channel count that does not agree, a file extension it cannot write -- and ioiocore runs it on the node's *first cycle*, which happens after ``start()`` has returned. A node that fails there logs the message, marks itself, and then returns from every later cycle without processing anything. What the user saw was a scope that never updated. The monitor thread does stop the run and print the message, but its passes are ``MONITORING_INTERVAL`` (1 s) apart and the banner goes to a console a windowed application may not have -- and ``start()`` had already returned normally, so a script carried on past a pipeline that was never going to produce anything. Best effort by construction: a node fed only by a sparse event stream sets up when its first event arrives, which may be never, so the wait is bounded and whatever sets up after the deadline stays with the monitor thread and :meth:`raise_if_failed`. Raises: RuntimeError: If a node failed to set up, naming the node and quoting what it wrote. """ from .core._private import threads if not threads.available(): # No first cycle can run while the only thread waits here, # so the wait would only delay start(). A setup failure then # reaches the monitor, as one after the deadline does. return deadline = time.monotonic() + self.SETUP_GRACE_S while time.monotonic() < deadline: if self._setup_failures(): break pending = [ node for node, _ in self._walk_nodes() if getattr(node, "get_counter", None) and node.get_counter() == 0 ] if not pending: break time.sleep(self.SETUP_POLL_S) failed = self._setup_failures() if not failed: return # Stopped here rather than left to the monitor thread's next # pass: a start() that raises must not leave an acquisition # thread running, a device claimed and a file half written for # up to a second afterwards. try: self.stop() except Exception: # The setup failure is the story. A stop that also fails # must not replace it with its own. pass entry = self.failure or self.get_last_error() detail = "." if entry is not None: message = entry["message"] if "message" in entry else entry detail = f": {message}" names = ", ".join( sorted(naming.public_name(type(n).__name__) for n in failed) ) error = RuntimeError( f"{names} could not set up, so the pipeline is not " f"running{detail} Nothing was emitted and no sink wrote a " f"row; fix what the node reported and start again." ) # Chained to what the node actually raised, so the traceback # names the line in the node -- and in the user's own node, if # they wrote one. Until ioiocore kept the exception this could # only quote the logged message, which tells a caller that # something failed and never where. # # `from` only when there is a cause: `raise X from None` # suppresses the chain, which would be a worse answer than # saying nothing about it. cause = self._first_failure(failed) if cause is not None: raise error from cause raise error @staticmethod def _first_failure(failed: list): """The exception behind the first node that has one. Best effort by construction. `failure()` arrived in ioiocore 5.0.0rc4, and a node built against an older one answers nothing -- in which case the message quoted above is all there ever was, and a chained `None` would not improve it. Args: failed (list): Nodes whose ``setup()`` raised. Returns: The first exception found, or None if none is available. """ for node in failed: getter = getattr(node, "failure", None) if not callable(getter): continue try: cause = getter() except Exception: # noqa: BLE001 - a probe, never a failure continue if cause is not None: return cause return None #: Safety stop for the batch driver. A batch source emits its whole #: recording in one cycle, so the loop below normally runs once; this #: only bounds a source that reports "more" forever, which would #: otherwise be an unkillable loop rather than an error. MAX_BATCH_CYCLES = 1_000_000
[docs] def execution_mode(self) -> str: """How this pipeline will be driven, as declared by its sources. Read from the sources rather than from their time base: a recording *paced to the clock* is an ordinary realtime pipeline, so the two properties are independent and only one of them says how to drive the pipeline. Returns: A value from :class:`Constants.ExecutionMode`. Raises: ValueError: If the pipeline has no source, or if its sources disagree -- a pipeline cannot be half batch. """ sources = self._sources() if not sources: raise ValueError( "This pipeline has no source, so there is no execution " "mode to speak of. Connect a source first." ) modes = { getattr( node, "EXECUTION_MODE", Constants.ExecutionMode.REALTIME ): node for node in sources } if len(modes) > 1: names = ", ".join( f"{type(node).__name__} ({mode})" for mode, node in sorted(modes.items()) ) raise ValueError( f"A pipeline is either realtime or batch, never both, " f"but its sources disagree: {names}." ) return next(iter(modes))
[docs] def run(self) -> Union[Result, dict, None]: """Process a recording from end to end and return the results. The batch counterpart of :meth:`start`. Where a realtime pipeline is started, left running and stopped by the caller, a batch pipeline is *driven*: this call returns when the whole recording has passed through the last node. It executes on the calling thread and creates none of its own, which is what makes an exception in a node arrive as an ordinary Python traceback through the caller's line rather than as a log entry from a monitoring thread. Returns: The :class:`~gpype.common.result.Result` of the single :class:`~gpype.backend.sinks.collector.Collector` in the pipeline; a dict of results keyed by node name if there is more than one; ``None`` if there is no Collector, in which case the run's output went wherever its sinks put it. Raises: ValueError: If the pipeline is not a batch pipeline -- see :meth:`execution_mode` and :meth:`_validate_batch`. Exception: Whatever a node raises, unchanged. The node's frame and the caller's line are both in the traceback. """ mode = self.execution_mode() if mode != Constants.ExecutionMode.BATCH: raise ValueError( f"run() drives a batch pipeline, and this one is " f"'{mode}'. Either build it with a source in batch mode " f"-- CsvReader(..., mode='batch') -- or use start() and " f"stop() to run it against the clock." ) self._validate_batch() # Before start(), because a node's setup() happens on its first # cycle and the flag decides whether a failure there is raised or # logged. Set on every node, chain internals included. self._set_batch(True) try: self.start() for source in self._sources(): cycles = 0 while not source.is_exhausted: source.cycle() cycles += 1 if cycles >= self.MAX_BATCH_CYCLES: raise RuntimeError( f"{type(source).__name__} still reports more " f"data after {cycles} cycles. A batch source " f"must eventually report is_exhausted." ) self._raise_node_failure() results = self._collect() finally: self.stop() self._set_batch(False) return results
[docs] def fit(self) -> Union[Result, dict, None]: """Fit every fittable node in this pipeline, then apply the result. The offline half of "fit offline, deploy online" (D-BATCH-16): drives a batch run exactly like :meth:`run`, except every :class:`~gpype.backend.core.fittable.Fittable` node in the pipeline fits on the whole recording first, instead of requiring state that was never loaded. A node not inheriting ``Fittable`` runs exactly as it would under :meth:`run`. Returns: Whatever :meth:`run` returns -- a fit run is a batch run, with the same single-Collector-or-dict-or-None contract. Raises: ValueError: If this pipeline holds no fittable node. Use :meth:`run` for an ordinary batch pipeline. Exception: Whatever :meth:`run` raises, unchanged. """ fittable = self._fittable_nodes() if not fittable: raise ValueError( "this pipeline holds no fittable node, so there is " "nothing for fit() to learn. Use run() to drive an " "ordinary batch pipeline." ) source_mark = self._single_source_mark() for node in fittable: node._gpype_fit_run = True node._gpype_external_mark = source_mark try: outcome = self.run() finally: for node in fittable: node._gpype_fit_run = False self._add_fitted_document(outcome) return outcome
def _add_fitted_document(self, outcome) -> None: """Give a fit run's results the document that carries its fit. The provenance record is stamped at ``start()``, before anything is fitted, so ``Result.document()`` rebuilt a pipeline that ``run()`` refused for want of a state. ``fitted_document`` is this pipeline's document after the fit, artifacts included. Args: outcome: What :meth:`run` returned. """ from ..common._private import provenance as provenance_module if isinstance(outcome, Result): results = [outcome] elif isinstance(outcome, dict): results = [r for r in outcome.values() if isinstance(r, Result)] else: results = [] if not results: return document = provenance_module.fitted_document(self) if document is None: return key = Constants.Keys.PROVENANCE for result in results: record = result._context.get(key) if isinstance(record, dict): result._context[key] = dict(record, fitted_document=document) def _single_source_mark(self) -> Optional[str]: """The entitlement mark this run's one input file carries, if any. A batch run has exactly one source (see :meth:`_validate_batch`). Where it is a :class:`~gpype.backend.sources.base.recording_reader.RecordingReader`, its own ``mark`` answers whether the run that *wrote* the file was itself marked -- history the file carries forward regardless of what this machine's own licence says now (D-BATCH-92). Not a port-context key: nothing besides this has needed one there. Returns: The mark, or None where this pipeline has no single source or that source declares none. """ sources = self._sources() if len(sources) != 1: return None return getattr(sources[0], "mark", None) #: Config key a batch reader's file name is stored under -- the #: literal string every reader's own constructor parameter writes #: (``CsvReader``, ``MatReader``, ``HDF5Reader``, ``EDFReader``, #: ``GtcReader``); there is no ``Constants.Keys`` entry for it. _FILE_NAME_KEY = "file_name"
[docs] def run_all(self, files: list, source: Optional[str] = None) -> dict: """Run this pipeline's document once per file, in input order. The many-file counterpart of :meth:`run` (LQ-R3): this pipeline is never driven itself. Instead, once per file, its **document** is rebuilt with that file's name in place of the one this pipeline was constructed with, given fresh ids (D-BATCH-72, so the several pipelines this builds in one process do not collide in ioiocore's id registry), deserialized into a fresh :class:`Pipeline`, and run. Each result carries its own provenance (LQ-P2), naming that file's own input digest. Owner's decision (document-rebuild form, D-BATCH-104): a factory form -- ``run_all(build_fn, files)``, calling *build_fn* fresh per file -- would also run a script-only pipeline (``Apply``), which this form cannot. Rebuilding from the document is what makes every file run the identical, already- validated pipeline rather than trusting a callback to reconstruct it consistently, and it is what lets a caller build the template pipeline once, from a document someone else wrote, with no author script to call back into at all. The failure policy is fail-fast: the first file that raises stops the run, with its path in the message and the original exception as ``__cause__`` (D-BATCH-104, beside D-BATCH-52's pure propagation). Nothing before it is retried, and nothing after it runs. Args: files: Paths to run the document against, in order. Each becomes the single batch source's ``file_name``. source: Name of the node whose ``file_name`` to replace. None (the default) finds this pipeline's one source -- a batch pipeline permits exactly one (:meth:`_validate_batch`) -- and needs naming only where a caller wants a specific node addressed by name regardless. Returns: dict: ``{file: result}``, in *files* order -- each value whatever :meth:`run` returns for that file (a :class:`~gpype.common.result.Result`, a dict of them, or None). Raises: ValueError: If this pipeline is not BATCH, if it does not have exactly one source and *source* names none, if *source* names a node that is not a source, if a file is listed twice (the results are keyed by file), or if the pipeline holds a function node (D-BATCH-70) -- checked once, before any file is read. Exception: Whatever the failing file's run raised, chained as ``__cause__`` of a ``RuntimeError`` naming that file. """ mode = self.execution_mode() if mode != Constants.ExecutionMode.BATCH: raise ValueError( f"run_all() drives a batch pipeline once per file, and " f"this one is '{mode}'. Build it with a source in batch " f"mode -- CsvReader(..., mode='batch') -- first." ) self._validate_batch() # Keyed by file, a file listed twice kept one result, silently. files = list(files) repeated = sorted( {str(f) for i, f in enumerate(files) if f in files[:i]} ) if repeated: raise ValueError( f"run_all() returns one result per file, keyed by file, " f"and these are listed more than once: " f"{', '.join(repeated)}." ) if source is not None: named = [n for n in self._sources() if n.name == source] if not named: sources = sorted(str(n.name) for n in self._sources()) raise ValueError( f"run_all(source={source!r}) must name this " f"pipeline's source, whose file it replaces; the " f"source is {', '.join(sources) or 'missing'}." ) # Refuses a function node up front, naming it (D-BATCH-70) -- # before any file in *files* is even opened. template = self.serialize() node_entries = list(template.get("nodes") or []) if source is None: # A source is written as its own core, not the chain that # wraps it (``_write_cores_not_chains``), so it is found by # class rather than by identity against ``self._imp._nodes``, # which holds the chains. Unique because a batch pipeline # permits exactly one source (``_validate_batch``, above). target = self._sources()[0] candidates = [ i for i, entry in enumerate(node_entries) if entry.get("class") == type(target).__name__ and entry.get("module") == type(target).__module__ ] if len(candidates) != 1: raise ValueError( f"run_all() could not find this pipeline's one " f"source ({type(target).__name__}) uniquely in its " f"own document. Pass source=<name> to say which " f"node's file_name to replace." ) index = candidates[0] else: index = next( ( i for i, entry in enumerate(node_entries) if (entry.get("config") or {}).get("name") == source ), None, ) if index is None: names = ", ".join( sorted( str((entry.get("config") or {}).get("name")) for entry in node_entries ) ) raise ValueError( f"run_all() found no node named {source!r} in this " f"pipeline. It holds: {names}." ) from ..common import document as document_module results: dict = {} for file in files: per_file = copy.deepcopy(template) nodes = list(per_file["nodes"]) node_entry = dict(nodes[index]) config = dict(node_entry.get("config") or {}) config[self._FILE_NAME_KEY] = str(file) node_entry["config"] = config nodes[index] = node_entry per_file["nodes"] = nodes per_file = document_module.with_fresh_ids(per_file) try: built = Pipeline.deserialize(per_file) try: results[file] = built.run() finally: built.close() except Exception as error: raise RuntimeError( f"run_all() failed on {file!r}: {error}" ) from error return results
def _validate_batch(self) -> None: """Reject a pipeline that cannot mean anything as a batch run. Two rules, and both exist because of what a monolithic block is: a second source would carry an independent time axis with nothing to align it to, and a node with two fed inputs would be merging two blocks whose only relationship is arrival order. Fan-*out* is fine and stays allowed: branches never rejoin, so nothing can be misaligned by one. Raises: ValueError: If the pipeline has more than one source, or any node has more than one connected input port. """ sources = self._sources() if len(sources) > 1: names = ", ".join(sorted(type(n).__name__ for n in sources)) raise ValueError( f"A batch pipeline takes exactly one source, because a " f"second one has its own time axis and nothing to align " f"it to. This pipeline has {len(sources)}: {names}." ) for node, _ in self._walk_nodes(): imp = getattr(node, "_imp", None) ports = getattr(imp, "_input_ports", None) or {} fed = [ name for name, port in ports.items() if getattr(port, "_connected_ports", None) ] if len(fed) > 1: raise ValueError( f"{type(node).__name__} has {len(fed)} connected " f"input ports ({', '.join(sorted(fed))}), which a " f"batch run cannot merge: each carries a whole " f"recording and their only relationship would be " f"arrival order. Offline, what a realtime pipeline " f"merges arrives as channels of one block." ) def _set_batch(self, value: bool) -> None: """Tell every node whether this run is a batch run. Args: value: True while a batch run is being driven. """ # Recorded on the pipeline as well as on the nodes: start() has # to know which way it is being driven before any node has # cycled, and walking the nodes there would be a search for an # answer run() already has. self._batch_driven = bool(value) for node, _ in self._walk_nodes(): imp = getattr(node, "_imp", None) if imp is not None and hasattr(type(imp), "batch"): imp.batch = value def _raise_node_failure(self) -> None: """Raise if a node ended the run in an error condition. A backstop rather than the main path: in a batch run a node's exception propagates to the caller directly. This catches the case where something set the error condition without raising -- a monitoring thread, or a node that reported rather than threw -- so that a failed run cannot return an empty result and look successful. Raises: RuntimeError: If any node is in an error condition. """ # get_condition() is new on ioiocore's ProcessingElement. Until # it existed the getattr below found nothing on any node -- the # condition was reachable only as Pipeline.get_condition and as # node._imp._condition -- so `failed` was always empty and this # backstop had never once fired. A batch run whose node reported # rather than threw still returned an empty result and looked # like a success. failed = [ node for node, _ in self._walk_nodes() if getattr(node, "get_condition", None) and node.get_condition() == ioc.Constants.Conditions.ERROR ] if not failed: return entry = self.failure or self.get_last_error() detail = "." if entry is not None: detail = f": {entry['message'] if 'message' in entry else entry}" # The public name: a chain's core is what carries the condition, # so this list read "_CsvWriterCore" -- a symbol the public API # does not have, sending the reader after a class they cannot # find. names = ", ".join( sorted(naming.public_name(type(n).__name__) for n in failed) ) raise RuntimeError(f"Batch run failed in {names}{detail}") def _collect(self) -> Union[Result, dict, None]: """Gather what the pipeline's collectors kept. A node opts in by declaring ``COLLECTS``, rather than by happening to expose a ``result`` member -- the same reason ``TIME_BASE`` and ``is_externally_fed`` are declared: an accidental match is a defect nobody would look for. Returns: One result, a dict of them keyed by node name, or None. """ found = {} for node, _ in self._walk_nodes(): if not getattr(type(node), "COLLECTS", False): continue result = node.result if result is not None: found[node.name] = result if not found: return None if len(found) == 1: return next(iter(found.values())) return found def _replays_only(self) -> bool: """Whether every source of this run replays a recording. The owner, 2026-10-03: "replay should not be device dongled" (D-ENT-99). A replay is one of the shipped readers -- a recording reader, a CSV reader, a batch source such as the GTC reader -- or a ``load_from`` replay core. Decided by class, not by the declared time base, which any source can set. Every source must replay: a recording reader beside a live source is refused at start(), and ``load_from`` replaces every source, so a mix that reaches here is one this method must not exempt. Returns: True if there is a source and every one of them replays. """ from .sources.base.batch_source import BatchSource from .sources.base.raw import _RawReplayCore from .sources.base.recording_reader import RecordingReader from .sources.csv_reader import CsvReader replaying = (RecordingReader, CsvReader, BatchSource, _RawReplayCore) sources = self._sources() return bool(sources) and all( isinstance(node, replaying) for node in sources ) def _begin_deferred_device_check(self, verdict=None) -> None: """Check for a required device without delaying start. Two cases, and they want opposite treatment: *An amplifier source attested.* Its handle was challenged inside start(), before any sink wrote a header, and answered: the requirement is met and there is nothing to look for. *None did* -- no amplifier is a source, or one did not answer the challenge -- and presence alone is the requirement. Finding a device costs 5-9 s, dominated by driver enumeration, and much more on a loaded machine, so the pipeline starts and the check runs behind it; if no device answers, the run is stopped a few seconds in and told why. A source merely holding a handle is not enough: a source of any make can hold one. The accepted residual: such a run does emit a few seconds of output before it is stopped. A truncated recording is not a usable artifact, which is what the requirement protects, so this is weaker than blocking and much stronger than nothing. Nothing happens at all unless entitlement.device_required() says a device is a precondition, which it does for a frozen standalone application only (D-ENT-98): development pays for no probe. Nor does a replay (D-ENT-99). Args: verdict: The run's resolved entitlement, whose ``attestation`` says whether a device answered during start(). """ self._device_check = None if not device_required( frozen=licence.is_frozen_deployment(), residency=LaunchConfig.get().residency, ): return if ( verdict is not None and verdict.attestation is Permission.FULL and not getattr(self, "_entitlement_supplied", False) ): return # A replay needs no device -- unless the licence cannot be # determined, when a device is what lets it run (D-ENT-100). if self._replays_only() and not getattr( self, "_licence_unverified", None ): return if self.execution_mode() == Constants.ExecutionMode.BATCH: # A batch run can be over before a deferred answer arrives, # so it waits: run() blocks its caller anyway, and nothing # has cycled yet. Afresh every run, as D latches for a run. presence = device_probe.probe(force=True) if not presence.reachable: raise PermissionError( f"this application runs only with a g.tec device " f"attached: {presence.detail}" ) return def on_absent(presence) -> None: self.log( f"stopping: this application runs only with a g.tec " f"device attached: {presence.detail}", type=ioc.Constants.LogTypes.ERROR, ) # Recorded as the run's failure, so an application polling # `failure` can tell this stop from an operator's. entry = self.get_last_error() if entry is not None: self._record_failure(entry) try: self.stop() except Exception: pass def on_present(presence) -> None: self.log( f"g.tec device reachable: {presence.serial}", type=ioc.Constants.LogTypes.INFO, ) self._device_check = device_probe.DeferredCheck( on_absent=on_absent, on_present=on_present, force=True ) self._device_check.start() def _cancel_deferred_device_check(self) -> None: """Stop caring about a check still in flight. An operator who stops the pipeline before the probe returns has already decided; reporting the answer then would stop an already-stopped pipeline and log a failure about nothing. """ check = getattr(self, "_device_check", None) if check is not None: check.cancel() self._device_check = None #: How long after start() a server waits before naming the streams #: nothing at the far end has answered, in seconds. The Link's own #: context budget: the processes of one pipeline may be started in #: any order, and within it an edge that has not connected is still #: starting rather than missing. STREAM_REPORT_S = CONTEXT_TIMEOUT_S def _begin_stream_report(self) -> None: """On a server, schedule :meth:`_report_unanswered_streams`. Nothing else reports a node whose edge is not running (D-NODE-53). Its stream never arrives, so its receiving Link never sets up and its context timeout never fires; a sink's stream is pushed into an empty subscriber set without complaint; and a node downstream that merges the silent stream with a live one queues the live one without bound, until the only message is a generic input backlog. A timer rather than a wait, because start() must not charge every server run the budget, and once rather than repeatedly: the report is about the deployment, which does not change mid-run. """ from .core._private import threads self._cancel_stream_report() if LaunchConfig.get().residency != Constants.Residency.SERVER: return self._stream_report_since = time.monotonic() if not threads.available(): # A Timer is a thread; the event loop holds the report # instead, and its handle cancels the same way. self._stream_report = threads.call_later( self.STREAM_REPORT_S, self._report_unanswered_streams ) return timer = threading.Timer( self.STREAM_REPORT_S, self._report_unanswered_streams ) timer.daemon = True self._stream_report = timer timer.start() def _cancel_stream_report(self) -> None: """Drop a report still pending; a stopped run has nothing to say.""" timer = getattr(self, "_stream_report", None) self._stream_report = None if timer is not None: timer.cancel() def _unanswered_streams(self) -> list: """Every Link stream here that nothing at the far end answered. Returns: ``(node name, edge id, stream id, sent)`` per stream, where ``sent`` is True for a stream this process sends and nobody receives, False for one it receives and nobody sends. """ from .core._private import assembly, wrapping from .core._private.link import Link found = [] for element in self._elements: core = wrapping.core_of(element) name = (core if core is not None else element).name edge_id = assembly.edge_id_of(core) nodes = [element, *(getattr(element, "internal_nodes", ()) or ())] for node in nodes: if not isinstance(node, Link): continue sent = node.is_externally_drained() for stream in node.unanswered_streams(): found.append((name, edge_id, stream, sent)) return found def _report_unanswered_streams(self) -> None: """Name each stream no edge has sent or subscribed to, with its node. Run by the timer :meth:`_begin_stream_report` starts, and skipped once the run is over. """ if self.get_state() != ioc.Constants.States.RUNNING: return found = self._unanswered_streams() if not found: return parts = [] for name, edge_id, stream, sent in found: where = "no edge_id" if edge_id is None else f"edge {edge_id!r}" what = "has no edge receiving it" if sent else "has sent nothing" parts.append(f"{name!r} ({where}, stream {stream!r}) {what}") since = getattr(self, "_stream_report_since", None) when = ( "Since start" if since is None else f"{time.monotonic() - since:.0f} s after start" ) self.log( f"{when}, " + "; ".join(parts) + ". An edge builds a node only when the node names it or no " "edge, with ids matched exactly, so no process is running as " "that edge, it has not started yet, or it was started with " "another id. A node fed by a stream that sends nothing waits " "for it without bound.", type=Constants.LogTypes.WARNING, )
[docs] def stop(self): """Stop the pipeline and terminate all data processing. Gracefully shuts down all nodes, releasing threads and hardware connections. Always call stop() before program termination, especially when using hardware interfaces. Logging resources outlive a stop() so that a pipeline can be started again; call close() to release those as well. """ self._await_failing_stop() # Before super().stop(), so a check that returns during teardown # cannot call stop() a second time from its own thread. self._cancel_deferred_device_check() self._cancel_stream_report() self._stop_attestation() # And before the nodes go: a probe holds a callable on a port, # and leaving it there would attach it to the *next* run of this # pipeline -- publishing to a stream id whose subscriber is long # gone, from a pipeline the operator never pointed it at. self._detach_all_probes() try: super().stop() finally: # Discard the timeline so a later pipeline in this process # starts with no master and no epoch. release_timeline(self) release_entitlement(self) # And a verdict supplied for a run that never started, so it # cannot entitle an unrelated later one (D-ENT-61). discard_supplied(self)
[docs] def close(self): """Dispose the pipeline and release its master timeline. ioiocore's close() stops the pipeline through its own implementation, bypassing this class's stop(), so the timeline has to be discarded here as well. Safe to call repeatedly. """ self._await_failing_stop() self._cancel_deferred_device_check() self._cancel_stream_report() self._stop_attestation() try: super().close() finally: release_timeline(self) release_entitlement(self) # And a verdict supplied for a run that never started, so it # cannot entitle an unrelated later one (D-ENT-61). discard_supplied(self) self._release_devices()
def _await_failing_stop(self) -> None: """Let the monitor finish stopping a failed run first. :attr:`failure` is readable while ioiocore's monitor is still stopping the nodes (D-CORE-71), and ioiocore's ``stop()`` takes no lock, so a caller reacting to it stopped every node a second time, concurrently. Under ERROR, ioiocore's ``stop()`` and ``close()`` join the monitor anyway; this joins it before the nodes are touched. A join rather than a wait for STOPPED: a ``stop()`` that raises in the monitor leaves the state RUNNING, and this caller's stop is then the retry. ``start()`` joins it too (D-CORE-73). While the state reads RUNNING, ioiocore skips the start; once it reads STOPPED, the monitor has still to record the failure, and would record it on the new run. """ if self.get_condition() != ioc.Constants.Conditions.ERROR: return thread = getattr(self._imp, "_monitor_thread", None) if thread is not None and thread is not threading.current_thread(): thread.join() def _release_devices(self) -> None: """Close every exclusive driver handle this pipeline opened. A source's own stop() releases its device on the ordinary path, but that path is not always taken: ioiocore stops the pipeline from its monitor thread when a node errors, and Pipeline.start() can raise before anything runs -- the entitlement gate does exactly that. A handle left open then locks the device out of the next run and the next process, because the driver refuses a second exclusive session. Called by close(), and by start() when the entitlement gate refuses the run. Until 2026-09-26 only close() called it, so a refused start kept the handle the gate had opened, and stop() does not reach a node that never ran. Never raises: this is teardown, where a second failure would mask the first. """ for node in self._sources(): release = getattr(node, "_release_device", None) if callable(release): try: release() except Exception: pass #: Key under which a package bundle travels inside the payload. BUNDLE_KEY = "bundle" #: Key under which a document's pins travel: a list of #: ``name==version`` strings (D-CORE-109). Absent when there are none. REQUIREMENTS_KEY = "requirements" #: Key under which every fitted node's state travels inside the #: document, keyed by its own content digest (D-BATCH-91, LQ-F2): #: ``{sha256: {encoding, data, meta}}``. A fitted node's own config #: carries only the digest, under ``fittable.ARTIFACT_KEY``. ARTIFACTS_KEY = "artifacts"
[docs] def serialize( self, packages: Optional[list[str]] = None, requirements: Optional[list[str]] = None, ) -> dict: """Serialize the pipeline configuration to a dictionary. Args: packages: Top-level packages to carry along, for nodes whose classes are not installed on the executing host. A node from an uninstalled package cannot be rebuilt there, however complete its configuration is. requirements: Exact pins, ``name==version``, of what the document needs beyond g.Pype, written under :attr:`REQUIREMENTS_KEY`. A deployment that opted in installs them when it loads the document. The pipeline keeps them: a later call without them, and the provenance record of each run, write them again (D-CORE-109). None writes the pins kept, from an earlier call or the document this pipeline was rebuilt from, as that document carried them; an empty list writes none, and keeps none. Returns: dict: Dictionary containing the complete pipeline configuration, including nodes, connections, parameters, and metadata. Raises: ValueError: If the pipeline holds a function node -- ``Apply`` or an ``@gp.node`` -- naming it. A document could not rebuild it (D-BATCH-70). RequirementError: If a requirement is not an exact pin, or names a distribution twice, naming every such entry. """ self._refuse_script_only_nodes() if requirements is not None: pins = [str(pin) for pin in _installer.parse_pins(requirements)] else: pins = copy.deepcopy(getattr(self, "_requirements", None)) data = super().serialize() self._write_cores_not_chains(data) self._write_fitted_artifacts(data) # ioiocore records itself in `writer` but cannot know about # g.Pype, and g.Pype is the half whose node behaviour has # actually changed under an unchanged document -- a filter's # initial state, a transform's scaling, a decimator gaining an # anti-alias stage. So the version that matters most for # attributing a difference is this one. from ioiocore.imp.pipeline_imp import PipelineImp writer_key = PipelineImp.SerializationKeys.WRITER writer = dict(data.get(writer_key) or {}) try: from ..__version__ import __version__ as gpype_version writer["gpype"] = str(gpype_version) except Exception: # pragma: no cover pass if writer: data[writer_key] = writer if packages: from ..common._private import bundle as bundle_module data[self.BUNDLE_KEY] = bundle_module.collect(packages) if pins: data[self.REQUIREMENTS_KEY] = pins if requirements is not None: # Kept once the document is written, so a refused call keeps # what was there. self._requirements = list(pins) or None return data
def _script_only_nodes(self) -> list: """The nodes a document of this pipeline could not rebuild. A node declaring ``SCRIPT_ONLY`` -- ``Apply``, and so every ``@gp.node`` -- holds a Python function as instance state. A document would rebuild it without one, and the load fails with ``missing 1 required positional argument: 'function'``. The one answer for every caller that must not write or re-run a document of such a pipeline, ``serialize()`` first. The pipeline's own nodes count, and the internal nodes of a chain that writes them into the document (``INTERNALS_ARE_DERIVED = False``). A chain whose internals are derived rebuilds them from its configuration, so a function node it holds is its class's to supply. Returns: The nodes, in the order the pipeline holds them. """ found: list = [] def visit(node) -> None: if getattr(type(node), "SCRIPT_ONLY", False): found.append(node) internal = getattr(node, "internal_nodes", None) if internal and not getattr( type(node), "INTERNALS_ARE_DERIVED", True ): for inner in internal: visit(inner) for node in list(getattr(self._imp, "_nodes", None) or []): visit(node) return found def _refuse_script_only_nodes(self) -> None: """Refuse to write a document that could not be rebuilt. Raises: ValueError: If the pipeline holds a function node, naming each. """ found = self._script_only_nodes() if not found: return names = ", ".join( f"'{node.name}' ({naming.public_name(type(node).__name__)})" for node in found ) raise ValueError( f"this pipeline cannot be written to a document: it holds a " f"function node, {names}. A function is not a configuration " f"value, so a document could not rebuild the node. Run the " f"pipeline from the script that builds it, or write a node " f"class whose parameters are configuration." ) def _write_cores_not_chains(self, data: dict) -> None: """Record the core each chain of ours carries, not the chain. **The rule a distributed pipeline already lives by**: one document describes the whole system, and each process builds its own part of it. A chain *is* that per-process part -- which Link exists, whether the core is run at all -- so a document that named the chain would be describing one process's answer and handing it to every other. Writing the core keeps the document saying what the author wrote, and leaves each reader to wrap it for its own residency. **Connections become name references.** A chain exposes its boundary internal node's ports as its own, so a source chain's output port id *is* its `Sync`'s -- a node built fresh on every load. Recorded as an id it could never resolve again. ioiocore resolves ``"eeg.out"`` by name for exactly this reason, and a name is what the author wrote in the first place. Does nothing when no element is one of our chains. Args: data: The document from ``super().serialize()``, updated in place. """ from ioiocore.imp.pipeline_imp import PipelineImp from .core._private import wrapping # Walked over ioiocore's own node list, which is what # ``super().serialize()`` built the document from -- element for # element, in that order. ``_elements`` is a different list: it # holds what passed through ``connect``, and ``add_node`` adds to # the pipeline without registering there. Indexing one against the # other replaced the wrong node, measured: an ``add_node``ed # Bandpass became the Generator core and the real chain stayed a # ``SourceChain``. elements = list(getattr(self._imp, "_nodes", None) or []) SK = PipelineImp.SerializationKeys nodes = list(data.get(SK.NODES) or []) if len(elements) != len(nodes): # pragma: no cover # Nothing can be matched up safely; leave the document as # ioiocore wrote it rather than corrupt it. return # Port id -> "node.port", for every boundary port of every chain # we are about to replace. renamed: dict = {} replaced = False seen: dict = {} for index, element in enumerate(elements): core = wrapping.core_of(element) if core is None: continue replaced = True nodes[index] = core.serialize() name = (core.config or {}).get("name") seen.setdefault(name, []).append(core) for port_id, port_name in wrapping.boundary_port_ids( element ).items(): renamed[port_id] = f"{name}.{port_name}" if not replaced: return self._refuse_ambiguous_names(seen) data[SK.NODES] = nodes data[SK.CONNECTIONS] = [ [renamed.get(source, source), renamed.get(target, target)] for source, target in (data.get(SK.CONNECTIONS) or []) ] @staticmethod def _refuse_ambiguous_names(by_name: dict) -> None: """Refuse to write a document whose endpoints would collide. **Node names become load-bearing the moment a chain is assembly.** A connection to one is written as ``"eeg.out"`` rather than as a port id, because the id belongs to a `Sync` rebuilt on every load (see ``_write_cores_not_chains``). Two nodes of one class that nobody named take ioiocore's default name -- their class name -- so they produce the *same* endpoint string, and the document stops recording which is which. Measured: a source feeding two unnamed `_CsvWriterCore`s writes ``[["eeg.out", "_CsvWriterCore.in"], ["eeg.out", "_CsvWriterCore.in"]]`` -- two byte-identical entries. Loading it does fail, because ioiocore drops an ambiguous name rather than picking the first, but it fails in whoever opens the file rather than in whoever wrote it, and by then the information is gone. So it is refused here, where the author can still fix it by naming the nodes. The same argument ``chain_params`` already makes about stream ids: an id is unique by construction, a name is not, and silently crossing two streams is worse than a message. Args: by_name: Node name to the cores carrying it. Raises: ValueError: If any name is carried by more than one wrapped node, naming the class and the count. """ clashes = { name: cores for name, cores in by_name.items() if len(cores) > 1 } if not clashes: return detail = "; ".join( f"{len(cores)} x {type(cores[0]).__name__} named {name!r}" for name, cores in sorted( clashes.items(), key=lambda kv: str(kv[0]) ) ) raise ValueError( f"this pipeline cannot be written to a document: {detail}. " f"A connection to a node the pipeline builds a chain around " f"is recorded by name, because the port ids such a chain " f"exposes belong to internal nodes rebuilt on every load. " f"Two nodes sharing a name therefore produce the same " f"endpoint and the document stops saying which is which. " f"Give each of them a distinct name." ) def _fittable_nodes(self) -> list: """Every :class:`~gpype.backend.core.fittable.Fittable` node here. Returns: The nodes, chain internals included (none are, today: a fittable node is a plain transform, never wrapped). """ from .core.fittable import Fittable return [ node for node, _ in self._walk_nodes() if isinstance(node, Fittable) ] def _write_fitted_artifacts(self, data: dict) -> None: """Write every fitted node's state into the document (LQ-F2). Each fitted node's state becomes one entry of ``data[Pipeline.ARTIFACTS_KEY]``, keyed by its own content digest, and that digest is written back into the node's own serialized config under ``fittable.ARTIFACT_KEY`` -- so a node with no state (never fitted) serializes exactly as before. Args: data: The document from ``super().serialize()``, updated in place. """ fitted = [n for n in self._fittable_nodes() if n.state is not None] if not fitted: return from ..common._private import artifacts as artifacts_module from .core.fittable import ARTIFACT_KEY elements = list(getattr(self._imp, "_nodes", None) or []) nodes = list(data.get("nodes") or []) section = dict(data.get(self.ARTIFACTS_KEY) or {}) for node in fitted: artifact_entry = artifacts_module.encode_state(node.state) sha256 = artifacts_module.digest_of(artifact_entry) section[sha256] = artifact_entry try: index = next( i for i, element in enumerate(elements) if element is node ) except StopIteration: # pragma: no cover - defensive continue # ioiocore's own ``Configuration`` is read-only # (``__setitem__`` refuses), so the node's serialized entry # and its config are rebuilt as plain dicts rather than # mutated in place. node_entry = dict(nodes[index]) config = dict(node_entry.get("config") or {}) config[ARTIFACT_KEY] = sha256 node_entry["config"] = config nodes[index] = node_entry data[self.ARTIFACTS_KEY] = section data["nodes"] = nodes @staticmethod def _rebuild(data: dict): """Rebuild the pipeline, wrapping each core on the way in. ``ioc.Pipeline.deserialize`` builds the nodes and wires them in one pass, with no point between the two where a node can be substituted -- and substituting is the whole job here, because a document records the **core** and every process has to put its own chain around it. So the pass is done here instead: build, wrap, add, then wire. Everything else is ioiocore's, deliberately. The format-version refusal, and the id-then-name endpoint resolution, are imported rather than reimplemented: a second implementation of *which documents load* is exactly the kind that drifts one release later and refuses a document the writer thought it had written. A document written before step 4 of ``chain-assembly`` names a class that is now the core, and records the chain's constructor call as its settings; it is built and wrapped like any other. A document naming a hand-written chain gets it back untouched. Args: data: The document. Returns: An ``ioc.Pipeline`` with every node in place. Raises: TypeError: If a connection endpoint cannot be resolved, naming the endpoints as the document wrote them. """ from ioiocore.imp.pipeline_imp import ( PipelineImp, _index_nodes, _resolve_port, ) from .core._private import assembly, wrapping SK = PipelineImp.SerializationKeys built = ioc.Pipeline(directory=_fallback_log_directory()) built.log("Starting pipeline deserialization...") PipelineImp._check_format_version(data, built) nodes = [] for setting in data[SK.NODES]: # Constructed from what the document stored, so a node that # checks its settings against its world can tell them from # an author's (D-NODE-61). with assembly.rebuilding(): node = ioc.Portable.deserialize(setting) node = wrapping.wrap(node) nodes.append(node) built.add_node(node) # Indexed on what was built, so a chain answers to the name its # core carried -- which is why a wrapper adopts it. by_name, ambiguous = _index_nodes(nodes) # An endpoint recorded as a port **id** names a port of the node # the document describes -- which, once that node is wrapped, is # a port *inside* the chain rather than the chain's own. It would # resolve, silently, to the wrong place: a consumer wired # upstream of the `Sync`, so the arrival-stamp channel is never # stripped and nothing is placed on the master timeline, with the # Sync left dangling and no error raised. # # Measured by rewriting a stored 4.0.0 document's class name to # its core, which is exactly what step 4 does to every document # already on disk: endpoint 2371E226... resolved to # `_GeneratorCore.out` while the chain's boundary `Sync.out` was # in no connection at all. # # Translated rather than refused, because those documents are # correct -- it is the meaning of "the node" that moved. Straight # to the chain's port, not through its name: a document that # connected by id may well hold two nodes of one class that # nobody named, and a name they share resolves to neither. translated = {} for node in nodes: core = wrapping.core_of(node) if core is None: continue for port_id, port_name in wrapping.boundary_port_ids(core).items(): translated[port_id] = (node, port_name) def resolve(ref, output: bool): hit = translated.get(ref) if hit is None: return _resolve_port(ref, by_name, ambiguous, output=output) chain, port_name = hit if output: return chain.get_output_port(port_name) return chain.get_input_port(port_name) for source, target in data[SK.CONNECTIONS]: try: oport = resolve(source, output=True) iport = resolve(target, output=False) oport.connect(iport) try: iport.connect(oport) except Exception: oport.disconnect(iport) raise built.log(f"Connected ports {source} and {target}.") except Exception as exc: message = ( f"Deserialization failed: {exc} " f"(connecting {source!r} to {target!r})" ) built.log(msg=message, type=Constants.LogTypes.ERROR) raise TypeError(message) from exc built.log("Deserialization complete.") return built @staticmethod def _named_modules(data: dict) -> list: """The modules a document's nodes name, in document order. Args: data: A serialized pipeline document. Returns: Every string module name, malformed entries skipped; the loader reports those in its own terms. """ from ioiocore.imp.pipeline_imp import PipelineImp from ioiocore.imp.portable_imp import PortableImp nodes = data.get(PipelineImp.SerializationKeys.NODES) or [] if not isinstance(nodes, (list, tuple)): return [] return [ setting.get(PortableImp.SerializationKeys.MODULE) for setting in nodes if isinstance(setting, dict) and isinstance( setting.get(PortableImp.SerializationKeys.MODULE), str ) ]
[docs] @staticmethod def check(data: dict, install: bool = False) -> None: """Refuse a document the allow-list or requirement check refuses. The document-level half of :meth:`deserialize`, which calls it first, so the order is written only here. The modules the document's nodes name are checked against the deployment's allow-list. Then, where the deployment installs a document's pins (``GPYPE_PACKAGE_INDEX`` is set) and the document has some, they are checked for form and against what is installed, and, with *install*, every absent one is installed. Last, for a document carrying packages, their code is checked for imports this host cannot satisfy. Nothing else happens: no bundle is unpacked, nothing is put on the import path, no module a node names is imported and no node is built. A validator can therefore ask whether a check would refuse a document, in the terms loading it uses, without loading it. The load can still refuse a document that passes: for its format revision, or with a ``BundleError`` as its bundle is unpacked. Without *install*, pip never runs, and a pin that is absent here may be what supplies a missing import; so while any pin is absent, the missing imports are not refused. The load installs the pins first and refuses them then. Args: data (dict): Serialized pipeline configuration dictionary. install (bool): Install the document's absent pins, where the deployment opted in. :meth:`deserialize` passes True. Raises: ModuleNotAllowedError: If the document names a module this deployment does not permit, naming every such module. RequirementError: If a pin is not exact, or a pinned wheel needs a distribution nothing pins or provides. RequirementConflictError: If a pin names a version other than the one installed here, or, with *install*, a pinned wheel needs another version of an installed distribution or would overwrite its files. InstallerError: If the pins cannot be installed. MissingRequirementsError: If the document's bundled code imports a package this host does not have, naming every such package. """ from ..common._private import allowlist as _allowlist _allowlist.check_document(data) pinned = _installer.for_document( data.get(Pipeline.REQUIREMENTS_KEY), install_absent=install ) if data.get(Pipeline.BUNDLE_KEY): from ..common._private import bundle as bundle_module try: bundle_module.check_requirements( data[Pipeline.BUNDLE_KEY], Pipeline._named_modules(data) ) except bundle_module.MissingRequirementsError as error: if pinned is not None and pinned["absent"] and not install: return hint = _installer.missing_hint( pinned, data.get(Pipeline.REQUIREMENTS_KEY) ) raise bundle_module.MissingRequirementsError( f"{error} {hint}" ) from None
[docs] @staticmethod def restore_requirements() -> dict: """Reinstall the pins the package cache records, with no network. For the process that starts a deployment: in a container the cache is a volume and the environment is the image's, so the pins installed before a restart are installed again from the cache, each wheel copied out of it and checked against the hash recorded at its download (D-CORE-88, D-CORE-117). Entry by entry: one whose distribution is installed at another version now is skipped and reported, as is one whose wheel this interpreter does not install, is missing or was replaced, one pip refuses or that would overwrite another distribution's files, and one whose own pip call does not finish within 600 s; the others are installed. Does nothing where ``GPYPE_PACKAGE_INDEX`` is not set. Whoever can write the cache can have a wheel installed here, with no document involved: the cache must be writable only by the deployment. Returns: dict: ``{"enabled": bool, "installed": [...], "present": [...], "skipped": [{"requirement": str, "reason": str}]}``, JSON-safe. Raises: InstallerError: Before anything is installed, if the cache's manifest cannot be read, another install holds the cache for longer than 1200 s, this interpreter's wheel tags cannot be read, or this interpreter, its site-packages or its pip cannot install. At any point, possibly after some entries were installed, which the next call reports as present, if pip cannot be started or the cache or a private directory beside pip cannot be used. """ return _installer.restore()
[docs] @staticmethod def installer_config() -> dict: """Whether this deployment installs a document's pins, and how. Read once from ``GPYPE_PACKAGE_INDEX`` and ``GPYPE_PACKAGE_CACHE``, for a control plane to report. Nothing can set it but the deployment's environment. Returns: dict: ``{"enabled": bool, "index": str or None, "cache": str}``, JSON-safe, the index's credentials redacted. """ return _installer.configuration()
[docs] @staticmethod def deserialize(data: dict) -> "Pipeline": """Deserialize a pipeline configuration from a dictionary. Rebuilding a node imports the module it names, so a document is executable input. It is checked first (:meth:`check`, with ``install=True``): the modules it names against the deployment's allow-list; then its pins, installing every absent one where the deployment opted in; then any bundled code for imports this host cannot satisfy -- all before any bundle is written to disk and put on the import path, so an untrusted payload is never unpacked for a document that was going to be refused. Without the opt-in, nothing is installed for it. Any packages carried in the payload are then unpacked and put on the import path, because rebuilding a node imports its class. The rebuilt pipeline keeps the document's pins as the document carried them, and its :meth:`serialize` writes them again; they are validated where they are installed. Args: data (dict): Serialized pipeline configuration dictionary. Returns: Pipeline: A new Pipeline instance with the specified configuration. Raises: ModuleNotAllowedError: If the document names a module this deployment does not permit. RequirementError, RequirementConflictError, InstallerError: If the deployment installs pins and this document's cannot be installed (:meth:`check`). MissingRequirementsError: If the document's bundled code imports a package this host does not have. ModuleNotFoundError: If a node's module cannot be imported; for a document with pins where the deployment installs none, saying that they were not installed. """ Pipeline.check(data, install=True) if data.get(Pipeline.BUNDLE_KEY): from ..common._private import bundle as bundle_module bundle_module.install(data[Pipeline.BUNDLE_KEY]) # Deserialize using parent class. A node module its pins would # have supplied fails here where the deployment installs none, # so the refusal says so (D-CORE-111). try: ioc_pipeline = Pipeline._rebuild(data) except ModuleNotFoundError as error: hint = _installer.pins_not_installed( data.get(Pipeline.REQUIREMENTS_KEY) ) if hint is None: raise raise ModuleNotFoundError( f"{error} {hint}", name=error.name, path=error.path ) from error # Create Pipeline instance without calling __init__ pipeline = object.__new__(Pipeline) # Copy all instance attributes from deserialized pipeline pipeline.__dict__.update(ioc_pipeline.__dict__) # __init__ was skipped, so this class's own state does not exist # yet. Established through the same method __init__ uses, never a # second hand-written list -- that is exactly how the previous # version fell behind. pipeline._init_gpype_state() pipeline._requirements = copy.deepcopy( data.get(Pipeline.REQUIREMENTS_KEY) ) # One thing genuinely differs from a fresh pipeline: the elements # come from ioiocore's own node list rather than starting empty. # They are what connect() would have recorded, and an empty list # would leave every Sync in a restored pipeline without a # timeline, so no event could ever be placed. imp = getattr(ioc_pipeline, "_imp", None) pipeline._elements = list(getattr(imp, "_nodes", None) or []) # The chains were built in _rebuild, before this pipeline # existed, so the map `chain_of` reads is empty unless it is # filled here. Without it a restored SERVER pipeline answers None # for every source it carries: the server runs no source core, # so walking `internal_nodes` finds nothing. from .core._private import wrapping for element in pipeline._elements: core = wrapping.core_of(element) if core is not None: pipeline._wrappers[id(core)] = element pipeline._load_fitted_artifacts(data) # Loading is when a server's reference is zeroed (D-TIME-20), so # a server that only deserializes -- a test, an example -- zeroes # it as the runtime does. Skipped while another pipeline here is # running rather than refused: a document read to be checked # must not fail, nor move a running pipeline's clock. launch = LaunchConfig.get() if ( launch.residency == Constants.Residency.SERVER and not Pipeline._running_pipelines() ): Pipeline.rezero_reference() return pipeline
def _load_fitted_artifacts(self, data: dict) -> None: """Load every fitted node's state out of the document (LQ-F2). Args: data: The document just rebuilt from. Raises: ValueError: If a node names an artifact the document does not carry, or one whose stored digest disagrees with its own content (edited or corrupted after it was written). """ section = data.get(self.ARTIFACTS_KEY) or {} if not section: return from ..common._private import artifacts as artifacts_module from .core.fittable import ARTIFACT_KEY for node in self._fittable_nodes(): sha256 = getattr(node, "_gpype_artifact_ref", None) if not sha256: continue label = naming.node_label(node) entry = section.get(sha256) if entry is None: raise ValueError( f"{label} names artifact {sha256!r} ({ARTIFACT_KEY} " f"in its configuration), but this document's " f"{self.ARTIFACTS_KEY!r} section does not carry it." ) state = artifacts_module.verify_and_decode(sha256, entry, label) node._gpype_state = state node._gpype_marked = bool(state.get("marked", False)) node.load_state(state)