From 21470012e8c480f35acd99717d08146f20025557 Mon Sep 17 00:00:00 2001 From: jegp Date: Tue, 23 Sep 2025 15:10:10 +0200 Subject: [PATCH 1/5] Added support for neuromrophic drivers --- examples/camera_to_aedat4.py | 7 -- examples/camera_to_stdout.py | 7 ++ pyproject.toml | 4 ++ python/faery/event_camera_input.py | 101 +++++++++++++++++++++-------- python/faery/events_stream.py | 1 + 5 files changed, 87 insertions(+), 33 deletions(-) delete mode 100644 examples/camera_to_aedat4.py create mode 100644 examples/camera_to_stdout.py diff --git a/examples/camera_to_aedat4.py b/examples/camera_to_aedat4.py deleted file mode 100644 index a6b52f3..0000000 --- a/examples/camera_to_aedat4.py +++ /dev/null @@ -1,7 +0,0 @@ -import faery - -( - faery.events_stream_from_camera("Inivation") # Open an Inivation camera - .crop(10, 110, 10, 110) # Remove events outside the region (10, 10) to (110, 110) - .to_file("some.aedat4") # Save the events to a file -) diff --git a/examples/camera_to_stdout.py b/examples/camera_to_stdout.py new file mode 100644 index 0000000..38af113 --- /dev/null +++ b/examples/camera_to_stdout.py @@ -0,0 +1,7 @@ +import faery + +( + faery.events_stream_from_camera() # Open an event camera + .crop(10, 110, 10, 110) # Remove events outside the region (10, 10) to (110, 110) + .to_stdout() # Print the events to stdout +) diff --git a/pyproject.toml b/pyproject.toml index 6211199..5ab7566 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,10 @@ dev = [ "pyright", "jupyter-book>=2.0.0a0", ] +camera = [ + "neuromorphic-drivers", + "event-camera-drivers" +] [project.scripts] faery = "faery:__main__.main" diff --git a/python/faery/event_camera_input.py b/python/faery/event_camera_input.py index f8d0742..e2c80d4 100644 --- a/python/faery/event_camera_input.py +++ b/python/faery/event_camera_input.py @@ -10,57 +10,106 @@ def has_event_camera_drivers(): return importlib.util.find_spec("event_camera_drivers") is not None +def has_neuromorphic_drivers(): + return importlib.util.find_spec("neuromorphic_drivers") is not None -def has_inivation_camera_drivers(): - if has_event_camera_drivers(): - import event_camera_drivers as evd # type: ignore - - return hasattr(evd, "InivationCamera") - return False - - -class InivationCameraStream(events_stream.EventsStream): - def __init__(self, buffer_size: int = 1024): - """Create an events stream from a connected Inivation camera +class EventCameraDriverStream(events_stream.EventsStream): + def __init__(self, + manufacturer: typing.Optional[typing.Literal["Prophesee", "Inivation"]] = None, + buffer_size: int = 1024 + ): + """Create an events stream using the event-camera-drivers library Args: - buffer_size: The size of the buffer to use for the event stream, defaults to 1024 + manufacturer (Optional[typing.Literal["Prophesee", "Inivation"]]): Camera manufacturer. + Defaults to automatic detection which might take some time. + buffer_size (int): The size of the buffer to use for the event stream, defaults to 1024 Returns: An (infinite) event stream from the camera Usage: - >>> stream = InivationCameraStream() # Open camera (will fail if no camera is connected) - >>> stream.map(...) # Use the stream as any other (infinite) event stream + >>> stream = EventCameraStream() # Open camera (will fail if no camera is connected) + >>> stream.map(...) # Use the stream as any other (infinite) event stream """ super().__init__() - try: import event_camera_drivers as evd # type: ignore self.camera = evd.InivationCamera(buffer_size=buffer_size) except ImportError as e: logging.error( - "Inivation camera drivers not available, please install the event_camera_drivers library" + "The event_camera_drivers library is not available, please install" ) raise e + except ValueError as e: + logging.warning("No camera found using libcaer") + raise e + + def is_running(self): + return self.camera.is_running() def __iter__(self) -> typing.Iterator[np.ndarray]: - while self.camera.is_running(): - v = next(self.camera) - yield v + while self.is_running(): + v = next(self.camera) + yield v def dimensions(self): return self.camera.resolution() +class NeuromorphicCameraStream(events_stream.EventsStream): + def __init__(self): + try: + import neuromorphic_drivers as nd + self.nd = nd + self.device_list = nd.list_devices() + if len(self.device_list) == 0: + raise RuntimeError("No event camera found, did you plug it in and install the udev rules?") + except ImportError as e: + logging.error( + "The neuromorphic_drivers library is not available, please install" + ) + except Exception as e: + raise e + + def __iter__(self) -> typing.Iterator[np.ndarray]: + with self.nd.open() as device: + for status, packet in device: + # TODO: Check status + yield packet + + def dimensions(self): + # TODO: Use the current camera, rather than device_list[0] + if self.device_list[0].name == self.nd.generated.enums.Name.INIVATION_DVXPLORER: + return (640, 480) + elif self.device_list[0].name == self.nd.generated.enums.Name.PROPHESEE_EVK4 or self.device_list[0].name == self.nd.generated.enums.Name.PROPHESEE_EVK3_HD: + return (1280, 720) + elif self.device_list[0].name == self.nd.generated.enums.Name.INIVATION_DAVIS346: + return (346, 260) + else: + raise ValueError("Unknown Camera", self.device_list[0].name) + def events_stream_from_camera( - manufacturer: typing.Literal["Inivation", "Prophesee"], buffer_size: int = 1024 + driver: typing.Optional[typing.Literal["EventCameraDrivers", "NeuromorphicDrivers", "Auto"]] = None, + manufacturer: typing.Optional[typing.Literal["Inivation", "Prophesee"]] = None, + buffer_size: int = 1024 ): - if manufacturer == "Inivation": - return InivationCameraStream(buffer_size) - elif manufacturer == "Prophesee": - raise NotImplementedError("Prophesee camera drivers are not implemented yet") - else: - raise ValueError(f"Unknown camera manufacturer: {manufacturer}") + stream = None + error = None + if driver is None or driver == "EventCameraDrivers": + try: + stream = EventCameraDriverStream(manufacturer=manufacturer, buffer_size=buffer_size) + except Exception as e: + error = e + if driver is None or driver == "NeuromorphicDrivers": + try: + stream = NeuromorphicCameraStream() + except Exception as e: + error = e + + if stream is None: + raise ValueError("No Event Camera found:", error) + + return stream diff --git a/python/faery/events_stream.py b/python/faery/events_stream.py index fa844af..9c0151a 100644 --- a/python/faery/events_stream.py +++ b/python/faery/events_stream.py @@ -176,6 +176,7 @@ def to_stdout( csv_header=csv_header, file_type="csv", on_progress=on_progress, # type: ignore + enforce_monotonic_timestamps=False ) def to_udp( From d57574de9def9ccea76cc6038fddb17dfd61f951 Mon Sep 17 00:00:00 2001 From: jegp Date: Wed, 24 Sep 2025 10:07:11 +0200 Subject: [PATCH 2/5] Properly cast ND events --- python/faery/event_camera_input.py | 59 +++++++++++++++++++----------- python/faery/file_decoder.py | 4 +- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/python/faery/event_camera_input.py b/python/faery/event_camera_input.py index e2c80d4..7472a2c 100644 --- a/python/faery/event_camera_input.py +++ b/python/faery/event_camera_input.py @@ -10,13 +10,16 @@ def has_event_camera_drivers(): return importlib.util.find_spec("event_camera_drivers") is not None + def has_neuromorphic_drivers(): return importlib.util.find_spec("neuromorphic_drivers") is not None + class EventCameraDriverStream(events_stream.EventsStream): - def __init__(self, + def __init__( + self, manufacturer: typing.Optional[typing.Literal["Prophesee", "Inivation"]] = None, - buffer_size: int = 1024 + buffer_size: int = 1024, ): """Create an events stream using the event-camera-drivers library @@ -32,8 +35,6 @@ def __init__(self, >>> stream = EventCameraStream() # Open camera (will fail if no camera is connected) >>> stream.map(...) # Use the stream as any other (infinite) event stream """ - - super().__init__() try: import event_camera_drivers as evd # type: ignore @@ -44,16 +45,13 @@ def __init__(self, ) raise e except ValueError as e: - logging.warning("No camera found using libcaer") + logging.info("No camera found using libcaer") raise e - def is_running(self): - return self.camera.is_running() - def __iter__(self) -> typing.Iterator[np.ndarray]: - while self.is_running(): - v = next(self.camera) - yield v + while self.camera.is_running(): + v = next(self.camera) + yield v def dimensions(self): return self.camera.resolution() @@ -62,11 +60,14 @@ def dimensions(self): class NeuromorphicCameraStream(events_stream.EventsStream): def __init__(self): try: - import neuromorphic_drivers as nd - self.nd = nd - self.device_list = nd.list_devices() - if len(self.device_list) == 0: - raise RuntimeError("No event camera found, did you plug it in and install the udev rules?") + import neuromorphic_drivers as nd + + self.nd = nd + self.device_list = nd.list_devices() + if len(self.device_list) == 0: + raise RuntimeError( + "No event camera found, did you plug it in and install the udev rules?" + ) except ImportError as e: logging.error( "The neuromorphic_drivers library is not available, please install" @@ -78,29 +79,43 @@ def __iter__(self) -> typing.Iterator[np.ndarray]: with self.nd.open() as device: for status, packet in device: # TODO: Check status - yield packet + events = packet.polarity_events + if events is not None: + # Neuromorphic drivers use similar dtype, so we can safely cast + yield events.astype(events_stream.EVENTS_DTYPE) def dimensions(self): # TODO: Use the current camera, rather than device_list[0] if self.device_list[0].name == self.nd.generated.enums.Name.INIVATION_DVXPLORER: return (640, 480) - elif self.device_list[0].name == self.nd.generated.enums.Name.PROPHESEE_EVK4 or self.device_list[0].name == self.nd.generated.enums.Name.PROPHESEE_EVK3_HD: + elif ( + self.device_list[0].name == self.nd.generated.enums.Name.PROPHESEE_EVK4 + or self.device_list[0].name + == self.nd.generated.enums.Name.PROPHESEE_EVK3_HD + ): return (1280, 720) - elif self.device_list[0].name == self.nd.generated.enums.Name.INIVATION_DAVIS346: + elif ( + self.device_list[0].name == self.nd.generated.enums.Name.INIVATION_DAVIS346 + ): return (346, 260) else: raise ValueError("Unknown Camera", self.device_list[0].name) + def events_stream_from_camera( - driver: typing.Optional[typing.Literal["EventCameraDrivers", "NeuromorphicDrivers", "Auto"]] = None, + driver: typing.Optional[ + typing.Literal["EventCameraDrivers", "NeuromorphicDrivers", "Auto"] + ] = None, manufacturer: typing.Optional[typing.Literal["Inivation", "Prophesee"]] = None, - buffer_size: int = 1024 + buffer_size: int = 1024, ): stream = None error = None if driver is None or driver == "EventCameraDrivers": try: - stream = EventCameraDriverStream(manufacturer=manufacturer, buffer_size=buffer_size) + stream = EventCameraDriverStream( + manufacturer=manufacturer, buffer_size=buffer_size + ) except Exception as e: error = e if driver is None or driver == "NeuromorphicDrivers": diff --git a/python/faery/file_decoder.py b/python/faery/file_decoder.py index 8662da3..f575a7c 100644 --- a/python/faery/file_decoder.py +++ b/python/faery/file_decoder.py @@ -296,9 +296,7 @@ def __iter__(self) -> collections.abc.Iterator[numpy.ndarray]: ) elif self.file_type == "es": assert self.path is not None - with es.Decoder( - path=self.path, t0=self.t0.to_microseconds() - ) as decoder: + with es.Decoder(path=self.path, t0=self.t0.to_microseconds()) as decoder: if self.event_type == "atis": for atis_events in decoder: mask = numpy.logical_not(atis_events["exposure"]) From 8608589e94af6304f2d5208717610e601a07df21 Mon Sep 17 00:00:00 2001 From: jegp Date: Wed, 24 Sep 2025 10:07:29 +0200 Subject: [PATCH 3/5] Fixed black version and formatted --- pyproject.toml | 2 +- python/faery/cli/colormaps.py | 2 +- python/faery/events_filter.py | 2 +- python/faery/events_render.py | 2 +- python/faery/events_stream.py | 2 +- python/faery/task.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ab7566..bcf2382 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dev = [ "maturin", "pytest", "isort", - "black", + "black==25.1.0", "pyright", "jupyter-book>=2.0.0a0", ] diff --git a/python/faery/cli/colormaps.py b/python/faery/cli/colormaps.py index aacaecf..dbbe00b 100644 --- a/python/faery/cli/colormaps.py +++ b/python/faery/cli/colormaps.py @@ -105,7 +105,7 @@ def run(self, arguments: list[str]): type_to_names_and_colormaps.keys(), key=lambda colormap_type: ( # show cyclic maps last - "\U0010FFFD" + "\U0010fffd" if colormap_type == "cyclic" else colormap_type ), diff --git a/python/faery/events_filter.py b/python/faery/events_filter.py index 23a5328..d145f63 100644 --- a/python/faery/events_filter.py +++ b/python/faery/events_filter.py @@ -34,7 +34,7 @@ def decorator(method): def typed_filter( - prefixes: set[typing.Literal["", "Finite", "Regular", "FiniteRegular"]] + prefixes: set[typing.Literal["", "Finite", "Regular", "FiniteRegular"]], ): def decorator(filter_class): attributes = [ diff --git a/python/faery/events_render.py b/python/faery/events_render.py index 189ac6e..996d6bd 100644 --- a/python/faery/events_render.py +++ b/python/faery/events_render.py @@ -25,7 +25,7 @@ def decorator(method): def typed_render( - prefixes: set[typing.Literal["", "Finite", "Regular", "FiniteRegular"]] + prefixes: set[typing.Literal["", "Finite", "Regular", "FiniteRegular"]], ): def decorator(render_class): attributes = [ diff --git a/python/faery/events_stream.py b/python/faery/events_stream.py index 9c0151a..6306788 100644 --- a/python/faery/events_stream.py +++ b/python/faery/events_stream.py @@ -176,7 +176,7 @@ def to_stdout( csv_header=csv_header, file_type="csv", on_progress=on_progress, # type: ignore - enforce_monotonic_timestamps=False + enforce_monotonic_timestamps=False, ) def to_udp( diff --git a/python/faery/task.py b/python/faery/task.py index 23ec1a6..6c0695e 100644 --- a/python/faery/task.py +++ b/python/faery/task.py @@ -78,7 +78,7 @@ def task_generator( timestamp.TimeOrTimecode, ], None, - ] + ], ) -> Task: class DecoratedTask(Task): From d2d0fb4974e6645e33c99ce4afe795d2ce786b2f Mon Sep 17 00:00:00 2001 From: jegp Date: Wed, 24 Sep 2025 10:16:13 +0200 Subject: [PATCH 4/5] Updated CLI api for camera --- examples/camera_to_stdout.py | 2 +- python/faery/cli/process.py | 27 +++++++++++++++++---------- python/faery/event_camera_input.py | 4 ++-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/examples/camera_to_stdout.py b/examples/camera_to_stdout.py index 38af113..d69f8fc 100644 --- a/examples/camera_to_stdout.py +++ b/examples/camera_to_stdout.py @@ -1,7 +1,7 @@ import faery ( - faery.events_stream_from_camera() # Open an event camera + faery.events_stream_from_camera() # Open an event camera, if available .crop(10, 110, 10, 110) # Remove events outside the region (10, 10) to (110, 110) .to_stdout() # Print the events to stdout ) diff --git a/python/faery/cli/process.py b/python/faery/cli/process.py index c0588c1..39c87f7 100644 --- a/python/faery/cli/process.py +++ b/python/faery/cli/process.py @@ -78,6 +78,21 @@ def add_csv_properties(parser: argparse.ArgumentParser): def input_parser() -> argparse.ArgumentParser: parser = base_parser("input") subparsers = parser.add_subparsers(required=True, dest="input") + # Camera subparser + subparser = subparsers.add_parser("camera") + subparser.add_argument( + "--buffer-size", + type=int, + default=1024, + help="Array buffer size (default: %(default)s)", + ) + subparser.add_argument( + "--driver", + type=str, + choices=["Auto", "NeuromorphicDrivers", "EventCameraDrivers"], + default="Auto", + help="The driver to detect the camera, defaults to Auto which tries both the NeuromorphicDrivers and EventCameraDrivers libraries" + ) # Stdin subparser subparser = subparsers.add_parser("stdin") subparser.add_argument( @@ -126,14 +141,6 @@ def input_parser() -> argparse.ArgumentParser: help="(default: %(default)s)", ) add_csv_properties(subparser) - # Inivation subparser - subparser = subparsers.add_parser("inivation") - subparser.add_argument( - "--buffer-size", - type=int, - default=1024, - help="Array buffer size (default: %(default)s)", - ) # UDP subparser subparser = subparsers.add_parser("udp") subparser.add_argument("address", type=list_filters.parse_udp) @@ -506,8 +513,8 @@ def set_input(self, arguments: list[str]): skip_errors=args.csv_skip_errors, ), ) - elif args.input == "inivation": - self.stream = faery.events_stream_from_camera("Inivation") + elif args.input == "camera": + self.stream = faery.events_stream_from_camera(driver=args.driver, buffer_size=args.buffer_size) elif args.input == "udp": self.stream = faery.events_stream_from_udp( dimensions=args.dimensions, diff --git a/python/faery/event_camera_input.py b/python/faery/event_camera_input.py index 7472a2c..6a06c8a 100644 --- a/python/faery/event_camera_input.py +++ b/python/faery/event_camera_input.py @@ -111,14 +111,14 @@ def events_stream_from_camera( ): stream = None error = None - if driver is None or driver == "EventCameraDrivers": + if driver is None or driver == "EventCameraDrivers" or driver == "Auto": try: stream = EventCameraDriverStream( manufacturer=manufacturer, buffer_size=buffer_size ) except Exception as e: error = e - if driver is None or driver == "NeuromorphicDrivers": + if driver is None or driver == "NeuromorphicDrivers" or driver == "Auto": try: stream = NeuromorphicCameraStream() except Exception as e: From 02ec2b49f10f22a21ef9a3f39183bca6644c62bd Mon Sep 17 00:00:00 2001 From: jegp Date: Wed, 24 Sep 2025 15:15:36 +0200 Subject: [PATCH 5/5] Improved docs and made a few quality-of-life changes --- README.md | 6 +- docs/concepts.md | 274 +++++++++++++++++++++ docs/dev.md | 2 +- docs/inputs/array.md | 383 +++++++++++++++++++++++++++++ docs/inputs/cameras.md | 172 +++++++++++++ docs/inputs/files.md | 207 ++++++++++++++++ docs/inputs/overview.md | 118 +++++++++ docs/inputs/stdin.md | 323 ++++++++++++++++++++++++ docs/inputs/udp.md | 213 ++++++++++++++++ docs/myst.yml | 15 +- docs/stream_types.md | 29 --- docs/stream_types_diagram.svg | 363 +++++++++++++++++++++++++++ pyproject.toml | 6 +- python/faery/cli/list_filters.py | 37 ++- python/faery/cli/process.py | 6 +- python/faery/event_camera_input.py | 4 + python/faery/events_filter.py | 22 +- python/faery/file_encoder.py | 9 + python/faery/frame_stream_state.py | 2 +- 19 files changed, 2138 insertions(+), 53 deletions(-) create mode 100644 docs/concepts.md create mode 100644 docs/inputs/array.md create mode 100644 docs/inputs/cameras.md create mode 100644 docs/inputs/files.md create mode 100644 docs/inputs/overview.md create mode 100644 docs/inputs/stdin.md create mode 100644 docs/inputs/udp.md delete mode 100644 docs/stream_types.md create mode 100644 docs/stream_types_diagram.svg diff --git a/README.md b/README.md index d5c54e1..4cde91e 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ faery input file events.aedat4 filter temporal --window-size 1000us output mp4 o 3. Stream data from an [Inivation camera](https://inivation.com/) to a UDP socket (note: [requires event camera drivers](https://aestream.github.io/faery/install)): ```sh -faery input inivation camera output udp localhost 7777 +faery input inivation camera output udp localhost:7777 ``` **Python library**: Faery provides a set of input functions to read event data from files, UDP streams, or other sources. You can chain methods to filter, render, or analyze the data. For example, to render an AEDAT4 event file as a real-time MP4 video: @@ -50,8 +50,8 @@ faery input inivation camera output udp localhost 7777 import faery faery.events_stream_from_file("input.aedat4") \ .regularize(frequency_hz=60.0) \ - .render(exponential_decay=0.2, style="starry_night") \ - .to_mp4("output.mp4") + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("output.mp4") ``` More information is available in the [command line usage documentation](https://aestream.github.io/faery/cli), the [Python library documentation](https://aestream.github.io/faery/python), and the [examples directory](https://github.com/aestream/faery/tree/main/examples). diff --git a/docs/concepts.md b/docs/concepts.md new file mode 100644 index 0000000..6ed005a --- /dev/null +++ b/docs/concepts.md @@ -0,0 +1,274 @@ +# Core concepts + +Understanding Faery's fundamental concepts is essential for effectively processing neuromorphic event data. This section introduces the key abstractions that power Faery's stream processing architecture. + +## Event data format + +Neuromorphic sensors generate **events** - discrete signals triggered by changes in the environment. Each event represents a pixel that detected a change at a specific time. + +### Event structure + +Every event in Faery contains four pieces of information: + +```python +Event = (timestamp, x_coordinate, y_coordinate, polarity) +``` + +- **Timestamp (t)**: When the event occurred (64-bit microseconds) +- **X coordinate (x)**: Horizontal pixel position (16-bit integer) +- **Y coordinate (y)**: Vertical pixel position (16-bit integer) +- **Polarity (p)**: Direction of change (boolean: True=increase, False=decrease in brightness) + +### Example events + +```python +import numpy as np +import faery + +# Create events manually +events = np.array([ + (1000, 320, 240, True), # Brightness increased at center pixel at t=1ms + (1500, 321, 240, False), # Brightness decreased at adjacent pixel at t=1.5ms + (2000, 319, 241, True), # Another increase nearby at t=2ms +], dtype=faery.EVENTS_DTYPE) +``` + +This sparse representation is highly efficient - only changing pixels generate events, unlike traditional cameras that capture every pixel at fixed intervals. + +## Stream processing architecture + +Faery processes events through **streams** - sequences of event packets that flow through processing pipelines. Understanding stream types is crucial for building effective data processing workflows. + +| Stream Type | Finite? | Regular? | Can `.to_array()`? | Can `.to_file()` video? | +|-------------|---------|----------|--------------------|-------------------------| +| `EventsStream` | ❌ | ❌ | ❌ | ❌ | +| `FiniteEventsStream` | ✅ | ❌ | ✅ | After `.regularize()` | +| `RegularEventsStream` | ❌ | ✅ | ❌ | ❌ | +| `FiniteRegularEventsStream` | ✅ | ✅ | ✅ | ✅ | + +Understanding these stream types and their sources enables you to choose the right input and build effective processing pipelines for your specific use case. + +### Stream hierarchy + +![Stream Type Relationships](stream_types_diagram.svg) + +Streams are organized into a hierarchy based on two key characteristics: + +1. **Finiteness**: Whether the stream has a definite end +2. **Regularity**: Whether packets arrive at fixed time intervals + +## Stream types explained + +### Base stream types + +**EventsStream** (Infinite, Irregular) +- Continues indefinitely +- Packet timing varies +- **Example**: Live camera feed, UDP stream +- **Use case**: Real-time processing, monitoring + +**FiniteEventsStream** (Finite, Irregular) +- Has a definite end +- Packet timing varies +- **Example**: File input, recorded data +- **Use case**: Batch processing, analysis + +### Regular stream types + +**RegularEventsStream** (Infinite, Regular) +- Continues indefinitely +- Fixed packet intervals +- **Example**: Regularized camera feed +- **Use case**: Real-time video generation + +**FiniteRegularEventsStream** (Finite, Regular) +- Has definite end +- Fixed packet intervals +- **Example**: Regularized file data +- **Use case**: Video generation from recordings + +## Implementation details + +All streams in Faery are implemented as a [`Stream`](https://github.com/aestream/faery/blob/main/python/faery/stream.py#L9) class which iterates over data packets. The data type characterizes what is contained in each packet: + +### Data types + +| Stream | Data Type | Characteristics | +|--------|-----------|-----------------| +| **EventsStream** | `np.ndarray` | Sparse event data represented as [timestamps, 2-d coordinates, and polarity bit](https://github.com/aestream/faery/blob/main/python/faery/events_stream.py#L16) `(t, x, y, p)`. | +| **FrameStream** | [`Frame`](https://github.com/aestream/faery/blob/main/python/faery/frame_stream.py#L19) | Dense frame data represented as `[timestamp, np.ndarray]`](https://github.com/aestream/faery/blob/main/python/faery/frame_stream.py#L165). | + +### Stream characteristics + +Apart from the data type, streams have characteristics that determine what operations are possible: + +| Stream Type | Description | Examples | +|-------------|-------------|----------| +| **InfiniteStream** | Requires continuous processing, no definite end | Reading from camera or UDP source | +| **FiniteStream** | Can be processed in a single pass, has definite end | Reading from file or finite source | +| **RegularStream** | Sends data at regular intervals | Filtering stream to output at fixed intervals | +| **FiniteRegularStream** | Both finite and regular | Filtering finite stream at regular intervals | + +## Stream type transformations + +Understanding how operations transform stream types is key to building valid processing pipelines: + +```python +import faery + +# File input creates FiniteEventsStream +stream = faery.events_stream_from_file("events.aedat4") # FiniteEventsStream + +# Regularization creates FiniteRegularEventsStream +regular_stream = stream.regularize(frequency_hz=30.0) # FiniteRegularEventsStream + +# Rendering creates a FrameStream +rendered_stream = regular_stream.render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) # FiniteRegularFrameStream + +# Video output requires a finite frame stream +rendered_stream.to_file("output.mp4") # ✓ Valid - finite stream can be saved to file +``` + +### Making Infinite Streams Finite + +Some operations require finite streams. Use slicing operations to convert: + +```python +# This would fail - infinite streams cannot be saved as complete video files +camera_stream = faery.events_stream_from_camera() # EventsStream (infinite) +camera_stream.regularize(30.0).render(...).to_file("video.mp4") # ❌ Error - no known end + +# Solution: Use time_slice to make it finite +camera_stream.time_slice(0 * faery.s, 10 * faery.s) # ✓ First 10 seconds + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.100000", colormap=faery.colormaps.devon) \ + .to_file("10_second_video.mp4") # ✓ Valid - finite duration + +# Or use event_slice for a specific number of events +camera_stream.event_slice(0, 100000) # ✓ First 100,000 events + .to_array() # ✓ Valid - finite number of events +``` + +## Processing Pipeline Patterns + +### Pattern 1: File Processing +```python +# Finite → Finite Regular → Output +faery.events_stream_from_file("input.aedat4") \ + .regularize(frequency_hz=60.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("output.mp4") +``` + +### Pattern 2: Real-time Processing +```python +# Infinite → Finite → Regular → Output +faery.events_stream_from_camera() \ + .time_slice(0 * faery.us, 5 * faery.s) \ + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.100000", colormap=faery.colormaps.devon) \ + .to_file("live.mp4") # Creates 5-second video file +``` + +### Pattern 3: Analysis Pipeline +```python +# Finite → Analysis +faery.events_stream_from_file("data.es") \ + .remove_off_events() \ + .to_event_rate(window_duration_us=100000) \ + .to_file("event_rates.csv") +``` + +## Stream Type Transformations + +Understanding how operations change stream types: + +### Making Infinite Streams Finite +```python +# ❌ This fails - infinite stream can't create complete video +faery.events_stream_from_camera().regularize(30.0).render(...).to_file("video.mp4") + +# ✅ This works - slice first to make finite +faery.events_stream_from_camera().time_slice(0 * faery.s, 5 * faery.s) \ + .regularize(30.0).render(...).to_file("video.mp4") +``` + +### Making Irregular Streams Regular +```python +# File (Finite + Irregular) → regularize() → (Finite + Regular) +faery.events_stream_from_file("data.es").regularize(frequency_hz=60.0) + +# Camera (Infinite + Irregular) → regularize() → (Infinite + Regular) +faery.events_stream_from_camera().regularize(frequency_hz=30.0) +``` + +## Stream State and Memory + +Faery streams are designed for memory efficiency: + +### Streaming Processing +```python +# Events are processed in small packets - memory usage stays constant +large_file_stream = faery.events_stream_from_file("10GB_events.aedat4") +processed = large_file_stream.regularize(frequency_hz=30.0) # Still memory-efficient +``` + +### Finite Collection +```python +# Only when explicitly collecting all data does memory usage grow +all_events = large_file_stream.to_array() # Loads entire file into memory +``` + +## Common Patterns by Stream Type + +### Finite Streams (Files, Arrays, stdin) +```python +# Pattern: Input → Filter → Regular → Output +source.filter_operation() \ + .regularize(frequency_hz) \ + .render(...) \ + .to_file("output.mp4") # ✅ Always works +``` + +### Infinite Streams (Cameras, UDP) +```python +# Pattern: Input → Slice → Regular → Output +source.time_slice(start, end) \ + .regularize(frequency_hz) \ + .render(...) \ + .to_file("output.mp4") # ✅ Works after slicing + +# Pattern: Input → Process → Stream +source.filter_operation() \ + .to_udp(address) # ✅ Stream-to-stream always works +``` + +## Practical Guidelines + +### Choose the Right Input +- **Files**: Use for analysis, batch processing, reproducible results +- **Cameras**: Use for real-time applications, live monitoring +- **UDP**: Use for distributed processing, network integration +- **Arrays**: Use for testing, simulation, synthetic data + +### Design Processing Pipelines +1. **Start with your input type** - determines initial stream type +2. **Apply filtering early** - reduce data volume for efficiency +3. **Regularize when needed** - required for video output and real-time processing +4. **Choose appropriate output** - match output requirements to stream type + +### Debug Stream Issues +```python +# Check stream type and properties +stream = faery.events_stream_from_file("data.es") +print(f"Dimensions: {stream.dimensions()}") +print(f"Stream type: {type(stream)}") + +# Preview first few packets +for i, packet in enumerate(stream): + print(f"Packet {i}: {len(packet)} events") + if i >= 2: # Just show first 3 packets + break +``` + +Understanding these concepts enables you to build efficient, type-safe processing pipelines that leverage Faery's full capabilities while avoiding common pitfalls. diff --git a/docs/dev.md b/docs/dev.md index 864888d..f6583df 100644 --- a/docs/dev.md +++ b/docs/dev.md @@ -28,7 +28,7 @@ maturin develop # or maturin develop --release to build with optimizations ```sh cargo fmt cargo clippy -pip install isort black pyright +pip install --group dev isort .; black .; pyright . ``` diff --git a/docs/inputs/array.md b/docs/inputs/array.md new file mode 100644 index 0000000..4af5d43 --- /dev/null +++ b/docs/inputs/array.md @@ -0,0 +1,383 @@ +# Arrays and static inputs + +Faery supports creating event and frame streams from programmatically generated data, making it useful for testing, simulations, and synthetic data generation. + +## Event streams from arrays + +Create event streams from NumPy arrays containing pre-computed event data. + +### Python API + +```python +import faery +import numpy as np + +# Create synthetic events +events = np.array([ + (1000, 100, 200, True), # t=1000μs, x=100, y=200, polarity=True + (2000, 150, 250, False), # t=2000μs, x=150, y=250, polarity=False + (3000, 200, 300, True), # t=3000μs, x=200, y=300, polarity=True +], dtype=faery.EVENTS_DTYPE) + +# Create stream from array +stream = faery.events_stream_from_array( + events=events, + dimensions=(640, 480) +) + +# Use like any other stream +stream.render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("synthetic.mp4") +``` + +### Event data format + +Events must use Faery's standard dtype: + +```python +import numpy as np +import faery + +# Standard event format +EVENTS_DTYPE = np.dtype([ + ("t", " np.ndarray: + """Generate a frame based on timestamp""" + # Convert timestamp to seconds + time_s = t.seconds() + + # Create animated pattern + frame = np.zeros((480, 640), dtype=np.uint8) + + # Moving circle + center_x = int(320 + 200 * np.sin(time_s * 2)) + center_y = int(240 + 100 * np.cos(time_s * 3)) + + y, x = np.ogrid[:480, :640] + distance = np.sqrt((x - center_x)**2 + (y - center_y)**2) + frame[distance < 30] = 255 + + return frame + +# Create procedural frame stream +stream = faery.frame_stream_from_function( + start_t=0 * faery.us, + frequency_hz=24.0, + dimensions=(640, 480), + frame_count=120, # 5 seconds at 24 FPS + get_frame=generate_frame +) + +stream.to_file("procedural_animation.mp4") +``` + +## Synthetic event patterns + +### Moving objects + +```python +import faery +import numpy as np + +def create_moving_object_events( + start_x: int, start_y: int, + velocity_x: float, velocity_y: float, + dimensions: tuple[int, int], + duration_us: int, + object_size: int = 5 +) -> np.ndarray: + """Create events for a moving object""" + events = [] + + for t in range(0, duration_us, 1000): # Every 1ms + # Current position + x = int(start_x + velocity_x * t / 1000000) # velocity in pixels/second + y = int(start_y + velocity_y * t / 1000000) + + # Create events around object position + for dx in range(-object_size, object_size + 1): + for dy in range(-object_size, object_size + 1): + px, py = x + dx, y + dy + if 0 <= px < dimensions[0] and 0 <= py < dimensions[1]: + # Random polarity for texture + polarity = np.random.choice([True, False]) + events.append((t, px, py, polarity)) + + return np.array(events, dtype=faery.EVENTS_DTYPE) + +# Create moving object +events = create_moving_object_events( + start_x=100, start_y=240, + velocity_x=200, # 200 pixels/second to the right + velocity_y=50, # 50 pixels/second downward + dimensions=(640, 480), + duration_us=3000000 # 3 seconds +) + +stream = faery.events_stream_from_array(events, dimensions=(640, 480)) +``` + +### Noise patterns + +```python +def create_noise_events( + dimensions: tuple[int, int], + duration_us: int, + event_rate_hz: float +) -> np.ndarray: + """Create random noise events""" + total_events = int(event_rate_hz * duration_us / 1000000) + + events = np.zeros(total_events, dtype=faery.EVENTS_DTYPE) + events["t"] = np.sort(np.random.randint(0, duration_us, total_events)) + events["x"] = np.random.randint(0, dimensions[0], total_events) + events["y"] = np.random.randint(0, dimensions[1], total_events) + events["p"] = np.random.choice([True, False], total_events) + + return events + +# Create noise background +noise = create_noise_events( + dimensions=(640, 480), + duration_us=5000000, # 5 seconds + event_rate_hz=10000 # 10k events/second +) + +stream = faery.events_stream_from_array(noise, dimensions=(640, 480)) +``` + +### Periodic patterns + +```python +def create_periodic_flash( + center: tuple[int, int], + radius: int, + period_us: int, + cycles: int, + dimensions: tuple[int, int] +) -> np.ndarray: + """Create periodic flashing pattern""" + events = [] + cx, cy = center + + for cycle in range(cycles): + # Flash on + flash_time = cycle * period_us + y, x = np.ogrid[:dimensions[1], :dimensions[0]] + distance = np.sqrt((x - cx)**2 + (y - cy)**2) + flash_pixels = np.where(distance < radius) + + for px, py in zip(flash_pixels[1], flash_pixels[0]): + events.append((flash_time, px, py, True)) + + # Flash off (half period later) + off_time = flash_time + period_us // 2 + for px, py in zip(flash_pixels[1], flash_pixels[0]): + events.append((off_time, px, py, False)) + + return np.array(events, dtype=faery.EVENTS_DTYPE) + +# Create blinking circle +flash_events = create_periodic_flash( + center=(320, 240), + radius=50, + period_us=500000, # 500ms period (2 Hz) + cycles=10, + dimensions=(640, 480) +) + +stream = faery.events_stream_from_array(flash_events, dimensions=(640, 480)) +``` + +## Testing and validation + +### Unit test events + +```python +import faery +import numpy as np + +# Create minimal test case +test_events = np.array([ + (0, 0, 0, True), + (1000, 639, 479, False), # Test corner cases +], dtype=faery.EVENTS_DTYPE) + +stream = faery.events_stream_from_array(test_events, dimensions=(640, 480)) + +# Verify stream properties +assert stream.dimensions() == (640, 480) + +# Test processing pipeline +processed = stream.crop(10, 10, 620, 460).take(100) +# ... additional validation +``` + +### Performance benchmarking + +```python +import time +import faery +import numpy as np + +# Create large synthetic dataset +num_events = 1000000 +events = np.zeros(num_events, dtype=faery.EVENTS_DTYPE) +events["t"] = np.arange(num_events) * 10 # 10μs intervals +events["x"] = np.random.randint(0, 1280, num_events) +events["y"] = np.random.randint(0, 720, num_events) +events["p"] = np.random.choice([True, False], num_events) + +# Benchmark processing speed +start_time = time.time() +stream = faery.events_stream_from_array(events, dimensions=(1280, 720)) +result = stream.regularize(frequency_hz=60.0).take(1000) +elapsed = time.time() - start_time + +print(f"Processed {num_events} events in {elapsed:.3f}s") +print(f"Processing rate: {num_events/elapsed:.0f} events/second") +``` + +## Integration with other libraries + +### From OpenCV + +```python +import cv2 +import numpy as np +import faery + +# Load video and convert to events +cap = cv2.VideoCapture('input_video.mp4') +events = [] +prev_frame = None +timestamp = 0 + +while True: + ret, frame = cap.read() + if not ret: + break + + gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + + if prev_frame is not None: + # Simple change detection + diff = cv2.absdiff(gray, prev_frame) + y_coords, x_coords = np.where(diff > 30) # Threshold + + for x, y in zip(x_coords, y_coords): + polarity = gray[y, x] > prev_frame[y, x] + events.append((timestamp, x, y, polarity)) + + prev_frame = gray + timestamp += 33333 # ~30 FPS (33.33ms intervals) + +cap.release() + +# Convert to Faery stream +events_array = np.array(events, dtype=faery.EVENTS_DTYPE) +stream = faery.events_stream_from_array(events_array, dimensions=(640, 480)) +``` + +### From simulation frameworks + +```python +# Example: Convert simulation data to Faery format +def simulation_to_events(sim_data, dimensions): + """Convert simulation output to event format""" + events = [] + + for time_step, positions, polarities in sim_data: + timestamp_us = int(time_step * 1000000) # Convert to microseconds + + for (x, y), polarity in zip(positions, polarities): + if 0 <= x < dimensions[0] and 0 <= y < dimensions[1]: + events.append((timestamp_us, int(x), int(y), bool(polarity))) + + return np.array(events, dtype=faery.EVENTS_DTYPE) + +# Usage with hypothetical simulation +# sim_results = run_my_simulation(...) +# events = simulation_to_events(sim_results, (640, 480)) +# stream = faery.events_stream_from_array(events, dimensions=(640, 480)) +``` + +## Common patterns + +### Combining multiple sources + +```python +import numpy as np +import faery + +# Create multiple event sources +source1 = create_moving_object_events(100, 200, 50, 25, (640, 480), 2000000) +source2 = create_noise_events((640, 480), 2000000, 5000) +source3 = create_periodic_flash((500, 300), 30, 200000, 10, (640, 480)) + +# Combine and sort by timestamp +all_events = np.concatenate([source1, source2, source3]) +sorted_indices = np.argsort(all_events["t"]) +combined_events = all_events[sorted_indices] + +# Create unified stream +stream = faery.events_stream_from_array(combined_events, dimensions=(640, 480)) +``` + +This approach allows you to create complex synthetic scenes with multiple objects, noise, and patterns for comprehensive testing and algorithm development. diff --git a/docs/inputs/cameras.md b/docs/inputs/cameras.md new file mode 100644 index 0000000..7efd9cb --- /dev/null +++ b/docs/inputs/cameras.md @@ -0,0 +1,172 @@ +# Event cameras inputs + +Faery supports direct streaming from event cameras, enabling real-time data processing and visualization. Note that this requires additional driver installations. + +## Supported cameras + +Faery supports event cameras from two major manufacturers through different driver systems: + +### Inivation cameras +- **DVXplorer** (640×480) +- **DAVIS346** (346×260) +- Requires: [`event_camera_drivers`](https://github.com/aestream/event-camera-drivers/) or [`neuromorphic_drivers`](https://github.com/neuromorphicsystems/neuromorphic-drivers) + +### Prophesee cameras +- **EVK4** (1280×720) +- **EVK3 HD** (1280×720) +- Requires: [`neuromorphic_drivers`](https://github.com/neuromorphicsystems/neuromorphic-drivers) + +## Driver installation + +Faery supports two driver systems. Install at least one: + +### Option 1: event_camera_drivers +```sh +pip install event_camera_drivers +``` +- Works well with Inivation cameras +- Uses existing libcaer drivers under the hood + +### Option 2: neuromorphic_drivers +```sh +pip install neuromorphic_drivers +``` +- Custom drivers +- Supports both Inivation and Prophesee cameras + +## Usage + +### Command line + +```sh +# Stream from any detected camera to file +faery input inivation camera output file recording.es + +# Stream to UDP for real-time processing +faery input inivation camera output udp localhost:7777 + +# Create real-time visualization +faery input inivation camera filter regularize 30.0 filter render exponential "00:00:00.100000" starry_night output mp4 live.mp4 +``` + +### Python + +```python +import faery + +# Auto-detect and connect to camera +stream = faery.events_stream_from_camera() + +# Specify driver system +stream = faery.events_stream_from_camera(driver="EventCameraDrivers") +stream = faery.events_stream_from_camera(driver="NeuromorphicDrivers") + +# Specify manufacturer (helps with faster detection) +stream = faery.events_stream_from_camera(manufacturer="Inivation") +stream = faery.events_stream_from_camera(manufacturer="Prophesee") + +# Configure buffer size for performance tuning +stream = faery.events_stream_from_camera(buffer_size=2048) + +# Use in a processing pipeline +faery.events_stream_from_camera() \ + .time_slice(0 * faery.s, 10 * faery.s) \ + .regularize(frequency_hz=30.0) \ + .render(tau="00:00:00.002000", decay="exponential", colormap=faery.colormaps.starry_night) \ + .to_file("live.mp4") +``` + +## Camera detection and selection + +Faery automatically detects available cameras and drivers: + +```python +# Check driver availability +import faery.event_camera_input as eci + +if eci.has_event_camera_drivers(): + print("event_camera_drivers available") + +if eci.has_neuromorphic_drivers(): + print("neuromorphic_drivers available") + +# Auto-detection tries drivers in order of preference +stream = faery.events_stream_from_camera(driver="Auto") # Default behavior +``` + +## Configuration options + +### Buffer size +Controls the internal buffer size for event streaming: + +```python +# Larger buffers reduce latency spikes but use more memory +stream = faery.events_stream_from_camera(buffer_size=4096) # Default: 1024 +``` + +### Manufacturer hint +Speeds up camera detection by specifying the expected manufacturer: + +```python +# Skip detection time by specifying manufacturer +stream = faery.events_stream_from_camera(manufacturer="Inivation") +``` + +## Real-time processing examples + +### Live visualization +```python +import faery + +# Record 10 seconds of camera data as video +faery.events_stream_from_camera() \ + .time_slice(0 * faery.s, 10 * faery.s) \ + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("live.mp4") +``` + +### Network streaming +```python +# Stream events over UDP for distributed processing +faery.events_stream_from_camera() \ + .to_udp(("localhost", 7777)) +``` + + +## Troubleshooting + +### No camera detected +``` +ValueError: No Event Camera found +``` +**Solutions:** +1. Check physical camera connection (USB) +2. Verify driver installation: `pip list | grep -E "(event_camera|neuromorphic)"` +3. For neuromorphic_drivers: ensure udev rules are installed +4. Try specifying manufacturer: `manufacturer="Inivation"` + +### Permission issues (Linux) +``` +PermissionError: [Errno 13] Permission denied +``` +**Solutions:** +1. Install udev rules for your camera - see the [guide on NeuromorphicDrivers' GitHub](https://github.com/neuromorphicsystems/neuromorphic-drivers?tab=readme-ov-file#udev-rules) +2. Add user to appropriate groups (dialout, plugdev) +3. Restart or re-login after group changes + +### Driver import errors +``` +ImportError: No module named 'event_camera_drivers' / 'neuromorphic_drivers' +``` +**Solutions:** +1. Install missing driver: `pip install event_camera_drivers` or `pip install neuromorphic_drivers` +2. Check virtual environment activation +3. Verify compatible Python version + +## Hardware requirements + +- **USB 3.0 or higher** recommended for high-resolution cameras +- **Available USB ports** (cameras may require significant bandwidth) +- **Sufficient RAM** for buffering (especially at high event rates) +- **Linux/Windows/macOS** support varies by driver system. Please submit issues in the respective driver repositories diff --git a/docs/inputs/files.md b/docs/inputs/files.md new file mode 100644 index 0000000..83b66ae --- /dev/null +++ b/docs/inputs/files.md @@ -0,0 +1,207 @@ +# File inputs + +Faery supports reading neuromorphic event data from multiple file formats. This section covers all supported file types and their specific handling requirements. + +## AEDAT4 files (.aedat4) + +AEDAT4 is the standard format from iniVation for event camera data. It's a container format that can hold multiple data streams. + +### Command line +```sh +# Basic file conversion +faery input file input.aedat4 output file output.es + +# Specify a specific track/stream (useful for multi-stream files) +faery input file input.aedat4 --track-id 1 output file output.es +``` + +### Python +```python +import faery + +# Read from AEDAT4 file +stream = faery.events_stream_from_file("input.aedat4") + +# Specify track ID for multi-stream files +stream = faery.events_stream_from_file("input.aedat4", track_id=1) +``` + +### Format details +- **Multi-stream support**: AEDAT4 files can contain multiple data streams. Use `track_id` to select a specific stream (defaults to the first event stream) +- **Metadata**: Contains sensor dimensions and other metadata +- **Compression**: Supports internal compression + +## ES files (.es) + +ES is Faery's native binary format, optimized for fast reading and minimal overhead. + +### Command line +```sh +# Convert to ES format +faery input file input.raw output file output.es + +# Read from ES with custom start time +faery input file input.es --t0 1000000 output file output.mp4 +``` + +### Python +```python +import faery + +# Read ES file +stream = faery.events_stream_from_file("input.es") + +# Specify start time offset (in microseconds) +stream = faery.events_stream_from_file("input.es", t0=1000000 * faery.us) +``` + +### Format details +- **Native format**: Optimized for Faery's internal event representation +- **Time offset**: Use `t0` parameter to specify initial timestamp +- **Compression**: Supports LZ4 and ZSTD compression + +## Prophesee RAW files (.raw) + +Prophesee RAW is the native format from Prophesee event cameras. + +### Command line +```sh +# Convert Prophesee RAW file +faery input file input.raw output file output.aedat4 + +# Specify dimensions if not in header +faery input file input.raw --dimensions-fallback 640x480 output file output.es + +# Override version detection +faery input file input.raw --version-fallback evt3 output file output.es +``` + +### Python +```python +import faery + +# Read Prophesee RAW file +stream = faery.events_stream_from_file("input.raw") + +# Specify fallback dimensions and version +stream = faery.events_stream_from_file( + "input.raw", + dimensions_fallback=(640, 480), + version_fallback="evt3" +) +``` + +### Format details +- **Header detection**: Faery automatically detects sensor dimensions and format version from the file header +- **Fallbacks**: Use `dimensions_fallback` and `version_fallback` when header information is missing +- **Versions**: Supports EVT2, EVT2.1, and EVT3 formats + +## DAT files (.dat) + +DAT is another format variant, similar to EVT but with different encoding. + +### Command line +```sh +# Convert DAT file +faery input file input.dat output file output.aedat4 + +# Specify fallback parameters +faery input file input.dat --dimensions-fallback 1280x720 --version-fallback dat2 output file output.es +``` + +### Python +```python +import faery + +# Read DAT file with fallbacks +stream = faery.events_stream_from_file( + "input.dat", + dimensions_fallback=(1280, 720), + version_fallback="dat2" +) +``` + +### Format details +- **Versions**: Supports DAT1 and DAT2 formats +- **Similar to EVT**: Uses similar fallback mechanisms as EVT files + +## CSV files (.csv) + +For human-readable data or custom formats, Faery supports CSV files with configurable column mapping. + +### Command line +```sh +# Read CSV with default column mapping (t,x,y,p) +faery input file events.csv output file output.es + +# Custom column indices and separator +faery input file events.csv --csv-t-index 0 --csv-x-index 1 --csv-y-index 2 --csv-p-index 3 --csv-separator ";" output file output.es + +# CSV without header +faery input file events.csv --no-csv-has-header output file output.es +``` + +### Python +```python +import faery + +# Read CSV with default settings +stream = faery.events_stream_from_file("events.csv") + +# Custom CSV properties +csv_props = faery.CsvProperties( + has_header=True, + separator=";", + t_index=0, + x_index=1, + y_index=2, + p_index=3 +) +stream = faery.events_stream_from_file("events.csv", csv_properties=csv_props) +``` + +### Format details +- **Column mapping**: Configure which columns contain timestamp, x, y, and polarity data +- **Headers**: Optionally skip the first row if it contains column headers +- **Separators**: Support for different delimiter characters + +## Format detection + +Faery automatically detects file formats based on file extensions: + +| Extension | Format | +|-----------|--------| +| `.aedat4` | AEDAT4 | +| `.es` | ES | +| `.raw` | Prophesee EVT | +| `.dat` | DAT | +| `.csv` | CSV | + +### Override format detection + +### Command line +```sh +# Force interpretation as specific format +faery input file mystery_file --file-type csv output file output.es +``` + +### Python +```python +# Override automatic detection +stream = faery.events_stream_from_file("mystery_file", file_type="csv") +``` + +## Common parameters + +All file input methods support these common parameters: + +- **`dimensions_fallback`**: Default sensor dimensions when not available in file header +- **`file_type`**: Override automatic format detection +- **`track_id`**: Select specific stream from multi-stream files (AEDAT4 only) + +## Troubleshooting + +**File not recognized**: Use `--file-type` or `file_type=` to override detection +**Dimension errors**: Specify `--dimensions-fallback` or `dimensions_fallback=` +**Multi-stream confusion**: Use `--track-id` or `track_id=` for AEDAT4 files +**CSV parsing issues**: Check column indices and separator settings diff --git a/docs/inputs/overview.md b/docs/inputs/overview.md new file mode 100644 index 0000000..5efe391 --- /dev/null +++ b/docs/inputs/overview.md @@ -0,0 +1,118 @@ +# Input sources + +Faery supports multiple input sources, each creating different types of streams. Understanding which input creates which stream type is crucial for building effective processing pipelines. + +## Stream type summary + +| Input Source | Stream Type | Finite? | Regular? | Use Cases | +|--------------|-------------|---------|----------|-----------| +| **Files** | `FiniteEventsStream` | ✅ Yes | ❌ No | Batch processing, analysis, reproducible results | +| **Event Cameras** | `EventsStream` | ❌ No | ❌ No | Real-time processing, live monitoring | +| **UDP Streams** | `EventsStream` | ❌ No | ❌ No | Network integration, distributed processing | +| **Arrays (Python)** | `FiniteEventsStream` | ✅ Yes | ❌ No | Testing, simulation, synthetic data | +| **Standard Input** | `FiniteEventsStream` | ✅ Yes | ❌ No | Pipeline integration, shell scripting | + +## Input source details + +### 📁 Files +**Stream Type**: `FiniteEventsStream` (Finite + Irregular) + +**Supported Formats**: AEDAT4, ES, Prophesee RAW, DAT, EVT, CSV + +**Characteristics**: +- Known duration and size +- Can be converted to arrays with `.to_array()` +- Perfect for video generation after regularization +- Reproducible processing + +**Example**: +```python +# File → Finite → Regular → Video +faery.events_stream_from_file("data.aedat4") \ + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.devon) \ + .to_file("output.mp4") +``` + +### 📷 Event Cameras +**Stream Type**: `EventsStream` (Infinite + Irregular) + +**Supported Hardware**: DVXplorer, DAVIS346, EVK4, EVK3 HD + +**Characteristics**: +- Continuous data stream +- Requires `.time_slice()` or `.event_slice()` for video output +- Real-time processing capabilities +- Live monitoring and streaming + +**Example**: +```python +# Camera → Slice → Finite → Regular → Video +faery.events_stream_from_camera() \ + .time_slice(0 * faery.s, 10 * faery.s) \ + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("camera_recording.mp4") +``` + +### 🌐 UDP Streams +**Stream Type**: `EventsStream` (Infinite + Irregular) + +**Supported Formats**: t64_x16_y16_on8, t32_x16_y15_on1 + +**Characteristics**: +- Network-based event streaming +- Low-latency for distributed systems +- Requires dimensions specification +- Can be finite or infinite depending on sender + +**Example**: +```python +# UDP → Process → Stream to another UDP endpoint +faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777) +).remove_off_events() \ + .to_udp(("processing-server", 8888)) +``` + +### 🔢 Arrays (Python) +**Stream Type**: `FiniteEventsStream` (Finite + Irregular) + +**Data Source**: NumPy arrays with `faery.EVENTS_DTYPE` + +**Characteristics**: +- Programmatically generated data +- Perfect for testing and validation +- Known size and duration +- Synthetic data generation + +**Example**: +```python +# Array → Direct video generation +events = create_synthetic_events() # Your function +faery.events_stream_from_array(events, dimensions=(640, 480)) \ + .regularize(frequency_hz=24.0) \ + .render(decay="exponential", tau="00:00:00.100000", colormap=faery.colormaps.batlow) \ + .to_file("synthetic.mp4") +``` + +### 📥 Standard Input +**Stream Type**: `FiniteEventsStream` (Finite + Irregular) + +**Supported Format**: CSV only + +**Characteristics**: +- Command-line pipeline integration +- Finite by nature (piped data has an end) +- Configurable CSV parsing +- Shell scripting integration + +**Example**: +```bash +# Shell pipeline → Faery processing +cat events.csv | faery input stdin --dimensions 640x480 \ + filter regularize 30.0 \ + filter render exponential "00:00:00.200000" starry_night \ + output file processed.mp4 +``` diff --git a/docs/inputs/stdin.md b/docs/inputs/stdin.md new file mode 100644 index 0000000..e7db28d --- /dev/null +++ b/docs/inputs/stdin.md @@ -0,0 +1,323 @@ +# Standard input (stdin) + +Faery supports reading event data from standard input, enabling integration with command-line pipelines and shell scripting workflows. + +## When to use stdin input + +Standard input is useful for: +- **Unix pipeline integration**: Chaining multiple command-line tools +- **Shell scripting**: Processing data streams in batch scripts +- **Data preprocessing**: Filtering or transforming data before Faery processing +- **Remote processing**: Receiving data through SSH pipes or network commands + +## Supported format + +Currently, stdin input only supports **CSV format**. The data must be structured as comma-separated values with configurable column mapping. + +## Basic usage + +### Command line + +```sh +# Read CSV from stdin and convert to file +cat events.csv | faery input stdin --dimensions 640x480 output file output.es + +# Process data from another command +generate_events.py | faery input stdin --dimensions 1280x720 output file output.aedat4 + +# Custom CSV format +cat custom_events.csv | faery input stdin --dimensions 640x480 \ + --csv-separator ";" \ + --csv-t-index 0 \ + --csv-x-index 1 \ + --csv-y-index 2 \ + --csv-p-index 3 \ + output file output.es + +# Skip header row +tail -n +2 events_with_header.csv | faery input stdin --dimensions 640x480 \ + --no-csv-has-header \ + output file output.es +``` + +### Python + +```python +import faery +import sys + +# Read from stdin (CSV format only) +stream = faery.events_stream_from_stdin( + dimensions=(640, 480) +) + +# Custom CSV properties +csv_props = faery.CsvProperties( + has_header=False, + separator=";", + t_index=0, + x_index=1, + y_index=2, + p_index=3 +) + +stream = faery.events_stream_from_stdin( + dimensions=(640, 480), + csv_properties=csv_props, + t0=1000000 * faery.us # Start time offset +) + +# Process the stream +stream.regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("stdin_output.mp4") +``` + +## CSV format requirements + +### Default format +```csv +t,x,y,p +1000,100,200,1 +2000,150,250,0 +3000,200,300,1 +``` + +- **Column 0 (t)**: Timestamp in microseconds +- **Column 1 (x)**: X coordinate +- **Column 2 (y)**: Y coordinate +- **Column 3 (p)**: Polarity (0/1 or True/False) + +### Custom column mapping + +Configure which columns contain which data: + +```sh +# Different column order: x,y,t,p +echo "150,200,1000,1" | faery input stdin --dimensions 640x480 \ + --csv-x-index 0 \ + --csv-y-index 1 \ + --csv-t-index 2 \ + --csv-p-index 3 \ + output file output.es +``` + +### Alternative separators + +```sh +# Tab-separated values +cat events.tsv | faery input stdin --dimensions 640x480 \ + --csv-separator $'\t' \ + output file output.es + +# Semicolon-separated +cat events.csv | faery input stdin --dimensions 640x480 \ + --csv-separator ";" \ + output file output.es +``` + +## Pipeline integration examples + +### Data preprocessing + +```sh +# Filter events by polarity before processing +awk -F, '$4==1' events.csv | faery input stdin --dimensions 640x480 \ + --no-csv-has-header \ + output file positive_events.es + +# Select time range +awk -F, '$1>=1000000 && $1<=2000000' events.csv | \ + faery input stdin --dimensions 640x480 \ + --no-csv-has-header \ + output file time_slice.es +``` + +### Multi-stage processing + +```sh +# Stage 1: Preprocess with custom script +cat raw_data.txt | ./preprocess.py | \ +# Stage 2: Convert to Faery format +faery input stdin --dimensions 1280x720 output file intermediate.es + +# Stage 3: Render video +faery input file intermediate.es \ + filter regularize 30.0 \ + filter render exponential 0.2 devon \ + output file final.mp4 +``` + +### Network integration + +```sh +# Receive data over SSH and process +ssh remote-host "cat /path/to/events.csv" | \ + faery input stdin --dimensions 640x480 output file remote_events.es + +# Process streaming data from network service +curl -s "http://api.example.com/events.csv" | \ + faery input stdin --dimensions 640x480 \ + filter regularize 60.0 \ + output udp localhost 7777 +``` + +### Database integration + +```sh +# Query database and process results +psql -d events_db -c "SELECT timestamp_us, x, y, polarity FROM events WHERE timestamp_us > 1000000" \ + --csv --no-align --field-separator=',' | \ + tail -n +2 | \ + faery input stdin --dimensions 1280x720 --no-csv-has-header output file db_events.es + +# MySQL export +mysql -u user -p -D events_db -e "SELECT t, x, y, p FROM events" \ + --batch --raw --silent | \ + faery input stdin --dimensions 640x480 --csv-separator $'\t' --no-csv-has-header \ + output file mysql_events.es +``` + +## Real-time stream processing + +### Continuous processing + +```sh +# Process events as they arrive (named pipe) +mkfifo event_pipe +./event_generator > event_pipe & +faery input stdin --dimensions 640x480 < event_pipe \ + filter regularize 30.0 \ + filter render exponential 0.1 starry_night \ + output file live_stream.mp4 +``` + +### Log file monitoring + +```sh +# Process new events from growing log file +tail -f /var/log/events.csv | \ + faery input stdin --dimensions 640x480 --no-csv-has-header \ + filter event_rate 100000 \ + output file event_rates.csv +``` + +## Error handling and validation + +### Input validation + +```sh +# Validate CSV format before processing +head -5 suspicious_data.csv | faery input stdin --dimensions 640x480 \ + output file test_output.es 2>&1 | grep -q "Error" && \ + echo "Invalid format detected" || echo "Format OK" +``` + +### Robust processing + +```sh +#!/bin/bash +# Robust stdin processing script + +set -e # Exit on any error + +# Check if stdin has data +if [ -t 0 ]; then + echo "Error: No data on stdin" >&2 + exit 1 +fi + +# Process with error handling +faery input stdin --dimensions 640x480 output file output.es || { + echo "Error: Failed to process stdin data" >&2 + exit 1 +} + +echo "Successfully processed stdin data" +``` + +## Performance considerations + +### Large data streams + +```sh +# Use buffering for large streams +stdbuf -o0 -e0 cat large_events.csv | \ + faery input stdin --dimensions 640x480 output file large_output.es + +# Process in chunks for memory efficiency +split -l 1000000 huge_events.csv chunk_ && \ +for chunk in chunk_*; do + cat "$chunk" | faery input stdin --dimensions 640x480 \ + output file "${chunk}.es" +done +``` + +### Progress monitoring + +```sh +# Monitor processing progress +pv large_events.csv | faery input stdin --dimensions 640x480 output file output.es + +# With line counting +cat events.csv | pv -l | faery input stdin --dimensions 640x480 output file output.es +``` + +## Troubleshooting + +### Common issues + +**"No data available"** +``` +ValueError: No data available on stdin +``` +- Verify data is being piped to stdin: `echo "test" | your_command` +- Check for empty input files +- Ensure proper pipeline construction + +**"Invalid CSV format"** +``` +ValueError: CSV parsing error +``` +- Verify column indices with `--csv-*-index` parameters +- Check separator with `--csv-separator` +- Validate data format: `head -5 data.csv` + +**"Dimension mismatch"** +``` +ValueError: Coordinates exceed dimensions +``` +- Check that x/y values fit within specified dimensions +- Verify coordinate columns are correctly mapped +- Consider using larger dimensions parameter + +### Debugging techniques + +```sh +# Preview data format +head -5 data.csv + +# Test with minimal data +echo "1000,100,200,1" | faery input stdin --dimensions 640x480 output file test.es + +# Validate column mapping +awk -F, '{print "t:"$1" x:"$2" y:"$3" p:"$4}' data.csv | head -5 + +# Check for invalid characters +cat data.csv | tr -cd '[:print:]\n' | head -5 +``` + +### Performance optimization + +```sh +# Disable header processing if not needed +faery input stdin --dimensions 640x480 --no-csv-has-header output file output.es + +# Use appropriate buffer sizes +stdbuf -i8192 -o8192 faery input stdin --dimensions 640x480 output file output.es + +# Parallel processing for multiple files +find . -name "*.csv" | xargs -P4 -I{} sh -c 'cat {} | faery input stdin --dimensions 640x480 output file {}.es' +``` + +Standard input processing in Faery provides a powerful way to integrate neuromorphic data processing into existing command-line workflows and enables seamless interoperability with other tools and systems. diff --git a/docs/inputs/udp.md b/docs/inputs/udp.md new file mode 100644 index 0000000..7423677 --- /dev/null +++ b/docs/inputs/udp.md @@ -0,0 +1,213 @@ +# Network (UDP) inputs + +Faery supports reading event streams from UDP network sources, enabling real-time distributed processing and camera streaming over networks. + +## When to use UDP input + +UDP streaming is useful for: +- **Real-time applications**: Processing events as they arrive from cameras or other sources +- **Distributed systems**: Receiving events from remote cameras or processing nodes +- **Low-latency pipelines**: Minimal protocol overhead for time-critical applications +- **Network integration**: Connecting Faery to existing UDP-based event systems + +## Supported UDP formats + +Faery supports two binary UDP formats for event transmission: + +### t64_x16_y16_on8 (Default) +- **Timestamp**: 64-bit (8 bytes) +- **X coordinate**: 16-bit (2 bytes) +- **Y coordinate**: 16-bit (2 bytes) +- **Polarity**: 8-bit (1 byte) +- **Total**: 13 bytes per event +- **Use case**: High precision timestamps, standard coordinate resolution + +### t32_x16_y15_on1 +- **Timestamp**: 32-bit (4 bytes) +- **X coordinate**: 16-bit (2 bytes) +- **Y coordinate**: 15-bit + 1-bit polarity (2 bytes total) +- **Total**: 8 bytes per event +- **Use case**: Bandwidth-constrained networks, lower timestamp precision acceptable + +## Basic usage + +### Command line + +```sh +# Listen on localhost port 7777 (IPv4) +faery input udp localhost:7777 --dimensions 640x480 output file received.es + +# Specify UDP format +faery input udp localhost:7777 --dimensions 640x480 --format t32_x16_y15_on1 output file received.es + +# IPv6 address +faery input udp [::1]:7777 --dimensions 640x480 output file received.es + +# Listen and stream to another UDP endpoint +faery input udp localhost:7777 --dimensions 640x480 output udp remote-host:8888 +``` + +### Python + +```python +import faery + +# Basic UDP input +stream = faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777) +) + +# Specify format +stream = faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777), + format="t32_x16_y15_on1" +) + +# IPv6 +stream = faery.events_stream_from_udp( + dimensions=(640, 480), + address=("::1", 7777, None, None) # IPv6 format +) + +# Use in processing pipeline +faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777) +).time_slice(0 * faery.s, 10 * faery.s) \ + .regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.starry_night) \ + .to_file("network_stream.mp4") +``` + +## Real-time processing examples + +### Camera to network pipeline +```python +# Send camera data over UDP +faery.events_stream_from_camera() \ + .to_udp("remote-host", 7777) + +# Receive and process on remote machine +faery.events_stream_from_udp( + dimensions=(640, 480), + address=("0.0.0.0", 7777) # Listen on all interfaces +).regularize(frequency_hz=60.0) \ + .render(decay="exponential", tau="00:00:00.100000", colormap=faery.colormaps.starry_night) \ + .to_file("remote_camera.mp4") +``` + +### Multi-hop processing +```python +# Stage 1: Basic filtering +faery.events_stream_from_udp( + dimensions=(1280, 720), + address=("localhost", 7777) +).remove_off_events() \ + .to_udp(("processing-node", 8888)) + +# Stage 2: Advanced processing +faery.events_stream_from_udp( + dimensions=(1280, 720), + address=("0.0.0.0", 8888) +).regularize(frequency_hz=30.0) \ + .render(decay="exponential", tau="00:00:00.200000", colormap=faery.colormaps.devon) \ + .to_file("processed_output.mp4") +``` + +### Event rate monitoring +```python +# Monitor network event rates +faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777) +).to_event_rate(window_duration_us=1000000) \ + .to_file("network_activity.csv") +``` + +## Firewall settings + +Your firewall can sometimes block UDP traffic, so ensure that your system opens the right ports. +Below are examples for Linux and Windows: + +```sh +# Allow UDP input on port 7777 (Linux iptables) +sudo iptables -A INPUT -p udp --dport 7777 -j ACCEPT + +# Windows Firewall (PowerShell) +New-NetFirewallRule -DisplayName "Faery UDP" -Direction Inbound -Protocol UDP -LocalPort 7777 -Action Allow +``` + +## Troubleshooting + +### Connection issues + +**Address already in use** +``` +OSError: [Errno 98] Address already in use +``` +- Check if another process is using the port: `netstat -ulnp | grep 7777` +- Try a different port + +**No route to host** +``` +OSError: [Errno 113] No route to host +``` +- Verify network connectivity: `ping target-host` +- Check firewall rules on both sender and receiver +- Ensure correct IP address and port + +### Data issues + +**No events received** +Either dump events to stdout or write a custom loop to check that you're actually seeing data. +```python +# Check if data is arriving (will block until events received) +for events in stream: + print(f"Received {len(events)} events") + break # Exit after first packet +``` + +**Malformed events** +- Verify sender and receiver use same UDP format +- Check network MTU settings (large packets may be fragmented) +- Ensure sender uses correct event encoding + +**High latency** +- Reduce processing complexity in pipeline +- Check network latency: `ping -c 10 target-host` +- Consider using faster UDP format (t32_x16_y15_on1) + +### Performance optimization + +**High CPU usage** +```python +# Add regularization to reduce processing frequency +stream = faery.events_stream_from_udp( + dimensions=(640, 480), + address=("localhost", 7777) +).regularize(frequency_hz=30.0) # Limit to 30 FPS processing +``` + +**Memory usage** +- Events are processed in streaming fashion (low memory footprint) +- For finite processing, consider `.take()` to limit event count +- Monitor with `top` or `htop` for actual memory usage + +**Network bandwidth** +```python +# Monitor received data rate +import time +start_time = time.time() +event_count = 0 + +for events in stream: + event_count += len(events) + elapsed = time.time() - start_time + if elapsed > 5.0: # Report every 5 seconds + rate = event_count / elapsed + print(f"Receiving {rate:.0f} events/second") + event_count = 0 + start_time = time.time() +``` diff --git a/docs/myst.yml b/docs/myst.yml index 8b0fa45..ed909bb 100644 --- a/docs/myst.yml +++ b/docs/myst.yml @@ -14,10 +14,23 @@ project: title: Faery - file: install.md - file: quickstart.md + - file: concepts.md + - title: Input Sources + file: inputs/overview.md + children: + - file: inputs/files.md + title: Files + - file: inputs/cameras.md + title: Event Cameras + - file: inputs/udp.md + title: UDP Streams + - file: inputs/array.md + title: Synthetic Data + - file: inputs/stdin.md + title: Standard Input - file: cli.md - file: python.md - file: batch.md - - file: stream_types.md - file: dev.md numbering: title: diff --git a/docs/stream_types.md b/docs/stream_types.md deleted file mode 100644 index b96b418..0000000 --- a/docs/stream_types.md +++ /dev/null @@ -1,29 +0,0 @@ -(stream-types)= -# Stream types in Faery - -Faery supports streams of various data types with different characteristics. -Understanding these streams is crucial for understanding the behavior of Faery---and in particular the Python API. - -## Stream types - -All streams are implemented as a [`Stream`](https://github.com/aestream/faery/blob/main/python/faery/stream.py#L9) which iterates over *some* data. -The data type characterizes what is contained in the stream. -We mostly operate with two types: events and frames: - -| Stream | Data Type | Characteristics | -|--------|-----------|-----------------| -| EventsStream | `np.ndarray` | Sparse event data represented as [timestamps, 2-d coordinates, and polarity bit](https://github.com/aestream/faery/blob/main/python/faery/events_stream.py#L16) `(t, x, y, p)`. | -| FrameStream | [`Frame`](https://github.com/aestream/faery/blob/main/python/faery/frame_stream.py#L19) | Dense frame data represented as `[timestamp, np.ndarray`](https://github.com/aestream/faery/blob/main/python/faery/frame_stream.py#L165). | - - -## Finite, infinite, and regular streams -Apart from the type of data, streams can have different characteristics that matter a great deal for what you can do with them. -For instance, a finite stream can be processed in a single pass, while an infinite stream requires some kind of continuous processing. -Additionally, streams can be structured in time by sending data at regular intervals. - -| Stream type | Description | Examples | -|-------------|-------------|-----------------| -| InfiniteStream | An infinite stream requires some kind of continuous processing. | Reading from a camera or UDP source. | -| FiniteStream | A finite stream can be processed in a single pass. | Reading from a file or a finite source. | -| RegularStream | A regular stream sends data at regular intervals. | Filtering an event stream to output events at regular intervals. | -| FiniteRegularStream | A finite regular stream sends data at regular intervals and can be processed in a single pass. | Filtering a finite event stream to output events at regular intervals. | diff --git a/docs/stream_types_diagram.svg b/docs/stream_types_diagram.svg new file mode 100644 index 0000000..4770c63 --- /dev/null +++ b/docs/stream_types_diagram.svg @@ -0,0 +1,363 @@ + + + + + + + + + + + Faery Stream Type Hierarchy + + ← Irregular Timing | Regular Timing → + ← Finite | Infinite → + + + + EventsStream + Infinite + Irregular + Camera, UDP + Real-time processing + + + RegularEventsStream + Infinite + Regular + Regularized camera + Live video output + + + FiniteEventsStream + Finite + Irregular + Files, stdin + Batch analysis + + + FiniteRegularEventsStream + Finite + Regular + Regularized files + Video generation + + + regularize() + + regularize() + + + time_slice() + + time_slice() + + + Common Operations + Available on all types: + .crop() .filter_polarity() + .to_array().to_event_rate() ... + .transpose() .apply() .map() + Type-specific: + + + Pipeline Examples + File → regularize() → render() → .to_file() + Camera → regularize() → .to_udp() + diff --git a/pyproject.toml b/pyproject.toml index bcf2382..aa99a36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,10 +19,10 @@ tests = ["pytest==8.3.3"] dev = [ "maturin", "pytest", - "isort", + "isort==6.0.1", "black==25.1.0", - "pyright", - "jupyter-book>=2.0.0a0", + "pyright==1.1.405", + "jupyter-book>=2.0.0b3", ] camera = [ "neuromorphic-drivers", diff --git a/python/faery/cli/list_filters.py b/python/faery/cli/list_filters.py index 87f7e52..12ba5d4 100644 --- a/python/faery/cli/list_filters.py +++ b/python/faery/cli/list_filters.py @@ -29,8 +29,10 @@ r"^\s*\(?\s*(\d+\.?\d*)\s*[,\s]\s*(\d+\.?\d*)\s*[,\s]\s*((?:\d+\.?\d*)\s*[,\s]\s*)?(\d+\.?\d*)\s*\)?\s*$" ) HEX_COLOR_PATTERN = re.compile(r"^\s*(#?[0-9A-Fa-f]{6}(?:[0-9A-Fa-f]{2})?)\s*$") -DIMENSIONS_PATTERN = re.compile(r"^\s*\(?\s*(\d+)\s*[,\s]\s*(\d+)\s*\)?\s*$") -UDP_PATTERN = re.compile(r"^\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{1,5})\s*$") +DIMENSIONS_TUPLE_PATTERN = re.compile(r"^\s*\(?\s*(\d+)\s*[,\s]\s*(\d+)\s*\)?\s*$") +DIMENSIONS_X_PATTERN = re.compile(r"^\s*(\d+)x(\d+)\s*$") +UDP_IPV4_PATTERN = re.compile(r"^\s*(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d{1,5})\s*$") +UDP_HOSTNAME_PATTERN = re.compile(r"^\s*([a-zA-Z0-9\-\.]+):(\d{1,5})\s*$") @dataclasses.dataclass @@ -106,20 +108,34 @@ def parse_color(string: str) -> faery.Color: def parse_dimensions(string: str) -> tuple[int, int]: - dimensions_match = DIMENSIONS_PATTERN.match(string) + # Try WIDTHxHEIGHT format first + dimensions_match = DIMENSIONS_X_PATTERN.match(string) if dimensions_match is not None: return (int(dimensions_match[1]), int(dimensions_match[2])) + + # Fall back to (width, height) tuple format + dimensions_match = DIMENSIONS_TUPLE_PATTERN.match(string) + if dimensions_match is not None: + return (int(dimensions_match[1]), int(dimensions_match[2])) + raise argparse.ArgumentTypeError( - f'parsing "{string}" failed (expected "(width, height)")' + f'parsing "{string}" failed (expected "WIDTHxHEIGHT" like "640x480" or "(width, height)" like "(640, 480)")' ) def parse_factor_or_minimum_dimensions( string: str, ) -> typing.Union[float, tuple[int, int]]: - dimensions_match = DIMENSIONS_PATTERN.match(string) + # Try WIDTHxHEIGHT format first + dimensions_match = DIMENSIONS_X_PATTERN.match(string) + if dimensions_match is not None: + return (int(dimensions_match[1]), int(dimensions_match[2])) + + # Fall back to (width, height) tuple format + dimensions_match = DIMENSIONS_TUPLE_PATTERN.match(string) if dimensions_match is not None: return (int(dimensions_match[1]), int(dimensions_match[2])) + return float(string) @@ -146,11 +162,18 @@ def parse_udp( ) -> typing.Union[ tuple[str, int], tuple[str, int, typing.Optional[int], typing.Optional[str]] ]: - match = UDP_PATTERN.match(string) + # Try IPv4 address pattern first + match = UDP_IPV4_PATTERN.match(string) + if match is not None: + return (match[1], int(match[2])) + + # Try hostname pattern + match = UDP_HOSTNAME_PATTERN.match(string) if match is not None: return (match[1], int(match[2])) + raise argparse.ArgumentTypeError( - f'parsing "{string}" failed (expected "a.b.c.d:port")' + f'parsing "{string}" failed (expected "host:port" where host can be an IP address like "192.168.1.1:7777" or hostname like "localhost:7777")' ) diff --git a/python/faery/cli/process.py b/python/faery/cli/process.py index 39c87f7..134786e 100644 --- a/python/faery/cli/process.py +++ b/python/faery/cli/process.py @@ -91,7 +91,7 @@ def input_parser() -> argparse.ArgumentParser: type=str, choices=["Auto", "NeuromorphicDrivers", "EventCameraDrivers"], default="Auto", - help="The driver to detect the camera, defaults to Auto which tries both the NeuromorphicDrivers and EventCameraDrivers libraries" + help="The driver to detect the camera, defaults to Auto which tries both the NeuromorphicDrivers and EventCameraDrivers libraries", ) # Stdin subparser subparser = subparsers.add_parser("stdin") @@ -514,7 +514,9 @@ def set_input(self, arguments: list[str]): ), ) elif args.input == "camera": - self.stream = faery.events_stream_from_camera(driver=args.driver, buffer_size=args.buffer_size) + self.stream = faery.events_stream_from_camera( + driver=args.driver, buffer_size=args.buffer_size + ) elif args.input == "udp": self.stream = faery.events_stream_from_udp( dimensions=args.dimensions, diff --git a/python/faery/event_camera_input.py b/python/faery/event_camera_input.py index 6a06c8a..a9a31b5 100644 --- a/python/faery/event_camera_input.py +++ b/python/faery/event_camera_input.py @@ -49,6 +49,7 @@ def __init__( raise e def __iter__(self) -> typing.Iterator[np.ndarray]: + logging.info(f"Starting streaming from event_camera_drivers: {self.camera}") while self.camera.is_running(): v = next(self.camera) yield v @@ -76,6 +77,9 @@ def __init__(self): raise e def __iter__(self) -> typing.Iterator[np.ndarray]: + logging.info( + f"Starting streaming from neuromorphic_drivers: {self.device_list[0]}" + ) with self.nd.open() as device: for status, packet in device: # TODO: Check status diff --git a/python/faery/events_filter.py b/python/faery/events_filter.py index d145f63..79080a2 100644 --- a/python/faery/events_filter.py +++ b/python/faery/events_filter.py @@ -197,11 +197,11 @@ class OffsetT(events_stream.FiniteRegularEventsFilter): """ -@typed_filter({"Finite"}) -class TimeSlice(events_stream.FiniteEventsFilter): # type: ignore +@typed_filter({"", "Finite"}) +class TimeSlice(events_stream.EventsFilter): # type: ignore def __init__( self, - parent: stream.FiniteStream[numpy.ndarray], + parent: stream.Stream[numpy.ndarray], start: timestamp.TimeOrTimecode, end: timestamp.TimeOrTimecode, zero: bool, @@ -212,12 +212,22 @@ def __init__( assert self.start < self.end, f"{start=} must be strictly smaller than {end=}" self.zero = zero + @restrict({"Finite"}) def time_range(self) -> tuple[timestamp.Time, timestamp.Time]: - parent_time_range = self.parent.time_range() + try: + parent_time_range = self.parent.time_range() + # If parent has time_range, intersect with slice bounds + actual_start = max(parent_time_range[0], self.start) + actual_end = min(parent_time_range[1], self.end) + except (AttributeError, NotImplementedError): + # Parent is infinite, use slice bounds + actual_start = self.start + actual_end = self.end + if self.zero: - return (timestamp.Time(microseconds=0), self.end - self.start) + return (timestamp.Time(microseconds=0), actual_end - actual_start) else: - return (self.start, self.end) + return (actual_start, actual_end) def __iter__(self) -> collections.abc.Iterator[numpy.ndarray]: for events in self.parent: diff --git a/python/faery/file_encoder.py b/python/faery/file_encoder.py index cf8545f..6f23647 100644 --- a/python/faery/file_encoder.py +++ b/python/faery/file_encoder.py @@ -376,6 +376,15 @@ def frames_to_file( state_manager = frame_stream_state.StateManager( stream=stream, on_progress=on_progress ) + + # Video files require finite streams (must have known duration) + if state_manager.time_range is None: + raise ValueError( + f"Cannot create video file '{path}' from infinite stream. " + "Use .time_slice() or .event_slice() to make the stream finite first. " + "Example: stream.time_slice(0 * faery.s, 10 * faery.s).to_file('video.mp4')" + ) + frames: list[frame_stream.Frame] = [] if file_type == "mp4": video_encoder = mp4.Encoder( diff --git a/python/faery/frame_stream_state.py b/python/faery/frame_stream_state.py index 4cac0cc..0598b46 100644 --- a/python/faery/frame_stream_state.py +++ b/python/faery/frame_stream_state.py @@ -156,7 +156,7 @@ def start(self): if self.frequency_hz is None: self.on_progress(FrameStreamState(frame="start")) else: - assert self.frame_count is not None + # For infinite regular streams, frame_count can be None self.on_progress( RegularFrameStreamState( frame="start",