diff --git a/CMakeLists.txt b/CMakeLists.txt index 5d5ade7..0c21d2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -93,3 +93,4 @@ add_subdirectory(simple_joystick_sender) add_subdirectory(simple_joystick_receiver) add_subdirectory(ping_pong_ping) add_subdirectory(ping_pong_pong) +add_subdirectory(user_timestamped_video) diff --git a/user_timestamped_video/CMakeLists.txt b/user_timestamped_video/CMakeLists.txt new file mode 100644 index 0000000..6fdcdd8 --- /dev/null +++ b/user_timestamped_video/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(producer) +add_subdirectory(consumer) diff --git a/user_timestamped_video/README.md b/user_timestamped_video/README.md new file mode 100644 index 0000000..5d3f5a1 --- /dev/null +++ b/user_timestamped_video/README.md @@ -0,0 +1,55 @@ +# UserTimestampedVideo + +This example is split into two executables and can demonstrate all four +producer/consumer combinations: + +- `UserTimestampedVideoProducer` publishes a synthetic video track named + `"timestamped-camera"` and stamps each frame with + `VideoCaptureOptions::metadata.user_timestamp_us`. +- `UserTimestampedVideoConsumer` subscribes to the remote + `"timestamped-camera"` track by name with either the rich or legacy callback + path. + +Run them in the same room with different participant identities: + +```sh +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoProducer +LIVEKIT_URL=ws://localhost:7880 LIVEKIT_TOKEN= ./UserTimestampedVideoConsumer +``` + +Flags: + +- Producer default: sends user timestamps +- Producer `--without-user-timestamp`: does not send user timestamps +- Consumer default: reads user timestamps through `setOnVideoFrameEventCallback` +- Consumer `--ignore-user-timestamp`: ignores metadata through the legacy + `setOnVideoFrameCallback` + +Matrix: + +```sh +# 1. Producer sends, consumer reads +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer + +# 2. Producer sends, consumer ignores +./UserTimestampedVideoProducer +./UserTimestampedVideoConsumer --ignore-user-timestamp + +# 3. Producer does not send, consumer ignores +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer --ignore-user-timestamp + +# 4. Producer does not send, consumer reads +./UserTimestampedVideoProducer --without-user-timestamp +./UserTimestampedVideoConsumer +``` + +Timestamp note: + +- `user_ts_us` is application metadata and is the value to compare end to end. +- `capture_ts_us` on the producer is the timestamp submitted to `captureFrame`. +- `capture_ts_us` on the consumer is the received WebRTC frame timestamp. +- Producer and consumer `capture_ts_us` values are not expected to match exactly, + because WebRTC may translate frame timestamps onto its own internal + capture-time timeline before delivery. diff --git a/user_timestamped_video/consumer/CMakeLists.txt b/user_timestamped_video/consumer/CMakeLists.txt new file mode 100644 index 0000000..d2f80a2 --- /dev/null +++ b/user_timestamped_video/consumer/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserTimestampedVideoConsumer + main.cpp +) + +target_include_directories(UserTimestampedVideoConsumer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserTimestampedVideoConsumer PRIVATE ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserTimestampedVideoConsumer) diff --git a/user_timestamped_video/consumer/main.cpp b/user_timestamped_video/consumer/main.cpp new file mode 100644 index 0000000..b54e527 --- /dev/null +++ b/user_timestamped_video/consumer/main.cpp @@ -0,0 +1,256 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserTimestampedVideoConsumer +/// +/// Receives remote video frames via `Room::setOnVideoFrameEventCallback()` and +/// logs any `VideoFrameMetadata::user_timestamp_us` values that arrive. Pair +/// with `UserTimestampedVideoProducer` running in another process. +/// +/// Usage: +/// UserTimestampedVideoConsumer [--ignore-user-timestamp] +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr const char *kTrackName = "timestamped-camera"; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +std::string +formatUserTimestamp(const std::optional &metadata) { + if (!metadata || !metadata->user_timestamp_us.has_value()) { + return "n/a"; + } + + return std::to_string(*metadata->user_timestamp_us); +} + +void printUsage(const char *program) { + std::cerr << "Usage:\n" + << " " << program + << " [--ignore-user-timestamp]\n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program + << " [--ignore-user-timestamp]\n"; +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token, + bool &read_user_timestamp) { + read_user_timestamp = true; + std::vector positional; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return false; + } + if (arg == "--ignore-user-timestamp") { + read_user_timestamp = false; + continue; + } + if (arg == "--read-user-timestamp") { + read_user_timestamp = true; + continue; + } + + positional.push_back(arg); + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() >= 2) { + url = positional[0]; + token = positional[1]; + } + + return !(url.empty() || token.empty()); +} + +class UserTimestampedVideoConsumerDelegate : public RoomDelegate { +public: + UserTimestampedVideoConsumerDelegate(Room &room, bool read_user_timestamp) + : room_(room), read_user_timestamp_(read_user_timestamp) {} + + void registerExistingParticipants() { + for (const auto &participant : room_.remoteParticipants()) { + if (participant) { + registerRemoteVideoCallback(participant->identity()); + } + } + } + + void onParticipantConnected(Room &, + const ParticipantConnectedEvent &event) override { + if (!event.participant) { + return; + } + + std::cout << "[consumer] participant connected: " + << event.participant->identity() << "\n"; + registerRemoteVideoCallback(event.participant->identity()); + } + + void onParticipantDisconnected( + Room &, const ParticipantDisconnectedEvent &event) override { + if (!event.participant) { + return; + } + + const std::string identity = event.participant->identity(); + room_.clearOnVideoFrameCallback(identity, std::string(kTrackName)); + + { + std::lock_guard lock(mutex_); + registered_identities_.erase(identity); + } + + std::cout << "[consumer] participant disconnected: " << identity << "\n"; + } + +private: + void registerRemoteVideoCallback(const std::string &identity) { + { + std::lock_guard lock(mutex_); + if (!registered_identities_.insert(identity).second) { + return; + } + } + + VideoStream::Options stream_options; + stream_options.format = VideoBufferType::RGBA; + + if (read_user_timestamp_) { + room_.setOnVideoFrameEventCallback( + identity, std::string(kTrackName), + [identity](const VideoFrameEvent &event) { + std::cout << "[consumer] from=" << identity + << " size=" << event.frame.width() << "x" + << event.frame.height() + << " capture_ts_us=" << event.timestamp_us + << " user_ts_us=" << formatUserTimestamp(event.metadata) + << " rotation=" << static_cast(event.rotation) + << "\n"; + }, + stream_options); + } else { + room_.setOnVideoFrameCallback( + identity, std::string(kTrackName), + [identity](const VideoFrame &frame, const std::int64_t timestamp_us) { + std::cout << "[consumer] from=" << identity + << " size=" << frame.width() << "x" << frame.height() + << " capture_ts_us=" << timestamp_us + << " user_ts_us=ignored\n"; + }, + stream_options); + } + + std::cout << "[consumer] listening for video frames from " << identity + << " track=\"" << kTrackName << "\" with user timestamp " + << (read_user_timestamp_ ? "enabled" : "ignored") << "\n"; + } + + Room &room_; + bool read_user_timestamp_; + std::mutex mutex_; + std::unordered_set registered_identities_; +}; + +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + bool read_user_timestamp = true; + + if (!parseArgs(argc, argv, url, token, read_user_timestamp)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + UserTimestampedVideoConsumerDelegate delegate(room, read_user_timestamp); + room.setDelegate(&delegate); + + std::cout << "[consumer] connecting to " << url << "\n"; + if (!room.Connect(url, token, options)) { + std::cerr << "[consumer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[consumer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "' with user timestamp " + << (read_user_timestamp ? "enabled" : "ignored") << "\n"; + + delegate.registerExistingParticipants(); + + while (g_running.load(std::memory_order_relaxed)) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + + for (const auto &participant : room.remoteParticipants()) { + if (participant) { + room.clearOnVideoFrameCallback(participant->identity(), + std::string(kTrackName)); + } + } + } + + room.setDelegate(nullptr); + } + + livekit::shutdown(); + return exit_code; +} diff --git a/user_timestamped_video/producer/CMakeLists.txt b/user_timestamped_video/producer/CMakeLists.txt new file mode 100644 index 0000000..5c573ea --- /dev/null +++ b/user_timestamped_video/producer/CMakeLists.txt @@ -0,0 +1,22 @@ +# Copyright 2026 LiveKit, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_executable(UserTimestampedVideoProducer + main.cpp +) + +target_include_directories(UserTimestampedVideoProducer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(UserTimestampedVideoProducer PRIVATE ${LIVEKIT_CORE_TARGET}) + +livekit_copy_windows_runtime_dlls(UserTimestampedVideoProducer) diff --git a/user_timestamped_video/producer/main.cpp b/user_timestamped_video/producer/main.cpp new file mode 100644 index 0000000..87da668 --- /dev/null +++ b/user_timestamped_video/producer/main.cpp @@ -0,0 +1,225 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/// UserTimestampedVideoProducer +/// +/// Publishes a synthetic camera track and stamps each frame with +/// `VideoCaptureOptions::metadata.user_timestamp_us`. Pair with +/// `UserTimestampedVideoConsumer` in another process to observe the user +/// timestamps flowing end to end. +/// +/// Usage: +/// UserTimestampedVideoProducer [--without-user-timestamp] +/// +/// Or via environment variables: +/// LIVEKIT_URL, LIVEKIT_TOKEN + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "livekit/livekit.h" + +using namespace livekit; + +namespace { + +constexpr int kFrameWidth = 640; +constexpr int kFrameHeight = 360; +constexpr int kFrameIntervalMs = 200; + +std::atomic g_running{true}; + +void handleSignal(int) { g_running.store(false); } + +std::string getenvOrEmpty(const char *name) { + const char *value = std::getenv(name); + return value ? std::string(value) : std::string{}; +} + +std::uint64_t nowEpochUs() { + return static_cast( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +void fillFrame(VideoFrame &frame, std::uint32_t frame_index) { + const std::uint8_t blue = static_cast((frame_index * 7) % 255); + const std::uint8_t green = + static_cast((frame_index * 13) % 255); + const std::uint8_t red = static_cast((frame_index * 29) % 255); + + std::uint8_t *data = frame.data(); + for (std::size_t i = 0; i < frame.dataSize(); i += 4) { + data[i + 0] = blue; + data[i + 1] = green; + data[i + 2] = red; + data[i + 3] = 255; + } +} + +void printUsage(const char *program) { + std::cerr << "Usage:\n" + << " " << program + << " [--without-user-timestamp]\n" + << "or:\n" + << " LIVEKIT_URL=... LIVEKIT_TOKEN=... " << program + << " [--without-user-timestamp]\n"; +} + +bool parseArgs(int argc, char *argv[], std::string &url, std::string &token, + bool &send_user_timestamp) { + send_user_timestamp = true; + std::vector positional; + + for (int i = 1; i < argc; ++i) { + const std::string arg = argv[i]; + if (arg == "-h" || arg == "--help") { + return false; + } + if (arg == "--without-user-timestamp") { + send_user_timestamp = false; + continue; + } + if (arg == "--with-user-timestamp") { + send_user_timestamp = true; + continue; + } + + positional.push_back(arg); + } + + url = getenvOrEmpty("LIVEKIT_URL"); + token = getenvOrEmpty("LIVEKIT_TOKEN"); + + if (positional.size() >= 2) { + url = positional[0]; + token = positional[1]; + } + + return !(url.empty() || token.empty()); +} + +} // namespace + +int main(int argc, char *argv[]) { + std::string url; + std::string token; + bool send_user_timestamp = true; + + if (!parseArgs(argc, argv, url, token, send_user_timestamp)) { + printUsage(argv[0]); + return 1; + } + + std::signal(SIGINT, handleSignal); +#ifdef SIGTERM + std::signal(SIGTERM, handleSignal); +#endif + + livekit::initialize(livekit::LogLevel::Info, livekit::LogSink::kConsole); + int exit_code = 0; + + { + Room room; + RoomOptions options; + options.auto_subscribe = true; + options.dynacast = false; + + std::cout << "[producer] connecting to " << url << "\n"; + if (!room.Connect(url, token, options)) { + std::cerr << "[producer] failed to connect\n"; + exit_code = 1; + } else { + std::cout << "[producer] connected as " + << room.localParticipant()->identity() << " to room '" + << room.room_info().name << "'\n"; + + auto source = std::make_shared(kFrameWidth, kFrameHeight); + auto track = + LocalVideoTrack::createLocalVideoTrack("timestamped-camera", source); + + try { + TrackPublishOptions publish_options; + publish_options.source = TrackSource::SOURCE_CAMERA; + publish_options.packet_trailer_features.user_timestamp = + send_user_timestamp; + + room.localParticipant()->publishTrack(track, publish_options); + std::cout << "[producer] published camera track with user timestamp " + << (send_user_timestamp ? "enabled" : "disabled") << "\n"; + + VideoFrame frame = VideoFrame::create(kFrameWidth, kFrameHeight, + VideoBufferType::BGRA); + const auto capture_start = std::chrono::steady_clock::now(); + std::uint32_t frame_index = 0; + auto next_frame_at = std::chrono::steady_clock::now(); + + while (g_running.load(std::memory_order_relaxed)) { + fillFrame(frame, frame_index); + + VideoCaptureOptions capture_options; + + // a steady_clock to align with other data/video frames + capture_options.timestamp_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - capture_start) + .count()); + capture_options.rotation = VideoRotation::VIDEO_ROTATION_0; + if (send_user_timestamp) { + capture_options.metadata = VideoFrameMetadata{}; + capture_options.metadata->user_timestamp_us = nowEpochUs(); + } + + source->captureFrame(frame, capture_options); + + if (frame_index % 5 == 0) { + std::cout << "[producer] frame=" << frame_index + << " capture_ts_us=" << capture_options.timestamp_us + << " user_ts_us=" + << (send_user_timestamp + ? std::to_string( + *capture_options.metadata->user_timestamp_us) + : std::string("disabled")) + << "\n"; + } + + ++frame_index; + next_frame_at += std::chrono::milliseconds(kFrameIntervalMs); + std::this_thread::sleep_until(next_frame_at); + } + } catch (const std::exception &error) { + std::cerr << "[producer] error: " << error.what() << "\n"; + exit_code = 1; + } + + if (track->publication()) { + room.localParticipant()->unpublishTrack(track->publication()->sid()); + } + } + } + + livekit::shutdown(); + return exit_code; +}