Dev/benchmarks

.gitattributes

@@ -0,0 +1,10 @@
+* text=auto eol=lf
+
+*.png binary
+*.jpg binary
+*.jpeg binary
+*.gif binary
+*.pdf binary
+*.so binary
+*.dll binary
+*.exe binary

.gitignore

@@ -4,4 +4,8 @@
 **/.idea/*
 .cache/
 bench/
-experiment/
+experiment/
+**/results
+**.pyc
+**/_pychache__
+.artifacts/

CMakeLists.txt

@@ -6,6 +6,19 @@ project(dsr
 
 include(GNUInstallDirs)
 
+# -DTSAN=ON  enables ThreadSanitizer across the whole build (library + tests/benchmarks).
+# Must be applied before any add_subdirectory so every TU is instrumented.
+# Incompatible with SANITIZER (ASan/UBSan) — enforce that here.
+option(TSAN "Enable ThreadSanitizer" OFF)
+if (TSAN)
+    if (SANITIZER)
+        message(FATAL_ERROR "TSAN and SANITIZER (ASan+UBSan) are mutually exclusive")
+    endif()
+    message(STATUS "ThreadSanitizer enabled")
+    add_compile_options(-fsanitize=thread -fno-omit-frame-pointer)
+    add_link_options(-fsanitize=thread)
+endif()
+
 add_definitions(-I/usr/include/x86_64-linux-gnu/qt6/QtOpenGLWidgets/)
 
 include_directories(/home/robocomp/robocomp/classes)
@@ -27,3 +40,9 @@ if (WITH_TESTS)
 add_subdirectory(tests)
 
 endif()
+
+if (WITH_BENCHMARKS)
+
+add_subdirectory(benchmarks)
+
+endif()

api/dsr_inner_eigen_api.cpp

@@ -123,10 +123,8 @@ std::optional<Mat::RTMat> InnerEigenAPI::get_transformation_matrix(const std::st
             }
         }
         // update node cache reference
-        uint64_t dst_id = G->get_node(dest).value().id();
-        node_map[dst_id].push_back(key);
-        uint64_t orig_id = G->get_node(orig).value().id();
-        node_map[orig_id].push_back(key);
+        node_map[bn.value().id()].push_back(key);
+        node_map[an.value().id()].push_back(key);
 
         // update cache
         auto ret = btotal.inverse() * atotal;
@@ -212,7 +210,6 @@ std::optional<Mat::Vector6d> InnerEigenAPI::transform_axis(const std::string &de
 
 std::optional<Mat::Vector6d> InnerEigenAPI::transform_axis( const std::string &dest,  const std::string & orig, std::uint64_t timestamp)
 {
-    Mat::Vector6d v;
 	return transform_axis(dest, Mat::Vector6d::Zero(), orig, timestamp);
 }
 

api/include/dsr/api/dsr_api.h

@@ -57,6 +57,7 @@ namespace DSR
     class DSRGraph : public QObject
     {
         friend RT_API;
+        friend class DSRGraphTestAccess;
 
         public:
         size_t size() const;
@@ -584,7 +585,6 @@ namespace DSR
         const bool copy;
         std::unique_ptr<Utilities> utils;
         std::unordered_set<std::string_view> ignored_attributes;
-        ThreadPool tp, tp_delta_attr;
         bool same_host;
         id_generator generator;
         GraphSettings::LOGLEVEL log_level;
@@ -677,6 +677,11 @@ namespace DSR
         std::unordered_multimap<uint64_t, std::tuple<uint64_t, std::string, mvreg<DSR::CRDTEdge>, uint64_t>> unprocessed_delta_edge_to;
         std::unordered_multimap<std::tuple<uint64_t, uint64_t, std::string>, std::tuple<std::string, mvreg<DSR::CRDTAttribute>, uint64_t>, hash_tuple> unprocessed_delta_edge_att;
 
+        // ThreadPools are declared after all data they access so that their
+        // destructors (which join worker threads) run before the data members
+        // are destroyed, preventing use-after-free data races on shutdown.
+        ThreadPool tp, tp_delta_attr;
+
         //Custom function for each rtps topic
         class NewMessageFunctor {
         public:

benchmarks/CMakeLists.txt

@@ -0,0 +1,160 @@
+cmake_minimum_required(VERSION 3.10)
+project(dsr_benchmarks
+        VERSION 2024.12.01
+        DESCRIPTION "DSR Benchmarking Suite"
+        LANGUAGES CXX)
+
+# Fetch Catch2 if not already available
+Include(FetchContent)
+
+FetchContent_Declare(
+  Catch2
+  GIT_REPOSITORY https://github.com/catchorg/Catch2.git
+  GIT_TAG        v3.8.0
+)
+
+FetchContent_Declare(
+  nanobench
+  GIT_REPOSITORY https://github.com/martinus/nanobench.git
+  GIT_TAG        v4.3.11
+)
+
+FetchContent_MakeAvailable(Catch2 nanobench)
+
+# Find required packages
+find_package(Boost REQUIRED)
+find_package(Qt6 COMPONENTS Core REQUIRED)
+find_package(Eigen3 3.3 REQUIRED NO_MODULE)
+
+# Collect source files
+set(BENCHMARK_SOURCES
+    benchmark_main.cpp
+
+    # Latency benchmarks
+    latency/delta_propagation_bench.cpp
+    latency/signal_latency_bench.cpp
+    latency/crdt_join_bench.cpp
+
+    # Throughput benchmarks
+    throughput/single_agent_ops_bench.cpp
+    throughput/concurrent_writers_bench.cpp
+    throughput/single_agent_ops_with_latency_bench.cpp
+    throughput/query_ops_bench.cpp
+
+    # Scalability benchmarks
+    scalability/multi_agent_sync_bench.cpp
+    scalability/graph_size_impact_bench.cpp
+    scalability/thread_scaling_bench.cpp
+    scalability/graph_size_scaling_bench.cpp
+    scalability/agent_scaling_bench.cpp
+
+    # Consistency benchmarks
+    consistency/convergence_time_bench.cpp
+    consistency/conflict_rate_bench.cpp
+)
+
+# Header files for IDE integration
+set(BENCHMARK_HEADERS
+    core/benchmark_config.h
+    core/timing_utils.h
+    core/metrics_collector.h
+    core/nanobench_adapter.h
+    core/report_generator.h
+    fixtures/multi_agent_fixture.h
+    fixtures/graph_generator.h
+)
+
+# Create benchmark executable
+add_executable(dsr_benchmarks
+    ${BENCHMARK_SOURCES}
+    ${BENCHMARK_HEADERS}
+)
+
+# Set C++ standard
+set_target_properties(dsr_benchmarks PROPERTIES
+    CMAKE_CXX_STANDARD 23
+    CXX_STANDARD_REQUIRED ON
+    CXX_EXTENSIONS ON
+)
+
+target_compile_options(dsr_benchmarks PUBLIC -g -std=c++23)
+
+# -DTSAN=ON  enables ThreadSanitizer.  Requires the dsr_api/dsr_core libraries
+# to also be built with TSAN=ON (via the root CMakeLists.txt), otherwise TSan
+# will report false positives from uninstrumented library code.
+option(TSAN "Enable ThreadSanitizer" OFF)
+if (TSAN)
+    message(STATUS "ThreadSanitizer enabled for benchmarks")
+    target_compile_options(dsr_benchmarks PRIVATE -fsanitize=thread -fno-omit-frame-pointer)
+    target_link_options(dsr_benchmarks PRIVATE -fsanitize=thread)
+endif()
+
+# Include directories
+target_include_directories(dsr_benchmarks PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}
+    ${CMAKE_CURRENT_SOURCE_DIR}/core
+    ${CMAKE_CURRENT_SOURCE_DIR}/fixtures
+)
+
+# Link libraries
+target_link_libraries(dsr_benchmarks PRIVATE
+    Catch2::Catch2
+    nanobench
+    dsr_api
+    dsr_core
+    Qt6::Core
+    Eigen3::Eigen
+    fastdds
+    fastcdr
+)
+
+# Create results directory
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/results)
+
+# Copy results directory structure
+add_custom_command(TARGET dsr_benchmarks POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:dsr_benchmarks>/results
+    COMMENT "Creating results directory"
+)
+
+# Flamegraph target — generates one SVG per benchmark test case via perf.
+# Requires: perf, and FlameGraph scripts (flamegraph.pl + stackcollapse-perf.pl).
+# Set FG_DIR to the FlameGraph checkout if the scripts aren't on PATH, e.g.:
+#   cmake --build . --target flamegraph -j1 -- FG_DIR=/opt/FlameGraph
+# Or pass a Catch2 filter to profile a subset:
+#   cmake --build . --target flamegraph -j1 -- BENCH_FILTER=[LATENCY]
+set(FLAMEGRAPH_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/flamegraph.sh)
+set(FLAMEGRAPH_OUTDIR ${CMAKE_CURRENT_BINARY_DIR}/results/flamegraphs)
+add_custom_target(flamegraph
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${FLAMEGRAPH_OUTDIR}
+    COMMAND env
+        FG_DIR=$ENV{FG_DIR}
+        ${FLAMEGRAPH_SCRIPT}
+            -b $<TARGET_FILE:dsr_benchmarks>
+            -o ${FLAMEGRAPH_OUTDIR}
+            $ENV{BENCH_FILTER}
+    DEPENDS dsr_benchmarks
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    COMMENT "Generating per-benchmark flamegraphs in ${FLAMEGRAPH_OUTDIR}"
+    USES_TERMINAL
+)
+
+# Register tests with CTest (optional)
+# Disabled auto-discovery as it requires running the binary at build time
+# which may fail if libraries are not in LD_LIBRARY_PATH
+# include(Catch)
+# catch_discover_tests(dsr_benchmarks)
+
+# Installation (optional)
+install(TARGETS dsr_benchmarks
+    RUNTIME DESTINATION bin
+)
+
+# Print configuration summary
+message(STATUS "")
+message(STATUS "DSR Benchmarks Configuration:")
+message(STATUS "  Build type: ${CMAKE_BUILD_TYPE}")
+message(STATUS "  C++ Standard: C++23")
+message(STATUS "  Catch2 version: 3.8.0")
+message(STATUS "  nanobench version: 4.3.11")
+message(STATUS "")

benchmarks/benchmark_main.cpp

@@ -0,0 +1,131 @@
+// DSR Benchmarking Suite
+// Main entry point using Catch2
+
+#define CATCH_CONFIG_RUNNER
+#include <catch2/catch_session.hpp>
+#include <QCoreApplication>
+#include <QtGlobal>
+#include <iostream>
+#include <stdexcept>
+
+// Custom Qt message handler to filter debug output during benchmarks
+static bool g_verbose = false;
+
+namespace {
+
+bool hasCliFlag(int argc, char* argv[], const char* flag) {
+    for (int i = 1; i < argc; ++i) {
+        if (std::string(argv[i]) == flag) {
+            return true;
+        }
+    }
+    return false;
+}
+
+bool shouldPrintBenchmarkPreamble(int argc, char* argv[]) {
+    return !hasCliFlag(argc, argv, "--help")
+        && !hasCliFlag(argc, argv, "-?")
+        && !hasCliFlag(argc, argv, "--list-tests")
+        && !hasCliFlag(argc, argv, "--list-tags")
+        && !hasCliFlag(argc, argv, "--list-reporters")
+        && !hasCliFlag(argc, argv, "--list-listeners");
+}
+
+}  // namespace
+
+void benchmarkMessageHandler(QtMsgType type, const QMessageLogContext& context, const QString& msg) {
+    // In non-verbose mode, only show warnings and above
+    if (!g_verbose) {
+        switch (type) {
+            case QtDebugMsg:
+            case QtInfoMsg:
+                return;  // Suppress debug and info messages
+            default:
+                break;
+        }
+    }
+
+    // Format and output remaining messages
+    QByteArray localMsg = msg.toLocal8Bit();
+    switch (type) {
+        case QtDebugMsg:
+            std::cout << "[DEBUG] " << localMsg.constData() << std::endl;
+            break;
+        case QtInfoMsg:
+            std::cout << "[INFO] " << localMsg.constData() << std::endl;
+            break;
+        case QtWarningMsg:
+            std::cout << "[WARNING] " << localMsg.constData() << std::endl;
+            break;
+        case QtCriticalMsg:
+            std::cout << "[CRITICAL] " << localMsg.constData() << std::endl;
+            break;
+        case QtFatalMsg:
+            std::cout << "[FATAL] " << localMsg.constData() << std::endl;
+            // Throw instead of abort() so the fixture's try/catch can catch it,
+            // mark the test as failed, and let Catch2 continue to the next test.
+            throw std::runtime_error(localMsg.constData());
+    }
+}
+
+int main(int argc, char* argv[]) {
+    // Install custom message handler before QCoreApplication
+    qInstallMessageHandler(benchmarkMessageHandler);
+
+    // Check for verbose flag
+    for (int i = 1; i < argc; ++i) {
+        if (std::string(argv[i]) == "--verbose" || std::string(argv[i]) == "-v") {
+            g_verbose = true;
+            break;
+        }
+    }
+
+    // Initialize Qt (required for signals/slots)
+    QCoreApplication app(argc, argv);
+    // Initialize Catch2
+    Catch::Session session;
+
+    // Set default reporter to console with colors
+    session.configData().showDurations = Catch::ShowDurations::Always;
+
+    // Apply command line arguments
+    int returnCode = session.applyCommandLine(argc, argv);
+    if (returnCode != 0) {
+        return returnCode;
+    }
+
+    if (shouldPrintBenchmarkPreamble(argc, argv)) {
+        std::cout << "=================================\n";
+        std::cout << " DSR Benchmarking Suite\n";
+        std::cout << "=================================\n\n";
+        std::cout << "Available benchmark categories:\n";
+        std::cout << "  [BASELINE]     - Curated low-noise regression baseline\n";
+        std::cout << "  [EXTENDED]     - Slower supplementary baseline coverage\n";
+        std::cout << "  [LATENCY]      - Signal emission, CRDT operations\n";
+        std::cout << "  [THROUGHPUT]   - Single agent insert/read/update/delete, concurrent writers\n";
+        std::cout << "  [CRDT]         - mvreg and dot_context micro-benchmarks\n";
+        std::cout << "  [SCALABILITY]  - Thread scaling, graph size impact\n";
+        std::cout << "  [CONSISTENCY]  - Convergence time, conflict rates\n";
+        std::cout << "  [PROFILE]      - Expensive profiling-focused cases\n";
+        std::cout << "  [LOAD]         - Work-under-load and concurrency-heavy cases\n";
+        std::cout << "  [MULTIAGENT]   - Multi-agent synchronization/consistency cases\n";
+        std::cout << "\n";
+        std::cout << "Usage examples:\n";
+        std::cout << "  ./dsr_benchmarks                    # Run all non-hidden benchmarks\n";
+        std::cout << "  ./dsr_benchmarks \"[BASELINE]\"       # Run curated baseline benchmarks\n";
+        std::cout << "  ./dsr_benchmarks \"[EXTENDED]\"       # Run slower supplementary coverage\n";
+        std::cout << "  ./dsr_benchmarks \"[LATENCY]\"        # Run latency benchmarks\n";
+        std::cout << "  ./dsr_benchmarks \"[THROUGHPUT]\"     # Run throughput benchmarks\n";
+        std::cout << "  ./dsr_benchmarks \"[CRDT]\"           # Run CRDT micro-benchmarks\n";
+        std::cout << "  ./dsr_benchmarks \"[PROFILE][LOAD]\"  # Run long load-heavy cases\n";
+        std::cout << "  ./dsr_benchmarks \"[PROFILE][MULTIAGENT]\" # Run multi-agent profiling cases\n";
+        std::cout << "  ./dsr_benchmarks \"[.multi]\"         # Run multi-agent tests (may timeout)\n";
+        std::cout << "  ./dsr_benchmarks -r json::out=x.json # Export to JSON\n";
+        std::cout << "  ./dsr_benchmarks --verbose          # Show Qt debug messages\n";
+        std::cout << "\n";
+        std::cout << "Note: [.multi] and [.extended] tests are hidden by default.\n";
+        std::cout << "\n";
+    }
+
+    return session.run();
+}