From 283fed5b688496b5931d6f1d8e04626c5945f867 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 24 Nov 2023 16:56:15 +0100 Subject: [PATCH 1/5] use collection adpaters for tensor buffer --- CMakeLists.txt | 2 +- src/batch_adapters.hpp | 41 --------------------- src/collection_adapters.hpp | 71 +++++++++++++++++++++++++++++++++++++ src/main.cpp | 20 +++-------- 4 files changed, 76 insertions(+), 58 deletions(-) delete mode 100644 src/batch_adapters.hpp create mode 100644 src/collection_adapters.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index aa2e41e..f2c2e49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ if(NOT DEFINED CMAKE_CXX_STANDARD) endif() include(FetchContent) -FetchContent_Declare(rerun_sdk URL https://github.com/rerun-io/rerun/releases/download/0.10.1/rerun_cpp_sdk.zip) +FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/d5153cb/rerun_cpp_sdk.zip) # TODO: 2023-11-24. Update to latest commit. FetchContent_MakeAvailable(rerun_sdk) find_package(Eigen3 REQUIRED) diff --git a/src/batch_adapters.hpp b/src/batch_adapters.hpp deleted file mode 100644 index f4e7180..0000000 --- a/src/batch_adapters.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#pragma once - -#include - -// Adapters so we can log eigen vectors as rerun positions: -template <> -struct rerun::ComponentBatchAdapter> { - ComponentBatch operator()(const std::vector& container) { - return ComponentBatch::borrow(container.data(), container.size()); - } - - ComponentBatch operator()(std::vector&& container) { - std::vector positions(container.size()); - memcpy(positions.data(), container.data(), container.size() * sizeof(Eigen::Vector3f)); - return ComponentBatch::take_ownership(std::move(positions)); - } -}; - -// Adapters so we can log an eigen matrix as rerun positions: -template <> -struct rerun::ComponentBatchAdapter { - ComponentBatch operator()(const Eigen::Matrix3Xf& matrix) { - // Sanity check that this is binary compatible. - static_assert( - sizeof(rerun::Position3D) == - sizeof(Eigen::Matrix3Xf::Scalar) * Eigen::Matrix3Xf::RowsAtCompileTime - ); - static_assert(alignof(rerun::Position3D) <= alignof(Eigen::Matrix3Xf::Scalar)); - return ComponentBatch::borrow( - // Cast to void because otherwise Rerun will try to do above sanity checks with the wrong type (scalar). - reinterpret_cast(matrix.data()), - matrix.cols() - ); - } - - ComponentBatch operator()(Eigen::Matrix3Xf&& matrix) { - std::vector positions(matrix.cols()); - memcpy(positions.data(), matrix.data(), matrix.size() * sizeof(rerun::Position3D)); - return ComponentBatch::take_ownership(std::move(positions)); - } -}; diff --git a/src/collection_adapters.hpp b/src/collection_adapters.hpp new file mode 100644 index 0000000..dd0e56c --- /dev/null +++ b/src/collection_adapters.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include +#include + +// Adapters so we can log eigen vectors as rerun positions: +template <> +struct rerun::CollectionAdapter> { + Collection operator()(const std::vector& container) { + return Collection::borrow(container.data(), container.size()); + } + + Collection operator()(std::vector&& container) { + std::vector positions(container.size()); + memcpy(positions.data(), container.data(), container.size() * sizeof(Eigen::Vector3f)); + return Collection::take_ownership(std::move(positions)); + } +}; + +// Adapters so we can log an eigen matrix as rerun positions: +template <> +struct rerun::CollectionAdapter { + Collection operator()(const Eigen::Matrix3Xf& matrix) { + // Sanity check that this is binary compatible. + static_assert( + sizeof(rerun::Position3D) == + sizeof(Eigen::Matrix3Xf::Scalar) * Eigen::Matrix3Xf::RowsAtCompileTime + ); + static_assert(alignof(rerun::Position3D) <= alignof(Eigen::Matrix3Xf::Scalar)); + return Collection::borrow( + // Cast to void because otherwise Rerun will try to do above sanity checks with the wrong type (scalar). + reinterpret_cast(matrix.data()), + matrix.cols() + ); + } + + Collection operator()(Eigen::Matrix3Xf&& matrix) { + std::vector positions(matrix.cols()); + memcpy(positions.data(), matrix.data(), matrix.size() * sizeof(rerun::Position3D)); + return Collection::take_ownership(std::move(positions)); + } +}; + +// Adapters so we can borrow an OpenCV image easily into Rerun images without copying: +template <> +struct rerun::CollectionAdapter { + Collection operator()(const cv::Mat& img) { + return Collection::borrow(img.data, img.total() * img.channels()); + } + + Collection operator()(cv::Mat&& img) { + std::vector img_vec(img.total() * img.channels()); + img_vec.assign(img.data, img.data + img.total() * img.channels()); + return Collection::take_ownership(std::move(img_vec)); + } +}; + +// Adapter for extracting tensor dimensions from an OpenCV matrix. +template <> +struct rerun::CollectionAdapter { + Collection operator()(const cv::Mat& img) { + // Only specify the const& operator since there is no way of borrowing the dimensions anyways. + return { + static_cast(img.rows), + static_cast(img.cols), + static_cast(img.channels()), + }; + } +}; \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index 5167a25..2416062 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -8,7 +8,7 @@ #include #include -#include "batch_adapters.hpp" +#include "collection_adapters.hpp" std::vector generate_random_points_vector(int num_points) { std::vector points(num_points); @@ -51,8 +51,8 @@ int main() { rec.log( "world/camera", rerun::Transform3D( - rerun::datatypes::Vec3D(camera_position.data()), - rerun::datatypes::Mat3x3(camera_orientation.data()) + rerun::Vec3D(camera_position.data()), + rerun::Mat3x3(camera_orientation.data()) ) ); @@ -66,19 +66,7 @@ int main() { // Log image to Rerun cv::cvtColor(img, img, cv::COLOR_BGR2RGB); // Rerun expects RGB format - // NOTE currently we need to construct a vector to log an image, this will change in the future - // see https://github.com/rerun-io/rerun/issues/3794 - std::vector img_vec(img.total() * img.channels()); - img_vec.assign(img.data, img.data + img.total() * img.channels()); - rec.log( - "image", - rerun::Image( - {static_cast(img.rows), - static_cast(img.cols), - static_cast(img.channels())}, - std::move(img_vec) - ) - ); + rec.log("image", rerun::Image(img, rerun::datatypes::TensorBuffer::u8(img))); return 0; } From 516b8f4263c5588ecfcd4165ec48225804de7299 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Fri, 24 Nov 2023 17:02:42 +0100 Subject: [PATCH 2/5] note on on to be removed datatypes:: namespace resolution --- src/collection_adapters.hpp | 1 + src/main.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/collection_adapters.hpp b/src/collection_adapters.hpp index dd0e56c..7a29845 100644 --- a/src/collection_adapters.hpp +++ b/src/collection_adapters.hpp @@ -58,6 +58,7 @@ struct rerun::CollectionAdapter { }; // Adapter for extracting tensor dimensions from an OpenCV matrix. +// TODO(https://github.com/rerun-io/rerun/pull/4331): remove `datatypes::` template <> struct rerun::CollectionAdapter { Collection operator()(const cv::Mat& img) { diff --git a/src/main.cpp b/src/main.cpp index 2416062..1cc89bf 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,6 +65,7 @@ int main() { } // Log image to Rerun + // TODO(https://github.com/rerun-io/rerun/pull/4331): remove `datatypes::` cv::cvtColor(img, img, cv::COLOR_BGR2RGB); // Rerun expects RGB format rec.log("image", rerun::Image(img, rerun::datatypes::TensorBuffer::u8(img))); From 6480a8281137bd159dc0e26b58f6f62d89f7f2bb Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Mon, 27 Nov 2023 13:14:36 +0100 Subject: [PATCH 3/5] add assertion, comment improvements --- src/collection_adapters.hpp | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/collection_adapters.hpp b/src/collection_adapters.hpp index 7a29845..03c3876 100644 --- a/src/collection_adapters.hpp +++ b/src/collection_adapters.hpp @@ -5,13 +5,17 @@ #include #include +#include + // Adapters so we can log eigen vectors as rerun positions: template <> struct rerun::CollectionAdapter> { + /// Borrow for non-temporary. Collection operator()(const std::vector& container) { return Collection::borrow(container.data(), container.size()); } + // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). Collection operator()(std::vector&& container) { std::vector positions(container.size()); memcpy(positions.data(), container.data(), container.size() * sizeof(Eigen::Vector3f)); @@ -22,12 +26,14 @@ struct rerun::CollectionAdapter> // Adapters so we can log an eigen matrix as rerun positions: template <> struct rerun::CollectionAdapter { + // Sanity check that this is binary compatible. + static_assert( + sizeof(rerun::Position3D) == + sizeof(Eigen::Matrix3Xf::Scalar) * Eigen::Matrix3Xf::RowsAtCompileTime + ); + + /// Borrow for non-temporary. Collection operator()(const Eigen::Matrix3Xf& matrix) { - // Sanity check that this is binary compatible. - static_assert( - sizeof(rerun::Position3D) == - sizeof(Eigen::Matrix3Xf::Scalar) * Eigen::Matrix3Xf::RowsAtCompileTime - ); static_assert(alignof(rerun::Position3D) <= alignof(Eigen::Matrix3Xf::Scalar)); return Collection::borrow( // Cast to void because otherwise Rerun will try to do above sanity checks with the wrong type (scalar). @@ -36,6 +42,7 @@ struct rerun::CollectionAdapter { ); } + // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). Collection operator()(Eigen::Matrix3Xf&& matrix) { std::vector positions(matrix.cols()); memcpy(positions.data(), matrix.data(), matrix.size() * sizeof(rerun::Position3D)); @@ -46,11 +53,21 @@ struct rerun::CollectionAdapter { // Adapters so we can borrow an OpenCV image easily into Rerun images without copying: template <> struct rerun::CollectionAdapter { + /// Borrow for non-temporary. Collection operator()(const cv::Mat& img) { + assert( + "OpenCV matrix was expected have bit depth CV_U8" && CV_MAT_DEPTH(img.type()) == CV_8U + ); + return Collection::borrow(img.data, img.total() * img.channels()); } + // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). Collection operator()(cv::Mat&& img) { + assert( + "OpenCV matrix was expected have bit depth CV_U8" && CV_MAT_DEPTH(img.type()) == CV_8U + ); + std::vector img_vec(img.total() * img.channels()); img_vec.assign(img.data, img.data + img.total() * img.channels()); return Collection::take_ownership(std::move(img_vec)); @@ -61,8 +78,8 @@ struct rerun::CollectionAdapter { // TODO(https://github.com/rerun-io/rerun/pull/4331): remove `datatypes::` template <> struct rerun::CollectionAdapter { + /// Only overload the const& operator since there is no way of borrowing the dimensions anyways. Collection operator()(const cv::Mat& img) { - // Only specify the const& operator since there is no way of borrowing the dimensions anyways. return { static_cast(img.rows), static_cast(img.cols), From 2e6876c2149fdc8bee2c1a3892f397ddf9e76b1d Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 28 Nov 2023 12:38:11 +0100 Subject: [PATCH 4/5] latest version. no datatypes --- CMakeLists.txt | 2 +- src/collection_adapters.hpp | 11 +++++------ src/main.cpp | 3 +-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2c2e49..b2b5ae1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,7 @@ if(NOT DEFINED CMAKE_CXX_STANDARD) endif() include(FetchContent) -FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/d5153cb/rerun_cpp_sdk.zip) # TODO: 2023-11-24. Update to latest commit. +FetchContent_Declare(rerun_sdk URL https://build.rerun.io/commit/746dbf3/rerun_cpp_sdk.zip) # TODO: 2023-11-28. Update to latest commit. FetchContent_MakeAvailable(rerun_sdk) find_package(Eigen3 REQUIRED) diff --git a/src/collection_adapters.hpp b/src/collection_adapters.hpp index 03c3876..e800576 100644 --- a/src/collection_adapters.hpp +++ b/src/collection_adapters.hpp @@ -15,7 +15,7 @@ struct rerun::CollectionAdapter> return Collection::borrow(container.data(), container.size()); } - // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). + // Do a full copy for temporaries (otherwise the data might be deleted when the temporary is destroyed). Collection operator()(std::vector&& container) { std::vector positions(container.size()); memcpy(positions.data(), container.data(), container.size() * sizeof(Eigen::Vector3f)); @@ -42,7 +42,7 @@ struct rerun::CollectionAdapter { ); } - // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). + // Do a full copy for temporaries (otherwise the data might be deleted when the temporary is destroyed). Collection operator()(Eigen::Matrix3Xf&& matrix) { std::vector positions(matrix.cols()); memcpy(positions.data(), matrix.data(), matrix.size() * sizeof(rerun::Position3D)); @@ -62,7 +62,7 @@ struct rerun::CollectionAdapter { return Collection::borrow(img.data, img.total() * img.channels()); } - // Do a full copy for temporaries (otherwise the data will be deleted when the temporary is destroyed). + // Do a full copy for temporaries (otherwise the data might be deleted when the temporary is destroyed). Collection operator()(cv::Mat&& img) { assert( "OpenCV matrix was expected have bit depth CV_U8" && CV_MAT_DEPTH(img.type()) == CV_8U @@ -75,11 +75,10 @@ struct rerun::CollectionAdapter { }; // Adapter for extracting tensor dimensions from an OpenCV matrix. -// TODO(https://github.com/rerun-io/rerun/pull/4331): remove `datatypes::` template <> -struct rerun::CollectionAdapter { +struct rerun::CollectionAdapter { /// Only overload the const& operator since there is no way of borrowing the dimensions anyways. - Collection operator()(const cv::Mat& img) { + Collection operator()(const cv::Mat& img) { return { static_cast(img.rows), static_cast(img.cols), diff --git a/src/main.cpp b/src/main.cpp index 1cc89bf..af80f30 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -65,9 +65,8 @@ int main() { } // Log image to Rerun - // TODO(https://github.com/rerun-io/rerun/pull/4331): remove `datatypes::` cv::cvtColor(img, img, cv::COLOR_BGR2RGB); // Rerun expects RGB format - rec.log("image", rerun::Image(img, rerun::datatypes::TensorBuffer::u8(img))); + rec.log("image", rerun::Image(img, rerun::TensorBuffer::u8(img))); return 0; } From a9d86500b2f9e685a359aed00a61f06405bad671 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Tue, 28 Nov 2023 12:42:59 +0100 Subject: [PATCH 5/5] show image logging with and without image data --- src/collection_adapters.hpp | 13 ------------- src/main.cpp | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/collection_adapters.hpp b/src/collection_adapters.hpp index e800576..8c6c897 100644 --- a/src/collection_adapters.hpp +++ b/src/collection_adapters.hpp @@ -73,16 +73,3 @@ struct rerun::CollectionAdapter { return Collection::take_ownership(std::move(img_vec)); } }; - -// Adapter for extracting tensor dimensions from an OpenCV matrix. -template <> -struct rerun::CollectionAdapter { - /// Only overload the const& operator since there is no way of borrowing the dimensions anyways. - Collection operator()(const cv::Mat& img) { - return { - static_cast(img.rows), - static_cast(img.cols), - static_cast(img.channels()), - }; - } -}; \ No newline at end of file diff --git a/src/main.cpp b/src/main.cpp index af80f30..6a80fdc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -18,6 +18,10 @@ std::vector generate_random_points_vector(int num_points) { return points; } +rerun::Collection tensor_shape(const cv::Mat& img) { + return {img.rows, img.cols, img.channels()}; +}; + int main() { const auto rec = rerun::RecordingStream("rerun_example_cpp"); rec.spawn().exit_on_failure(); @@ -64,9 +68,15 @@ int main() { return 1; } - // Log image to Rerun - cv::cvtColor(img, img, cv::COLOR_BGR2RGB); // Rerun expects RGB format - rec.log("image", rerun::Image(img, rerun::TensorBuffer::u8(img))); + // Rerun expects RGB format + cv::cvtColor(img, img, cv::COLOR_BGR2RGB); + + // Log image to rerun using the tensor buffer adapter defined in `collection_adapters.hpp`. + rec.log("image0", rerun::Image(tensor_shape(img), rerun::TensorBuffer::u8(img))); + + // Or by passing a pointer to the image data. + // The pointer cast here is redundant since `data` is already uint8_t in this case, but if you have e.g. a float image it may be necessary to cast to float*. + rec.log("image1", rerun::Image(tensor_shape(img), reinterpret_cast(img.data))); return 0; }