From ae6e5079de1b63b02bf758997f55ae6cad9f3f3f Mon Sep 17 00:00:00 2001 From: 36000 Date: Sun, 30 Aug 2026 20:19:32 -0700 Subject: [PATCH] support for large trx --- include/trx/detail/dtype_helpers.h | 12 +- include/trx/trx.h | 35 ++-- include/trx/trx.tpp | 271 +++++++++++++++-------------- src/trx.cpp | 32 ++-- 4 files changed, 180 insertions(+), 170 deletions(-) diff --git a/include/trx/detail/dtype_helpers.h b/include/trx/detail/dtype_helpers.h index e1a0d96..6b1543c 100644 --- a/include/trx/detail/dtype_helpers.h +++ b/include/trx/detail/dtype_helpers.h @@ -3,6 +3,8 @@ #include +#include + #include #include #include @@ -16,25 +18,25 @@ namespace detail { // // MapType must be an Eigen::Map> type. template -inline void remap(MapType &map, void *data, int rows, int cols) { +inline void remap(MapType &map, void *data, std::int64_t rows, std::int64_t cols) { using Scalar = typename MapType::Scalar; new (&map) MapType(reinterpret_cast(data), rows, cols); // NOLINT } // Overload for const data pointers (read-only maps). template -inline void remap(MapType &map, const void *data, int rows, int cols) { +inline void remap(MapType &map, const void *data, std::int64_t rows, std::int64_t cols) { using Scalar = typename MapType::Scalar; new (&map) MapType(const_cast(reinterpret_cast(data)), rows, cols); // NOLINT } // Convenience overloads that unpack a (rows, cols) shape tuple. template -inline void remap(MapType &map, void *data, const std::tuple &shape) { +inline void remap(MapType &map, void *data, const std::tuple &shape) { remap(map, data, std::get<0>(shape), std::get<1>(shape)); } template -inline void remap(MapType &map, const void *data, const std::tuple &shape) { +inline void remap(MapType &map, const void *data, const std::tuple &shape) { remap(map, data, std::get<0>(shape), std::get<1>(shape)); } @@ -45,7 +47,7 @@ std::tuple _split_ext_with_dimensionality(const s template inline Eigen::Matrix _compute_lengths(const Eigen::MatrixBase
&offsets, - int nb_vertices) { + std::int64_t nb_vertices) { static_cast(nb_vertices); if (offsets.size() > 1) { const auto casted = offsets.template cast(); diff --git a/include/trx/trx.h b/include/trx/trx.h index 97babf6..747fc34 100644 --- a/include/trx/trx.h +++ b/include/trx/trx.h @@ -79,6 +79,10 @@ inline json::object _json_object(const json &value) { return json::object(); } +inline int64_t _json_int64(const json &value) { + return static_cast(value.number_value()); +} + inline json _json_set(const json &value, const std::string &key, const json &field) { auto obj = _json_object(value); obj[key] = field; @@ -242,8 +246,8 @@ template class TrxFile { public: struct GroupBackingInfo { std::string filename; - int rows = 0; - int cols = 0; + int64_t rows = 0; + int64_t cols = 0; std::string dtype; long long mem_offset = 0; }; @@ -264,8 +268,8 @@ template class TrxFile { // Member Functions() // TrxFile(int nb_vertices = 0, int nb_streamlines = 0); - TrxFile(int nb_vertices = 0, - int nb_streamlines = 0, + TrxFile(int64_t nb_vertices = 0, + int64_t nb_streamlines = 0, const TrxFile
*init_as = nullptr, std::string reference = ""); ~TrxFile(); @@ -303,7 +307,7 @@ template class TrxFile { * @param nb_vertices The number of vertices to keep * @param delete_dpg Remove data_per_group when resizing */ - void resize(int nb_streamlines = -1, int nb_vertices = -1, bool delete_dpg = false); + void resize(int64_t nb_streamlines = -1, int64_t nb_vertices = -1, bool delete_dpg = false); /** * @brief Save a TrxFile @@ -368,7 +372,7 @@ template class TrxFile { return static_cast(streamlines->_data.rows()); } if (header["NB_VERTICES"].is_number()) { - return static_cast(header["NB_VERTICES"].int_value()); + return static_cast(_json_int64(header["NB_VERTICES"])); } return 0; } @@ -381,7 +385,7 @@ template class TrxFile { return static_cast(streamlines->_lengths.size()); } if (header["NB_STREAMLINES"].is_number()) { - return static_cast(header["NB_STREAMLINES"].int_value()); + return static_cast(_json_int64(header["NB_STREAMLINES"])); } return 0; } @@ -519,8 +523,8 @@ template class TrxFile { * @return std::tuple A tuple representing the end of the copied streamlines and end * of copied points */ - std::tuple - _copy_fixed_arrays_from(TrxFile
*trx, int strs_start = 0, int pts_start = 0, int nb_strs_to_copy = -1); + std::tuple + _copy_fixed_arrays_from(TrxFile
*trx, int64_t strs_start = 0, int64_t pts_start = 0, int64_t nb_strs_to_copy = -1); int len(); private: @@ -533,7 +537,7 @@ template class TrxFile { * @return std::tuple A tuple representing the index of the last streamline and the * total length of all the streamlines */ - std::tuple _get_real_len(); + std::tuple _get_real_len(); }; namespace detail { @@ -568,8 +572,8 @@ inline std::string make_unique_temp_path(const std::string &prefix) { struct TypedArray { std::string dtype; - int rows = 0; - int cols = 0; + int64_t rows = 0; + int64_t cols = 0; mio::shared_mmap_sink mmap; std::vector owned; @@ -1183,7 +1187,8 @@ TrxFile
::compute_group_connectivity(ConnectivityMeasure measure, const std:: if (b == this->group_backing_info_.end()) { continue; } - const size_t expected_ids = static_cast(std::max(0, b->second.rows)) * static_cast(std::max(0, b->second.cols)); + const size_t expected_ids = static_cast(std::max(0, b->second.rows)) * + static_cast(std::max(0, b->second.cols)); tmp_ids.resize(expected_ids); if (expected_ids > 0) { std::ifstream in(b->second.filename, std::ios::binary); @@ -1264,7 +1269,7 @@ void allocate_file(const std::string &path, std::size_t size); // Known limitations: only row-major order supported; shape uses tuple (sufficient for 2D); // dtype parameter is used only for byte-size computation. mio::shared_mmap_sink _create_memmap(std::string filename, - const std::tuple &shape, + const std::tuple &shape, const std::string &mode = "r", const std::string &dtype = "float32", long long offset = 0); @@ -1282,7 +1287,7 @@ std::string _generate_filename_from_data(const Eigen::MatrixBase
&arr, const */ template std::unique_ptr> -_initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile
*init_as = nullptr); +_initialize_empty_trx(int64_t nb_streamlines, int64_t nb_vertices, const TrxFile
*init_as = nullptr); template void ediff1d(Eigen::Matrix &lengths, diff --git a/include/trx/trx.tpp b/include/trx/trx.tpp index ac92ef7..8800c89 100644 --- a/include/trx/trx.tpp +++ b/include/trx/trx.tpp @@ -211,7 +211,7 @@ std::unique_ptr> TrxFile
::make_empty_like() const { } template -TrxFile
::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile
*init_as, std::string reference) { +TrxFile
::TrxFile(int64_t nb_vertices, int64_t nb_streamlines, const TrxFile
*init_as, std::string reference) { std::vector> affine(4); std::vector dimensions(3); @@ -275,15 +275,16 @@ TrxFile
::TrxFile(int nb_vertices, int nb_streamlines, const TrxFile
*ini json::object header_obj; header_obj["VOXEL_TO_RASMM"] = affine; header_obj["DIMENSIONS"] = dimensions; - header_obj["NB_VERTICES"] = nb_vertices; - header_obj["NB_STREAMLINES"] = nb_streamlines; + // json11 has no 64-bit integer constructor; store counts as double + header_obj["NB_VERTICES"] = static_cast(nb_vertices); + header_obj["NB_STREAMLINES"] = static_cast(nb_streamlines); this->header = json(header_obj); this->_copy_safe = true; } template -std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile
*init_as) { +std::unique_ptr> _initialize_empty_trx(int64_t nb_streamlines, int64_t nb_vertices, const TrxFile
*init_as) { auto trx = std::make_unique>(); std::string tmp_dir = make_temp_dir("trx"); @@ -292,8 +293,8 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve if (init_as != nullptr) { header = init_as->header; } - header = _json_set(header, "NB_VERTICES", nb_vertices); - header = _json_set(header, "NB_STREAMLINES", nb_streamlines); + header = _json_set(header, "NB_VERTICES", static_cast(nb_vertices)); + header = _json_set(header, "NB_STREAMLINES", static_cast(nb_streamlines)); std::string positions_dtype; std::string offsets_dtype; @@ -313,7 +314,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve std::string positions_filename(tmp_dir); positions_filename += "/positions.3." + positions_dtype; - std::tuple shape = std::make_tuple(nb_vertices, 3); + std::tuple shape = std::make_tuple(nb_vertices, static_cast(3)); trx->streamlines = std::make_unique>(); trx->streamlines->mmap_pos = trx::_create_memmap(positions_filename, shape, "w+", positions_dtype); @@ -323,7 +324,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve std::string offsets_filename(tmp_dir); offsets_filename += "/offsets." + offsets_dtype; - std::tuple shape_off = std::make_tuple(nb_streamlines + 1, 1); + std::tuple shape_off = std::make_tuple(nb_streamlines + 1, static_cast(1)); trx->streamlines->mmap_off = trx::_create_memmap(offsets_filename, shape_off, "w+", offsets_dtype); trx::detail::remap(trx->streamlines->_offsets, trx->streamlines->mmap_off.data(), shape_off); @@ -344,7 +345,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve } for (auto const &x : init_as->data_per_vertex) { - int rows, cols; + int64_t rows, cols; std::string dpv_dtype = dtype_from_scalar
(); Map> tmp_as = init_as->data_per_vertex.find(x.first)->second->_data; @@ -360,20 +361,20 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve dpv_filename = dpv_dirname + x.first + "." + std::to_string(cols) + "." + dpv_dtype; } - std::tuple dpv_shape = std::make_tuple(rows, cols); + std::tuple dpv_shape = std::make_tuple(rows, cols); trx->data_per_vertex[x.first] = std::make_unique>(); trx->data_per_vertex[x.first]->mmap_pos = trx::_create_memmap(dpv_filename, dpv_shape, "w+", dpv_dtype); trx::detail::remap(trx->data_per_vertex[x.first]->_data, trx->data_per_vertex[x.first]->mmap_pos.data(), rows, cols); trx::detail::remap(trx->data_per_vertex[x.first]->_offsets, trx->streamlines->_offsets.data(), - int(trx->streamlines->_offsets.rows()), int(trx->streamlines->_offsets.cols())); + trx->streamlines->_offsets.rows(), trx->streamlines->_offsets.cols()); trx->data_per_vertex[x.first]->_lengths = trx->streamlines->_lengths; } for (auto const &x : init_as->data_per_streamline) { std::string dps_dtype = dtype_from_scalar
(); - int rows, cols; + int64_t rows, cols; Map> tmp_as = init_as->data_per_streamline.find(x.first)->second->_matrix; std::string dps_filename; @@ -388,7 +389,7 @@ std::unique_ptr> _initialize_empty_trx(int nb_streamlines, int nb_ve dps_filename = dps_dirname + x.first + "." + std::to_string(cols) + "." + dps_dtype; } - std::tuple dps_shape = std::make_tuple(rows, cols); + std::tuple dps_shape = std::make_tuple(rows, cols); trx->data_per_streamline[x.first] = std::make_unique>(); trx->data_per_streamline[x.first]->mmap = trx::_create_memmap(dps_filename, dps_shape, std::string("w+"), dps_dtype); @@ -436,7 +437,7 @@ TrxFile
::_create_trx_from_pointer(json header, long long size = std::get<1>(x->second); if (base == "positions" && (folder.empty() || folder == ".")) { - const auto nb_vertices = static_cast(trx->header["NB_VERTICES"].int_value()); + const auto nb_vertices = _json_int64(trx->header["NB_VERTICES"]); const auto expected = nb_vertices * 3; if (size != expected || dim != 3) { throw TrxFormatError("Wrong data size/dimensionality: size=" + std::to_string(size) + @@ -444,7 +445,7 @@ TrxFile
::_create_trx_from_pointer(json header, " filename=" + elem_filename); } - std::tuple shape = std::make_tuple(static_cast(trx->header["NB_VERTICES"].int_value()), 3); + std::tuple shape = std::make_tuple(nb_vertices, static_cast(3)); trx->streamlines->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_adress); @@ -452,8 +453,8 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (base == "offsets" && (folder.empty() || folder == ".")) { - const auto nb_streamlines = static_cast(trx->header["NB_STREAMLINES"].int_value()); - const auto nb_vertices = static_cast(trx->header["NB_VERTICES"].int_value()); + const auto nb_streamlines = _json_int64(trx->header["NB_STREAMLINES"]); + const auto nb_vertices = static_cast(_json_int64(trx->header["NB_VERTICES"])); const auto expected = nb_streamlines + 1; const bool missing_sentinel = (size == nb_streamlines && dim == 1); if ((size != expected && !missing_sentinel) || dim != 1) { @@ -462,17 +463,17 @@ TrxFile
::_create_trx_from_pointer(json header, " filename=" + elem_filename); } - const int nb_str = static_cast(trx->header["NB_STREAMLINES"].int_value()); - const int offsets_rows = missing_sentinel ? (nb_str + 1) : static_cast(size); - std::tuple shape = std::make_tuple(offsets_rows, 1); - trx->streamlines->mmap_off = trx::_create_memmap(filename, std::make_tuple(static_cast(size), 1), "r+", - ext, mem_adress); + const int64_t nb_str = nb_streamlines; + const int64_t offsets_rows = missing_sentinel ? (nb_str + 1) : static_cast(size); + std::tuple shape = std::make_tuple(offsets_rows, static_cast(1)); + trx->streamlines->mmap_off = trx::_create_memmap(filename, std::make_tuple(static_cast(size), static_cast(1)), + "r+", ext, mem_adress); if (ext == "uint64") { if (missing_sentinel) { trx->streamlines->_offsets_owned.resize(static_cast(offsets_rows)); auto *src = reinterpret_cast(trx->streamlines->mmap_off.data()); // NOLINT - for (int i = 0; i < static_cast(size); ++i) { + for (long long i = 0; i < size; ++i) { trx->streamlines->_offsets_owned[static_cast(i)] = src[i]; } trx->streamlines->_offsets_owned.back() = nb_vertices; @@ -483,7 +484,7 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (ext == "uint32") { trx->streamlines->_offsets_owned.resize(static_cast(offsets_rows)); auto *src = reinterpret_cast(trx->streamlines->mmap_off.data()); // NOLINT - for (int i = 0; i < static_cast(size); ++i) { + for (long long i = 0; i < size; ++i) { trx->streamlines->_offsets_owned[static_cast(i)] = static_cast(src[i]); } if (missing_sentinel) { @@ -496,18 +497,19 @@ TrxFile
::_create_trx_from_pointer(json header, Matrix offsets = trx->streamlines->_offsets; trx->streamlines->_lengths = - trx::detail::_compute_lengths(offsets, static_cast(trx->header["NB_VERTICES"].int_value())); + trx::detail::_compute_lengths(offsets, _json_int64(trx->header["NB_VERTICES"])); } else if (folder == "dps") { - std::tuple shape; + std::tuple shape; trx->data_per_streamline[base] = std::make_unique>(); - int nb_scalar = size / static_cast(trx->header["NB_STREAMLINES"].int_value()); + const int64_t nb_streamlines = _json_int64(trx->header["NB_STREAMLINES"]); + const int64_t nb_scalar = nb_streamlines > 0 ? size / nb_streamlines : 0; - if (size % static_cast(trx->header["NB_STREAMLINES"].int_value()) != 0 || nb_scalar != dim) { + if (nb_streamlines == 0 || size % nb_streamlines != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dps size/dimensionality"); } else { - shape = std::make_tuple(static_cast(trx->header["NB_STREAMLINES"].int_value()), nb_scalar); + shape = std::make_tuple(nb_streamlines, nb_scalar); } trx->data_per_streamline[base]->mmap = trx::_create_memmap(filename, shape, "r+", ext, mem_adress); const std::string expected_dtype = dtype_from_scalar
(); @@ -525,14 +527,15 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (folder == "dpv") { - std::tuple shape; + std::tuple shape; trx->data_per_vertex[base] = std::make_unique>(); - int nb_scalar = size / static_cast(trx->header["NB_VERTICES"].int_value()); + const int64_t nb_vertices = _json_int64(trx->header["NB_VERTICES"]); + const int64_t nb_scalar = nb_vertices > 0 ? size / nb_vertices : 0; - if (size % static_cast(trx->header["NB_VERTICES"].int_value()) != 0 || nb_scalar != dim) { + if (nb_vertices == 0 || size % nb_vertices != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dpv size/dimensionality"); } else { - shape = std::make_tuple(static_cast(trx->header["NB_VERTICES"].int_value()), nb_scalar); + shape = std::make_tuple(nb_vertices, nb_scalar); } trx->data_per_vertex[base]->mmap_pos = trx::_create_memmap(filename, shape, "r+", ext, mem_adress); const std::string expected_dtype = dtype_from_scalar
(); @@ -546,18 +549,18 @@ TrxFile
::_create_trx_from_pointer(json header, trx->data_per_vertex[base]->mmap_pos.unmap(); } trx::detail::remap(trx->data_per_vertex[base]->_offsets, trx->streamlines->_offsets.data(), - int(trx->streamlines->_offsets.rows()), int(trx->streamlines->_offsets.cols())); + trx->streamlines->_offsets.rows(), trx->streamlines->_offsets.cols()); trx->data_per_vertex[base]->_lengths = trx->streamlines->_lengths; } else if (folder.rfind("dpg", 0) == 0) { - std::tuple shape; + std::tuple shape; if (size != dim) { throw TrxFormatError("Wrong dpg size/dimensionality"); } else { - shape = std::make_tuple(1, static_cast(size)); + shape = std::make_tuple(static_cast(1), static_cast(size)); } std::string data_name = path_basename(base); @@ -581,11 +584,11 @@ TrxFile
::_create_trx_from_pointer(json header, } else if (folder == "groups") { - std::tuple shape; + std::tuple shape; if (dim != 1) { throw TrxFormatError("Wrong group dimensionality"); } else { - shape = std::make_tuple(static_cast(size), 1); + shape = std::make_tuple(static_cast(size), static_cast(1)); } trx->groups[base] = nullptr; typename TrxFile
::GroupBackingInfo info; @@ -616,15 +619,16 @@ template std::unique_ptr> TrxFile
::deepcopy() { // Determine effective counts (handle sliced/non-copy-safe data) json tmp_header = this->header; - int nb_streamlines, nb_vertices; + int64_t nb_streamlines = 0; + int64_t nb_vertices = 0; if (!this->_copy_safe) { - nb_streamlines = static_cast(this->num_streamlines()); - nb_vertices = static_cast(this->streamlines->_data.size() / 3); - tmp_header = _json_set(tmp_header, "NB_STREAMLINES", nb_streamlines); - tmp_header = _json_set(tmp_header, "NB_VERTICES", nb_vertices); + nb_streamlines = static_cast(this->num_streamlines()); + nb_vertices = static_cast(this->streamlines->_data.size() / 3); + tmp_header = _json_set(tmp_header, "NB_STREAMLINES", static_cast(nb_streamlines)); + tmp_header = _json_set(tmp_header, "NB_VERTICES", static_cast(nb_vertices)); } else { - nb_streamlines = tmp_header["NB_STREAMLINES"].int_value(); - nb_vertices = tmp_header["NB_VERTICES"].int_value(); + nb_streamlines = _json_int64(tmp_header["NB_STREAMLINES"]); + nb_vertices = _json_int64(tmp_header["NB_VERTICES"]); } // Allocate a fresh TrxFile with memory-mapped storage @@ -675,12 +679,12 @@ template std::unique_ptr> TrxFile
::deepcopy() { } for (auto const &kv : this->groups) { std::string group_dtype = dtype_from_scalar(); - int rows = static_cast(kv.second->_matrix.rows()); - int cols = static_cast(kv.second->_matrix.cols()); + const int64_t rows = static_cast(kv.second->_matrix.rows()); + const int64_t cols = static_cast(kv.second->_matrix.cols()); std::string group_filename = groups_dirname + kv.first; group_filename = _generate_filename_from_data(kv.second->_matrix, group_filename); - std::tuple group_shape = std::make_tuple(rows, cols); + std::tuple group_shape = std::make_tuple(rows, cols); copy->groups[kv.first] = std::make_unique>(); copy->groups[kv.first]->mmap = _create_memmap(group_filename, group_shape, "w+", group_dtype); trx::detail::remap(copy->groups[kv.first]->_matrix, copy->groups[kv.first]->mmap.data(), rows, cols); @@ -698,12 +702,12 @@ template std::unique_ptr> TrxFile
::deepcopy() { } for (auto const &field : group_kv.second) { std::string dpg_dtype = dtype_from_scalar
(); - int rows = static_cast(field.second->_matrix.rows()); - int cols = static_cast(field.second->_matrix.cols()); + const int64_t rows = static_cast(field.second->_matrix.rows()); + const int64_t cols = static_cast(field.second->_matrix.cols()); std::string dpg_filename = dpg_subdirname + SEPARATOR + field.first; dpg_filename = _generate_filename_from_data(field.second->_matrix, dpg_filename); - std::tuple dpg_shape = std::make_tuple(rows, cols); + std::tuple dpg_shape = std::make_tuple(rows, cols); copy->data_per_group[group_kv.first][field.first] = std::make_unique>(); copy->data_per_group[group_kv.first][field.first]->mmap = _create_memmap(dpg_filename, dpg_shape, "w+", dpg_dtype); @@ -718,34 +722,36 @@ template std::unique_ptr> TrxFile
::deepcopy() { /// Compute the used range in a preallocated TrxFile by finding the last non-zero length. /// Returns (nb_streamlines_used, nb_vertices_used). -template std::tuple TrxFile
::_get_real_len() { +template std::tuple TrxFile
::_get_real_len() { if (this->streamlines->_lengths.size() == 0) - return std::make_tuple(0, 0); + return std::make_tuple(static_cast(0), static_cast(0)); - int last_elem_pos = trx::detail::_dichotomic_search(this->streamlines->_lengths); + int64_t last_elem_pos = trx::detail::_dichotomic_search(this->streamlines->_lengths); if (last_elem_pos != -1) { - int strs_end = last_elem_pos + 1; - int pts_end = this->streamlines->_lengths(Eigen::seq(0, last_elem_pos), 0).sum(); + const int64_t strs_end = last_elem_pos + 1; + const int64_t pts_end = + this->streamlines->_lengths(Eigen::seq(0, last_elem_pos), 0).template cast().sum(); return std::make_tuple(strs_end, pts_end); } - return std::make_tuple(0, 0); + return std::make_tuple(static_cast(0), static_cast(0)); } template -std::tuple -TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int strs_start, int pts_start, int nb_strs_to_copy) { - int curr_strs_len, curr_pts_len; +std::tuple +TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int64_t strs_start, int64_t pts_start, int64_t nb_strs_to_copy) { + int64_t curr_strs_len = 0; + int64_t curr_pts_len = 0; if (nb_strs_to_copy == -1) { - std::tuple curr = this->_get_real_len(); + std::tuple curr = this->_get_real_len(); curr_strs_len = std::get<0>(curr); curr_pts_len = std::get<1>(curr); } else { curr_strs_len = nb_strs_to_copy; - curr_pts_len = trx->streamlines->_lengths(Eigen::seq(0, curr_strs_len - 1)).sum(); + curr_pts_len = trx->streamlines->_lengths(Eigen::seq(0, curr_strs_len - 1)).template cast().sum(); } if (pts_start == -1) { @@ -755,8 +761,8 @@ TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int strs_start, int pts_s strs_start = 0; } - int strs_end = strs_start + curr_strs_len; - int pts_end = pts_start + curr_pts_len; + const int64_t strs_end = strs_start + curr_strs_len; + const int64_t pts_end = pts_start + curr_pts_len; if (curr_pts_len == 0) return std::make_tuple(strs_start, pts_start); @@ -775,8 +781,8 @@ TrxFile
::_copy_fixed_arrays_from(TrxFile
*trx, int strs_start, int pts_s pts_start, 0, curr_pts_len, this->data_per_vertex[x.first]->_data.cols()) = trx->data_per_vertex[x.first]->_data.block(0, 0, curr_pts_len, trx->data_per_vertex[x.first]->_data.cols()); trx::detail::remap(this->data_per_vertex[x.first]->_offsets, trx->data_per_vertex[x.first]->_offsets.data(), - static_cast(trx->data_per_vertex[x.first]->_offsets.rows()), - static_cast(trx->data_per_vertex[x.first]->_offsets.cols())); + trx->data_per_vertex[x.first]->_offsets.rows(), + trx->data_per_vertex[x.first]->_offsets.cols()); this->data_per_vertex[x.first]->_lengths = trx->data_per_vertex[x.first]->_lengths; } @@ -841,21 +847,21 @@ void TrxFile
::_cleanup_temporary_directory() { template // Caveats: downsizing vertices is not supported; reducing streamlines truncates data; same-size // resize is a no-op. -void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { +void TrxFile
::resize(int64_t nb_streamlines, int64_t nb_vertices, bool delete_dpg) { if (!this->_copy_safe) { throw TrxArgumentError("Cannot resize a sliced dataset."); } - std::tuple sp_end = this->_get_real_len(); - int strs_end = std::get<0>(sp_end); - int ptrs_end = std::get<1>(sp_end); + std::tuple sp_end = this->_get_real_len(); + int64_t strs_end = std::get<0>(sp_end); + int64_t ptrs_end = std::get<1>(sp_end); if (nb_streamlines != -1 && nb_streamlines < strs_end) { strs_end = nb_streamlines; } if (nb_vertices == -1) { - ptrs_end = this->streamlines->_lengths.sum(); + ptrs_end = this->streamlines->_lengths.template cast().sum(); nb_vertices = ptrs_end; } else if (nb_vertices < ptrs_end) { return; @@ -865,14 +871,14 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { nb_streamlines = strs_end; } - if (nb_streamlines == this->header["NB_STREAMLINES"].int_value() && - nb_vertices == this->header["NB_VERTICES"].int_value()) { + if (nb_streamlines == _json_int64(this->header["NB_STREAMLINES"]) && + nb_vertices == _json_int64(this->header["NB_VERTICES"])) { return; } auto trx = _initialize_empty_trx(nb_streamlines, nb_vertices, this); - if (nb_streamlines < this->header["NB_STREAMLINES"].int_value()) + if (nb_streamlines < _json_int64(this->header["NB_STREAMLINES"])) trx->_copy_fixed_arrays_from(this, -1, -1, nb_streamlines); else { trx->_copy_fixed_arrays_from(this); @@ -889,15 +895,16 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { std::string group_dtype = dtype_from_scalar(); std::string group_name = group_dir + x.first + "." + group_dtype; - int ori_length = this->groups[x.first]->_matrix.size(); + const Eigen::Index ori_length = this->groups[x.first]->_matrix.size(); + static_cast(ori_length); - std::vector keep_rows; - std::vector keep_cols = {0}; + std::vector keep_rows; + std::vector keep_cols = {0}; // Slicing - for (int i = 0; i < x.second->_matrix.rows(); ++i) { - for (int j = 0; j < x.second->_matrix.cols(); ++j) { - if (static_cast(x.second->_matrix(i, j)) < strs_end) { + for (Eigen::Index i = 0; i < x.second->_matrix.rows(); ++i) { + for (Eigen::Index j = 0; j < x.second->_matrix.cols(); ++j) { + if (static_cast(x.second->_matrix(i, j)) < strs_end) { keep_rows.push_back(i); } } @@ -905,15 +912,15 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { // std::cout << "Cols " << keep_rows.at(1) << std::endl; Matrix tmp = this->groups[x.first]->_matrix(keep_rows, keep_cols); - std::tuple group_shape = std::make_tuple(tmp.size(), 1); + std::tuple group_shape = std::make_tuple(static_cast(tmp.size()), static_cast(1)); trx->groups[x.first] = std::make_unique>(); trx->groups[x.first]->mmap = trx::_create_memmap(group_name, group_shape, "w+", group_dtype); trx::detail::remap(trx->groups[x.first]->_matrix, trx->groups[x.first]->mmap.data(), group_shape); // update values - for (int i = 0; i < trx->groups[x.first]->_matrix.rows(); ++i) { - for (int j = 0; j < trx->groups[x.first]->_matrix.cols(); ++j) { + for (Eigen::Index i = 0; i < trx->groups[x.first]->_matrix.rows(); ++i) { + for (Eigen::Index j = 0; j < trx->groups[x.first]->_matrix.cols(); ++j) { trx->groups[x.first]->_matrix(i, j) = tmp(i, j); } } @@ -944,8 +951,9 @@ void TrxFile
::resize(int nb_streamlines, int nb_vertices, bool delete_dpg) { std::string dpg_filename = dpg_subdir + SEPARATOR + y.first; dpg_filename = _generate_filename_from_data(this->data_per_group[x.first][y.first]->_matrix, dpg_filename); - std::tuple dpg_shape = std::make_tuple(this->data_per_group[x.first][y.first]->_matrix.rows(), - this->data_per_group[x.first][y.first]->_matrix.cols()); + std::tuple dpg_shape = + std::make_tuple(static_cast(this->data_per_group[x.first][y.first]->_matrix.rows()), + static_cast(this->data_per_group[x.first][y.first]->_matrix.cols())); if (trx->data_per_group[x.first].find(y.first) == trx->data_per_group[x.first].end()) { trx->data_per_group[x.first][y.first] = std::make_unique>(); @@ -1164,13 +1172,8 @@ template void TrxFile
::normalize_for_save() { if (used_vertices > data_rows) { throw TrxFormatError("TRX offsets exceed positions row count"); } - if (used_vertices > static_cast(std::numeric_limits::max()) || - used_streamlines > static_cast(std::numeric_limits::max())) { - throw TrxFormatError("TRX normalize_for_save exceeds supported int range"); - } - if (used_streamlines < total_streamlines || used_vertices < data_rows) { - this->resize(static_cast(used_streamlines), static_cast(used_vertices)); + this->resize(static_cast(used_streamlines), static_cast(used_vertices)); } const size_t normalized_streamlines = this->num_streamlines(); @@ -1187,8 +1190,8 @@ template void TrxFile
::normalize_for_save() { this->streamlines->_lengths(static_cast(i)) = static_cast(diff); } - this->header = _json_set(this->header, "NB_STREAMLINES", static_cast(normalized_streamlines)); - this->header = _json_set(this->header, "NB_VERTICES", static_cast(this->num_vertices())); + this->header = _json_set(this->header, "NB_STREAMLINES", static_cast(normalized_streamlines)); + this->header = _json_set(this->header, "NB_VERTICES", static_cast(this->num_vertices())); } template void TrxFile
::save(const std::string &filename, const TrxSaveOptions &options) { @@ -1204,13 +1207,13 @@ template void TrxFile
::save(const std::string &filename, const throw TrxFormatError("Cannot save TRX without offsets data"); } if (save_trx->header["NB_STREAMLINES"].is_number()) { - const auto nb_streamlines = static_cast(save_trx->header["NB_STREAMLINES"].int_value()); + const auto nb_streamlines = static_cast(_json_int64(save_trx->header["NB_STREAMLINES"])); if (save_trx->streamlines->_offsets.size() != static_cast(nb_streamlines + 1)) { throw TrxFormatError("TRX offsets size does not match NB_STREAMLINES"); } } if (save_trx->header["NB_VERTICES"].is_number()) { - const auto nb_vertices = static_cast(save_trx->header["NB_VERTICES"].int_value()); + const auto nb_vertices = static_cast(_json_int64(save_trx->header["NB_VERTICES"])); const auto last = static_cast(save_trx->num_vertices()); if (last != nb_vertices) { throw TrxFormatError("TRX offsets sentinel does not match NB_VERTICES"); @@ -1389,7 +1392,7 @@ void TrxFile
::add_dps_from_vector(const std::string &name, const std::string if (this->streamlines) { nb_streamlines = static_cast(this->streamlines->_lengths.size()); } else if (this->header["NB_STREAMLINES"].is_number()) { - nb_streamlines = static_cast(this->header["NB_STREAMLINES"].int_value()); + nb_streamlines = static_cast(_json_int64(this->header["NB_STREAMLINES"])); } if (values.size() != nb_streamlines) { @@ -1413,10 +1416,10 @@ void TrxFile
::add_dps_from_vector(const std::string &name, const std::string this->data_per_streamline.erase(existing); } - const int rows = static_cast(nb_streamlines); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); - const size_t n = static_cast(rows * cols); + const int64_t rows = static_cast(nb_streamlines); + const int64_t cols = 1; + std::tuple shape = std::make_tuple(rows, cols); + const size_t n = static_cast(rows) * static_cast(cols); auto matrix = std::make_unique>(); matrix->mmap = trx::_create_memmap(dps_filename, shape, "w+", dtype_norm); @@ -1477,7 +1480,7 @@ void TrxFile
::add_dpv_from_vector(const std::string &name, const std::string if (this->streamlines) { nb_vertices = static_cast(this->streamlines->_data.rows()); } else if (this->header["NB_VERTICES"].is_number()) { - nb_vertices = static_cast(this->header["NB_VERTICES"].int_value()); + nb_vertices = static_cast(_json_int64(this->header["NB_VERTICES"])); } if (values.size() != nb_vertices) { @@ -1501,10 +1504,10 @@ void TrxFile
::add_dpv_from_vector(const std::string &name, const std::string this->data_per_vertex.erase(existing); } - const int rows = static_cast(nb_vertices); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); - const size_t n = static_cast(rows * cols); + const int64_t rows = static_cast(nb_vertices); + const int64_t cols = 1; + std::tuple shape = std::make_tuple(rows, cols); + const size_t n = static_cast(rows) * static_cast(cols); auto seq = std::make_unique>(); seq->mmap_pos = trx::_create_memmap(dpv_filename, shape, "w+", dtype_norm); @@ -1557,7 +1560,7 @@ void TrxFile
::add_group_from_indices(const std::string &name, const std::vec if (this->streamlines) { nb_streamlines = static_cast(this->streamlines->_lengths.size()); } else if (this->header["NB_STREAMLINES"].is_number()) { - nb_streamlines = static_cast(this->header["NB_STREAMLINES"].int_value()); + nb_streamlines = static_cast(_json_int64(this->header["NB_STREAMLINES"])); } for (const auto idx : indices) { @@ -1586,14 +1589,14 @@ void TrxFile
::add_group_from_indices(const std::string &name, const std::vec this->group_backing_info_.erase(backing); } - const int rows = static_cast(indices.size()); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); + const int64_t rows = static_cast(indices.size()); + const int64_t cols = 1; + std::tuple shape = std::make_tuple(rows, cols); auto group = std::make_unique>(); group->mmap = trx::_create_memmap(group_filename, shape, "w+", "uint32"); trx::detail::remap(group->_matrix, group->mmap.data(), shape); - for (int i = 0; i < rows; ++i) { + for (int64_t i = 0; i < rows; ++i) { group->_matrix(i, 0) = indices[static_cast(i)]; } this->groups[name] = std::move(group); @@ -2003,11 +2006,11 @@ template void TrxStream::finalize(const std::string &filename, Trx const size_t nb_streamlines = lengths_.size(); const size_t nb_vertices = total_vertices_; - TrxFile
trx(static_cast(nb_vertices), static_cast(nb_streamlines)); + TrxFile
trx(static_cast(nb_vertices), static_cast(nb_streamlines)); json header_out = header; - header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); - header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); + header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); + header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); trx.header = header_out; auto &positions = trx.streamlines->_data; @@ -2145,8 +2148,8 @@ inline void TrxStream::finalize_directory_impl(const std::string &directory, boo ec.clear(); json header_out = header; - header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); - header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); + header_out = _json_set(header_out, "NB_VERTICES", static_cast(nb_vertices)); + header_out = _json_set(header_out, "NB_STREAMLINES", static_cast(nb_streamlines)); const std::string header_path = directory + SEPARATOR + "header.json"; std::ofstream out_header(header_path, std::ios::out | std::ios::trunc); if (!out_header.is_open()) { @@ -2523,21 +2526,21 @@ void TrxFile
::add_dpv_from_tsf(const std::string &name, const std::string &d this->data_per_vertex.erase(existing); } - const int rows = static_cast(nb_vertices); - const int cols = 1; - std::tuple shape = std::make_tuple(rows, cols); + const int64_t rows = static_cast(nb_vertices); + const int64_t cols = 1; + std::tuple shape = std::make_tuple(rows, cols); auto seq = std::make_unique>(); seq->mmap_pos = trx::_create_memmap(dpv_filename, shape, "w+", dtype_norm); trx::detail::remap(seq->_data, seq->mmap_pos.data(), rows, cols); - for (int i = 0; i < rows; ++i) { + for (int64_t i = 0; i < rows; ++i) { seq->_data(i, 0) = static_cast
(values[static_cast(i)]); } trx::detail::remap(seq->_offsets, this->streamlines->_offsets.data(), - static_cast(this->streamlines->_offsets.rows()), - static_cast(this->streamlines->_offsets.cols())); + this->streamlines->_offsets.rows(), + this->streamlines->_offsets.cols()); seq->_lengths = this->streamlines->_lengths; this->data_per_vertex[name] = std::move(seq); @@ -2863,11 +2866,11 @@ const MMappedMatrix *TrxFile
::get_group_members(const std::string if (b == this->group_backing_info_.end()) { return nullptr; } - const int rows = b->second.rows; - const int cols = b->second.cols; - std::tuple shape = std::make_tuple(rows, cols); + const int64_t rows = b->second.rows; + const int64_t cols = b->second.cols; + std::tuple shape = std::make_tuple(rows, cols); it->second = std::make_unique>(); - const size_t n = static_cast(std::max(0, rows)) * static_cast(std::max(0, cols)); + const size_t n = static_cast(std::max(0, rows)) * static_cast(std::max(0, cols)); it->second->_matrix_owned.resize(n); if (n > 0) { @@ -3009,7 +3012,7 @@ void TrxFile
::add_dpg_from_vector(const std::string &group, auto &group_map = this->data_per_group[group]; group_map.erase(name); - std::tuple shape = std::make_tuple(rows, cols); + std::tuple shape = std::make_tuple(static_cast(rows), static_cast(cols)); group_map[name] = std::make_unique>(); group_map[name]->mmap = _create_memmap(dpg_filename, shape, "w+", dtype_norm); @@ -3139,21 +3142,21 @@ std::unique_ptr> TrxFile
::subset_streamlines(const std::vectormake_empty_like(); } - std::vector old_to_new(nb_streamlines, -1); + std::vector old_to_new(nb_streamlines, -1); size_t total_vertices = 0; for (size_t i = 0; i < selected.size(); ++i) { const uint32_t idx = selected[i]; - old_to_new[idx] = static_cast(i); + old_to_new[idx] = static_cast(i); const uint64_t start = offsets[idx]; const uint64_t end = offsets[idx + 1]; total_vertices += static_cast(end - start); } - auto out = std::make_unique>(static_cast(total_vertices), - static_cast(selected.size()), + auto out = std::make_unique>(static_cast(total_vertices), + static_cast(selected.size()), this); - out->header = _json_set(this->header, "NB_VERTICES", static_cast(total_vertices)); - out->header = _json_set(out->header, "NB_STREAMLINES", static_cast(selected.size())); + out->header = _json_set(this->header, "NB_VERTICES", static_cast(total_vertices)); + out->header = _json_set(out->header, "NB_STREAMLINES", static_cast(selected.size())); auto &out_positions = out->streamlines->_data; auto &out_offsets = out->streamlines->_offsets; @@ -3258,8 +3261,8 @@ std::unique_ptr> TrxFile
::subset_streamlines(const std::vector_matrix, dpg_filename); - std::tuple dpg_shape = std::make_tuple(field_kv.second->_matrix.rows(), - field_kv.second->_matrix.cols()); + std::tuple dpg_shape = std::make_tuple(static_cast(field_kv.second->_matrix.rows()), + static_cast(field_kv.second->_matrix.cols())); out->data_per_group[group_name][field_name] = std::make_unique>(); out->data_per_group[group_name][field_name]->mmap = diff --git a/src/trx.cpp b/src/trx.cpp index d53d207..af29eaa 100644 --- a/src/trx.cpp +++ b/src/trx.cpp @@ -178,7 +178,7 @@ std::array read_xyz_as_double(const TypedArray &positions, size_t row throw TrxDTypeError("Unsupported positions dtype for streamline extraction: " + positions.dtype); } -TypedArray make_typed_array(const std::string &filename, int rows, int cols, const std::string &dtype) { +TypedArray make_typed_array(const std::string &filename, int64_t rows, int64_t cols, const std::string &dtype) { TypedArray array; array.dtype = dtype; array.rows = rows; @@ -277,7 +277,7 @@ size_t AnyTrxFile::num_vertices() const { return static_cast(positions.rows); } if (header["NB_VERTICES"].is_number()) { - return static_cast(header["NB_VERTICES"].int_value()); + return static_cast(_json_int64(header["NB_VERTICES"])); } return 0; } @@ -287,7 +287,7 @@ size_t AnyTrxFile::num_streamlines() const { return lengths.size(); } if (header["NB_STREAMLINES"].is_number()) { - return static_cast(header["NB_STREAMLINES"].int_value()); + return static_cast(_json_int64(header["NB_STREAMLINES"])); } return 0; } @@ -453,8 +453,8 @@ AnyTrxFile::_create_from_pointer(json header, throw TrxFormatError("Missing NB_VERTICES or NB_STREAMLINES in header.json"); } - const int nb_vertices = header["NB_VERTICES"].int_value(); - const int nb_streamlines = header["NB_STREAMLINES"].int_value(); + const int64_t nb_vertices = _json_int64(header["NB_VERTICES"]); + const int64_t nb_streamlines = _json_int64(header["NB_STREAMLINES"]); for (auto x = dict_pointer_size.rbegin(); x != dict_pointer_size.rend(); ++x) { const std::string elem_filename = x->first; @@ -484,7 +484,7 @@ AnyTrxFile::_create_from_pointer(json header, } trx.offsets = make_typed_array(elem_filename, nb_streamlines + 1, 1, ext); } else if (folder == "dps") { - const int nb_scalar = nb_streamlines > 0 ? static_cast(size / nb_streamlines) : 0; + const int64_t nb_scalar = nb_streamlines > 0 ? size / nb_streamlines : 0; if (nb_streamlines == 0 || size % nb_streamlines != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dps size/dimensionality"); } @@ -492,7 +492,7 @@ AnyTrxFile::_create_from_pointer(json header, arr.materialize_to_owned(); trx.data_per_streamline.emplace(base, std::move(arr)); } else if (folder == "dpv") { - const int nb_scalar = nb_vertices > 0 ? static_cast(size / nb_vertices) : 0; + const int64_t nb_scalar = nb_vertices > 0 ? size / nb_vertices : 0; if (nb_vertices == 0 || size % nb_vertices != 0 || nb_scalar != dim) { throw TrxFormatError("Wrong dpv size/dimensionality"); } @@ -505,7 +505,7 @@ AnyTrxFile::_create_from_pointer(json header, } std::string data_name = path_basename(base); std::string sub_folder = path_basename(folder); - auto arr = make_typed_array(elem_filename, 1, static_cast(size), ext); + auto arr = make_typed_array(elem_filename, 1, static_cast(size), ext); arr.materialize_to_owned(); trx.data_per_group[sub_folder].emplace(data_name, std::move(arr)); } else if (folder == "groups") { @@ -515,7 +515,7 @@ AnyTrxFile::_create_from_pointer(json header, if (ext != "uint32") { throw TrxDTypeError("Unsupported group dtype: " + ext); } - auto arr = make_typed_array(elem_filename, static_cast(size), 1, ext); + auto arr = make_typed_array(elem_filename, static_cast(size), 1, ext); arr.materialize_to_owned(); trx.groups.emplace(base, std::move(arr)); } else { @@ -643,13 +643,13 @@ void AnyTrxFile::save(const std::string &filename, const TrxSaveOptions &options throw TrxFormatError("Cannot save TRX without decoded offsets"); } if (header["NB_STREAMLINES"].is_number()) { - const auto nb_streamlines = static_cast(header["NB_STREAMLINES"].int_value()); + const auto nb_streamlines = static_cast(_json_int64(header["NB_STREAMLINES"])); if (offsets_u64.size() != nb_streamlines + 1) { throw TrxFormatError("TRX offsets size does not match NB_STREAMLINES"); } } if (header["NB_VERTICES"].is_number()) { - const auto nb_vertices = static_cast(header["NB_VERTICES"].int_value()); + const auto nb_vertices = static_cast(_json_int64(header["NB_VERTICES"])); const auto last = offsets_u64.back(); if (last != nb_vertices) { throw TrxFormatError("TRX offsets sentinel does not match NB_VERTICES"); @@ -906,7 +906,7 @@ void allocate_file(const std::string &path, std::size_t size) { } mio::shared_mmap_sink _create_memmap(std::string filename, - const std::tuple &shape, + const std::tuple &shape, const std::string &mode, const std::string &dtype, long long offset) { @@ -1653,8 +1653,8 @@ void merge_trx_shards(const MergeTrxShardsOptions &options) { ensure_schema_match("groups", groups_schema, shard_dir); const json shard_header = read_header(shard_dir); - const uint64_t shard_vertices = static_cast(shard_header["NB_VERTICES"].int_value()); - const uint64_t shard_streamlines = static_cast(shard_header["NB_STREAMLINES"].int_value()); + const uint64_t shard_vertices = static_cast(_json_int64(shard_header["NB_VERTICES"])); + const uint64_t shard_streamlines = static_cast(_json_int64(shard_header["NB_STREAMLINES"])); const std::string shard_positions = find_file_with_prefix(shard_dir, "positions."); const std::string shard_offsets = find_file_with_prefix(shard_dir, "offsets."); @@ -1691,8 +1691,8 @@ void merge_trx_shards(const MergeTrxShardsOptions &options) { total_streamlines += shard_streamlines; } - merged_header = _json_set(merged_header, "NB_VERTICES", static_cast(total_vertices)); - merged_header = _json_set(merged_header, "NB_STREAMLINES", static_cast(total_streamlines)); + merged_header = _json_set(merged_header, "NB_VERTICES", static_cast(total_vertices)); + merged_header = _json_set(merged_header, "NB_STREAMLINES", static_cast(total_streamlines)); { const std::string merged_header_path = output_dir + SEPARATOR + "header.json"; std::ofstream out(merged_header_path, std::ios::out | std::ios::trunc);