Skip to content

Support larger TRX files #55

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
36000 wants to merge 1 commit into from
Closed

Support larger TRX files #55

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions include/trx/detail/dtype_helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@

#include <Eigen/Core>

#include <cstdint>

#include <new>
#include <string>
#include <tuple>
Expand All @@ -16,25 +18,25 @@ namespace detail {
//
// MapType must be an Eigen::Map<Matrix<...>> type.
template <typename MapType>
inline void remap(MapType &map, void *data, int rows, int cols) {
inline void remap(MapType &map, void *data, std::int64_t rows, std::int64_t cols) {
using Scalar = typename MapType::Scalar;
new (&map) MapType(reinterpret_cast<Scalar *>(data), rows, cols); // NOLINT
}

// Overload for const data pointers (read-only maps).
template <typename MapType>
inline void remap(MapType &map, const void *data, int rows, int cols) {
inline void remap(MapType &map, const void *data, std::int64_t rows, std::int64_t cols) {
using Scalar = typename MapType::Scalar;
new (&map) MapType(const_cast<Scalar *>(reinterpret_cast<const Scalar *>(data)), rows, cols); // NOLINT
}

// Convenience overloads that unpack a (rows, cols) shape tuple.
template <typename MapType>
inline void remap(MapType &map, void *data, const std::tuple<int, int> &shape) {
inline void remap(MapType &map, void *data, const std::tuple<std::int64_t, std::int64_t> &shape) {
remap(map, data, std::get<0>(shape), std::get<1>(shape));
}
template <typename MapType>
inline void remap(MapType &map, const void *data, const std::tuple<int, int> &shape) {
inline void remap(MapType &map, const void *data, const std::tuple<std::int64_t, std::int64_t> &shape) {
remap(map, data, std::get<0>(shape), std::get<1>(shape));
}

Expand All @@ -45,7 +47,7 @@ std::tuple<std::string, int, std::string> _split_ext_with_dimensionality(const s

template <typename DT>
inline Eigen::Matrix<uint32_t, Eigen::Dynamic, 1> _compute_lengths(const Eigen::MatrixBase<DT> &offsets,
int nb_vertices) {
std::int64_t nb_vertices) {
static_cast<void>(nb_vertices);
if (offsets.size() > 1) {
const auto casted = offsets.template cast<uint64_t>();
Expand Down
35 changes: 20 additions & 15 deletions include/trx/trx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ inline json::object _json_object(const json &value) {
return json::object();
}

inline int64_t _json_int64(const json &value) {
return static_cast<int64_t>(value.number_value());
}
Comment on lines +82 to +84

inline json _json_set(const json &value, const std::string &key, const json &field) {
auto obj = _json_object(value);
obj[key] = field;
Expand Down Expand Up @@ -242,8 +246,8 @@ template <typename DT> class TrxFile {
public:
struct GroupBackingInfo {
std::string filename;
int rows = 0;
int cols = 0;
int64_t rows = 0;
int64_t cols = 0;
std::string dtype;
long long mem_offset = 0;
};
Expand All @@ -264,8 +268,8 @@ template <typename DT> class TrxFile {

// Member Functions()
// TrxFile(int nb_vertices = 0, int nb_streamlines = 0);
TrxFile(int nb_vertices = 0,
int nb_streamlines = 0,
TrxFile(int64_t nb_vertices = 0,
int64_t nb_streamlines = 0,
const TrxFile<DT> *init_as = nullptr,
std::string reference = "");
~TrxFile();
Expand Down Expand Up @@ -303,7 +307,7 @@ template <typename DT> class TrxFile {
* @param nb_vertices The number of vertices to keep
* @param delete_dpg Remove data_per_group when resizing
*/
void resize(int nb_streamlines = -1, int nb_vertices = -1, bool delete_dpg = false);
void resize(int64_t nb_streamlines = -1, int64_t nb_vertices = -1, bool delete_dpg = false);

/**
* @brief Save a TrxFile
Expand Down Expand Up @@ -368,7 +372,7 @@ template <typename DT> class TrxFile {
return static_cast<size_t>(streamlines->_data.rows());
}
if (header["NB_VERTICES"].is_number()) {
return static_cast<size_t>(header["NB_VERTICES"].int_value());
return static_cast<size_t>(_json_int64(header["NB_VERTICES"]));
}
return 0;
}
Expand All @@ -381,7 +385,7 @@ template <typename DT> class TrxFile {
return static_cast<size_t>(streamlines->_lengths.size());
}
if (header["NB_STREAMLINES"].is_number()) {
return static_cast<size_t>(header["NB_STREAMLINES"].int_value());
return static_cast<size_t>(_json_int64(header["NB_STREAMLINES"]));
}
return 0;
}
Expand Down Expand Up @@ -519,8 +523,8 @@ template <typename DT> class TrxFile {
* @return std::tuple<int, int> A tuple representing the end of the copied streamlines and end
* of copied points
*/
std::tuple<int, int>
_copy_fixed_arrays_from(TrxFile<DT> *trx, int strs_start = 0, int pts_start = 0, int nb_strs_to_copy = -1);
std::tuple<int64_t, int64_t>
_copy_fixed_arrays_from(TrxFile<DT> *trx, int64_t strs_start = 0, int64_t pts_start = 0, int64_t nb_strs_to_copy = -1);
int len();

private:
Expand All @@ -533,7 +537,7 @@ template <typename DT> class TrxFile {
* @return std::tuple<int, int> A tuple representing the index of the last streamline and the
* total length of all the streamlines
*/
std::tuple<int, int> _get_real_len();
std::tuple<int64_t, int64_t> _get_real_len();
};

namespace detail {
Expand Down Expand Up @@ -568,8 +572,8 @@ inline std::string make_unique_temp_path(const std::string &prefix) {

struct TypedArray {
std::string dtype;
int rows = 0;
int cols = 0;
int64_t rows = 0;
int64_t cols = 0;
mio::shared_mmap_sink mmap;
std::vector<std::uint8_t> owned;

Expand Down Expand Up @@ -1183,7 +1187,8 @@ TrxFile<DT>::compute_group_connectivity(ConnectivityMeasure measure, const std::
if (b == this->group_backing_info_.end()) {
continue;
}
const size_t expected_ids = static_cast<size_t>(std::max(0, b->second.rows)) * static_cast<size_t>(std::max(0, b->second.cols));
const size_t expected_ids = static_cast<size_t>(std::max<int64_t>(0, b->second.rows)) *
static_cast<size_t>(std::max<int64_t>(0, b->second.cols));
tmp_ids.resize(expected_ids);
if (expected_ids > 0) {
std::ifstream in(b->second.filename, std::ios::binary);
Expand Down Expand Up @@ -1264,7 +1269,7 @@ void allocate_file(const std::string &path, std::size_t size);
// Known limitations: only row-major order supported; shape uses tuple (sufficient for 2D);
// dtype parameter is used only for byte-size computation.
mio::shared_mmap_sink _create_memmap(std::string filename,
const std::tuple<int, int> &shape,
const std::tuple<int64_t, int64_t> &shape,
const std::string &mode = "r",
const std::string &dtype = "float32",
long long offset = 0);
Expand All @@ -1282,7 +1287,7 @@ std::string _generate_filename_from_data(const Eigen::MatrixBase<DT> &arr, const
*/
template <typename DT>
std::unique_ptr<TrxFile<DT>>
_initialize_empty_trx(int nb_streamlines, int nb_vertices, const TrxFile<DT> *init_as = nullptr);
_initialize_empty_trx(int64_t nb_streamlines, int64_t nb_vertices, const TrxFile<DT> *init_as = nullptr);

template <typename DT>
void ediff1d(Eigen::Matrix<DT, Eigen::Dynamic, 1> &lengths,
Expand Down
Loading
Loading