Performance improvements when serializing to rds

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changelog
 
-## Version 0.10.0 - 0.10.2
+## Version 0.10.0 - 0.10.3
 
 - Added methods to write to RDS/RData files.
 - Supports atomic types, generic dictionaries/lists, and **BiocPy objects**.

lib/src/rdswrapper.cpp

@@ -289,12 +289,12 @@ std::unique_ptr<rds2cpp::RObject> py_to_robject(const py::object& obj, std::vect
         // bool arrays
         if (dtype.is(py::dtype::of<bool>())) {
             auto buf = arr.cast<py::array_t<bool, py::array::c_style | py::array::forcecast>>();
-            auto r = buf.unchecked<1>();
             auto vec = std::make_unique<rds2cpp::LogicalVector>();
 
-            vec->data.reserve(r.shape(0));
-            for (ssize_t i = 0; i < r.shape(0); ++i) {
-                vec->data.push_back(r(i) ? 1 : 0);
+            vec->data.reserve(buf.size());
+            const bool* ptr = buf.data();
+            for (size_t i = 0; i < buf.size(); ++i) {
+                vec->data.push_back(ptr[i] ? 1 : 0);
             }
 
             return vec;
@@ -306,28 +306,17 @@ std::unique_ptr<rds2cpp::RObject> py_to_robject(const py::object& obj, std::vect
             py::isinstance<py::array_t<int16_t>>(arr) ||
             py::isinstance<py::array_t<int8_t>>(arr)) {
             auto buf = arr.cast<py::array_t<int32_t, py::array::c_style | py::array::forcecast>>();
-            auto r = buf.unchecked<1>();
             auto vec = std::make_unique<rds2cpp::IntegerVector>();
-
-            vec->data.reserve(r.shape(0));
-            for (ssize_t i = 0; i < r.shape(0); ++i) {
-                vec->data.push_back(r(i));
-            }
-
+            vec->data.assign(buf.data(), buf.data() + buf.size());
             return vec;
         }
 
         // float arrays
         if (py::isinstance<py::array_t<double>>(arr) ||
             py::isinstance<py::array_t<float>>(arr)) {
             auto buf = arr.cast<py::array_t<double, py::array::c_style | py::array::forcecast>>();
-            auto r = buf.unchecked<1>();
             auto vec = std::make_unique<rds2cpp::DoubleVector>();
-
-            vec->data.reserve(r.shape(0));
-            for (ssize_t i = 0; i < r.shape(0); ++i) {
-                vec->data.push_back(r(i));
-            }
+            vec->data.assign(buf.data(), buf.data() + buf.size());
             return vec;
         }
 
@@ -376,9 +365,7 @@ std::unique_ptr<rds2cpp::RObject> py_to_robject(const py::object& obj, std::vect
                     auto data_obj = d["data"];
                     if (py::isinstance<py::array>(data_obj)) {
                         auto arr = data_obj.cast<py::array_t<int32_t, py::array::c_style | py::array::forcecast>>();
-                        auto r = arr.unchecked<1>();
-                        vec->data.reserve(r.shape(0));
-                        for (ssize_t i = 0; i < r.shape(0); ++i) vec->data.push_back(r(i));
+                        vec->data.assign(arr.data(), arr.data() + arr.size());
                     } else {
                         auto seq = data_obj.cast<py::sequence>();
                         vec->data.reserve(py::len(seq));
@@ -409,9 +396,7 @@ std::unique_ptr<rds2cpp::RObject> py_to_robject(const py::object& obj, std::vect
                     auto data_obj = d["data"];
                     if (py::isinstance<py::array>(data_obj)) {
                         auto arr = data_obj.cast<py::array_t<double, py::array::c_style | py::array::forcecast>>();
-                        auto r = arr.unchecked<1>();
-                        vec->data.reserve(r.shape(0));
-                        for (ssize_t i = 0; i < r.shape(0); ++i) vec->data.push_back(r(i));
+                        vec->data.assign(arr.data(), arr.data() + arr.size());
                     } else {
                         auto seq = data_obj.cast<py::sequence>();
                         vec->data.reserve(py::len(seq));
@@ -442,9 +427,11 @@ std::unique_ptr<rds2cpp::RObject> py_to_robject(const py::object& obj, std::vect
                     auto data_obj = d["data"];
                     if (py::isinstance<py::array>(data_obj)) {
                         auto arr = data_obj.cast<py::array_t<bool, py::array::c_style | py::array::forcecast>>();
-                        auto r = arr.unchecked<1>();
-                        vec->data.reserve(r.shape(0));
-                        for (ssize_t i = 0; i < r.shape(0); ++i) vec->data.push_back(r(i) ? 1 : 0);
+                        vec->data.reserve(arr.size());
+                        const bool* ptr = arr.data();
+                        for (size_t i = 0; i < arr.size(); ++i) {
+                            vec->data.push_back(ptr[i] ? 1 : 0);
+                        }
                     } else {
                         auto seq = data_obj.cast<py::sequence>();
                         vec->data.reserve(py::len(seq));

src/rds2py/PyRdsReader.py

@@ -122,9 +122,12 @@ def _handle_r_special_cases(self, data: np.ndarray, rtype: str, size: int) -> Un
         """Handle special R data representations."""
         try:
             # Special handling for R integer containing NA
-            if size != 2:
-                if any(data == self.R_MIN):
-                    return np.array([np.nan if x == self.R_MIN else x for x in data])
+            if rtype == "integer" and size != 2:
+                mask = data == self.R_MIN
+                if mask.any():
+                    float_data = data.astype(np.float64)
+                    float_data[mask] = np.nan
+                    return float_data
 
             # Special handling for R integer sequences
             if rtype == "integer" and size == 2 and data[0] == self.R_MIN and data[1] < 0:
@@ -140,7 +143,15 @@ def _get_numeric_data(self, obj: RdsReader, rtype: str) -> np.ndarray:
         try:
             data = obj.get_numeric_data()
             if rtype == "boolean":
-                return data.astype(bool)
+                mask = data == self.R_MIN
+                if mask.any():
+                    res = data.astype(object)
+                    res[mask] = None
+                    non_na_mask = ~mask
+                    res[non_na_mask] = res[non_na_mask].astype(bool)
+                    return res
+                else:
+                    return data.astype(bool)
 
             return data
         except Exception as e:

tests/test_atomics.py

@@ -34,6 +34,8 @@ def test_read_atomic_logical_na():
     assert arr is not None
     assert isinstance(arr, BooleanList)
     assert len(arr) > 0
+    assert None in arr
+    assert sum(1 for x in arr if x is None) == 10
 
 
 ## Doubles/Floats
@@ -123,6 +125,7 @@ def test_save_names_directly():
 
 def test_save_numpy_scalars():
     import numpy as np
+
     from rds2py import save_rds
 
     # Test integer scalar