Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions packages/db-dtypes/db_dtypes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
date_dtype_name = "dbdate"
time_dtype_name = "dbtime"
_EPOCH = datetime.datetime(1970, 1, 1)
_NPEPOCH = numpy.datetime64(_EPOCH)
_NPEPOCH = numpy.datetime64(_EPOCH, "ns")
_NP_DTYPE = "datetime64[ns]"

# Numpy converts datetime64 scalars to datetime.datetime only if microsecond or
Expand Down Expand Up @@ -119,7 +119,7 @@ def _datetime(
)

if pandas.isna(scalar):
return numpy.datetime64("NaT")
return numpy.datetime64("NaT", "ns")
if isinstance(scalar, datetime.time):
return pandas.Timestamp(
year=1970,
Expand Down Expand Up @@ -172,9 +172,12 @@ def _box_func(self, x):
__return_deltas = {"timedelta", "timedelta64", "timedelta64[ns]", "<m8", _NP_DTYPE}

def astype(self, dtype, copy=True):
deltas = self._ndarray - _NPEPOCH
matched_epoch = _NPEPOCH.astype(self._ndarray.dtype)
deltas = self._ndarray - matched_epoch
stype = str(dtype)
if stype in self.__return_deltas:
if "ns" in stype and "ns" not in str(deltas.dtype):
return deltas.astype("timedelta64[ns]")
return deltas
Comment thread
parthea marked this conversation as resolved.
elif stype.startswith("timedelta64[") or stype.startswith("<m8["):
return deltas.astype(dtype, copy=False)
Expand Down Expand Up @@ -250,7 +253,9 @@ def _datetime(
scalar = scalar.as_py()

if pandas.isna(scalar):
return numpy.datetime64("NaT")
# Use day-level "D" resolution for DateArray missing values
# to match the underlying storage unit and avoid unit mismatches.
return numpy.datetime64("NaT", "D")
elif isinstance(scalar, numpy.datetime64):
dateObj = pandas.Timestamp(scalar)
elif isinstance(scalar, datetime.date):
Expand Down Expand Up @@ -321,7 +326,8 @@ def __add__(self, other):
return self.astype("object") + other

if isinstance(other, TimeArray):
return (other._ndarray - _NPEPOCH) + self._ndarray
matched_epoch = _NPEPOCH.astype(other._ndarray.dtype)
return (other._ndarray - matched_epoch) + self._ndarray

return super().__add__(other) # type: ignore[misc]

Expand Down
13 changes: 12 additions & 1 deletion packages/db-dtypes/db_dtypes/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@

from db_dtypes import pandas_backports

from typing import cast

pandas_release = pandas_backports.pandas_release


Expand All @@ -48,7 +50,16 @@ class BaseDatetimeArray(pandas_backports.OpsMixin, _mixins.NDArrayBackedExtensio
# Categorical, iNaT for Period. Outside of object dtype, self.isna() should
# be exactly locations in self._ndarray with _internal_fill_value. See:
# https://github.com/pandas-dev/pandas/blob/main/pandas/core/arrays/_mixins.py
_internal_fill_value = numpy.datetime64("NaT")

# This is defined dynamically to match the underlying unit of self._ndarray
# (e.g., "D" for DateArray, "ns" for TimeArray) to avoid unit mismatches.
@property
def _internal_fill_value(self):
ndarray = getattr(self, "_ndarray", None)
if ndarray is not None:
unit = cast(Any, numpy.datetime_data(ndarray.dtype)[0])
return numpy.datetime64("NaT", unit)
return numpy.datetime64("NaT", "ns")

_box_func: Callable[[Any], Any]
_from_backing_data: Callable[[Any], Any]
Expand Down
1 change: 0 additions & 1 deletion packages/db-dtypes/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,6 @@ def default(session, tests_path):
"-W default::FutureWarning",
f"--junitxml={os.path.split(tests_path)[-1]}_{session.python}_sponge_log.xml",
"--cov=db_dtypes",
"--cov=tests/unit",
"--cov-append",
"--cov-config=.coveragerc",
"--cov-report=",
Expand Down
19 changes: 19 additions & 0 deletions packages/db-dtypes/tests/unit/test_dtypes.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import datetime

import numpy
import pytest

pd = pytest.importorskip("pandas")
Expand Down Expand Up @@ -668,3 +669,21 @@ def test_date_sub():
do = pd.Series([pd.DateOffset(days=i) for i in range(4)])
expect = dates.astype("object") - do
np.testing.assert_array_equal(dates - do, expect)


def test_internal_fill_value_without_ndarray():
"""Ensure fallback return is reached when _ndarray is missing."""
from db_dtypes.core import BaseDatetimeArray

# Create a bare instance using __new__ to bypass __init__ and avoid setting _ndarray
instance = BaseDatetimeArray.__new__(BaseDatetimeArray)

# Extract the value
fill_value = instance._internal_fill_value

# Use numpy.isnat() to check for NaT safely
assert numpy.isnat(fill_value)

# (Optional) Verify the unit metadata is still exactly what you expect
unit, _ = numpy.datetime_data(fill_value.dtype)
assert unit == "ns"
Loading