Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/run-tox.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
name: "data-validation-framework-py${{ matrix.python-version }}-${{ matrix.min_versions }}"
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload test artifacts
uses: actions/upload-artifact@v6
uses: actions/upload-artifact@v7
if: always()
with:
name: tests-${{ matrix.python-version }}-${{ matrix.min_versions }}
Expand Down
11 changes: 10 additions & 1 deletion data_validation_framework/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,15 @@ def _apply_to_df_internal(data):
return num, df.apply(_tqdm_wrapper, axis=1, args=args, **kwargs)


def _restore_object_nulls(result_df, template_df):
"""Restore None values for columns that were object-typed in the input."""
for col, dtype in template_df.dtypes.items():
if col in result_df.columns and pd.api.types.is_object_dtype(dtype):
result_df[col] = result_df[col].astype(object)
result_df.loc[result_df[col].isna(), col] = None
return result_df


def tqdm_worker(progress_bar, tqdm_queue):
"""Update progress bar using the Queue."""
while True:
Expand Down Expand Up @@ -155,7 +164,7 @@ def apply_to_df(df, func, *args, nb_processes=None, redirect_stdout=None, **kwar
progress_bar.close()

# Return the results
return all_res
return _restore_object_nulls(all_res, df)


def try_operation(row, func, *args, **kwargs):
Expand Down
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

reqs = [
"luigi>=3.1",
"luigi-tools>=0.0.18",
"luigi-tools>=0.3.5",
"numpy>=1.21",
"pandas>=1.3",
"rst2pdf>=0.99",
Expand All @@ -17,7 +17,7 @@
doc_reqs = [
"docutils<0.21", # Temporary fix for m2r2
"m2r2",
"sphinx",
"sphinx>=7.4,<9",
"sphinx-bluebrain-theme",
]
test_reqs = [
Expand Down
9 changes: 7 additions & 2 deletions tests/test_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@

SKIP_IF_NO_LATEXMK = not which("latexmk")
REASON_NO_LATEXMK = "The command latexmk is not available."
# Newer Sphinx/TeX stacks slightly shift pagination while keeping the report content intact.
LATEXPDF_DIFF_THRESHOLD = 15


@pytest.fixture
Expand Down Expand Up @@ -2107,7 +2109,9 @@ def test_latexpdf(
assert (root / "TestWorkflow" / "report.csv").exists()
assert (root / "report_TestWorkflow.pdf").exists()
assert pdfdiff(
root / "report_TestWorkflow.pdf", data_dir / "test_report" / "report_latexpdf.pdf"
root / "report_TestWorkflow.pdf",
data_dir / "test_report" / "report_latexpdf.pdf",
threshold=LATEXPDF_DIFF_THRESHOLD,
)

def test_fail_element_no_exception(
Expand Down Expand Up @@ -2443,7 +2447,7 @@ def test_latexpdf(self, tmpdir, dataset_df_path, data_dir, TestWorkflow):
assert pdfdiff(
root / "TestWorkflow_specifications.pdf",
data_dir / "test_report_before_run" / "report_latexpdf.pdf",
threshold=15,
threshold=LATEXPDF_DIFF_THRESHOLD,
)

@pytest.fixture
Expand Down Expand Up @@ -2502,6 +2506,7 @@ def test_latexpdf_with_config(
assert pdfdiff(
root / "TestWorkflow_specifications.pdf",
data_dir / "test_report_before_run" / "report_latexpdf_with_config.pdf",
threshold=LATEXPDF_DIFF_THRESHOLD,
)

def test_nested_workflows(
Expand Down
17 changes: 12 additions & 5 deletions tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,18 @@ commands =
[testenv:min_versions]
basepython = python3.9
deps =
Requirements-Builder
uv
allowlist_externals =
uv
commands_pre =
requirements-builder --level=min --extras=test -o {envtmpdir}/requirements_min.txt setup.py
pip install -r {envtmpdir}/requirements_min.txt
pip freeze
uv pip compile \
--python {envpython} \
--resolution lowest-direct \
--extra test \
--output-file {envtmpdir}/requirements_min.txt \
setup.py
uv pip install --python {envpython} -r {envtmpdir}/requirements_min.txt
uv pip freeze --python {envpython}

[testenv:lint]
basepython = python3.9
Expand All @@ -81,7 +88,7 @@ commands =
pre-commit run --all-files

[testenv:docs]
basepython = python3.9
basepython = python3.11
changedir = docs
extras = docs
allowlist_externals =
Expand Down
Loading