diff --git a/.github/workflows/cicd_tests.yml b/.github/workflows/cicd_tests.yml index d2ae3f2b43..ae3694f276 100644 --- a/.github/workflows/cicd_tests.yml +++ b/.github/workflows/cicd_tests.yml @@ -56,7 +56,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - opt: ["codeformat", "pytype", "mypy"] + opt: ["codeformat", "mypy"] # "pytype" omitted for being essentially deprecated, see #8865 steps: - name: Clean unused tools run: | @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, macOS-latest, ubuntu-latest] + os: [windows-latest, ubuntu-latest] # macOS-latest omitted for now for being very slow, see #8864 timeout-minutes: 120 env: QUICKTEST: True @@ -220,7 +220,6 @@ jobs: - name: Install the complete dependencies run: | python -m pip install --user --upgrade pip wheel pybind11 # TODO: pybind11 added for macOS, may not be needed - #python -m pip install torch==${PYTORCH_VER1} torchvision==${TORCHVISION_VER1} cat "requirements-dev.txt" python -m pip install --no-build-isolation -r requirements-dev.txt python -m pip list @@ -267,8 +266,8 @@ jobs: python -m pip install torch==${PYTORCH_VER1} torchvision --extra-index-url https://download.pytorch.org/whl/cpu - name: Check packages run: | - pip uninstall monai - pip list | grep -iv monai + python -m pip uninstall -y monai + python -m pip list | grep -iv monai git fetch --depth=1 origin +refs/tags/*:refs/tags/* set -e @@ -306,12 +305,3 @@ jobs: python -m pip install ${name}[all] --extra-index-url https://download.pytorch.org/whl/cpu python -c 'import monai; monai.config.print_config()' 2>&1 | grep -iv "unknown" python -c 'import monai; print(monai.__file__)' - - name: Quick test - working-directory: ${{ steps.mktemp.outputs.tmp_dir }} - run: | - # run min tests - cp ${{ steps.root.outputs.pwd }}/requirements*.txt . - cp -r ${{ steps.root.outputs.pwd }}/tests . - ls -al - python -m pip install --no-build-isolation -r requirements-dev.txt --extra-index-url https://download.pytorch.org/whl/cpu - python -m unittest -v diff --git a/.github/workflows/cron-ngc-bundle.yml b/.github/workflows/cron-ngc-bundle.yml index 35c9deba53..b07d3c333e 100644 --- a/.github/workflows/cron-ngc-bundle.yml +++ b/.github/workflows/cron-ngc-bundle.yml @@ -2,6 +2,9 @@ name: cron-ngc-bundle on: + pull_request: + branches: + - dev # temporary for testing in the PR schedule: - cron: "0 2 * * *" # at 02:00 UTC # Allows you to run this workflow manually from the Actions tab @@ -22,23 +25,18 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.10' - - name: cache weekly timestamp - id: pip-cache - run: echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - uses: actions/cache@v5 - id: cache - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} + cache: pip + - name: Install CPU PyTorch + run: | + python -m pip install torch==2.8.0 torchvision --extra-index-url https://download.pytorch.org/whl/cpu - name: Install dependencies run: | rm -rf /github/home/.cache/torch/hub/bundle/ - python -m pip install --upgrade pip wheel - python -m pip install -r requirements-dev.txt + python -m pip install --no-build-isolation --upgrade pip wheel wheel-stub + python -m pip install --no-build-isolation -r requirements-dev.txt - name: Loading Bundles run: | # clean up temporary files $(pwd)/runtests.sh --build --clean # run tests - python -m tests.ngc_bundle_download + PYTHONPATH=. python -m unittest tests.bundle.test_bundle_download.TestNgcBundleDownload diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml index f1ff708a43..3bdfe12715 100644 --- a/.github/workflows/cron.yml +++ b/.github/workflows/cron.yml @@ -214,7 +214,7 @@ jobs: python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" python -c 'import torch; print(torch.rand(5,3, device=torch.device("cuda:0")))' ngc --version - BUILD_MONAI=1 ./runtests.sh --build --coverage --pytype --unittests --disttests # unit tests with pytype checks, coverage report + BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests # unit tests with pytype checks, coverage report BUILD_MONAI=1 ./runtests.sh --build --coverage --net # integration tests with coverage report coverage xml --ignore-errors if pgrep python; then pkill python; fi diff --git a/.github/workflows/pythonapp-gpu.yml b/.github/workflows/pythonapp-gpu.yml index 7de9936e9b..4a7069e7f9 100644 --- a/.github/workflows/pythonapp-gpu.yml +++ b/.github/workflows/pythonapp-gpu.yml @@ -1,14 +1,14 @@ # Jenkinsfile.monai-premerge name: premerge-gpu -on: - # quick tests for pull requests and the releasing branches - push: - branches: - - main - - releasing/* - pull_request: - types: [opened, synchronize, closed] +# on: +# # quick tests for pull requests and the releasing branches +# push: +# branches: +# - main +# - releasing/* +# pull_request: +# types: [opened, synchronize, closed] concurrency: # automatically cancel the previously triggered workflows when there's a newer version diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7df47ae19..c6d1a63345 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml index 63162b2dac..176e66b639 100644 --- a/.github/workflows/setupapp.yml +++ b/.github/workflows/setupapp.yml @@ -9,6 +9,9 @@ on: - releasing/* - feature/* - dev + pull_request: + branches: + - dev # temporary for testing in the PR concurrency: # automatically cancel the previously triggered workflows when there's a newer version @@ -20,62 +23,63 @@ jobs: # - docker-py3-pip- (shared) # - ubuntu 37 38 39 310-pip- # - os-latest-pip (shared) - coverage-py3: - # if: github.repository == 'Project-MONAI/MONAI' - if: ${{ false }} # disable self-hosted job project-monai/monai#7039 - container: - image: nvcr.io/nvidia/pytorch:22.04-py3 - options: --gpus all - runs-on: [self-hosted, linux, x64, integration] - steps: - - uses: actions/checkout@v6 - - name: cache weekly timestamp - id: pip-cache - run: | - echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - if: ${{ startsWith(github.ref, 'refs/heads/dev') }} - uses: actions/cache@v5 - id: cache - with: - path: | - ~/.cache/pip - ~/.cache/torch - key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} - - name: Install the dependencies - run: | - which python - python -m pip install --upgrade pip wheel - python -m pip install --upgrade torch torchvision - python -m pip install -r requirements-dev.txt - - name: Run unit tests report coverage - env: - NGC_API_KEY: ${{ secrets.NGC_API_KEY }} - NGC_ORG: ${{ secrets.NGC_ORG }} - NGC_TEAM: ${{ secrets.NGC_TEAM }} - run: | - python -m pip list - git config --global --add safe.directory /__w/MONAI/MONAI - git clean -ffdx - df -h - # python -m pip cache info - nvidia-smi - export CUDA_VISIBLE_DEVICES=$(python -m tests.utils | tail -n 1) - echo $CUDA_VISIBLE_DEVICES - trap 'if pgrep python; then pkill python; fi;' ERR - python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null & - python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" - python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' - BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests # unit tests with coverage report - BUILD_MONAI=1 ./runtests.sh --build --coverage --net # integration tests with coverage report - coverage xml --ignore-errors - if pgrep python; then pkill python; fi - shell: bash - - name: Upload coverage - uses: codecov/codecov-action@v6 - with: - fail_ci_if_error: false - files: ./coverage.xml + + # coverage-py3: + # # if: github.repository == 'Project-MONAI/MONAI' + # if: ${{ false }} # disable self-hosted job project-monai/monai#7039 + # container: + # image: nvcr.io/nvidia/pytorch:22.04-py3 + # options: --gpus all + # runs-on: [self-hosted, linux, x64, integration] + # steps: + # - uses: actions/checkout@v6 + # - name: cache weekly timestamp + # id: pip-cache + # run: | + # echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT + # - name: cache for pip + # if: ${{ startsWith(github.ref, 'refs/heads/dev') }} + # uses: actions/cache@v5 + # id: cache + # with: + # path: | + # ~/.cache/pip + # ~/.cache/torch + # key: docker-py3-pip-${{ steps.pip-cache.outputs.datew }} + # - name: Install the dependencies + # run: | + # which python + # python -m pip install --upgrade pip wheel wheel-stub + # python -m pip install --upgrade torch torchvision + # python -m pip install -r requirements-dev.txt + # - name: Run unit tests report coverage + # env: + # NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + # NGC_ORG: ${{ secrets.NGC_ORG }} + # NGC_TEAM: ${{ secrets.NGC_TEAM }} + # run: | + # python -m pip list + # git config --global --add safe.directory /__w/MONAI/MONAI + # git clean -ffdx + # df -h + # # python -m pip cache info + # nvidia-smi + # export CUDA_VISIBLE_DEVICES=$(python -m tests.utils | tail -n 1) + # echo $CUDA_VISIBLE_DEVICES + # trap 'if pgrep python; then pkill python; fi;' ERR + # python -c $'import torch\na,b=torch.zeros(1,device="cuda:0"),torch.zeros(1,device="cuda:1");\nwhile True:print(a,b)' > /dev/null & + # python -c "import torch; print(torch.__version__); print('{} of GPUs available'.format(torch.cuda.device_count()))" + # python -c 'import torch; print(torch.rand(5, 3, device=torch.device("cuda:0")))' + # BUILD_MONAI=1 ./runtests.sh --build --coverage --unittests --disttests # unit tests with coverage report + # BUILD_MONAI=1 ./runtests.sh --build --coverage --net # integration tests with coverage report + # coverage xml --ignore-errors + # if pgrep python; then pkill python; fi + # shell: bash + # - name: Upload coverage + # uses: codecov/codecov-action@v6 + # with: + # fail_ci_if_error: false + # files: ./coverage.xml test-py3x: runs-on: ubuntu-latest @@ -91,20 +95,26 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: pip + - name: Install CPU PyTorch + run: | + python -m pip install --upgrade pip wheel wheel-stub + python -m pip install torch==2.8.0 torchvision --extra-index-url https://download.pytorch.org/whl/cpu - name: Install the dependencies run: | find /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \; - python -m pip install --upgrade pip wheel + python -m pip install --no-build-isolation -r requirements-min.txt # necessary only here for some reason? python -m pip install --no-build-isolation -r requirements-dev.txt - name: Run quick tests CPU ubuntu env: NGC_API_KEY: ${{ secrets.NGC_API_KEY }} NGC_ORG: ${{ secrets.NGC_ORG }} NGC_TEAM: ${{ secrets.NGC_TEAM }} + TRANSFORMERS_VERBOSITY: error # stifle huggingface transformers warnings about aliases run: | python -m pip list python -c 'import torch; print(torch.__version__); print(torch.rand(5,3))' BUILD_MONAI=0 ./runtests.sh --build --coverage --quick --unittests + ./runtests.sh --clean BUILD_MONAI=1 ./runtests.sh --build --coverage --quick --min coverage xml --ignore-errors - name: Upload coverage @@ -120,18 +130,9 @@ jobs: uses: actions/setup-python@v6 with: python-version: '3.10' - - name: cache weekly timestamp - id: pip-cache + - name: Install CPU PyTorch run: | - echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - uses: actions/cache@v5 - id: cache - with: - path: | - ~/.cache/pip - ~/.cache/torch - key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} + python -m pip install torch==2.8.0 torchvision --extra-index-url https://download.pytorch.org/whl/cpu - name: Install the default branch no build (dev branch only) if: github.ref == 'refs/heads/dev' run: | diff --git a/.github/workflows/weekly-preview.yml b/.github/workflows/weekly-preview.yml index a74585aa38..6a2d07386f 100644 --- a/.github/workflows/weekly-preview.yml +++ b/.github/workflows/weekly-preview.yml @@ -1,42 +1,44 @@ name: weekly-preview +permissions: + contents: read + on: schedule: - cron: "0 2 * * 0" # 02:00 of every Sunday jobs: - flake8-py3: + static-checks: runs-on: ubuntu-latest strategy: matrix: - opt: ["codeformat", "pytype", "mypy"] + opt: ["codeformat", "mypy"] steps: + - name: Clean unused tools + run: | + find /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \; + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc /usr/local/.ghcup + sudo docker system prune -f + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Set up Python 3.10 uses: actions/setup-python@v6 with: python-version: '3.10' - - name: cache weekly timestamp - id: pip-cache - run: | - echo "datew=$(date '+%Y-%V')" >> $GITHUB_OUTPUT - - name: cache for pip - uses: actions/cache@v5 - id: cache - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ steps.pip-cache.outputs.datew }} + cache: 'pip' - name: Install dependencies run: | - find /opt/hostedtoolcache/* -maxdepth 0 ! -name 'Python' -exec rm -rf {} \; python -m pip install --upgrade pip wheel - python -m pip install -r requirements-dev.txt + python -m pip install --no-build-isolation -r requirements-dev.txt - name: Lint and type check run: | # clean up temporary files $(pwd)/runtests.sh --build --clean - # Github actions have 2 cores, so parallelize pytype - $(pwd)/runtests.sh --build --${{ matrix.opt }} -j 2 + $(pwd)/runtests.sh --build --${{ matrix.opt }} packaging: if: github.repository == 'Project-MONAI/MONAI' @@ -46,6 +48,7 @@ jobs: with: ref: dev fetch-depth: 0 + persist-credentials: false - name: Set up Python 3.10 uses: actions/setup-python@v6 with: diff --git a/environment-dev.yml b/environment-dev.yml index 9358cdc83b..b2457006c8 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -6,7 +6,7 @@ channels: - conda-forge dependencies: - numpy>=1.24,<3.0 - - pytorch>=2.3.0 + - pytorch>=2.8.0 - torchio - torchvision - pytorch-cuda>=11.6 diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 1edca210af..2511ce2219 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -609,24 +609,26 @@ def __init__( # the cache is created without multi-threading self._read_env: Any | None = None # this runs on the primary thread/process - self._read_env = self._fill_cache_start_reader(show_progress=self.progress) - print(f"Accessing lmdb file: {self.db_file.absolute()}.") + read_env = self._fill_cache_start_reader(show_progress=self.progress) + read_env.close() def set_data(self, data: Sequence): """ Set the input data and delete all the out-dated cache content. - """ + self.close() super().set_data(data=data) self._read_env = self._fill_cache_start_reader(show_progress=self.progress) def _safe_serialize(self, val): + """Serialize the tensor/array `val` using the pickle protocol, and return its bytes object.""" out = BytesIO() torch.save(convert_to_tensor(val), out, pickle_protocol=self.pickle_protocol) out.seek(0) return out.read() def _safe_deserialize(self, val): + """Load the object from the given bytes data, this must be loadable as weights only using `torch.load`.""" return torch.load(BytesIO(val), map_location="cpu", weights_only=True) def _fill_cache_start_reader(self, show_progress=True): @@ -693,10 +695,7 @@ def _fill_cache_start_reader(self, show_progress=True): return lmdb.open(path=f"{self.db_file}", subdir=False, **self.lmdb_kwargs) def _cachecheck(self, item_transformed): - """ - if the item is not found in the lmdb file, resolves to the persistent cache default behaviour. - - """ + """If the item is not found in the lmdb file, resolves to the persistent cache default behaviour.""" if self._read_env is None: # this runs on multiple processes, each one should have its own env. self._read_env = self._fill_cache_start_reader(show_progress=False) @@ -721,8 +720,15 @@ def info(self): out = dict(self._read_env.info()) out["size"] = len(self.data) out["filename"] = f"{self.db_file.absolute()}" + self.close() return out + def close(self): + """Close the read environment and set it to None, if it exists.""" + if self._read_env: + self._read_env.close() + self._read_env = None + class CacheDataset(Dataset): """ diff --git a/requirements-dev.txt b/requirements-dev.txt index 13c79a2393..f0dbb4ed0f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -17,7 +17,6 @@ pyflakes black>=26.3.1 isort>=5.1, <6, !=6.0.0 ruff>=0.14.11,<0.15 -pytype>=2020.6.1, <=2024.4.11; platform_system != "Windows" pybind11 types-setuptools mypy>=1.5.0, <1.12.0 diff --git a/tests/data/test_lmdbdataset.py b/tests/data/test_lmdbdataset.py index 04fbf95cc3..a71395af4f 100644 --- a/tests/data/test_lmdbdataset.py +++ b/tests/data/test_lmdbdataset.py @@ -19,9 +19,8 @@ import numpy as np from parameterized import parameterized -from monai.data import LMDBDataset, json_hashing +from monai.data import LMDBDataset, json_hashing, pickle_hashing from monai.transforms import Compose, LoadImaged, SimulateDelayd, Transform -from tests.test_utils import skip_if_windows TEST_CASE_1 = [ Compose( @@ -76,7 +75,7 @@ SimulateDelayd(keys=["image", "label", "extra"], delay_time=[1e-7, 1e-6, 1e-5]), ], (128, 128, 128), - {"db_name": "testdb", "lmdb_kwargs": {"map_size": 2 * 1024**2}}, + {"db_name": "testdb", "lmdb_kwargs": {"map_size": 50 * 1024**2}}, ] @@ -89,38 +88,17 @@ def __call__(self, data): return data -@skip_if_windows class TestLMDBDataset(unittest.TestCase): - def test_cache(self): - """testing no inplace change to the hashed item""" + @parameterized.expand([(pickle_hashing,), (json_hashing,)]) + def test_cache(self, hash_func): + """Testing no inplace change to the hashed item.""" items = [[list(range(i))] for i in range(5)] with tempfile.TemporaryDirectory() as tempdir: - ds = LMDBDataset(items, transform=_InplaceXform(), cache_dir=tempdir, lmdb_kwargs={"map_size": 10 * 1024}) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - ds1 = LMDBDataset(items, transform=_InplaceXform(), cache_dir=tempdir, lmdb_kwargs={"map_size": 10 * 1024}) - self.assertEqual(list(ds1), list(ds)) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - - ds = LMDBDataset( - items, - transform=_InplaceXform(), - cache_dir=tempdir, - lmdb_kwargs={"map_size": 10 * 1024}, - hash_func=json_hashing, - ) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - ds1 = LMDBDataset( - items, - transform=_InplaceXform(), - cache_dir=tempdir, - lmdb_kwargs={"map_size": 10 * 1024}, - hash_func=json_hashing, - ) - self.assertEqual(list(ds1), list(ds)) - self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]]) - - self.assertTrue(isinstance(ds1.info(), dict)) + ds = LMDBDataset(items, transform=_InplaceXform(), cache_dir=tempdir, hash_func=hash_func, progress=False) + self.assertEqual(items, [[[]], [[0]], [[0, 1]], [[0, 1, 2]], [[0, 1, 2, 3]]], "Input dataset mutated.") + self.assertNotEqual(items, list(ds), "Output data unmodified by transform.") + self.assertIsInstance(ds.info(), dict) @parameterized.expand([TEST_CASE_1, TEST_CASE_2, TEST_CASE_3, TEST_CASE_4, TEST_CASE_5, TEST_CASE_6, TEST_CASE_7]) def test_shape(self, transform, expected_shape, kwargs=None): @@ -146,16 +124,30 @@ def test_shape(self, transform, expected_shape, kwargs=None): }, ] + # update the data to cache + test_data_new = [ + { + "image": os.path.join(tempdir, "test_image1_new.nii.gz"), + "label": os.path.join(tempdir, "test_label1_new.nii.gz"), + "extra": os.path.join(tempdir, "test_extra1_new.nii.gz"), + }, + { + "image": os.path.join(tempdir, "test_image2_new.nii.gz"), + "label": os.path.join(tempdir, "test_label2_new.nii.gz"), + "extra": os.path.join(tempdir, "test_extra2_new.nii.gz"), + }, + ] + cache_dir = os.path.join(os.path.join(tempdir, "cache"), "data") - dataset_precached = LMDBDataset( - data=test_data, transform=transform, progress=False, cache_dir=cache_dir, **kwargs - ) + ds_args = dict(data=test_data, transform=transform, progress=False, cache_dir=cache_dir, **kwargs) + + dataset_precached = LMDBDataset(**ds_args) data1_precached = dataset_precached[0] data2_precached = dataset_precached[1] - dataset_postcached = LMDBDataset( - data=test_data, transform=transform, progress=False, cache_dir=cache_dir, **kwargs - ) + dataset_precached.close() + + dataset_postcached = LMDBDataset(**ds_args) data1_postcached = dataset_postcached[0] data2_postcached = dataset_postcached[1] @@ -179,19 +171,6 @@ def test_shape(self, transform, expected_shape, kwargs=None): self.assertTupleEqual(data2_postcached["label"].shape, expected_shape) self.assertTupleEqual(data2_postcached["extra"].shape, expected_shape) - # update the data to cache - test_data_new = [ - { - "image": os.path.join(tempdir, "test_image1_new.nii.gz"), - "label": os.path.join(tempdir, "test_label1_new.nii.gz"), - "extra": os.path.join(tempdir, "test_extra1_new.nii.gz"), - }, - { - "image": os.path.join(tempdir, "test_image2_new.nii.gz"), - "label": os.path.join(tempdir, "test_label2_new.nii.gz"), - "extra": os.path.join(tempdir, "test_extra2_new.nii.gz"), - }, - ] # test new exchanged cache content if transform is None: dataset_postcached.set_data(data=test_data_new) @@ -202,6 +181,8 @@ def test_shape(self, transform, expected_shape, kwargs=None): with self.assertRaises(RuntimeError): dataset_postcached.set_data(data=test_data_new) # filename list updated, files do not exist + dataset_postcached.close() # open environments are fragile, cleanup is needed for tests + if __name__ == "__main__": unittest.main()