From 346cf00298d355822004d2226d8f97da74bf041a Mon Sep 17 00:00:00 2001 From: maxtext authors Date: Sun, 14 Jun 2026 22:38:04 -0700 Subject: [PATCH] Internal change PiperOrigin-RevId: 932236591 --- src/maxtext/configs/pyconfig.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/maxtext/configs/pyconfig.py b/src/maxtext/configs/pyconfig.py index f4a8e3965f..b018f95493 100644 --- a/src/maxtext/configs/pyconfig.py +++ b/src/maxtext/configs/pyconfig.py @@ -531,6 +531,21 @@ def _initialize_pydantic(argv: list[str] | None = None, **kwargs) -> MaxTextConf pydantic_kwargs = _prepare_for_pydantic(raw_keys_dict) + # Resolve relative tokenizer_path against the config directory + # (fileset root on Borg) + if pydantic_kwargs.get("tokenizer_path"): + fileset_root = os.path.dirname(config_path) + candidate_path = os.path.join( + fileset_root, pydantic_kwargs["tokenizer_path"] + ) + if os.path.exists(candidate_path): + logger.info( + "Resolved tokenizer_path %s to %s under fileset root", + pydantic_kwargs["tokenizer_path"], + candidate_path, + ) + pydantic_kwargs["tokenizer_path"] = candidate_path + if pydantic_kwargs.get("use_tokamax_splash") and pydantic_kwargs.get("use_jax_splash"): raise ValueError("At most one of `use_tokamax_splash` and `use_jax_splash` can be set to True.")