diff --git a/CHANGELOG.md b/CHANGELOG.md index f39b0a94..86a8e9e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,8 @@ The format is based on Keep a Changelog and this project adheres to Semantic Ver ## [0.1.96] - 2026-07-09 -- **Add opt-in TTL caching for `get_prompt`** - `Netra.prompts.get_prompt` now accepts `use_cache` and `cache_ttl` parameters for in-memory caching. Configure the default TTL via `cache_ttl_seconds` in `Netra.init()` or the `NETRA_CACHE_TTL_SECONDS` environment variable. Use `Netra.prompts.clear_cache()` to invalidate cached entries. +- **Add opt-in TTL caching for `get_prompt`** - `Netra.prompts.get_prompt` now accepts `use_cache` and `cache_ttl` parameters for in-memory caching. Default TTL is `PROMPT_CACHE_TTL_SECONDS` (60); override per call with `cache_ttl`. Use `Netra.prompts.clear_cache()` to invalidate cached entries. +- **Add opt-in TTL caching for `get_model_pricing`** - `Netra.models.get_model_pricing` now accepts `use_cache` and `cache_ttl` parameters for in-memory caching. Default TTL is `MODEL_PRICING_CACHE_TTL_SECONDS` (300); override per call with `cache_ttl`. Use `Netra.models.clear_cache()` to invalidate cached entries. ## [0.1.95] - 2026-06-26 diff --git a/README.md b/README.md index fddd8e2f..798702c1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ - 🌐 **HTTP Client Instrumentation**: Automatic tracing for aiohttp and httpx - 💾 **Vector Database Support**: Weaviate, Qdrant, and other vector DB instrumentation - 📋 **Prompt Management**: Fetch managed prompts from Netra with optional in-memory TTL caching +- 💰 **Model Pricing**: Fetch model pricing from Netra with optional in-memory TTL caching ## 📦 Installation @@ -50,7 +51,6 @@ Netra.init( trace_content=True, environment="Your Application environment", instruments={InstrumentSet.OPENAI, InstrumentSet.ANTHROPIC}, - cache_ttl_seconds=60, # default TTL for opt-in prompt caching (env: NETRA_CACHE_TTL_SECONDS) ) ``` @@ -323,7 +323,7 @@ Action tracking follows this schema: ## 📋 Prompt Management -Fetch managed prompt versions from Netra via `Netra.prompts`. Caching is opt-in and disabled by default. +Fetch managed prompt versions from Netra via `Netra.prompts`. Caching is opt-in and disabled by default. Default TTL is **60 seconds** (`PROMPT_CACHE_TTL_SECONDS`); override per call with `cache_ttl`. ```python from netra import Netra @@ -332,13 +332,12 @@ from netra.instrumentation.instruments import InstrumentSet Netra.init( app_name="My App", instruments={InstrumentSet.OPENAI}, - cache_ttl_seconds=60, # default TTL for cached prompt reads ) # Fetch a prompt (calls the API on every request by default) prompt = Netra.prompts.get_prompt("my-prompt", label="production") -# Opt in to in-memory caching to reduce API calls +# Opt in to in-memory caching to reduce API calls (default TTL: 60s) prompt = Netra.prompts.get_prompt("my-prompt", label="production", use_cache=True) # Override TTL for a single call (seconds) @@ -351,10 +350,49 @@ Netra.prompts.clear_cache() Caching notes: - `use_cache` defaults to `False`; enable it per call when you want caching. +- Default TTL is the module constant `PROMPT_CACHE_TTL_SECONDS` (60); override with `cache_ttl`. - Cache keys are scoped by prompt `name` and `label`. - Empty or failed responses are not stored in the cache. - The prompt cache is cleared automatically when `Netra.shutdown()` is called. +## 💰 Model Pricing + +Fetch model details and pricing for your project via `Netra.models`. Caching is opt-in and disabled by default. Default TTL is **300 seconds** (`MODEL_PRICING_CACHE_TTL_SECONDS`); override per call with `cache_ttl`. + +```python +from netra import Netra +from netra.instrumentation.instruments import InstrumentSet + +Netra.init( + app_name="My App", + instruments={InstrumentSet.OPENAI}, +) + +# Fetch all model pricing (calls the API on every request by default) +models = Netra.models.get_model_pricing() + +# Fetch pricing for a specific model +models = Netra.models.get_model_pricing(name="gpt-4o") + +# Opt in to in-memory caching to reduce API calls (default TTL: 300s) +models = Netra.models.get_model_pricing(use_cache=True) + +# Override TTL for a single call (seconds) +models = Netra.models.get_model_pricing(use_cache=True, cache_ttl=600) + +# Clear cached entries after pricing updates +Netra.models.clear_cache() +``` + +Caching notes: + +- `use_cache` defaults to `False`; enable it per call when you want caching. +- Default TTL is the module constant `MODEL_PRICING_CACHE_TTL_SECONDS` (300); override with `cache_ttl`. +- Cache keys are scoped by model `name` (or `"all"` when no name is passed). +- Empty or failed responses are not stored in the cache. +- When `use_cache` is `True`, do not mutate the returned list or nested dicts — the same objects may be served on later cache hits. +- The model pricing cache is cleared automatically when `Netra.shutdown()` is called. + ## 🔧 Advanced Configuration ### Environment Variables @@ -373,7 +411,6 @@ Netra SDK can be configured using the following environment variables: | `NETRA_TRACE_CONTENT` | Whether to capture prompt/completion content (`true`/`false`) | `true` | | `NETRA_ENV` | Deployment environment (e.g., `prod`, `staging`, `dev`) | `local` | | `NETRA_RESOURCE_ATTRS` | JSON string of custom resource attributes | `{}` | -| `NETRA_CACHE_TTL_SECONDS` | Default TTL in seconds for opt-in SDK read caches (e.g. `get_prompt`) | `60` | #### Standard OpenTelemetry Variables diff --git a/netra/__init__.py b/netra/__init__.py index 8f8c7130..d36b346e 100644 --- a/netra/__init__.py +++ b/netra/__init__.py @@ -76,7 +76,6 @@ def init( metrics_export_interval_ms: Optional[int] = None, export_auto_metrics: Optional[bool] = None, root_instruments: Optional[AbstractSet[NetraInstruments]] = None, - cache_ttl_seconds: Optional[int] = None, ) -> None: """ Thread-safe initialization of Netra. @@ -111,8 +110,6 @@ def init( span is blocked, its entire subtree is discarded. Pass a set containing ``NetraInstruments.ALL`` to allow all instrumentations to produce root spans (legacy behaviour). - cache_ttl_seconds: Default TTL in seconds for opt-in read caches - (default: 60, env: NETRA_CACHE_TTL_SECONDS) Returns: None @@ -137,7 +134,6 @@ def init( enable_metrics=enable_metrics, metrics_export_interval_ms=metrics_export_interval_ms, export_auto_metrics=export_auto_metrics, - cache_ttl_seconds=cache_ttl_seconds, ) # Configure logging based on debug mode @@ -306,6 +302,11 @@ def shutdown(cls) -> None: cls.prompts.clear_cache() except Exception: pass + if hasattr(cls, "models") and cls.models is not None: + try: + cls.models.clear_cache() + except Exception: + pass @classmethod def get_meter(cls, name: str = "netra", version: Optional[str] = None) -> otel_metrics.Meter: diff --git a/netra/config.py b/netra/config.py index 935ad0ce..fe233a9e 100644 --- a/netra/config.py +++ b/netra/config.py @@ -35,7 +35,6 @@ def __init__( enable_metrics: Optional[bool] = None, metrics_export_interval_ms: Optional[int] = None, export_auto_metrics: Optional[bool] = None, - cache_ttl_seconds: Optional[int] = None, ): """ Initialize the configuration. @@ -53,7 +52,6 @@ def __init__( enable_metrics: Whether to enable custom metrics export via OTLP (default: False) metrics_export_interval_ms: How often to push metrics to the collector in ms (default: 60000) export_auto_metrics: Whether to export OTel auto-instrumented system metrics (default: False) - cache_ttl_seconds: Default TTL in seconds for opt-in SDK read caches (default: 60, env: NETRA_CACHE_TTL_SECONDS) """ self.app_name = self._get_app_name(app_name) self.otlp_endpoint = self._get_otlp_endpoint() @@ -79,9 +77,6 @@ def __init__( self.metrics_export_interval_ms = self._get_int_config( metrics_export_interval_ms, "NETRA_METRICS_EXPORT_INTERVAL", default=60000 ) - self.cache_ttl_seconds = self._get_int_config( - cache_ttl_seconds, "NETRA_CACHE_TTL_SECONDS", default=60 - ) self._set_trace_content_env() diff --git a/netra/models/api.py b/netra/models/api.py index 81ce7c94..ab1a746f 100644 --- a/netra/models/api.py +++ b/netra/models/api.py @@ -1,11 +1,14 @@ import logging from typing import Any, List, Optional +from netra.cache import TTLCache from netra.config import Config from netra.models.client import ModelsHttpClient logger = logging.getLogger(__name__) +MODEL_PRICING_CACHE_TTL_SECONDS = 300 + class Models: """Public entry-point exposed as Netra.models""" @@ -19,25 +22,53 @@ def __init__(self, config: Config) -> None: """ self._config = config self._client = ModelsHttpClient(config) + self._cache: TTLCache[Any] = TTLCache(default_ttl=MODEL_PRICING_CACHE_TTL_SECONDS) + + def clear_cache(self) -> None: + """Clear all cached model pricing entries.""" + self._cache.clear() - def get_model_pricing(self, name: Optional[str] = None) -> List[Any] | Any: + def get_model_pricing( + self, + name: Optional[str] = None, + use_cache: bool = False, + cache_ttl: Optional[int] = None, + ) -> List[Any] | Any: """ Fetch models for the project associated with the configured API key. Args: name: Optional model name to filter results. + use_cache: When True, read/write the in-memory cache (default: False). + cache_ttl: Per-call cache TTL in seconds (default: 300). Returns: - List of model dicts from the API response, or None on failure. + List of model dicts from the API response, or empty list on failure. + When use_cache is True, do not mutate the returned list or nested + dicts/prices — the same objects may be served on later cache hits. """ + cache_key = f"model:pricing:{name or 'all'}" + + if use_cache: + cached = self._cache.get(cache_key) + if cached is not None: + return cached + result = self._client.get_model_pricing(name=name) if not isinstance(result, dict): return result + # Client failure sentinel is {}; do not treat as a successful empty list. + if not result: + return [] + items = result.get("data", []) or [] if not isinstance(items, list): logger.error("netra.models: Unexpected response format; 'data' is not a list") return [] + if use_cache: + self._cache.set(cache_key, items, cache_ttl) + return items diff --git a/netra/prompts/api.py b/netra/prompts/api.py index b173b96b..81f15747 100644 --- a/netra/prompts/api.py +++ b/netra/prompts/api.py @@ -7,6 +7,8 @@ logger = logging.getLogger(__name__) +PROMPT_CACHE_TTL_SECONDS = 60 + class Prompts: """ @@ -22,7 +24,7 @@ def __init__(self, cfg: Config) -> None: """ self._config = cfg self._client = PromptsHttpClient(cfg) - self._cache: TTLCache[Any] = TTLCache(default_ttl=cfg.cache_ttl_seconds) + self._cache: TTLCache[Any] = TTLCache(default_ttl=PROMPT_CACHE_TTL_SECONDS) def clear_cache(self) -> None: """Clear all cached prompt entries.""" @@ -42,7 +44,7 @@ def get_prompt( name: Name of the prompt label: Label of the prompt version (default: "production") use_cache: When True, read/write the in-memory cache (default: False) - cache_ttl: Per-call cache TTL in seconds (default: init cache_ttl_seconds) + cache_ttl: Per-call cache TTL in seconds (default: PROMPT_CACHE_TTL_SECONDS) Returns: Prompt version data or None/empty dict if not found diff --git a/tests/test_models_cache.py b/tests/test_models_cache.py new file mode 100644 index 00000000..ecdaca92 --- /dev/null +++ b/tests/test_models_cache.py @@ -0,0 +1,171 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from netra import Netra +from netra.config import Config +from netra.models.api import MODEL_PRICING_CACHE_TTL_SECONDS, Models + + +@pytest.fixture +def models() -> Models: + cfg = Config() + client = MagicMock() + instance = Models(cfg) + instance._client = client + return instance + + +class TestModelsGetModelPricingCaching: + def test_use_cache_omitted_calls_http_every_time(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing() + models.get_model_pricing() + + assert models._client.get_model_pricing.call_count == 2 + + def test_use_cache_true_second_call_skips_http(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + first = models.get_model_pricing(use_cache=True) + second = models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 1 + assert first == [{"name": "gpt-4o"}] + assert second == [{"name": "gpt-4o"}] + + def test_use_cache_true_different_names_use_separate_entries(self, models: Models) -> None: + models._client.get_model_pricing.side_effect = [ + {"data": [{"name": "gpt-4o"}]}, + {"data": [{"name": "claude-3"}]}, + ] + + gpt = models.get_model_pricing("gpt-4o", use_cache=True) + claude = models.get_model_pricing("claude-3", use_cache=True) + + assert models._client.get_model_pricing.call_count == 2 + assert gpt == [{"name": "gpt-4o"}] + assert claude == [{"name": "claude-3"}] + + def test_name_none_uses_all_cache_key(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing(use_cache=True) + models.get_model_pricing(name=None, use_cache=True) + + assert models._client.get_model_pricing.call_count == 1 + + def test_empty_list_is_cached(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": []} + + first = models.get_model_pricing(use_cache=True) + second = models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 1 + assert first == [] + assert second == [] + + def test_api_failure_empty_dict_does_not_store_in_cache(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {} + + models.get_model_pricing(use_cache=True) + models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 2 + + def test_api_none_response_does_not_store_in_cache(self, models: Models) -> None: + models._client.get_model_pricing.return_value = None + + models.get_model_pricing(use_cache=True) + models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 2 + + def test_non_list_data_does_not_store_in_cache(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": {"name": "bad"}} + + models.get_model_pricing(use_cache=True) + models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 2 + + def test_per_call_cache_ttl_expires_before_default(self, models: Models) -> None: + with patch("netra.cache.time.monotonic", side_effect=[0.0, 0.0, 1.1, 1.1]): + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing(use_cache=True, cache_ttl=1) + assert models._client.get_model_pricing.call_count == 1 + + models.get_model_pricing(use_cache=True, cache_ttl=1) + assert models._client.get_model_pricing.call_count == 1 + + models.get_model_pricing(use_cache=True, cache_ttl=1) + assert models._client.get_model_pricing.call_count == 2 + + def test_zero_cache_ttl_skips_cache_write(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing(use_cache=True, cache_ttl=0) + models.get_model_pricing(use_cache=True, cache_ttl=0) + + assert models._client.get_model_pricing.call_count == 2 + + def test_use_cache_false_with_cache_ttl_ignores_cache(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing(use_cache=False, cache_ttl=30) + models.get_model_pricing(use_cache=False, cache_ttl=30) + + assert models._client.get_model_pricing.call_count == 2 + + def test_clear_cache_forces_next_call_to_hit_http(self, models: Models) -> None: + models._client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + + models.get_model_pricing(use_cache=True) + models.clear_cache() + models.get_model_pricing(use_cache=True) + + assert models._client.get_model_pricing.call_count == 2 + + def test_default_ttl_is_models_owned_constant(self, models: Models) -> None: + assert MODEL_PRICING_CACHE_TTL_SECONDS == 300 + assert models._cache._default_ttl == MODEL_PRICING_CACHE_TTL_SECONDS + assert models._cache._default_ttl != 60 + + +class TestModelsCacheShutdown: + def setup_method(self) -> None: + with Netra._init_lock: + Netra._initialized = False + + def teardown_method(self) -> None: + with Netra._init_lock: + Netra._initialized = False + + @patch("netra.init_instrumentations") + @patch("netra.Tracer") + @patch("netra.Config") + def test_shutdown_clears_models_cache( + self, + mock_config: MagicMock, + mock_tracer: MagicMock, + mock_init_instrumentations: MagicMock, + ) -> None: + mock_cfg = MagicMock() + mock_config.return_value = mock_cfg + + Netra.init() + + mock_client = MagicMock() + mock_client.get_model_pricing.return_value = {"data": [{"name": "gpt-4o"}]} + Netra.models._client = mock_client + + Netra.models.get_model_pricing(use_cache=True) + Netra.models.get_model_pricing(use_cache=True) + assert mock_client.get_model_pricing.call_count == 1 + + Netra.shutdown() + + Netra.models.get_model_pricing(use_cache=True) + assert mock_client.get_model_pricing.call_count == 2 diff --git a/tests/test_netra_init.py b/tests/test_netra_init.py index 93398deb..e223777b 100644 --- a/tests/test_netra_init.py +++ b/tests/test_netra_init.py @@ -64,7 +64,6 @@ def test_init_with_default_parameters( enable_metrics=None, metrics_export_interval_ms=None, export_auto_metrics=None, - cache_ttl_seconds=None, ) # Verify Tracer was initialized @@ -101,7 +100,6 @@ def test_init_with_custom_parameters( "enable_metrics": None, "metrics_export_interval_ms": None, "export_auto_metrics": None, - "cache_ttl_seconds": None, } app_name = "test-app" diff --git a/tests/test_prompts_cache.py b/tests/test_prompts_cache.py index 35d39785..f06de871 100644 --- a/tests/test_prompts_cache.py +++ b/tests/test_prompts_cache.py @@ -4,12 +4,12 @@ from netra import Netra from netra.config import Config -from netra.prompts.api import Prompts +from netra.prompts.api import PROMPT_CACHE_TTL_SECONDS, Prompts @pytest.fixture def prompts() -> Prompts: - cfg = Config(cache_ttl_seconds=60) + cfg = Config() client = MagicMock() instance = Prompts(cfg) instance._client = client @@ -102,6 +102,11 @@ def test_clear_cache_forces_next_call_to_hit_http(self, prompts: Prompts) -> Non assert prompts._client.get_prompt_version.call_count == 2 + def test_default_ttl_is_prompts_owned_constant(self, prompts: Prompts) -> None: + assert PROMPT_CACHE_TTL_SECONDS == 60 + assert prompts._cache._default_ttl == PROMPT_CACHE_TTL_SECONDS + assert prompts._cache._default_ttl != 300 + class TestPromptsCacheShutdown: def setup_method(self) -> None: @@ -122,7 +127,6 @@ def test_shutdown_clears_prompt_cache( mock_init_instrumentations: MagicMock, ) -> None: mock_cfg = MagicMock() - mock_cfg.cache_ttl_seconds = 60 mock_config.return_value = mock_cfg Netra.init()