From abb03fa72117cdddeba100904c43e0b1939c74cd Mon Sep 17 00:00:00 2001 From: Amey Pawar <138877912+ameyypawar@users.noreply.github.com> Date: Thu, 2 Jul 2026 00:26:44 +0530 Subject: [PATCH 1/2] feat(fetch): make the request timeout configurable The fetch server hardcoded a 30s httpx timeout, which is too short for large downloads and slow endpoints and too long for quick health checks, with no way to override it (#4448). Make the timeout configurable, most-specific wins: - per-request: an optional `timeout` argument on the fetch tool; - server default: `--timeout` CLI flag or the `FETCH_TIMEOUT` env var; - falls back to the existing 30s default. The timeout is in seconds (float), matching httpx and the previous hardcoded value rather than the milliseconds suggested in the issue. The robots.txt preflight honors the same timeout so it does not become the bottleneck on slow hosts. Addresses #4448. --- src/fetch/src/mcp_server_fetch/__init__.py | 15 ++++++- src/fetch/src/mcp_server_fetch/server.py | 49 +++++++++++++++++++--- src/fetch/tests/test_server.py | 44 +++++++++++++++++++ 3 files changed, 100 insertions(+), 8 deletions(-) diff --git a/src/fetch/src/mcp_server_fetch/__init__.py b/src/fetch/src/mcp_server_fetch/__init__.py index 09744ce319..32bc8bf039 100644 --- a/src/fetch/src/mcp_server_fetch/__init__.py +++ b/src/fetch/src/mcp_server_fetch/__init__.py @@ -1,10 +1,11 @@ -from .server import serve +from .server import serve, DEFAULT_REQUEST_TIMEOUT def main(): """MCP Fetch Server - HTTP fetching functionality for MCP""" import argparse import asyncio + import os parser = argparse.ArgumentParser( description="give a model the ability to make web requests" @@ -17,8 +18,18 @@ def main(): ) parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests") + env_timeout = os.environ.get("FETCH_TIMEOUT") + parser.add_argument( + "--timeout", + type=float, + default=float(env_timeout) if env_timeout else DEFAULT_REQUEST_TIMEOUT, + help="Request timeout in seconds (default: 30, or the FETCH_TIMEOUT env var)", + ) + args = parser.parse_args() - asyncio.run(serve(args.user_agent, args.ignore_robots_txt, args.proxy_url)) + asyncio.run( + serve(args.user_agent, args.ignore_robots_txt, args.proxy_url, args.timeout) + ) if __name__ == "__main__": diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index b42c7b1f6b..6a3cffadf6 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -23,6 +23,12 @@ DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)" DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)" +# Default per-request timeout, in seconds, for outbound HTTP requests. Matches +# httpx's timeout unit and the value previously hardcoded here. Overridable via +# the serve() timeout argument, the --timeout CLI flag, the FETCH_TIMEOUT +# environment variable, or the per-request "timeout" tool argument. +DEFAULT_REQUEST_TIMEOUT = 30.0 + def extract_content_from_html(html: str) -> str: """Extract and convert HTML content to Markdown format. @@ -63,7 +69,12 @@ def get_robots_txt_url(url: str) -> str: return robots_url -async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None: +async def check_may_autonomously_fetch_url( + url: str, + user_agent: str, + proxy_url: str | None = None, + timeout: float = DEFAULT_REQUEST_TIMEOUT, +) -> None: """ Check if the URL can be fetched by the user agent according to the robots.txt file. Raises a McpError if not. @@ -78,6 +89,7 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, + timeout=timeout, ) except HTTPError: raise McpError(ErrorData( @@ -109,7 +121,11 @@ async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: async def fetch_url( - url: str, user_agent: str, force_raw: bool = False, proxy_url: str | None = None + url: str, + user_agent: str, + force_raw: bool = False, + proxy_url: str | None = None, + timeout: float = DEFAULT_REQUEST_TIMEOUT, ) -> Tuple[str, str]: """ Fetch the URL and return the content in a form ready for the LLM, as well as a prefix string with status information. @@ -122,7 +138,7 @@ async def fetch_url( url, follow_redirects=True, headers={"User-Agent": user_agent}, - timeout=30, + timeout=timeout, ) except HTTPError as e: raise McpError(ErrorData(code=INTERNAL_ERROR, message=f"Failed to fetch {url}: {e!r}")) @@ -176,12 +192,21 @@ class Fetch(BaseModel): description="Get the actual HTML content of the requested page, without simplification.", ), ] + timeout: Annotated[ + float | None, + Field( + default=None, + description="Request timeout in seconds for this fetch. Overrides the server-wide default when set.", + gt=0, + ), + ] async def serve( custom_user_agent: str | None = None, ignore_robots_txt: bool = False, proxy_url: str | None = None, + timeout: float = DEFAULT_REQUEST_TIMEOUT, ) -> None: """Run the fetch MCP server. @@ -189,6 +214,8 @@ async def serve( custom_user_agent: Optional custom User-Agent string to use for requests ignore_robots_txt: Whether to ignore robots.txt restrictions proxy_url: Optional proxy URL to use for requests + timeout: Default request timeout in seconds, used when a fetch call does + not specify its own timeout """ server = Server("mcp-fetch") user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS @@ -231,11 +258,19 @@ async def call_tool(name, arguments: dict) -> list[TextContent]: if not url: raise McpError(ErrorData(code=INVALID_PARAMS, message="URL is required")) + request_timeout = args.timeout if args.timeout is not None else timeout + if not ignore_robots_txt: - await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url) + await check_may_autonomously_fetch_url( + url, user_agent_autonomous, proxy_url, timeout=request_timeout + ) content, prefix = await fetch_url( - url, user_agent_autonomous, force_raw=args.raw, proxy_url=proxy_url + url, + user_agent_autonomous, + force_raw=args.raw, + proxy_url=proxy_url, + timeout=request_timeout, ) original_length = len(content) if args.start_index >= original_length: @@ -262,7 +297,9 @@ async def get_prompt(name: str, arguments: dict | None) -> GetPromptResult: url = arguments["url"] try: - content, prefix = await fetch_url(url, user_agent_manual, proxy_url=proxy_url) + content, prefix = await fetch_url( + url, user_agent_manual, proxy_url=proxy_url, timeout=timeout + ) # TODO: after SDK bug is addressed, don't catch the exception except McpError as e: return GetPromptResult( diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py index 96c1cb38c7..b0173628d1 100644 --- a/src/fetch/tests/test_server.py +++ b/src/fetch/tests/test_server.py @@ -10,6 +10,7 @@ check_may_autonomously_fetch_url, fetch_url, DEFAULT_USER_AGENT_AUTONOMOUS, + DEFAULT_REQUEST_TIMEOUT, ) @@ -324,3 +325,46 @@ async def test_fetch_with_proxy(self): # Verify AsyncClient was called with proxy mock_client_class.assert_called_once_with(proxy="http://proxy.example.com:8080") + + @pytest.mark.asyncio + async def test_fetch_uses_default_timeout(self): + """Test that fetch_url applies the default timeout to the request.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"data": "test"}' + mock_response.headers = {"content-type": "application/json"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + ) + + assert mock_client.get.call_args.kwargs["timeout"] == DEFAULT_REQUEST_TIMEOUT + + @pytest.mark.asyncio + async def test_fetch_uses_custom_timeout(self): + """Test that an explicit timeout is passed through to the request.""" + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.text = '{"data": "test"}' + mock_response.headers = {"content-type": "application/json"} + + with patch("httpx.AsyncClient") as mock_client_class: + mock_client = AsyncMock() + mock_client.get = AsyncMock(return_value=mock_response) + mock_client_class.return_value.__aenter__ = AsyncMock(return_value=mock_client) + mock_client_class.return_value.__aexit__ = AsyncMock(return_value=None) + + await fetch_url( + "https://api.example.com/data", + DEFAULT_USER_AGENT_AUTONOMOUS, + timeout=60.0, + ) + + assert mock_client.get.call_args.kwargs["timeout"] == 60.0 From 04ee6bb9840b931b42214e1ae431e8bc3c9aa6e5 Mon Sep 17 00:00:00 2001 From: Amey Pawar <138877912+ameyypawar@users.noreply.github.com> Date: Wed, 8 Jul 2026 16:27:12 +0530 Subject: [PATCH 2/2] refactor(fetch): scope timeout to the content fetch and validate it Address review feedback on the configurable-timeout change: - Revert the robots.txt preflight to its previous (httpx-default) timeout; only the content fetch honors the configurable timeout, keeping the change scoped to what #4448 describes and avoiding a behavior change to the preflight. - Reject non-positive --timeout / FETCH_TIMEOUT values (matching the per-request field's gt=0), with a clean error instead of a traceback. - Add tests for the --timeout flag, the FETCH_TIMEOUT env var, and their precedence. - Document --timeout / FETCH_TIMEOUT and the per-request timeout argument in the README. --- src/fetch/README.md | 7 ++++ src/fetch/src/mcp_server_fetch/__init__.py | 19 ++++++++-- src/fetch/src/mcp_server_fetch/server.py | 24 +++++-------- src/fetch/tests/test_server.py | 42 ++++++++++++++++++++++ 4 files changed, 74 insertions(+), 18 deletions(-) diff --git a/src/fetch/README.md b/src/fetch/README.md index 2c3e048927..48e3a9def4 100644 --- a/src/fetch/README.md +++ b/src/fetch/README.md @@ -16,6 +16,7 @@ The fetch tool will truncate the response, but by using the `start_index` argume - `max_length` (integer, optional): Maximum number of characters to return (default: 5000) - `start_index` (integer, optional): Start content from this character index (default: 0) - `raw` (boolean, optional): Get raw content without markdown conversion (default: false) + - `timeout` (number, optional): Request timeout in seconds for this fetch; overrides the server default (default: 30) ### Prompts @@ -170,6 +171,12 @@ This can be customized by adding the argument `--user-agent=YourUserAgent` to th The server can be configured to use a proxy by using the `--proxy-url` argument. +### Customization - Timeout + +By default, requests time out after 30 seconds. The default can be changed with the `--timeout` argument (in seconds) or +the `FETCH_TIMEOUT` environment variable, and a single request can override it with the `timeout` tool argument +(most specific wins). + ## Windows Configuration If you're experiencing timeout issues on Windows, you may need to set the `PYTHONIOENCODING` environment variable to ensure proper character encoding: diff --git a/src/fetch/src/mcp_server_fetch/__init__.py b/src/fetch/src/mcp_server_fetch/__init__.py index 32bc8bf039..ced500633f 100644 --- a/src/fetch/src/mcp_server_fetch/__init__.py +++ b/src/fetch/src/mcp_server_fetch/__init__.py @@ -18,11 +18,26 @@ def main(): ) parser.add_argument("--proxy-url", type=str, help="Proxy URL to use for requests") + def positive_timeout(value: str) -> float: + seconds = float(value) + if seconds <= 0: + raise argparse.ArgumentTypeError( + "timeout must be a positive number of seconds" + ) + return seconds + env_timeout = os.environ.get("FETCH_TIMEOUT") + try: + default_timeout = ( + positive_timeout(env_timeout) if env_timeout else DEFAULT_REQUEST_TIMEOUT + ) + except (ValueError, argparse.ArgumentTypeError) as error: + parser.error(f"invalid FETCH_TIMEOUT environment variable: {error}") + parser.add_argument( "--timeout", - type=float, - default=float(env_timeout) if env_timeout else DEFAULT_REQUEST_TIMEOUT, + type=positive_timeout, + default=default_timeout, help="Request timeout in seconds (default: 30, or the FETCH_TIMEOUT env var)", ) diff --git a/src/fetch/src/mcp_server_fetch/server.py b/src/fetch/src/mcp_server_fetch/server.py index 6a3cffadf6..7806b40f2e 100644 --- a/src/fetch/src/mcp_server_fetch/server.py +++ b/src/fetch/src/mcp_server_fetch/server.py @@ -23,10 +23,10 @@ DEFAULT_USER_AGENT_AUTONOMOUS = "ModelContextProtocol/1.0 (Autonomous; +https://github.com/modelcontextprotocol/servers)" DEFAULT_USER_AGENT_MANUAL = "ModelContextProtocol/1.0 (User-Specified; +https://github.com/modelcontextprotocol/servers)" -# Default per-request timeout, in seconds, for outbound HTTP requests. Matches -# httpx's timeout unit and the value previously hardcoded here. Overridable via -# the serve() timeout argument, the --timeout CLI flag, the FETCH_TIMEOUT -# environment variable, or the per-request "timeout" tool argument. +# Default per-request timeout, in seconds, for the outbound content fetch. +# Matches httpx's timeout unit and the value previously hardcoded here. +# Overridable via the serve() timeout argument, the --timeout CLI flag, the +# FETCH_TIMEOUT environment variable, or the per-request "timeout" tool argument. DEFAULT_REQUEST_TIMEOUT = 30.0 @@ -69,12 +69,7 @@ def get_robots_txt_url(url: str) -> str: return robots_url -async def check_may_autonomously_fetch_url( - url: str, - user_agent: str, - proxy_url: str | None = None, - timeout: float = DEFAULT_REQUEST_TIMEOUT, -) -> None: +async def check_may_autonomously_fetch_url(url: str, user_agent: str, proxy_url: str | None = None) -> None: """ Check if the URL can be fetched by the user agent according to the robots.txt file. Raises a McpError if not. @@ -89,7 +84,6 @@ async def check_may_autonomously_fetch_url( robot_txt_url, follow_redirects=True, headers={"User-Agent": user_agent}, - timeout=timeout, ) except HTTPError: raise McpError(ErrorData( @@ -214,8 +208,8 @@ async def serve( custom_user_agent: Optional custom User-Agent string to use for requests ignore_robots_txt: Whether to ignore robots.txt restrictions proxy_url: Optional proxy URL to use for requests - timeout: Default request timeout in seconds, used when a fetch call does - not specify its own timeout + timeout: Default request timeout in seconds for the content fetch, used + when a fetch call does not specify its own timeout """ server = Server("mcp-fetch") user_agent_autonomous = custom_user_agent or DEFAULT_USER_AGENT_AUTONOMOUS @@ -261,9 +255,7 @@ async def call_tool(name, arguments: dict) -> list[TextContent]: request_timeout = args.timeout if args.timeout is not None else timeout if not ignore_robots_txt: - await check_may_autonomously_fetch_url( - url, user_agent_autonomous, proxy_url, timeout=request_timeout - ) + await check_may_autonomously_fetch_url(url, user_agent_autonomous, proxy_url) content, prefix = await fetch_url( url, diff --git a/src/fetch/tests/test_server.py b/src/fetch/tests/test_server.py index b0173628d1..ef491caf32 100644 --- a/src/fetch/tests/test_server.py +++ b/src/fetch/tests/test_server.py @@ -1,9 +1,13 @@ """Tests for the fetch MCP server.""" +import os +import sys + import pytest from unittest.mock import AsyncMock, patch, MagicMock from mcp.shared.exceptions import McpError +from mcp_server_fetch import main from mcp_server_fetch.server import ( extract_content_from_html, get_robots_txt_url, @@ -368,3 +372,41 @@ async def test_fetch_uses_custom_timeout(self): ) assert mock_client.get.call_args.kwargs["timeout"] == 60.0 + + +class TestServerTimeoutConfiguration: + """Tests for timeout configuration via the CLI flag and environment variable.""" + + def _serve_timeout_for(self, argv, env=None): + """Run main() with the given argv/env and return the timeout passed to serve().""" + import mcp_server_fetch + + with patch.object(mcp_server_fetch, "serve", new=AsyncMock()) as mock_serve, \ + patch.dict(os.environ, env or {}, clear=False), \ + patch.object(sys, "argv", argv): + if not (env and "FETCH_TIMEOUT" in env): + os.environ.pop("FETCH_TIMEOUT", None) + main() + + # serve(custom_user_agent, ignore_robots_txt, proxy_url, timeout) + return mock_serve.call_args.args[3] + + def test_default_timeout_when_unset(self): + """The default timeout is used when neither flag nor env var is set.""" + assert self._serve_timeout_for(["mcp-server-fetch"]) == DEFAULT_REQUEST_TIMEOUT + + def test_timeout_from_cli_flag(self): + """The --timeout flag sets the server default.""" + assert self._serve_timeout_for(["mcp-server-fetch", "--timeout", "45"]) == 45.0 + + def test_timeout_from_env_var(self): + """FETCH_TIMEOUT sets the server default when no flag is given.""" + assert self._serve_timeout_for( + ["mcp-server-fetch"], {"FETCH_TIMEOUT": "50"} + ) == 50.0 + + def test_cli_flag_overrides_env_var(self): + """The --timeout flag takes precedence over FETCH_TIMEOUT.""" + assert self._serve_timeout_for( + ["mcp-server-fetch", "--timeout", "45"], {"FETCH_TIMEOUT": "50"} + ) == 45.0