diff --git a/README.md b/README.md index 5f9baa780e..89721eea52 100644 --- a/README.md +++ b/README.md @@ -859,12 +859,14 @@ The following sets of tools are available: - **list_issue_types** - List available issue types - **Required OAuth Scopes**: `read:org` - **Accepted OAuth Scopes**: `admin:org`, `read:org`, `write:org` + - `fields`: Subset of issue type fields to return for each issue type. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `owner`: The organization owner of the repository (string, required) - **list_issues** - List issues - **Required OAuth Scopes**: `repo` - `after`: Cursor for pagination. Use the endCursor from the previous page's PageInfo for GraphQL APIs. (string, optional) - `direction`: Order direction. If provided, the 'orderBy' also needs to be provided. (string, optional) + - `fields`: Subset of issue fields to return for each issue. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `labels`: Filter by labels (string[], optional) - `orderBy`: Order issues by field. If provided, the 'direction' also needs to be provided. (string, optional) - `owner`: Repository owner (string, required) @@ -875,6 +877,7 @@ The following sets of tools are available: - **search_issues** - Search issues - **Required OAuth Scopes**: `repo` + - `fields`: Subset of issue fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `order`: Sort order (string, optional) - `owner`: Optional repository owner. If provided with repo, only issues for this repository are listed. (string, optional) - `page`: Page number for pagination (min 1) (number, optional) @@ -1074,6 +1077,7 @@ The following sets of tools are available: - **Required OAuth Scopes**: `repo` - `base`: Filter by base branch (string, optional) - `direction`: Sort direction (string, optional) + - `fields`: Subset of pull request fields to return for each pull request. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `head`: Filter by head user/org and branch (string, optional) - `owner`: Repository owner (string, required) - `page`: Page number for pagination (min 1) (number, optional) @@ -1123,6 +1127,7 @@ The following sets of tools are available: - **search_pull_requests** - Search pull requests - **Required OAuth Scopes**: `repo` + - `fields`: Subset of pull request fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `order`: Sort order (string, optional) - `owner`: Optional repository owner. If provided with repo, only pull requests for this repository are listed. (string, optional) - `page`: Page number for pagination (min 1) (number, optional) @@ -1232,6 +1237,7 @@ The following sets of tools are available: - **list_branches** - List branches - **Required OAuth Scopes**: `repo` + - `fields`: Subset of branch fields to return for each branch. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `owner`: Repository owner (string, required) - `page`: Page number for pagination (min 1) (number, optional) - `perPage`: Results per page for pagination (min 1, max 100) (number, optional) @@ -1240,6 +1246,7 @@ The following sets of tools are available: - **list_commits** - List commits - **Required OAuth Scopes**: `repo` - `author`: Author username or email address to filter commits by (string, optional) + - `fields`: Subset of commit fields to return for each commit. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `owner`: Repository owner (string, required) - `page`: Page number for pagination (min 1) (number, optional) - `path`: Only commits containing this file path will be returned (string, optional) @@ -1251,6 +1258,7 @@ The following sets of tools are available: - **list_releases** - List releases - **Required OAuth Scopes**: `repo` + - `fields`: Subset of release fields to return for each release. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `owner`: Repository owner (string, required) - `page`: Page number for pagination (min 1) (number, optional) - `perPage`: Results per page for pagination (min 1, max 100) (number, optional) @@ -1258,6 +1266,7 @@ The following sets of tools are available: - **list_tags** - List tags - **Required OAuth Scopes**: `repo` + - `fields`: Subset of tag fields to return for each tag. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields. (string[], optional) - `owner`: Repository owner (string, required) - `page`: Page number for pagination (min 1) (number, optional) - `perPage`: Results per page for pagination (min 1, max 100) (number, optional) diff --git a/evals/.gitignore b/evals/.gitignore new file mode 100644 index 0000000000..20d931c542 --- /dev/null +++ b/evals/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +out/ +.env +.env.local +# Captured fixtures contain real (large) GitHub data; keep only the example committed. +fixtures/*.json +!fixtures/example_*.json diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000000..ccc303cc94 --- /dev/null +++ b/evals/README.md @@ -0,0 +1,206 @@ +# Phase 1 evals: context cost vs. benefit of output schemas + field filtering + +Small, deterministic scripts to get the numbers we care about, compared across +**three shippable configurations**: + +| Scenario | output schema | `fields` param | what it represents | +|----------|:---:|:---:|--------------------| +| **S1 baseline** | ✗ | ✗ | today's behavior (no experiment) | +| **S2 schema+fields** | ✓ | ✓ | the full experiment | +| **S3 fields-only** | ✗ | ✓ | hypothesized sweet spot: the model can filter without carrying the heavy schema | + +The intuition behind **S3**: the model doesn't need the full output schema to +filter — it just needs the `fields` param (whose enum already tells it which +fields exist). So S3 may capture almost all of the benefit at a fraction of the +fixed cost. + +The two numbers we derive: + +1. **Fixed tax** — extra tokens added to the `tools/list` payload (paid once at + client init) by each scenario. +2. **Per-call savings** — tokens saved when the model filters a tool response to + a subset of fields. + +From those: **break-even calls = fixed_tax / avg_savings_per_call**, computed per +scenario. + +No LLM required for the offline numbers (tokenization falls back to a chars/4 +proxy if `tiktoken` isn't installed). The online A/B (step 4) runs the three +scenarios through a real model over realistic multi-tool sessions. + +## Setup + +```bash +cd evals +python3 -m venv .venv && source .venv/bin/activate +pip install -r requirements.txt # tiktoken (+ anthropic for the online A/B) +``` + +### Token & secrets + +Live tool calls and the online A/B need a real GitHub token. **Never hardcode or +commit it.** Provide it via the environment only; `.env*` is gitignored: + +```bash +export GITHUB_PERSONAL_ACCESS_TOKEN=ghp_xxx # read-only scope is enough +``` + +A dummy token is used automatically for `capture_tools.py` (it never calls the API). + + +## 1) Fixed tax + +Capture the tool list WITH the experiment enabled, then let the analyzer derive +all three scenarios by stripping `outputSchema` and/or the `fields` property: + +```bash +python3 capture_tools.py --features output_schemas --toolsets all \ + --out out/tools.treatment.json + +python3 fixed_tax.py --tools out/tools.treatment.json --json-out out/fixed_tax.json +# add --approx if offline without tiktoken +``` + +`fixed_tax.py` prints the payload tokens for each scenario (S1/S2/S3), the fixed +tax of each vs the S1 baseline, a component breakdown (schema vs `fields`), and a +per-tool breakdown. `--json-out` writes the per-scenario taxes for step 3. + +> Tip: measure with the `--toolsets` you'd actually ship. The tax is fixed in +> absolute tokens but its *relative* size shrinks the more tools you expose. + +## 2) Per-call savings (real data) + +Capture real full vs filtered responses for the affected tools, straight from +live GitHub (read-only), then token-diff them: + +```bash +export GITHUB_PERSONAL_ACCESS_TOKEN=ghp_xxx +python3 capture_fixtures.py --owner github --repo github-mcp-server --org github +python3 response_savings.py --fixed-tax-json out/fixed_tax.json +``` + +`capture_fixtures.py` calls each tool twice (no `fields` vs a small subset) and +writes `fixtures/.full.json` / `.filtered.json`. Use a busy repo for a +realistic upper bound, **and** a small repo for the lower bound — report a range. +An example pair is included so the script also runs offline. + +## 3) Break-even (per scenario) + +`response_savings.py --fixed-tax-json out/fixed_tax.json` prints break-even for +both taxed scenarios: + +``` +break_even_calls = scenario_fixed_tax / avg_saved_per_call +``` + +Interpretation: +- A session with **more** filtered list/search calls than `break_even_calls` is + net-positive on context for that scenario. +- **S3 (fields-only)** has a far smaller tax than S2, so its break-even is tiny — + this is the configuration to scrutinize first. +- Short sessions (few tool calls) are where the fixed tax dominates — call this + out in the writeup. + +## 4) Online A/B (Phase 2 — real multi-tool sessions, all 3 scenarios) + +Runs the same tasks through a real model across all three scenarios, measuring +cumulative prompt tokens. This is the only way to confirm the model actually +*uses* `fields` and to get the true net effect — including whether S3 really is +the sweet spot. + +**Use a model with a real context window.** The harness talks to any +**OpenAI-compatible** endpoint, so you don't need a paid third-party key: + +- **GitHub Models** (default) — authenticated with your GitHub token, no extra + key. Convenient, but the free tier caps requests at **16,000 tokens**, so large + unfiltered responses error out (`413`). Fine for a smoke test; **not** for the + headline numbers. +- **A Copilot / internal proxy** — point `--base-url` at any OpenAI-compatible + endpoint you already have access to and pass its token via `--api-key-env`. This + is how to run a large-context model (e.g. `claude-opus-4-6`) with no request cap + and no out-of-pocket billing. + +```bash +export GITHUB_PERSONAL_ACCESS_TOKEN=ghp_xxx # always: the MCP server uses this + +# Smoke test: GitHub Models gpt-5 (16k cap — expect overflow failures on big repos) +python3 schema_fields_eval.py --model openai/gpt-5 --toolsets issues,pull_requests --repeat 3 + +# Recommended: a large-context model via your OpenAI-compatible endpoint +export COPILOT_TOKEN=... # whatever token that endpoint needs +python3 schema_fields_eval.py \ + --base-url https://your-openai-compatible-endpoint/v1 \ + --api-key-env COPILOT_TOKEN \ + --model claude-opus-4-6 --toolsets issues,pull_requests --repeat 3 + +# --base-url any OpenAI-compatible endpoint +# --api-key-env VAR env var holding that endpoint's token +# --repo owner/repo target repo for the tasks (default cli/cli; see below) +# --tasks-file mytasks.txt one task per line, optionally 'tagtask' +``` + +> Target a readable repo: the tasks run against a **live** repo, so point `--repo` +> at a large, **public, SAML-free** repo a plain PAT can read (default `cli/cli`). +> If you aim at a SAML-protected org repo (e.g. `github/github-mcp-server`), every +> call 403s, the model only ever sees tiny error payloads, and the `fields` arms +> look like pure overhead because there's nothing to filter — the experiment then +> measures only the fixed schema/param tax, not the filtering payoff. Such runs +> are now flagged as failures (a tool returning `isError`) and excluded from the +> token comparison rather than silently counted as valid. + +The server is booted with `--features output_schemas` so the **S2** arm has a real +schema to embed; the `fields` param and server-side filtering are present in every +arm regardless, so only what each arm shows the *model* differs. + +It prints, per scenario: cumulative prompt/completion tokens, tool-call counts, +`fields` adoption, the net delta vs the S1 baseline, and a **per-task-type +breakdown** (narrow / full / neutral) so you can see *where* each config helps. +Only task-runs where all three arms succeeded count toward the token comparison +(so on the capped GitHub Models endpoint, the biggest filtering wins — tasks where +the unfiltered baseline overflowed — show up in the failure counts, not the token +table; another reason to use a large-context endpoint). Use `--repeat >= 3` to +average out model nondeterminism. Per-run detail is written to +`out/schema_fields_eval.jsonl`. + +> Task design matters: the default tasks are intentionally **neutral** (they do +> not tell the model to "return only X"). Biasing prompts toward terse answers +> would inflate the filtering arms. Keep a balanced mix of narrow/full/neutral. + +> Cost control: the default toolsets are narrow on purpose. The relevant +> differences live in the affected tools, so you don't need all 79 tools loaded +> each turn. Use `fixed_tax.py` (all toolsets) for the init-tax number and the +> online run for the savings/net dynamic. + + + +## Honesty notes + +- Tokenizers differ across providers; report **deltas** and state the tokenizer. +- Step 2 assumes the model actually uses `fields`. That adoption rate can only be + confirmed by the Phase 2 online A/B — Phase 1 is an upper bound on benefit. +- Real response sizes vary a lot by repo; capture fixtures from both a small and a + large/busy repo and report a range, not a single number. +- The `fields` param and the server-side filtering are **not** gated by the + `output_schemas` feature flag in the server — only the `outputSchema` and the + response's `structuredContent` are. So S1 (baseline) here means "pre-experiment + main", and "flag off" in production today would still ship the `fields` param. + Reconcile the scenario you measure with the toggle you'd actually ship. +- With output schemas on, each tool result also carries a `structuredContent` + duplicate of the payload. The online A/B forwards only the text content to the + model (so all arms see identical response bytes); a client that also feeds + `structuredContent` to the model would pay more in the S2 arm. State this + assumption when you report. + +## Files + +| File | Purpose | +|------|---------| +| `capture_tools.py` | Boot server over stdio, dump `tools/list` result | +| `fixed_tax.py` | Per-scenario token-diff (S1/S2/S3); `--json-out` for break-even | +| `capture_fixtures.py` | Capture real full/filtered tool responses (live GitHub) | +| `response_savings.py` | Token-diff full vs filtered responses; per-scenario break-even | +| `schema_fields_eval.py` | 3-scenario (A/B/C) multi-tool agent eval, prompt-token accounting | +| `_mcp_client.py` | Shared MCP stdio client | +| `_tokenize.py` | Tokenizer helper (tiktoken or chars/4 fallback) | +| `fixtures/` | Response pairs (example + captured) | + diff --git a/evals/_mcp_client.py b/evals/_mcp_client.py new file mode 100644 index 0000000000..c83d3cd65e --- /dev/null +++ b/evals/_mcp_client.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Minimal MCP stdio client used by the eval scripts. + +Spawns the GitHub MCP server, performs the MCP handshake, and exposes +`list_tools()` and `call_tool()`. stdout is newline-delimited JSON-RPC; server +logs go to stderr. + +Security: never hardcode a token. The token is read from the process environment +(GITHUB_PERSONAL_ACCESS_TOKEN). A dummy non-`ghp_` token is used only as a +fallback so `tools/list` works offline without hitting GitHub. +""" + +from __future__ import annotations + +import json +import os +import select +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parent.parent +PROTOCOL_VERSION = "2025-06-18" + + +def text_content(result: dict) -> str: + """Concatenate the text parts of a tools/call result.""" + parts = [c.get("text", "") for c in result.get("content", []) if c.get("type") == "text"] + return "".join(parts) + + +class MCPServer: + def __init__( + self, + server_cmd: str = "go run ./cmd/github-mcp-server stdio", + extra_args: list[str] | None = None, + env: dict[str, str] | None = None, + cwd: Path = REPO_ROOT, + timeout: float = 180.0, + ) -> None: + self.cmd = shlex.split(server_cmd) + list(extra_args or []) + self.env = {**os.environ, **(env or {})} + self.env.setdefault("GITHUB_PERSONAL_ACCESS_TOKEN", "dummy_token_no_network") + self.cwd = str(cwd) + self.timeout = timeout + self.proc: subprocess.Popen | None = None + self._id = 0 + + def __enter__(self) -> "MCPServer": + self.start() + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + def start(self) -> None: + print(f"[mcp] starting: {' '.join(self.cmd)}", file=sys.stderr) + self.proc = subprocess.Popen( + self.cmd, + cwd=self.cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=sys.stderr, + text=True, + env=self.env, + ) + self._request( + "initialize", + { + "protocolVersion": PROTOCOL_VERSION, + "capabilities": {}, + "clientInfo": {"name": "evals", "version": "0"}, + }, + ) + self._notify("notifications/initialized") + + # -- JSON-RPC plumbing ------------------------------------------------- + def _send(self, payload: dict) -> None: + assert self.proc and self.proc.stdin + self.proc.stdin.write(json.dumps(payload) + "\n") + self.proc.stdin.flush() + + def _notify(self, method: str, params: dict | None = None) -> None: + self._send({"jsonrpc": "2.0", "method": method, "params": params or {}}) + + def _read(self) -> dict: + assert self.proc and self.proc.stdout + while True: + ready, _, _ = select.select([self.proc.stdout], [], [], self.timeout) + if not ready: + raise TimeoutError("timed out waiting for server (see stderr above)") + line = self.proc.stdout.readline() + if line == "": + raise EOFError("server closed stdout unexpectedly") + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue # ignore stray non-JSON output + + def _request(self, method: str, params: dict) -> dict: + self._id += 1 + req_id = self._id + self._send({"jsonrpc": "2.0", "id": req_id, "method": method, "params": params}) + while True: + msg = self._read() + if msg.get("id") == req_id: + if "error" in msg: + raise RuntimeError(f"{method} error: {msg['error']}") + return msg["result"] + + # -- High-level API ---------------------------------------------------- + def list_tools(self) -> list[dict]: + tools: list[dict] = [] + cursor = None + while True: + params = {} if cursor is None else {"cursor": cursor} + result = self._request("tools/list", params) + tools.extend(result.get("tools", [])) + cursor = result.get("nextCursor") + if not cursor: + return tools + + def call_tool(self, name: str, arguments: dict) -> dict: + return self._request("tools/call", {"name": name, "arguments": arguments}) + + def close(self) -> None: + if not self.proc: + return + try: + if self.proc.stdin: + self.proc.stdin.close() + except Exception: # noqa: BLE001 + pass + self.proc.terminate() + try: + self.proc.wait(timeout=10) + except Exception: # noqa: BLE001 + self.proc.kill() + self.proc = None diff --git a/evals/_tokenize.py b/evals/_tokenize.py new file mode 100644 index 0000000000..320d4c6078 --- /dev/null +++ b/evals/_tokenize.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Shared helpers for the eval scripts: tokenization that works offline. + +Prefers tiktoken (o200k_base, a good proxy for modern GPT/Claude tokenizers). +Falls back to a chars/4 heuristic if tiktoken is unavailable or --approx is set. +The break-even analysis only needs *relative* deltas, so the proxy is fine as +long as the same tokenizer is used for both arms. +""" + +from __future__ import annotations + +import json +from typing import Any + +_ENCODER = None +_MODE = None + + +def get_tokenizer(approx: bool = False): + """Return (count_fn, mode_label). count_fn(text:str) -> int.""" + global _ENCODER, _MODE + if approx: + return (lambda s: max(1, len(s) // 4), "approx(chars/4)") + if _ENCODER is None: + try: + import tiktoken # type: ignore + + _ENCODER = tiktoken.get_encoding("o200k_base") + _MODE = "tiktoken(o200k_base)" + except Exception: # noqa: BLE001 - any failure -> fall back + _ENCODER = False + _MODE = "approx(chars/4)" + if _ENCODER is False: + return (lambda s: max(1, len(s) // 4), _MODE) + return (lambda s: len(_ENCODER.encode(s)), _MODE) + + +def to_wire(obj: Any) -> str: + """Serialize like a compact JSON-RPC payload on the wire. + + Whitespace/key-order are normalized so both arms are compared identically. + """ + return json.dumps(obj, separators=(",", ":"), ensure_ascii=False, sort_keys=True) + + +def count_obj_tokens(obj: Any, approx: bool = False) -> int: + count, _ = get_tokenizer(approx) + return count(to_wire(obj)) diff --git a/evals/capture_fixtures.py b/evals/capture_fixtures.py new file mode 100644 index 0000000000..c535215ca9 --- /dev/null +++ b/evals/capture_fixtures.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Capture REAL tool-response payloads (full vs field-filtered) for the affected +tools, writing them as fixtures consumed by response_savings.py. + +Requires a real token in the environment (read-only is enough): + export GITHUB_PERSONAL_ACCESS_TOKEN=ghp_xxx # do NOT commit this + python3 capture_fixtures.py --owner github --repo github-mcp-server --org github + +Each scenario is run twice: once with no `fields` (all fields) and once asking for +a small subset, mirroring how the model would call it. The server does the actual +filtering, so these are genuine payloads. +""" + +from __future__ import annotations + +import argparse +import os +import sys +from pathlib import Path + +from _mcp_client import MCPServer, text_content + +FIXTURES = Path(__file__).parent / "fixtures" + + +def scenarios(owner: str, repo: str, org: str, query_issue: str, query_pr: str): + """Return list of (name, tool, full_args, filtered_args).""" + issue_fields = ["number", "title"] + return [ + ( + "list_issue_types", + "list_issue_types", + {"owner": org}, + {"owner": org, "fields": ["name"]}, + ), + ( + "list_issues", + "list_issues", + {"owner": owner, "repo": repo}, + {"owner": owner, "repo": repo, "fields": issue_fields}, + ), + ( + "search_issues", + "search_issues", + {"query": query_issue}, + {"query": query_issue, "fields": issue_fields}, + ), + ( + "list_pull_requests", + "list_pull_requests", + {"owner": owner, "repo": repo}, + {"owner": owner, "repo": repo, "fields": issue_fields}, + ), + ( + "search_pull_requests", + "search_pull_requests", + {"query": query_pr}, + {"query": query_pr, "fields": issue_fields}, + ), + ] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--owner", default="github") + parser.add_argument("--repo", default="github-mcp-server") + parser.add_argument("--org", default="github", help="Org for list_issue_types") + parser.add_argument("--issue-query", default="repo:github/github-mcp-server lockdown") + parser.add_argument("--pr-query", default="repo:github/github-mcp-server is:open") + parser.add_argument("--server-cmd", default="go run ./cmd/github-mcp-server stdio") + args = parser.parse_args() + + if not os.environ.get("GITHUB_PERSONAL_ACCESS_TOKEN"): + print( + "ERROR: set GITHUB_PERSONAL_ACCESS_TOKEN in the environment " + "(real token needed for live tool calls).", + file=sys.stderr, + ) + return 2 + + FIXTURES.mkdir(parents=True, exist_ok=True) + # Read-only server; tools we use are all reads. + with MCPServer(server_cmd=args.server_cmd, extra_args=["--toolsets", "all", "--read-only"]) as server: + for name, tool, full_args, filt_args in scenarios( + args.owner, args.repo, args.org, args.issue_query, args.pr_query + ): + full = text_content(server.call_tool(tool, full_args)) + filtered = text_content(server.call_tool(tool, filt_args)) + (FIXTURES / f"{name}.full.json").write_text(full) + (FIXTURES / f"{name}.filtered.json").write_text(filtered) + print( + f"[fixtures] {name}: full={len(full)}B filtered={len(filtered)}B", + file=sys.stderr, + ) + + print("[fixtures] done. Now run: python3 response_savings.py", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/capture_tools.py b/evals/capture_tools.py new file mode 100644 index 0000000000..cde9ab9ae2 --- /dev/null +++ b/evals/capture_tools.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Boot the GitHub MCP server over stdio, do the MCP handshake, and dump the +`tools/list` result to a JSON file. + +This captures exactly what an MCP client receives at initialization, so it is the +ground truth for the "fixed tax" measurement (see fixed_tax.py). + +Example (WITH output schemas + the new `fields` params): + python3 capture_tools.py --features output_schemas --toolsets all \ + --out out/tools.treatment.json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from _mcp_client import MCPServer + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", required=True, help="Output JSON file path") + parser.add_argument("--features", default="", help="Comma-separated feature flags") + parser.add_argument("--toolsets", default="all", help="Comma-separated toolsets / 'all'") + parser.add_argument("--read-only", action="store_true") + parser.add_argument("--server-cmd", default="go run ./cmd/github-mcp-server stdio") + parser.add_argument("--timeout", type=float, default=180.0) + args = parser.parse_args() + + extra = ["--toolsets", args.toolsets] + if args.features: + extra += ["--features", args.features] + if args.read_only: + extra += ["--read-only"] + + with MCPServer(server_cmd=args.server_cmd, extra_args=extra, timeout=args.timeout) as server: + tools = server.list_tools() + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + json.dumps( + { + "config": { + "features": args.features, + "toolsets": args.toolsets, + "read_only": args.read_only, + }, + "tool_count": len(tools), + "tools": tools, + }, + indent=2, + ) + ) + have_schema = sum(1 for t in tools if "outputSchema" in t) + have_fields = sum( + 1 for t in tools if "fields" in (t.get("inputSchema", {}).get("properties", {}) or {}) + ) + print( + f"[capture] wrote {len(tools)} tools -> {out_path} " + f"({have_schema} with outputSchema, {have_fields} with a `fields` param)", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/fixed_tax.py b/evals/fixed_tax.py new file mode 100644 index 0000000000..8ae163d8a8 --- /dev/null +++ b/evals/fixed_tax.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Compute the "fixed tax": how many extra tokens the output schemas and the new +`fields` params add to the `tools/list` payload a client receives at init. + +Reports the three shippable configurations measured by the online A/B, so the +offline init-cost numbers line up with the online session numbers: + + * S1 baseline -- no output schema, no `fields` param + * S2 schema+fields -- output schema AND `fields` param (full experiment) + * S3 fields-only -- `fields` param but NO output schema (hypothesized sweet spot) + +Takes a single capture produced WITH the experiment on (output schemas enabled), +then derives the other configurations by stripping `outputSchema` and/or the +`fields` property from each tool's `inputSchema`. + +Usage: + python3 fixed_tax.py --tools out/tools.treatment.json + python3 fixed_tax.py --tools out/tools.treatment.json --approx # offline + python3 fixed_tax.py --tools out/tools.treatment.json --json-out out/fixed_tax.json +""" + +from __future__ import annotations + +import argparse +import copy +import json +from pathlib import Path + +from _tokenize import count_obj_tokens, get_tokenizer + + +def strip_output_schema(tool: dict) -> dict: + t = copy.deepcopy(tool) + t.pop("outputSchema", None) + return t + + +def strip_fields_param(tool: dict) -> dict: + t = copy.deepcopy(tool) + props = t.get("inputSchema", {}).get("properties") + if props and "fields" in props: + del props["fields"] + req = t.get("inputSchema", {}).get("required") + if isinstance(req, list) and "fields" in req: + req.remove("fields") + return t + + +def strip_both(tool: dict) -> dict: + return strip_fields_param(strip_output_schema(tool)) + + +def pct(part: int, whole: int) -> str: + return f"{(100.0 * part / whole):.2f}%" if whole else "n/a" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tools", required=True, help="Capture JSON (experiment ON)") + parser.add_argument("--approx", action="store_true", help="Force chars/4 proxy") + parser.add_argument( + "--json-out", + help="Optional path to write the three scenario taxes as JSON " + "(consumed by response_savings.py --fixed-tax-json for break-even).", + ) + args = parser.parse_args() + + data = json.loads(Path(args.tools).read_text()) + tools = data["tools"] + _, mode = get_tokenizer(args.approx) + + full = [count_obj_tokens(t, args.approx) for t in tools] + no_schema = [count_obj_tokens(strip_output_schema(t), args.approx) for t in tools] + no_fields = [count_obj_tokens(strip_fields_param(t), args.approx) for t in tools] + baseline = [count_obj_tokens(strip_both(t), args.approx) for t in tools] + + sum_full = sum(full) + sum_no_schema = sum(no_schema) + sum_no_fields = sum(no_fields) + sum_baseline = sum(baseline) + + schema_cost = sum_full - sum_no_schema + fields_cost = sum_full - sum_no_fields + total_tax = sum_full - sum_baseline + + # Per-scenario payload sizes and their fixed tax vs the S1 baseline. + # S1 baseline = strip both (sum_baseline) + # S2 schema+fields = full (sum_full) + # S3 fields-only = strip output schema (sum_no_schema) + s2_tax = sum_full - sum_baseline + s3_tax = sum_no_schema - sum_baseline + + print(f"tokenizer: {mode}") + print(f"tools in payload: {len(tools)}") + print(f" with outputSchema: {sum(1 for t in tools if 'outputSchema' in t)}") + print( + f" with `fields` param: " + f"{sum(1 for t in tools if 'fields' in (t.get('inputSchema', {}).get('properties', {}) or {}))}" + ) + print() + print("PAYLOAD TOKENS PER SCENARIO (whole tools/list)") + print(f" S1 baseline (no schema/no fields): {sum_baseline}") + print(f" S2 schema+fields (full): {sum_full}") + print(f" S3 fields-only (no schema): {sum_no_schema}") + print() + print("FIXED TAX vs S1 baseline (extra tokens paid once at init)") + print(f" S2 schema+fields: +{s2_tax:>6} ({pct(s2_tax, sum_baseline)} of baseline)") + print(f" S3 fields-only: +{s3_tax:>6} ({pct(s3_tax, sum_baseline)} of baseline)") + print() + print("FIXED TAX BY COMPONENT") + print(f" output schemas: +{schema_cost:>6} ({pct(schema_cost, sum_baseline)} of baseline)") + print(f" `fields` params: +{fields_cost:>6} ({pct(fields_cost, sum_baseline)} of baseline)") + print(f" combined: +{total_tax:>6} ({pct(total_tax, sum_baseline)} of baseline)") + print() + + # Per-tool breakdown, only for tools that changed. + rows = [] + for i, t in enumerate(tools): + delta = full[i] - baseline[i] + if delta: + rows.append((t.get("name", "?"), full[i] - no_schema[i], full[i] - no_fields[i], delta)) + rows.sort(key=lambda r: r[3], reverse=True) + if rows: + print("PER-TOOL ADDED TOKENS") + print(f" {'tool':<24}{'schema':>8}{'fields':>8}{'total':>8}") + for name, sc, fl, tot in rows: + print(f" {name:<24}{sc:>8}{fl:>8}{tot:>8}") + + if args.json_out: + out = { + "tokenizer": mode, + "tool_count": len(tools), + "scenarios": { + "baseline": {"payload_tokens": sum_baseline, "fixed_tax": 0}, + "schema_fields": {"payload_tokens": sum_full, "fixed_tax": s2_tax}, + "fields_only": {"payload_tokens": sum_no_schema, "fixed_tax": s3_tax}, + }, + } + out_path = Path(args.json_out) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out, indent=2)) + print(f"\nwrote scenario taxes -> {out_path}") + + print() + print( + "Next: run response_savings.py (optionally --fixed-tax-json out/fixed_tax.json)\n" + "to get avg tokens saved per filtered call and the per-scenario break-even." + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/fixtures/example_issue_types.filtered.json b/evals/fixtures/example_issue_types.filtered.json new file mode 100644 index 0000000000..24c29bf81b --- /dev/null +++ b/evals/fixtures/example_issue_types.filtered.json @@ -0,0 +1 @@ +{"issue_types":[{"name":"Bug"},{"name":"Feature"},{"name":"Task"}]} diff --git a/evals/fixtures/example_issue_types.full.json b/evals/fixtures/example_issue_types.full.json new file mode 100644 index 0000000000..cce3d01228 --- /dev/null +++ b/evals/fixtures/example_issue_types.full.json @@ -0,0 +1 @@ +{"issue_types":[{"id":1,"node_id":"IT_kwDOABCD01","name":"Bug","description":"Something isn't working as expected","color":"RED","created_at":"2024-01-10T12:00:00Z","updated_at":"2024-03-02T09:30:00Z"},{"id":2,"node_id":"IT_kwDOABCD02","name":"Feature","description":"A new feature or enhancement request","color":"GREEN","created_at":"2024-01-10T12:00:00Z","updated_at":"2024-03-02T09:30:00Z"},{"id":3,"node_id":"IT_kwDOABCD03","name":"Task","description":"A chore or piece of work to be done","color":"BLUE","created_at":"2024-01-10T12:00:00Z","updated_at":"2024-03-02T09:30:00Z"}]} diff --git a/evals/requirements.txt b/evals/requirements.txt new file mode 100644 index 0000000000..c481504499 --- /dev/null +++ b/evals/requirements.txt @@ -0,0 +1,8 @@ +# Optional: accurate tokenization. If not installed (or offline), the scripts +# fall back to a chars/4 approximation. Install with: +# python3 -m venv .venv && source .venv/bin/activate && pip install -r requirements.txt +tiktoken>=0.7.0 +# Online harness (Phase 2) only. Used by schema_fields_eval.py against any +# OpenAI-compatible endpoint (GitHub Models with your GitHub token, or a +# Copilot / internal proxy via --base-url + --api-key-env). No third-party key. +openai>=1.30.0 diff --git a/evals/response_savings.py b/evals/response_savings.py new file mode 100644 index 0000000000..514cf4fc4d --- /dev/null +++ b/evals/response_savings.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Measure tokens saved per tool call when the model requests a subset of fields. + +Pairs of fixture files represent the *text* payload a tool returns: + fixtures/.full.json # response with no `fields` filter (all fields) + fixtures/.filtered.json # response when the model asked for a few fields + +For each pair it prints tokens(full), tokens(filtered) and the savings, plus an +overall average used for the break-even calculation. + +Usage: + python3 response_savings.py # uses ./fixtures + python3 response_savings.py --dir fixtures --approx + +Capture real fixtures with the bundled mcpcurl, e.g.: + ./mcpcurl --stdio-server-cmd "go run ./cmd/github-mcp-server stdio" \ + tools list_issue_types --owner github > fixtures/list_issue_types.full.json + ... and again with --fields name for the filtered version. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from _tokenize import get_tokenizer + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dir", default=str(Path(__file__).parent / "fixtures")) + parser.add_argument("--approx", action="store_true", help="Force chars/4 proxy") + parser.add_argument( + "--fixed-tax-json", + help="Optional fixed_tax.py --json-out file; prints per-scenario break-even.", + ) + args = parser.parse_args() + + count, mode = get_tokenizer(args.approx) + fixtures_dir = Path(args.dir) + fulls = sorted(fixtures_dir.glob("*.full.json")) + if not fulls: + print(f"No *.full.json fixtures found in {fixtures_dir}") + return 1 + + print(f"tokenizer: {mode}\n") + print(f" {'scenario':<28}{'full':>8}{'filtered':>10}{'saved':>8}{'saved%':>8}") + total_saved = 0 + pairs = 0 + for full_path in fulls: + name = full_path.name[: -len(".full.json")] + filtered_path = fixtures_dir / f"{name}.filtered.json" + if not filtered_path.exists(): + print(f" {name:<28} (missing {filtered_path.name}, skipped)") + continue + full_tokens = count(full_path.read_text()) + filtered_tokens = count(filtered_path.read_text()) + saved = full_tokens - filtered_tokens + savedpct = f"{(100.0 * saved / full_tokens):.1f}%" if full_tokens else "n/a" + print(f" {name:<28}{full_tokens:>8}{filtered_tokens:>10}{saved:>8}{savedpct:>8}") + total_saved += saved + pairs += 1 + + if not pairs: + return 0 + + avg = total_saved / pairs + print() + print(f"avg saved per filtered call: {avg:.0f} tokens (over {pairs} scenarios)") + + if args.fixed_tax_json: + data = json.loads(Path(args.fixed_tax_json).read_text()) + scenarios = data.get("scenarios", {}) + print() + print("BREAK-EVEN (filtered list/search calls needed to repay the init tax)") + print(f" {'scenario':<20}{'fixed tax':>11}{'break-even calls':>18}") + # Show the two configurations that actually carry a tax. + for key, label in (("fields_only", "S3 fields-only"), ("schema_fields", "S2 schema+fields")): + tax = scenarios.get(key, {}).get("fixed_tax") + if tax is None: + continue + be = (tax / avg) if avg else float("inf") + print(f" {label:<20}{tax:>11}{be:>18.2f}") + print( + " (A session with more filtered calls than the break-even is net-positive " + "on context for that scenario.)" + ) + else: + print("break-even calls = fixed_tax / avg_saved_per_call") + print(" (pass --fixed-tax-json out/fixed_tax.json to compute it per scenario)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/evals/schema_fields_eval.py b/evals/schema_fields_eval.py new file mode 100644 index 0000000000..7dc53503f3 --- /dev/null +++ b/evals/schema_fields_eval.py @@ -0,0 +1,753 @@ +#!/usr/bin/env python3 +"""Phase 2 online eval: measure real context (prompt token) usage over multi-tool +agent sessions across THREE shippable tool-definition configurations (an A/B/C +test of the schema/fields knobs). + +Uses GitHub Models (OpenAI-compatible) so it authenticates with your GitHub +token -- no third-party API key, no separate billing. Point `--base-url` at any +other OpenAI-compatible endpoint if you have an internal one. + +IMPORTANT -- target repo: the tasks hit a live repo, so point them at a large, +public, SAML-free repo (default: cli/cli) that a plain PAT can read. If you aim +at a SAML-protected org repo, every call 403s, the model only ever sees tiny +error payloads, and the `fields` arms look like pure overhead because there is +nothing to filter. Override with --repo / --org. + +All three arms run the SAME tasks against the SAME live MCP server. The only +difference is the tool definitions presented to the model: + + * S1 baseline -- no output schema, no `fields` param. The model cannot + filter and always receives full responses. (today's behavior) + * S2 schema+fields -- output schema folded into the description AND the `fields` + param exposed. The model can filter, and pays the full + documented fixed tax in context. + * S3 fields-only -- the `fields` param exposed but NO output schema. The + hypothesized sweet spot: the model still knows it can + filter (and which fields exist, via the param's enum) but + doesn't carry the heavy schema text. + +The server attaches the `fields` param unconditionally and performs the filtering +regardless of any feature flag, so every arm gets correct server behavior; we only +vary what each arm shows the MODEL. The server is booted WITH output schemas +enabled so the S2 arm actually has a schema to embed (and pays its real tax). + +Headline metric: cumulative `usage.prompt_tokens` across every turn of a session +(= total context the model had to read). We also track completion tokens, tool +calls, and how often the model actually used `fields`. Use `--repeat` to average +out model nondeterminism, and read the per-task-type breakdown to see WHERE each +configuration helps or hurts. + +Requirements: + pip install -r requirements.txt # openai + export GITHUB_PERSONAL_ACCESS_TOKEN=ghp_... # real token; do NOT commit + +Run (GitHub Models gpt-5; the free tier caps requests at 16k -- fine for a smoke +test, but big unfiltered responses will error): + python3 schema_fields_eval.py --model openai/gpt-5 --toolsets issues,pull_requests --repeat 3 + +For a large-context model with no request cap, point --base-url at any +OpenAI-compatible endpoint you already have access to. The Copilot API +(https://api.githubcopilot.com) works out of the box for Copilot users -- the +short-lived Copilot token is minted automatically from your GitHub token, no +out-of-pocket third-party key: + # discover the exact model id first: + python3 schema_fields_eval.py --base-url https://api.githubcopilot.com --list-models + # then run: + python3 schema_fields_eval.py --base-url https://api.githubcopilot.com \ + --model claude-opus-4 --toolsets issues,pull_requests --repeat 3 +""" + +from __future__ import annotations + +import argparse +import copy +import json +import os +import sys +from pathlib import Path + +from _mcp_client import MCPServer, text_content + +DEFAULT_BASE_URL = "https://models.github.ai/inference" +DEFAULT_MODEL = "openai/gpt-5" + +# Default target: a large, public, SAML-free repo so a plain PAT gets real, +# sizeable responses to filter. Pick a repo busy enough that unfiltered payloads +# are genuinely big -- that's what reveals the filtering payoff. Override with +# --repo / --org. (Do NOT point at a SAML-protected org repo like +# github/github-mcp-server: a PAT 403s, every task returns a tiny error, and the +# `fields` arms then look like pure overhead because there's nothing to trim.) +DEFAULT_REPO = "cli/cli" + +# Balanced, intentionally NEUTRAL task set, templated on {repo}/{org} so the same +# tasks can target any repo. We deliberately do NOT instruct the model to "return +# only X" -- that would bias it toward using `fields` and inflate the treatment +# arms. Instead we mix tasks whose faithful answer needs just a field or two with +# tasks that need full objects, and let the model's own filtering decisions be +# what we measure. Each task is tagged with the kind of answer it implies (used +# only for the per-type breakdown; never sent to the model): +# narrow -> a faithful answer needs just a field or two (filtering should help) +# full -> a faithful answer needs rich fields/bodies (filtering shouldn't help) +# neutral -> genuinely ambiguous; the model decides +TASK_TEMPLATES: list[tuple[str, str]] = [ + ("narrow", "How many open issues are there in {repo}, and what are their titles?"), + ("narrow", "What are the numbers of the currently open pull requests in {repo}?"), + ("narrow", "What issue types are configured for the {org} organization?"), + ("narrow", "What are the names of the branches in {repo}?"), + ("narrow", "List the tag names in {repo}."), + ("narrow", "What are the tag names of the most recent releases in {repo}?"), + ("full", "Summarize what the most recently updated open issue in {repo} is about."), + ("full", "Look at the open pull requests in {repo} and tell me which one looks most substantial, and why."), + ("full", "Find issues mentioning 'lockdown' in {repo} and explain what they're asking for."), + ("full", "Summarize what changed in the most recent release of {repo}."), + ("full", "Summarize the last few commits on the default branch of {repo} and what they changed."), + ("neutral", "Give me an overview of recent activity in {repo}'s open issues."), + ("neutral", "Search for open pull requests about tests in {repo} and tell me what they do."), + ("neutral", "Give me an overview of recent commit activity in {repo}."), +] + + +def build_tasks(repo: str, org: str) -> list[tuple[str, str]]: + """Render the templated task set for a concrete repo/org.""" + return [(tag, text.format(repo=repo, org=org)) for tag, text in TASK_TEMPLATES] + +SYSTEM_PROMPT = ( + "You are an assistant with access to GitHub tools. Use the tools to answer the " + "user's request, then stop when you can answer." +) + +# The three shippable configurations under test, each defined by two independent +# knobs: +# keep_fields -- expose the `fields` param so the model CAN filter responses +# embed_schema -- fold the tool's output schema into its description (the +# documented fixed tax). Only has an effect when the server was +# booted with output schemas enabled so the schema is present. +ARMS: dict[str, dict[str, bool]] = { + "baseline": {"keep_fields": False, "embed_schema": False}, + "fields_only": {"keep_fields": True, "embed_schema": False}, + "schema_fields": {"keep_fields": True, "embed_schema": True}, +} +# Presentation order: cheapest fixed tax -> most expensive, so deltas read naturally. +ARM_ORDER = ["baseline", "fields_only", "schema_fields"] +SCENARIO_LABEL = { + "baseline": "S1 no-schema/no-fields", + "fields_only": "S3 fields-only", + "schema_fields": "S2 schema+fields", +} + + +def mcp_tool_to_openai(tool: dict, *, keep_fields: bool, embed_output_schema: bool) -> dict: + schema = copy.deepcopy(tool.get("inputSchema", {"type": "object"})) + if not keep_fields: + props = schema.get("properties") + if props and "fields" in props: + del props["fields"] + req = schema.get("required") + if isinstance(req, list) and "fields" in req: + req.remove("fields") + description = tool.get("description", "") + if embed_output_schema and "outputSchema" in tool: + description += "\n\nReturns (output schema): " + json.dumps( + tool["outputSchema"], separators=(",", ":") + ) + return { + "type": "function", + "function": { + "name": tool["name"], + "description": description, + "parameters": schema, + }, + } + + +def build_toolset(tools: list[dict], *, arm: str) -> list[dict]: + cfg = ARMS[arm] + return [ + mcp_tool_to_openai( + t, keep_fields=cfg["keep_fields"], embed_output_schema=cfg["embed_schema"] + ) + for t in tools + ] + + +def run_task(client, server: MCPServer, openai_tools: list[dict], task: str, *, model: str, max_turns: int, allow_fields: bool) -> dict: + messages = [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": task}, + ] + prompt_tokens = completion_tokens = turns = tool_calls = fields_calls = 0 + tool_errors = 0 + final_text = "" + error = "" + + for _ in range(max_turns): + try: + resp = client.chat.completions.create( + model=model, + messages=messages, + tools=openai_tools, + tool_choice="auto", + ) + except Exception as exc: # noqa: BLE001 - record (e.g. context/rate limit) and stop + error = f"{type(exc).__name__}: {str(exc)[:200]}" + break + turns += 1 + if resp.usage: + prompt_tokens += resp.usage.prompt_tokens + completion_tokens += resp.usage.completion_tokens + + msg = resp.choices[0].message + if not msg.tool_calls: + final_text = msg.content or "" + break + + messages.append( + { + "role": "assistant", + "content": msg.content, + "tool_calls": [ + { + "id": tc.id, + "type": "function", + "function": {"name": tc.function.name, "arguments": tc.function.arguments}, + } + for tc in msg.tool_calls + ], + } + ) + for tc in msg.tool_calls: + tool_calls += 1 + try: + args = json.loads(tc.function.arguments or "{}") + except json.JSONDecodeError: + args = {} + if not isinstance(args, dict): + args = {} + # In the baseline arm the model has no `fields` param; defensively drop + # it in case the model invents one, so baseline always gets full responses. + if not allow_fields: + args.pop("fields", None) + elif "fields" in args: + fields_calls += 1 + try: + result = server.call_tool(tc.function.name, args) + content = text_content(result) or json.dumps(result) + # An `isError` result (e.g. 403/SAML, not-found, rate limit) means + # the model never saw a real payload to filter. Record it so this + # task-run is excluded from the apples-to-apples token comparison + # instead of silently counting as a success. + if result.get("isError"): + tool_errors += 1 + if not error: + error = "tool_error: " + (content or "isError")[:200] + except Exception as exc: # noqa: BLE001 + tool_errors += 1 + if not error: + error = f"{type(exc).__name__}: {str(exc)[:200]}" + content = f"ERROR: {exc}" + messages.append({"role": "tool", "tool_call_id": tc.id, "content": content}) + + return { + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "turns": turns, + "tool_calls": tool_calls, + "fields_calls": fields_calls, + "tool_errors": tool_errors, + "final_text": final_text, + "error": error, + } + + +def summarize(records: list[dict], *, model: str, base_url: str, out_path: Path) -> None: + """Print a 3-scenario comparison plus a per-task-type breakdown. + + A task-run only counts toward the token comparison if ALL three arms + succeeded for it, so every comparison is apples-to-apples. + """ + from collections import defaultdict + + by_key: dict[tuple, dict] = defaultdict(dict) + for r in records: + by_key[(r["run"], r["task"])][r["arm"]] = r + + keys = list(by_key) + valid = [ + k for k in keys + if all(a in by_key[k] and not by_key[k][a]["error"] for a in ARM_ORDER) + ] + + print("\n=== 3-SCENARIO ONLINE A/B (cumulative prompt tokens) ===") + print(f"model: {model} @ {base_url}") + print(f"task-runs: {len(keys)} valid (all 3 arms ok): {len(valid)}") + for arm in ARM_ORDER: + errs = sum(1 for k in keys if arm not in by_key[k] or by_key[k][arm]["error"]) + print(f" failures[{SCENARIO_LABEL[arm]:<22}]: {errs}") + print( + " NOTE: a failure means an arm errored on a task-run (a tool call returned " + "isError -- e.g. 403/SAML/not-found/rate-limit -- or an unfiltered response " + "overflowed the model's input limit). Such runs are excluded from the token " + "comparison below so it stays apples-to-apples." + ) + + if not valid: + print("\nNo task-runs where all three arms succeeded -- nothing to compare.") + print(f"per-run JSONL: {out_path}") + return + + def arm_prompt(arm: str) -> int: + return sum(by_key[k][arm]["prompt_tokens"] for k in valid) + + base_tot = arm_prompt("baseline") + print(f"\nVALID COMPARISON over {len(valid)} task-runs (lower prompt tokens = better):") + print(f" {'scenario':<24}{'prompt_tok':>11}{'Δ vs S1':>10}{'Δ%':>8}{'cheaper':>10}{'fields_use':>12}") + for arm in ARM_ORDER: + tot = arm_prompt(arm) + delta = tot - base_tot + pct = (100.0 * delta / base_tot) if base_tot else 0.0 + cheaper = sum( + 1 for k in valid + if by_key[k][arm]["prompt_tokens"] < by_key[k]["baseline"]["prompt_tokens"] + ) + fc = sum(by_key[k][arm]["fields_calls"] for k in valid) + tc = sum(by_key[k][arm]["tool_calls"] for k in valid) + sign = "+" if delta >= 0 else "" + print( + f" {SCENARIO_LABEL[arm]:<24}{tot:>11}{sign + str(delta):>10}" + f"{sign + f'{pct:.1f}':>8}{f'{cheaper}/{len(valid)}':>10}{f'{fc}/{tc}':>12}" + ) + + # Where does the benefit live? Mean prompt tokens per task-run, by task type. + tags = sorted({by_key[k]["baseline"]["tag"] for k in valid}) + print("\nBY TASK TYPE (mean prompt tokens per task-run):") + header = f" {'tag':<10}" + "".join(f"{SCENARIO_LABEL[a]:>24}" for a in ARM_ORDER) + print(header) + for tag in tags: + tag_keys = [k for k in valid if by_key[k]["baseline"]["tag"] == tag] + cells = "".join( + f"{sum(by_key[k][arm]['prompt_tokens'] for k in tag_keys) / len(tag_keys):>24.0f}" + for arm in ARM_ORDER + ) + print(f" {tag:<10}{cells}") + + # ---- Per-task analytics -------------------------------------------------- + # The cumulative table above is dominated by a few heavy tasks (e.g. listing + # hundreds of issues), which can hide the experience on small tasks a user may + # run far more often. Here we average every task across its runs and also give + # each task equal weight, so one huge high-savings task can't overshadow the + # rest. + from statistics import mean, median + + per_task: dict[str, list] = defaultdict(list) + task_tag: dict[str, str] = {} + for k in valid: + task = by_key[k]["baseline"]["task"] + per_task[task].append(k) + task_tag[task] = by_key[k]["baseline"]["tag"] + + def task_arm_mean(task: str, arm: str) -> float: + ks = per_task[task] + return sum(by_key[k][arm]["prompt_tokens"] for k in ks) / len(ks) + + print("\nPER-TASK (mean prompt tokens per run; Δ% vs S1, negative = cheaper):") + print(f" {'tag':<8}{'S1':>9}{'S3':>9}{'S2':>9}{'S3 Δ%':>8}{'S2 Δ%':>8} task") + # Heaviest S1 tasks first, so big and small tasks are both visible. + for task in sorted(per_task, key=lambda t: task_arm_mean(t, "baseline"), reverse=True): + s1 = task_arm_mean(task, "baseline") + s3 = task_arm_mean(task, "fields_only") + s2 = task_arm_mean(task, "schema_fields") + s3p = 100.0 * (s3 - s1) / s1 if s1 else 0.0 + s2p = 100.0 * (s2 - s1) / s1 if s1 else 0.0 + print( + f" {task_tag[task]:<8}{s1:>9.0f}{s3:>9.0f}{s2:>9.0f}" + f"{s3p:>+8.1f}{s2p:>+8.1f} {task[:60]}" + ) + + # Equal-weight view: each task counts once, so a single huge task can't dominate. + print("\nEQUAL-WEIGHT ACROSS TASKS (each task counts once, regardless of size):") + print(f" {'scenario':<24}{'mean/task':>10}{'median/task':>13}{'mean Δ% vs S1':>15}") + for arm in ARM_ORDER: + per_task_means = [task_arm_mean(t, arm) for t in per_task] + eq_mean = mean(per_task_means) + eq_med = median(per_task_means) + pct = [ + 100.0 * (task_arm_mean(t, arm) - task_arm_mean(t, "baseline")) / task_arm_mean(t, "baseline") + for t in per_task + if task_arm_mean(t, "baseline") + ] + eq_pct = mean(pct) if pct else 0.0 + print(f" {SCENARIO_LABEL[arm]:<24}{eq_mean:>10.0f}{eq_med:>13.0f}{eq_pct:>+15.1f}") + print( + " (Unlike the cumulative table, here every task -- including cheap,\n" + " frequently-run ones -- counts equally, better reflecting a typical mix.)" + ) + + print(f"\nper-run JSONL: {out_path}") + + +def parse_tasks_file(path: str, *, repo: str, org: str) -> list[tuple[str, str]]: + """Read a tasks file: one task per line, optionally 'tagtask text'. + + Task text may use {repo}/{org} placeholders, rendered with the run's target. + """ + tasks: list[tuple[str, str]] = [] + for line in Path(path).read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + if "\t" in line: + tag, text = line.split("\t", 1) + tasks.append((tag.strip() or "neutral", text.strip().format(repo=repo, org=org))) + else: + tasks.append(("neutral", line.format(repo=repo, org=org))) + return tasks + + +def get_copilot_token(github_token: str) -> str: + """Return a short-lived Copilot API bearer token (no out-of-pocket key). + + The Copilot API needs a token minted from a Copilot-entitled GitHub OAuth + token -- a plain PAT can't mint one (the mint endpoint 404s). We obtain the + OAuth token, in priority order, from: + 1. $COPILOT_OAUTH_TOKEN (a gho_... token you already have), or + 2. the editor's stored token (~/.config/github-copilot/apps.json|hosts.json), or + 3. a cached token from a previous device login, or + 4. an interactive GitHub device login (one-time; cached afterwards). + Then it exchanges that for the short-lived bearer the Copilot API accepts. + """ + import urllib.error + + oauth = _copilot_oauth_token() + try: + return _mint_copilot_bearer(oauth) + except urllib.error.HTTPError as exc: + if exc.code in (401, 404) and _OAUTH_CACHE.exists(): + # Cached OAuth token likely expired/revoked -- re-login once. + _OAUTH_CACHE.unlink(missing_ok=True) + oauth = _device_login() + _save_cached_oauth(oauth) + return _mint_copilot_bearer(oauth) + if exc.code == 404: + raise RuntimeError( + "Copilot token mint returned 404. The OAuth token used isn't " + "Copilot-entitled. Sign in via the device flow (rerun and follow the " + "prompt) or set COPILOT_OAUTH_TOKEN to a gho_ token from your editor." + ) from exc + raise RuntimeError(f"Copilot token mint failed: HTTP {exc.code}") from exc + + +# GitHub Copilot (editor) OAuth app client id -- the same public id editors use to +# authenticate Copilot. Used only for the device-login flow below. +_COPILOT_OAUTH_CLIENT_ID = "Iv1.b507a08c87ecfe98" +_OAUTH_CACHE = Path.home() / ".config" / "github-mcp-evals" / "copilot_oauth.json" + + +def _mint_copilot_bearer(oauth_token: str) -> str: + import urllib.request + + req = urllib.request.Request( + "https://api.github.com/copilot_internal/v2/token", + headers={ + "Authorization": f"token {oauth_token}", + "Accept": "application/json", + "Editor-Version": "GitHubMCPServerEvals/1.0", + "User-Agent": "GitHubMCPServerEvals/1.0", + }, + ) + with urllib.request.urlopen(req) as resp: # noqa: S310 - fixed, trusted URL + data = json.loads(resp.read().decode()) + token = data.get("token") + if not token: + raise RuntimeError(f"unexpected Copilot token response keys: {sorted(data)}") + return token + + +def _copilot_oauth_token() -> str: + """Resolve a Copilot-entitled GitHub OAuth (gho_) token; device-login if needed.""" + env = os.environ.get("COPILOT_OAUTH_TOKEN") + if env: + return env + editor = _find_editor_oauth_token() + if editor: + return editor + cached = _load_cached_oauth() + if cached: + return cached + token = _device_login() + _save_cached_oauth(token) + return token + + +def _find_editor_oauth_token() -> str | None: + base = Path(os.environ.get("XDG_CONFIG_HOME") or (Path.home() / ".config")) / "github-copilot" + for name in ("apps.json", "hosts.json"): + path = base / name + if not path.exists(): + continue + try: + data = json.loads(path.read_text()) + except Exception: # noqa: BLE001 + continue + for key, val in data.items(): + if "github.com" in key and isinstance(val, dict) and val.get("oauth_token"): + return val["oauth_token"] + return None + + +def _load_cached_oauth() -> str | None: + if not _OAUTH_CACHE.exists(): + return None + try: + return json.loads(_OAUTH_CACHE.read_text()).get("oauth_token") + except Exception: # noqa: BLE001 + return None + + +def _save_cached_oauth(token: str) -> None: + _OAUTH_CACHE.parent.mkdir(parents=True, exist_ok=True) + _OAUTH_CACHE.write_text(json.dumps({"oauth_token": token})) + try: + os.chmod(_OAUTH_CACHE, 0o600) + except OSError: + pass + + +def _device_login() -> str: + """Interactive GitHub device-login for the Copilot OAuth app; returns gho_ token.""" + import time + import urllib.parse + import urllib.request + + def post(url: str, data: dict) -> dict: + body = urllib.parse.urlencode(data).encode() + req = urllib.request.Request( + url, + data=body, + headers={"Accept": "application/json", "User-Agent": "GitHubMCPServerEvals/1.0"}, + ) + with urllib.request.urlopen(req) as resp: # noqa: S310 - fixed, trusted URLs + return json.loads(resp.read().decode()) + + dc = post( + "https://github.com/login/device/code", + {"client_id": _COPILOT_OAUTH_CLIENT_ID, "scope": "read:user"}, + ) + print( + f"\n[copilot auth] Open {dc['verification_uri']} and enter code: " + f"{dc['user_code']}\n[copilot auth] Waiting for authorization...", + file=sys.stderr, + ) + interval = int(dc.get("interval", 5)) + deadline = time.time() + int(dc.get("expires_in", 900)) + while time.time() < deadline: + time.sleep(interval) + res = post( + "https://github.com/login/oauth/access_token", + { + "client_id": _COPILOT_OAUTH_CLIENT_ID, + "device_code": dc["device_code"], + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + }, + ) + if res.get("access_token"): + print("[copilot auth] Authorized.", file=sys.stderr) + return res["access_token"] + err = res.get("error") + if err == "authorization_pending": + continue + if err == "slow_down": + interval += 5 + continue + raise RuntimeError(f"device login failed: {res}") + raise RuntimeError("device login timed out before authorization") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", + default=DEFAULT_MODEL, + help="Any model the endpoint serves, e.g. openai/gpt-5 or claude-opus-4-6.", + ) + parser.add_argument( + "--base-url", + default=DEFAULT_BASE_URL, + help="OpenAI-compatible endpoint. Defaults to GitHub Models; point it at a " + "Copilot / internal proxy for a large-context model with no request cap.", + ) + parser.add_argument( + "--api-key-env", + default=None, + help="Env var holding the API key/token for the endpoint. If unset, tries " + "GITHUB_MODELS_TOKEN, GITHUB_COPILOT_TOKEN, OPENAI_API_KEY, GITHUB_TOKEN, " + "then GITHUB_PERSONAL_ACCESS_TOKEN.", + ) + parser.add_argument("--toolsets", default="issues,pull_requests,repos") + parser.add_argument("--max-turns", type=int, default=8) + parser.add_argument( + "--repo", + default=DEFAULT_REPO, + help="Target owner/repo for the default tasks and {repo} placeholders. Use a " + "large, public, SAML-free repo so a plain PAT gets real, sizeable responses " + f"to filter. Default: {DEFAULT_REPO}.", + ) + parser.add_argument( + "--org", + default=None, + help="Organization for the issue-types task and {org} placeholders. Defaults " + "to the owner of --repo.", + ) + parser.add_argument( + "--repeat", + type=int, + default=1, + help="Run every task in every arm this many times to average out model " + "nondeterminism. Use >=3 for a result you'd present.", + ) + parser.add_argument( + "--tasks-file", + help="Optional tasks file: one task per line, optionally prefixed with " + "'tag' where tag is narrow|full|neutral.", + ) + parser.add_argument("--out", default="out/schema_fields_eval.jsonl") + parser.add_argument("--server-cmd", default="go run ./cmd/github-mcp-server stdio") + parser.add_argument( + "--list-models", + action="store_true", + help="List the models the endpoint exposes (handy to find the exact " + "Copilot model id) and exit.", + ) + parser.add_argument( + "--copilot-integration-id", + default="vscode-chat", + help="Copilot-Integration-Id header; only sent to the Copilot endpoint.", + ) + parser.add_argument( + "--editor-version", + default="GitHubMCPServerEvals/1.0", + help="Editor-Version header; only sent to the Copilot endpoint.", + ) + args = parser.parse_args() + + gh_token = os.environ.get("GITHUB_PERSONAL_ACCESS_TOKEN") + if not gh_token and not args.list_models: + print("ERROR: set GITHUB_PERSONAL_ACCESS_TOKEN (real token for the MCP server).", file=sys.stderr) + return 2 + try: + from openai import OpenAI # type: ignore + except ImportError: + print("ERROR: pip install openai", file=sys.stderr) + return 2 + + # The Copilot API is OpenAI-compatible but needs a short-lived token (minted + # from your GitHub token) plus integration headers -- no out-of-pocket key. + is_copilot = "githubcopilot.com" in args.base_url + + def build_client(): + headers: dict[str, str] = {} + if args.api_key_env: + key = os.environ.get(args.api_key_env) + if not key: + raise RuntimeError(f"{args.api_key_env} is not set") + elif is_copilot: + key = get_copilot_token(gh_token) + else: + key = ( + os.environ.get("GITHUB_MODELS_TOKEN") + or os.environ.get("GITHUB_COPILOT_TOKEN") + or os.environ.get("OPENAI_API_KEY") + or os.environ.get("GITHUB_TOKEN") + or gh_token + ) + if is_copilot: + headers = { + "Copilot-Integration-Id": args.copilot_integration_id, + "Editor-Version": args.editor_version, + } + return OpenAI(base_url=args.base_url, api_key=key, default_headers=headers or None) + + try: + client = build_client() + except Exception as exc: # noqa: BLE001 + print(f"ERROR: failed to build model client: {exc}", file=sys.stderr) + return 2 + + if args.list_models: + try: + models = client.models.list() + ids = sorted(getattr(m, "id", str(m)) for m in models.data) + except Exception as exc: # noqa: BLE001 + print(f"ERROR: failed to list models: {exc}", file=sys.stderr) + return 2 + print(f"Models available at {args.base_url}:") + for mid in ids: + print(f" {mid}") + return 0 + + org = args.org or args.repo.split("/", 1)[0] + tasks = ( + parse_tasks_file(args.tasks_file, repo=args.repo, org=org) + if args.tasks_file + else build_tasks(args.repo, org) + ) + + out_path = Path(args.out) + out_path.parent.mkdir(parents=True, exist_ok=True) + + records: list[dict] = [] + # Boot the server WITH output schemas enabled so the `schema_fields` arm has a + # real outputSchema to embed (and thus pays its real fixed tax). The server + # exposes the `fields` param and filters regardless of the flag, so all three + # arms see correct server behavior; only the tool defs shown to the model vary. + with MCPServer( + server_cmd=args.server_cmd, + extra_args=["--toolsets", args.toolsets, "--read-only", "--features", "output_schemas"], + ) as server: + mcp_tools = server.list_tools() + have_schema = sum(1 for t in mcp_tools if "outputSchema" in t) + if have_schema == 0: + print( + "WARNING: no tool reported an outputSchema, so the schema_fields arm " + "is identical to fields_only. Is `--features output_schemas` wired up " + "in the server build?", + file=sys.stderr, + ) + arms = {arm: build_toolset(mcp_tools, arm=arm) for arm in ARM_ORDER} + + with out_path.open("w") as fh: + for run_idx in range(args.repeat): + for tag, task in tasks: + # Copilot tokens are short-lived; mint a fresh one per task so a + # long --repeat run doesn't fail midway with an expired token. + if is_copilot: + try: + client = build_client() + except Exception as exc: # noqa: BLE001 + print(f"WARNING: Copilot token refresh failed: {exc}", file=sys.stderr) + for arm in ARM_ORDER: + m = run_task( + client, + server, + arms[arm], + task, + model=args.model, + max_turns=args.max_turns, + allow_fields=ARMS[arm]["keep_fields"], + ) + m.update({"arm": arm, "tag": tag, "task": task, "run": run_idx}) + records.append(m) + fh.write(json.dumps(m) + "\n") + print( + f"[{SCENARIO_LABEL[arm]:>22}] run{run_idx} " + f"prompt={m['prompt_tokens']:>7} calls={m['tool_calls']} " + f"fields={m['fields_calls']} " + f"{'ERR:' + m['error'] if m['error'] else ''} :: {task[:46]}", + file=sys.stderr, + ) + + summarize(records, model=args.model, base_url=args.base_url, out_path=out_path) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/out/online_ab.jsonl b/out/online_ab.jsonl new file mode 100644 index 0000000000..a6a00b2ea2 --- /dev/null +++ b/out/online_ab.jsonl @@ -0,0 +1,72 @@ +{"prompt_tokens": 1715, "completion_tokens": 105, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 0} +{"prompt_tokens": 13161, "completion_tokens": 6068, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "There are 156 open issues. Their titles are:\n\n- Add support for MCP OAuth for Visual Studio (GHE MCP)\n- list_discussions and get_discussion should include upvoteCount\n- Skill catalog: missing find-prs and find-issues skills\n- issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned\n- Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column)\n- Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability\n- The badge URL points to the CI workflow status of `modelcontextprotocol/registry`, not this repository. If the intent is to show this repo\u2019s CI status, update the badge and link to this repo\u2019s workflow instead of an external repository.\n- Insiders Feedback: [bug] MCP app getting 422 error and create pull request UI stays while PR is silently created\n- bug: `get_me` tool fails without parameters due to empty JSON object `{}` not being accepted as valid input\n- discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp)\n- Add file mode (executable bit) support to `push_files`\n- Insiders Feedback: No types\n- Add installation and configuration documentation for Antigravity CLI\n- `pull_request_review_write` combines create/submit/delete into one tool, making fine-grained permissions by method impossible\n- Clarify that pull_request_review_write create without event creates a pending review\n- Add support for creating and updating repository security advisories\n- Feature: allow review comments on any file line, not just lines within the diff\n- Intermittent commit/push failures when using GitHub MCP connector for file operations and branch management, via Grok official Connector\n- Add update_pull_request_comment and delete_pull_request_comment tools\n- Add support for raw GraphQL queries\n- Interest check: GHES Manage API toolset?\n- Schema quality: 20 required parameters missing descriptions, 92 optional parameters undescribed\n- List user's recent activities\n- feat: add labels support to create_pull_request tool\n- Add bulk/batch issue label operations\n- Add auto-merge support to pull request tools\n- issue_write body parameter overwrites issue content on update \u2014 easily confused with adding a comment\n- Add addBlockedBy / removeBlockedBy mutations (native issue dependencies) to MCP server\n- get_job_logs: run_id with failed_only=false should return all job logs; failed_only should be a consistent modifier\n- Hosted MCP `get_check_runs` fails with 403 where the equivalent REST call succeeds\n- docs: Add `go install` as a first-class installation method\n- Tool request: download workflow run artifact contents\n- \"Application Error\" upon selecting the \"Install in Insiders\" link\n- Feature request: batch-fetch merged PR metadata for release notes / changelog generation\n- Feature request: batch-fetch pull request review comment threads for an explicit PR list\n- Feature Request: `get_agent_card` tool \u2014 load ALP-compatible agents directly from GitHub repos\n- bug: merge_pull_request ignores mergeMethod parameter, always attempts merge commit\n- Missing author_association field in list_issues and pull request responses\n- get_diff / get_files: JSON serialization inflates diff payload far beyond token limits\n- Support dynamic per-session provider tokens for the local GitHub MCP server\n- Copilot-generated PR review code suggestions (\"Suggest\" changesets) are not returned by pull_request_read or any API\n- Suggestion: add a recommended security policy / rate limiting guide\n- Hide write-oriented prompts and instructions when write tools are filtered\n- Classify secondary and abuse rate limits separately from generic GitHub failures\n- http: classify missing token, invalid token, and insufficient scope distinctly\n- Invalidate token-scope cache context when authorization token changes\n- GitHub MCP issue read appears to drop code block text in angle brackets\n- Inventory: return stable hidden-tool reason codes across filter gates\n- Add read-only regression coverage for dynamic toolsets and deprecated aliases\n- push_files takes ~70s per call (both success and fast-fail cases)\n- Security policy template for GitHub MCP server\n- Fail closed in strict mode on unknown toolset names\n- get_copilot_space tool does not return attached file contents from Copilot Spaces\n- Add deterministic toolset resolution ordering for default and custom toolsets\n- Enforce fail-closed startup when PAT/OAuth scope requirements are unmet\n- Add deterministic CI state classifier for GitHub MCP server PR triage\n- Feature: Auto-select token based on repository owner (multi-account support)\n- No built in safeguard against getting rate limited\n- Create repository from template\n- Feature: compare_file_contents tool with semantic diffs\n- Feature Request: Add get_user tool to fetch a user profile by username\n- Github MCP tools for Runners\n- Add multi-account support for client-side account switching\n- Tool name `get_file_contents` conflicts with Claude Desktop built-in tool\n- Add `minimal_output` parameter to list_pull_requests and search_pull_requests\n- Reply tool calls with structuredContent\n- Feature Request: Add support for GitHub Security APIs (Dependabot, Code Scanning, Secret Scanning)\n- Add organization member and repository discovery tools\n- Add compare_commits tool for branch/commit diff analysis\n- Add release management operations to releases toolset\n- Add retry logic with jitter for sub_issue_write to handle parallel calls\n- `get_copilot_job_status` fails with JSON unmarshal error for `response_status_code`\n- Consider workflow scope or user-escalation pattern for merging PRs that modify workflow files\n- Feature Request: `get_labels` or `get_all_labels`\n- Add setting to restrict actions to the current repo\n- Restrict MCP access to private repositories\n- Addressing GitHub MCP's Key Issues: Token Scopes, Verbose Prompts, and Lack of Project Context\n- feat: Issue Preview with manual edit in the chat view before publish\n- Add project ( add_project tool )\n- google/go-licenses currently not maintained\n- Toolset not listing for Github MCP Server | Unstable behaviour of test chat pannel in copilot studio\n- Mutliple github accounts with .ssh/config workaround. Owner & repo cannot be extracted\n- Add `add_repository_collaborator` tool for managing repository access\n- perf: integrate go-sdk schema caching optimization (Phase 1)\n- Add create_discussion tool to discussions toolset\n- Add issue_graph tool to visualize issue/PR relationships and hierarchy\n- list_copilot_spaces / get_copilot_space tools don't return any results in Android Studio\n- Query on availability of Delete Branch tool\n- Proxima pull request action links to wrong url\n- Consider shipping MCPB bundle\n- VSCode MCP Registry integration does not support latest MCP registry\n- list_project_fields fails with JSON unmarshal error for single_select fields with options\n- VSCode: Toolset name mismatch: 'github/*' fails with Unknown tool, but 'github' warns it's deprecated\n- Assign custom agent to issue via assign_copilot_to_issue\n- Github MCP tool doesnt work with Gemini CLI in codespaces\n- Github MCP Tools don't work with Gemini 2.5 models familly and langchain/deepagents javascript library\n- Unable to read the contents of a file, but can write and edit when used in ChatGPT.\n- When using the 'Get Copilot Space' tool, files are found but content cannot be read\n- When using 'Get Copilot Space', files are not found from space for uploaded files\n- GitHub MCP Server Tools Not Working in Copilot Spaces When Used in VS Code\n- Support GitHub App authentication with client credentials\n- Get PR timeline details\n- Add tools for listing organization members and outside collaborators\n- No projects v2 access using GitHub remote MCP server in ChatGPT\n- Docker MCP Toolkit OAuth integration returns 403 on write operations\n- Add optional \"jq\" field to every MCP tool\n- Static MCP Registry question\n- Set a title in `serverInfo.title` for nicer display\n- Copilot Spaces Tool\n- 32600 error code when trying to connect to mcp with openai responses API\n- How to access EMU organization repositories using custom OAuth app credentials with the remote GitHub MCP server?\n- Add GitHub Packages Support to MCP Server\n- Add Claude Code Plugin / Marketplace\n- get_file_contents throws MCP error: resource not found\n- Tool for Creating and Modifying GitHub Discussions via MCP Server\n- Feature Request: Expose GitHub Project Item History/Timeline Events\n- Add support for updating pull request reviews without dismissal\n- list_issues in Continue extension in VS Code with GitHub Docker MCP\n- Resource protected by organization SAML enforcement\n- [Bug] MCP tool works and responds, but Claude Desktop pop-up shows \"Could not attach to MCP server\"\n- get_pull_request_review_comments doesn't return resolved state\n- missing required parameter: query\n- Add `update_issue_comment` tool\n- Cannot use access_token from GitHub App to call create_pull_request_with_copilot\n- Cannot configure Remote GitHub MCP Server in ChatGPT (OAuth discovery error)\n- `Get pull request comments` not fetching comments\n- Remote MCP OAuth + Gemini CLI Errors Out\n- MCP Server not returning results for Copilot when invoked fom Copilot agent\n- MCP Governance - Allow/Deny List for MCP for Organizations and Teams\n- Support sort and comment author filters for get_issue_comments\n- MCP server as part of GitHub Desktop application\n- Add CreateRelease tool for programmatic release creation\n- Control access/auth for github MCP server(remote-github hosted) using MS IDP.\n- Make it easy to create & submit PRs to open source repos\n- Add tool to get_issue_timeline\n- Add tools to get issue_events and get_issue_event\n- Support `--edit-last` functionality\n- Add ability to fetch issue relationships (parent, blocked by, etc..)\n- Error invoking MCP tool search_users (someone else reported the exact same issue)\n- VS Code Github Copilot calling create_pull_request_with_copilot on remote MCP using fine-grained PAT gets 401\n- Invalid URL protocol\n- Feature Request: Add tool to fetch job logs by job name (avoid listing all jobs)\n- Error to MCP server connection\n- Feature Request: Add support for editing issue comments\n- Enable Custom Workflow Registry for Reusable Prompt + Tool Definitions\n- Add support for list_repository_contributors\n- [FEAT] Add support for rulesets and custom properties.\n- Improving access permissions for assets in a GitHub private repository\n- Allow uploading files as attachments\n- Wiki access\n- Add pagination support for get_pull_request_diff to handle large PRs\n- allow users to create repositories from a repository template.\n- Support subdomain isolation for GHES hosts\n- Please add an optional \"Project\" field when Creating an Issue\n- Support creating repositories from repository templates\n- Non PAT auth method", "error": "", "arm": "fields_only", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 0} +{"prompt_tokens": 4878, "completion_tokens": 233, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 0} +{"prompt_tokens": 1713, "completion_tokens": 41, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 0} +{"prompt_tokens": 32910, "completion_tokens": 5225, "turns": 4, "tool_calls": 3, "fields_calls": 3, "final_text": "2673, 2672, 2670, 2667, 2666, 2665, 2663, 2659, 2658, 2656, 2655, 2652, 2650, 2639, 2637, 2634, 2633, 2632, 2631, 2630, 2629, 2627, 2626, 2625, 2624, 2623, 2622, 2620, 2604, 2601, 2599, 2591, 2588, 2586, 2579, 2577, 2572, 2569, 2562, 2482, 2468, 2465, 2447, 2443, 2428, 2416, 2413, 2408, 2406, 2382, 2374, 2340, 2294, 2285, 2279, 2265, 2261, 2246, 2230, 2220, 2214, 2198, 2193, 2184, 2183, 2129, 2118, 2105, 2086, 2083, 2076, 2038, 2031, 2010, 1984, 1983, 1982, 1981, 1980, 1974, 1971, 1959, 1956, 1941, 1928, 1927, 1915, 1867, 1846, 1843, 1837, 1836, 1825, 1800, 1797, 1794, 1734, 1700, 1678, 1677, 1661, 1579, 1574, 1564, 1560, 1542, 1527, 1525, 1519, 1512, 1511, 1508, 1491, 1490, 1406, 1363, 1345, 1339, 1338, 1283, 1226, 1209, 1197, 1190, 1149, 1139, 1130, 1098, 1095, 1087, 1072, 1037, 1015, 1004, 1002, 962, 816, 494, 211, 210", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 0} +{"prompt_tokens": 32614, "completion_tokens": 3885, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "2673, 2672, 2670, 2667, 2666, 2665, 2663, 2659, 2658, 2656, 2655, 2652, 2650, 2639, 2637, 2634, 2633, 2632, 2631, 2630, 2629, 2627, 2626, 2625, 2624, 2623, 2622, 2620, 2604, 2601, 2599, 2591, 2588, 2586, 2579, 2577, 2572, 2569, 2562, 2482, 2468, 2465, 2447, 2443, 2428, 2416, 2413, 2408, 2406, 2382, 2374, 2340, 2294, 2285, 2279, 2265, 2261, 2246, 2230, 2220, 2214, 2198, 2193, 2184, 2183, 2129, 2118, 2105, 2086, 2083, 2076, 2038, 2031, 2010, 1984, 1983, 1982, 1981, 1980, 1974, 1971, 1959, 1956, 1941, 1928, 1927, 1915, 1867, 1846, 1843, 1837, 1836, 1825, 1800, 1797, 1794, 1734, 1700, 1678, 1677, 1661, 1579, 1574, 1564, 1560, 1542, 1527, 1525, 1519, 1512, 1511, 1508, 1491, 1490, 1406, 1363, 1345, 1339, 1338, 1283, 1226, 1209, 1197, 1190, 1149, 1139, 1130, 1098, 1095, 1087, 1072, 1037, 1015, 1004, 1002, 962, 816, 494, 211, 210", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 0} +{"prompt_tokens": 1706, "completion_tokens": 290, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Could you share the GitHub organization name (owner)? I can then look up the issue types configured for that org.", "error": "", "arm": "baseline", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 0} +{"prompt_tokens": 2319, "completion_tokens": 432, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Which GitHub organization do you mean? Please provide the org handle (owner), e.g., \u201cgithub\u201d or your company\u2019s org name, and I\u2019ll list the configured issue types.", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 0} +{"prompt_tokens": 4869, "completion_tokens": 638, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Which GitHub organization should I check? Please provide the org (owner) name, for example: octo-org.\n\nOptional: If you only need certain fields (e.g., name, description, color), let me know so I can limit the response.", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 0} +{"prompt_tokens": 4193, "completion_tokens": 842, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "- Issue #2448: \u201cAdd update_pull_request_comment and delete_pull_request_comment tools\u201d\n- Problem: The MCP server currently supports replying to PR review comments but not editing or deleting them, forcing agents to post corrective replies (creating thread noise) or bypass the server with direct gh api calls.\n- Proposal: Add two tools:\n - update_pull_request_comment (PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id}) with params owner, repo, comment_id, body.\n - delete_pull_request_comment (DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id}) with params owner, repo, comment_id.\n - Constraints: Only operate on comments authored by the authenticated user; gh equivalents provided.\n- Context: A real incident highlighted confusion between discussion_rN identifiers and API comment IDs (related to #2235) and the lack of edit/delete created noisy threads.\n- Benefit: Lets agents correct or retract their own review comments and clean up mistakes programmatically, improving reliability.", "error": "", "arm": "baseline", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 0} +{"prompt_tokens": 5437, "completion_tokens": 858, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "The most recently updated open issue (#2448, \u201cAdd update_pull_request_comment and delete_pull_request_comment tools\u201d) requests adding two MCP server tools to edit and delete pull request review comments. Today, the server supports only replying to an existing review comment, so agents can\u2019t correct or retract their own comments without posting noisy follow-up replies or bypassing the MCP server via direct REST calls. The proposal is to add:\n- update_pull_request_comment (PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id}) to change a comment\u2019s body\n- delete_pull_request_comment (DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id}) to remove a comment\n\nBoth would be limited to comments authored by the authenticated user. The issue cites real-world noise from forced \u201creply\u201d workarounds and notes confusion between UI discussion identifiers and API comment IDs. The goal is cleaner threads and the ability for agents to programmatically fix their own mistakes.", "error": "", "arm": "fields_only", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 0} +{"prompt_tokens": 10534, "completion_tokens": 852, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "The latest updated open issue (#2448) requests adding two MCP server tools to manage pull request review comments: update_pull_request_comment (PATCH) and delete_pull_request_comment (DELETE). Today the server only supports add_reply_to_pull_request_comment, so agents can\u2019t edit or remove their own review comments\u2014leading to stale comments and noisy threads. The proposal maps directly to existing GitHub REST endpoints and would be restricted to modifying/deleting comments authored by the authenticated user. Additional context notes real-world cases where replies were used to correct mistakes and confusion between discussion_rN URLs and the actual API comment IDs (see related #2235).", "error": "", "arm": "schema_fields", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 0} +{"prompt_tokens": 1720, "completion_tokens": 369, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 0} +{"prompt_tokens": 2333, "completion_tokens": 352, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 0} +{"prompt_tokens": 4883, "completion_tokens": 410, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 0} +{"prompt_tokens": 9727, "completion_tokens": 1799, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here are the issues in github/github-mcp-server that mention \u201clockdown,\u201d with what they\u2019re asking for:\n\n- #2197 \u2013 Inventory: return stable hidden-tool reason codes across filter gates (open)\n - Ask: Unify and expose a single, deterministic \u201creason code\u201d whenever tools are hidden by filtering, across all filter paths (toolset, PAT scope, read-only, lockdown). Specifically includes adding a lockdown-specific code (lockdown_filtered), adding tests for each hide path, and ensuring classifications are consistent across inventory and request filtering. Scope touches internal/ghmcp/server.go, pkg/inventory/builder.go, pkg/scopes/fetcher.go, pkg/lockdown/lockdown.go.\n - Link: https://github.com/github/github-mcp-server/issues/2197\n\n- #1870 \u2013 Error: MCP error 0: lockdown cache is not configured (closed \u2013 completed)\n - Ask: Fix a bug where using the remote MCP with X-MCP-Lockdown: true fails with \u201clockdown cache is not configured\u201d when invoking GitHub tools (e.g., PR details/comments). The reporter expected either normal results or a permission error, not an internal lockdown cache error. The issue was resolved.\n - Link: https://github.com/github/github-mcp-server/issues/1870\n\n- #1714 \u2013 get_job_logs fails (closed)\n - Ask: Implement or correctly register the get_job_logs tool; the client received \u201cunknown tool\u201d for get_job_logs. Lockdown is only mentioned incidentally in the sample headers (X-MCP-Lockdown: false); the request itself is about enabling direct job log access (or clarifying supported tools).\n - Link: https://github.com/github/github-mcp-server/issues/1714\n\n- #1545 \u2013 Hello from the github-mcp-server maintainers! \u2764\ufe0f (closed \u2013 not planned)\n - Ask: Announcement and feedback thread about two initiatives (content filtering and expanding openWorldHint). It specifically calls out a planned \u201cconfigurable lockdown mode\u201d to ensure only content from users with push access is returned, referencing PR #1371. The post invites questions/feedback on the approach.\n - Link: https://github.com/github/github-mcp-server/issues/1545", "error": "", "arm": "baseline", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 0} +{"prompt_tokens": 26697, "completion_tokens": 3800, "turns": 3, "tool_calls": 3, "fields_calls": 0, "final_text": "Here are the issues in github/github-mcp-server that mention \u201clockdown,\u201d with what each is asking for:\n\n- #2197 (open) \u2014 Inventory: return stable hidden-tool reason codes across filter gates\n - Ask: Unify and expose deterministic \u201chidden tool\u201d reason codes across all filtering paths (toolset, PAT scope, read-only, lockdown), e.g., toolset_filtered, scope_filtered, readonly_filtered, lockdown_filtered. Add tests, and keep classification consistent between inventory building and request-time filtering. Scope touches pkg/lockdown/lockdown.go among others.\n\n- #1685 (open) \u2014 Restrict MCP access to private repositories\n - Ask: Feature to restrict a local MCP deployment to only specific repositories (e.g., only private org repos), ideally via flags for org and repo. A maintainer suggested \u201clockdown mode\u201d as a partial mitigation (limits returns to content from trusted contributors), but the original request is stricter, repo-level access control.\n\n- #1870 (closed, completed) \u2014 Error: MCP error 0: lockdown cache is not configured\n - Ask: Fix a bug where enabling X-MCP-Lockdown: true on the remote server caused a \u201clockdown cache is not configured\u201d error on basic read operations (e.g., PR details/comments). Expected behavior: either succeed or fail with a proper permission error, not a configuration error. Resolved.\n\n- #844 (closed, completed) \u2014 Exfiltrate information from private repositories\n - Ask: Address data exfiltration via prompt injection (from public content causing leakage from private repos). The maintainers\u2019 response lists mitigations now available:\n - Lockdown mode (helps prevent prompt injection by restricting returned content to trusted contributors)\n - Secret scanning on each tools/call to public repos\n - Read-only mode\n - OAuth 2.1 support\n - Use strict fine-grained tokens\n - They noted this was not treated as a specific vulnerability in MCP itself (no CVE) but a broader agent safety issue; ongoing mitigation work continues.\n\n- #1714 (closed) \u2014 get_job_logs fails\n - Ask: Fix \u201cunknown tool\u201d error for get_job_logs. Lockdown is only mentioned in a sample config block (X-MCP-Lockdown: false); the issue is not about lockdown itself. Resolved.\n\n- #1545 (closed, not planned) \u2014 Hello from the github-mcp-server maintainers! \u2764\ufe0f\n - Ask/Announcement: Outline security initiatives including a configurable lockdown mode (return only content from users with push access) as part of a broader content-filtering pipeline, plus expanding openWorldHint usage. Links to the lockdown PR (#1371). Closed as an announcement thread.\n\nSummary of \u201clockdown\u201d across issues:\n- Lockdown mode is a server-side safeguard that restricts tool responses to content from trusted contributors (e.g., users with push access) to reduce prompt-injection and exfiltration risks (#1545, #844).\n- There was a bug in lockdown initialization for the remote server that\u2019s now fixed (#1870).\n- They\u2019re also improving observability by exposing consistent reason codes when tools are hidden due to lockdown or other gates (#2197).\n- In one feature request, maintainers suggested lockdown as a mitigation while the user asked for stricter, repo-level access controls (#1685).", "error": "", "arm": "fields_only", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 0} +{"prompt_tokens": 18091, "completion_tokens": 2169, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here are the issues that mention \u201clockdown\u201d in github/github-mcp-server, with what each is asking for:\n\n- #2197 \u2014 Inventory: return stable hidden-tool reason codes across filter gates (open)\n Link: https://github.com/github/github-mcp-server/issues/2197\n Ask: Ensure that when tools are hidden by filtering (e.g., due to toolset, PAT scope, read-only mode, or lockdown), callers consistently receive one deterministic reason code (like toolset_filtered, scope_filtered, readonly_filtered, lockdown_filtered). Add tests for each hide path and keep classification consistent across inventory and request filtering.\n\n- #1685 \u2014 Restrict MCP access to private repositories (open)\n Link: https://github.com/github/github-mcp-server/issues/1685\n Ask: Provide a deterministic way (e.g., flags/config) to restrict a local MCP deployment to only specific repositories (by org and optionally repo), rather than relying on prompts or broad token scopes. This is related to tightening access and could align with lockdown-style controls.\n\n- #1870 \u2014 Error: MCP error 0: lockdown cache is not configured (closed \u2013 completed)\n Link: https://github.com/github/github-mcp-server/issues/1870\n Ask: Fix a bug where using the remote MCP with X-MCP-Lockdown: true causes calls like \u201cget PR details\u201d or \u201cget review comments\u201d to fail with \u201clockdown cache is not configured.\u201d Expectation: either return data (subject to permissions) or a proper permission error, not a configuration error.\n\n- #844 \u2014 Exfiltrate information from private repositories (closed \u2013 completed)\n Link: https://github.com/github/github-mcp-server/issues/844\n Ask: Respond to a reported vulnerability where prompt injection in a public repo could cause leaking info from private repos. The reporter asks whether there\u2019s a fix/CVE and how users can protect private repos. This led to hardening measures (lockdown-related protections are part of the broader mitigation story).\n\n- #1714 \u2014 get_job_logs fails (closed \u2013 completed)\n Link: https://github.com/github/github-mcp-server/issues/1714\n Ask: Fix \u201cunknown tool get_job_logs\u201d when using the remote MCP. Lockdown is only mentioned in the example headers (X-MCP-Lockdown: \"false\"); the core ask is to support or correctly route the get_job_logs tool.\n\n- #1545 \u2014 Hello from the github-mcp-server maintainers! \u2764\ufe0f (closed \u2013 not planned)\n Link: https://github.com/github/github-mcp-server/issues/1545\n Ask: Announcement and request for feedback on new security measures, including content filtering and introducing a configurable lockdown mode to ensure only content from users with push access to the repository is returned. They invite questions/feedback on the approach.", "error": "", "arm": "schema_fields", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 0} +{"prompt_tokens": 14712, "completion_tokens": 3322, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here\u2019s a concise overview of recent activity in github/github-mcp-server\u2019s open issues (156 open total; below highlights are the 20 most recently updated, sorted by latest activity):\n\nNew or newly active today (2026-06-11)\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) \u2014 enhancement, request ai review. New issue focused on enabling OAuth for VS with enterprise policy constraints. https://github.com/github/github-mcp-server/issues/2669\n- #2448 Add update_pull_request_comment and delete_pull_request_comment tools \u2014 feature request to support editing/deleting PR review comments via tools. Updated today. https://github.com/github/github-mcp-server/issues/2448\n- #132 Non PAT auth method \u2014 long-running enhancement asking for alternatives to PAT (e.g., device_code/OAuth); still active with 37 comments. https://github.com/github/github-mcp-server/issues/132\n\nNew in the last 1\u20132 days (2026-06-10)\n- #2661 list_discussions and get_discussion should include upvoteCount \u2014 request to expose upvoteCount to better prioritize discussions. https://github.com/github/github-mcp-server/issues/2661\n- #985 Add tool to get_issue_timeline \u2014 older enhancement resurfaced (linked PR #211), updated yesterday. https://github.com/github/github-mcp-server/issues/985\n- #2647 Skill catalog: missing find-prs and find-issues skills \u2014 asks for discoverable higher-level search skills wrapping search APIs. https://github.com/github/github-mcp-server/issues/2647\n\nNotable bugs and reliability issues (recently active)\n- #2636 issue_write update (state: closed) hangs \u2014 mutation succeeds but tool never returns; client must timeout. High-impact reliability bug. https://github.com/github/github-mcp-server/issues/2636\n- #2202 Issue read drops code in angle brackets inside fenced code \u2014 possible encoding/sanitization issue. https://github.com/github/github-mcp-server/issues/2202\n- #2235 Copilot-generated PR \u201cSuggest\u201d changesets not returned by any API \u2014 upstream/platform limitation; MCP returns prose only. https://github.com/github/github-mcp-server/issues/2235\n- #2594 Insiders: 422 from create pull request UI; UI persists while PR is actually created \u2014 UX/consistency bug. https://github.com/github/github-mcp-server/issues/2594\n- #1079 Get pull request comments not fetching comments outside review context \u2014 likely API surface mismatch; marked stale but recently pinged. https://github.com/github/github-mcp-server/issues/1079\n- #1082 401 with GitHub App access_token for create_pull_request_with_copilot \u2014 works only with classic PAT; auth surface mismatch; marked stale. https://github.com/github/github-mcp-server/issues/1082\n\nFeature requests and tooling gaps (recently active)\n- #2448 Edit/delete PR review comments tools \u2014 closes mutation gap for agents to correct their own comments. https://github.com/github/github-mcp-server/issues/2448\n- #985 get_issue_timeline tool \u2014 adds REST /issues/timeline support. https://github.com/github/github-mcp-server/issues/985\n- #2661 Include upvoteCount in discussions tools \u2014 improves prioritization signal. https://github.com/github/github-mcp-server/issues/2661\n- #2647 Add find-prs and find-issues skills \u2014 improves skill catalog discoverability. https://github.com/github/github-mcp-server/issues/2647\n- #2603 High-level helper tools for Projects V2 (e.g., add_issue_to_project, move_project_item/status) \u2014 reduce GraphQL ID plumbing for agents. request ai review. https://github.com/github/github-mcp-server/issues/2603\n\nAuth/OAuth and ecosystem integration (theme across multiple issues)\n- #132 Non-PAT auth method \u2014 device_code/OAuth alternatives requested. https://github.com/github/github-mcp-server/issues/132\n- #2669 MCP OAuth for VS (GHE) \u2014 enable VS OAuth similar to VS Code. https://github.com/github/github-mcp-server/issues/2669\n- #1081 ChatGPT custom connector: \u201cError fetching OAuth configuration.\u201d \u2014 OAuth discovery failure; stale but still referenced. https://github.com/github/github-mcp-server/issues/1081\n- #1075 Remote MCP OAuth + Gemini CLI errors out \u2014 OAuth authenticated but write calls fail/parse errors. https://github.com/github/github-mcp-server/issues/1075\n- #998 Control access/auth to remote MCP via MS IDP \u2014 enterprise access restrictions for hosted server. https://github.com/github/github-mcp-server/issues/998\n\nOther recent items\n- #2136 Security policy template for all 83 tools (PolicyLayer/Intercept) \u2014 request ai review; proposes guardrails template. https://github.com/github/github-mcp-server/issues/2136\n- #2602 Insiders: VS Code Copilot Chat issue creation widget UX feedback \u2014 duplicate/phantom UI after disconnect/reconnect. https://github.com/github/github-mcp-server/issues/2602\n- #2595 Badge URL points to modelcontextprotocol/registry CI instead of this repo \u2014 housekeeping. https://github.com/github/github-mcp-server/issues/2595\n- #1069 MCP server returns different search_code results via agent vs direct \u2014 inconsistency report. https://github.com/github/github-mcp-server/issues/1069\n\nTrends and takeaways\n- Strong momentum around OAuth/enterprise auth: multiple fresh issues (#2669, continued activity on #132; plus client-specific OAuth problems #1081, #1075, #998).\n- Several actionable tool gaps that would improve agent ergonomics: PR comment edit/delete (#2448), discussions upvoteCount (#2661), Projects V2 high-level helpers (#2603), find-prs/find-issues catalog entries (#2647), and issue timeline (#985).\n- A critical reliability bug (#2636) where issue_close updates hang despite successful mutation.\n- Content fidelity gaps for agents: code blocks with angle brackets in issue_read (#2202) and missing Copilot suggestion diffs (#2235; likely needs platform support).\n- Ongoing Insiders UX issues for creation flows (#2594, #2602).\n\nIf you want, I can drill into any of these (e.g., reproduce steps, current labels/comments, or propose fix approaches) or track updates beyond this top-20 window.", "error": "", "arm": "baseline", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 0} +{"prompt_tokens": 2326, "completion_tokens": 549, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 0} +{"prompt_tokens": 13836, "completion_tokens": 4594, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s a concise snapshot of recent activity in github/github-mcp-server\u2019s open issues.\n\nOverall\n- Open issues: 156\n- Most recently updated issues are heavily focused on authentication, tooling gaps (comments, discussions, project helpers), and a few regressions/bugs.\n- Time window below highlights items updated or opened over roughly the last week.\n\nNew issues opened in the last 7 days\n- #2669 \u2014 Add support for MCP OAuth for Visual Studio (GHE MCP) (labels: enhancement, request ai review) \u2014 opened Jun 11\n https://github.com/github/github-mcp-server/issues/2669\n- #2661 \u2014 list_discussions and get_discussion should include upvoteCount \u2014 opened Jun 10\n https://github.com/github/github-mcp-server/issues/2661\n- #2647 \u2014 Skill catalog: missing find-prs and find-issues skills \u2014 opened Jun 8\n https://github.com/github/github-mcp-server/issues/2647\n- #2636 \u2014 issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned \u2014 opened Jun 7\n https://github.com/github/github-mcp-server/issues/2636\n- #2603 \u2014 Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column) (label: request ai review) \u2014 opened Jun 4\n https://github.com/github/github-mcp-server/issues/2603\n\nOlder issues with fresh updates (past week)\n- #2448 \u2014 Add update_pull_request_comment and delete_pull_request_comment tools \u2014 updated Jun 11\n https://github.com/github/github-mcp-server/issues/2448\n- #132 \u2014 Non PAT auth method (label: enhancement, 37 comments) \u2014 updated Jun 11\n https://github.com/github/github-mcp-server/issues/132\n- #985 \u2014 Add tool to get_issue_timeline (label: enhancement) \u2014 updated Jun 10\n https://github.com/github/github-mcp-server/issues/985\n- #2136 \u2014 Security policy template for GitHub MCP server (label: request ai review) \u2014 updated Jun 7\n https://github.com/github/github-mcp-server/issues/2136\n- #2202 \u2014 GitHub MCP issue read appears to drop code block text in angle brackets (labels: bug, request ai review) \u2014 updated Jun 6\n https://github.com/github/github-mcp-server/issues/2202\n- #2235 \u2014 Copilot-generated PR review \u201cSuggest\u201d changesets not returned by pull_request_read or any API (labels: bug, request ai review) \u2014 updated Jun 4\n https://github.com/github/github-mcp-server/issues/2235\n\nRecent bug reports and regressions\n- #2636 \u2014 issue_write update hangs after successful mutation (opened Jun 7)\n- #2587 \u2014 get_me tool fails without parameters due to empty {} not accepted (labels: bug, request ai review) \u2014 updated Jun 1\n- #2594 \u2014 MCP app 422 error; PR quietly created while UI remains \u2014 opened Jun 2\n- #2202 \u2014 Issue read drops content inside angle brackets \u2014 updated Jun 6\n- Several long-running auth/CLI bugs saw activity on Jun 1\u20133: #1069 (Copilot agent returns no results), #1075 (Remote MCP OAuth + Gemini CLI errors), #1079 (\u201cGet pull request comments\u201d not fetching), #1081 (OAuth discovery error in ChatGPT)\n\nFeature requests and enhancements in motion\n- Comments/PR tooling: #2448 (add update/delete PR comment tools)\n- Discussions: #2661 (include upvoteCount)\n- Projects V2 helpers: #2603 (high-level tools like move_issue_to_column)\n- File operations: #2578 (support executable bit in push_files)\n- Auth improvements: #132 (non-PAT auth), #1333 (GitHub App auth via client credentials)\n- Issue history: #985 (get_issue_timeline)\n\nHosted/remote service and catalog gaps\n- #2584 \u2014 Discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp) \u2014 updated May 31\n- #2647 \u2014 Skill catalog missing find-prs/find-issues \u2014 opened Jun 8\n\nDocs/quality/housekeeping\n- #2425 \u2014 Schema quality: missing parameter descriptions \u2014 updated Jun 1\n- #2595 \u2014 Badge URL points to wrong repository CI \u2014 opened Jun 2\n- #2529 \u2014 Antigravity CLI installation/config docs request \u2014 opened May 22\n\nIf you want, I can drill into any of these (labels, comment threads, or fetch the latest comments) or expand the window beyond the last 50 most recently updated items.", "error": "", "arm": "schema_fields", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 0} +{"prompt_tokens": 1716, "completion_tokens": 519, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 0} +{"prompt_tokens": 2329, "completion_tokens": 399, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 0} +{"prompt_tokens": 4879, "completion_tokens": 660, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 0} +{"prompt_tokens": 1715, "completion_tokens": 233, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 1} +{"prompt_tokens": 26723, "completion_tokens": 6914, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "There are 156 open issues. Their titles are:\n\n- Add support for MCP OAuth for Visual Studio (GHE MCP)\n- list_discussions and get_discussion should include upvoteCount\n- Skill catalog: missing find-prs and find-issues skills\n- issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned\n- Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column)\n- Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability\n- The badge URL points to the CI workflow status of `modelcontextprotocol/registry`, not this repository. If the intent is to show this repo\u2019s CI status, update the badge and link to this repo\u2019s workflow instead of an external repository.\n- Insiders Feedback: [bug] MCP app getting 422 error and create pull request UI stays while PR is silently created\n- bug: `get_me` tool fails without parameters due to empty JSON object `{}` not being accepted as valid input\n- discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp)\n- Add file mode (executable bit) support to `push_files`\n- Insiders Feedback: No types\n- Add installation and configuration documentation for Antigravity CLI\n- `pull_request_review_write` combines create/submit/delete into one tool, making fine-grained permissions by method impossible\n- Clarify that pull_request_review_write create without event creates a pending review\n- Add support for creating and updating repository security advisories\n- Feature: allow review comments on any file line, not just lines within the diff\n- Intermittent commit/push failures when using GitHub MCP connector for file operations and branch management, via Grok official Connector\n- Add update_pull_request_comment and delete_pull_request_comment tools\n- Add support for raw GraphQL queries\n- Interest check: GHES Manage API toolset?\n- Schema quality: 20 required parameters missing descriptions, 92 optional parameters undescribed\n- List user's recent activities\n- feat: add labels support to create_pull_request tool\n- Add bulk/batch issue label operations\n- Add auto-merge support to pull request tools\n- issue_write body parameter overwrites issue content on update \u2014 easily confused with adding a comment\n- Add addBlockedBy / removeBlockedBy mutations (native issue dependencies) to MCP server\n- get_job_logs: run_id with failed_only=false should return all job logs; failed_only should be a consistent modifier\n- Hosted MCP `get_check_runs` fails with 403 where the equivalent REST call succeeds\n- docs: Add `go install` as a first-class installation method\n- Tool request: download workflow run artifact contents\n- \"Application Error\" upon selecting the \"Install in Insiders\" link\n- Feature request: batch-fetch merged PR metadata for release notes / changelog generation\n- Feature request: batch-fetch pull request review comment threads for an explicit PR list\n- Feature Request: `get_agent_card` tool \u2014 load ALP-compatible agents directly from GitHub repos\n- bug: merge_pull_request ignores mergeMethod parameter, always attempts merge commit\n- Missing author_association field in list_issues and pull request responses\n- get_diff / get_files: JSON serialization inflates diff payload far beyond token limits\n- Support dynamic per-session provider tokens for the local GitHub MCP server\n- Copilot-generated PR review code suggestions (\"Suggest\" changesets) are not returned by pull_request_read or any API\n- Suggestion: add a recommended security policy / rate limiting guide\n- Hide write-oriented prompts and instructions when write tools are filtered\n- Classify secondary and abuse rate limits separately from generic GitHub failures\n- http: classify missing token, invalid token, and insufficient scope distinctly\n- Invalidate token-scope cache context when authorization token changes\n- GitHub MCP issue read appears to drop code block text in angle brackets\n- Inventory: return stable hidden-tool reason codes across filter gates\n- Add read-only regression coverage for dynamic toolsets and deprecated aliases\n- push_files takes ~70s per call (both success and fast-fail cases)\n- Security policy template for GitHub MCP server\n- Fail closed in strict mode on unknown toolset names\n- get_copilot_space tool does not return attached file contents from Copilot Spaces\n- Add deterministic toolset resolution ordering for default and custom toolsets\n- Enforce fail-closed startup when PAT/OAuth scope requirements are unmet\n- Add deterministic CI state classifier for GitHub MCP server PR triage\n- Feature: Auto-select token based on repository owner (multi-account support)\n- No built in safeguard against getting rate limited\n- Create repository from template\n- Feature: compare_file_contents tool with semantic diffs\n- Feature Request: Add get_user tool to fetch a user profile by username\n- Github MCP tools for Runners\n- Add multi-account support for client-side account switching\n- Tool name `get_file_contents` conflicts with Claude Desktop built-in tool\n- Add `minimal_output` parameter to list_pull_requests and search_pull_requests\n- Reply tool calls with structuredContent\n- Feature Request: Add support for GitHub Security APIs (Dependabot, Code Scanning, Secret Scanning)\n- Add organization member and repository discovery tools\n- Add compare_commits tool for branch/commit diff analysis\n- Add release management operations to releases toolset\n- Add retry logic with jitter for sub_issue_write to handle parallel calls\n- `get_copilot_job_status` fails with JSON unmarshal error for `response_status_code`\n- Consider workflow scope or user-escalation pattern for merging PRs that modify workflow files\n- Feature Request: `get_labels` or `get_all_labels`\n- Add setting to restrict actions to the current repo\n- Restrict MCP access to private repositories\n- Addressing GitHub MCP's Key Issues: Token Scopes, Verbose Prompts, and Lack of Project Context\n- feat: Issue Preview with manual edit in the chat view before publish\n- Add project ( add_project tool )\n- google/go-licenses currently not maintained\n- Toolset not listing for Github MCP Server | Unstable behaviour of test chat pannel in copilot studio\n- Mutliple github accounts with .ssh/config workaround. Owner & repo cannot be extracted\n- Add `add_repository_collaborator` tool for managing repository access\n- perf: integrate go-sdk schema caching optimization (Phase 1)\n- Add create_discussion tool to discussions toolset\n- Add issue_graph tool to visualize issue/PR relationships and hierarchy\n- list_copilot_spaces / get_copilot_space tools don't return any results in Android Studio\n- Query on availability of Delete Branch tool\n- Proxima pull request action links to wrong url\n- Consider shipping MCPB bundle\n- VSCode MCP Registry integration does not support latest MCP registry\n- list_project_fields fails with JSON unmarshal error for single_select fields with options\n- VSCode: Toolset name mismatch: 'github/*' fails with Unknown tool, but 'github' warns it's deprecated\n- Assign custom agent to issue via assign_copilot_to_issue\n- Github MCP tool doesnt work with Gemini CLI in codespaces\n- Github MCP Tools don't work with Gemini 2.5 models familly and langchain/deepagents javascript library\n- Unable to read the contents of a file, but can write and edit when used in ChatGPT.\n- When using the 'Get Copilot Space' tool, files are found but content cannot be read\n- When using 'Get Copilot Space', files are not found from space for uploaded files\n- GitHub MCP Server Tools Not Working in Copilot Spaces When Used in VS Code\n- Support GitHub App authentication with client credentials\n- Get PR timeline details\n- Add tools for listing organization members and outside collaborators\n- No projects v2 access using GitHub remote MCP server in ChatGPT\n- Docker MCP Toolkit OAuth integration returns 403 on write operations\n- Add optional \"jq\" field to every MCP tool\n- Static MCP Registry question\n- Set a title in `serverInfo.title` for nicer display\n- Copilot Spaces Tool\n- 32600 error code when trying to connect to mcp with openai responses API\n- How to access EMU organization repositories using custom OAuth app credentials with the remote GitHub MCP server?\n- Add GitHub Packages Support to MCP Server\n- Add Claude Code Plugin / Marketplace\n- get_file_contents throws MCP error: resource not found\n- Tool for Creating and Modifying GitHub Discussions via MCP Server\n- Feature Request: Expose GitHub Project Item History/Timeline Events\n- Add support for updating pull request reviews without dismissal\n- list_issues in Continue extension in VS Code with GitHub Docker MCP\n- Resource protected by organization SAML enforcement\n- [Bug] MCP tool works and responds, but Claude Desktop pop-up shows \"Could not attach to MCP server\"\n- get_pull_request_review_comments doesn't return resolved state\n- missing required parameter: query\n- Add `update_issue_comment` tool\n- Cannot use access_token from GitHub App to call create_pull_request_with_copilot\n- Cannot configure Remote GitHub MCP Server in ChatGPT (OAuth discovery error)\n- `Get pull request comments` not fetching comments\n- Remote MCP OAuth + Gemini CLI Errors Out\n- MCP Server not returning results for Copilot when invoked fom Copilot agent\n- MCP Governance - Allow/Deny List for MCP for Organizations and Teams\n- Support sort and comment author filters for get_issue_comments\n- MCP server as part of GitHub Desktop application\n- Add CreateRelease tool for programmatic release creation\n- Control access/auth for github MCP server(remote-github hosted) using MS IDP.\n- Make it easy to create & submit PRs to open source repos\n- Add tool to get_issue_timeline\n- Add tools to get issue_events and get_issue_event\n- Support `--edit-last` functionality\n- Add ability to fetch issue relationships (parent, blocked by, etc..)\n- Error invoking MCP tool search_users (someone else reported the exact same issue)\n- VS Code Github Copilot calling create_pull_request_with_copilot on remote MCP using fine-grained PAT gets 401\n- Invalid URL protocol\n- Feature Request: Add tool to fetch job logs by job name (avoid listing all jobs)\n- Error to MCP server connection\n- Feature Request: Add support for editing issue comments\n- Enable Custom Workflow Registry for Reusable Prompt + Tool Definitions\n- Add support for list_repository_contributors\n- [FEAT] Add support for rulesets and custom properties.\n- Improving access permissions for assets in a GitHub private repository\n- Allow uploading files as attachments\n- Wiki access\n- Add pagination support for get_pull_request_diff to handle large PRs\n- allow users to create repositories from a repository template.\n- Support subdomain isolation for GHES hosts\n- Please add an optional \"Project\" field when Creating an Issue\n- Support creating repositories from repository templates\n- Non PAT auth method", "error": "", "arm": "fields_only", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 1} +{"prompt_tokens": 21874, "completion_tokens": 7060, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "There are 156 open issues in github/github-mcp-server. Their titles are:\n\n- Add support for MCP OAuth for Visual Studio (GHE MCP)\n- list_discussions and get_discussion should include upvoteCount\n- Skill catalog: missing find-prs and find-issues skills\n- issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned\n- Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column)\n- Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability\n- The badge URL points to the CI workflow status of `modelcontextprotocol/registry`, not this repository. If the intent is to show this repo\u2019s CI status, update the badge and link to this repo\u2019s workflow instead of an external repository.\n- Insiders Feedback: [bug] MCP app getting 422 error and create pull request UI stays while PR is silently created\n- bug: `get_me` tool fails without parameters due to empty JSON object `{}` not being accepted as valid input\n- discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp)\n- Add file mode (executable bit) support to `push_files`\n- Insiders Feedback: No types\n- Add installation and configuration documentation for Antigravity CLI\n- `pull_request_review_write` combines create/submit/delete into one tool, making fine-grained permissions by method impossible\n- Clarify that pull_request_review_write create without event creates a pending review\n- Add support for creating and updating repository security advisories\n- Feature: allow review comments on any file line, not just lines within the diff\n- Intermittent commit/push failures when using GitHub MCP connector for file operations and branch management, via Grok official Connector\n- Add update_pull_request_comment and delete_pull_request_comment tools\n- Add support for raw GraphQL queries\n- Interest check: GHES Manage API toolset?\n- Schema quality: 20 required parameters missing descriptions, 92 optional parameters undescribed\n- List user's recent activities\n- feat: add labels support to create_pull_request tool\n- Add bulk/batch issue label operations\n- Add auto-merge support to pull request tools\n- issue_write body parameter overwrites issue content on update \u2014 easily confused with adding a comment\n- Add addBlockedBy / removeBlockedBy mutations (native issue dependencies) to MCP server\n- get_job_logs: run_id with failed_only=false should return all job logs; failed_only should be a consistent modifier\n- Hosted MCP `get_check_runs` fails with 403 where the equivalent REST call succeeds\n- docs: Add `go install` as a first-class installation method\n- Tool request: download workflow run artifact contents\n- \"Application Error\" upon selecting the \"Install in Insiders\" link\n- Feature request: batch-fetch merged PR metadata for release notes / changelog generation\n- Feature request: batch-fetch pull request review comment threads for an explicit PR list\n- Feature Request: `get_agent_card` tool \u2014 load ALP-compatible agents directly from GitHub repos\n- bug: merge_pull_request ignores mergeMethod parameter, always attempts merge commit\n- Missing author_association field in list_issues and pull request responses\n- get_diff / get_files: JSON serialization inflates diff payload far beyond token limits\n- Support dynamic per-session provider tokens for the local GitHub MCP server\n- Copilot-generated PR review code suggestions (\"Suggest\" changesets) are not returned by pull_request_read or any API\n- Suggestion: add a recommended security policy / rate limiting guide\n- Hide write-oriented prompts and instructions when write tools are filtered\n- Classify secondary and abuse rate limits separately from generic GitHub failures\n- http: classify missing token, invalid token, and insufficient scope distinctly\n- Invalidate token-scope cache context when authorization token changes\n- GitHub MCP issue read appears to drop code block text in angle brackets\n- Inventory: return stable hidden-tool reason codes across filter gates\n- Add read-only regression coverage for dynamic toolsets and deprecated aliases\n- push_files takes ~70s per call (both success and fast-fail cases)\n- Security policy template for GitHub MCP server\n- Fail closed in strict mode on unknown toolset names\n- get_copilot_space tool does not return attached file contents from Copilot Spaces\n- Add deterministic toolset resolution ordering for default and custom toolsets\n- Enforce fail-closed startup when PAT/OAuth scope requirements are unmet\n- Add deterministic CI state classifier for GitHub MCP server PR triage\n- Feature: Auto-select token based on repository owner (multi-account support)\n- No built in safeguard against getting rate limited\n- Create repository from template\n- Feature: compare_file_contents tool with semantic diffs\n- Feature Request: Add get_user tool to fetch a user profile by username\n- Github MCP tools for Runners\n- Add multi-account support for client-side account switching\n- Tool name `get_file_contents` conflicts with Claude Desktop built-in tool\n- Add `minimal_output` parameter to list_pull_requests and search_pull_requests\n- Reply tool calls with structuredContent\n- Feature Request: Add support for GitHub Security APIs (Dependabot, Code Scanning, Secret Scanning)\n- Add organization member and repository discovery tools\n- Add compare_commits tool for branch/commit diff analysis\n- Add release management operations to releases toolset\n- Add retry logic with jitter for sub_issue_write to handle parallel calls\n- `get_copilot_job_status` fails with JSON unmarshal error for `response_status_code`\n- Consider workflow scope or user-escalation pattern for merging PRs that modify workflow files\n- Feature Request: `get_labels` or `get_all_labels`\n- Add setting to restrict actions to the current repo\n- Restrict MCP access to private repositories\n- Addressing GitHub MCP's Key Issues: Token Scopes, Verbose Prompts, and Lack of Project Context\n- feat: Issue Preview with manual edit in the chat view before publish\n- Add project ( add_project tool )\n- google/go-licenses currently not maintained\n- Toolset not listing for Github MCP Server | Unstable behaviour of test chat pannel in copilot studio\n- Mutliple github accounts with .ssh/config workaround. Owner & repo cannot be extracted\n- Add `add_repository_collaborator` tool for managing repository access\n- perf: integrate go-sdk schema caching optimization (Phase 1)\n- Add create_discussion tool to discussions toolset\n- Add issue_graph tool to visualize issue/PR relationships and hierarchy\n- list_copilot_spaces / get_copilot_space tools don't return any results in Android Studio\n- Query on availability of Delete Branch tool\n- Proxima pull request action links to wrong url\n- Consider shipping MCPB bundle\n- VSCode MCP Registry integration does not support latest MCP registry\n- list_project_fields fails with JSON unmarshal error for single_select fields with options\n- VSCode: Toolset name mismatch: 'github/*' fails with Unknown tool, but 'github' warns it's deprecated\n- Assign custom agent to issue via assign_copilot_to_issue\n- Github MCP tool doesnt work with Gemini CLI in codespaces\n- Github MCP Tools don't work with Gemini 2.5 models familly and langchain/deepagents javascript library\n- Unable to read the contents of a file, but can write and edit when used in ChatGPT.\n- When using the 'Get Copilot Space' tool, files are found but content cannot be read\n- When using 'Get Copilot Space', files are not found from space for uploaded files\n- GitHub MCP Server Tools Not Working in Copilot Spaces When Used in VS Code\n- Support GitHub App authentication with client credentials\n- Get PR timeline details\n- Add tools for listing organization members and outside collaborators\n- No projects v2 access using GitHub remote MCP server in ChatGPT\n- Docker MCP Toolkit OAuth integration returns 403 on write operations\n- Add optional \"jq\" field to every MCP tool\n- Static MCP Registry question\n- Set a title in `serverInfo.title` for nicer display\n- Copilot Spaces Tool\n- 32600 error code when trying to connect to mcp with openai responses API\n- How to access EMU organization repositories using custom OAuth app credentials with the remote GitHub MCP server?\n- Add GitHub Packages Support to MCP Server\n- Add Claude Code Plugin / Marketplace\n- get_file_contents throws MCP error: resource not found\n- Tool for Creating and Modifying GitHub Discussions via MCP Server\n- Feature Request: Expose GitHub Project Item History/Timeline Events\n- Add support for updating pull request reviews without dismissal\n- list_issues in Continue extension in VS Code with GitHub Docker MCP\n- Resource protected by organization SAML enforcement\n- [Bug] MCP tool works and responds, but Claude Desktop pop-up shows \"Could not attach to MCP server\"\n- get_pull_request_review_comments doesn't return resolved state\n- missing required parameter: query\n- Add `update_issue_comment` tool\n- Cannot use access_token from GitHub App to call create_pull_request_with_copilot\n- Cannot configure Remote GitHub MCP Server in ChatGPT (OAuth discovery error)\n- `Get pull request comments` not fetching comments\n- Remote MCP OAuth + Gemini CLI Errors Out\n- MCP Server not returning results for Copilot when invoked fom Copilot agent\n- MCP Governance - Allow/Deny List for MCP for Organizations and Teams\n- Support sort and comment author filters for get_issue_comments\n- MCP server as part of GitHub Desktop application\n- Add CreateRelease tool for programmatic release creation\n- Control access/auth for github MCP server(remote-github hosted) using MS IDP.\n- Make it easy to create & submit PRs to open source repos\n- Add tool to get_issue_timeline\n- Add tools to get issue_events and get_issue_event\n- Support `--edit-last` functionality\n- Add ability to fetch issue relationships (parent, blocked by, etc..)\n- Error invoking MCP tool search_users (someone else reported the exact same issue)\n- VS Code Github Copilot calling create_pull_request_with_copilot on remote MCP using fine-grained PAT gets 401\n- Invalid URL protocol\n- Feature Request: Add tool to fetch job logs by job name (avoid listing all jobs)\n- Error to MCP server connection\n- Feature Request: Add support for editing issue comments\n- Enable Custom Workflow Registry for Reusable Prompt + Tool Definitions\n- Add support for list_repository_contributors\n- [FEAT] Add support for rulesets and custom properties.\n- Improving access permissions for assets in a GitHub private repository\n- Allow uploading files as attachments\n- Wiki access\n- Add pagination support for get_pull_request_diff to handle large PRs\n- allow users to create repositories from a repository template.\n- Support subdomain isolation for GHES hosts\n- Please add an optional \"Project\" field when Creating an Issue\n- Support creating repositories from repository templates\n- Non PAT auth method", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 1} +{"prompt_tokens": 1713, "completion_tokens": 169, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 1} +{"prompt_tokens": 32922, "completion_tokens": 3053, "turns": 4, "tool_calls": 3, "fields_calls": 3, "final_text": "2673, 2672, 2670, 2667, 2666, 2665, 2663, 2659, 2658, 2656, 2655, 2652, 2650, 2639, 2637, 2634, 2633, 2632, 2631, 2630, 2629, 2627, 2626, 2625, 2624, 2623, 2622, 2620, 2604, 2601, 2599, 2591, 2588, 2586, 2579, 2577, 2572, 2569, 2562, 2482, 2468, 2465, 2447, 2443, 2428, 2416, 2413, 2408, 2406, 2382, 2374, 2340, 2294, 2285, 2279, 2265, 2261, 2246, 2230, 2220, 2214, 2198, 2193, 2184, 2183, 2129, 2118, 2105, 2086, 2083, 2076, 2038, 2031, 2010, 1984, 1983, 1982, 1981, 1980, 1974, 1971, 1959, 1956, 1941, 1928, 1927, 1915, 1867, 1846, 1843, 1837, 1836, 1825, 1800, 1797, 1794, 1734, 1700, 1678, 1677, 1661, 1579, 1574, 1564, 1560, 1542, 1527, 1525, 1519, 1512, 1511, 1508, 1491, 1490, 1406, 1363, 1345, 1339, 1338, 1283, 1226, 1209, 1197, 1190, 1149, 1139, 1130, 1098, 1095, 1087, 1072, 1037, 1015, 1004, 1002, 962, 816, 494, 211, 210", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 1} +{"prompt_tokens": 21735, "completion_tokens": 2968, "turns": 4, "tool_calls": 3, "fields_calls": 3, "final_text": "The currently open pull request numbers in github/github-mcp-server are:\n2673, 2672, 2670, 2667, 2666, 2665, 2663, 2659, 2658, 2656, 2655, 2652, 2650, 2639, 2637, 2634, 2633, 2632, 2631, 2630, 2629, 2627, 2626, 2625, 2624, 2623, 2622, 2620, 2604, 2601, 2599, 2591, 2588, 2586, 2579, 2577, 2572, 2569, 2562, 2482, 2468, 2465, 2447, 2443, 2428, 2416, 2413, 2408, 2406, 2382, 2374, 2340, 2294, 2285, 2279, 2265, 2261, 2246, 2230, 2220, 2214, 2198, 2193, 2184, 2183, 2129, 2118, 2105, 2086, 2083, 2076, 2038, 2031, 2010, 1984, 1983, 1982, 1981, 1980, 1974, 1971, 1959, 1956, 1941, 1928, 1927, 1915, 1867, 1846, 1843, 1837, 1836, 1825, 1800, 1797, 1794, 1734, 1700, 1678, 1677, 1661, 1579, 1574, 1564, 1560, 1542, 1527, 1525, 1519, 1512, 1511, 1508, 1491, 1490, 1406, 1363, 1345, 1339, 1338, 1283, 1226, 1209, 1197, 1190, 1149, 1139, 1130, 1098, 1095, 1087, 1072, 1037, 1015, 1004, 1002, 962, 816, 494, 211, 210.", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 1} +{"prompt_tokens": 1706, "completion_tokens": 423, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Which GitHub organization are you referring to? Please provide the organization (owner) name, and I\u2019ll list the configured issue types for it.", "error": "", "arm": "baseline", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 1} +{"prompt_tokens": 5105, "completion_tokens": 885, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here are the issue types currently configured for the github organization:\n\n- Epic \u2014 A group of related work that typically contributes to an initiative\n- Feature \u2014 A request, improvement, or new functionality\n- Task \u2014 A specific piece of work\n- Bug \u2014 An unexpected problem or behavior\n- Batch \u2014 A group of work that can be completed relatively quickly\n- Initiative \u2014 A 1\u20132 quarter, multi-release investment that delivers measurable customer and business outcomes\n- Risk \u2014 An uncertain event or condition that requires monitoring and/or mitigation\n\nIf you meant a different organization, let me know the org name and I\u2019ll check its configured issue types.", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 1} +{"prompt_tokens": 4869, "completion_tokens": 632, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Could you tell me which GitHub organization you mean? Please provide the org handle (owner). If you mean the public organization named \u201cgithub\u201d (owner: github), let me know and I\u2019ll list its configured issue types.", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 1} +{"prompt_tokens": 4193, "completion_tokens": 737, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "The latest open issue proposes adding two MCP server tools to manage pull request review comments: update_pull_request_comment and delete_pull_request_comment. Today, the server only supports add_reply_to_pull_request_comment, so agents can\u2019t edit or delete their own review comments\u2014forcing noisy reply threads or fallback to raw gh api calls. The suggested tools would map to existing GitHub REST endpoints (PATCH/DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id}), include auth constraints (only operate on the authenticated user\u2019s comments), and accept owner, repo, comment_id (and body for updates). The request stems from real-world cases where agents needed to correct or retract comments; it also notes confusion between discussion_rN URLs and the underlying API comment id, with a related reading-gap issue referenced as #2235.", "error": "", "arm": "baseline", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 1} +{"prompt_tokens": 5444, "completion_tokens": 723, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "It\u2019s a feature request to add two tools to the MCP server for pull request review comments: one to update an existing review comment and another to delete one. Today the server only supports adding a reply to a review comment, which forces agents to leave the original (possibly incorrect) comment in place or fall back to direct REST calls via gh api. The proposal maps directly to existing GitHub REST endpoints (PATCH/DELETE for /repos/{owner}/{repo}/pulls/comments/{comment_id}), limited to editing/deleting comments authored by the authenticated user, to reduce thread noise and let agents clean up mistakes programmatically.", "error": "", "arm": "fields_only", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 1} +{"prompt_tokens": 10514, "completion_tokens": 736, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s the most recently updated open issue in github/github-mcp-server:\n\n- Issue #2448: \u201cAdd update_pull_request_comment and delete_pull_request_comment tools\u201d\n Link: https://github.com/github/github-mcp-server/issues/2448\n Summary:\n - Proposes two new MCP tools to manage PR review comments: update_pull_request_comment (PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id}) and delete_pull_request_comment (DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id}).\n - Motivation: The server currently only supports add_reply_to_pull_request_comment, so agents can\u2019t edit or delete their own review comments. This leads to posting corrective replies instead of updating/removing the original, creating thread noise.\n - Constraints: Edits/deletes limited to comments authored by the authenticated user. Equivalent gh api commands are provided.\n - Context: A real incident highlighted the gap; also notes confusion between discussion_rN URLs and the actual API comment IDs.", "error": "", "arm": "schema_fields", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 1} +{"prompt_tokens": 1720, "completion_tokens": 369, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 1} +{"prompt_tokens": 2333, "completion_tokens": 216, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 1} +{"prompt_tokens": 4883, "completion_tokens": 220, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 1} +{"prompt_tokens": 9735, "completion_tokens": 1497, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here are the issues that mention \u201clockdown\u201d in github/github-mcp-server, with what each is asking for:\n\n- #2197 \u2014 Inventory: return stable hidden-tool reason codes across filter gates (open)\n - Ask: Ensure that when tools are hidden by filtering (toolset, PAT scope, read-only, or lockdown), callers receive one deterministic reason code. Proposed codes include toolset_filtered, scope_filtered, readonly_filtered, and lockdown_filtered. Add tests for each hide path and keep the classification consistent across inventory and request-time filtering.\n\n- #1870 \u2014 Error: MCP error 0: lockdown cache is not configured (closed, completed)\n - Ask: Fix a runtime error that occurs when using the remote MCP server with X-MCP-Lockdown: true, where requests fail with \u201clockdown cache is not configured.\u201d Expected behavior is either successful execution or a permission error, not a configuration error. In short: make lockdown mode work (or fail gracefully) on the hosted server.\n\n- #1714 \u2014 get_job_logs fails (closed)\n - Ask: Implement or expose the get_job_logs tool (or fix its discovery) because clients receive \u201cunknown tool\u201d for get_job_logs. Lockdown is only mentioned incidentally in the provided header example; the actual request is about adding/fixing the tool.\n\n- #1545 \u2014 Hello from the github-mcp-server maintainers! \u2764\ufe0f (closed, not planned)\n - Ask: Announcement/roadmap issue describing two initiatives: content filtering and openWorldHint expansion. It specifically calls out introducing a configurable lockdown mode to ensure only content from users with push access is returned, and references the related PR for lockdown mode (#1371).", "error": "", "arm": "baseline", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 1} +{"prompt_tokens": 37404, "completion_tokens": 3980, "turns": 4, "tool_calls": 3, "fields_calls": 0, "final_text": "Here are the issues that mention \u201clockdown\u201d in github/github-mcp-server, with what each is asking for:\n\n- #2197 \u2013 Inventory: return stable hidden-tool reason codes across filter gates (open)\n Ask: Standardize and return deterministic reason codes when tools are hidden by filters, including a specific lockdown classification (e.g., lockdown_filtered). Also add tests and keep the classification consistent across inventory and request filtering.\n\n- #1685 \u2013 Restrict MCP access to private repositories (open)\n Ask: A feature to restrict a local/remote MCP deployment to specific repos (by org/repo), especially private repos. Maintainers suggested using lockdown mode as a partial mitigation while scoping is considered.\n\n- #1870 \u2013 Error: MCP error 0: lockdown cache is not configured (closed \u2013 completed)\n Ask: Fix a bug where using the remote MCP with X-MCP-Lockdown: true causes \u201clockdown cache is not configured.\u201d The reporter expected either normal behavior or a clear permission error, not a misconfiguration error. This was resolved.\n\n- #844 \u2013 Exfiltrate information from private repositories (closed \u2013 completed)\n Ask: Address a reported data exfiltration scenario via prompt injection and clarify response (fixes, CVE, mitigation). Maintainers replied this wasn\u2019t a specific vulnerability unique to this server and listed mitigations now available, including lockdown mode, secret scanning on public-repo calls, read-only mode, OAuth 2.1 support, and fine\u2011grained tokens.\n\n- #1714 \u2013 get_job_logs fails (closed \u2013 completed)\n Ask: Implement/fix the get_job_logs tool (error: unknown tool). Lockdown appears only in the provided config snippet (X-MCP-Lockdown set to false); the issue itself is about the missing tool.\n\n- #1545 \u2013 Hello from the github-mcp-server maintainers! \u2764\ufe0f (closed \u2013 not planned)\n Ask: Announcement/feedback thread about security initiatives, including a planned \u201clockdown mode\u201d to ensure only content from users with push access is returned, plus broader content filtering and openWorldHint coverage. It invited feedback and pointed to related PRs.\n\nIn short: \u201cLockdown\u201d is referenced as a mitigation/feature to constrain returned content to trusted contributors (reducing prompt\u2011injection risk), needs proper server configuration (cache) when enabled on the remote server, and is part of broader efforts to make responses safer and more predictable.", "error": "", "arm": "fields_only", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 1} +{"prompt_tokens": 13415, "completion_tokens": 1849, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here are the issues in github/github-mcp-server that mention \u201clockdown,\u201d and what they\u2019re asking for:\n\n- #2197 (open) \u2013 Inventory: return stable hidden-tool reason codes across filter gates\n - What it asks: Unify and return deterministic reason codes whenever tools are hidden by filters. Specifically calls out including lockdown as a first-class reason (e.g., lockdown_filtered) alongside toolset, scope, and readonly filters. Requests tests for each hide path and consistency across inventory and request-time filtering. Mentions relevant code areas, including pkg/lockdown/lockdown.go.\n\n- #1870 (closed) \u2013 Error: MCP error 0: lockdown cache is not configured\n - What it asks: Fix a bug where enabling lockdown via the X-MCP-Lockdown: \"true\" header causes requests to fail with \u201clockdown cache is not configured.\u201d The expectation is either successful execution or a clear permission-denied response, not an internal configuration error. Repro steps and logs are provided.\n\n- #1714 (closed) \u2013 get_job_logs fails\n - Where lockdown appears: Only in the sample headers (X-MCP-Lockdown: \"false\"); it\u2019s incidental to the report.\n - What it asks: Address \u201cunknown tool\u201d for get_job_logs (not specifically about lockdown). Notes a workaround via get_workflow_run_logs_url.\n\n- #1545 (closed) \u2013 Hello from the github-mcp-server maintainers! \u2764\ufe0f\n - What it asks (among broader updates): Announces and seeks feedback on a planned, configurable lockdown mode intended to ensure the server only returns content originating from users with push access to the repository. References a related PR for lockdown mode (#1371).", "error": "", "arm": "schema_fields", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 1} +{"prompt_tokens": 1713, "completion_tokens": 307, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 1} +{"prompt_tokens": 7511, "completion_tokens": 6410, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s a concise snapshot of recent activity in github/github-mcp-server\u2019s open issues, focusing on updates since 2026-05-28:\n\nOverall\n- 36 open issues have been updated since 2026-05-28.\n- 12 new issues were opened in this window.\n\nNewly opened issues (since 2026-05-28)\n- 2669: Add support for MCP OAuth for Visual Studio (GHE MCP) [labels: enhancement, request ai review] \u2014 opened 2026-06-11\n- 2661: list_discussions and get_discussion should include upvoteCount \u2014 opened 2026-06-10\n- 2647: Skill catalog: missing find-prs and find-issues skills \u2014 opened 2026-06-08\n- 2636: issue_write update (state: closed) hangs \u2014 mutation succeeds but no response \u2014 opened 2026-06-07\n- 2603: Feature Request: Helpers for Projects V2 (e.g., move_issue_to_column) [request ai review] \u2014 opened 2026-06-04\n- 2602: Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability \u2014 opened 2026-06-03\n- 2595: Badge URL points to external repo\u2019s CI instead of this repo \u2014 opened 2026-06-02\n- 2594: Insiders Feedback: 422 error; create PR UI stays while PR is created \u2014 opened 2026-06-02\n- 2587: bug: get_me fails without parameters ({} not accepted) [bug, request ai review] \u2014 opened 2026-06-01\n- 2584: discussions toolset not exposed via hosted remote endpoint \u2014 opened 2026-05-31\n- 2578: Add file mode (executable bit) support to push_files \u2014 opened 2026-05-30\n- 2575: Insiders Feedback: No types \u2014 opened 2026-05-29\n\nOlder issues with fresh updates\n- 2448: Add update_pull_request_comment and delete_pull_request_comment tools \u2014 updated 2026-06-11\n- 132: Non PAT auth method [enhancement] \u2014 long-running, active thread (37 comments) \u2014 updated 2026-06-11\n- 985: Add tool to get_issue_timeline [enhancement] \u2014 updated 2026-06-10\n- 2136: Security policy template for GitHub MCP server [request ai review] \u2014 updated 2026-06-07\n- 2202: GitHub MCP issue_read drops code in angle brackets [bug, request ai review] \u2014 updated 2026-06-06\n- 2235: Copilot PR \u201cSuggest\u201d changesets not returned by pull_request_read [bug, request ai review] \u2014 updated 2026-06-04\n- 1082: Cannot use GitHub App access_token with create_pull_request_with_copilot [bug, stale] (8 comments) \u2014 updated 2026-06-03\n- 1081: OAuth discovery error configuring Remote GitHub MCP in ChatGPT [bug, stale] (7 comments) \u2014 updated 2026-06-02\n- 1079: \u201cGet pull request comments\u201d not fetching comments [bug, stale] \u2014 updated 2026-06-02\n- 998: Control access/auth for remote GitHub MCP using MS IDP \u2014 updated 2026-06-01\n- 1075: Remote MCP OAuth + Gemini CLI errors [bug, stale] (6 comments) \u2014 updated 2026-06-01\n- 1069: MCP Server not returning results for Copilot (agent) [bug] \u2014 updated 2026-06-01\n- 2425: Schema quality: missing descriptions for parameters \u2014 updated 2026-06-01\n- 1476: Query on availability of Delete Branch tool [tool-proposal] \u2014 updated 2026-06-01\n- 1047: Support sort and comment author filters for get_issue_comments [enhancement, stale] \u2014 updated 2026-05-31\n- 1020: MCP server as part of GitHub Desktop [enhancement, discussion, stale] \u2014 updated 2026-05-31\n- 942: VS Code Copilot calling create_pull_request_with_copilot via fine-grained PAT gets 401 [bug, stale] \u2014 updated 2026-05-31\n- 1333: Support GitHub App auth with client credentials [enhancement] \u2014 updated 2026-05-31\n- 980: Support --edit-last functionality [enhancement] \u2014 updated 2026-05-30\n- 1935: Tool name get_file_contents conflicts with Claude Desktop built-in tool \u2014 updated 2026-05-30\n- 1012: Add CreateRelease tool [stale] (10 comments) \u2014 updated 2026-05-30\n- 987: Make it easy to create & submit PRs to OSS repos [enhancement, stale] \u2014 updated 2026-05-30\n- 984: Add tools to get issue_events and get_issue_event [enhancement, stale] \u2014 updated 2026-05-30\n- 950: Add ability to fetch issue relationships (parent, blocked by, etc.) [enhancement] \u2014 updated 2026-05-28\n\nThemes and trends\n- Authentication and OAuth: Active conversations around alternatives to PATs and broader auth support (132, 1333, 998), plus issues with OAuth flows and tool behavior across clients (1081, 1075, 1069). A new request to support MCP OAuth for Visual Studio and GHE MCP (2669).\n- Discussions tooling: Requests to expose discussions tools via the hosted endpoint (2584) and to include upvoteCount in discussion APIs (2661).\n- New tools and capabilities: Multiple enhancement requests including update/delete PR comments (2448), Projects V2 helpers (2603), issue timeline and events tools (985, 984), CreateRelease (1012), relationships between issues (950), push_files executable bit support (2578), and UX helpers like --edit-last (980).\n- Bug reports and reliability: Reports include hanging mutations on issue_write update (2636), get_me parameter handling (2587), angle-bracket code dropped in issue_read (2202), PR review \u201cSuggest\u201d changesets not retrievable (2235), and various client-specific failures or errors (1079, 1082, 942, 2594).\n- Insiders feedback: Several fresh items tied to Copilot/VS Code experiences (2602, 2594, 2575).\n- Metadata quality and housekeeping: Schema description gaps called out (2425), and a CI badge pointing to the wrong repo (2595).\n\nMost discussed among these updated issues (by comment count in the list)\n- 132: Non PAT auth method \u2014 37 comments\n- 1012: Add CreateRelease tool \u2014 10 comments\n- 1082: GitHub App access_token with create_pull_request_with_copilot \u2014 8 comments\n- 1081: OAuth discovery error (Remote MCP in ChatGPT) \u2014 7 comments\n- 1075: Remote MCP OAuth + Gemini CLI errors \u2014 6 comments\n\nIf you want, I can drill into any of these issues (details, labels, or comments) or pull the latest comments for specific ones.", "error": "", "arm": "fields_only", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 1} +{"prompt_tokens": 10750, "completion_tokens": 3546, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s a concise overview of open issues in github/github-mcp-server with recent activity (updated since 2026-06-04). Total: 11 issues.\n\nNewly opened this week\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) \u2014 enhancement, request ai review \u2014 opened/updated 2026-06-11 by dagirard\n https://github.com/github/github-mcp-server/issues/2669\n- #2661 list_discussions and get_discussion should include upvoteCount \u2014 opened 2026-06-10 by carlhye\n https://github.com/github/github-mcp-server/issues/2661\n- #2647 Skill catalog: missing find-prs and find-issues skills \u2014 opened 2026-06-08, updated 2026-06-10 by SamMorrowDrums\n https://github.com/github/github-mcp-server/issues/2647\n- #2636 issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned \u2014 opened/updated 2026-06-07 by edimuj\n https://github.com/github/github-mcp-server/issues/2636\n- #2603 Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column) \u2014 request ai review \u2014 opened/updated 2026-06-04 by kidrov\n https://github.com/github/github-mcp-server/issues/2603\n\nOlder issues with fresh updates\n- #2448 Add update_pull_request_comment and delete_pull_request_comment tools \u2014 updated 2026-06-11 (1 comment) by HaleTom\n https://github.com/github/github-mcp-server/issues/2448\n- #132 Non PAT auth method \u2014 enhancement \u2014 updated 2026-06-11 (37 comments) by aSapien\n https://github.com/github/github-mcp-server/issues/132\n- #985 Add tool to get_issue_timeline \u2014 enhancement \u2014 updated 2026-06-10 (3 comments) by tommaso-moro\n https://github.com/github/github-mcp-server/issues/985\n- #2136 Security policy template for GitHub MCP server \u2014 request ai review \u2014 updated 2026-06-07 (3 comments) by L1AD\n https://github.com/github/github-mcp-server/issues/2136\n- #2202 GitHub MCP issue read appears to drop code block text in angle brackets \u2014 bug, request ai review \u2014 updated 2026-06-06 (3 comments) by fmun\n https://github.com/github/github-mcp-server/issues/2202\n- #2235 Copilot-generated PR review code suggestions (\u201cSuggest\u201d changesets) are not returned by pull_request_read or any API \u2014 bug, request ai review \u2014 updated 2026-06-04 (3 comments) by VooDisss\n https://github.com/github/github-mcp-server/issues/2235\n\nThemes and signals from this week\n- New feature requests: OAuth support for GHE/VS MCP, upvoteCount in discussions APIs, PR comment management tools, Projects V2 helper tools, and skill catalog gaps.\n- Active long-running thread: Non-PAT authentication (#132) continues to see discussion.\n- Bug reports getting attention: content fidelity in issue reads (angle brackets), missing Copilot \u201cSuggest\u201d changes via APIs, and an issue_write update hang.\n- Several items flagged request ai review, indicating maintainers seeking AI-assisted triage/review.\n\nIf you want, I can drill into any of these for details (discussion highlights, proposed solutions, or who\u2019s participating).", "error": "", "arm": "schema_fields", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 1} +{"prompt_tokens": 1716, "completion_tokens": 711, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 1} +{"prompt_tokens": 2329, "completion_tokens": 332, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 1} +{"prompt_tokens": 4879, "completion_tokens": 411, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 1} +{"prompt_tokens": 1715, "completion_tokens": 169, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 2} +{"prompt_tokens": 24055, "completion_tokens": 8206, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "There are 156 open issues. Their titles are:\n- Add support for MCP OAuth for Visual Studio (GHE MCP)\n- list_discussions and get_discussion should include upvoteCount\n- Skill catalog: missing find-prs and find-issues skills\n- issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned\n- Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column)\n- Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability\n- The badge URL points to the CI workflow status of `modelcontextprotocol/registry`, not this repository. If the intent is to show this repo\u2019s CI status, update the badge and link to this repo\u2019s workflow instead of an external repository.\n- Insiders Feedback: [bug] MCP app getting 422 error and create pull request UI stays while PR is silently created\n- bug: `get_me` tool fails without parameters due to empty JSON object `{}` not being accepted as valid input\n- discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp)\n- Add file mode (executable bit) support to `push_files`\n- Insiders Feedback: No types\n- Add installation and configuration documentation for Antigravity CLI\n- `pull_request_review_write` combines create/submit/delete into one tool, making fine-grained permissions by method impossible\n- Clarify that pull_request_review_write create without event creates a pending review\n- Add support for creating and updating repository security advisories\n- Feature: allow review comments on any file line, not just lines within the diff\n- Intermittent commit/push failures when using GitHub MCP connector for file operations and branch management, via Grok official Connector\n- Add update_pull_request_comment and delete_pull_request_comment tools\n- Add support for raw GraphQL queries\n- Interest check: GHES Manage API toolset?\n- Schema quality: 20 required parameters missing descriptions, 92 optional parameters undescribed\n- List user's recent activities\n- feat: add labels support to create_pull_request tool\n- Add bulk/batch issue label operations\n- Add auto-merge support to pull request tools\n- issue_write body parameter overwrites issue content on update \u2014 easily confused with adding a comment\n- Add addBlockedBy / removeBlockedBy mutations (native issue dependencies) to MCP server\n- get_job_logs: run_id with failed_only=false should return all job logs; failed_only should be a consistent modifier\n- Hosted MCP `get_check_runs` fails with 403 where the equivalent REST call succeeds\n- docs: Add `go install` as a first-class installation method\n- Tool request: download workflow run artifact contents\n- \"Application Error\" upon selecting the \"Install in Insiders\" link\n- Feature request: batch-fetch merged PR metadata for release notes / changelog generation\n- Feature request: batch-fetch pull request review comment threads for an explicit PR list\n- Feature Request: `get_agent_card` tool \u2014 load ALP-compatible agents directly from GitHub repos\n- bug: merge_pull_request ignores mergeMethod parameter, always attempts merge commit\n- Missing author_association field in list_issues and pull request responses\n- get_diff / get_files: JSON serialization inflates diff payload far beyond token limits\n- Support dynamic per-session provider tokens for the local GitHub MCP server\n- Copilot-generated PR review code suggestions (\"Suggest\" changesets) are not returned by pull_request_read or any API\n- Suggestion: add a recommended security policy / rate limiting guide\n- Hide write-oriented prompts and instructions when write tools are filtered\n- Classify secondary and abuse rate limits separately from generic GitHub failures\n- http: classify missing token, invalid token, and insufficient scope distinctly\n- Invalidate token-scope cache context when authorization token changes\n- GitHub MCP issue read appears to drop code block text in angle brackets\n- Inventory: return stable hidden-tool reason codes across filter gates\n- Add read-only regression coverage for dynamic toolsets and deprecated aliases\n- push_files takes ~70s per call (both success and fast-fail cases)\n- Security policy template for GitHub MCP server\n- Fail closed in strict mode on unknown toolset names\n- get_copilot_space tool does not return attached file contents from Copilot Spaces\n- Add deterministic toolset resolution ordering for default and custom toolsets\n- Enforce fail-closed startup when PAT/OAuth scope requirements are unmet\n- Add deterministic CI state classifier for GitHub MCP server PR triage\n- Feature: Auto-select token based on repository owner (multi-account support)\n- No built in safeguard against getting rate limited\n- Create repository from template\n- Feature: compare_file_contents tool with semantic diffs\n- Feature Request: Add get_user tool to fetch a user profile by username\n- Github MCP tools for Runners\n- Add multi-account support for client-side account switching\n- Tool name `get_file_contents` conflicts with Claude Desktop built-in tool\n- Add `minimal_output` parameter to list_pull_requests and search_pull_requests\n- Reply tool calls with structuredContent\n- Feature Request: Add support for GitHub Security APIs (Dependabot, Code Scanning, Secret Scanning)\n- Add organization member and repository discovery tools\n- Add compare_commits tool for branch/commit diff analysis\n- Add release management operations to releases toolset\n- Add retry logic with jitter for sub_issue_write to handle parallel calls\n- `get_copilot_job_status` fails with JSON unmarshal error for `response_status_code`\n- Consider workflow scope or user-escalation pattern for merging PRs that modify workflow files\n- Feature Request: `get_labels` or `get_all_labels`\n- Add setting to restrict actions to the current repo\n- Restrict MCP access to private repositories\n- Addressing GitHub MCP's Key Issues: Token Scopes, Verbose Prompts, and Lack of Project Context\n- feat: Issue Preview with manual edit in the chat view before publish\n- Add project ( add_project tool )\n- google/go-licenses currently not maintained\n- Toolset not listing for Github MCP Server | Unstable behaviour of test chat pannel in copilot studio\n- Mutliple github accounts with .ssh/config workaround. Owner & repo cannot be extracted\n- Add `add_repository_collaborator` tool for managing repository access\n- perf: integrate go-sdk schema caching optimization (Phase 1)\n- Add create_discussion tool to discussions toolset\n- Add issue_graph tool to visualize issue/PR relationships and hierarchy\n- list_copilot_spaces / get_copilot_space tools don't return any results in Android Studio\n- Query on availability of Delete Branch tool\n- Proxima pull request action links to wrong url\n- Consider shipping MCPB bundle\n- VSCode MCP Registry integration does not support latest MCP registry\n- list_project_fields fails with JSON unmarshal error for single_select fields with options\n- VSCode: Toolset name mismatch: 'github/*' fails with Unknown tool, but 'github' warns it's deprecated\n- Assign custom agent to issue via assign_copilot_to_issue\n- Github MCP tool doesnt work with Gemini CLI in codespaces\n- Github MCP Tools don't work with Gemini 2.5 models familly and langchain/deepagents javascript library\n- Unable to read the contents of a file, but can write and edit when used in ChatGPT.\n- When using the 'Get Copilot Space' tool, files are found but content cannot be read\n- When using 'Get Copilot Space', files are not found from space for uploaded files\n- GitHub MCP Server Tools Not Working in Copilot Spaces When Used in VS Code\n- Support GitHub App authentication with client credentials\n- Get PR timeline details\n- Add tools for listing organization members and outside collaborators\n- No projects v2 access using GitHub remote MCP server in ChatGPT\n- Docker MCP Toolkit OAuth integration returns 403 on write operations\n- Add optional \"jq\" field to every MCP tool\n- Static MCP Registry question\n- Set a title in `serverInfo.title` for nicer display\n- Copilot Spaces Tool\n- 32600 error code when trying to connect to mcp with openai responses API\n- How to access EMU organization repositories using custom OAuth app credentials with the remote GitHub MCP server?\n- Add GitHub Packages Support to MCP Server\n- Add Claude Code Plugin / Marketplace\n- get_file_contents throws MCP error: resource not found\n- Tool for Creating and Modifying GitHub Discussions via MCP Server\n- Feature Request: Expose GitHub Project Item History/Timeline Events\n- Add support for updating pull request reviews without dismissal\n- list_issues in Continue extension in VS Code with GitHub Docker MCP\n- Resource protected by organization SAML enforcement\n- [Bug] MCP tool works and responds, but Claude Desktop pop-up shows \"Could not attach to MCP server\"\n- get_pull_request_review_comments doesn't return resolved state\n- missing required parameter: query\n- Add `update_issue_comment` tool\n- Cannot use access_token from GitHub App to call create_pull_request_with_copilot\n- Cannot configure Remote GitHub MCP Server in ChatGPT (OAuth discovery error)\n- `Get pull request comments` not fetching comments\n- Remote MCP OAuth + Gemini CLI Errors Out\n- MCP Server not returning results for Copilot when invoked fom Copilot agent\n- MCP Governance - Allow/Deny List for MCP for Organizations and Teams\n- Support sort and comment author filters for get_issue_comments\n- MCP server as part of GitHub Desktop application\n- Add CreateRelease tool for programmatic release creation\n- Control access/auth for github MCP server(remote-github hosted) using MS IDP.\n- Make it easy to create & submit PRs to open source repos\n- Add tool to get_issue_timeline\n- Add tools to get issue_events and get_issue_event\n- Support `--edit-last` functionality\n- Add ability to fetch issue relationships (parent, blocked by, etc..)\n- Error invoking MCP tool search_users (someone else reported the exact same issue)\n- VS Code Github Copilot calling create_pull_request_with_copilot on remote MCP using fine-grained PAT gets 401\n- Invalid URL protocol\n- Feature Request: Add tool to fetch job logs by job name (avoid listing all jobs)\n- Error to MCP server connection\n- Feature Request: Add support for editing issue comments\n- Enable Custom Workflow Registry for Reusable Prompt + Tool Definitions\n- Add support for list_repository_contributors\n- [FEAT] Add support for rulesets and custom properties.\n- Improving access permissions for assets in a GitHub private repository\n- Allow uploading files as attachments\n- Wiki access\n- Add pagination support for get_pull_request_diff to handle large PRs\n- allow users to create repositories from a repository template.\n- Support subdomain isolation for GHES hosts\n- Please add an optional \"Project\" field when Creating an Issue\n- Support creating repositories from repository templates\n- Non PAT auth method", "error": "", "arm": "fields_only", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 2} +{"prompt_tokens": 21841, "completion_tokens": 9086, "turns": 3, "tool_calls": 2, "fields_calls": 2, "final_text": "There are 156 open issues in github/github-mcp-server. Their titles are:\n\n- Add support for MCP OAuth for Visual Studio (GHE MCP)\n- list_discussions and get_discussion should include upvoteCount\n- Skill catalog: missing find-prs and find-issues skills\n- issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned\n- Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column)\n- Insiders Feedback: Issue creation widget in VS Code Copilot Chat usability\n- The badge URL points to the CI workflow status of `modelcontextprotocol/registry`, not this repository. If the intent is to show this repo\u2019s CI status, update the badge and link to this repo\u2019s workflow instead of an external repository.\n- Insiders Feedback: [bug] MCP app getting 422 error and create pull request UI stays while PR is silently created\n- bug: `get_me` tool fails without parameters due to empty JSON object `{}` not being accepted as valid input\n- discussions toolset not exposed via hosted remote endpoint (api.githubcopilot.com/mcp)\n- Add file mode (executable bit) support to `push_files`\n- Insiders Feedback: No types\n- Add installation and configuration documentation for Antigravity CLI\n- `pull_request_review_write` combines create/submit/delete into one tool, making fine-grained permissions by method impossible\n- Clarify that pull_request_review_write create without event creates a pending review\n- Add support for creating and updating repository security advisories\n- Feature: allow review comments on any file line, not just lines within the diff\n- Intermittent commit/push failures when using GitHub MCP connector for file operations and branch management, via Grok official Connector\n- Add update_pull_request_comment and delete_pull_request_comment tools\n- Add support for raw GraphQL queries\n- Interest check: GHES Manage API toolset?\n- Schema quality: 20 required parameters missing descriptions, 92 optional parameters undescribed\n- List user's recent activities\n- feat: add labels support to create_pull_request tool\n- Add bulk/batch issue label operations\n- Add auto-merge support to pull request tools\n- issue_write body parameter overwrites issue content on update \u2014 easily confused with adding a comment\n- Add addBlockedBy / removeBlockedBy mutations (native issue dependencies) to MCP server\n- get_job_logs: run_id with failed_only=false should return all job logs; failed_only should be a consistent modifier\n- Hosted MCP `get_check_runs` fails with 403 where the equivalent REST call succeeds\n- docs: Add `go install` as a first-class installation method\n- Tool request: download workflow run artifact contents\n- \"Application Error\" upon selecting the \"Install in Insiders\" link\n- Feature request: batch-fetch merged PR metadata for release notes / changelog generation\n- Feature request: batch-fetch pull request review comment threads for an explicit PR list\n- Feature Request: `get_agent_card` tool \u2014 load ALP-compatible agents directly from GitHub repos\n- bug: merge_pull_request ignores mergeMethod parameter, always attempts merge commit\n- Missing author_association field in list_issues and pull request responses\n- get_diff / get_files: JSON serialization inflates diff payload far beyond token limits\n- Support dynamic per-session provider tokens for the local GitHub MCP server\n- Copilot-generated PR review code suggestions (\"Suggest\" changesets) are not returned by pull_request_read or any API\n- Suggestion: add a recommended security policy / rate limiting guide\n- Hide write-oriented prompts and instructions when write tools are filtered\n- Classify secondary and abuse rate limits separately from generic GitHub failures\n- http: classify missing token, invalid token, and insufficient scope distinctly\n- Invalidate token-scope cache context when authorization token changes\n- GitHub MCP issue read appears to drop code block text in angle brackets\n- Inventory: return stable hidden-tool reason codes across filter gates\n- Add read-only regression coverage for dynamic toolsets and deprecated aliases\n- push_files takes ~70s per call (both success and fast-fail cases)\n- Security policy template for GitHub MCP server\n- Fail closed in strict mode on unknown toolset names\n- get_copilot_space tool does not return attached file contents from Copilot Spaces\n- Add deterministic toolset resolution ordering for default and custom toolsets\n- Enforce fail-closed startup when PAT/OAuth scope requirements are unmet\n- Add deterministic CI state classifier for GitHub MCP server PR triage\n- Feature: Auto-select token based on repository owner (multi-account support)\n- No built in safeguard against getting rate limited\n- Create repository from template\n- Feature: compare_file_contents tool with semantic diffs\n- Feature Request: Add get_user tool to fetch a user profile by username\n- Github MCP tools for Runners\n- Add multi-account support for client-side account switching\n- Tool name `get_file_contents` conflicts with Claude Desktop built-in tool\n- Add `minimal_output` parameter to list_pull_requests and search_pull_requests\n- Reply tool calls with structuredContent\n- Feature Request: Add support for GitHub Security APIs (Dependabot, Code Scanning, Secret Scanning)\n- Add organization member and repository discovery tools\n- Add compare_commits tool for branch/commit diff analysis\n- Add release management operations to releases toolset\n- Add retry logic with jitter for sub_issue_write to handle parallel calls\n- `get_copilot_job_status` fails with JSON unmarshal error for `response_status_code`\n- Consider workflow scope or user-escalation pattern for merging PRs that modify workflow files\n- Feature Request: `get_labels` or `get_all_labels`\n- Add setting to restrict actions to the current repo\n- Restrict MCP access to private repositories\n- Addressing GitHub MCP's Key Issues: Token Scopes, Verbose Prompts, and Lack of Project Context\n- feat: Issue Preview with manual edit in the chat view before publish\n- Add project ( add_project tool )\n- google/go-licenses currently not maintained\n- Toolset not listing for Github MCP Server | Unstable behaviour of test chat pannel in copilot studio\n- Mutliple github accounts with .ssh/config workaround. Owner & repo cannot be extracted\n- Add `add_repository_collaborator` tool for managing repository access\n- perf: integrate go-sdk schema caching optimization (Phase 1)\n- Add create_discussion tool to discussions toolset\n- Add issue_graph tool to visualize issue/PR relationships and hierarchy\n- list_copilot_spaces / get_copilot_space tools don't return any results in Android Studio\n- Query on availability of Delete Branch tool\n- Proxima pull request action links to wrong url\n- Consider shipping MCPB bundle\n- VSCode MCP Registry integration does not support latest MCP registry\n- list_project_fields fails with JSON unmarshal error for single_select fields with options\n- VSCode: Toolset name mismatch: 'github/*' fails with Unknown tool, but 'github' warns it's deprecated\n- Assign custom agent to issue via assign_copilot_to_issue\n- Github MCP tool doesnt work with Gemini CLI in codespaces\n- Github MCP Tools don't work with Gemini 2.5 models familly and langchain/deepagents javascript library\n- Unable to read the contents of a file, but can write and edit when used in ChatGPT.\n- When using the 'Get Copilot Space' tool, files are found but content cannot be read\n- When using 'Get Copilot Space', files are not found from space for uploaded files\n- GitHub MCP Server Tools Not Working in Copilot Spaces When Used in VS Code\n- Support GitHub App authentication with client credentials\n- Get PR timeline details\n- Add tools for listing organization members and outside collaborators\n- No projects v2 access using GitHub remote MCP server in ChatGPT\n- Docker MCP Toolkit OAuth integration returns 403 on write operations\n- Add optional \"jq\" field to every MCP tool\n- Static MCP Registry question\n- Set a title in `serverInfo.title` for nicer display\n- Copilot Spaces Tool\n- 32600 error code when trying to connect to mcp with openai responses API\n- How to access EMU organization repositories using custom OAuth app credentials with the remote GitHub MCP server?\n- Add GitHub Packages Support to MCP Server\n- Add Claude Code Plugin / Marketplace\n- get_file_contents throws MCP error: resource not found\n- Tool for Creating and Modifying GitHub Discussions via MCP Server\n- Feature Request: Expose GitHub Project Item History/Timeline Events\n- Add support for updating pull request reviews without dismissal\n- list_issues in Continue extension in VS Code with GitHub Docker MCP\n- Resource protected by organization SAML enforcement\n- [Bug] MCP tool works and responds, but Claude Desktop pop-up shows \"Could not attach to MCP server\"\n- get_pull_request_review_comments doesn't return resolved state\n- missing required parameter: query\n- Add `update_issue_comment` tool\n- Cannot use access_token from GitHub App to call create_pull_request_with_copilot\n- Cannot configure Remote GitHub MCP Server in ChatGPT (OAuth discovery error)\n- `Get pull request comments` not fetching comments\n- Remote MCP OAuth + Gemini CLI Errors Out\n- MCP Server not returning results for Copilot when invoked fom Copilot agent\n- MCP Governance - Allow/Deny List for MCP for Organizations and Teams\n- Support sort and comment author filters for get_issue_comments\n- MCP server as part of GitHub Desktop application\n- Add CreateRelease tool for programmatic release creation\n- Control access/auth for github MCP server(remote-github hosted) using MS IDP.\n- Make it easy to create & submit PRs to open source repos\n- Add tool to get_issue_timeline\n- Add tools to get issue_events and get_issue_event\n- Support `--edit-last` functionality\n- Add ability to fetch issue relationships (parent, blocked by, etc..)\n- Error invoking MCP tool search_users (someone else reported the exact same issue)\n- VS Code Github Copilot calling create_pull_request_with_copilot on remote MCP using fine-grained PAT gets 401\n- Invalid URL protocol\n- Feature Request: Add tool to fetch job logs by job name (avoid listing all jobs)\n- Error to MCP server connection\n- Feature Request: Add support for editing issue comments\n- Enable Custom Workflow Registry for Reusable Prompt + Tool Definitions\n- Add support for list_repository_contributors\n- [FEAT] Add support for rulesets and custom properties.\n- Improving access permissions for assets in a GitHub private repository\n- Allow uploading files as attachments\n- Wiki access\n- Add pagination support for get_pull_request_diff to handle large PRs\n- allow users to create repositories from a repository template.\n- Support subdomain isolation for GHES hosts\n- Please add an optional \"Project\" field when Creating an Issue\n- Support creating repositories from repository templates\n- Non PAT auth method", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "How many open issues are there in github/github-mcp-server, and what are their titles?", "run": 2} +{"prompt_tokens": 1713, "completion_tokens": 173, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 2} +{"prompt_tokens": 32910, "completion_tokens": 3945, "turns": 4, "tool_calls": 3, "fields_calls": 3, "final_text": "2673, 2672, 2670, 2667, 2666, 2665, 2663, 2659, 2658, 2656, 2655, 2652, 2650, 2639, 2637, 2634, 2633, 2632, 2631, 2630, 2629, 2627, 2626, 2625, 2624, 2623, 2622, 2620, 2604, 2601, 2599, 2591, 2588, 2586, 2579, 2577, 2572, 2569, 2562, 2482, 2468, 2465, 2447, 2443, 2428, 2416, 2413, 2408, 2406, 2382, 2374, 2340, 2294, 2285, 2279, 2265, 2261, 2246, 2230, 2220, 2214, 2198, 2193, 2184, 2183, 2129, 2118, 2105, 2086, 2083, 2076, 2038, 2031, 2010, 1984, 1983, 1982, 1981, 1980, 1974, 1971, 1959, 1956, 1941, 1928, 1927, 1915, 1867, 1846, 1843, 1837, 1836, 1825, 1800, 1797, 1794, 1734, 1700, 1678, 1677, 1661, 1579, 1574, 1564, 1560, 1542, 1527, 1525, 1519, 1512, 1511, 1508, 1491, 1490, 1406, 1363, 1345, 1339, 1338, 1283, 1226, 1209, 1197, 1190, 1149, 1139, 1130, 1098, 1095, 1087, 1072, 1037, 1015, 1004, 1002, 962, 816, 494, 211, 210", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 2} +{"prompt_tokens": 4876, "completion_tokens": 41, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "narrow", "task": "What are the numbers of the currently open pull requests in github/github-mcp-server?", "run": 2} +{"prompt_tokens": 3942, "completion_tokens": 971, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "The github organization has the following issue types configured:\n- Epic \u2014 A group of related work that typically contributes to an initiative\n- Feature \u2014 A request, improvement, or new functionality\n- Task \u2014 A specific piece of work\n- Bug \u2014 An unexpected problem or behavior\n- Batch \u2014 A group of work that can be completed relatively quickly\n- Initiative \u2014 A 1\u20132 quarter, multi-release investment that delivers measurable customer and business outcomes\n- Risk \u2014 An uncertain event or condition that requires monitoring and/or mitigation", "error": "", "arm": "baseline", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 2} +{"prompt_tokens": 2319, "completion_tokens": 424, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Which GitHub organization are you referring to? Please provide the org handle (e.g., my-org), and I\u2019ll list the configured issue types.", "error": "", "arm": "fields_only", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 2} +{"prompt_tokens": 4869, "completion_tokens": 512, "turns": 1, "tool_calls": 0, "fields_calls": 0, "final_text": "Which GitHub organization (owner) are you referring to? Please provide the org name (e.g., github, vercel, your-org). If you have a preference for specific fields (e.g., just names vs. full details), let me know as well.", "error": "", "arm": "schema_fields", "tag": "narrow", "task": "What issue types are configured for the github organization?", "run": 2} +{"prompt_tokens": 4193, "completion_tokens": 868, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "- Issue #2448: \u201cAdd update_pull_request_comment and delete_pull_request_comment tools\u201d\n- What it\u2019s about: A feature request to let the MCP server edit and delete existing pull request review comments, not just add replies.\n- Why: Without edit/delete, agents must post replies to correct mistakes (creating thread noise) or bypass the MCP server with gh api PATCH/DELETE calls.\n- Proposed solution: Add update_pull_request_comment (PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id}, with body) and delete_pull_request_comment (DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id}), restricted to modifying comments authored by the authenticated user.\n- Context: Addresses practical issues encountered by agents; also notes confusion between discussion_rN URLs and API comment IDs (related to #2235).", "error": "", "arm": "baseline", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 2} +{"prompt_tokens": 5451, "completion_tokens": 1120, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "- Issue #2448: \u201cAdd update_pull_request_comment and delete_pull_request_comment tools.\u201d\n- What it\u2019s about: A feature request to let the MCP server edit and delete existing pull request review comments, not just add replies.\n- Why: Today, agents can only reply to a review comment via add_reply_to_pull_request_comment, which forces noisy reply threads when an edit or retraction is needed, or pushes users to bypass MCP with gh api.\n- Proposed solution: Add two tools that map to existing GitHub REST endpoints\u2014update_pull_request_comment (PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id} with new body) and delete_pull_request_comment (DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id})\u2014restricted to modifying/deleting only the authenticated user\u2019s own comments.\n- Extra context: Notes confusion between discussion_rN URL tokens and the API comment id and references a related gap in #2235.", "error": "", "arm": "fields_only", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 2} +{"prompt_tokens": 10534, "completion_tokens": 1319, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Most recently updated open issue: #2448 \u2014 \u201cAdd update_pull_request_comment and delete_pull_request_comment tools\u201d\n\nSummary:\n- It\u2019s a feature request to let the MCP server edit or delete existing pull request review comments, not just add replies.\n- Today, agents must either post a new reply (leaving the incorrect original in place and adding thread noise) or bypass the MCP server by calling the GitHub REST API directly via gh api.\n- Proposed tools map to existing REST endpoints: PATCH /repos/{owner}/{repo}/pulls/comments/{comment_id} (update_pull_request_comment) and DELETE /repos/{owner}/{repo}/pulls/comments/{comment_id} (delete_pull_request_comment), with the constraint that users can only modify their own comments.\n- The request cites a real incident where lack of edit/delete led to noisy threads and notes confusion between discussion_r{N} identifiers and API comment IDs; it references a related gap in #2235.\n\nLink: https://github.com/github/github-mcp-server/issues/2448", "error": "", "arm": "schema_fields", "tag": "full", "task": "Summarize what the most recently updated open issue in github/github-mcp-server is about.", "run": 2} +{"prompt_tokens": 1720, "completion_tokens": 297, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 2} +{"prompt_tokens": 2333, "completion_tokens": 224, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 2} +{"prompt_tokens": 4883, "completion_tokens": 210, "turns": 1, "tool_calls": 1, "fields_calls": 1, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "full", "task": "Look at the open pull requests in github/github-mcp-server and tell me which one looks most substantial, and why.", "run": 2} +{"prompt_tokens": 11765, "completion_tokens": 2221, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here are all the issues in github/github-mcp-server that mention \u201clockdown,\u201d with what each is asking for:\n\n- #2197 (open) \u2014 Inventory: return stable hidden-tool reason codes across filter gates\n Ask: Provide deterministic, stable reason codes when tools are hidden by filters (toolset, PAT scope, read-only mode, or lockdown). Specifically include a consistent lockdown_filtered code and unify classification across inventory and request filtering, with tests to validate each hide path.\n Link: https://github.com/github/github-mcp-server/issues/2197\n\n- #1685 (open) \u2014 Restrict MCP access to private repositories\n Ask: Add configuration to restrict a local MCP deployment to only specific orgs/repos (and not implicitly include public repos). The user wants a deterministic way to limit repository access (beyond prompt instructions), essentially a scoping/lockdown-style control via flags.\n Link: https://github.com/github/github-mcp-server/issues/1685\n\n- #1870 (closed \u2013 completed) \u2014 Error: MCP error 0: lockdown cache is not configured\n Ask: Fix a bug where enabling X-MCP-Lockdown: true on the remote server caused an immediate error (\u201clockdown cache is not configured\u201d). The expectation was either normal results or a permission error, not a server misconfiguration error.\n Link: https://github.com/github/github-mcp-server/issues/1870\n\n- #844 (closed \u2013 completed) \u2014 Exfiltrate information from private repositories\n Ask: Address a reported vulnerability where prompt injection in a public repo could cause agents to publish information from private repos. The reporter asks if there was a response/fix/CVE and how users can protect private repos. This implicitly calls for stronger controls (e.g., lockdown modes) to prevent cross-repo data leakage.\n Link: https://github.com/github/github-mcp-server/issues/844\n\n- #1714 (closed \u2013 completed) \u2014 get_job_logs fails\n Ask: Fix \u201cunknown tool get_job_logs.\u201d The issue incidentally shows X-MCP-Lockdown: false in the sample config but is not about lockdown; it\u2019s a request to support or correctly expose the get_job_logs tool.\n Link: https://github.com/github/github-mcp-server/issues/1714\n\n- #1545 (closed \u2013 not planned) \u2014 Hello from the github-mcp-server maintainers! \u2764\ufe0f\n Ask: Announcement seeking feedback about two initiatives, including a \u201cconfigurable lockdown mode\u201d to restrict content to users with push access. While not a request to change a specific bug/feature in this issue itself, it asks for community input on the proposed lockdown feature (referencing PR #1371).\n Link: https://github.com/github/github-mcp-server/issues/1545", "error": "", "arm": "baseline", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 2} +{"prompt_tokens": 24050, "completion_tokens": 3288, "turns": 3, "tool_calls": 2, "fields_calls": 0, "final_text": "Here are the issues in github/github-mcp-server that mention \u201clockdown,\u201d with what each is asking for:\n\n- #2197 (open) \u2013 \u201cInventory: return stable hidden-tool reason codes across filter gates\u201d\n - Ask: Standardize and return deterministic \u201creason codes\u201d whenever a tool is hidden by filtering, including a specific code for lockdown (\u201clockdown_filtered\u201d), plus tests to cover each hide path. Goal: make filtering/auditability predictable across inventory and request filtering.\n\n- #1870 (closed \u2013 completed) \u2013 \u201cError: MCP error 0: lockdown cache is not configured\u201d\n - Ask: Fix a bug where enabling lockdown via the X-MCP-Lockdown header caused requests to fail with \u201clockdown cache is not configured.\u201d The reporter expects proper behavior (either successful data return under lockdown or a permission-style error), not a server misconfiguration error.\n\n- #1545 (closed \u2013 not planned) \u2013 \u201cHello from the github-mcp-server maintainers! \u2764\ufe0f\u201d\n - Ask: Announcement/update inviting feedback on two initiatives, one of which is a \u201cconfigurable lockdown mode\u201d to restrict returns to content from users with push access. Not a bug or feature request per se, but it documents the planned lockdown capability and asks for community input.\n\n- #1685 (open) \u2013 \u201cRestrict MCP access to private repositories\u201d\n - Ask: Feature request to restrict a deployment to specific repos (e.g., only certain private org repos). Lockdown is suggested in a maintainer comment as a partial mitigation (limiting content to trusted contributors), but the primary request is a configuration flag to restrict which repos are accessible.\n\n- #844 (closed \u2013 completed) \u2013 \u201cExfiltrate information from private repositories\u201d\n - Ask: Security report asking whether a prompt-injection exploit was addressed (fix? CVE?) and how users can protect private repos. While the body doesn\u2019t focus on lockdown, related maintainer work (including lockdown mode) is part of broader mitigations to reduce leakage from untrusted content.\n\n- #1714 (closed \u2013 completed) \u2013 \u201cget_job_logs fails\u201d\n - Ask: Bug report that a specific tool (\u201cget_job_logs\u201d) returned \u201cunknown tool.\u201d The issue includes an example config showing X-MCP-Lockdown: false, but the request itself isn\u2019t about lockdown; it matched because the snippet contained the header string. The fix was to support or route the tool properly.\n\nIn short, the direct asks around \u201clockdown\u201d are: finalize and surface clear lockdown-related filter reasons (#2197), fix lockdown server behavior when enabled (#1870), and the maintainers\u2019 announcement of introducing a configurable lockdown mode (#1545). Other issues reference lockdown as a mitigation or incidentally via config snippets.", "error": "", "arm": "fields_only", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 2} +{"prompt_tokens": 16053, "completion_tokens": 1744, "turns": 2, "tool_calls": 1, "fields_calls": 0, "final_text": "Here are the issues in github/github-mcp-server that mention \u201clockdown,\u201d along with what they\u2019re asking for:\n\n- #2197 \u2013 Inventory: return stable hidden-tool reason codes across filter gates (open)\n Link: https://github.com/github/github-mcp-server/issues/2197\n Ask: Standardize and return deterministic reason codes whenever tools are hidden by filtering, including when hidden due to lockdown. They want a single stable classification per hidden decision (e.g., toolset_filtered, scope_filtered, readonly_filtered, lockdown_filtered), tests covering each hide path, and consistent behavior across inventory and request filtering.\n\n- #1870 \u2013 Error: MCP error 0: lockdown cache is not configured (closed \u2013 completed)\n Link: https://github.com/github/github-mcp-server/issues/1870\n Ask: Fix a bug where enabling lockdown (X-MCP-Lockdown: true) produced a server error about a missing lockdown cache. The expectation was either normal tool behavior or a clear permission error, not a configuration error. Essentially, ensure lockdown mode works without throwing \u201clockdown cache is not configured,\u201d by configuring/initializing the cache or handling it gracefully.\n\n- #1714 \u2013 get_job_logs fails (closed \u2013 completed)\n Link: https://github.com/github/github-mcp-server/issues/1714\n Ask: Not really about lockdown (it\u2019s just present in the sample headers as false). The request was to address \u201cunknown tool\u201d for get_job_logs or provide the functionality. Lockdown is incidental here.\n\n- #1545 \u2013 Hello from the github-mcp-server maintainers! \u2764\ufe0f (closed \u2013 not planned)\n Link: https://github.com/github/github-mcp-server/issues/1545\n Ask: Announcement/coordination issue outlining security and predictability initiatives, including introducing a configurable lockdown mode to only return content from users with push access. They were soliciting feedback and pointing to related PRs (e.g., lockdown mode #1371).", "error": "", "arm": "schema_fields", "tag": "full", "task": "Find issues mentioning 'lockdown' in github/github-mcp-server and explain what they're asking for.", "run": 2} +{"prompt_tokens": 1713, "completion_tokens": 1187, "turns": 1, "tool_calls": 2, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 2} +{"prompt_tokens": 8537, "completion_tokens": 4511, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s a concise overview of what\u2019s been happening recently in github/github-mcp-server\u2019s open issues (sorted by most recently updated). Total open issues: 156.\n\nWhat changed today (most recent updates)\n- #2448 Add update_pull_request_comment and delete_pull_request_comment tools \u2013 active discussion today.\n- #132 Non PAT auth method (37 comments) \u2013 ongoing conversation, remains one of the most active auth threads.\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) \u2013 newly opened and immediately updated.\n\nNew issues opened in the last week\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) \u2013 auth/enterprise integration.\n- #2661 list_discussions and get_discussion should include upvoteCount \u2013 discussions API feature request.\n- #2647 Skill catalog: missing find-prs and find-issues skills \u2013 tool catalog parity.\n- #2636 issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned \u2013 reliability bug.\n- #2603 Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column) \u2013 Projects V2 helpers.\n\nOther notable items updated in the last few days\n- Reliability/bugs\n - #2202 GitHub MCP issue read appears to drop code block text in angle brackets \u2013 parsing/data loss bug.\n - #2587 bug: get_me tool fails without parameters due to {} not being accepted \u2013 input validation issue.\n - Several older bug threads still open and recently touched (many labeled stale): #1082, #1081, #1079, #1075, #1069, #942, #945, #934.\n\n- Comments and reviews tooling\n - #2448 Add update/delete PR comment tools \u2013 capability gap being discussed.\n - #2235 Copilot-generated PR \u201cSuggest changes\u201d not returned by pull_request_read or any API \u2013 coverage gap.\n - #2525 pull_request_review_write combines create/submit/delete, making fine-grained permissions hard.\n - #2524 Clarify that pull_request_review_write create without event creates a pending review \u2013 docs/behavior clarity.\n\n- Auth and enterprise\n - #132 Non PAT auth method \u2013 long-running thread, very active.\n - #1333 Support GitHub App authentication with client credentials \u2013 auth expansion request.\n\n- Discussions and hosted endpoint parity\n - #2661 Add upvoteCount to discussion tools \u2013 richer data.\n - #2584 discussions toolset not exposed via hosted remote endpoint \u2013 hosted parity gap.\n\n- Developer experience and Insiders feedback\n - #2602 Insiders: Issue creation widget in VS Code Copilot Chat usability.\n - #2594 Insiders: 422 error; PR is created but UI stays \u2013 UX/integration bug.\n - #2575 Insiders Feedback: No types \u2013 typing/SDK feedback.\n - #2425 Schema quality: missing descriptions for many params \u2013 schema hygiene.\n\n- Misc feature requests\n - #2578 Add file mode (executable bit) support to push_files.\n - Issue timelines/events: #985 get_issue_timeline, #984 tools for issue_events.\n - Projects/organization workflows and governance: #1048 org allow/deny list for MCP; #500 subdomain isolation for GHES.\n - Security APIs: #1921 add Dependabot/Code Scanning/Secret Scanning support.\n\n- Documentation and repo hygiene\n - #2529 Add installation/config docs for Antigravity CLI.\n - #2595 Badge URL points to the wrong repo\u2019s workflow status \u2013 needs correction.\n\nClusters/trends\n- Authentication continues to be a hot topic (Non-PAT auth, GitHub App auth, enterprise OAuth).\n- Gaps in review/comment tooling and discussions data are getting attention (update/delete PR comments, \u201cSuggest changes,\u201d upvote counts).\n- Endpoint/tooling parity and schema/documentation quality remain active workstreams.\n- A handful of long-standing bug reports are being revisited but remain open.\n\nYou can open any issue directly via:\nhttps://github.com/github/github-mcp-server/issues/ISSUE_NUMBER\nExample: https://github.com/github/github-mcp-server/issues/2448\n\nIf you want, I can drill into any of these (details, latest comments, or label breakdown), or filter by category like bugs, enhancements, or \u201crequest ai review.\u201d", "error": "", "arm": "fields_only", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 2} +{"prompt_tokens": 17622, "completion_tokens": 5775, "turns": 2, "tool_calls": 1, "fields_calls": 1, "final_text": "Here\u2019s a concise snapshot of recent activity in github/github-mcp-server\u2019s open issues, based on the latest 100 open issues ordered by most recently updated (156 open in total).\n\nMost recently updated (past 48 hours)\n- #2448 Add update_pull_request_comment and delete_pull_request_comment tools (updated 2026-06-11 13:36 UTC, 1 comment) \u2013 proposal to add write operations for PR comments.\n- #132 Non PAT auth method [enhancement] (updated 2026-06-11 10:56 UTC, 37 comments) \u2013 long-running discussion on alternatives to PAT-based auth.\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) [enhancement, request ai review] (created/updated 2026-06-11) \u2013 new request focused on enterprise/VS integration.\n- #2661 list_discussions and get_discussion should include upvoteCount (created/updated 2026-06-10) \u2013 feature request for discussions metadata.\n- #985 Add tool to get_issue_timeline [enhancement] (updated 2026-06-10) \u2013 renewed interest in richer issue history.\n- #2647 Skill catalog: missing find-prs and find-issues skills (updated 2026-06-10) \u2013 gap in skill exposure/catalog.\n\nNew issues created in the last 7 days\n- #2669 Add support for MCP OAuth for Visual Studio (GHE MCP) [enhancement, request ai review]\n- #2661 list_discussions and get_discussion should include upvoteCount\n- #2647 Skill catalog: missing find-prs and find-issues skills\n- #2636 issue_write update (state: closed) hangs \u2014 mutation succeeds but no response is ever returned (likely bug)\n- #2603 Feature Request: High-level helper tools for GitHub Projects V2 (e.g. move_issue_to_column) [request ai review]\n\nOther notable activity in the past week\n- Bugs and regressions\n - #2636 issue_write update hangs (mutation succeeds, no response).\n - #2587 get_me tool fails with empty JSON input {} [bug, request ai review] (created 2026-06-01, still open).\n - #2202 Issue read drops code in angle brackets [bug, request ai review] (active 2026-06-06).\n - #2258 merge_pull_request ignores mergeMethod (always merge commit) (updated late March; still relevant for merge behavior).\n - #2381 Hosted MCP get_check_runs returns 403 where REST succeeds [bug, request ai review].\n- Authentication, authorization, and enterprise support\n - #132 Non PAT auth method (highly active).\n - #1333 Support GitHub App authentication with client credentials [enhancement].\n - #1121 Resource protected by organization SAML enforcement.\n - #2213 Classify missing token/invalid token/insufficient scope distinctly.\n - #2203 Invalidate token-scope cache when auth token changes.\n- Enhancements to tools and capabilities\n - PR tools: #2448 update/delete PR comments; #2411 auto-merge support; #2415 labels in create_pull_request; #1910 compare_commits; #625 pagination for get_pull_request_diff; #1086 PR review comments missing \u201cresolved\u201d state.\n - Issues/discussions/labels: #2412 bulk/batch label ops; #985 issue_timeline; #984 issue_events; #950 issue relationships (parent/blocked-by); #2661 discussions upvoteCount.\n - Files/attachments: #738 upload attachments; #2578 executable bit in push_files.\n - GraphQL and batch ops: #2437 raw GraphQL queries; #2301 batch-fetch PR review comment threads; #2302 batch-fetch merged PR metadata for release notes.\n - Templates/releases/security: #446/#1992/#522 create repos from templates; #1909 release management operations; #1921 security APIs (Dependabot, code/secret scanning); #2299 get_agent_card.\n- Insiders and UX feedback\n - #2602 VS Code Copilot Chat issue creation widget usability (Insiders feedback).\n - #2594 MCP app 422 with PR UI lingering (Insiders bug).\n - #2575 \u201cNo types\u201d (Insiders feedback).\n - #2595 Badge URL points to external repo\u2019s CI status.\n\nLabel trends and themes\n- \u201cenhancement\u201d and \u201cbug\u201d dominate recent updates, with several tagged \u201crequest ai review.\u201d\n- Active themes: authentication/enterprise (OAuth, GitHub App creds, SAML), improving PR/issue/discussion tooling (labels, timelines, comments, batch/bulk operations), and stability/consistency (tool input handling, API parity, error classification).\n\nTotals and scope\n- Open issues: 156.\n- This overview summarizes the most recent 100, emphasizing items updated in the last 48 hours and new issues from the last 7 days.\n\nIf you want, I can drill into any of the above (e.g., full details on a specific issue, only bugs, or only enhancement requests), or expand the view beyond the latest 100.", "error": "", "arm": "schema_fields", "tag": "neutral", "task": "Give me an overview of recent activity in github/github-mcp-server's open issues.", "run": 2} +{"prompt_tokens": 1716, "completion_tokens": 764, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "baseline", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 2} +{"prompt_tokens": 2329, "completion_tokens": 239, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "fields_only", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 2} +{"prompt_tokens": 4879, "completion_tokens": 893, "turns": 1, "tool_calls": 1, "fields_calls": 0, "final_text": "", "error": "APIStatusError: Error code: 413 - {'error': {'code': 'tokens_limit_reached', 'message': 'Request body too large for gpt-5 model. Max size: 16000 tokens.', 'details': 'Request body too large for gpt-5 model. Max size:", "arm": "schema_fields", "tag": "neutral", "task": "Search for open pull requests about tests in github/github-mcp-server and tell me what they do.", "run": 2} diff --git a/pkg/github/__toolsnaps__/list_branches.snap b/pkg/github/__toolsnaps__/list_branches.snap index 883a6fffcb..4db47b6a1c 100644 --- a/pkg/github/__toolsnaps__/list_branches.snap +++ b/pkg/github/__toolsnaps__/list_branches.snap @@ -6,6 +6,18 @@ "description": "List branches in a GitHub repository", "inputSchema": { "properties": { + "fields": { + "description": "Subset of branch fields to return for each branch. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "name", + "sha", + "protected" + ], + "type": "string" + }, + "type": "array" + }, "owner": { "description": "Repository owner", "type": "string" diff --git a/pkg/github/__toolsnaps__/list_commits.snap b/pkg/github/__toolsnaps__/list_commits.snap index 1a773f217e..84640e4369 100644 --- a/pkg/github/__toolsnaps__/list_commits.snap +++ b/pkg/github/__toolsnaps__/list_commits.snap @@ -10,6 +10,22 @@ "description": "Author username or email address to filter commits by", "type": "string" }, + "fields": { + "description": "Subset of commit fields to return for each commit. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "sha", + "html_url", + "commit", + "author", + "committer", + "stats", + "files" + ], + "type": "string" + }, + "type": "array" + }, "owner": { "description": "Repository owner", "type": "string" diff --git a/pkg/github/__toolsnaps__/list_issue_types.snap b/pkg/github/__toolsnaps__/list_issue_types.snap index f1f1377a81..af6926d944 100644 --- a/pkg/github/__toolsnaps__/list_issue_types.snap +++ b/pkg/github/__toolsnaps__/list_issue_types.snap @@ -6,6 +6,22 @@ "description": "List supported issue types for repository owner (organization).", "inputSchema": { "properties": { + "fields": { + "description": "Subset of issue type fields to return for each issue type. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "id", + "node_id", + "name", + "description", + "color", + "created_at", + "updated_at" + ], + "type": "string" + }, + "type": "array" + }, "owner": { "description": "The organization owner of the repository", "type": "string" diff --git a/pkg/github/__toolsnaps__/list_issues.snap b/pkg/github/__toolsnaps__/list_issues.snap index a4be59bb0c..b87ebaa9ce 100644 --- a/pkg/github/__toolsnaps__/list_issues.snap +++ b/pkg/github/__toolsnaps__/list_issues.snap @@ -18,6 +18,35 @@ ], "type": "string" }, + "fields": { + "description": "Subset of issue fields to return for each issue. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "number", + "title", + "body", + "state", + "state_reason", + "draft", + "locked", + "html_url", + "user", + "author_association", + "labels", + "assignees", + "milestone", + "comments", + "reactions", + "created_at", + "updated_at", + "closed_at", + "closed_by", + "issue_type" + ], + "type": "string" + }, + "type": "array" + }, "labels": { "description": "Filter by labels", "items": { diff --git a/pkg/github/__toolsnaps__/list_pull_requests.snap b/pkg/github/__toolsnaps__/list_pull_requests.snap index 25f1268c64..48626df97d 100644 --- a/pkg/github/__toolsnaps__/list_pull_requests.snap +++ b/pkg/github/__toolsnaps__/list_pull_requests.snap @@ -18,6 +18,40 @@ ], "type": "string" }, + "fields": { + "description": "Subset of pull request fields to return for each pull request. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "number", + "title", + "body", + "state", + "draft", + "merged", + "mergeable_state", + "html_url", + "user", + "labels", + "assignees", + "requested_reviewers", + "merged_by", + "head", + "base", + "additions", + "deletions", + "changed_files", + "commits", + "comments", + "created_at", + "updated_at", + "closed_at", + "merged_at", + "milestone" + ], + "type": "string" + }, + "type": "array" + }, "head": { "description": "Filter by head user/org and branch", "type": "string" diff --git a/pkg/github/__toolsnaps__/list_releases.snap b/pkg/github/__toolsnaps__/list_releases.snap index 57502c3c86..c7bb6e9a68 100644 --- a/pkg/github/__toolsnaps__/list_releases.snap +++ b/pkg/github/__toolsnaps__/list_releases.snap @@ -6,6 +6,24 @@ "description": "List releases in a GitHub repository", "inputSchema": { "properties": { + "fields": { + "description": "Subset of release fields to return for each release. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "id", + "tag_name", + "name", + "body", + "html_url", + "published_at", + "prerelease", + "draft", + "author" + ], + "type": "string" + }, + "type": "array" + }, "owner": { "description": "Repository owner", "type": "string" diff --git a/pkg/github/__toolsnaps__/list_tags.snap b/pkg/github/__toolsnaps__/list_tags.snap index 1e66d2c1f6..b14dd511ae 100644 --- a/pkg/github/__toolsnaps__/list_tags.snap +++ b/pkg/github/__toolsnaps__/list_tags.snap @@ -6,6 +6,17 @@ "description": "List git tags in a GitHub repository", "inputSchema": { "properties": { + "fields": { + "description": "Subset of tag fields to return for each tag. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "name", + "sha" + ], + "type": "string" + }, + "type": "array" + }, "owner": { "description": "Repository owner", "type": "string" diff --git a/pkg/github/__toolsnaps__/search_issues.snap b/pkg/github/__toolsnaps__/search_issues.snap index beaa5b7376..f824c294c2 100644 --- a/pkg/github/__toolsnaps__/search_issues.snap +++ b/pkg/github/__toolsnaps__/search_issues.snap @@ -6,6 +6,35 @@ "description": "Search for issues in GitHub repositories using issues search syntax already scoped to is:issue", "inputSchema": { "properties": { + "fields": { + "description": "Subset of issue fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "number", + "title", + "body", + "state", + "state_reason", + "draft", + "locked", + "html_url", + "user", + "author_association", + "labels", + "assignees", + "milestone", + "comments", + "reactions", + "created_at", + "updated_at", + "closed_at", + "closed_by", + "issue_type" + ], + "type": "string" + }, + "type": "array" + }, "order": { "description": "Sort order", "enum": [ diff --git a/pkg/github/__toolsnaps__/search_pull_requests.snap b/pkg/github/__toolsnaps__/search_pull_requests.snap index 05376c0065..6b1bf6e3ff 100644 --- a/pkg/github/__toolsnaps__/search_pull_requests.snap +++ b/pkg/github/__toolsnaps__/search_pull_requests.snap @@ -6,6 +6,35 @@ "description": "Search for pull requests in GitHub repositories using issues search syntax already scoped to is:pr", "inputSchema": { "properties": { + "fields": { + "description": "Subset of pull request fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + "items": { + "enum": [ + "number", + "title", + "body", + "state", + "state_reason", + "draft", + "locked", + "html_url", + "user", + "author_association", + "labels", + "assignees", + "milestone", + "comments", + "reactions", + "created_at", + "updated_at", + "closed_at", + "closed_by", + "issue_type" + ], + "type": "string" + }, + "type": "array" + }, "order": { "description": "Sort order", "enum": [ diff --git a/pkg/github/issues.go b/pkg/github/issues.go index 82807f32be..a3e4d86e82 100644 --- a/pkg/github/issues.go +++ b/pkg/github/issues.go @@ -564,6 +564,14 @@ func ListIssueTypes(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "The organization owner of the repository", }, + "fields": { + Type: "array", + Description: "Subset of issue type fields to return for each issue type. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: issueTypeFieldEnum, + }, + }, }, Required: []string{"owner"}, }, @@ -575,6 +583,11 @@ func ListIssueTypes(t translations.TranslationHelperFunc) inventory.ServerTool { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } + client, err := deps.GetClient(ctx) if err != nil { return utils.NewToolResultErrorFromErr("failed to get GitHub client", err), nil, nil @@ -600,7 +613,16 @@ func ListIssueTypes(t translations.TranslationHelperFunc) inventory.ServerTool { } } - result, err := structuredTextResult(ctx, deps, issueTypes, MinimalIssueTypesResponse{IssueTypes: minimalIssueTypes}) + var payload any = MinimalIssueTypesResponse{IssueTypes: minimalIssueTypes} + if len(fields) > 0 { + filtered, err := filterEachField(minimalIssueTypes, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter issue types", err), nil, nil + } + payload = map[string]any{"issue_types": filtered} + } + + result, err := structuredTextResult(ctx, deps, payload, payload) if err != nil { return utils.NewToolResultErrorFromErr("failed to marshal issue types", err), nil, nil } @@ -967,6 +989,14 @@ func SearchIssues(t translations.TranslationHelperFunc) inventory.ServerTool { Description: "Sort order", Enum: []any{"asc", "desc"}, }, + "fields": { + Type: "array", + Description: "Subset of issue fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: issueFieldEnum, + }, + }, }, Required: []string{"query"}, } @@ -1425,6 +1455,14 @@ func ListIssues(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "Filter by date (ISO 8601 timestamp)", }, + "fields": { + Type: "array", + Description: "Subset of issue fields to return for each issue. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: issueFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, } @@ -1508,6 +1546,11 @@ func ListIssues(t translations.TranslationHelperFunc) inventory.ServerTool { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } + // There are two optional parameters: since and labels. var sinceTime time.Time var hasSince bool @@ -1597,7 +1640,20 @@ func ListIssues(t translations.TranslationHelperFunc) inventory.ServerTool { isPrivate = queryResult.GetIsPrivate() } - result, err := structuredTextResult(ctx, deps, resp, resp) + var payload any = resp + if len(fields) > 0 { + filteredItems, err := filterEachField(resp.Issues, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter issues", err), nil, nil + } + payload = map[string]any{ + "issues": filteredItems, + "totalCount": resp.TotalCount, + "pageInfo": resp.PageInfo, + } + } + + result, err := structuredTextResult(ctx, deps, payload, payload) if err != nil { return nil, nil, err } diff --git a/pkg/github/issues_test.go b/pkg/github/issues_test.go index 49ce2dde9c..f2b6f5591b 100644 --- a/pkg/github/issues_test.go +++ b/pkg/github/issues_test.go @@ -693,6 +693,57 @@ func Test_SearchIssues(t *testing.T) { } } +func Test_SearchIssues_FieldFiltering(t *testing.T) { + mockSearchResult := &github.IssuesSearchResult{ + Total: github.Ptr(1), + IncompleteResults: github.Ptr(false), + Issues: []*github.Issue{ + { + Number: github.Ptr(42), + Title: github.Ptr("Bug: Something is broken"), + Body: github.Ptr("This is a bug report"), + State: github.Ptr("open"), + HTMLURL: github.Ptr("https://github.com/owner/repo/issues/42"), + }, + }, + } + + serverTool := SearchIssues(translations.NullTranslationHelper) + client := github.NewClient(MockHTTPClientWithHandlers(map[string]http.HandlerFunc{ + GetSearchIssues: mockResponse(t, http.StatusOK, mockSearchResult), + })) + deps := BaseDeps{Client: client} + handler := serverTool.Handler(deps) + + request := createMCPRequest(map[string]any{ + "query": "repo:owner/repo is:open", + "fields": []any{"number", "title"}, + }) + + result, err := handler(ContextWithDeps(context.Background(), deps), &request) + require.NoError(t, err) + require.False(t, result.IsError) + + textContent := getTextResult(t, result) + + // The wrapper metadata is preserved while each item is reduced to the + // requested fields only. + var returned struct { + TotalCount int `json:"total_count"` + IncompleteResults bool `json:"incomplete_results"` + Items []map[string]any `json:"items"` + } + require.NoError(t, json.Unmarshal([]byte(textContent.Text), &returned)) + assert.Equal(t, 1, returned.TotalCount) + require.Len(t, returned.Items, 1) + require.Len(t, returned.Items[0], 2) + assert.Contains(t, returned.Items[0], "number") + assert.Contains(t, returned.Items[0], "title") + + assert.NotContains(t, textContent.Text, "html_url") + assert.NotContains(t, textContent.Text, "This is a bug report") +} + func Test_CreateIssue(t *testing.T) { // Verify tool definition once serverTool := IssueWrite(translations.NullTranslationHelper) @@ -1352,6 +1403,86 @@ func Test_ListIssues(t *testing.T) { } } +func Test_ListIssues_FieldFiltering(t *testing.T) { + serverTool := ListIssues(translations.NullTranslationHelper) + + mockIssues := []map[string]any{ + { + "number": 123, + "title": "First Issue", + "body": "This is the first test issue", + "state": "OPEN", + "databaseId": 1001, + "createdAt": "2023-01-01T00:00:00Z", + "updatedAt": "2023-01-01T00:00:00Z", + "author": map[string]any{"login": "user1"}, + "labels": map[string]any{ + "nodes": []map[string]any{ + {"name": "bug", "id": "label1", "description": "Bug label"}, + }, + }, + "comments": map[string]any{"totalCount": 5}, + }, + } + + mockResponse := githubv4mock.DataResponse(map[string]any{ + "repository": map[string]any{ + "issues": map[string]any{ + "nodes": mockIssues, + "pageInfo": map[string]any{ + "hasNextPage": false, + "hasPreviousPage": false, + "startCursor": "", + "endCursor": "", + }, + "totalCount": 1, + }, + "isPrivate": false, + }, + }) + + query := "query($after:String$direction:OrderDirection!$first:Int!$orderBy:IssueOrderField!$owner:String!$repo:String!$states:[IssueState!]!){repository(owner: $owner, name: $repo){issues(first: $first, after: $after, states: $states, orderBy: {field: $orderBy, direction: $direction}){nodes{number,title,body,state,databaseId,author{login},createdAt,updatedAt,labels(first: 100){nodes{name,id,description}},comments{totalCount}},pageInfo{hasNextPage,hasPreviousPage,startCursor,endCursor},totalCount},isPrivate}}" + vars := map[string]any{ + "owner": "owner", + "repo": "repo", + "states": []any{"OPEN", "CLOSED"}, + "orderBy": "CREATED_AT", + "direction": "DESC", + "first": float64(30), + "after": (*string)(nil), + } + + httpClient := githubv4mock.NewMockedHTTPClient(githubv4mock.NewQueryMatcher(query, vars, mockResponse)) + deps := BaseDeps{GQLClient: githubv4.NewClient(httpClient)} + handler := serverTool.Handler(deps) + + req := createMCPRequest(map[string]any{ + "owner": "owner", + "repo": "repo", + "fields": []any{"number", "title"}, + }) + res, err := handler(ContextWithDeps(context.Background(), deps), &req) + require.NoError(t, err) + require.False(t, res.IsError) + + text := getTextResult(t, res).Text + + // Wrapper metadata is preserved while each issue is reduced to the + // requested fields only. + var response struct { + Issues []map[string]any `json:"issues"` + TotalCount int `json:"totalCount"` + } + require.NoError(t, json.Unmarshal([]byte(text), &response)) + assert.Equal(t, 1, response.TotalCount) + require.Len(t, response.Issues, 1) + require.Len(t, response.Issues[0], 2) + assert.Contains(t, response.Issues[0], "number") + assert.Contains(t, response.Issues[0], "title") + + assert.NotContains(t, text, "This is the first test issue") +} + func Test_ListIssues_IFC_InsidersMode(t *testing.T) { t.Parallel() @@ -3367,19 +3498,75 @@ func Test_ListIssueTypes(t *testing.T) { textContent := getTextResult(t, result) // Unmarshal and verify the result - var returnedIssueTypes []*github.IssueType - err = json.Unmarshal([]byte(textContent.Text), &returnedIssueTypes) + var returnedResponse MinimalIssueTypesResponse + err = json.Unmarshal([]byte(textContent.Text), &returnedResponse) require.NoError(t, err) + returnedIssueTypes := returnedResponse.IssueTypes if tc.expectedIssueTypes != nil { require.Equal(t, len(tc.expectedIssueTypes), len(returnedIssueTypes)) for i, expected := range tc.expectedIssueTypes { - assert.Equal(t, *expected.Name, *returnedIssueTypes[i].Name) - assert.Equal(t, *expected.Description, *returnedIssueTypes[i].Description) - assert.Equal(t, *expected.Color, *returnedIssueTypes[i].Color) - assert.Equal(t, *expected.ID, *returnedIssueTypes[i].ID) + assert.Equal(t, *expected.Name, returnedIssueTypes[i].Name) + assert.Equal(t, *expected.Description, returnedIssueTypes[i].Description) + assert.Equal(t, *expected.Color, returnedIssueTypes[i].Color) + assert.Equal(t, *expected.ID, returnedIssueTypes[i].ID) } } }) } } + +func Test_ListIssueTypes_FieldFiltering(t *testing.T) { + mockIssueTypes := []*github.IssueType{ + { + ID: github.Ptr(int64(1)), + Name: github.Ptr("bug"), + Description: github.Ptr("Something isn't working"), + Color: github.Ptr("d73a4a"), + }, + { + ID: github.Ptr(int64(2)), + Name: github.Ptr("feature"), + Description: github.Ptr("New feature or enhancement"), + Color: github.Ptr("a2eeef"), + }, + } + + serverTool := ListIssueTypes(translations.NullTranslationHelper) + client := github.NewClient(MockHTTPClientWithHandlers(map[string]http.HandlerFunc{ + "GET /orgs/testorg/issue-types": mockResponse(t, http.StatusOK, mockIssueTypes), + })) + deps := BaseDeps{Client: client} + handler := serverTool.Handler(deps) + + request := createMCPRequest(map[string]any{ + "owner": "testorg", + "fields": []any{"name"}, + }) + + result, err := handler(ContextWithDeps(context.Background(), deps), &request) + require.NoError(t, err) + require.NotNil(t, result) + require.False(t, result.IsError) + + textContent := getTextResult(t, result) + + var returnedResponse MinimalIssueTypesResponse + require.NoError(t, json.Unmarshal([]byte(textContent.Text), &returnedResponse)) + require.Len(t, returnedResponse.IssueTypes, 2) + + // Only the requested "name" field should be populated; all others are + // dropped from the response (omitempty zeroes them out). + for i, expected := range mockIssueTypes { + got := returnedResponse.IssueTypes[i] + assert.Equal(t, *expected.Name, got.Name) + assert.Zero(t, got.ID) + assert.Empty(t, got.Description) + assert.Empty(t, got.Color) + assert.Empty(t, got.NodeID) + } + + // The raw JSON should not contain the filtered-out keys. + assert.NotContains(t, textContent.Text, "description") + assert.NotContains(t, textContent.Text, "color") +} diff --git a/pkg/github/minimal_types.go b/pkg/github/minimal_types.go index 815798dc11..ba8560c297 100644 --- a/pkg/github/minimal_types.go +++ b/pkg/github/minimal_types.go @@ -1,6 +1,8 @@ package github import ( + "bytes" + "encoding/json" "time" "github.com/google/go-github/v82/github" @@ -908,6 +910,73 @@ func convertToMinimalIssueType(issueType *github.IssueType) MinimalIssueType { return m } +// issueTypeFieldEnum lists the selectable fields for the list_issue_types tool, +// matching the JSON field names of MinimalIssueType. +var issueTypeFieldEnum = []any{"id", "node_id", "name", "description", "color", "created_at", "updated_at"} + +// issueFieldEnum lists the selectable fields for issue list and search results, +// matching the JSON field names of MinimalIssue. +var issueFieldEnum = []any{"number", "title", "body", "state", "state_reason", "draft", "locked", "html_url", "user", "author_association", "labels", "assignees", "milestone", "comments", "reactions", "created_at", "updated_at", "closed_at", "closed_by", "issue_type"} + +// pullRequestFieldEnum lists the selectable fields for pull request list results, +// matching the JSON field names of MinimalPullRequest. +var pullRequestFieldEnum = []any{"number", "title", "body", "state", "draft", "merged", "mergeable_state", "html_url", "user", "labels", "assignees", "requested_reviewers", "merged_by", "head", "base", "additions", "deletions", "changed_files", "commits", "comments", "created_at", "updated_at", "closed_at", "merged_at", "milestone"} + +// commitFieldEnum lists the selectable fields for list_commits results, +// matching the JSON field names of MinimalCommit. +var commitFieldEnum = []any{"sha", "html_url", "commit", "author", "committer", "stats", "files"} + +// branchFieldEnum lists the selectable fields for list_branches results, +// matching the JSON field names of MinimalBranch. +var branchFieldEnum = []any{"name", "sha", "protected"} + +// tagFieldEnum lists the selectable fields for list_tags results, +// matching the JSON field names of MinimalTag. +var tagFieldEnum = []any{"name", "sha"} + +// releaseFieldEnum lists the selectable fields for list_releases results, +// matching the JSON field names of MinimalRelease. +var releaseFieldEnum = []any{"id", "tag_name", "name", "body", "html_url", "published_at", "prerelease", "draft", "author"} + +// filterFields marshals v to a JSON object and returns a map containing only the +// requested fields. Fields that are unknown or absent from the JSON (for example +// empty values dropped via omitempty) are skipped. +func filterFields(v any, fields []string) (map[string]any, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() // preserve integer precision for fields such as IDs + var object map[string]any + if err := decoder.Decode(&object); err != nil { + return nil, err + } + + picked := make(map[string]any, len(fields)) + for _, field := range fields { + if value, ok := object[field]; ok { + picked[field] = value + } + } + return picked, nil +} + +// filterEachField applies filterFields to every item, returning a slice in which +// each element contains only the requested fields. +func filterEachField[T any](items []T, fields []string) ([]map[string]any, error) { + filtered := make([]map[string]any, 0, len(items)) + for _, item := range items { + picked, err := filterFields(item, fields) + if err != nil { + return nil, err + } + filtered = append(filtered, picked) + } + return filtered, nil +} + func convertToMinimalCodeSearchResult(result *github.CodeSearchResult) MinimalCodeSearchResult { minimalResult := MinimalCodeSearchResult{ TotalCount: result.GetTotal(), diff --git a/pkg/github/pullrequests.go b/pkg/github/pullrequests.go index 2c76100078..013f9e1e76 100644 --- a/pkg/github/pullrequests.go +++ b/pkg/github/pullrequests.go @@ -1121,6 +1121,14 @@ func ListPullRequests(t translations.TranslationHelperFunc) inventory.ServerTool Description: "Sort direction", Enum: []any{"asc", "desc"}, }, + "fields": { + Type: "array", + Description: "Subset of pull request fields to return for each pull request. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: pullRequestFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, } @@ -1167,6 +1175,10 @@ func ListPullRequests(t translations.TranslationHelperFunc) inventory.ServerTool if err != nil { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil, nil @@ -1226,14 +1238,25 @@ func ListPullRequests(t translations.TranslationHelperFunc) inventory.ServerTool } } - r, err := json.Marshal(minimalPRs) + var textPayload any = minimalPRs + var structured any = MinimalPullRequestsResponse{PullRequests: minimalPRs} + if len(fields) > 0 { + filteredPRs, err := filterEachField(minimalPRs, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter pull requests", err), nil, nil + } + textPayload = filteredPRs + structured = map[string]any{"pull_requests": filteredPRs} + } + + r, err := json.Marshal(textPayload) if err != nil { return utils.NewToolResultErrorFromErr("failed to marshal response", err), nil, nil } result := utils.NewToolResultText(string(r)) if outputSchemasEnabled(ctx, deps) { - result.StructuredContent = MinimalPullRequestsResponse{PullRequests: minimalPRs} + result.StructuredContent = structured } return result, nil, nil @@ -1388,6 +1411,14 @@ func SearchPullRequests(t translations.TranslationHelperFunc) inventory.ServerTo Description: "Sort order", Enum: []any{"asc", "desc"}, }, + "fields": { + Type: "array", + Description: "Subset of pull request fields to return for each result. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: issueFieldEnum, + }, + }, }, Required: []string{"query"}, } diff --git a/pkg/github/pullrequests_test.go b/pkg/github/pullrequests_test.go index 4f0ec9493b..b69eb61bf0 100644 --- a/pkg/github/pullrequests_test.go +++ b/pkg/github/pullrequests_test.go @@ -684,6 +684,48 @@ func Test_ListPullRequests(t *testing.T) { } } +func Test_ListPullRequests_FieldFiltering(t *testing.T) { + mockPRs := []*github.PullRequest{ + { + Number: github.Ptr(42), + Title: github.Ptr("First PR"), + State: github.Ptr("open"), + Body: github.Ptr("Body of the first PR"), + HTMLURL: github.Ptr("https://github.com/owner/repo/pull/42"), + }, + } + + serverTool := ListPullRequests(translations.NullTranslationHelper) + client := github.NewClient(MockHTTPClientWithHandlers(map[string]http.HandlerFunc{ + GetReposPullsByOwnerByRepo: mockResponse(t, http.StatusOK, mockPRs), + })) + deps := BaseDeps{Client: client} + handler := serverTool.Handler(deps) + + request := createMCPRequest(map[string]any{ + "owner": "owner", + "repo": "repo", + "fields": []any{"number", "title"}, + }) + + result, err := handler(ContextWithDeps(context.Background(), deps), &request) + require.NoError(t, err) + require.False(t, result.IsError) + + textContent := getTextResult(t, result) + + var returned []map[string]any + require.NoError(t, json.Unmarshal([]byte(textContent.Text), &returned)) + require.Len(t, returned, 1) + require.Len(t, returned[0], 2) + assert.Contains(t, returned[0], "number") + assert.Contains(t, returned[0], "title") + + // Fields that were not requested must be absent from the payload. + assert.NotContains(t, textContent.Text, "html_url") + assert.NotContains(t, textContent.Text, "Body of the first PR") +} + func Test_MergePullRequest(t *testing.T) { // Verify tool definition once serverTool := MergePullRequest(translations.NullTranslationHelper) @@ -1085,6 +1127,54 @@ func Test_SearchPullRequests(t *testing.T) { } +func Test_SearchPullRequests_FieldFiltering(t *testing.T) { + mockSearchResult := &github.IssuesSearchResult{ + Total: github.Ptr(1), + IncompleteResults: github.Ptr(false), + Issues: []*github.Issue{ + { + Number: github.Ptr(42), + Title: github.Ptr("Add new feature"), + Body: github.Ptr("This is a pull request body"), + State: github.Ptr("open"), + HTMLURL: github.Ptr("https://github.com/owner/repo/pull/42"), + }, + }, + } + + serverTool := SearchPullRequests(translations.NullTranslationHelper) + client := github.NewClient(MockHTTPClientWithHandlers(map[string]http.HandlerFunc{ + GetSearchIssues: mockResponse(t, http.StatusOK, mockSearchResult), + })) + deps := BaseDeps{Client: client} + handler := serverTool.Handler(deps) + + request := createMCPRequest(map[string]any{ + "query": "repo:owner/repo is:open", + "fields": []any{"number", "title"}, + }) + + result, err := handler(ContextWithDeps(context.Background(), deps), &request) + require.NoError(t, err) + require.False(t, result.IsError) + + textContent := getTextResult(t, result) + + var returned struct { + TotalCount int `json:"total_count"` + Items []map[string]any `json:"items"` + } + require.NoError(t, json.Unmarshal([]byte(textContent.Text), &returned)) + assert.Equal(t, 1, returned.TotalCount) + require.Len(t, returned.Items, 1) + require.Len(t, returned.Items[0], 2) + assert.Contains(t, returned.Items[0], "number") + assert.Contains(t, returned.Items[0], "title") + + assert.NotContains(t, textContent.Text, "html_url") + assert.NotContains(t, textContent.Text, "This is a pull request body") +} + func Test_GetPullRequestFiles(t *testing.T) { // Verify tool definition once serverTool := PullRequestRead(translations.NullTranslationHelper) diff --git a/pkg/github/repositories.go b/pkg/github/repositories.go index da1dace9d6..400eda0996 100644 --- a/pkg/github/repositories.go +++ b/pkg/github/repositories.go @@ -159,6 +159,14 @@ func ListCommits(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "Only commits before this date will be returned (ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ or YYYY-MM-DD)", }, + "fields": { + Type: "array", + Description: "Subset of commit fields to return for each commit. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: commitFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, }), @@ -193,6 +201,10 @@ func ListCommits(t translations.TranslationHelperFunc) inventory.ServerTool { if err != nil { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil, nil @@ -254,7 +266,18 @@ func ListCommits(t translations.TranslationHelperFunc) inventory.ServerTool { minimalCommits[i] = convertToMinimalCommit(commit, false) } - result, err := structuredTextResult(ctx, deps, minimalCommits, MinimalCommitsResponse{Commits: minimalCommits}) + var textPayload any = minimalCommits + var structuredPayload any = MinimalCommitsResponse{Commits: minimalCommits} + if len(fields) > 0 { + filtered, err := filterEachField(minimalCommits, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter commits", err), nil, nil + } + textPayload = filtered + structuredPayload = map[string]any{"commits": filtered} + } + + result, err := structuredTextResult(ctx, deps, textPayload, structuredPayload) if err != nil { return nil, nil, fmt.Errorf("failed to marshal response: %w", err) } @@ -286,6 +309,14 @@ func ListBranches(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "Repository name", }, + "fields": { + Type: "array", + Description: "Subset of branch fields to return for each branch. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: branchFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, }), @@ -300,6 +331,10 @@ func ListBranches(t translations.TranslationHelperFunc) inventory.ServerTool { if err != nil { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil, nil @@ -341,7 +376,18 @@ func ListBranches(t translations.TranslationHelperFunc) inventory.ServerTool { minimalBranches = append(minimalBranches, convertToMinimalBranch(branch)) } - result, err := structuredTextResult(ctx, deps, minimalBranches, MinimalBranchesResponse{Branches: minimalBranches}) + var textPayload any = minimalBranches + var structuredPayload any = MinimalBranchesResponse{Branches: minimalBranches} + if len(fields) > 0 { + filtered, err := filterEachField(minimalBranches, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter branches", err), nil, nil + } + textPayload = filtered + structuredPayload = map[string]any{"branches": filtered} + } + + result, err := structuredTextResult(ctx, deps, textPayload, structuredPayload) if err != nil { return nil, nil, fmt.Errorf("failed to marshal response: %w", err) } @@ -1527,6 +1573,14 @@ func ListTags(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "Repository name", }, + "fields": { + Type: "array", + Description: "Subset of tag fields to return for each tag. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: tagFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, }), @@ -1541,6 +1595,10 @@ func ListTags(t translations.TranslationHelperFunc) inventory.ServerTool { if err != nil { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil, nil @@ -1581,7 +1639,18 @@ func ListTags(t translations.TranslationHelperFunc) inventory.ServerTool { } } - result, err := structuredTextResult(ctx, deps, minimalTags, MinimalTagsResponse{Tags: minimalTags}) + var textPayload any = minimalTags + var structuredPayload any = MinimalTagsResponse{Tags: minimalTags} + if len(fields) > 0 { + filtered, err := filterEachField(minimalTags, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter tags", err), nil, nil + } + textPayload = filtered + structuredPayload = map[string]any{"tags": filtered} + } + + result, err := structuredTextResult(ctx, deps, textPayload, structuredPayload) if err != nil { return nil, nil, fmt.Errorf("failed to marshal response: %w", err) } @@ -1719,6 +1788,14 @@ func ListReleases(t translations.TranslationHelperFunc) inventory.ServerTool { Type: "string", Description: "Repository name", }, + "fields": { + Type: "array", + Description: "Subset of release fields to return for each release. If omitted, all fields are returned. Use this to reduce response size when you only need specific fields.", + Items: &jsonschema.Schema{ + Type: "string", + Enum: releaseFieldEnum, + }, + }, }, Required: []string{"owner", "repo"}, }), @@ -1733,6 +1810,10 @@ func ListReleases(t translations.TranslationHelperFunc) inventory.ServerTool { if err != nil { return utils.NewToolResultError(err.Error()), nil, nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil, nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil, nil @@ -1769,7 +1850,18 @@ func ListReleases(t translations.TranslationHelperFunc) inventory.ServerTool { } } - result, err := structuredTextResult(ctx, deps, minimalReleases, MinimalReleasesResponse{Releases: minimalReleases}) + var textPayload any = minimalReleases + var structuredPayload any = MinimalReleasesResponse{Releases: minimalReleases} + if len(fields) > 0 { + filtered, err := filterEachField(minimalReleases, fields) + if err != nil { + return utils.NewToolResultErrorFromErr("failed to filter releases", err), nil, nil + } + textPayload = filtered + structuredPayload = map[string]any{"releases": filtered} + } + + result, err := structuredTextResult(ctx, deps, textPayload, structuredPayload) if err != nil { return nil, nil, fmt.Errorf("failed to marshal response: %w", err) } diff --git a/pkg/github/search_utils.go b/pkg/github/search_utils.go index 78b7985d16..2bd0924bf9 100644 --- a/pkg/github/search_utils.go +++ b/pkg/github/search_utils.go @@ -74,6 +74,10 @@ func searchHandler( if err != nil { return utils.NewToolResultError(err.Error()), nil } + fields, err := OptionalStringArrayParam(args, "fields") + if err != nil { + return utils.NewToolResultError(err.Error()), nil + } pagination, err := OptionalPaginationParams(args) if err != nil { return utils.NewToolResultError(err.Error()), nil @@ -112,5 +116,22 @@ func searchHandler( return utils.NewToolResultErrorFromErr(errorPrefix+": failed to marshal response", err), nil } + if len(fields) > 0 { + minimal := convertToMinimalSearchIssuesResult(result) + filteredItems, err := filterEachField(minimal.Items, fields) + if err != nil { + return utils.NewToolResultErrorFromErr(errorPrefix+": failed to filter response", err), nil + } + filtered := map[string]any{ + "total_count": minimal.TotalCount, + "incomplete_results": minimal.IncompleteResults, + "items": filteredItems, + } + toolResult, err = structuredTextResult(ctx, deps, filtered, filtered) + if err != nil { + return utils.NewToolResultErrorFromErr(errorPrefix+": failed to marshal response", err), nil + } + } + return toolResult, nil }