From 882cc291edb438f66a4a2b7c364162bef6b52da4 Mon Sep 17 00:00:00 2001 From: mornyx Date: Sun, 19 Apr 2026 20:05:39 +0800 Subject: [PATCH 1/6] feat(fs): add filesystem operations for TiDB Cloud FS Add ticloud fs command group for managing files and directories in TiDB Cloud FS, including: - Basic file operations: ls, cat, cp, mkdir, mv, rm, stat - Search: grep, find - SQL execution via fs sql - Interactive shell via fs shell - Secret vault management: set, get, exec, ls, rm, grant, revoke, audit - FUSE mount support - FS client with multipart upload and resume - Documentation for TiDB Cloud FS usage Also adds fs-related config properties: - fs.cluster-id - fs.zero-instance-id - fs-endpoint --- docs/fs.md | 338 +++++ go.mod | 3 + go.sum | 8 +- internal/cli/fs/README.md | 367 ++++++ internal/cli/fs/cat.go | 54 + internal/cli/fs/cp.go | 128 ++ internal/cli/fs/find.go | 135 ++ internal/cli/fs/fs.go | 220 ++++ internal/cli/fs/grep.go | 68 + internal/cli/fs/init.go | 68 + internal/cli/fs/ls.go | 81 ++ internal/cli/fs/mkdir.go | 45 + internal/cli/fs/mount.go | 134 ++ internal/cli/fs/mv.go | 52 + internal/cli/fs/rm.go | 46 + internal/cli/fs/secret.go | 600 +++++++++ internal/cli/fs/shell.go | 228 ++++ internal/cli/fs/sql.go | 73 ++ internal/cli/fs/stat.go | 54 + internal/cli/root.go | 2 + internal/config/config.go | 9 +- internal/config/profile.go | 19 + internal/flag/flag.go | 5 + internal/iostream/iostream.go | 3 + internal/prop/property.go | 5 +- internal/service/fs/client.go | 1596 +++++++++++++++++++++++ internal/service/fs/client_auth_test.go | 133 ++ internal/service/fs/fuse/fs.go | 497 +++++++ internal/service/fs/fuse/mount.go | 107 ++ 29 files changed, 5071 insertions(+), 7 deletions(-) create mode 100644 docs/fs.md create mode 100644 internal/cli/fs/README.md create mode 100644 internal/cli/fs/cat.go create mode 100644 internal/cli/fs/cp.go create mode 100644 internal/cli/fs/find.go create mode 100644 internal/cli/fs/fs.go create mode 100644 internal/cli/fs/grep.go create mode 100644 internal/cli/fs/init.go create mode 100644 internal/cli/fs/ls.go create mode 100644 internal/cli/fs/mkdir.go create mode 100644 internal/cli/fs/mount.go create mode 100644 internal/cli/fs/mv.go create mode 100644 internal/cli/fs/rm.go create mode 100644 internal/cli/fs/secret.go create mode 100644 internal/cli/fs/shell.go create mode 100644 internal/cli/fs/sql.go create mode 100644 internal/cli/fs/stat.go create mode 100644 internal/service/fs/client.go create mode 100644 internal/service/fs/client_auth_test.go create mode 100644 internal/service/fs/fuse/fs.go create mode 100644 internal/service/fs/fuse/mount.go diff --git a/docs/fs.md b/docs/fs.md new file mode 100644 index 00000000..02768697 --- /dev/null +++ b/docs/fs.md @@ -0,0 +1,338 @@ +# TiDB Cloud FS (ticloud fs) 使用指南 + +**TiDB Cloud FS** 是 TiDB Cloud 提供的托管文件存储服务，为 TiDB 实例提供统一的文件系统抽象。你可以像操作本地文件系统一样在 TiDB Cloud 中存储、读取和管理文件，同时支持 SQL 查询、密钥保险箱（Vault）以及 FUSE 挂载等高级能力。 + +**`ticloud fs`** 是 TiDB Cloud CLI 中用于操作 TiDB Cloud FS 的命令组，支持上传下载、目录浏览、SQL 执行、密钥管理以及 FUSE 挂载等功能。 + +--- + +## 前置依赖：关联一个 TiDB 实例 + +FS 的数据存储依赖于一个具体的 TiDB 实例。根据你拥有的实例类型，需要将其 ID 绑定到 CLI 配置中，并完成一次性的 FS 初始化（`init`）。 + +### 使用 Serverless TiDB + +如果你已经有一个 **TiDB Cloud Serverless** 集群（可以通过 [TiDB Cloud Web Console](https://tidbcloud.com/) 创建，也可以使用 `ticloud serverless create` 命令创建），请按以下步骤配置： + +1. **配置鉴权** + + Serverless 集群受 TiDB Cloud 平台统一管控，访问其 FS 服务时必须提供有效的平台鉴权信息。你可以选择以下任一方式： + + - **OAuth 登录（推荐）** + + ```bash + ticloud auth login + ``` + + - **API Key 鉴权** + + ```bash + ticloud config set public-key + ticloud config set private-key + ``` + +2. **配置 cluster-id** + + ```bash + ticloud config set fs.cluster-id + ``` + +3. **初始化 FS** + + ```bash + ticloud fs init --user admin --password + ``` + + > 每个新的 Serverless 集群在首次使用 FS 前，都必须执行 `init` 来为其开通 FS 租户。 + +### 使用 Zero TiDB + +如果你使用的是 **TiDB Zero** 实例，配置方式如下。如果你还不了解 TiDB Zero，可以访问 [https://zero.tidbcloud.com/](https://zero.tidbcloud.com/) 了解更多。 + +1. **配置 zero-instance-id** + + ```bash + ticloud config set fs.zero-instance-id + ``` + + TiDB Zero 采用独立的部署与访问模型，**不需要**执行 `ticloud auth login`，也**不需要**配置 `public-key` / `private-key`。 + +2. **初始化 FS** + + ```bash + ticloud fs init --user admin --password + ``` + + > 与 Serverless 相同，新的 Zero 实例也需要执行一次 `init` 来开通 FS 租户。 + +--- + +## 路径格式说明 + +FS 使用 `:/` 前缀表示远程路径： + +- `:/` — 远程根目录 +- `:/path/to/file.txt` — 远程文件路径 + +--- + +## 子命令手册 + +### 初始化与配置 + +在使用任何文件操作之前，必须先为关联的数据库创建 FS 租户。这一步只需要执行一次。 + +#### `ticloud fs init` + +为当前关联的数据库初始化 FS 租户，每个新数据库在首次使用 FS 前必须执行一次。 + +```bash +ticloud fs init --user admin --password secret +``` + +### 基础文件操作 + +`ticloud fs` 提供了一组类似 Unix 文件系统的命令，用于管理远程文件和目录。你可以像操作本地文件一样进行上传、下载、浏览和删除。 + +#### `ticloud fs ls` + +列出远程目录内容。支持 `-l` 查看详情。 + +```bash +# 查看根目录 +ticloud fs ls :/ + +# 查看某个文件夹，并显示文件大小等详细信息 +ticloud fs ls -l :/myfolder +``` + +#### `ticloud fs cat` + +显示远程文件内容，类似于 Unix 的 `cat` 命令，适合快速查看文本文件。 + +```bash +ticloud fs cat :/readme.txt +``` + +#### `ticloud fs cp` + +复制文件。支持本地↔远程、远程↔远程、从标准输入上传，以及断点续传大文件。 + +```bash +# 上传本地文件到远程目录 +ticloud fs cp local.txt :/remote/ + +# 从远程下载文件到当前目录 +ticloud fs cp :/remote/file.txt . + +# 通过管道将标准输入内容写入远程文件 +echo "hello" | ticloud fs cp - :/file.txt + +# 断点续传大文件，适合网络不稳定场景 +ticloud fs cp --resume large.zip :/remote/ +``` + +#### `ticloud fs mkdir` + +创建远程目录，支持多级路径。 + +```bash +ticloud fs mkdir :/mydir +``` + +#### `ticloud fs mv` + +移动或重命名远程文件/目录。 + +```bash +ticloud fs mv :/oldname :/newname +``` + +#### `ticloud fs rm` + +删除远程文件或空目录。 + +```bash +ticloud fs rm :/file.txt +``` + +#### `ticloud fs stat` + +查看远程文件或目录的元数据，例如大小、修改时间等。 + +```bash +ticloud fs stat :/file.txt +``` + +### 搜索与查询 + +除了基础文件操作，`ticloud fs` 还支持直接在远程存储中搜索文件内容和执行 SQL 查询，方便你快速定位信息或进行数据分析。 + +#### `ticloud fs grep` + +在远程文件中搜索匹配内容。支持 `--limit` 限制结果数量。 + +```bash +ticloud fs grep "TODO" :/project --limit 20 +``` + +#### `ticloud fs find` + +按条件查找远程文件。支持 `-name`、`-size`、`-newer` 等过滤条件，类似 Unix 的 `find` 命令。 + +```bash +# 按文件名后缀查找 +ticloud fs find :/ -name "*.go" + +# 查找大于 1MB 的文件 +ticloud fs find :/ -size +1M +``` + +#### `ticloud fs sql` + +在 FS 后端执行 SQL 查询，可以直接对存储的数据进行结构化查询。 + +```bash +ticloud fs sql -q "SELECT 1" +``` + +### Secret 管理 + +TiDB Cloud FS 内置了一个密钥保险箱（Vault），用于安全地存储敏感信息，例如数据库连接串、API Key 等。你可以通过子命令对这些密钥进行增删改查，还可以为第三方 Agent 颁发受限访问令牌。 + +#### `ticloud fs secret set` + +创建或更新密钥。字段值可以直接赋值、从文件读取（`@file`）或从标准输入读取（`-`）。 + +```bash +# 直接设置字段 +ticloud fs secret set myapp DATABASE_URL=mysql://... + +# 从文件读取字段值，并从标准输入读取密码 +ticloud fs secret set myapp key=@secret.txt password=- +``` + +#### `ticloud fs secret get` + +读取密钥或指定字段。支持 `--json` 和 `--env` 输出格式。 + +```bash +# 读取整个密钥 +ticloud fs secret get myapp + +# 读取指定字段，并以 JSON 格式输出 +ticloud fs secret get myapp/password --json +``` + +#### `ticloud fs secret exec` + +将密钥字段作为环境变量注入后执行指定命令，常用于在本地脚本或 CI 流程中安全地使用密钥。 + +```bash +ticloud fs secret exec myapp -- ./run.sh +``` + +#### `ticloud fs secret ls` + +列出所有密钥。支持 `--json` 输出。 + +```bash +ticloud fs secret ls +``` + +#### `ticloud fs secret rm` + +删除指定密钥。 + +```bash +ticloud fs secret rm myapp +``` + +#### `ticloud fs secret grant` + +为指定 Agent 颁发受限能力令牌，限制其只能访问特定的密钥范围。需要指定 `--agent` 和 `--ttl`。 + +```bash +ticloud fs secret grant --agent myagent --ttl 1h myapp/password +``` + +#### `ticloud fs secret revoke` + +撤销一个已颁发的能力令牌，使其立即失效。 + +```bash +ticloud fs secret revoke tok_abc123 +``` + +#### `ticloud fs secret audit` + +查询密钥审计日志，追踪谁在什么时间访问了哪些密钥。支持按 `--secret`、`--agent`、`--since` 过滤，以及 `--limit` 限制条数。 + +```bash +# 查看最近的 50 条审计记录 +ticloud fs secret audit --limit 50 + +# 查看某个密钥最近 24 小时内的访问记录 +ticloud fs secret audit --secret myapp --since 24h +``` + +### 交互式 Shell + +如果你需要频繁执行多条 FS 命令，可以使用交互式 Shell，避免每次都要输入完整命令前缀。 + +#### `ticloud fs shell` + +启动交互式 FS Shell。支持 `cd`、`pwd`、`ls`、`cat`、`cp`、`mkdir`、`mv`、`rm`、`sql`、`stat`、`help`、`exit` 等命令。提示符为 `ticloud:fs>`。 + +```bash +ticloud fs shell +``` + +### FUSE 挂载 + +通过 FUSE 挂载，你可以将远程 FS 映射为本地目录，直接使用系统自带的文件管理器或命令行工具访问。 + +#### `ticloud fs mount` + +将远程 FS 挂载到本地目录。当前版本默认只读。支持 `--debug`（开启调试日志）和 `--allow-other`（允许其他用户访问）。 + +```bash +ticloud fs mount /mnt/tidbcloud +``` + +#### `ticloud fs umount` + +卸载本地挂载点，安全断开与远程 FS 的连接。 + +```bash +ticloud fs umount /mnt/tidbcloud +``` + +--- + +## 配置项速查 + +| 配置项 | 说明 | 默认值 | +|--------|------|--------| +| `fs.cluster-id` | 关联的 Serverless 集群 ID | — | +| `fs.zero-instance-id` | 关联的 Zero 实例 ID | — | +| `fs-endpoint` | FS 服务端点地址 | `https://fs.tidbapi.com/` | +| `public-key` / `private-key` | Serverless 场景下的 API Key 鉴权 | — | + +也支持通过环境变量覆盖： + +- `TICLOUD_FS_ENDPOINT` — 覆盖 `fs-endpoint` +- `TICLOUD_FS_CLUSTER_ID` — 覆盖 `fs.cluster-id` +- `TICLOUD_FS_ZERO_INSTANCE_ID` — 覆盖 `fs.zero-instance-id` + +--- + +## 常见问题 + +**Q: 执行 `ticloud fs ls :/` 时提示 `tenant not found`？** + +A: 说明当前关联的数据库尚未初始化 FS。请运行 `ticloud fs init --user --password ` 完成初始化。 + +**Q: Serverless 集群报 401 Unauthorized？** + +A: 请确认已完成 `ticloud auth login` 或正确配置了 `public-key` / `private-key`。 diff --git a/go.mod b/go.mod index 48e010f3..d17a5398 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/go-sql-driver/mysql v1.8.1 github.com/google/go-github/v49 v49.0.0 + github.com/hanwen/go-fuse/v2 v2.9.0 github.com/hashicorp/go-version v1.6.0 github.com/icholy/digest v0.1.22 github.com/juju/errors v1.0.0 @@ -124,3 +125,5 @@ require ( ) replace github.com/tidbcloud/tidbcloud-cli/pkg => ./pkg + +replace github.com/golang-module/carbon/v2 => github.com/dromara/carbon/v2 v2.3.12 diff --git a/go.sum b/go.sum index 0b7e1191..11d67a0e 100644 --- a/go.sum +++ b/go.sum @@ -269,6 +269,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dromara/carbon/v2 v2.3.12 h1:Fj9164o0w5jfcqDjvEOZSYdoajsBxpslm7PvhATcokc= +github.com/dromara/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= @@ -341,8 +343,6 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k= -github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -386,6 +386,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58= +github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= @@ -518,6 +520,8 @@ github.com/mithrandie/ternary v1.1.1 h1:k/joD6UGVYxHixYmSR8EGgDFNONBMqyD373xT4QR github.com/mithrandie/ternary v1.1.1/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= diff --git a/internal/cli/fs/README.md b/internal/cli/fs/README.md new file mode 100644 index 00000000..c10edea8 --- /dev/null +++ b/internal/cli/fs/README.md @@ -0,0 +1,367 @@ +# TiDB Cloud FS Commands + +The `ticloud fs` command group provides filesystem-like operations for managing files and directories in TiDB Cloud FS. + +## Overview + +TiDB Cloud FS is a managed file storage service that allows you to store and retrieve files using familiar filesystem semantics. The CLI provides both direct API operations and FUSE mount capabilities. + +## Prerequisites + +- TiDB Cloud CLI (`ticloud`) installed +- TiDB Cloud authentication configured (OAuth login or API key pair) + +## Configuration + +The `fs` command reuses your existing TiDB Cloud CLI authentication: + +```bash +# OAuth login +ticloud auth login + +# Or configure API keys +ticloud config set public-key +ticloud config set private-key +``` + +### Database Association + +FS operations can be associated with a specific database: + +```bash +# Associate with a TiDB Cloud serverless cluster +ticloud config set fs.cluster-id + +# Associate with a TiDB Zero instance +ticloud config set fs.zero-instance-id +``` + +These are sent as `X-TIDBCLOUD-CLUSTER-ID` and `X-TIDBCLOUD-ZERO-INSTANCE-ID` headers respectively. If both are set, `cluster-id` takes precedence. + +### Server URL + +The default FS server URL is `https://fs.tidbapi.com/`. You can override it: + +```bash +ticloud config set fs-endpoint +# or +export TICLOUD_FS_SERVER= +``` + +## Path Format + +Remote paths use the `:/` prefix: + +- `:/` - Root directory +- `:/path/to/file.txt` - File path + +## Commands + +### Basic Operations + +#### List Directory + +```bash +# List root directory +ticloud fs ls :/ + +# List with details +ticloud fs ls -l :/myfolder + +# List subdirectory +ticloud fs ls :/docs +``` + +#### Display File Contents + +```bash +ticloud fs cat :/readme.txt +ticloud fs cat :/docs/guide.md +``` + +#### Copy Files + +Upload local to remote: +```bash +ticloud fs cp local.txt :/remote/ +ticloud fs cp ./data.json :/backup/ +``` + +Download remote to local: +```bash +ticloud fs cp :/readme.txt ./ +ticloud fs cp :/docs/guide.md ./local-docs/ +``` + +Server-side copy: +```bash +ticloud fs cp :/old/path :/new/path +``` + +From stdin: +```bash +echo "content" | ticloud fs cp - :/file.txt +``` + +Resume interrupted upload: +```bash +ticloud fs cp --resume large.zip :/remote/ +``` + +#### Create Directory + +```bash +ticloud fs mkdir :/mydir +ticloud fs mkdir :/parent/child +``` + +#### Move/Rename + +```bash +ticloud fs mv :/oldname :/newname +ticloud fs mv :/folder/file.txt :/other/file.txt +``` + +#### Remove + +```bash +ticloud fs rm :/file.txt +ticloud fs rm :/empty_folder/ +``` + +#### File Metadata + +```bash +ticloud fs stat :/file.txt +``` + +### SQL Execution + +Execute SQL queries against the FS backend: + +```bash +ticloud fs sql -q "SELECT 1" +ticloud fs sql -f query.sql +``` + +### Secret Management + +Manage secrets in the TiDB Cloud FS vault: + +```bash +# Set a secret +ticloud fs secret set myapp DATABASE_URL=mysql://... + +# Get a secret +ticloud fs secret get myapp +ticloud fs secret get myapp/password + +# List secrets +ticloud fs secret ls + +# Delete a secret +ticloud fs secret rm myapp + +# Run a command with secret env vars +ticloud fs secret exec myapp -- ./run.sh + +# Issue a scoped capability token +ticloud fs secret grant --agent myagent --ttl 1h myapp/password + +# Revoke a token +ticloud fs secret revoke tok_abc123 + +# Query audit events +ticloud fs secret audit --limit 50 +``` + +### Search Operations + +#### Search File Contents (Grep) + +```bash +# Search for pattern +ticloud fs grep "function main" :/ + +# Search with limit +ticloud fs grep "TODO" :/myproject --limit 20 +``` + +#### Find Files + +```bash +# Find by name pattern +ticloud fs find :/ -name "*.go" + +# Find by size (larger than 1MB) +ticloud fs find :/ -size +1M + +# Find modified after date +ticloud fs find :/ -newer 2024-01-01 + +# Combined filters +ticloud fs find :/data -name "*.json" -newer 2024-01-01 +``` + +### Interactive Shell + +Start an interactive shell for filesystem operations: + +```bash +ticloud fs shell +``` + +Shell commands: +- `cd ` - Change directory +- `pwd` - Print current directory +- `ls [path]` - List directory +- `cat ` - Display file +- `cp ` - Copy files +- `mkdir ` - Create directory +- `mv ` - Move/rename +- `rm ` - Remove +- `sql ` - Execute SQL query +- `stat ` - Show metadata +- `help` - Show help +- `exit` - Exit shell + +Prompt: `ticloud:fs>` + +### FUSE Mount + +Mount the remote filesystem as a local directory using FUSE. + +#### Prerequisites + +**Linux:** +- Install FUSE: `sudo apt-get install fuse3` (Debian/Ubuntu) or `sudo yum install fuse3` (RHEL/CentOS) + +**macOS:** +- Install macFUSE from https://osxfuse.github.io/ + +#### Mount + +```bash +# Create mount point +mkdir -p /mnt/tidbcloud + +# Mount (read-only, default for MVP) +ticloud fs mount /mnt/tidbcloud + +# Mount with debug logging +ticloud fs mount /mnt/tidbcloud --debug + +# Mount allowing other users +ticloud fs mount /mnt/tidbcloud --allow-other +``` + +#### Use Mounted Filesystem + +Once mounted, use standard filesystem commands: + +```bash +# List files +ls /mnt/tidbcloud + +# Read files +cat /mnt/tidbcloud/readme.txt + +# Copy from mount +cp /mnt/tidbcloud/file.txt ./ +``` + +#### Unmount + +```bash +ticloud fs umount /mnt/tidbcloud +``` + +Or use system commands: +```bash +# Linux +fusermount3 -u /mnt/tidbcloud + +# macOS +umount /mnt/tidbcloud +``` + +## Examples + +### Backup Workflow + +```bash +# Create backup directory +ticloud fs ls :/backups 2>/dev/null || ticloud fs mkdir :/backups + +# Upload backup +tar czf - /data | ticloud fs cp - :/backups/data-$(date +%Y%m%d).tar.gz + +# List backups +ticloud fs ls :/backups +``` + +### Development Workflow + +```bash +# Start interactive shell +ticloud fs shell + +# In shell: +ticloud:fs> cd /myproject +ticloud:fs> ls +ticloud:fs> cat config.json +ticloud:fs> exit +``` + +### Search and Filter + +```bash +# Find all Go files modified recently +ticloud fs find :/ -name "*.go" -newer $(date -v-7d +%Y-%m-%d) + +# Search for function definitions +ticloud fs grep "^func " :/src --limit 50 +``` + +## Limitations + +### MVP (Current Version) + +- **FUSE mount is read-only** - Write operations via FUSE will be added in a future release +- **No symlink support** - Symlinks are not supported +- **Limited concurrent uploads** - Large file uploads may be slower + +### Planned Features + +- Full read-write FUSE mount +- Streaming large file operations +- Progress bars for transfers +- Batch operations +- File versioning + +## Troubleshooting + +### FUSE Mount Issues + +**Error: "fuse: device not found"** +- Ensure FUSE is installed: `modprobe fuse` (Linux) or install macFUSE (macOS) + +**Error: "permission denied"** +- Check mount point permissions +- Try with `--allow-other` flag +- On Linux, add user to `fuse` group: `sudo usermod -a -G fuse $USER` + +**Error: "transport endpoint is not connected"** +- The mount is stale, try unmounting and remounting + +### Connection Issues + +**Error: "cannot reach FS server"** +- Check network connectivity +- Verify `fs-endpoint` config: `ticloud config describe` +- Check TiDB Cloud auth status: `ticloud auth status` + +## See Also + +- `ticloud config` - Manage CLI configuration +- `ticloud auth` - Authentication commands diff --git a/internal/cli/fs/cat.go b/internal/cli/fs/cat.go new file mode 100644 index 00000000..901c1dd5 --- /dev/null +++ b/internal/cli/fs/cat.go @@ -0,0 +1,54 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + "io" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func catCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "cat ", + Short: "Display file contents", + Example: ` ticloud fs cat :/file.txt + ticloud fs cat :/myfolder/data.json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + rp := ParseRemotePath(args[0]) + path := rp.Path + + ctx := context.Background() + reader, err := client.ReadStream(ctx, path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("read %s: %w", path, err)) + } + defer reader.Close() + + _, err = io.Copy(h.IOStreams.Out, reader) + return suggestInitIfTenantNotFound(err) + }, + } +} diff --git a/internal/cli/fs/cp.go b/internal/cli/fs/cp.go new file mode 100644 index 00000000..4758dd4d --- /dev/null +++ b/internal/cli/fs/cp.go @@ -0,0 +1,128 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func cpCmd(h *internal.Helper) *cobra.Command { + var resume bool + var cmd = &cobra.Command{ + Use: "cp ", + Short: "Copy files between local and remote", + Long: `Copy files between local and remote filesystems. + +Source and destination can be: + - Local paths (any path not starting with :) + - Remote paths (starting with :/) + - - for stdin (source) or stdout (destination) + +Examples: + ticloud fs cp local.txt :/remote/ # Upload + ticloud fs cp :/remote/file.txt . # Download + ticloud fs cp :/a/b :/c/d # Server-side copy + cat data.txt | ticloud fs cp - :/remote/ # Upload from stdin`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return suggestInitIfTenantNotFound(err) + } + + src, dst := args[0], args[1] + srcRemote := IsRemote(src) + dstRemote := IsRemote(dst) + + ctx := context.Background() + + switch { + case !srcRemote && !dstRemote: + return fmt.Errorf("local-to-local copy not supported") + + case !srcRemote && dstRemote: + // Upload + dstPath := ParseRemotePath(dst).Path + if src == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("read stdin: %w", err) + } + return client.Write(dstPath, data) + } + if strings.HasSuffix(dstPath, "/") { + dstPath = filepath.Join(dstPath, filepath.Base(src)) + } + f, err := os.Open(src) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("open %s: %w", src, err)) + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("stat %s: %w", src, err)) + } + if resume { + return suggestInitIfTenantNotFound(client.ResumeUpload(ctx, dstPath, f, stat.Size(), nil)) + } + return suggestInitIfTenantNotFound(client.WriteStream(ctx, dstPath, f, stat.Size(), nil)) + + case srcRemote && !dstRemote: + // Download + srcPath := ParseRemotePath(src).Path + reader, err := client.ReadStream(ctx, srcPath) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("read %s: %w", src, err)) + } + defer reader.Close() + + if dst == "-" { + _, err = io.Copy(h.IOStreams.Out, reader) + return suggestInitIfTenantNotFound(err) + } + f, err := os.Create(dst) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("create %s: %w", dst, err)) + } + defer f.Close() + _, err = io.Copy(f, reader) + return suggestInitIfTenantNotFound(err) + + case srcRemote && dstRemote: + // Server-side copy + srcPath := ParseRemotePath(src).Path + dstPath := ParseRemotePath(dst).Path + if strings.HasSuffix(dstPath, "/") { + dstPath = filepath.Join(dstPath, filepath.Base(srcPath)) + } + return suggestInitIfTenantNotFound(client.Copy(srcPath, dstPath)) + } + + return nil + }, + } + + cmd.Flags().BoolVar(&resume, "resume", false, "Resume interrupted upload") + return cmd +} diff --git a/internal/cli/fs/find.go b/internal/cli/fs/find.go new file mode 100644 index 00000000..d01902e9 --- /dev/null +++ b/internal/cli/fs/find.go @@ -0,0 +1,135 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func findCmd(h *internal.Helper) *cobra.Command { + var ( + nameGlob string + tagFilter string + newer string + older string + sizeFilter string + ) + + var cmd = &cobra.Command{ + Use: "find [path]", + Short: "Find files by attributes", + Long: `Find files matching specified criteria. + +Filters: + --name Match filename pattern + --tag Match tag + --newer Modified after date (YYYY-MM-DD) + --older Modified before date (YYYY-MM-DD) + --size <+N|-N> Size filter (+N = larger than, -N = smaller than) + +Examples: + ticloud fs find :/ --name "*.go" + ticloud fs find :/ --newer 2024-01-01 --size +1M + ticloud fs find :/myfolder --tag env=production`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := "/" + if len(args) > 0 { + path = ParseRemotePath(args[0]).Path + } + + params := url.Values{} + + if nameGlob != "" { + params.Set("name", nameGlob) + } + + if tagFilter != "" { + parts := strings.SplitN(tagFilter, "=", 2) + if len(parts) == 2 { + params.Set("tag_key", parts[0]) + params.Set("tag_value", parts[1]) + } + } + + if newer != "" { + t, err := time.Parse("2006-01-02", newer) + if err != nil { + return fmt.Errorf("invalid -newer date: %w", err) + } + params.Set("modified_after", strconv.FormatInt(t.Unix(), 10)) + } + + if older != "" { + t, err := time.Parse("2006-01-02", older) + if err != nil { + return fmt.Errorf("invalid -older date: %w", err) + } + params.Set("modified_before", strconv.FormatInt(t.Unix(), 10)) + } + + if sizeFilter != "" { + if strings.HasPrefix(sizeFilter, "+") { + sizeStr := sizeFilter[1:] + size, err := humanize.ParseBytes(sizeStr) + if err != nil { + return fmt.Errorf("invalid -size value: %w", err) + } + params.Set("min_size", strconv.FormatUint(size, 10)) + } else if strings.HasPrefix(sizeFilter, "-") { + sizeStr := sizeFilter[1:] + size, err := humanize.ParseBytes(sizeStr) + if err != nil { + return fmt.Errorf("invalid -size value: %w", err) + } + params.Set("max_size", strconv.FormatUint(size, 10)) + } + } + + results, err := client.Find(path, params) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("find: %w", err)) + } + + for _, r := range results { + size := humanize.IBytes(uint64(r.SizeBytes)) + fmt.Printf("%s\t%s\n", size, r.Path) + } + return nil + }, + } + + cmd.Flags().StringVar(&nameGlob, "name", "", "Match filename pattern") + cmd.Flags().StringVar(&tagFilter, "tag", "", "Match tag (key=value)") + cmd.Flags().StringVar(&newer, "newer", "", "Modified after date (YYYY-MM-DD)") + cmd.Flags().StringVar(&older, "older", "", "Modified before date (YYYY-MM-DD)") + cmd.Flags().StringVar(&sizeFilter, "size", "", "Size filter (+N or -N, e.g., +1M, -100K)") + + return cmd +} diff --git a/internal/cli/fs/fs.go b/internal/cli/fs/fs.go new file mode 100644 index 00000000..ddbe9a1c --- /dev/null +++ b/internal/cli/fs/fs.go @@ -0,0 +1,220 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "net/http" + "os" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + "github.com/tidbcloud/tidbcloud-cli/internal/config" + "github.com/tidbcloud/tidbcloud-cli/internal/config/store" + "github.com/tidbcloud/tidbcloud-cli/internal/flag" + "github.com/tidbcloud/tidbcloud-cli/internal/service/cloud" + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + "github.com/juju/errors" + "github.com/spf13/cobra" + "github.com/zalando/go-keyring" +) + +// FSCmd returns the fs command. +func FSCmd(h *internal.Helper) *cobra.Command { + var fsCmd = &cobra.Command{ + Use: "fs", + Short: "Filesystem operations for TiDB Cloud FS", + Long: `Filesystem operations for TiDB Cloud FS. + +This command provides filesystem-like operations for managing files +and directories in TiDB Cloud FS. + +Path format: :/path/to/file + - :/ denotes the root of the remote filesystem + +Authentication: + The fs command reuses your TiDB Cloud CLI authentication (OAuth or API key). + Please login with 'ticloud auth login' or configure API keys with: + ticloud config set public-key + ticloud config set private-key + +Database Association: + FS operations can be associated with a specific database: + - Serverless cluster: ticloud config set fs.cluster-id + (sent as X-TIDBCLOUD-CLUSTER-ID header) + - TiDB Zero instance: ticloud config set fs.zero-instance-id + (sent as X-TIDBCLOUD-ZERO-INSTANCE-ID header) + If both are set, cluster-id takes precedence. + +Configuration: + - Server URL: ticloud config set fs-endpoint (default: https://fs.tidbapi.com/) + - Env override: TICLOUD_FS_ENDPOINT, TICLOUD_FS_CLUSTER_ID, TICLOUD_FS_ZERO_INSTANCE_ID + +Examples: + ticloud fs ls :/ # List root directory + ticloud fs ls -l :/myfolder # List with details + ticloud fs cat :/file.txt # Display file contents + ticloud fs cp local.txt :/remote/ # Upload file + ticloud fs cp :/remote/file.txt . # Download file + ticloud fs mv :/old :/new # Rename/move + ticloud fs rm :/file.txt # Remove file + ticloud fs mkdir :/mydir # Create directory + ticloud fs stat :/file.txt # Show metadata + ticloud fs sql -q "SELECT 1" # Execute SQL + ticloud fs secret ls # List secrets + ticloud fs grep "pattern" :/ # Search content + ticloud fs find :/ -name "*.txt" # Find files + ticloud fs shell # Interactive shell +`, + } + + fsCmd.AddCommand(initCmd(h)) + fsCmd.AddCommand(lsCmd(h)) + fsCmd.AddCommand(catCmd(h)) + fsCmd.AddCommand(cpCmd(h)) + fsCmd.AddCommand(mkdirCmd(h)) + fsCmd.AddCommand(statCmd(h)) + fsCmd.AddCommand(mvCmd(h)) + fsCmd.AddCommand(rmCmd(h)) + fsCmd.AddCommand(sqlCmd(h)) + fsCmd.AddCommand(secretCmd(h)) + fsCmd.AddCommand(shellCmd(h)) + fsCmd.AddCommand(grepCmd(h)) + fsCmd.AddCommand(findCmd(h)) + fsCmd.AddCommand(mountCmd(h)) + fsCmd.AddCommand(umountCmd(h)) + + fsCmd.PersistentFlags().String(flag.FSEndpoint, "", "FS server URL (overrides config)") + fsCmd.PersistentFlags().String(flag.FSClusterID, "", "Associated TiDB Cloud cluster ID") + fsCmd.PersistentFlags().String(flag.FSZeroInstanceID, "", "Associated TiDB Zero instance ID") + + return fsCmd +} + +// newClient creates a new FS client using the active TiDB Cloud CLI authentication. +// It reuses OAuth tokens or API key pairs from the current profile. +func newClient(cmd *cobra.Command) (*fs.Client, error) { + // Resolve server URL: flag > env > config > default + server, _ := cmd.Flags().GetString(flag.FSEndpoint) + if server == "" { + server = os.Getenv("TICLOUD_FS_ENDPOINT") + } + if server == "" { + server = config.GetFSEndpoint() + } + + // Resolve database association: flag > env > config + clusterID, _ := cmd.Flags().GetString(flag.FSClusterID) + if clusterID == "" { + clusterID = os.Getenv("TICLOUD_FS_CLUSTER_ID") + } + if clusterID == "" { + clusterID = config.GetFSClusterID() + } + + zeroInstanceID, _ := cmd.Flags().GetString(flag.FSZeroInstanceID) + if zeroInstanceID == "" { + zeroInstanceID = os.Getenv("TICLOUD_FS_ZERO_INSTANCE_ID") + } + if zeroInstanceID == "" { + zeroInstanceID = config.GetFSZeroInstanceID() + } + + // Build HTTP transport using the same auth logic as the root CLI. + var transport http.RoundTripper + publicKey, privateKey := config.GetPublicKey(), config.GetPrivateKey() + if publicKey != "" && privateKey != "" { + transport = cloud.NewDigestTransport(publicKey, privateKey) + } else { + if err := config.ValidateToken(); err != nil { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys with 'ticloud config set public-key ' and 'ticloud config set private-key '") + } + token, err := config.GetAccessToken() + if err != nil { + if errors.Is(err, keyring.ErrNotFound) || errors.Is(err, store.ErrNotSupported) { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys") + } + return nil, errors.Trace(err) + } + transport = cloud.NewBearTokenTransport(token) + } + + httpClient := &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && req.URL.Host != via[0].URL.Host { + req.Header.Del("Authorization") + } + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + return nil + }, + } + + return fs.NewClient(server, httpClient, clusterID, zeroInstanceID), nil +} + +// suggestInitIfTenantNotFound wraps errors that indicate the tenant has not been +// provisioned yet with a helpful hint to run 'ticloud fs init'. +func suggestInitIfTenantNotFound(err error) error { + if err == nil { + return nil + } + msg := err.Error() + if strings.Contains(strings.ToLower(msg), "tenant not found") { + return fmt.Errorf("%w\n\nFS tenant not found for this database. Please run 'ticloud fs init' first.", err) + } + return err +} + +// RemotePath represents a parsed remote path. +type RemotePath struct { + Context string + Path string +} + +// ParseRemotePath parses a remote path string. +// Format: [context:]/path or :/path (uses default context) +func ParseRemotePath(s string) RemotePath { + if !strings.HasPrefix(s, ":") { + return RemotePath{Path: s} + } + s = s[1:] // remove leading : + if !strings.HasPrefix(s, "/") { + // Check for context prefix + if idx := strings.Index(s, ":/"); idx != -1 { + return RemotePath{ + Context: s[:idx], + Path: s[idx+1:], + } + } + } + return RemotePath{Path: s} +} + +func (rp RemotePath) String() string { + if rp.Context != "" { + return ":" + rp.Context + ":" + rp.Path + } + return ":" + rp.Path +} + +// IsRemote returns true if the path is a remote path (starts with :). +func IsRemote(s string) bool { + return strings.HasPrefix(s, ":") +} + diff --git a/internal/cli/fs/grep.go b/internal/cli/fs/grep.go new file mode 100644 index 00000000..9b2d5fb3 --- /dev/null +++ b/internal/cli/fs/grep.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func grepCmd(h *internal.Helper) *cobra.Command { + var limit int + var cmd = &cobra.Command{ + Use: "grep [path]", + Short: "Search file contents", + Long: `Search for a pattern in file contents across the filesystem. + +Results are ranked by relevance score. Only text files are searched. + +Examples: + ticloud fs grep "function main" :/ + ticloud fs grep "TODO" :/myproject --limit 20`, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + pattern := args[0] + path := "/" + if len(args) > 1 { + path = ParseRemotePath(args[1]).Path + } + + results, err := client.Grep(pattern, path, limit) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("grep: %w", err)) + } + + for _, r := range results { + if r.Score != nil { + fmt.Printf("%.3f %s\n", *r.Score, r.Path) + } else { + fmt.Println(r.Path) + } + } + return nil + }, + } + + cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of results") + return cmd +} diff --git a/internal/cli/fs/init.go b/internal/cli/fs/init.go new file mode 100644 index 00000000..6195dc8d --- /dev/null +++ b/internal/cli/fs/init.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func initCmd(h *internal.Helper) *cobra.Command { + var ( + user string + password string + ) + + var cmd = &cobra.Command{ + Use: "init", + Short: "Initialize FS for the associated database", + Long: `Provision the FS tenant for the configured database. + +This command must be run once before any file operations when using a new +TiDB Cloud cluster or TiDB Zero instance. + +Examples: + ticloud fs init --user admin --password secret`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + if user == "" || password == "" { + return fmt.Errorf("--user and --password are required") + } + + result, err := client.Provision(context.Background(), user, password) + if err != nil { + return fmt.Errorf("provision failed: %w", err) + } + + fmt.Fprintf(h.IOStreams.Out, "FS initialized for tenant %s (status: %s)\n", result.TenantID, result.Status) + return nil + }, + } + + cmd.Flags().StringVar(&user, "user", "", "FS admin user") + cmd.Flags().StringVar(&password, "password", "", "FS admin password") + _ = cmd.MarkFlagRequired("user") + _ = cmd.MarkFlagRequired("password") + + return cmd +} diff --git a/internal/cli/fs/ls.go b/internal/cli/fs/ls.go new file mode 100644 index 00000000..61a63e13 --- /dev/null +++ b/internal/cli/fs/ls.go @@ -0,0 +1,81 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func lsCmd(h *internal.Helper) *cobra.Command { + var long bool + var cmd = &cobra.Command{ + Use: "ls [path]", + Short: "List directory contents", + Example: ` ticloud fs ls :/ + ticloud fs ls -l :/myfolder`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := "/" + if len(args) > 0 { + rp := ParseRemotePath(args[0]) + path = rp.Path + } + if path == "" { + path = "/" + } + + entries, err := client.List(path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("list %s: %w", path, err)) + } + + if long { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, e := range entries { + typeChar := "-" + if e.IsDir { + typeChar = "d" + } + fmt.Fprintf(w, "%s\t%s\t%s\n", typeChar, humanize.IBytes(uint64(e.Size)), e.Name) + } + return w.Flush() + } + + for _, e := range entries { + name := e.Name + if e.IsDir { + name += "/" + } + fmt.Println(name) + } + return nil + }, + } + + cmd.Flags().BoolVarP(&long, "long", "l", false, "Use long listing format") + return cmd +} diff --git a/internal/cli/fs/mkdir.go b/internal/cli/fs/mkdir.go new file mode 100644 index 00000000..5d5c91d3 --- /dev/null +++ b/internal/cli/fs/mkdir.go @@ -0,0 +1,45 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mkdirCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mkdir ", + Short: "Create a directory", + Example: ` ticloud fs mkdir :/mydir + ticloud fs mkdir :/parent/child`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := ParseRemotePath(args[0]).Path + if err := client.Mkdir(path); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("mkdir %s: %w", path, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/mount.go b/internal/cli/fs/mount.go new file mode 100644 index 00000000..cf189466 --- /dev/null +++ b/internal/cli/fs/mount.go @@ -0,0 +1,134 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + fusepkg "github.com/tidbcloud/tidbcloud-cli/internal/service/fs/fuse" + + "github.com/spf13/cobra" +) + +func mountCmd(h *internal.Helper) *cobra.Command { + var ( + readOnly bool + debug bool + allowOther bool + ) + + var cmd = &cobra.Command{ + Use: "mount ", + Short: "Mount remote filesystem via FUSE", + Long: `Mount the remote filesystem as a local FUSE mount. + +WARNING: This feature requires FUSE support and may need additional setup. +On Linux, you need fusermount or fusermount3 installed. +On macOS, you need macFUSE installed. + +Examples: + ticloud fs mount /mnt/tidbcloud + ticloud fs mount /mnt/tidbcloud --read-only --debug`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + mountpoint := args[0] + + // Check if mountpoint exists + if _, err := os.Stat(mountpoint); os.IsNotExist(err) { + return fmt.Errorf("mountpoint %s does not exist", mountpoint) + } + + // Check if already mounted + if isMounted(mountpoint) { + return fmt.Errorf("%s is already mounted", mountpoint) + } + + client, err := newClient(cmd) + if err != nil { + return suggestInitIfTenantNotFound(err) + } + + // For now, only support read-only mode (MVP) + // Full read-write support will be added later + if !readOnly { + fmt.Fprintln(h.IOStreams.Out, "Note: Mounting in read-only mode (MVP). Write support coming soon.") + readOnly = true + } + + opts := &fusepkg.MountOptions{ + Client: client, + MountPoint: mountpoint, + ReadOnly: readOnly, + Debug: debug, + AllowOther: allowOther, + } + + return suggestInitIfTenantNotFound(fusepkg.Mount(opts)) + }, + } + + cmd.Flags().BoolVarP(&readOnly, "read-only", "r", true, "Mount as read-only (default: true for MVP)") + cmd.Flags().BoolVar(&debug, "debug", false, "Enable FUSE debug logging") + cmd.Flags().BoolVar(&allowOther, "allow-other", false, "Allow other users to access mount") + + return cmd +} + +func umountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "umount ", + Short: "Unmount FUSE filesystem", + Example: ` ticloud fs umount /mnt/tidbcloud`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + mountpoint := args[0] + + var umountCmd *exec.Cmd + if runtime.GOOS == "darwin" { + umountCmd = exec.Command("umount", mountpoint) + } else { + // Try fusermount3 first, then fusermount, then umount + if _, err := exec.LookPath("fusermount3"); err == nil { + umountCmd = exec.Command("fusermount3", "-u", mountpoint) + } else if _, err := exec.LookPath("fusermount"); err == nil { + umountCmd = exec.Command("fusermount", "-u", mountpoint) + } else { + umountCmd = exec.Command("umount", mountpoint) + } + } + + output, err := umountCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("umount failed: %s", string(output)) + } + + fmt.Fprintf(h.IOStreams.Out, "Unmounted %s\n", mountpoint) + return nil + }, + } +} + +func isMounted(mountpoint string) bool { + data, err := os.ReadFile("/proc/mounts") + if err != nil { + return false + } + return strings.Contains(string(data), mountpoint) +} diff --git a/internal/cli/fs/mv.go b/internal/cli/fs/mv.go new file mode 100644 index 00000000..bc4a21e0 --- /dev/null +++ b/internal/cli/fs/mv.go @@ -0,0 +1,52 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mvCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mv ", + Short: "Move or rename a file or directory", + Long: `Move or rename a file or directory in the remote filesystem. + +This is a metadata-only operation with zero data transfer cost. + +Examples: + ticloud fs mv :/oldname :/newname + ticloud fs mv :/folder/file.txt :/other/file.txt`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + oldPath := ParseRemotePath(args[0]).Path + newPath := ParseRemotePath(args[1]).Path + + if err := client.Rename(oldPath, newPath); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("move %s to %s: %w", oldPath, newPath, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/rm.go b/internal/cli/fs/rm.go new file mode 100644 index 00000000..6117e510 --- /dev/null +++ b/internal/cli/fs/rm.go @@ -0,0 +1,46 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func rmCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "rm ", + Short: "Remove a file or directory", + Example: ` ticloud fs rm :/file.txt + ticloud fs rm :/empty_folder/`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := ParseRemotePath(args[0]).Path + + if err := client.Delete(path); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("remove %s: %w", path, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/secret.go b/internal/cli/fs/secret.go new file mode 100644 index 00000000..bee6df97 --- /dev/null +++ b/internal/cli/fs/secret.go @@ -0,0 +1,600 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal" + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + "github.com/spf13/cobra" +) + +const ( + defaultAuditLimit = 100 + maxClientAuditLimit = 1000 +) + +func secretCmd(h *internal.Helper) *cobra.Command { + var secretCmd = &cobra.Command{ + Use: "secret", + Short: "Secret management operations", + Long: `Manage secrets in TiDB Cloud FS vault.`, + } + + secretCmd.AddCommand(secretSetCmd(h)) + secretCmd.AddCommand(secretGetCmd(h)) + secretCmd.AddCommand(secretExecCmd(h)) + secretCmd.AddCommand(secretLsCmd(h)) + secretCmd.AddCommand(secretRmCmd(h)) + secretCmd.AddCommand(secretGrantCmd(h)) + secretCmd.AddCommand(secretRevokeCmd(h)) + secretCmd.AddCommand(secretAuditCmd(h)) + + return secretCmd +} + +func secretSetCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "set ...", + Short: "Create or update a secret", + Example: ` ticloud fs secret set myapp DATABASE_URL=mysql://... + ticloud fs secret set myapp key=@secret.txt password=-`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + fields, err := parseSecretFields(args[1:]) + if err != nil { + return err + } + + ctx := context.Background() + if _, err := client.CreateVaultSecret(ctx, name, fields); err != nil { + // If conflict, update instead + if !strings.Contains(err.Error(), "conflict") { + return suggestInitIfTenantNotFound(fmt.Errorf("set secret: %w", err)) + } + _, err = client.UpdateVaultSecret(ctx, name, fields) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("update secret: %w", err)) + } + } + return nil + }, + } +} + +func secretGetCmd(h *internal.Helper) *cobra.Command { + var asJSON, asEnv bool + var cmd = &cobra.Command{ + Use: "get ", + Short: "Read a secret or one field", + Example: ` ticloud fs secret get myapp + ticloud fs secret get myapp/password --json + ticloud fs secret get myapp/api_key --env`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + ref := args[0] + name, field, err := parseSecretRef(ref) + if err != nil { + return err + } + if asJSON && asEnv { + return fmt.Errorf("--json and --env are mutually exclusive") + } + + ctx := context.Background() + if field != "" { + value, err := client.ReadVaultSecretField(ctx, name, field) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret field: %w", err)) + } + switch { + case asEnv: + envKey, err := normalizeSecretEnvKey(field) + if err != nil { + return err + } + fmt.Fprintf(h.IOStreams.Out, "%s=%s\n", envKey, value) + case asJSON: + return writeJSON(h, map[string]string{field: value}) + default: + fmt.Fprintln(h.IOStreams.Out, value) + } + return nil + } + + fields, err := client.ReadVaultSecret(ctx, name) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret: %w", err)) + } + if asEnv { + return printEnv(h, fields) + } + return writeJSON(h, fields) + }, + } + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + cmd.Flags().BoolVar(&asEnv, "env", false, "Output as environment variables") + return cmd +} + +func secretExecCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "exec -- ", + Short: "Run a command with secret fields injected as env vars", + Example: ` ticloud fs secret exec myapp -- ./run.sh`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + + sep := -1 + for i := 1; i < len(args); i++ { + if args[i] == "--" { + sep = i + break + } + } + if sep < 0 || sep == len(args)-1 { + return fmt.Errorf("usage: ticloud fs secret exec -- ") + } + cmdArgs := args[sep+1:] + + fields, err := client.ReadVaultSecret(context.Background(), name) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret: %w", err)) + } + + c := exec.Command(cmdArgs[0], cmdArgs[1:]...) + c.Stdin = os.Stdin + c.Stdout = h.IOStreams.Out + c.Stderr = h.IOStreams.Err + envMap, err := buildSecretEnvMap(fields) + if err != nil { + return err + } + c.Env = mergeEnv(os.Environ(), envMap) + return c.Run() + }, + } +} + +func secretLsCmd(h *internal.Helper) *cobra.Command { + var asJSON bool + var cmd = &cobra.Command{ + Use: "ls", + Short: "List secrets", + Example: ` ticloud fs secret ls --json`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + secrets, err := client.ListVaultSecrets(context.Background()) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("list secrets: %w", err)) + } + + names := make([]string, 0, len(secrets)) + for _, sec := range secrets { + names = append(names, sec.Name) + } + sort.Strings(names) + + if asJSON { + return writeJSON(h, map[string]any{"secrets": names}) + } + for _, name := range names { + fmt.Fprintln(h.IOStreams.Out, name) + } + return nil + }, + } + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + return cmd +} + +func secretRmCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "rm ", + Short: "Delete a secret", + Example: ` ticloud fs secret rm myapp`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + if err := client.DeleteVaultSecret(context.Background(), name); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("delete secret: %w", err)) + } + return nil + }, + } +} + +func secretGrantCmd(h *internal.Helper) *cobra.Command { + var ( + agentID string + taskID string + ttlRaw string + asJSON bool + tokenOnly bool + ) + var cmd = &cobra.Command{ + Use: "grant --agent --ttl [--task ] ", + Short: "Issue a scoped capability token", + Example: ` ticloud fs secret grant --agent myagent --ttl 1h myapp/password`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + scope := args + for _, entry := range scope { + if _, _, err := parseSecretRef(entry); err != nil { + return fmt.Errorf("invalid scope %q: %w", entry, err) + } + } + if asJSON && tokenOnly { + return fmt.Errorf("--json and --token-only are mutually exclusive") + } + if agentID == "" { + return fmt.Errorf("--agent is required") + } + if ttlRaw == "" { + return fmt.Errorf("--ttl is required") + } + ttl, err := time.ParseDuration(ttlRaw) + if err != nil { + return fmt.Errorf("invalid --ttl %q: %w", ttlRaw, err) + } + if ttl <= 0 { + return fmt.Errorf("--ttl must be positive") + } + + resp, err := client.IssueVaultToken(context.Background(), agentID, taskID, scope, ttl) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("grant token: %w", err)) + } + switch { + case tokenOnly: + fmt.Fprintln(h.IOStreams.Out, resp.Token) + case asJSON: + return writeJSON(h, resp) + default: + fmt.Fprintf(h.IOStreams.Out, "token=%s\n", resp.Token) + fmt.Fprintf(h.IOStreams.Out, "token_id=%s\n", resp.TokenID) + fmt.Fprintf(h.IOStreams.Out, "expires_at=%s\n", resp.ExpiresAt.Format(time.RFC3339)) + } + return nil + }, + } + cmd.Flags().StringVar(&agentID, "agent", "", "Agent ID") + cmd.Flags().StringVar(&taskID, "task", "", "Task ID") + cmd.Flags().StringVar(&ttlRaw, "ttl", "", "Token TTL (e.g., 1h, 30m)") + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + cmd.Flags().BoolVar(&tokenOnly, "token-only", false, "Output only the token string") + return cmd +} + +func secretRevokeCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "revoke ", + Short: "Revoke a capability token", + Example: ` ticloud fs secret revoke tok_abc123`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + if err := client.RevokeVaultToken(context.Background(), args[0]); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("revoke token: %w", err)) + } + return nil + }, + } +} + +func secretAuditCmd(h *internal.Helper) *cobra.Command { + var ( + secretName string + agentID string + sinceRaw string + limit = defaultAuditLimit + asJSON bool + ) + var cmd = &cobra.Command{ + Use: "audit [--secret ] [--agent ] [--since ] [--limit ]", + Short: "Query vault audit events", + Example: ` ticloud fs secret audit --limit 50 + ticloud fs secret audit --secret myapp --since 24h`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + queryLimit := limit + if agentID != "" || sinceRaw != "" { + queryLimit = maxClientAuditLimit + } + if queryLimit > maxClientAuditLimit { + queryLimit = maxClientAuditLimit + } + + events, err := client.QueryVaultAudit(context.Background(), secretName, queryLimit) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("audit query: %w", err)) + } + + if sinceRaw != "" { + d, err := time.ParseDuration(sinceRaw) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", sinceRaw, err) + } + if d <= 0 { + return fmt.Errorf("--since must be positive") + } + events = filterAuditEvents(events, agentID, time.Now().Add(-d)) + } else if agentID != "" { + events = filterAuditEvents(events, agentID, time.Time{}) + } + if len(events) > limit { + events = events[:limit] + } + + if asJSON { + return writeJSON(h, map[string]any{"events": events}) + } + printAudit(h, events) + return nil + }, + } + cmd.Flags().StringVar(&secretName, "secret", "", "Filter by secret name") + cmd.Flags().StringVar(&agentID, "agent", "", "Filter by agent ID") + cmd.Flags().StringVar(&sinceRaw, "since", "", "Filter events newer than duration (e.g., 24h)") + cmd.Flags().IntVar(&limit, "limit", defaultAuditLimit, "Maximum number of events") + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + return cmd +} + +func validateSecretName(name string) error { + if name == "" { + return fmt.Errorf("secret name is required") + } + if strings.Contains(name, "/") { + return fmt.Errorf("secret name %q must be flat; use only for reads and scopes", name) + } + if strings.Contains(name, "*") { + return fmt.Errorf("wildcard scope entries are not supported: %q", name) + } + return nil +} + +func parseSecretRef(raw string) (string, string, error) { + if raw == "" { + return "", "", fmt.Errorf("secret reference is required") + } + parts := strings.SplitN(raw, "/", 2) + name := parts[0] + if err := validateSecretName(name); err != nil { + return "", "", err + } + if len(parts) == 1 { + return name, "", nil + } + field := parts[1] + if field == "" { + return "", "", fmt.Errorf("field name is required in %q", raw) + } + if strings.Contains(field, "*") { + return "", "", fmt.Errorf("wildcard scope entries are not supported: %q", raw) + } + return name, field, nil +} + +func parseSecretFields(args []string) (map[string]string, error) { + fields := make(map[string]string, len(args)) + var stdinValue []byte + var stdinRead bool + for _, arg := range args { + key, valueSpec, ok := strings.Cut(arg, "=") + if !ok || key == "" { + return nil, fmt.Errorf("field assignment must be field=value, field=@file, or field=-: %q", arg) + } + var value string + switch { + case valueSpec == "-": + if !stdinRead { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, fmt.Errorf("read stdin: %w", err) + } + stdinValue = data + stdinRead = true + } + value = string(stdinValue) + case strings.HasPrefix(valueSpec, "@"): + data, err := os.ReadFile(valueSpec[1:]) + if err != nil { + return nil, fmt.Errorf("read %s: %w", valueSpec[1:], err) + } + value = string(data) + default: + value = valueSpec + } + fields[key] = value + } + return fields, nil +} + +func writeJSON(h *internal.Helper, v any) error { + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func printEnv(h *internal.Helper, fields map[string]string) error { + envMap, err := buildSecretEnvMap(fields) + if err != nil { + return err + } + keys := make([]string, 0, len(envMap)) + for k := range envMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + fmt.Fprintf(h.IOStreams.Out, "%s=%s\n", key, envMap[key]) + } + return nil +} + +func printAudit(h *internal.Helper, events []fs.VaultAuditEvent) { + w := tabwriter.NewWriter(h.IOStreams.Out, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "TIME\tAGENT\tACTION\tSECRET\tFIELD") + for _, ev := range events { + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", + ev.Timestamp.Format(time.RFC3339), + ev.AgentID, + ev.EventType, + ev.SecretName, + ev.FieldName, + ) + } + _ = w.Flush() +} + +func buildSecretEnvMap(fields map[string]string) (map[string]string, error) { + env := make(map[string]string, len(fields)) + owners := make(map[string]string, len(fields)) + for field, value := range fields { + envKey, err := normalizeSecretEnvKey(field) + if err != nil { + return nil, err + } + if prevField, exists := owners[envKey]; exists { + return nil, fmt.Errorf("secret fields %q and %q both normalize to env var %q", prevField, field, envKey) + } + owners[envKey] = field + env[envKey] = value + } + return env, nil +} + +func filterAuditEvents(events []fs.VaultAuditEvent, agentID string, since time.Time) []fs.VaultAuditEvent { + filtered := make([]fs.VaultAuditEvent, 0, len(events)) + for _, ev := range events { + if agentID != "" && ev.AgentID != agentID { + continue + } + if !since.IsZero() && ev.Timestamp.Before(since) { + continue + } + filtered = append(filtered, ev) + } + return filtered +} + +func mergeEnv(base []string, overrides map[string]string) []string { + merged := make(map[string]string, len(base)+len(overrides)) + for _, entry := range base { + key, value, ok := strings.Cut(entry, "=") + if ok { + merged[key] = value + } + } + for k, v := range overrides { + merged[k] = v + } + keys := make([]string, 0, len(merged)) + for k := range merged { + keys = append(keys, k) + } + sort.Strings(keys) + env := make([]string, 0, len(keys)) + for _, k := range keys { + env = append(env, k+"="+merged[k]) + } + return env +} + +func normalizeSecretEnvKey(field string) (string, error) { + if field == "" { + return "", fmt.Errorf("secret field name is required") + } + var b strings.Builder + b.Grow(len(field) + 1) + for i := 0; i < len(field); i++ { + ch := field[i] + switch { + case ch >= 'a' && ch <= 'z': + b.WriteByte(ch - ('a' - 'A')) + case ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9', ch == '_': + b.WriteByte(ch) + default: + b.WriteByte('_') + } + } + key := b.String() + if key == "" { + return "", fmt.Errorf("secret field name is required") + } + if key[0] >= '0' && key[0] <= '9' { + key = "_" + key + } + return key, nil +} diff --git a/internal/cli/fs/shell.go b/internal/cli/fs/shell.go new file mode 100644 index 00000000..c4d37ae8 --- /dev/null +++ b/internal/cli/fs/shell.go @@ -0,0 +1,228 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func shellCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "shell", + Short: "Interactive filesystem shell", + Long: `Start an interactive shell for filesystem operations. + +Commands: + cd Change directory + pwd Print current directory + ls [path] List directory + cat Display file contents + cp Copy files (remote only) + mkdir Create directory + mv Move/rename + rm Remove file + sql Execute SQL query + stat Show metadata + help Show help + exit Exit shell + +The prompt shows the current remote directory.`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + cwd := "/" + ctx := context.Background() + scanner := bufio.NewScanner(h.IOStreams.In) + + fmt.Fprintln(h.IOStreams.Out, "TiDB Cloud FS Shell. Type 'help' for commands, 'exit' to quit.") + + for { + fmt.Fprint(h.IOStreams.Out, "ticloud:fs> ") + if !scanner.Scan() { + break + } + + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + + cmd := parts[0] + args := parts[1:] + + switch cmd { + case "exit", "quit": + return nil + + case "help": + fmt.Fprintln(h.IOStreams.Out, `Commands: + cd Change directory + pwd Print current directory + ls [path] List directory + cat Display file contents + cp Copy files (remote only) + mkdir Create directory + mv Move/rename + rm Remove file + sql Execute SQL query + stat Show metadata + help Show this help + exit Exit shell`) + + case "pwd": + fmt.Fprintln(h.IOStreams.Out, cwd) + + case "cd": + if len(args) == 0 { + cwd = "/" + } else { + cwd = resolvePath(cwd, args[0]) + } + + case "ls": + path := cwd + if len(args) > 0 { + path = resolvePath(cwd, args[0]) + } + entries, err := client.List(path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("list: %w", err))) + continue + } + for _, e := range entries { + name := e.Name + if e.IsDir { + name += "/" + } + fmt.Fprintln(h.IOStreams.Out, name) + } + + case "cat": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: cat ") + continue + } + path := resolvePath(cwd, args[0]) + reader, err := client.ReadStream(ctx, path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("read: %w", err))) + continue + } + _, err = io.Copy(h.IOStreams.Out, reader) + reader.Close() + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", err) + } + + case "cp": + if len(args) < 2 { + fmt.Fprintln(h.IOStreams.Err, "Usage: cp ") + continue + } + src, dst := resolvePath(cwd, args[0]), resolvePath(cwd, args[1]) + if err := client.Copy(src, dst); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("copy: %w", err))) + } + + case "mkdir": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: mkdir ") + continue + } + path := resolvePath(cwd, args[0]) + if err := client.Mkdir(path); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("mkdir: %w", err))) + } + + case "mv": + if len(args) < 2 { + fmt.Fprintln(h.IOStreams.Err, "Usage: mv ") + continue + } + old, newPath := resolvePath(cwd, args[0]), resolvePath(cwd, args[1]) + if err := client.Rename(old, newPath); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("mv: %w", err))) + } + + case "rm": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: rm ") + continue + } + path := resolvePath(cwd, args[0]) + if err := client.Delete(path); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("rm: %w", err))) + } + + case "sql": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: sql ") + continue + } + query := strings.Join(args, " ") + rows, err := client.SQL(query) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("sql: %w", err))) + continue + } + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + if err := enc.Encode(rows); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", err) + } + + case "stat": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: stat ") + continue + } + path := resolvePath(cwd, args[0]) + info, err := client.Stat(path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("stat: %w", err))) + continue + } + fmt.Fprintf(h.IOStreams.Out, "Size: %d, Dir: %v, Rev: %d\n", info.Size, info.IsDir, info.Revision) + + default: + fmt.Fprintf(h.IOStreams.Err, "Unknown command: %s\n", cmd) + } + } + + return nil + }, + } +} + +func resolvePath(cwd, p string) string { + if strings.HasPrefix(p, "/") { + return p + } + return filepath.Join(cwd, p) +} diff --git a/internal/cli/fs/sql.go b/internal/cli/fs/sql.go new file mode 100644 index 00000000..c0ad35c7 --- /dev/null +++ b/internal/cli/fs/sql.go @@ -0,0 +1,73 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func sqlCmd(h *internal.Helper) *cobra.Command { + var query, file string + var cmd = &cobra.Command{ + Use: "sql", + Short: "Execute a SQL query", + Long: `Execute a SQL query against the FS backend database. + +You can provide the query directly with -q or from a file with -f.`, + Example: ` ticloud fs sql -q "SELECT 1" + ticloud fs sql -f query.sql`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + if query == "" && file == "" { + return fmt.Errorf("either -q or -f is required") + } + if query != "" && file != "" { + return fmt.Errorf("-q and -f are mutually exclusive") + } + + if file != "" { + data, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + query = string(data) + } + + rows, err := client.SQL(query) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("sql: %w", err)) + } + + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + return enc.Encode(rows) + }, + } + + cmd.Flags().StringVarP(&query, "query", "q", "", "SQL query string") + cmd.Flags().StringVarP(&file, "file", "f", "", "Path to a SQL file") + + return cmd +} diff --git a/internal/cli/fs/stat.go b/internal/cli/fs/stat.go new file mode 100644 index 00000000..7f2a3645 --- /dev/null +++ b/internal/cli/fs/stat.go @@ -0,0 +1,54 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func statCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "stat ", + Short: "Display file metadata", + Example: ` ticloud fs stat :/file.txt + ticloud fs stat :/myfolder/`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + rp := ParseRemotePath(args[0]) + path := rp.Path + + info, err := client.Stat(path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("stat %s: %w", path, err)) + } + + fmt.Printf("Path: %s\n", path) + fmt.Printf("Size: %s (%d bytes)\n", humanize.IBytes(uint64(info.Size)), info.Size) + fmt.Printf("Type: %s\n", map[bool]string{true: "directory", false: "file"}[info.IsDir]) + fmt.Printf("Version: %d\n", info.Revision) + return nil + }, + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 847c9b22..4e8fe139 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -24,6 +24,7 @@ import ( "github.com/tidbcloud/tidbcloud-cli/internal" "github.com/tidbcloud/tidbcloud-cli/internal/cli/auth" configCmd "github.com/tidbcloud/tidbcloud-cli/internal/cli/config" + "github.com/tidbcloud/tidbcloud-cli/internal/cli/fs" "github.com/tidbcloud/tidbcloud-cli/internal/cli/project" "github.com/tidbcloud/tidbcloud-cli/internal/cli/serverless" "github.com/tidbcloud/tidbcloud-cli/internal/cli/upgrade" @@ -194,6 +195,7 @@ func RootCmd(h *internal.Helper) *cobra.Command { rootCmd.AddCommand(auth.AuthCmd(h)) rootCmd.AddCommand(configCmd.ConfigCmd(h)) + rootCmd.AddCommand(fs.FSCmd(h)) rootCmd.AddCommand(serverless.Cmd(h)) rootCmd.AddCommand(project.ProjectCmd(h)) rootCmd.AddCommand(version.VersionCmd(h)) diff --git a/internal/config/config.go b/internal/config/config.go index 1baf0012..743c68e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -28,10 +28,11 @@ const ( cliNameInTiUP = "cloud" HomePath = ".ticloud" - Confirmed = "yes" - OAuthEndpoint = "https://oauth.tidbcloud.com" - ClientID = "wiSBy7f27zWBaBCxS16tDm7DDj2T3POgwFFbefTrgx8FAXKhzaPzv1Uta9NTck2r" - ClientSecret = "ieKdWDao0QFmHbfYXpQJHuYZ9nLdpRptfE9d3W30WettIFLZL66JKewznvVkY39IkEbBQiZK60pjcnm7BN7Lj9uRiCSpIC4n2aI3IUyHlLKtvxZLrrfXugHsC7qhb1Js" + Confirmed = "yes" + OAuthEndpoint = "https://oauth.tidbcloud.com" + DefaultFSEndpoint = "https://fs.tidbapi.com" + ClientID = "wiSBy7f27zWBaBCxS16tDm7DDj2T3POgwFFbefTrgx8FAXKhzaPzv1Uta9NTck2r" + ClientSecret = "ieKdWDao0QFmHbfYXpQJHuYZ9nLdpRptfE9d3W30WettIFLZL66JKewznvVkY39IkEbBQiZK60pjcnm7BN7Lj9uRiCSpIC4n2aI3IUyHlLKtvxZLrrfXugHsC7qhb1Js" ) var ( diff --git a/internal/config/profile.go b/internal/config/profile.go index 8736052c..f72b6b21 100644 --- a/internal/config/profile.go +++ b/internal/config/profile.go @@ -124,6 +124,25 @@ func (p *Profile) GetIAMEndpoint() string { return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.IAMEndpoint)) } +func GetFSEndpoint() string { return activeProfile.GetFSEndpoint() } +func (p *Profile) GetFSEndpoint() string { + endpoint := viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSEndpoint)) + if endpoint == "" { + return DefaultFSEndpoint + } + return endpoint +} + +func GetFSClusterID() string { return activeProfile.GetFSClusterID() } +func (p *Profile) GetFSClusterID() string { + return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSClusterID)) +} + +func GetFSZeroInstanceID() string { return activeProfile.GetFSZeroInstanceID() } +func (p *Profile) GetFSZeroInstanceID() string { + return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSZeroInstanceID)) +} + func GetOAuthEndpoint() (apiUrl string) { return activeProfile.GetOAuthEndpoint() } func (p *Profile) GetOAuthEndpoint() (newApiUrl string) { newApiUrl = viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.OAuthEndpoint)) diff --git a/internal/flag/flag.go b/internal/flag/flag.go index e81a11df..b960be0d 100644 --- a/internal/flag/flag.go +++ b/internal/flag/flag.go @@ -134,6 +134,11 @@ const ( MigrationConfigFile string = "config-file" MigrationMode string = "mode" DryRun string = "dry-run" + + // FS flags + FSEndpoint string = "fs-endpoint" + FSClusterID string = "fs.cluster-id" + FSZeroInstanceID string = "fs.zero-instance-id" ) const OutputHelp = "Output format, one of [\"human\" \"json\"]. For the complete result, please use json format." diff --git a/internal/iostream/iostream.go b/internal/iostream/iostream.go index 7d136b5c..8261321e 100644 --- a/internal/iostream/iostream.go +++ b/internal/iostream/iostream.go @@ -23,6 +23,7 @@ import ( ) type IOStreams struct { + In io.Reader Out io.Writer Err io.Writer CanPrompt bool @@ -32,6 +33,7 @@ func System() *IOStreams { canPrompt := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) return &IOStreams{ + In: os.Stdin, Out: os.Stdout, Err: os.Stderr, CanPrompt: canPrompt, @@ -40,6 +42,7 @@ func System() *IOStreams { func Test() *IOStreams { return &IOStreams{ + In: &bytes.Buffer{}, Out: &bytes.Buffer{}, Err: &bytes.Buffer{}, CanPrompt: false, diff --git a/internal/prop/property.go b/internal/prop/property.go index 102f23ef..c7c4d47c 100644 --- a/internal/prop/property.go +++ b/internal/prop/property.go @@ -30,6 +30,9 @@ const ( OAuthClientID string = "oauth-client-id" OAuthClientSecret string = "oauth-client-secret" TelemetryEnabled string = "telemetry-enabled" + FSEndpoint string = "fs-endpoint" + FSClusterID string = "fs.cluster-id" + FSZeroInstanceID string = "fs.zero-instance-id" // shall not be set by user TokenExpiredAt string = "token-expired-at" @@ -42,7 +45,7 @@ func GlobalProperties() []string { } func ProfileProperties() []string { - return []string{PublicKey, PrivateKey, ServerlessEndpoint, IAMEndpoint, OAuthEndpoint, OAuthClientID, OAuthClientSecret, TelemetryEnabled} + return []string{PublicKey, PrivateKey, ServerlessEndpoint, IAMEndpoint, OAuthEndpoint, OAuthClientID, OAuthClientSecret, TelemetryEnabled, FSEndpoint, FSClusterID, FSZeroInstanceID} } func ValidateApiUrl(value string) (*url.URL, error) { diff --git a/internal/service/fs/client.go b/internal/service/fs/client.go new file mode 100644 index 00000000..4ba67c53 --- /dev/null +++ b/internal/service/fs/client.go @@ -0,0 +1,1596 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fs provides the filesystem client for TiDB Cloud FS operations. +package fs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // PartSize is the default S3 multipart part size (8 MiB). + PartSize = 8 * 1024 * 1024 + + // DefaultSmallFileThreshold matches the server's threshold for direct PUT vs multipart. + DefaultSmallFileThreshold = 50_000 // 50,000 bytes + + // Upload concurrency limits + uploadMaxConcurrency = 16 + uploadMaxBufferBytes = 256 * 1024 * 1024 // 256 MB +) + +// Part represents a single part in a multipart upload. +type Part struct { + Number int64 + Size int64 +} + +// CalcParts calculates the parts needed for a file of given size. +func CalcParts(totalSize, partSize int64) []Part { + if totalSize <= 0 { + return nil + } + if partSize <= 0 { + partSize = PartSize + } + parts := make([]Part, 0, (totalSize+partSize-1)/partSize) + var offset int64 + for offset < totalSize { + sz := partSize + if offset+sz > totalSize { + sz = totalSize - offset + } + parts = append(parts, Part{ + Number: int64(len(parts) + 1), + Size: sz, + }) + offset += sz + } + return parts +} + +// Client is the FS HTTP client. +type Client struct { + baseURL string + httpClient *http.Client + clusterID string + zeroInstanceID string + smallFileThreshold int64 // 0 means use DefaultSmallFileThreshold +} + +// GetBaseURL returns the base URL of the client. +func (c *Client) GetBaseURL() string { + return c.baseURL +} + +// GetClusterID returns the associated TiDB Cloud cluster ID. +func (c *Client) GetClusterID() string { + return c.clusterID +} + +// GetZeroInstanceID returns the associated TiDB Zero instance ID. +func (c *Client) GetZeroInstanceID() string { + return c.zeroInstanceID +} + +// ProvisionResult is the response from the FS provision endpoint. +type ProvisionResult struct { + TenantID string `json:"tenant_id"` + APIKey string `json:"api_key"` + Status string `json:"status"` +} + +// Provision initializes the FS tenant for the associated database. +func (c *Client) Provision(ctx context.Context, user, password string) (*ProvisionResult, error) { + body, err := json.Marshal(map[string]string{ + "user": user, + "password": password, + }) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/provision", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusAccepted { + return nil, readError(resp) + } + var result ProvisionResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode provision response: %w", err) + } + return &result, nil +} + +// NewClient creates a new FS client. +// The httpClient should already be configured with authentication (e.g., Bearer or Digest transport). +// clusterID and zeroInstanceID are sent as X-TIDBCLOUD-CLUSTER-ID and X-TIDBCLOUD-ZERO-INSTANCE-ID headers. +func NewClient(baseURL string, httpClient *http.Client, clusterID, zeroInstanceID string) *Client { + if httpClient == nil { + httpClient = &http.Client{} + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + httpClient: httpClient, + clusterID: clusterID, + zeroInstanceID: zeroInstanceID, + } +} + +// FileInfo represents a file entry from a directory listing. +type FileInfo struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"isDir"` +} + +// StatResult represents file metadata from HEAD. +type StatResult struct { + Size int64 + IsDir bool + Revision int64 +} + +func (c *Client) url(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return c.baseURL + "/v1/fs" + path +} + +// RawPost sends a raw POST request to the specified endpoint. +func (c *Client) RawPost(endpoint string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, c.baseURL+endpoint, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req) +} + +func (c *Client) do(req *http.Request) (*http.Response, error) { + return c.doWithClient(c.httpClient, req) +} + +func (c *Client) doWithClient(client *http.Client, req *http.Request) (*http.Response, error) { + // Inject TiDB Cloud native provider headers. Cluster ID takes precedence over Zero instance ID. + if c.clusterID != "" { + req.Header.Set("X-TIDBCLOUD-CLUSTER-ID", c.clusterID) + } else if c.zeroInstanceID != "" { + req.Header.Set("X-TIDBCLOUD-ZERO-INSTANCE-ID", c.zeroInstanceID) + } + return client.Do(req) +} + +// Write uploads data to a remote path. +func (c *Client) Write(path string, data []byte) error { + req, err := http.NewRequest(http.MethodPut, c.url(path), bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Read downloads a file's content. +func (c *Client) Read(path string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + return io.ReadAll(resp.Body) +} + +// List returns the entries in a directory. +func (c *Client) List(path string) ([]FileInfo, error) { + // Use an explicit value to avoid intermediaries dropping bare "?list". + req, err := http.NewRequest(http.MethodGet, c.url(path)+"?list=1", nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Entries []FileInfo `json:"entries"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return result.Entries, nil +} + +// Stat returns metadata for a path. +func (c *Client) Stat(path string) (*StatResult, error) { + req, err := http.NewRequest(http.MethodHead, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + _ = resp.Body.Close() + if resp.StatusCode == 404 { + return nil, fmt.Errorf("not found: %s", path) + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + s := &StatResult{ + IsDir: resp.Header.Get("X-FS-IsDir") == "true", + } + if cl := resp.Header.Get("Content-Length"); cl != "" { + s.Size, _ = strconv.ParseInt(cl, 10, 64) + } + if rev := resp.Header.Get("X-FS-Revision"); rev != "" { + s.Revision, _ = strconv.ParseInt(rev, 10, 64) + } + return s, nil +} + +// Delete removes a file or directory. +func (c *Client) Delete(path string) error { + req, err := http.NewRequest(http.MethodDelete, c.url(path), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Copy performs a server-side zero-copy (same file_id, new path). +func (c *Client) Copy(srcPath, dstPath string) error { + req, err := http.NewRequest(http.MethodPost, c.url(dstPath)+"?copy", nil) + if err != nil { + return err + } + req.Header.Set("X-Dat9-Copy-Source", srcPath) + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Rename moves/renames a file or directory (metadata-only). +func (c *Client) Rename(oldPath, newPath string) error { + req, err := http.NewRequest(http.MethodPost, c.url(newPath)+"?rename", nil) + if err != nil { + return err + } + req.Header.Set("X-Dat9-Rename-Source", oldPath) + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Mkdir creates a directory. +func (c *Client) Mkdir(path string) error { + req, err := http.NewRequest(http.MethodPost, c.url(path)+"?mkdir", nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +func readError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + var errResp struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &errResp) == nil && errResp.Error != "" { + return fmt.Errorf("%s", errResp.Error) + } + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) +} + +// SQL executes a SQL query. +func (c *Client) SQL(query string) ([]map[string]interface{}, error) { + body, err := json.Marshal(map[string]string{"query": query}) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, c.baseURL+"/v1/sql", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var rows []map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return rows, nil +} + +// SearchResult represents a search result. +type SearchResult struct { + Path string `json:"path"` + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` + Score *float64 `json:"score,omitempty"` +} + +// Grep searches for a pattern in files. +func (c *Client) Grep(query, pathPrefix string, limit int) ([]SearchResult, error) { + u := c.url(pathPrefix) + "?grep=" + url.QueryEscape(query) + if limit > 0 { + u += "&limit=" + strconv.Itoa(limit) + } + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var results []SearchResult + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, err + } + return results, nil +} + +// Find searches for files matching criteria. +func (c *Client) Find(pathPrefix string, params url.Values) ([]SearchResult, error) { + params.Set("find", "") + u := c.url(pathPrefix) + "?" + params.Encode() + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var results []SearchResult + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, err + } + return results, nil +} + +// ProgressFunc is called after each part upload completes. +type ProgressFunc func(partNumber, totalParts int, bytesUploaded int64) + +// UploadPlan is the server's 202 response for large file uploads. +type UploadPlan struct { + UploadID string `json:"upload_id"` + PartSize int64 `json:"part_size"` + Parts []PartURL `json:"parts"` +} + +// PartURL is a presigned URL for uploading one part. +type PartURL struct { + Number int `json:"number"` + URL string `json:"url"` + Size int64 `json:"size"` + ChecksumSHA256 string `json:"checksum_sha256,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + ExpiresAt string `json:"expires_at"` +} + +// WriteStream uploads data from a reader. For small files (size < threshold), +// it does a direct PUT with body. For large files, it sends a Content-Length-only +// PUT to get a 202 with presigned URLs, then uploads parts concurrently. +func (c *Client) WriteStream(ctx context.Context, path string, r io.Reader, size int64, progress ProgressFunc) error { + threshold := int64(DefaultSmallFileThreshold) + if c.smallFileThreshold > 0 { + threshold = c.smallFileThreshold + } + if size < threshold { + // Small file: direct PUT with body + data, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("read data: %w", err) + } + return c.Write(path, data) + } + ra, ok := r.(io.ReaderAt) + if !ok { + return fmt.Errorf("large uploads require an io.ReaderAt (seekable source) to compute per-part checksums") + } + checksums, err := computePartChecksumsFromReaderAt(ra, size, PartSize) + if err != nil { + return fmt.Errorf("compute part checksums: %w", err) + } + plan, err := c.initiateUpload(ctx, path, size, checksums) + if err != nil { + return err + } + return c.uploadParts(ctx, plan, ra, progress) +} + +type uploadInitiateRequest struct { + Path string `json:"path"` + TotalSize int64 `json:"total_size"` + PartChecksums []string `json:"part_checksums"` +} + +type uploadResumeRequest struct { + PartChecksums []string `json:"part_checksums"` +} + +func (c *Client) initiateUpload(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, error) { + plan, resp, err := c.initiateUploadByBody(ctx, path, size, checksums) + if err == nil { + return plan, nil + } + if resp != nil { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed { + return c.initiateUploadLegacy(ctx, path, size, checksums) + } + if resp.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(err.Error()), "unknown upload action") { + return c.initiateUploadLegacy(ctx, path, size, checksums) + } + return UploadPlan{}, err + } + return UploadPlan{}, err +} + +func (c *Client) initiateUploadByBody(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, *http.Response, error) { + body, err := json.Marshal(uploadInitiateRequest{Path: path, TotalSize: size, PartChecksums: checksums}) + if err != nil { + return UploadPlan{}, nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/uploads/initiate", bytes.NewReader(body)) + if err != nil { + return UploadPlan{}, nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return UploadPlan{}, nil, err + } + if resp.StatusCode != http.StatusAccepted { + return UploadPlan{}, resp, readError(resp) + } + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + _ = resp.Body.Close() + return UploadPlan{}, nil, fmt.Errorf("decode upload plan: %w", err) + } + _ = resp.Body.Close() + return plan, nil, nil +} + +func (c *Client) initiateUploadLegacy(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.url(path), http.NoBody) + if err != nil { + return UploadPlan{}, err + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("X-FS-Content-Length", fmt.Sprintf("%d", size)) + if len(checksums) > 0 { + req.Header.Set("X-FS-Part-Checksums", strings.Join(checksums, ",")) + } + + resp, err := c.do(req) + if err != nil { + return UploadPlan{}, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusAccepted { + return UploadPlan{}, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return UploadPlan{}, fmt.Errorf("decode upload plan: %w", err) + } + return plan, nil +} + +func uploadParallelism(partSize int64) int { + if partSize <= 0 { + partSize = PartSize + } + byMemory := int(uploadMaxBufferBytes / partSize) + if byMemory < 1 { + byMemory = 1 + } + return min(byMemory, uploadMaxConcurrency) +} + +func checksumParallelism(partSize int64, partCount int) int { + if partSize <= 0 { + partSize = PartSize + } + byMemory := int(uploadMaxBufferBytes / partSize) + if byMemory < 1 { + byMemory = 1 + } + return min(runtime.GOMAXPROCS(0), partCount, byMemory) +} + +func (c *Client) uploadParts(ctx context.Context, plan UploadPlan, ra io.ReaderAt, progress ProgressFunc) error { + stdPartSize := plan.PartSize + if stdPartSize <= 0 && len(plan.Parts) > 0 { + stdPartSize = plan.Parts[0].Size + } + if stdPartSize <= 0 { + stdPartSize = PartSize + } + maxConcurrency := uploadParallelism(stdPartSize) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 1) + sem := make(chan struct{}, maxConcurrency) + var wg sync.WaitGroup + + for _, part := range plan.Parts { + select { + case err := <-errCh: + cancel() + wg.Wait() + return err + default: + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + wg.Add(1) + go func(p PartURL) { + defer wg.Done() + defer func() { <-sem }() + + data := make([]byte, p.Size) + offset := int64(p.Number-1) * stdPartSize + n, err := ra.ReadAt(data, offset) + if err != nil && err != io.EOF { + select { + case errCh <- fmt.Errorf("read part %d: %w", p.Number, err): + default: + } + cancel() + return + } + if int64(n) != p.Size { + select { + case errCh <- fmt.Errorf("short read for part %d: got %d want %d", p.Number, n, p.Size): + default: + } + cancel() + return + } + + _, err = c.uploadOnePart(ctx, p, data) + if err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + cancel() + return + } + + if progress != nil { + progress(p.Number, len(plan.Parts), int64(len(data))) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) uploadOnePart(ctx context.Context, part PartURL, data []byte) (string, error) { + checksum := part.ChecksumSHA256 + if checksum == "" { + h := sha256.Sum256(data) + checksum = base64.StdEncoding.EncodeToString(h[:]) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) + if err != nil { + return "", err + } + for k, v := range part.Headers { + if strings.EqualFold(k, "host") { + continue + } + req.Header.Set(k, v) + } + req.ContentLength = int64(len(data)) + req.Header.Set("x-amz-checksum-sha256", checksum) + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + return resp.Header.Get("ETag"), nil +} + +func (c *Client) completeUpload(ctx context.Context, uploadID string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/complete", nil) + if err != nil { + return err + } + + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// ReadStream reads a file, following 302 redirects for large files. +func (c *Client) ReadStream(ctx context.Context, path string) (io.ReadCloser, error) { + noRedirectClient := *c.httpClient + noRedirectClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.doWithClient(&noRedirectClient, req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect: + _ = resp.Body.Close() + location := resp.Header.Get("Location") + if location == "" { + return nil, fmt.Errorf("302 without Location header") + } + req2, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return nil, err + } + resp2, err := c.httpClient.Do(req2) + if err != nil { + return nil, err + } + if resp2.StatusCode >= 300 { + defer func() { _ = resp2.Body.Close() }() + return nil, readError(resp2) + } + return resp2.Body, nil + + case resp.StatusCode >= 300: + defer func() { _ = resp.Body.Close() }() + return nil, readError(resp) + + default: + return resp.Body, nil + } +} + +// ReadStreamRange reads a byte range from a remote file. +func (c *Client) ReadStreamRange(ctx context.Context, path string, offset, length int64) (io.ReadCloser, error) { + if length <= 0 { + return io.NopCloser(bytes.NewReader(nil)), nil + } + + noRedirectClient := *c.httpClient + noRedirectClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.doWithClient(&noRedirectClient, req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect: + _ = resp.Body.Close() + location := resp.Header.Get("Location") + if location == "" { + return nil, fmt.Errorf("302 without Location header") + } + req2, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return nil, err + } + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) + resp2, err := c.httpClient.Do(req2) + if err != nil { + return nil, err + } + + switch resp2.StatusCode { + case http.StatusPartialContent: + return resp2.Body, nil + case http.StatusRequestedRangeNotSatisfiable: + defer func() { _ = resp2.Body.Close() }() + return io.NopCloser(bytes.NewReader(nil)), nil + default: + if resp2.StatusCode >= 300 { + defer func() { _ = resp2.Body.Close() }() + return nil, readError(resp2) + } + return c.sliceBody(resp2.Body, offset, length) + } + + case resp.StatusCode >= 300: + defer func() { _ = resp.Body.Close() }() + return nil, readError(resp) + + default: + return c.sliceBody(resp.Body, offset, length) + } +} + +func (c *Client) sliceBody(rc io.ReadCloser, offset, length int64) (io.ReadCloser, error) { + if offset > 0 { + if _, err := io.CopyN(io.Discard, rc, offset); err != nil { + _ = rc.Close() + if err == io.EOF { + return io.NopCloser(strings.NewReader("")), nil + } + return nil, fmt.Errorf("skip to offset: %w", err) + } + } + return &limitedReadCloser{r: io.LimitReader(rc, length), c: rc}, nil +} + +type limitedReadCloser struct { + r io.Reader + c io.Closer +} + +func (l *limitedReadCloser) Read(p []byte) (int, error) { return l.r.Read(p) } +func (l *limitedReadCloser) Close() error { return l.c.Close() } + +// UploadMeta is the server's response for querying active uploads. +type UploadMeta struct { + UploadID string `json:"upload_id"` + PartsTotal int `json:"parts_total"` + Status string `json:"status"` + ExpiresAt string `json:"expires_at"` +} + +// ResumeUpload queries for an incomplete upload and resumes it. +func (c *Client) ResumeUpload(ctx context.Context, path string, r io.ReaderAt, totalSize int64, progress ProgressFunc) error { + meta, err := c.queryUpload(ctx, path) + if err != nil { + return err + } + + checksums, err := computePartChecksumsFromReaderAt(r, totalSize, PartSize) + if err != nil { + return fmt.Errorf("compute part checksums: %w", err) + } + plan, err := c.requestResume(ctx, meta.UploadID, checksums) + if err != nil { + return err + } + + if len(plan.Parts) == 0 { + return c.completeUpload(ctx, plan.UploadID) + } + + if err := c.uploadMissingParts(ctx, *plan, r, meta.PartsTotal, progress); err != nil { + return err + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) queryUpload(ctx context.Context, path string) (*UploadMeta, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.baseURL+"/v1/uploads?path="+path+"&status=UPLOADING", nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + + var envelope struct { + Uploads []UploadMeta `json:"uploads"` + } + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + return nil, fmt.Errorf("decode upload meta: %w", err) + } + if len(envelope.Uploads) == 0 { + return nil, fmt.Errorf("no active upload for %s", path) + } + return &envelope.Uploads[0], nil +} + +func (c *Client) requestResume(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, error) { + plan, resp, err := c.requestResumeByBody(ctx, uploadID, checksums) + if err == nil { + return plan, nil + } + if resp != nil { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(err.Error()), "missing x-fs-part-checksums header") { + return c.requestResumeLegacy(ctx, uploadID, checksums) + } + return nil, err + } + return nil, err +} + +func (c *Client) requestResumeByBody(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, *http.Response, error) { + body, err := json.Marshal(uploadResumeRequest{PartChecksums: checksums}) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/resume", bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, nil, err + } + + if resp.StatusCode == http.StatusGone { + _ = resp.Body.Close() + return nil, nil, fmt.Errorf("upload %s has expired", uploadID) + } + if resp.StatusCode >= 300 { + return nil, resp, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + _ = resp.Body.Close() + return nil, nil, fmt.Errorf("decode resume plan: %w", err) + } + _ = resp.Body.Close() + return &plan, nil, nil +} + +func (c *Client) requestResumeLegacy(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/resume", nil) + if err != nil { + return nil, err + } + if len(checksums) > 0 { + req.Header.Set("X-FS-Part-Checksums", strings.Join(checksums, ",")) + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusGone { + return nil, fmt.Errorf("upload %s has expired", uploadID) + } + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return nil, fmt.Errorf("decode resume plan: %w", err) + } + return &plan, nil +} + +func (c *Client) uploadMissingParts(ctx context.Context, plan UploadPlan, r io.ReaderAt, totalParts int, progress ProgressFunc) error { + stdPartSize := plan.PartSize + if stdPartSize <= 0 { + stdPartSize = PartSize + } + maxConcurrency := uploadParallelism(stdPartSize) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sem := make(chan struct{}, maxConcurrency) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + for _, part := range plan.Parts { + select { + case err := <-errCh: + cancel() + wg.Wait() + return err + default: + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + wg.Add(1) + go func(p PartURL) { + defer wg.Done() + defer func() { <-sem }() + + data := make([]byte, p.Size) + offset := int64(p.Number-1) * stdPartSize + n, err := r.ReadAt(data, offset) + if err != nil && err != io.EOF { + select { + case errCh <- fmt.Errorf("read part %d at offset %d: %w", p.Number, offset, err): + default: + } + cancel() + return + } + if int64(n) != p.Size { + select { + case errCh <- fmt.Errorf("short read for part %d at offset %d: got %d want %d", p.Number, offset, n, p.Size): + default: + } + cancel() + return + } + + _, err = c.uploadOnePart(ctx, p, data) + if err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + cancel() + return + } + if progress != nil { + progress(p.Number, totalParts, int64(len(data))) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return nil +} + +func computePartChecksumsFromReaderAt(r io.ReaderAt, totalSize int64, partSize int64) ([]string, error) { + if totalSize <= 0 { + return nil, nil + } + parts := CalcParts(totalSize, partSize) + checksums := make([]string, len(parts)) + workers := checksumParallelism(partSize, len(parts)) + + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + partCh := make(chan int, len(parts)) + + for i := range parts { + partCh <- i + } + close(partCh) + + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, partSize) + for i := range partCh { + p := parts[i] + data := buf[:p.Size] + offset := int64(p.Number-1) * partSize + n, err := r.ReadAt(data, offset) + if err != nil && err != io.EOF { + errOnce.Do(func() { firstErr = fmt.Errorf("read part %d: %w", p.Number, err) }) + return + } + if int64(n) != p.Size { + errOnce.Do(func() { + firstErr = fmt.Errorf("short read for part %d: got %d want %d", p.Number, n, p.Size) + }) + return + } + h := sha256.Sum256(data) + checksums[i] = base64.StdEncoding.EncodeToString(h[:]) + } + }() + } + wg.Wait() + + if firstErr != nil { + return nil, firstErr + } + return checksums, nil +} + +// PatchPlan mirrors the server's response for a PATCH request. +type PatchPlan struct { + UploadID string `json:"upload_id"` + PartSize int64 `json:"part_size"` + UploadParts []*PatchPartURL `json:"upload_parts"` + CopiedParts []int `json:"copied_parts"` +} + +// PatchPartURL describes one dirty part the client must upload. +type PatchPartURL struct { + Number int `json:"number"` + URL string `json:"url"` + Size int64 `json:"size"` + Headers map[string]string `json:"headers,omitempty"` + ExpiresAt string `json:"expires_at"` + ReadURL string `json:"read_url,omitempty"` + ReadHeaders map[string]string `json:"read_headers,omitempty"` +} + +// PatchFile performs a partial update of a large file. +func (c *Client) PatchFile(ctx context.Context, path string, newSize int64, dirtyParts []int, readPart func(partNumber int, partSize int64, origData []byte) ([]byte, error), progress ProgressFunc) error { + reqBody, err := json.Marshal(map[string]any{ + "new_size": newSize, + "dirty_parts": dirtyParts, + }) + if err != nil { + return fmt.Errorf("marshal patch request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, c.url(path), bytes.NewReader(reqBody)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusAccepted { + return readError(resp) + } + + var plan PatchPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return fmt.Errorf("decode patch plan: %w", err) + } + + const maxConcurrency = 4 + sem := make(chan struct{}, maxConcurrency) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + totalParts := len(plan.UploadParts) + len(plan.CopiedParts) + + for _, part := range plan.UploadParts { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + select { + case err := <-errCh: + wg.Wait() + return err + default: + } + + wg.Add(1) + go func(p *PatchPartURL) { + defer wg.Done() + defer func() { <-sem }() + + if err := c.uploadPatchPart(ctx, p, readPart); err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + return + } + + if progress != nil { + progress(p.Number, totalParts, p.Size) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) uploadPatchPart(ctx context.Context, part *PatchPartURL, readPart func(int, int64, []byte) ([]byte, error)) error { + var origData []byte + if part.ReadURL != "" { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, part.ReadURL, nil) + if err != nil { + return fmt.Errorf("create read request: %w", err) + } + for k, v := range part.ReadHeaders { + if !strings.EqualFold(k, "host") { + req.Header.Set(k, v) + } + } + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("download original part: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return fmt.Errorf("download original part: HTTP %d", resp.StatusCode) + } + origData, err = io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read original part body: %w", err) + } + } + + data, err := readPart(part.Number, part.Size, origData) + if err != nil { + return fmt.Errorf("readPart callback: %w", err) + } + + h := sha256.Sum256(data) + checksum := base64.StdEncoding.EncodeToString(h[:]) + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) + if err != nil { + return err + } + for k, v := range part.Headers { + if strings.EqualFold(k, "host") { + continue + } + req.Header.Set(k, v) + } + req.ContentLength = int64(len(data)) + req.Header.Set("x-amz-checksum-sha256", checksum) + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload part: HTTP %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// VaultSecret holds secret metadata returned by the management API. +type VaultSecret struct { + Name string `json:"name"` + SecretType string `json:"secret_type"` + Revision int64 `json:"revision"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// VaultTokenIssueResponse is returned when issuing a scoped capability token. +type VaultTokenIssueResponse struct { + Token string `json:"token"` + TokenID string `json:"token_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +// VaultAuditEvent is an audit event returned by the vault audit API. +type VaultAuditEvent struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + TokenID string `json:"token_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + TaskID string `json:"task_id,omitempty"` + SecretName string `json:"secret_name,omitempty"` + FieldName string `json:"field_name,omitempty"` + Adapter string `json:"adapter,omitempty"` + Detail map[string]any `json:"detail,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +func (c *Client) vaultURL(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return c.baseURL + "/v1/vault" + path +} + +// CreateVaultSecret creates a new secret via the management API. +func (c *Client) CreateVaultSecret(ctx context.Context, name string, fields map[string]string) (*VaultSecret, error) { + body, err := json.Marshal(map[string]any{ + "name": name, + "fields": fields, + "created_by": "ticloud-cli", + }) + if err != nil { + return nil, fmt.Errorf("marshal secret create request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.vaultURL("/secrets"), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var sec VaultSecret + if err := json.NewDecoder(resp.Body).Decode(&sec); err != nil { + return nil, fmt.Errorf("decode secret create response: %w", err) + } + return &sec, nil +} + +// UpdateVaultSecret rotates a secret via the management API. +func (c *Client) UpdateVaultSecret(ctx context.Context, name string, fields map[string]string) (*VaultSecret, error) { + body, err := json.Marshal(map[string]any{ + "fields": fields, + "updated_by": "ticloud-cli", + }) + if err != nil { + return nil, fmt.Errorf("marshal secret update request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.vaultURL("/secrets/"+url.PathEscape(name)), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var sec VaultSecret + if err := json.NewDecoder(resp.Body).Decode(&sec); err != nil { + return nil, fmt.Errorf("decode secret update response: %w", err) + } + return &sec, nil +} + +// DeleteVaultSecret deletes a secret via the management API. +func (c *Client) DeleteVaultSecret(ctx context.Context, name string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.vaultURL("/secrets/"+url.PathEscape(name)), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// ListVaultSecrets lists secret metadata via the management API. +func (c *Client) ListVaultSecrets(ctx context.Context) ([]VaultSecret, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/secrets"), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Secrets []VaultSecret `json:"secrets"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode secret list response: %w", err) + } + if result.Secrets == nil { + result.Secrets = []VaultSecret{} + } + return result.Secrets, nil +} + +// IssueVaultToken issues a scoped capability token via the management API. +func (c *Client) IssueVaultToken(ctx context.Context, agentID, taskID string, scope []string, ttl time.Duration) (*VaultTokenIssueResponse, error) { + ttlSeconds := int(ttl / time.Second) + body, err := json.Marshal(map[string]any{ + "agent_id": agentID, + "task_id": taskID, + "scope": scope, + "ttl_seconds": ttlSeconds, + }) + if err != nil { + return nil, fmt.Errorf("marshal token issue request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.vaultURL("/tokens"), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result VaultTokenIssueResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode token issue response: %w", err) + } + return &result, nil +} + +// RevokeVaultToken revokes a capability token via the management API. +func (c *Client) RevokeVaultToken(ctx context.Context, tokenID string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.vaultURL("/tokens/"+url.PathEscape(tokenID)), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// QueryVaultAudit queries the audit log via the management API. +func (c *Client) QueryVaultAudit(ctx context.Context, secretName string, limit int) ([]VaultAuditEvent, error) { + u, err := url.Parse(c.vaultURL("/audit")) + if err != nil { + return nil, err + } + q := u.Query() + if secretName != "" { + q.Set("secret", secretName) + } + if limit > 0 { + q.Set("limit", fmt.Sprintf("%d", limit)) + } + u.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Events []VaultAuditEvent `json:"events"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode audit response: %w", err) + } + if result.Events == nil { + result.Events = []VaultAuditEvent{} + } + return result.Events, nil +} + +// ListReadableVaultSecrets enumerates secrets visible to the bearer capability token. +func (c *Client) ListReadableVaultSecrets(ctx context.Context) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read"), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Secrets []string `json:"secrets"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode readable secret list response: %w", err) + } + if result.Secrets == nil { + result.Secrets = []string{} + } + return result.Secrets, nil +} + +// ReadVaultSecret reads all fields of a secret using the consumption API. +func (c *Client) ReadVaultSecret(ctx context.Context, name string) (map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read/"+url.PathEscape(name)), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result map[string]string + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode secret read response: %w", err) + } + if result == nil { + result = map[string]string{} + } + return result, nil +} + +// ReadVaultSecretField reads a single field via the consumption API. +func (c *Client) ReadVaultSecretField(ctx context.Context, name, field string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read/"+url.PathEscape(name)+"/"+url.PathEscape(field)), nil) + if err != nil { + return "", err + } + resp, err := c.do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return "", readError(resp) + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read field response: %w", err) + } + return string(data), nil +} diff --git a/internal/service/fs/client_auth_test.go b/internal/service/fs/client_auth_test.go new file mode 100644 index 00000000..1ae12452 --- /dev/null +++ b/internal/service/fs/client_auth_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestClientInjectsZeroInstanceIDHeader(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + instanceID := "test-instance-123" + client := NewClient(server.URL, &http.Client{}, "", instanceID) + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders == nil { + t.Fatal("No request was made to server") + } + + instanceHeader := capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") + if instanceHeader != instanceID { + t.Errorf("X-TIDBCLOUD-ZERO-INSTANCE-ID header = %q, want %q", instanceHeader, instanceID) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != "" { + t.Error("X-TIDBCLOUD-CLUSTER-ID should not be set when zero instance ID is used") + } +} + +func TestClientInjectsClusterIDHeader(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + clusterID := "test-cluster-456" + client := NewClient(server.URL, &http.Client{}, clusterID, "") + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders == nil { + t.Fatal("No request was made to server") + } + + clusterHeader := capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") + if clusterHeader != clusterID { + t.Errorf("X-TIDBCLOUD-CLUSTER-ID header = %q, want %q", clusterHeader, clusterID) + } + + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when cluster ID is used") + } +} + +func TestClientClusterIDWinsOverZeroInstanceID(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + clusterID := "test-cluster-456" + instanceID := "test-instance-123" + client := NewClient(server.URL, &http.Client{}, clusterID, instanceID) + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != clusterID { + t.Errorf("X-TIDBCLOUD-CLUSTER-ID header = %q, want %q", capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID"), clusterID) + } + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when both IDs are provided (cluster wins)") + } +} + +func TestClientNoDBHeadersWhenNotConfigured(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + client := NewClient(server.URL, &http.Client{}, "", "") + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != "" { + t.Error("X-TIDBCLOUD-CLUSTER-ID should not be set when not configured") + } + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when not configured") + } +} diff --git a/internal/service/fs/fuse/fs.go b/internal/service/fs/fuse/fs.go new file mode 100644 index 00000000..e502f5cf --- /dev/null +++ b/internal/service/fs/fuse/fs.go @@ -0,0 +1,497 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fuse + +import ( + "context" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + gofuse "github.com/hanwen/go-fuse/v2/fuse" +) + +// ReadOnlyFS is a read-only FUSE filesystem for TiDB Cloud FS. +type ReadOnlyFS struct { + gofuse.RawFileSystem + + client *fs.Client + opts *MountOptions + inodes *inodeManager + dirs *dirCache + uid uint32 + gid uint32 +} + +// inodeManager manages inode to path mappings. +type inodeManager struct { + mu sync.RWMutex + byInode map[uint64]*inodeEntry + byPath map[string]uint64 + nextIno uint64 +} + +type inodeEntry struct { + Path string + IsDir bool + RefCnt uint64 + Attr *fs.StatResult + AttrTime time.Time +} + +// dirCache caches directory listings. +type dirCache struct { + mu sync.RWMutex + entries map[string]*dirCacheEntry + ttl time.Duration +} + +type dirCacheEntry struct { + Items []fs.FileInfo + Timestamp time.Time +} + +// NewReadOnlyFS creates a new read-only FUSE filesystem. +func NewReadOnlyFS(client *fs.Client, opts *MountOptions) *ReadOnlyFS { + fsys := &ReadOnlyFS{ + RawFileSystem: gofuse.NewDefaultRawFileSystem(), + client: client, + opts: opts, + inodes: &inodeManager{ + byInode: make(map[uint64]*inodeEntry), + byPath: make(map[string]uint64), + nextIno: gofuse.FUSE_ROOT_ID, + }, + dirs: &dirCache{ + entries: make(map[string]*dirCacheEntry), + ttl: 5 * time.Second, + }, + uid: uint32(os.Getuid()), + gid: uint32(os.Getgid()), + } + + // Initialize root inode + fsys.inodes.byPath["/"] = gofuse.FUSE_ROOT_ID + fsys.inodes.byInode[gofuse.FUSE_ROOT_ID] = &inodeEntry{ + Path: "/", + IsDir: true, + RefCnt: 1, + } + + return fsys +} + +// GetAttr implements GetAttr. +func (f *ReadOnlyFS) GetAttr(cancel <-chan struct{}, in *gofuse.GetAttrIn, out *gofuse.AttrOut) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + // Check cache + if entry.Attr != nil && time.Since(entry.AttrTime) < time.Second { + f.fillAttr(entry, out) + return gofuse.OK + } + + // Special-case root directory: backend may not support Stat("/") + if entry.Path == "/" { + entry.IsDir = true + entry.Attr = &fs.StatResult{IsDir: true, Size: 0} + entry.AttrTime = time.Now() + f.fillAttr(entry, out) + return gofuse.OK + } + + // Fetch from server + stat, err := f.client.Stat(entry.Path) + if err != nil { + return gofuse.ENOENT + } + + entry.Attr = stat + entry.AttrTime = time.Now() + entry.IsDir = stat.IsDir + + f.fillAttr(entry, out) + return gofuse.OK +} + +func (f *ReadOnlyFS) fillAttr(entry *inodeEntry, out *gofuse.AttrOut) { + out.Attr.Ino = f.inodes.getIno(entry.Path) + if entry.Attr != nil { + out.Attr.Size = uint64(entry.Attr.Size) + } else { + out.Attr.Size = 0 + } + out.Attr.Mode = 0o755 + if entry.IsDir { + out.Attr.Mode |= syscall.S_IFDIR + out.Attr.Nlink = 2 + } else { + out.Attr.Mode |= syscall.S_IFREG + out.Attr.Nlink = 1 + } + out.Attr.Uid = f.uid + out.Attr.Gid = f.gid + out.SetTimeout(1 * time.Second) +} + +// Lookup implements Lookup. +func (f *ReadOnlyFS) Lookup(cancel <-chan struct{}, header *gofuse.InHeader, name string, out *gofuse.EntryOut) gofuse.Status { + parentIno := header.NodeId + + f.inodes.mu.RLock() + parentEntry, ok := f.inodes.byInode[parentIno] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + path := filepath.Join(parentEntry.Path, name) + + // Get or create inode + ino := f.inodes.GetOrCreate(path) + + f.inodes.mu.RLock() + entry := f.inodes.byInode[ino] + f.inodes.mu.RUnlock() + + // Fetch attr + stat, err := f.client.Stat(path) + if err != nil { + return gofuse.ENOENT + } + + entry.Attr = stat + entry.AttrTime = time.Now() + entry.IsDir = stat.IsDir + + out.NodeId = ino + out.Generation = 1 + out.Attr.Ino = ino + if entry.Attr != nil { + out.Attr.Size = uint64(entry.Attr.Size) + } else { + out.Attr.Size = 0 + } + out.Attr.Mode = 0o755 + if entry.IsDir { + out.Attr.Mode |= syscall.S_IFDIR + } else { + out.Attr.Mode |= syscall.S_IFREG + } + out.Attr.Nlink = 1 + out.Attr.Uid = f.uid + out.Attr.Gid = f.gid + + return gofuse.OK +} + +// Access implements Access. +func (f *ReadOnlyFS) Access(cancel <-chan struct{}, in *gofuse.AccessIn) gofuse.Status { + return gofuse.OK +} + +// OpenDir implements OpenDir. +func (f *ReadOnlyFS) OpenDir(cancel <-chan struct{}, in *gofuse.OpenIn, out *gofuse.OpenOut) gofuse.Status { + out.Fh = in.NodeId + return gofuse.OK +} + +// ReadDir implements ReadDir. +func (f *ReadOnlyFS) ReadDir(cancel <-chan struct{}, in *gofuse.ReadIn, out *gofuse.DirEntryList) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + realEntries, err := f.getDirEntries(entry.Path) + if err != nil { + return gofuse.EIO + } + + dotdotIno := in.NodeId + if entry.Path != "/" { + parentPath := filepath.Dir(entry.Path) + dotdotIno = f.inodes.getIno(parentPath) + if dotdotIno == 0 { + dotdotIno = f.inodes.GetOrCreate(parentPath) + } + } + + type dirItem struct { + name string + ino uint64 + mode uint32 + } + + var items []dirItem + for _, e := range realEntries { + if e.Name == "" || e.Name == ":" { + continue + } + childPath := filepath.Join(entry.Path, e.Name) + childIno := f.inodes.GetOrCreate(childPath) + mode := uint32(0o755) + if e.IsDir { + mode |= syscall.S_IFDIR + } else { + mode |= syscall.S_IFREG + } + items = append(items, dirItem{e.Name, childIno, mode}) + } + + offset := int(in.Offset) + for i, item := range items { + if i < offset { + continue + } + de := gofuse.DirEntry{ + Name: item.name, + Ino: item.ino, + Mode: item.mode, + Off: uint64(i + 1), + } + if !out.AddDirEntry(de) { + break + } + } + + return gofuse.OK +} + +// ReadDirPlus implements ReadDirPlus. +func (f *ReadOnlyFS) ReadDirPlus(cancel <-chan struct{}, in *gofuse.ReadIn, out *gofuse.DirEntryList) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + realEntries, err := f.getDirEntries(entry.Path) + if err != nil { + return gofuse.EIO + } + + dotdotIno := in.NodeId + if entry.Path != "/" { + parentPath := filepath.Dir(entry.Path) + dotdotIno = f.inodes.getIno(parentPath) + if dotdotIno == 0 { + dotdotIno = f.inodes.GetOrCreate(parentPath) + } + } + + type dirItem struct { + name string + ino uint64 + mode uint32 + } + + var items []dirItem + for _, e := range realEntries { + if e.Name == "" || e.Name == ":" { + continue + } + childPath := filepath.Join(entry.Path, e.Name) + childIno := f.inodes.GetOrCreate(childPath) + mode := uint32(0o755) + if e.IsDir { + mode |= syscall.S_IFDIR + } else { + mode |= syscall.S_IFREG + } + items = append(items, dirItem{e.Name, childIno, mode}) + } + + offset := int(in.Offset) + for i, item := range items { + if i < offset { + continue + } + de := gofuse.DirEntry{ + Name: item.name, + Ino: item.ino, + Mode: item.mode, + Off: uint64(i + 1), + } + entryOut := out.AddDirLookupEntry(de) + if entryOut == nil { + break + } + entryOut.NodeId = item.ino + entryOut.Generation = 1 + entryOut.Attr.Ino = item.ino + entryOut.Attr.Size = 0 + if item.mode&syscall.S_IFDIR != 0 { + entryOut.Attr.Mode = syscall.S_IFDIR | 0o755 + } else { + entryOut.Attr.Mode = syscall.S_IFREG | 0o755 + } + if item.mode&syscall.S_IFDIR != 0 { + entryOut.Attr.Nlink = 2 + } else { + entryOut.Attr.Nlink = 1 + } + entryOut.Attr.Uid = f.uid + entryOut.Attr.Gid = f.gid + entryOut.SetEntryTimeout(1 * time.Second) + entryOut.SetAttrTimeout(1 * time.Second) + } + + return gofuse.OK +} + +// Open implements Open. +func (f *ReadOnlyFS) Open(cancel <-chan struct{}, in *gofuse.OpenIn, out *gofuse.OpenOut) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + if entry.IsDir { + return gofuse.EISDIR + } + + out.Fh = in.NodeId + out.OpenFlags = gofuse.FOPEN_KEEP_CACHE + + return gofuse.OK +} + +// Read implements Read. +func (f *ReadOnlyFS) Read(cancel <-chan struct{}, in *gofuse.ReadIn, buf []byte) (gofuse.ReadResult, gofuse.Status) { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return nil, gofuse.ENOENT + } + + reader, err := f.client.ReadStreamRange(context.Background(), entry.Path, int64(in.Offset), int64(len(buf))) + if err != nil { + return nil, gofuse.EIO + } + defer reader.Close() + + n, err := reader.Read(buf) + if err != nil && err != io.EOF { + return nil, gofuse.EIO + } + + return gofuse.ReadResultData(buf[:n]), gofuse.OK +} + +// Release implements Release. +func (f *ReadOnlyFS) Release(cancel <-chan struct{}, in *gofuse.ReleaseIn) { + // Nothing to do for read-only +} + +// ReleaseDir implements ReleaseDir. +func (f *ReadOnlyFS) ReleaseDir(in *gofuse.ReleaseIn) { + // Nothing to do +} + +// Forget implements Forget. +func (f *ReadOnlyFS) Forget(nodeID uint64, nlookup uint64) { + f.inodes.Forget(nodeID, nlookup) +} + +// getDirEntries gets directory entries with caching. +func (f *ReadOnlyFS) getDirEntries(path string) ([]fs.FileInfo, error) { + f.dirs.mu.RLock() + cache, ok := f.dirs.entries[path] + f.dirs.mu.RUnlock() + + if ok && time.Since(cache.Timestamp) < f.dirs.ttl { + return cache.Items, nil + } + + entries, err := f.client.List(path) + if err != nil { + return nil, err + } + + f.dirs.mu.Lock() + f.dirs.entries[path] = &dirCacheEntry{ + Items: entries, + Timestamp: time.Now(), + } + f.dirs.mu.Unlock() + + return entries, nil +} + +// inodeManager methods + +func (im *inodeManager) GetOrCreate(path string) uint64 { + im.mu.Lock() + defer im.mu.Unlock() + + if ino, ok := im.byPath[path]; ok { + return ino + } + + ino := atomic.AddUint64(&im.nextIno, 1) + im.byPath[path] = ino + im.byInode[ino] = &inodeEntry{ + Path: path, + RefCnt: 1, + } + return ino +} + +func (im *inodeManager) getIno(path string) uint64 { + im.mu.RLock() + defer im.mu.RUnlock() + return im.byPath[path] +} + +func (im *inodeManager) Forget(ino uint64, nlookup uint64) { + im.mu.Lock() + defer im.mu.Unlock() + + entry, ok := im.byInode[ino] + if !ok { + return + } + + if entry.RefCnt <= nlookup { + delete(im.byInode, ino) + delete(im.byPath, entry.Path) + } else { + entry.RefCnt -= nlookup + } +} diff --git a/internal/service/fs/fuse/mount.go b/internal/service/fs/fuse/mount.go new file mode 100644 index 00000000..d3d8586f --- /dev/null +++ b/internal/service/fs/fuse/mount.go @@ -0,0 +1,107 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fuse provides FUSE filesystem implementation for TiDB Cloud FS. +package fuse + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + gofuse "github.com/hanwen/go-fuse/v2/fuse" +) + +// MountOptions configures the FUSE mount. +type MountOptions struct { + Client *fs.Client // FS client (with auth and headers configured) + MountPoint string // Local mount point + ReadOnly bool // Mount as read-only (MVP default) + Debug bool // Enable FUSE debug logging + AllowOther bool // Allow other users to access mount +} + +// Validate checks the mount options. +func (o *MountOptions) Validate() error { + if o.Client == nil { + return fmt.Errorf("FS client is required") + } + if o.MountPoint == "" { + return fmt.Errorf("mount point is required") + } + return nil +} + +// Mount creates and serves a FUSE mount. It blocks until the filesystem +// is unmounted or a signal (SIGINT, SIGTERM) is received. +func Mount(opts *MountOptions) error { + if err := opts.Validate(); err != nil { + return err + } + + // Create mount point if not exists + if err := os.MkdirAll(opts.MountPoint, 0o755); err != nil { + return fmt.Errorf("create mount point: %w", err) + } + + // Test connection + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := opts.Client.List("/"); err != nil { + return fmt.Errorf("cannot reach FS server: %w", err) + } + _ = ctx + + // Create FUSE filesystem + fsys := NewReadOnlyFS(opts.Client, opts) + + // Configure mount options + fuseOpts := &gofuse.MountOptions{ + FsName: "ticloudfs", + Name: "ticloudfs", + MaxReadAhead: 128 * 1024, + Debug: opts.Debug, + AllowOther: opts.AllowOther, + } + if opts.ReadOnly { + fuseOpts.Options = append(fuseOpts.Options, "ro") + } + + // Create FUSE server + server, err := gofuse.NewServer(fsys, opts.MountPoint, fuseOpts) + if err != nil { + return fmt.Errorf("fuse mount: %w", err) + } + + // Handle signals + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigCh + fmt.Fprintf(os.Stderr, "\nUnmounting %s...\n", opts.MountPoint) + if err := server.Unmount(); err != nil { + fmt.Fprintf(os.Stderr, "Unmount error: %v\n", err) + } + }() + + fmt.Fprintf(os.Stderr, "Mounted on %s (server: %s)\n", opts.MountPoint, opts.Client.GetBaseURL()) + server.Serve() + return nil +} From c39506eb4b1605288fca2de84057112d4ac8988a Mon Sep 17 00:00:00 2001 From: mornyx Date: Sun, 19 Apr 2026 20:19:42 +0800 Subject: [PATCH 2/6] docs(fs): translate docs/fs.md to English --- docs/fs.md | 178 ++++++++++++++++++++++++++--------------------------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/docs/fs.md b/docs/fs.md index 02768697..1a507a18 100644 --- a/docs/fs.md +++ b/docs/fs.md @@ -1,114 +1,114 @@ -# TiDB Cloud FS (ticloud fs) 使用指南 +# TiDB Cloud FS (ticloud fs) User Guide -**TiDB Cloud FS** 是 TiDB Cloud 提供的托管文件存储服务，为 TiDB 实例提供统一的文件系统抽象。你可以像操作本地文件系统一样在 TiDB Cloud 中存储、读取和管理文件，同时支持 SQL 查询、密钥保险箱（Vault）以及 FUSE 挂载等高级能力。 +**TiDB Cloud FS** is a managed file storage service provided by TiDB Cloud that offers a unified filesystem abstraction for TiDB instances. You can store, read, and manage files in TiDB Cloud just like a local filesystem, with advanced capabilities such as SQL queries, a secret vault, and FUSE mount support. -**`ticloud fs`** 是 TiDB Cloud CLI 中用于操作 TiDB Cloud FS 的命令组，支持上传下载、目录浏览、SQL 执行、密钥管理以及 FUSE 挂载等功能。 +**`ticloud fs`** is the command group in the TiDB Cloud CLI for operating TiDB Cloud FS, supporting upload/download, directory browsing, SQL execution, secret management, and FUSE mount. --- -## 前置依赖：关联一个 TiDB 实例 +## Prerequisites: Associate a TiDB Instance -FS 的数据存储依赖于一个具体的 TiDB 实例。根据你拥有的实例类型，需要将其 ID 绑定到 CLI 配置中，并完成一次性的 FS 初始化（`init`）。 +FS data storage depends on a specific TiDB instance. Depending on the type of instance you have, you need to bind its ID to the CLI configuration and complete a one-time FS initialization (`init`). -### 使用 Serverless TiDB +### Using Serverless TiDB -如果你已经有一个 **TiDB Cloud Serverless** 集群（可以通过 [TiDB Cloud Web Console](https://tidbcloud.com/) 创建，也可以使用 `ticloud serverless create` 命令创建），请按以下步骤配置： +If you already have a **TiDB Cloud Serverless** cluster (created via the [TiDB Cloud Web Console](https://tidbcloud.com/) or the `ticloud serverless create` command), configure it as follows: -1. **配置鉴权** +1. **Configure Authentication** - Serverless 集群受 TiDB Cloud 平台统一管控，访问其 FS 服务时必须提供有效的平台鉴权信息。你可以选择以下任一方式： + Serverless clusters are managed by the TiDB Cloud platform, so you must provide valid platform credentials to access their FS service. Choose one of the following methods: - - **OAuth 登录（推荐）** + - **OAuth Login (Recommended)** ```bash ticloud auth login ``` - - **API Key 鉴权** + - **API Key Authentication** ```bash ticloud config set public-key ticloud config set private-key ``` -2. **配置 cluster-id** +2. **Configure cluster-id** ```bash ticloud config set fs.cluster-id ``` -3. **初始化 FS** +3. **Initialize FS** ```bash ticloud fs init --user admin --password ``` - > 每个新的 Serverless 集群在首次使用 FS 前，都必须执行 `init` 来为其开通 FS 租户。 + > Every new Serverless cluster must run `init` before using FS for the first time to provision the FS tenant. -### 使用 Zero TiDB +### Using Zero TiDB -如果你使用的是 **TiDB Zero** 实例，配置方式如下。如果你还不了解 TiDB Zero，可以访问 [https://zero.tidbcloud.com/](https://zero.tidbcloud.com/) 了解更多。 +If you are using a **TiDB Zero** instance, configure it as follows. If you are not familiar with TiDB Zero, visit [https://zero.tidbcloud.com/](https://zero.tidbcloud.com/) to learn more. -1. **配置 zero-instance-id** +1. **Configure zero-instance-id** ```bash ticloud config set fs.zero-instance-id ``` - TiDB Zero 采用独立的部署与访问模型，**不需要**执行 `ticloud auth login`，也**不需要**配置 `public-key` / `private-key`。 + TiDB Zero uses an independent deployment and access model. You **do not** need to run `ticloud auth login` or configure `public-key` / `private-key`. -2. **初始化 FS** +2. **Initialize FS** ```bash ticloud fs init --user admin --password ``` - > 与 Serverless 相同，新的 Zero 实例也需要执行一次 `init` 来开通 FS 租户。 + > Like Serverless, every new Zero instance must run `init` once to provision the FS tenant. --- -## 路径格式说明 +## Path Format -FS 使用 `:/` 前缀表示远程路径： +FS uses the `:/` prefix to denote remote paths: -- `:/` — 远程根目录 -- `:/path/to/file.txt` — 远程文件路径 +- `:/` — Remote root directory +- `:/path/to/file.txt` — Remote file path --- -## 子命令手册 +## Subcommand Manual -### 初始化与配置 +### Initialization and Configuration -在使用任何文件操作之前，必须先为关联的数据库创建 FS 租户。这一步只需要执行一次。 +Before performing any file operations, you must create an FS tenant for the associated database. This step only needs to be performed once. #### `ticloud fs init` -为当前关联的数据库初始化 FS 租户，每个新数据库在首次使用 FS 前必须执行一次。 +Initialize the FS tenant for the currently associated database. This must be run before using FS for the first time on a new database. ```bash ticloud fs init --user admin --password secret ``` -### 基础文件操作 +### Basic File Operations -`ticloud fs` 提供了一组类似 Unix 文件系统的命令，用于管理远程文件和目录。你可以像操作本地文件一样进行上传、下载、浏览和删除。 +`ticloud fs` provides a set of Unix-like filesystem commands for managing remote files and directories. You can upload, download, browse, and delete files as if they were local. #### `ticloud fs ls` -列出远程目录内容。支持 `-l` 查看详情。 +List remote directory contents. Supports `-l` for detailed view. ```bash -# 查看根目录 +# List root directory ticloud fs ls :/ -# 查看某个文件夹，并显示文件大小等详细信息 +# List a folder with size details ticloud fs ls -l :/myfolder ``` #### `ticloud fs cat` -显示远程文件内容，类似于 Unix 的 `cat` 命令，适合快速查看文本文件。 +Display remote file contents, similar to the Unix `cat` command. Useful for quickly viewing text files. ```bash ticloud fs cat :/readme.txt @@ -116,25 +116,25 @@ ticloud fs cat :/readme.txt #### `ticloud fs cp` -复制文件。支持本地↔远程、远程↔远程、从标准输入上传，以及断点续传大文件。 +Copy files. Supports local↔remote, remote↔remote, stdin upload, and resuming large file transfers. ```bash -# 上传本地文件到远程目录 +# Upload a local file to remote ticloud fs cp local.txt :/remote/ -# 从远程下载文件到当前目录 +# Download a remote file to current directory ticloud fs cp :/remote/file.txt . -# 通过管道将标准输入内容写入远程文件 +# Upload from stdin via pipe echo "hello" | ticloud fs cp - :/file.txt -# 断点续传大文件，适合网络不稳定场景 +# Resume a large file upload (useful for unstable networks) ticloud fs cp --resume large.zip :/remote/ ``` #### `ticloud fs mkdir` -创建远程目录，支持多级路径。 +Create a remote directory. Supports nested paths. ```bash ticloud fs mkdir :/mydir @@ -142,7 +142,7 @@ ticloud fs mkdir :/mydir #### `ticloud fs mv` -移动或重命名远程文件/目录。 +Move or rename remote files/directories. ```bash ticloud fs mv :/oldname :/newname @@ -150,7 +150,7 @@ ticloud fs mv :/oldname :/newname #### `ticloud fs rm` -删除远程文件或空目录。 +Delete remote files or empty directories. ```bash ticloud fs rm :/file.txt @@ -158,19 +158,19 @@ ticloud fs rm :/file.txt #### `ticloud fs stat` -查看远程文件或目录的元数据，例如大小、修改时间等。 +View remote file or directory metadata, such as size and modification time. ```bash ticloud fs stat :/file.txt ``` -### 搜索与查询 +### Search and Query -除了基础文件操作，`ticloud fs` 还支持直接在远程存储中搜索文件内容和执行 SQL 查询，方便你快速定位信息或进行数据分析。 +In addition to basic file operations, `ticloud fs` supports searching file contents and executing SQL queries directly on remote storage, making it easy to locate information or perform data analysis. #### `ticloud fs grep` -在远程文件中搜索匹配内容。支持 `--limit` 限制结果数量。 +Search for matching content in remote files. Supports `--limit` to restrict the number of results. ```bash ticloud fs grep "TODO" :/project --limit 20 @@ -178,55 +178,55 @@ ticloud fs grep "TODO" :/project --limit 20 #### `ticloud fs find` -按条件查找远程文件。支持 `-name`、`-size`、`-newer` 等过滤条件，类似 Unix 的 `find` 命令。 +Find remote files by criteria. Supports `-name`, `-size`, `-newer` filters, similar to the Unix `find` command. ```bash -# 按文件名后缀查找 +# Find by file extension ticloud fs find :/ -name "*.go" -# 查找大于 1MB 的文件 +# Find files larger than 1MB ticloud fs find :/ -size +1M ``` #### `ticloud fs sql` -在 FS 后端执行 SQL 查询，可以直接对存储的数据进行结构化查询。 +Execute SQL queries against the FS backend for structured queries on stored data. ```bash ticloud fs sql -q "SELECT 1" ``` -### Secret 管理 +### Secret Management -TiDB Cloud FS 内置了一个密钥保险箱（Vault），用于安全地存储敏感信息，例如数据库连接串、API Key 等。你可以通过子命令对这些密钥进行增删改查，还可以为第三方 Agent 颁发受限访问令牌。 +TiDB Cloud FS includes a built-in secret vault for securely storing sensitive information such as database connection strings and API keys. You can manage these secrets via subcommands, and issue scoped capability tokens for third-party agents. #### `ticloud fs secret set` -创建或更新密钥。字段值可以直接赋值、从文件读取（`@file`）或从标准输入读取（`-`）。 +Create or update a secret. Field values can be set directly, read from a file (`@file`), or read from stdin (`-`). ```bash -# 直接设置字段 +# Set fields directly ticloud fs secret set myapp DATABASE_URL=mysql://... -# 从文件读取字段值，并从标准输入读取密码 +# Read a field from file and password from stdin ticloud fs secret set myapp key=@secret.txt password=- ``` #### `ticloud fs secret get` -读取密钥或指定字段。支持 `--json` 和 `--env` 输出格式。 +Read a secret or a specific field. Supports `--json` and `--env` output formats. ```bash -# 读取整个密钥 +# Read the entire secret ticloud fs secret get myapp -# 读取指定字段，并以 JSON 格式输出 +# Read a specific field and output as JSON ticloud fs secret get myapp/password --json ``` #### `ticloud fs secret exec` -将密钥字段作为环境变量注入后执行指定命令，常用于在本地脚本或 CI 流程中安全地使用密钥。 +Execute a command with secret fields injected as environment variables. Commonly used in local scripts or CI pipelines to securely use secrets. ```bash ticloud fs secret exec myapp -- ./run.sh @@ -234,7 +234,7 @@ ticloud fs secret exec myapp -- ./run.sh #### `ticloud fs secret ls` -列出所有密钥。支持 `--json` 输出。 +List all secrets. Supports `--json` output. ```bash ticloud fs secret ls @@ -242,7 +242,7 @@ ticloud fs secret ls #### `ticloud fs secret rm` -删除指定密钥。 +Delete a specific secret. ```bash ticloud fs secret rm myapp @@ -250,7 +250,7 @@ ticloud fs secret rm myapp #### `ticloud fs secret grant` -为指定 Agent 颁发受限能力令牌，限制其只能访问特定的密钥范围。需要指定 `--agent` 和 `--ttl`。 +Issue a scoped capability token for a specific agent, restricting access to certain secrets. Requires `--agent` and `--ttl`. ```bash ticloud fs secret grant --agent myagent --ttl 1h myapp/password @@ -258,7 +258,7 @@ ticloud fs secret grant --agent myagent --ttl 1h myapp/password #### `ticloud fs secret revoke` -撤销一个已颁发的能力令牌，使其立即失效。 +Revoke an issued capability token, invalidating it immediately. ```bash ticloud fs secret revoke tok_abc123 @@ -266,35 +266,35 @@ ticloud fs secret revoke tok_abc123 #### `ticloud fs secret audit` -查询密钥审计日志，追踪谁在什么时间访问了哪些密钥。支持按 `--secret`、`--agent`、`--since` 过滤，以及 `--limit` 限制条数。 +Query the secret audit log to track who accessed which secrets and when. Supports filtering by `--secret`, `--agent`, `--since`, and limiting results with `--limit`. ```bash -# 查看最近的 50 条审计记录 +# View the latest 50 audit events ticloud fs secret audit --limit 50 -# 查看某个密钥最近 24 小时内的访问记录 +# View audit events for a secret in the last 24 hours ticloud fs secret audit --secret myapp --since 24h ``` -### 交互式 Shell +### Interactive Shell -如果你需要频繁执行多条 FS 命令，可以使用交互式 Shell，避免每次都要输入完整命令前缀。 +If you need to execute multiple FS commands frequently, you can use the interactive shell to avoid typing the full command prefix each time. #### `ticloud fs shell` -启动交互式 FS Shell。支持 `cd`、`pwd`、`ls`、`cat`、`cp`、`mkdir`、`mv`、`rm`、`sql`、`stat`、`help`、`exit` 等命令。提示符为 `ticloud:fs>`。 +Launch the interactive FS Shell. Supports commands such as `cd`, `pwd`, `ls`, `cat`, `cp`, `mkdir`, `mv`, `rm`, `sql`, `stat`, `help`, `exit`. The prompt is `ticloud:fs>`. ```bash ticloud fs shell ``` -### FUSE 挂载 +### FUSE Mount -通过 FUSE 挂载，你可以将远程 FS 映射为本地目录，直接使用系统自带的文件管理器或命令行工具访问。 +Via FUSE mount, you can map the remote FS to a local directory and access it using the system file manager or standard command-line tools. #### `ticloud fs mount` -将远程 FS 挂载到本地目录。当前版本默认只读。支持 `--debug`（开启调试日志）和 `--allow-other`（允许其他用户访问）。 +Mount the remote FS to a local directory. The current version defaults to read-only. Supports `--debug` (enable debug logging) and `--allow-other` (allow other users to access). ```bash ticloud fs mount /mnt/tidbcloud @@ -302,7 +302,7 @@ ticloud fs mount /mnt/tidbcloud #### `ticloud fs umount` -卸载本地挂载点，安全断开与远程 FS 的连接。 +Unmount the local mount point, safely disconnecting from the remote FS. ```bash ticloud fs umount /mnt/tidbcloud @@ -310,29 +310,29 @@ ticloud fs umount /mnt/tidbcloud --- -## 配置项速查 +## Configuration Quick Reference -| 配置项 | 说明 | 默认值 | -|--------|------|--------| -| `fs.cluster-id` | 关联的 Serverless 集群 ID | — | -| `fs.zero-instance-id` | 关联的 Zero 实例 ID | — | -| `fs-endpoint` | FS 服务端点地址 | `https://fs.tidbapi.com/` | -| `public-key` / `private-key` | Serverless 场景下的 API Key 鉴权 | — | +| Property | Description | Default | +|----------|-------------|---------| +| `fs.cluster-id` | Associated Serverless cluster ID | — | +| `fs.zero-instance-id` | Associated Zero instance ID | — | +| `fs-endpoint` | FS server endpoint | `https://fs.tidbapi.com/` | +| `public-key` / `private-key` | API Key authentication for Serverless | — | -也支持通过环境变量覆盖： +You can also override these via environment variables: -- `TICLOUD_FS_ENDPOINT` — 覆盖 `fs-endpoint` -- `TICLOUD_FS_CLUSTER_ID` — 覆盖 `fs.cluster-id` -- `TICLOUD_FS_ZERO_INSTANCE_ID` — 覆盖 `fs.zero-instance-id` +- `TICLOUD_FS_ENDPOINT` — overrides `fs-endpoint` +- `TICLOUD_FS_CLUSTER_ID` — overrides `fs.cluster-id` +- `TICLOUD_FS_ZERO_INSTANCE_ID` — overrides `fs.zero-instance-id` --- -## 常见问题 +## FAQ -**Q: 执行 `ticloud fs ls :/` 时提示 `tenant not found`？** +**Q: Running `ticloud fs ls :/` returns `tenant not found`?** -A: 说明当前关联的数据库尚未初始化 FS。请运行 `ticloud fs init --user --password ` 完成初始化。 +A: The associated database has not been initialized for FS yet. Please run `ticloud fs init --user --password ` to complete initialization. -**Q: Serverless 集群报 401 Unauthorized？** +**Q: Serverless cluster returns 401 Unauthorized?** -A: 请确认已完成 `ticloud auth login` 或正确配置了 `public-key` / `private-key`。 +A: Please confirm you have completed `ticloud auth login` or correctly configured `public-key` / `private-key`. From 517b6685a1b60acf50679d183f10298a25f28f77 Mon Sep 17 00:00:00 2001 From: mornyx Date: Mon, 20 Apr 2026 17:25:19 +0800 Subject: [PATCH 3/6] chore(fs): fix go fmt issues --- internal/cli/fs/fs.go | 1 - internal/cli/fs/secret.go | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/internal/cli/fs/fs.go b/internal/cli/fs/fs.go index ddbe9a1c..450f670e 100644 --- a/internal/cli/fs/fs.go +++ b/internal/cli/fs/fs.go @@ -217,4 +217,3 @@ func (rp RemotePath) String() string { func IsRemote(s string) bool { return strings.HasPrefix(s, ":") } - diff --git a/internal/cli/fs/secret.go b/internal/cli/fs/secret.go index bee6df97..0da7d48e 100644 --- a/internal/cli/fs/secret.go +++ b/internal/cli/fs/secret.go @@ -156,8 +156,8 @@ func secretGetCmd(h *internal.Helper) *cobra.Command { func secretExecCmd(h *internal.Helper) *cobra.Command { return &cobra.Command{ - Use: "exec -- ", - Short: "Run a command with secret fields injected as env vars", + Use: "exec -- ", + Short: "Run a command with secret fields injected as env vars", Example: ` ticloud fs secret exec myapp -- ./run.sh`, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -271,8 +271,8 @@ func secretGrantCmd(h *internal.Helper) *cobra.Command { tokenOnly bool ) var cmd = &cobra.Command{ - Use: "grant --agent --ttl [--task ] ", - Short: "Issue a scoped capability token", + Use: "grant --agent --ttl [--task ] ", + Short: "Issue a scoped capability token", Example: ` ticloud fs secret grant --agent myagent --ttl 1h myapp/password`, Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { From 537ddd9c1b8fa321827bd148454bd32a456dea19 Mon Sep 17 00:00:00 2001 From: mornyx Date: Mon, 20 Apr 2026 17:36:44 +0800 Subject: [PATCH 4/6] chore(fs): fix lint and Windows build issues - Fix ineffectual assignment to dotdotIno in fuse/fs.go - Pre-allocate items slices in fuse/fs.go - Add //nolint:gosec for exec.Command in secret.go - Add //go:build !windows tags to fuse package files - Split mount.go into mount_unix.go and mount_windows.go stubs --- internal/cli/fs/{mount.go => mount_unix.go} | 2 + internal/cli/fs/mount_windows.go | 48 +++++++++++++++++++++ internal/cli/fs/secret.go | 1 + internal/service/fs/fuse/fs.go | 10 ++++- internal/service/fs/fuse/mount.go | 2 + 5 files changed, 61 insertions(+), 2 deletions(-) rename internal/cli/fs/{mount.go => mount_unix.go} (99%) create mode 100644 internal/cli/fs/mount_windows.go diff --git a/internal/cli/fs/mount.go b/internal/cli/fs/mount_unix.go similarity index 99% rename from internal/cli/fs/mount.go rename to internal/cli/fs/mount_unix.go index cf189466..9af75ce1 100644 --- a/internal/cli/fs/mount.go +++ b/internal/cli/fs/mount_unix.go @@ -1,3 +1,5 @@ +//go:build !windows + // Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/internal/cli/fs/mount_windows.go b/internal/cli/fs/mount_windows.go new file mode 100644 index 00000000..01e3d87b --- /dev/null +++ b/internal/cli/fs/mount_windows.go @@ -0,0 +1,48 @@ +//go:build windows + +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mount ", + Short: "Mount remote filesystem via FUSE", + Long: `Mount the remote filesystem as a local FUSE mount.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("FUSE mount is not supported on Windows") + }, + } +} + +func umountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "umount ", + Short: "Unmount FUSE filesystem", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("FUSE umount is not supported on Windows") + }, + } +} diff --git a/internal/cli/fs/secret.go b/internal/cli/fs/secret.go index 0da7d48e..1d806df2 100644 --- a/internal/cli/fs/secret.go +++ b/internal/cli/fs/secret.go @@ -188,6 +188,7 @@ func secretExecCmd(h *internal.Helper) *cobra.Command { return suggestInitIfTenantNotFound(fmt.Errorf("get secret: %w", err)) } + //nolint:gosec // user-provided command execution is intentional c := exec.Command(cmdArgs[0], cmdArgs[1:]...) c.Stdin = os.Stdin c.Stdout = h.IOStreams.Out diff --git a/internal/service/fs/fuse/fs.go b/internal/service/fs/fuse/fs.go index e502f5cf..4ec2747e 100644 --- a/internal/service/fs/fuse/fs.go +++ b/internal/service/fs/fuse/fs.go @@ -1,3 +1,5 @@ +//go:build !windows + // Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -251,7 +253,9 @@ func (f *ReadOnlyFS) ReadDir(cancel <-chan struct{}, in *gofuse.ReadIn, out *gof mode uint32 } - var items []dirItem + items := make([]dirItem, 0, 2+len(realEntries)) + items = append(items, dirItem{".", in.NodeId, syscall.S_IFDIR | 0o555}) + items = append(items, dirItem{"..", dotdotIno, syscall.S_IFDIR | 0o555}) for _, e := range realEntries { if e.Name == "" || e.Name == ":" { continue @@ -316,7 +320,9 @@ func (f *ReadOnlyFS) ReadDirPlus(cancel <-chan struct{}, in *gofuse.ReadIn, out mode uint32 } - var items []dirItem + items := make([]dirItem, 0, 2+len(realEntries)) + items = append(items, dirItem{".", in.NodeId, syscall.S_IFDIR | 0o555}) + items = append(items, dirItem{"..", dotdotIno, syscall.S_IFDIR | 0o555}) for _, e := range realEntries { if e.Name == "" || e.Name == ":" { continue diff --git a/internal/service/fs/fuse/mount.go b/internal/service/fs/fuse/mount.go index d3d8586f..a6a933c1 100644 --- a/internal/service/fs/fuse/mount.go +++ b/internal/service/fs/fuse/mount.go @@ -1,3 +1,5 @@ +//go:build !windows + // Copyright 2026 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); From cd799999df82b205b989e75aa3c4e8634b916a5b Mon Sep 17 00:00:00 2001 From: mornyx Date: Wed, 22 Apr 2026 11:11:52 +0800 Subject: [PATCH 5/6] fix(fs): skip auth for zero-instance-id --- internal/cli/fs/fs.go | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/internal/cli/fs/fs.go b/internal/cli/fs/fs.go index 450f670e..6f0d21e1 100644 --- a/internal/cli/fs/fs.go +++ b/internal/cli/fs/fs.go @@ -134,22 +134,27 @@ func newClient(cmd *cobra.Command) (*fs.Client, error) { } // Build HTTP transport using the same auth logic as the root CLI. + // For Zero instances, skip auth entirely. var transport http.RoundTripper - publicKey, privateKey := config.GetPublicKey(), config.GetPrivateKey() - if publicKey != "" && privateKey != "" { - transport = cloud.NewDigestTransport(publicKey, privateKey) + if clusterID == "" && zeroInstanceID != "" { + transport = http.DefaultTransport } else { - if err := config.ValidateToken(); err != nil { - return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys with 'ticloud config set public-key ' and 'ticloud config set private-key '") - } - token, err := config.GetAccessToken() - if err != nil { - if errors.Is(err, keyring.ErrNotFound) || errors.Is(err, store.ErrNotSupported) { - return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys") + publicKey, privateKey := config.GetPublicKey(), config.GetPrivateKey() + if publicKey != "" && privateKey != "" { + transport = cloud.NewDigestTransport(publicKey, privateKey) + } else { + if err := config.ValidateToken(); err != nil { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys with 'ticloud config set public-key ' and 'ticloud config set private-key '") + } + token, err := config.GetAccessToken() + if err != nil { + if errors.Is(err, keyring.ErrNotFound) || errors.Is(err, store.ErrNotSupported) { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys") + } + return nil, errors.Trace(err) } - return nil, errors.Trace(err) + transport = cloud.NewBearTokenTransport(token) } - transport = cloud.NewBearTokenTransport(token) } httpClient := &http.Client{ From 7a0167c867f7b3ecccda1fc83704f80206f0fecd Mon Sep 17 00:00:00 2001 From: mornyx Date: Wed, 29 Apr 2026 11:08:28 +0800 Subject: [PATCH 6/6] docs: demo user 'admin' -> 'root' Signed-off-by: mornyx --- docs/fs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/fs.md b/docs/fs.md index 1a507a18..cfb07325 100644 --- a/docs/fs.md +++ b/docs/fs.md @@ -40,7 +40,7 @@ If you already have a **TiDB Cloud Serverless** cluster (created via the [TiDB C 3. **Initialize FS** ```bash - ticloud fs init --user admin --password + ticloud fs init --user root --password ``` > Every new Serverless cluster must run `init` before using FS for the first time to provision the FS tenant. @@ -60,7 +60,7 @@ If you are using a **TiDB Zero** instance, configure it as follows. If you are n 2. **Initialize FS** ```bash - ticloud fs init --user admin --password + ticloud fs init --user root --password ``` > Like Serverless, every new Zero instance must run `init` once to provision the FS tenant. @@ -87,7 +87,7 @@ Before performing any file operations, you must create an FS tenant for the asso Initialize the FS tenant for the currently associated database. This must be run before using FS for the first time on a new database. ```bash -ticloud fs init --user admin --password secret +ticloud fs init --user root --password secret ``` ### Basic File Operations