diff --git a/docs/fs.md b/docs/fs.md new file mode 100644 index 00000000..cfb07325 --- /dev/null +++ b/docs/fs.md @@ -0,0 +1,338 @@ +# TiDB Cloud FS (ticloud fs) User Guide + +**TiDB Cloud FS** is a managed file storage service provided by TiDB Cloud that offers a unified filesystem abstraction for TiDB instances. You can store, read, and manage files in TiDB Cloud just like a local filesystem, with advanced capabilities such as SQL queries, a secret vault, and FUSE mount support. + +**`ticloud fs`** is the command group in the TiDB Cloud CLI for operating TiDB Cloud FS, supporting upload/download, directory browsing, SQL execution, secret management, and FUSE mount. + +--- + +## Prerequisites: Associate a TiDB Instance + +FS data storage depends on a specific TiDB instance. Depending on the type of instance you have, you need to bind its ID to the CLI configuration and complete a one-time FS initialization (`init`). + +### Using Serverless TiDB + +If you already have a **TiDB Cloud Serverless** cluster (created via the [TiDB Cloud Web Console](https://tidbcloud.com/) or the `ticloud serverless create` command), configure it as follows: + +1. **Configure Authentication** + + Serverless clusters are managed by the TiDB Cloud platform, so you must provide valid platform credentials to access their FS service. Choose one of the following methods: + + - **OAuth Login (Recommended)** + + ```bash + ticloud auth login + ``` + + - **API Key Authentication** + + ```bash + ticloud config set public-key + ticloud config set private-key + ``` + +2. **Configure cluster-id** + + ```bash + ticloud config set fs.cluster-id + ``` + +3. **Initialize FS** + + ```bash + ticloud fs init --user root --password + ``` + + > Every new Serverless cluster must run `init` before using FS for the first time to provision the FS tenant. + +### Using Zero TiDB + +If you are using a **TiDB Zero** instance, configure it as follows. If you are not familiar with TiDB Zero, visit [https://zero.tidbcloud.com/](https://zero.tidbcloud.com/) to learn more. + +1. **Configure zero-instance-id** + + ```bash + ticloud config set fs.zero-instance-id + ``` + + TiDB Zero uses an independent deployment and access model. You **do not** need to run `ticloud auth login` or configure `public-key` / `private-key`. + +2. **Initialize FS** + + ```bash + ticloud fs init --user root --password + ``` + + > Like Serverless, every new Zero instance must run `init` once to provision the FS tenant. + +--- + +## Path Format + +FS uses the `:/` prefix to denote remote paths: + +- `:/` — Remote root directory +- `:/path/to/file.txt` — Remote file path + +--- + +## Subcommand Manual + +### Initialization and Configuration + +Before performing any file operations, you must create an FS tenant for the associated database. This step only needs to be performed once. + +#### `ticloud fs init` + +Initialize the FS tenant for the currently associated database. This must be run before using FS for the first time on a new database. + +```bash +ticloud fs init --user root --password secret +``` + +### Basic File Operations + +`ticloud fs` provides a set of Unix-like filesystem commands for managing remote files and directories. You can upload, download, browse, and delete files as if they were local. + +#### `ticloud fs ls` + +List remote directory contents. Supports `-l` for detailed view. + +```bash +# List root directory +ticloud fs ls :/ + +# List a folder with size details +ticloud fs ls -l :/myfolder +``` + +#### `ticloud fs cat` + +Display remote file contents, similar to the Unix `cat` command. Useful for quickly viewing text files. + +```bash +ticloud fs cat :/readme.txt +``` + +#### `ticloud fs cp` + +Copy files. Supports local↔remote, remote↔remote, stdin upload, and resuming large file transfers. + +```bash +# Upload a local file to remote +ticloud fs cp local.txt :/remote/ + +# Download a remote file to current directory +ticloud fs cp :/remote/file.txt . + +# Upload from stdin via pipe +echo "hello" | ticloud fs cp - :/file.txt + +# Resume a large file upload (useful for unstable networks) +ticloud fs cp --resume large.zip :/remote/ +``` + +#### `ticloud fs mkdir` + +Create a remote directory. Supports nested paths. + +```bash +ticloud fs mkdir :/mydir +``` + +#### `ticloud fs mv` + +Move or rename remote files/directories. + +```bash +ticloud fs mv :/oldname :/newname +``` + +#### `ticloud fs rm` + +Delete remote files or empty directories. + +```bash +ticloud fs rm :/file.txt +``` + +#### `ticloud fs stat` + +View remote file or directory metadata, such as size and modification time. + +```bash +ticloud fs stat :/file.txt +``` + +### Search and Query + +In addition to basic file operations, `ticloud fs` supports searching file contents and executing SQL queries directly on remote storage, making it easy to locate information or perform data analysis. + +#### `ticloud fs grep` + +Search for matching content in remote files. Supports `--limit` to restrict the number of results. + +```bash +ticloud fs grep "TODO" :/project --limit 20 +``` + +#### `ticloud fs find` + +Find remote files by criteria. Supports `-name`, `-size`, `-newer` filters, similar to the Unix `find` command. + +```bash +# Find by file extension +ticloud fs find :/ -name "*.go" + +# Find files larger than 1MB +ticloud fs find :/ -size +1M +``` + +#### `ticloud fs sql` + +Execute SQL queries against the FS backend for structured queries on stored data. + +```bash +ticloud fs sql -q "SELECT 1" +``` + +### Secret Management + +TiDB Cloud FS includes a built-in secret vault for securely storing sensitive information such as database connection strings and API keys. You can manage these secrets via subcommands, and issue scoped capability tokens for third-party agents. + +#### `ticloud fs secret set` + +Create or update a secret. Field values can be set directly, read from a file (`@file`), or read from stdin (`-`). + +```bash +# Set fields directly +ticloud fs secret set myapp DATABASE_URL=mysql://... + +# Read a field from file and password from stdin +ticloud fs secret set myapp key=@secret.txt password=- +``` + +#### `ticloud fs secret get` + +Read a secret or a specific field. Supports `--json` and `--env` output formats. + +```bash +# Read the entire secret +ticloud fs secret get myapp + +# Read a specific field and output as JSON +ticloud fs secret get myapp/password --json +``` + +#### `ticloud fs secret exec` + +Execute a command with secret fields injected as environment variables. Commonly used in local scripts or CI pipelines to securely use secrets. + +```bash +ticloud fs secret exec myapp -- ./run.sh +``` + +#### `ticloud fs secret ls` + +List all secrets. Supports `--json` output. + +```bash +ticloud fs secret ls +``` + +#### `ticloud fs secret rm` + +Delete a specific secret. + +```bash +ticloud fs secret rm myapp +``` + +#### `ticloud fs secret grant` + +Issue a scoped capability token for a specific agent, restricting access to certain secrets. Requires `--agent` and `--ttl`. + +```bash +ticloud fs secret grant --agent myagent --ttl 1h myapp/password +``` + +#### `ticloud fs secret revoke` + +Revoke an issued capability token, invalidating it immediately. + +```bash +ticloud fs secret revoke tok_abc123 +``` + +#### `ticloud fs secret audit` + +Query the secret audit log to track who accessed which secrets and when. Supports filtering by `--secret`, `--agent`, `--since`, and limiting results with `--limit`. + +```bash +# View the latest 50 audit events +ticloud fs secret audit --limit 50 + +# View audit events for a secret in the last 24 hours +ticloud fs secret audit --secret myapp --since 24h +``` + +### Interactive Shell + +If you need to execute multiple FS commands frequently, you can use the interactive shell to avoid typing the full command prefix each time. + +#### `ticloud fs shell` + +Launch the interactive FS Shell. Supports commands such as `cd`, `pwd`, `ls`, `cat`, `cp`, `mkdir`, `mv`, `rm`, `sql`, `stat`, `help`, `exit`. The prompt is `ticloud:fs>`. + +```bash +ticloud fs shell +``` + +### FUSE Mount + +Via FUSE mount, you can map the remote FS to a local directory and access it using the system file manager or standard command-line tools. + +#### `ticloud fs mount` + +Mount the remote FS to a local directory. The current version defaults to read-only. Supports `--debug` (enable debug logging) and `--allow-other` (allow other users to access). + +```bash +ticloud fs mount /mnt/tidbcloud +``` + +#### `ticloud fs umount` + +Unmount the local mount point, safely disconnecting from the remote FS. + +```bash +ticloud fs umount /mnt/tidbcloud +``` + +--- + +## Configuration Quick Reference + +| Property | Description | Default | +|----------|-------------|---------| +| `fs.cluster-id` | Associated Serverless cluster ID | — | +| `fs.zero-instance-id` | Associated Zero instance ID | — | +| `fs-endpoint` | FS server endpoint | `https://fs.tidbapi.com/` | +| `public-key` / `private-key` | API Key authentication for Serverless | — | + +You can also override these via environment variables: + +- `TICLOUD_FS_ENDPOINT` — overrides `fs-endpoint` +- `TICLOUD_FS_CLUSTER_ID` — overrides `fs.cluster-id` +- `TICLOUD_FS_ZERO_INSTANCE_ID` — overrides `fs.zero-instance-id` + +--- + +## FAQ + +**Q: Running `ticloud fs ls :/` returns `tenant not found`?** + +A: The associated database has not been initialized for FS yet. Please run `ticloud fs init --user --password ` to complete initialization. + +**Q: Serverless cluster returns 401 Unauthorized?** + +A: Please confirm you have completed `ticloud auth login` or correctly configured `public-key` / `private-key`. diff --git a/go.mod b/go.mod index 48e010f3..d17a5398 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/go-sql-driver/mysql v1.8.1 github.com/google/go-github/v49 v49.0.0 + github.com/hanwen/go-fuse/v2 v2.9.0 github.com/hashicorp/go-version v1.6.0 github.com/icholy/digest v0.1.22 github.com/juju/errors v1.0.0 @@ -124,3 +125,5 @@ require ( ) replace github.com/tidbcloud/tidbcloud-cli/pkg => ./pkg + +replace github.com/golang-module/carbon/v2 => github.com/dromara/carbon/v2 v2.3.12 diff --git a/go.sum b/go.sum index 0b7e1191..11d67a0e 100644 --- a/go.sum +++ b/go.sum @@ -269,6 +269,8 @@ github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dromara/carbon/v2 v2.3.12 h1:Fj9164o0w5jfcqDjvEOZSYdoajsBxpslm7PvhATcokc= +github.com/dromara/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= @@ -341,8 +343,6 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/golang-module/carbon/v2 v2.3.12 h1:VC1DwN1kBwJkh5MjXmTFryjs5g4CWyoM8HAHffZPX/k= -github.com/golang-module/carbon/v2 v2.3.12/go.mod h1:HNsedGzXGuNciZImYP2OMnpiwq/vhIstR/vn45ib5cI= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= @@ -386,6 +386,8 @@ github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8 github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/hanwen/go-fuse/v2 v2.9.0 h1:0AOGUkHtbOVeyGLr0tXupiid1Vg7QB7M6YUcdmVdC58= +github.com/hanwen/go-fuse/v2 v2.9.0/go.mod h1:yE6D2PqWwm3CbYRxFXV9xUd8Md5d6NG0WBs5spCswmI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= @@ -518,6 +520,8 @@ github.com/mithrandie/ternary v1.1.1 h1:k/joD6UGVYxHixYmSR8EGgDFNONBMqyD373xT4QR github.com/mithrandie/ternary v1.1.1/go.mod h1:0D9Ba3+09K2TdSZO7/bFCC0GjSXetCvYuYq0u8FY/1g= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= diff --git a/internal/cli/fs/README.md b/internal/cli/fs/README.md new file mode 100644 index 00000000..c10edea8 --- /dev/null +++ b/internal/cli/fs/README.md @@ -0,0 +1,367 @@ +# TiDB Cloud FS Commands + +The `ticloud fs` command group provides filesystem-like operations for managing files and directories in TiDB Cloud FS. + +## Overview + +TiDB Cloud FS is a managed file storage service that allows you to store and retrieve files using familiar filesystem semantics. The CLI provides both direct API operations and FUSE mount capabilities. + +## Prerequisites + +- TiDB Cloud CLI (`ticloud`) installed +- TiDB Cloud authentication configured (OAuth login or API key pair) + +## Configuration + +The `fs` command reuses your existing TiDB Cloud CLI authentication: + +```bash +# OAuth login +ticloud auth login + +# Or configure API keys +ticloud config set public-key +ticloud config set private-key +``` + +### Database Association + +FS operations can be associated with a specific database: + +```bash +# Associate with a TiDB Cloud serverless cluster +ticloud config set fs.cluster-id + +# Associate with a TiDB Zero instance +ticloud config set fs.zero-instance-id +``` + +These are sent as `X-TIDBCLOUD-CLUSTER-ID` and `X-TIDBCLOUD-ZERO-INSTANCE-ID` headers respectively. If both are set, `cluster-id` takes precedence. + +### Server URL + +The default FS server URL is `https://fs.tidbapi.com/`. You can override it: + +```bash +ticloud config set fs-endpoint +# or +export TICLOUD_FS_SERVER= +``` + +## Path Format + +Remote paths use the `:/` prefix: + +- `:/` - Root directory +- `:/path/to/file.txt` - File path + +## Commands + +### Basic Operations + +#### List Directory + +```bash +# List root directory +ticloud fs ls :/ + +# List with details +ticloud fs ls -l :/myfolder + +# List subdirectory +ticloud fs ls :/docs +``` + +#### Display File Contents + +```bash +ticloud fs cat :/readme.txt +ticloud fs cat :/docs/guide.md +``` + +#### Copy Files + +Upload local to remote: +```bash +ticloud fs cp local.txt :/remote/ +ticloud fs cp ./data.json :/backup/ +``` + +Download remote to local: +```bash +ticloud fs cp :/readme.txt ./ +ticloud fs cp :/docs/guide.md ./local-docs/ +``` + +Server-side copy: +```bash +ticloud fs cp :/old/path :/new/path +``` + +From stdin: +```bash +echo "content" | ticloud fs cp - :/file.txt +``` + +Resume interrupted upload: +```bash +ticloud fs cp --resume large.zip :/remote/ +``` + +#### Create Directory + +```bash +ticloud fs mkdir :/mydir +ticloud fs mkdir :/parent/child +``` + +#### Move/Rename + +```bash +ticloud fs mv :/oldname :/newname +ticloud fs mv :/folder/file.txt :/other/file.txt +``` + +#### Remove + +```bash +ticloud fs rm :/file.txt +ticloud fs rm :/empty_folder/ +``` + +#### File Metadata + +```bash +ticloud fs stat :/file.txt +``` + +### SQL Execution + +Execute SQL queries against the FS backend: + +```bash +ticloud fs sql -q "SELECT 1" +ticloud fs sql -f query.sql +``` + +### Secret Management + +Manage secrets in the TiDB Cloud FS vault: + +```bash +# Set a secret +ticloud fs secret set myapp DATABASE_URL=mysql://... + +# Get a secret +ticloud fs secret get myapp +ticloud fs secret get myapp/password + +# List secrets +ticloud fs secret ls + +# Delete a secret +ticloud fs secret rm myapp + +# Run a command with secret env vars +ticloud fs secret exec myapp -- ./run.sh + +# Issue a scoped capability token +ticloud fs secret grant --agent myagent --ttl 1h myapp/password + +# Revoke a token +ticloud fs secret revoke tok_abc123 + +# Query audit events +ticloud fs secret audit --limit 50 +``` + +### Search Operations + +#### Search File Contents (Grep) + +```bash +# Search for pattern +ticloud fs grep "function main" :/ + +# Search with limit +ticloud fs grep "TODO" :/myproject --limit 20 +``` + +#### Find Files + +```bash +# Find by name pattern +ticloud fs find :/ -name "*.go" + +# Find by size (larger than 1MB) +ticloud fs find :/ -size +1M + +# Find modified after date +ticloud fs find :/ -newer 2024-01-01 + +# Combined filters +ticloud fs find :/data -name "*.json" -newer 2024-01-01 +``` + +### Interactive Shell + +Start an interactive shell for filesystem operations: + +```bash +ticloud fs shell +``` + +Shell commands: +- `cd ` - Change directory +- `pwd` - Print current directory +- `ls [path]` - List directory +- `cat ` - Display file +- `cp ` - Copy files +- `mkdir ` - Create directory +- `mv ` - Move/rename +- `rm ` - Remove +- `sql ` - Execute SQL query +- `stat ` - Show metadata +- `help` - Show help +- `exit` - Exit shell + +Prompt: `ticloud:fs>` + +### FUSE Mount + +Mount the remote filesystem as a local directory using FUSE. + +#### Prerequisites + +**Linux:** +- Install FUSE: `sudo apt-get install fuse3` (Debian/Ubuntu) or `sudo yum install fuse3` (RHEL/CentOS) + +**macOS:** +- Install macFUSE from https://osxfuse.github.io/ + +#### Mount + +```bash +# Create mount point +mkdir -p /mnt/tidbcloud + +# Mount (read-only, default for MVP) +ticloud fs mount /mnt/tidbcloud + +# Mount with debug logging +ticloud fs mount /mnt/tidbcloud --debug + +# Mount allowing other users +ticloud fs mount /mnt/tidbcloud --allow-other +``` + +#### Use Mounted Filesystem + +Once mounted, use standard filesystem commands: + +```bash +# List files +ls /mnt/tidbcloud + +# Read files +cat /mnt/tidbcloud/readme.txt + +# Copy from mount +cp /mnt/tidbcloud/file.txt ./ +``` + +#### Unmount + +```bash +ticloud fs umount /mnt/tidbcloud +``` + +Or use system commands: +```bash +# Linux +fusermount3 -u /mnt/tidbcloud + +# macOS +umount /mnt/tidbcloud +``` + +## Examples + +### Backup Workflow + +```bash +# Create backup directory +ticloud fs ls :/backups 2>/dev/null || ticloud fs mkdir :/backups + +# Upload backup +tar czf - /data | ticloud fs cp - :/backups/data-$(date +%Y%m%d).tar.gz + +# List backups +ticloud fs ls :/backups +``` + +### Development Workflow + +```bash +# Start interactive shell +ticloud fs shell + +# In shell: +ticloud:fs> cd /myproject +ticloud:fs> ls +ticloud:fs> cat config.json +ticloud:fs> exit +``` + +### Search and Filter + +```bash +# Find all Go files modified recently +ticloud fs find :/ -name "*.go" -newer $(date -v-7d +%Y-%m-%d) + +# Search for function definitions +ticloud fs grep "^func " :/src --limit 50 +``` + +## Limitations + +### MVP (Current Version) + +- **FUSE mount is read-only** - Write operations via FUSE will be added in a future release +- **No symlink support** - Symlinks are not supported +- **Limited concurrent uploads** - Large file uploads may be slower + +### Planned Features + +- Full read-write FUSE mount +- Streaming large file operations +- Progress bars for transfers +- Batch operations +- File versioning + +## Troubleshooting + +### FUSE Mount Issues + +**Error: "fuse: device not found"** +- Ensure FUSE is installed: `modprobe fuse` (Linux) or install macFUSE (macOS) + +**Error: "permission denied"** +- Check mount point permissions +- Try with `--allow-other` flag +- On Linux, add user to `fuse` group: `sudo usermod -a -G fuse $USER` + +**Error: "transport endpoint is not connected"** +- The mount is stale, try unmounting and remounting + +### Connection Issues + +**Error: "cannot reach FS server"** +- Check network connectivity +- Verify `fs-endpoint` config: `ticloud config describe` +- Check TiDB Cloud auth status: `ticloud auth status` + +## See Also + +- `ticloud config` - Manage CLI configuration +- `ticloud auth` - Authentication commands diff --git a/internal/cli/fs/cat.go b/internal/cli/fs/cat.go new file mode 100644 index 00000000..901c1dd5 --- /dev/null +++ b/internal/cli/fs/cat.go @@ -0,0 +1,54 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + "io" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func catCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "cat ", + Short: "Display file contents", + Example: ` ticloud fs cat :/file.txt + ticloud fs cat :/myfolder/data.json`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + rp := ParseRemotePath(args[0]) + path := rp.Path + + ctx := context.Background() + reader, err := client.ReadStream(ctx, path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("read %s: %w", path, err)) + } + defer reader.Close() + + _, err = io.Copy(h.IOStreams.Out, reader) + return suggestInitIfTenantNotFound(err) + }, + } +} diff --git a/internal/cli/fs/cp.go b/internal/cli/fs/cp.go new file mode 100644 index 00000000..4758dd4d --- /dev/null +++ b/internal/cli/fs/cp.go @@ -0,0 +1,128 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func cpCmd(h *internal.Helper) *cobra.Command { + var resume bool + var cmd = &cobra.Command{ + Use: "cp ", + Short: "Copy files between local and remote", + Long: `Copy files between local and remote filesystems. + +Source and destination can be: + - Local paths (any path not starting with :) + - Remote paths (starting with :/) + - - for stdin (source) or stdout (destination) + +Examples: + ticloud fs cp local.txt :/remote/ # Upload + ticloud fs cp :/remote/file.txt . # Download + ticloud fs cp :/a/b :/c/d # Server-side copy + cat data.txt | ticloud fs cp - :/remote/ # Upload from stdin`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return suggestInitIfTenantNotFound(err) + } + + src, dst := args[0], args[1] + srcRemote := IsRemote(src) + dstRemote := IsRemote(dst) + + ctx := context.Background() + + switch { + case !srcRemote && !dstRemote: + return fmt.Errorf("local-to-local copy not supported") + + case !srcRemote && dstRemote: + // Upload + dstPath := ParseRemotePath(dst).Path + if src == "-" { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("read stdin: %w", err) + } + return client.Write(dstPath, data) + } + if strings.HasSuffix(dstPath, "/") { + dstPath = filepath.Join(dstPath, filepath.Base(src)) + } + f, err := os.Open(src) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("open %s: %w", src, err)) + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("stat %s: %w", src, err)) + } + if resume { + return suggestInitIfTenantNotFound(client.ResumeUpload(ctx, dstPath, f, stat.Size(), nil)) + } + return suggestInitIfTenantNotFound(client.WriteStream(ctx, dstPath, f, stat.Size(), nil)) + + case srcRemote && !dstRemote: + // Download + srcPath := ParseRemotePath(src).Path + reader, err := client.ReadStream(ctx, srcPath) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("read %s: %w", src, err)) + } + defer reader.Close() + + if dst == "-" { + _, err = io.Copy(h.IOStreams.Out, reader) + return suggestInitIfTenantNotFound(err) + } + f, err := os.Create(dst) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("create %s: %w", dst, err)) + } + defer f.Close() + _, err = io.Copy(f, reader) + return suggestInitIfTenantNotFound(err) + + case srcRemote && dstRemote: + // Server-side copy + srcPath := ParseRemotePath(src).Path + dstPath := ParseRemotePath(dst).Path + if strings.HasSuffix(dstPath, "/") { + dstPath = filepath.Join(dstPath, filepath.Base(srcPath)) + } + return suggestInitIfTenantNotFound(client.Copy(srcPath, dstPath)) + } + + return nil + }, + } + + cmd.Flags().BoolVar(&resume, "resume", false, "Resume interrupted upload") + return cmd +} diff --git a/internal/cli/fs/find.go b/internal/cli/fs/find.go new file mode 100644 index 00000000..d01902e9 --- /dev/null +++ b/internal/cli/fs/find.go @@ -0,0 +1,135 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "net/url" + "strconv" + "strings" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func findCmd(h *internal.Helper) *cobra.Command { + var ( + nameGlob string + tagFilter string + newer string + older string + sizeFilter string + ) + + var cmd = &cobra.Command{ + Use: "find [path]", + Short: "Find files by attributes", + Long: `Find files matching specified criteria. + +Filters: + --name Match filename pattern + --tag Match tag + --newer Modified after date (YYYY-MM-DD) + --older Modified before date (YYYY-MM-DD) + --size <+N|-N> Size filter (+N = larger than, -N = smaller than) + +Examples: + ticloud fs find :/ --name "*.go" + ticloud fs find :/ --newer 2024-01-01 --size +1M + ticloud fs find :/myfolder --tag env=production`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := "/" + if len(args) > 0 { + path = ParseRemotePath(args[0]).Path + } + + params := url.Values{} + + if nameGlob != "" { + params.Set("name", nameGlob) + } + + if tagFilter != "" { + parts := strings.SplitN(tagFilter, "=", 2) + if len(parts) == 2 { + params.Set("tag_key", parts[0]) + params.Set("tag_value", parts[1]) + } + } + + if newer != "" { + t, err := time.Parse("2006-01-02", newer) + if err != nil { + return fmt.Errorf("invalid -newer date: %w", err) + } + params.Set("modified_after", strconv.FormatInt(t.Unix(), 10)) + } + + if older != "" { + t, err := time.Parse("2006-01-02", older) + if err != nil { + return fmt.Errorf("invalid -older date: %w", err) + } + params.Set("modified_before", strconv.FormatInt(t.Unix(), 10)) + } + + if sizeFilter != "" { + if strings.HasPrefix(sizeFilter, "+") { + sizeStr := sizeFilter[1:] + size, err := humanize.ParseBytes(sizeStr) + if err != nil { + return fmt.Errorf("invalid -size value: %w", err) + } + params.Set("min_size", strconv.FormatUint(size, 10)) + } else if strings.HasPrefix(sizeFilter, "-") { + sizeStr := sizeFilter[1:] + size, err := humanize.ParseBytes(sizeStr) + if err != nil { + return fmt.Errorf("invalid -size value: %w", err) + } + params.Set("max_size", strconv.FormatUint(size, 10)) + } + } + + results, err := client.Find(path, params) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("find: %w", err)) + } + + for _, r := range results { + size := humanize.IBytes(uint64(r.SizeBytes)) + fmt.Printf("%s\t%s\n", size, r.Path) + } + return nil + }, + } + + cmd.Flags().StringVar(&nameGlob, "name", "", "Match filename pattern") + cmd.Flags().StringVar(&tagFilter, "tag", "", "Match tag (key=value)") + cmd.Flags().StringVar(&newer, "newer", "", "Modified after date (YYYY-MM-DD)") + cmd.Flags().StringVar(&older, "older", "", "Modified before date (YYYY-MM-DD)") + cmd.Flags().StringVar(&sizeFilter, "size", "", "Size filter (+N or -N, e.g., +1M, -100K)") + + return cmd +} diff --git a/internal/cli/fs/fs.go b/internal/cli/fs/fs.go new file mode 100644 index 00000000..6f0d21e1 --- /dev/null +++ b/internal/cli/fs/fs.go @@ -0,0 +1,224 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "net/http" + "os" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + "github.com/tidbcloud/tidbcloud-cli/internal/config" + "github.com/tidbcloud/tidbcloud-cli/internal/config/store" + "github.com/tidbcloud/tidbcloud-cli/internal/flag" + "github.com/tidbcloud/tidbcloud-cli/internal/service/cloud" + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + "github.com/juju/errors" + "github.com/spf13/cobra" + "github.com/zalando/go-keyring" +) + +// FSCmd returns the fs command. +func FSCmd(h *internal.Helper) *cobra.Command { + var fsCmd = &cobra.Command{ + Use: "fs", + Short: "Filesystem operations for TiDB Cloud FS", + Long: `Filesystem operations for TiDB Cloud FS. + +This command provides filesystem-like operations for managing files +and directories in TiDB Cloud FS. + +Path format: :/path/to/file + - :/ denotes the root of the remote filesystem + +Authentication: + The fs command reuses your TiDB Cloud CLI authentication (OAuth or API key). + Please login with 'ticloud auth login' or configure API keys with: + ticloud config set public-key + ticloud config set private-key + +Database Association: + FS operations can be associated with a specific database: + - Serverless cluster: ticloud config set fs.cluster-id + (sent as X-TIDBCLOUD-CLUSTER-ID header) + - TiDB Zero instance: ticloud config set fs.zero-instance-id + (sent as X-TIDBCLOUD-ZERO-INSTANCE-ID header) + If both are set, cluster-id takes precedence. + +Configuration: + - Server URL: ticloud config set fs-endpoint (default: https://fs.tidbapi.com/) + - Env override: TICLOUD_FS_ENDPOINT, TICLOUD_FS_CLUSTER_ID, TICLOUD_FS_ZERO_INSTANCE_ID + +Examples: + ticloud fs ls :/ # List root directory + ticloud fs ls -l :/myfolder # List with details + ticloud fs cat :/file.txt # Display file contents + ticloud fs cp local.txt :/remote/ # Upload file + ticloud fs cp :/remote/file.txt . # Download file + ticloud fs mv :/old :/new # Rename/move + ticloud fs rm :/file.txt # Remove file + ticloud fs mkdir :/mydir # Create directory + ticloud fs stat :/file.txt # Show metadata + ticloud fs sql -q "SELECT 1" # Execute SQL + ticloud fs secret ls # List secrets + ticloud fs grep "pattern" :/ # Search content + ticloud fs find :/ -name "*.txt" # Find files + ticloud fs shell # Interactive shell +`, + } + + fsCmd.AddCommand(initCmd(h)) + fsCmd.AddCommand(lsCmd(h)) + fsCmd.AddCommand(catCmd(h)) + fsCmd.AddCommand(cpCmd(h)) + fsCmd.AddCommand(mkdirCmd(h)) + fsCmd.AddCommand(statCmd(h)) + fsCmd.AddCommand(mvCmd(h)) + fsCmd.AddCommand(rmCmd(h)) + fsCmd.AddCommand(sqlCmd(h)) + fsCmd.AddCommand(secretCmd(h)) + fsCmd.AddCommand(shellCmd(h)) + fsCmd.AddCommand(grepCmd(h)) + fsCmd.AddCommand(findCmd(h)) + fsCmd.AddCommand(mountCmd(h)) + fsCmd.AddCommand(umountCmd(h)) + + fsCmd.PersistentFlags().String(flag.FSEndpoint, "", "FS server URL (overrides config)") + fsCmd.PersistentFlags().String(flag.FSClusterID, "", "Associated TiDB Cloud cluster ID") + fsCmd.PersistentFlags().String(flag.FSZeroInstanceID, "", "Associated TiDB Zero instance ID") + + return fsCmd +} + +// newClient creates a new FS client using the active TiDB Cloud CLI authentication. +// It reuses OAuth tokens or API key pairs from the current profile. +func newClient(cmd *cobra.Command) (*fs.Client, error) { + // Resolve server URL: flag > env > config > default + server, _ := cmd.Flags().GetString(flag.FSEndpoint) + if server == "" { + server = os.Getenv("TICLOUD_FS_ENDPOINT") + } + if server == "" { + server = config.GetFSEndpoint() + } + + // Resolve database association: flag > env > config + clusterID, _ := cmd.Flags().GetString(flag.FSClusterID) + if clusterID == "" { + clusterID = os.Getenv("TICLOUD_FS_CLUSTER_ID") + } + if clusterID == "" { + clusterID = config.GetFSClusterID() + } + + zeroInstanceID, _ := cmd.Flags().GetString(flag.FSZeroInstanceID) + if zeroInstanceID == "" { + zeroInstanceID = os.Getenv("TICLOUD_FS_ZERO_INSTANCE_ID") + } + if zeroInstanceID == "" { + zeroInstanceID = config.GetFSZeroInstanceID() + } + + // Build HTTP transport using the same auth logic as the root CLI. + // For Zero instances, skip auth entirely. + var transport http.RoundTripper + if clusterID == "" && zeroInstanceID != "" { + transport = http.DefaultTransport + } else { + publicKey, privateKey := config.GetPublicKey(), config.GetPrivateKey() + if publicKey != "" && privateKey != "" { + transport = cloud.NewDigestTransport(publicKey, privateKey) + } else { + if err := config.ValidateToken(); err != nil { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys with 'ticloud config set public-key ' and 'ticloud config set private-key '") + } + token, err := config.GetAccessToken() + if err != nil { + if errors.Is(err, keyring.ErrNotFound) || errors.Is(err, store.ErrNotSupported) { + return nil, errors.New("no valid auth found: please login with 'ticloud auth login' or configure API keys") + } + return nil, errors.Trace(err) + } + transport = cloud.NewBearTokenTransport(token) + } + } + + httpClient := &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && req.URL.Host != via[0].URL.Host { + req.Header.Del("Authorization") + } + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + return nil + }, + } + + return fs.NewClient(server, httpClient, clusterID, zeroInstanceID), nil +} + +// suggestInitIfTenantNotFound wraps errors that indicate the tenant has not been +// provisioned yet with a helpful hint to run 'ticloud fs init'. +func suggestInitIfTenantNotFound(err error) error { + if err == nil { + return nil + } + msg := err.Error() + if strings.Contains(strings.ToLower(msg), "tenant not found") { + return fmt.Errorf("%w\n\nFS tenant not found for this database. Please run 'ticloud fs init' first.", err) + } + return err +} + +// RemotePath represents a parsed remote path. +type RemotePath struct { + Context string + Path string +} + +// ParseRemotePath parses a remote path string. +// Format: [context:]/path or :/path (uses default context) +func ParseRemotePath(s string) RemotePath { + if !strings.HasPrefix(s, ":") { + return RemotePath{Path: s} + } + s = s[1:] // remove leading : + if !strings.HasPrefix(s, "/") { + // Check for context prefix + if idx := strings.Index(s, ":/"); idx != -1 { + return RemotePath{ + Context: s[:idx], + Path: s[idx+1:], + } + } + } + return RemotePath{Path: s} +} + +func (rp RemotePath) String() string { + if rp.Context != "" { + return ":" + rp.Context + ":" + rp.Path + } + return ":" + rp.Path +} + +// IsRemote returns true if the path is a remote path (starts with :). +func IsRemote(s string) bool { + return strings.HasPrefix(s, ":") +} diff --git a/internal/cli/fs/grep.go b/internal/cli/fs/grep.go new file mode 100644 index 00000000..9b2d5fb3 --- /dev/null +++ b/internal/cli/fs/grep.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func grepCmd(h *internal.Helper) *cobra.Command { + var limit int + var cmd = &cobra.Command{ + Use: "grep [path]", + Short: "Search file contents", + Long: `Search for a pattern in file contents across the filesystem. + +Results are ranked by relevance score. Only text files are searched. + +Examples: + ticloud fs grep "function main" :/ + ticloud fs grep "TODO" :/myproject --limit 20`, + Args: cobra.RangeArgs(1, 2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + pattern := args[0] + path := "/" + if len(args) > 1 { + path = ParseRemotePath(args[1]).Path + } + + results, err := client.Grep(pattern, path, limit) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("grep: %w", err)) + } + + for _, r := range results { + if r.Score != nil { + fmt.Printf("%.3f %s\n", *r.Score, r.Path) + } else { + fmt.Println(r.Path) + } + } + return nil + }, + } + + cmd.Flags().IntVarP(&limit, "limit", "n", 0, "Maximum number of results") + return cmd +} diff --git a/internal/cli/fs/init.go b/internal/cli/fs/init.go new file mode 100644 index 00000000..6195dc8d --- /dev/null +++ b/internal/cli/fs/init.go @@ -0,0 +1,68 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func initCmd(h *internal.Helper) *cobra.Command { + var ( + user string + password string + ) + + var cmd = &cobra.Command{ + Use: "init", + Short: "Initialize FS for the associated database", + Long: `Provision the FS tenant for the configured database. + +This command must be run once before any file operations when using a new +TiDB Cloud cluster or TiDB Zero instance. + +Examples: + ticloud fs init --user admin --password secret`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + if user == "" || password == "" { + return fmt.Errorf("--user and --password are required") + } + + result, err := client.Provision(context.Background(), user, password) + if err != nil { + return fmt.Errorf("provision failed: %w", err) + } + + fmt.Fprintf(h.IOStreams.Out, "FS initialized for tenant %s (status: %s)\n", result.TenantID, result.Status) + return nil + }, + } + + cmd.Flags().StringVar(&user, "user", "", "FS admin user") + cmd.Flags().StringVar(&password, "password", "", "FS admin password") + _ = cmd.MarkFlagRequired("user") + _ = cmd.MarkFlagRequired("password") + + return cmd +} diff --git a/internal/cli/fs/ls.go b/internal/cli/fs/ls.go new file mode 100644 index 00000000..61a63e13 --- /dev/null +++ b/internal/cli/fs/ls.go @@ -0,0 +1,81 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func lsCmd(h *internal.Helper) *cobra.Command { + var long bool + var cmd = &cobra.Command{ + Use: "ls [path]", + Short: "List directory contents", + Example: ` ticloud fs ls :/ + ticloud fs ls -l :/myfolder`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := "/" + if len(args) > 0 { + rp := ParseRemotePath(args[0]) + path = rp.Path + } + if path == "" { + path = "/" + } + + entries, err := client.List(path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("list %s: %w", path, err)) + } + + if long { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for _, e := range entries { + typeChar := "-" + if e.IsDir { + typeChar = "d" + } + fmt.Fprintf(w, "%s\t%s\t%s\n", typeChar, humanize.IBytes(uint64(e.Size)), e.Name) + } + return w.Flush() + } + + for _, e := range entries { + name := e.Name + if e.IsDir { + name += "/" + } + fmt.Println(name) + } + return nil + }, + } + + cmd.Flags().BoolVarP(&long, "long", "l", false, "Use long listing format") + return cmd +} diff --git a/internal/cli/fs/mkdir.go b/internal/cli/fs/mkdir.go new file mode 100644 index 00000000..5d5c91d3 --- /dev/null +++ b/internal/cli/fs/mkdir.go @@ -0,0 +1,45 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mkdirCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mkdir ", + Short: "Create a directory", + Example: ` ticloud fs mkdir :/mydir + ticloud fs mkdir :/parent/child`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := ParseRemotePath(args[0]).Path + if err := client.Mkdir(path); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("mkdir %s: %w", path, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/mount_unix.go b/internal/cli/fs/mount_unix.go new file mode 100644 index 00000000..9af75ce1 --- /dev/null +++ b/internal/cli/fs/mount_unix.go @@ -0,0 +1,136 @@ +//go:build !windows + +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + fusepkg "github.com/tidbcloud/tidbcloud-cli/internal/service/fs/fuse" + + "github.com/spf13/cobra" +) + +func mountCmd(h *internal.Helper) *cobra.Command { + var ( + readOnly bool + debug bool + allowOther bool + ) + + var cmd = &cobra.Command{ + Use: "mount ", + Short: "Mount remote filesystem via FUSE", + Long: `Mount the remote filesystem as a local FUSE mount. + +WARNING: This feature requires FUSE support and may need additional setup. +On Linux, you need fusermount or fusermount3 installed. +On macOS, you need macFUSE installed. + +Examples: + ticloud fs mount /mnt/tidbcloud + ticloud fs mount /mnt/tidbcloud --read-only --debug`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + mountpoint := args[0] + + // Check if mountpoint exists + if _, err := os.Stat(mountpoint); os.IsNotExist(err) { + return fmt.Errorf("mountpoint %s does not exist", mountpoint) + } + + // Check if already mounted + if isMounted(mountpoint) { + return fmt.Errorf("%s is already mounted", mountpoint) + } + + client, err := newClient(cmd) + if err != nil { + return suggestInitIfTenantNotFound(err) + } + + // For now, only support read-only mode (MVP) + // Full read-write support will be added later + if !readOnly { + fmt.Fprintln(h.IOStreams.Out, "Note: Mounting in read-only mode (MVP). Write support coming soon.") + readOnly = true + } + + opts := &fusepkg.MountOptions{ + Client: client, + MountPoint: mountpoint, + ReadOnly: readOnly, + Debug: debug, + AllowOther: allowOther, + } + + return suggestInitIfTenantNotFound(fusepkg.Mount(opts)) + }, + } + + cmd.Flags().BoolVarP(&readOnly, "read-only", "r", true, "Mount as read-only (default: true for MVP)") + cmd.Flags().BoolVar(&debug, "debug", false, "Enable FUSE debug logging") + cmd.Flags().BoolVar(&allowOther, "allow-other", false, "Allow other users to access mount") + + return cmd +} + +func umountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "umount ", + Short: "Unmount FUSE filesystem", + Example: ` ticloud fs umount /mnt/tidbcloud`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + mountpoint := args[0] + + var umountCmd *exec.Cmd + if runtime.GOOS == "darwin" { + umountCmd = exec.Command("umount", mountpoint) + } else { + // Try fusermount3 first, then fusermount, then umount + if _, err := exec.LookPath("fusermount3"); err == nil { + umountCmd = exec.Command("fusermount3", "-u", mountpoint) + } else if _, err := exec.LookPath("fusermount"); err == nil { + umountCmd = exec.Command("fusermount", "-u", mountpoint) + } else { + umountCmd = exec.Command("umount", mountpoint) + } + } + + output, err := umountCmd.CombinedOutput() + if err != nil { + return fmt.Errorf("umount failed: %s", string(output)) + } + + fmt.Fprintf(h.IOStreams.Out, "Unmounted %s\n", mountpoint) + return nil + }, + } +} + +func isMounted(mountpoint string) bool { + data, err := os.ReadFile("/proc/mounts") + if err != nil { + return false + } + return strings.Contains(string(data), mountpoint) +} diff --git a/internal/cli/fs/mount_windows.go b/internal/cli/fs/mount_windows.go new file mode 100644 index 00000000..01e3d87b --- /dev/null +++ b/internal/cli/fs/mount_windows.go @@ -0,0 +1,48 @@ +//go:build windows + +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mount ", + Short: "Mount remote filesystem via FUSE", + Long: `Mount the remote filesystem as a local FUSE mount.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("FUSE mount is not supported on Windows") + }, + } +} + +func umountCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "umount ", + Short: "Unmount FUSE filesystem", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return fmt.Errorf("FUSE umount is not supported on Windows") + }, + } +} diff --git a/internal/cli/fs/mv.go b/internal/cli/fs/mv.go new file mode 100644 index 00000000..bc4a21e0 --- /dev/null +++ b/internal/cli/fs/mv.go @@ -0,0 +1,52 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func mvCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "mv ", + Short: "Move or rename a file or directory", + Long: `Move or rename a file or directory in the remote filesystem. + +This is a metadata-only operation with zero data transfer cost. + +Examples: + ticloud fs mv :/oldname :/newname + ticloud fs mv :/folder/file.txt :/other/file.txt`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + oldPath := ParseRemotePath(args[0]).Path + newPath := ParseRemotePath(args[1]).Path + + if err := client.Rename(oldPath, newPath); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("move %s to %s: %w", oldPath, newPath, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/rm.go b/internal/cli/fs/rm.go new file mode 100644 index 00000000..6117e510 --- /dev/null +++ b/internal/cli/fs/rm.go @@ -0,0 +1,46 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func rmCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "rm ", + Short: "Remove a file or directory", + Example: ` ticloud fs rm :/file.txt + ticloud fs rm :/empty_folder/`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + path := ParseRemotePath(args[0]).Path + + if err := client.Delete(path); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("remove %s: %w", path, err)) + } + return nil + }, + } +} diff --git a/internal/cli/fs/secret.go b/internal/cli/fs/secret.go new file mode 100644 index 00000000..1d806df2 --- /dev/null +++ b/internal/cli/fs/secret.go @@ -0,0 +1,601 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal" + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + "github.com/spf13/cobra" +) + +const ( + defaultAuditLimit = 100 + maxClientAuditLimit = 1000 +) + +func secretCmd(h *internal.Helper) *cobra.Command { + var secretCmd = &cobra.Command{ + Use: "secret", + Short: "Secret management operations", + Long: `Manage secrets in TiDB Cloud FS vault.`, + } + + secretCmd.AddCommand(secretSetCmd(h)) + secretCmd.AddCommand(secretGetCmd(h)) + secretCmd.AddCommand(secretExecCmd(h)) + secretCmd.AddCommand(secretLsCmd(h)) + secretCmd.AddCommand(secretRmCmd(h)) + secretCmd.AddCommand(secretGrantCmd(h)) + secretCmd.AddCommand(secretRevokeCmd(h)) + secretCmd.AddCommand(secretAuditCmd(h)) + + return secretCmd +} + +func secretSetCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "set ...", + Short: "Create or update a secret", + Example: ` ticloud fs secret set myapp DATABASE_URL=mysql://... + ticloud fs secret set myapp key=@secret.txt password=-`, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + fields, err := parseSecretFields(args[1:]) + if err != nil { + return err + } + + ctx := context.Background() + if _, err := client.CreateVaultSecret(ctx, name, fields); err != nil { + // If conflict, update instead + if !strings.Contains(err.Error(), "conflict") { + return suggestInitIfTenantNotFound(fmt.Errorf("set secret: %w", err)) + } + _, err = client.UpdateVaultSecret(ctx, name, fields) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("update secret: %w", err)) + } + } + return nil + }, + } +} + +func secretGetCmd(h *internal.Helper) *cobra.Command { + var asJSON, asEnv bool + var cmd = &cobra.Command{ + Use: "get ", + Short: "Read a secret or one field", + Example: ` ticloud fs secret get myapp + ticloud fs secret get myapp/password --json + ticloud fs secret get myapp/api_key --env`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + ref := args[0] + name, field, err := parseSecretRef(ref) + if err != nil { + return err + } + if asJSON && asEnv { + return fmt.Errorf("--json and --env are mutually exclusive") + } + + ctx := context.Background() + if field != "" { + value, err := client.ReadVaultSecretField(ctx, name, field) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret field: %w", err)) + } + switch { + case asEnv: + envKey, err := normalizeSecretEnvKey(field) + if err != nil { + return err + } + fmt.Fprintf(h.IOStreams.Out, "%s=%s\n", envKey, value) + case asJSON: + return writeJSON(h, map[string]string{field: value}) + default: + fmt.Fprintln(h.IOStreams.Out, value) + } + return nil + } + + fields, err := client.ReadVaultSecret(ctx, name) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret: %w", err)) + } + if asEnv { + return printEnv(h, fields) + } + return writeJSON(h, fields) + }, + } + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + cmd.Flags().BoolVar(&asEnv, "env", false, "Output as environment variables") + return cmd +} + +func secretExecCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "exec -- ", + Short: "Run a command with secret fields injected as env vars", + Example: ` ticloud fs secret exec myapp -- ./run.sh`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + + sep := -1 + for i := 1; i < len(args); i++ { + if args[i] == "--" { + sep = i + break + } + } + if sep < 0 || sep == len(args)-1 { + return fmt.Errorf("usage: ticloud fs secret exec -- ") + } + cmdArgs := args[sep+1:] + + fields, err := client.ReadVaultSecret(context.Background(), name) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("get secret: %w", err)) + } + + //nolint:gosec // user-provided command execution is intentional + c := exec.Command(cmdArgs[0], cmdArgs[1:]...) + c.Stdin = os.Stdin + c.Stdout = h.IOStreams.Out + c.Stderr = h.IOStreams.Err + envMap, err := buildSecretEnvMap(fields) + if err != nil { + return err + } + c.Env = mergeEnv(os.Environ(), envMap) + return c.Run() + }, + } +} + +func secretLsCmd(h *internal.Helper) *cobra.Command { + var asJSON bool + var cmd = &cobra.Command{ + Use: "ls", + Short: "List secrets", + Example: ` ticloud fs secret ls --json`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + secrets, err := client.ListVaultSecrets(context.Background()) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("list secrets: %w", err)) + } + + names := make([]string, 0, len(secrets)) + for _, sec := range secrets { + names = append(names, sec.Name) + } + sort.Strings(names) + + if asJSON { + return writeJSON(h, map[string]any{"secrets": names}) + } + for _, name := range names { + fmt.Fprintln(h.IOStreams.Out, name) + } + return nil + }, + } + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + return cmd +} + +func secretRmCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "rm ", + Short: "Delete a secret", + Example: ` ticloud fs secret rm myapp`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + name := args[0] + if err := validateSecretName(name); err != nil { + return err + } + if err := client.DeleteVaultSecret(context.Background(), name); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("delete secret: %w", err)) + } + return nil + }, + } +} + +func secretGrantCmd(h *internal.Helper) *cobra.Command { + var ( + agentID string + taskID string + ttlRaw string + asJSON bool + tokenOnly bool + ) + var cmd = &cobra.Command{ + Use: "grant --agent --ttl [--task ] ", + Short: "Issue a scoped capability token", + Example: ` ticloud fs secret grant --agent myagent --ttl 1h myapp/password`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + scope := args + for _, entry := range scope { + if _, _, err := parseSecretRef(entry); err != nil { + return fmt.Errorf("invalid scope %q: %w", entry, err) + } + } + if asJSON && tokenOnly { + return fmt.Errorf("--json and --token-only are mutually exclusive") + } + if agentID == "" { + return fmt.Errorf("--agent is required") + } + if ttlRaw == "" { + return fmt.Errorf("--ttl is required") + } + ttl, err := time.ParseDuration(ttlRaw) + if err != nil { + return fmt.Errorf("invalid --ttl %q: %w", ttlRaw, err) + } + if ttl <= 0 { + return fmt.Errorf("--ttl must be positive") + } + + resp, err := client.IssueVaultToken(context.Background(), agentID, taskID, scope, ttl) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("grant token: %w", err)) + } + switch { + case tokenOnly: + fmt.Fprintln(h.IOStreams.Out, resp.Token) + case asJSON: + return writeJSON(h, resp) + default: + fmt.Fprintf(h.IOStreams.Out, "token=%s\n", resp.Token) + fmt.Fprintf(h.IOStreams.Out, "token_id=%s\n", resp.TokenID) + fmt.Fprintf(h.IOStreams.Out, "expires_at=%s\n", resp.ExpiresAt.Format(time.RFC3339)) + } + return nil + }, + } + cmd.Flags().StringVar(&agentID, "agent", "", "Agent ID") + cmd.Flags().StringVar(&taskID, "task", "", "Task ID") + cmd.Flags().StringVar(&ttlRaw, "ttl", "", "Token TTL (e.g., 1h, 30m)") + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + cmd.Flags().BoolVar(&tokenOnly, "token-only", false, "Output only the token string") + return cmd +} + +func secretRevokeCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "revoke ", + Short: "Revoke a capability token", + Example: ` ticloud fs secret revoke tok_abc123`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + if err := client.RevokeVaultToken(context.Background(), args[0]); err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("revoke token: %w", err)) + } + return nil + }, + } +} + +func secretAuditCmd(h *internal.Helper) *cobra.Command { + var ( + secretName string + agentID string + sinceRaw string + limit = defaultAuditLimit + asJSON bool + ) + var cmd = &cobra.Command{ + Use: "audit [--secret ] [--agent ] [--since ] [--limit ]", + Short: "Query vault audit events", + Example: ` ticloud fs secret audit --limit 50 + ticloud fs secret audit --secret myapp --since 24h`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + queryLimit := limit + if agentID != "" || sinceRaw != "" { + queryLimit = maxClientAuditLimit + } + if queryLimit > maxClientAuditLimit { + queryLimit = maxClientAuditLimit + } + + events, err := client.QueryVaultAudit(context.Background(), secretName, queryLimit) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("audit query: %w", err)) + } + + if sinceRaw != "" { + d, err := time.ParseDuration(sinceRaw) + if err != nil { + return fmt.Errorf("invalid --since %q: %w", sinceRaw, err) + } + if d <= 0 { + return fmt.Errorf("--since must be positive") + } + events = filterAuditEvents(events, agentID, time.Now().Add(-d)) + } else if agentID != "" { + events = filterAuditEvents(events, agentID, time.Time{}) + } + if len(events) > limit { + events = events[:limit] + } + + if asJSON { + return writeJSON(h, map[string]any{"events": events}) + } + printAudit(h, events) + return nil + }, + } + cmd.Flags().StringVar(&secretName, "secret", "", "Filter by secret name") + cmd.Flags().StringVar(&agentID, "agent", "", "Filter by agent ID") + cmd.Flags().StringVar(&sinceRaw, "since", "", "Filter events newer than duration (e.g., 24h)") + cmd.Flags().IntVar(&limit, "limit", defaultAuditLimit, "Maximum number of events") + cmd.Flags().BoolVar(&asJSON, "json", false, "Output as JSON") + return cmd +} + +func validateSecretName(name string) error { + if name == "" { + return fmt.Errorf("secret name is required") + } + if strings.Contains(name, "/") { + return fmt.Errorf("secret name %q must be flat; use only for reads and scopes", name) + } + if strings.Contains(name, "*") { + return fmt.Errorf("wildcard scope entries are not supported: %q", name) + } + return nil +} + +func parseSecretRef(raw string) (string, string, error) { + if raw == "" { + return "", "", fmt.Errorf("secret reference is required") + } + parts := strings.SplitN(raw, "/", 2) + name := parts[0] + if err := validateSecretName(name); err != nil { + return "", "", err + } + if len(parts) == 1 { + return name, "", nil + } + field := parts[1] + if field == "" { + return "", "", fmt.Errorf("field name is required in %q", raw) + } + if strings.Contains(field, "*") { + return "", "", fmt.Errorf("wildcard scope entries are not supported: %q", raw) + } + return name, field, nil +} + +func parseSecretFields(args []string) (map[string]string, error) { + fields := make(map[string]string, len(args)) + var stdinValue []byte + var stdinRead bool + for _, arg := range args { + key, valueSpec, ok := strings.Cut(arg, "=") + if !ok || key == "" { + return nil, fmt.Errorf("field assignment must be field=value, field=@file, or field=-: %q", arg) + } + var value string + switch { + case valueSpec == "-": + if !stdinRead { + data, err := io.ReadAll(os.Stdin) + if err != nil { + return nil, fmt.Errorf("read stdin: %w", err) + } + stdinValue = data + stdinRead = true + } + value = string(stdinValue) + case strings.HasPrefix(valueSpec, "@"): + data, err := os.ReadFile(valueSpec[1:]) + if err != nil { + return nil, fmt.Errorf("read %s: %w", valueSpec[1:], err) + } + value = string(data) + default: + value = valueSpec + } + fields[key] = value + } + return fields, nil +} + +func writeJSON(h *internal.Helper, v any) error { + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + return enc.Encode(v) +} + +func printEnv(h *internal.Helper, fields map[string]string) error { + envMap, err := buildSecretEnvMap(fields) + if err != nil { + return err + } + keys := make([]string, 0, len(envMap)) + for k := range envMap { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + fmt.Fprintf(h.IOStreams.Out, "%s=%s\n", key, envMap[key]) + } + return nil +} + +func printAudit(h *internal.Helper, events []fs.VaultAuditEvent) { + w := tabwriter.NewWriter(h.IOStreams.Out, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(w, "TIME\tAGENT\tACTION\tSECRET\tFIELD") + for _, ev := range events { + _, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", + ev.Timestamp.Format(time.RFC3339), + ev.AgentID, + ev.EventType, + ev.SecretName, + ev.FieldName, + ) + } + _ = w.Flush() +} + +func buildSecretEnvMap(fields map[string]string) (map[string]string, error) { + env := make(map[string]string, len(fields)) + owners := make(map[string]string, len(fields)) + for field, value := range fields { + envKey, err := normalizeSecretEnvKey(field) + if err != nil { + return nil, err + } + if prevField, exists := owners[envKey]; exists { + return nil, fmt.Errorf("secret fields %q and %q both normalize to env var %q", prevField, field, envKey) + } + owners[envKey] = field + env[envKey] = value + } + return env, nil +} + +func filterAuditEvents(events []fs.VaultAuditEvent, agentID string, since time.Time) []fs.VaultAuditEvent { + filtered := make([]fs.VaultAuditEvent, 0, len(events)) + for _, ev := range events { + if agentID != "" && ev.AgentID != agentID { + continue + } + if !since.IsZero() && ev.Timestamp.Before(since) { + continue + } + filtered = append(filtered, ev) + } + return filtered +} + +func mergeEnv(base []string, overrides map[string]string) []string { + merged := make(map[string]string, len(base)+len(overrides)) + for _, entry := range base { + key, value, ok := strings.Cut(entry, "=") + if ok { + merged[key] = value + } + } + for k, v := range overrides { + merged[k] = v + } + keys := make([]string, 0, len(merged)) + for k := range merged { + keys = append(keys, k) + } + sort.Strings(keys) + env := make([]string, 0, len(keys)) + for _, k := range keys { + env = append(env, k+"="+merged[k]) + } + return env +} + +func normalizeSecretEnvKey(field string) (string, error) { + if field == "" { + return "", fmt.Errorf("secret field name is required") + } + var b strings.Builder + b.Grow(len(field) + 1) + for i := 0; i < len(field); i++ { + ch := field[i] + switch { + case ch >= 'a' && ch <= 'z': + b.WriteByte(ch - ('a' - 'A')) + case ch >= 'A' && ch <= 'Z', ch >= '0' && ch <= '9', ch == '_': + b.WriteByte(ch) + default: + b.WriteByte('_') + } + } + key := b.String() + if key == "" { + return "", fmt.Errorf("secret field name is required") + } + if key[0] >= '0' && key[0] <= '9' { + key = "_" + key + } + return key, nil +} diff --git a/internal/cli/fs/shell.go b/internal/cli/fs/shell.go new file mode 100644 index 00000000..c4d37ae8 --- /dev/null +++ b/internal/cli/fs/shell.go @@ -0,0 +1,228 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func shellCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "shell", + Short: "Interactive filesystem shell", + Long: `Start an interactive shell for filesystem operations. + +Commands: + cd Change directory + pwd Print current directory + ls [path] List directory + cat Display file contents + cp Copy files (remote only) + mkdir Create directory + mv Move/rename + rm Remove file + sql Execute SQL query + stat Show metadata + help Show help + exit Exit shell + +The prompt shows the current remote directory.`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + cwd := "/" + ctx := context.Background() + scanner := bufio.NewScanner(h.IOStreams.In) + + fmt.Fprintln(h.IOStreams.Out, "TiDB Cloud FS Shell. Type 'help' for commands, 'exit' to quit.") + + for { + fmt.Fprint(h.IOStreams.Out, "ticloud:fs> ") + if !scanner.Scan() { + break + } + + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + + cmd := parts[0] + args := parts[1:] + + switch cmd { + case "exit", "quit": + return nil + + case "help": + fmt.Fprintln(h.IOStreams.Out, `Commands: + cd Change directory + pwd Print current directory + ls [path] List directory + cat Display file contents + cp Copy files (remote only) + mkdir Create directory + mv Move/rename + rm Remove file + sql Execute SQL query + stat Show metadata + help Show this help + exit Exit shell`) + + case "pwd": + fmt.Fprintln(h.IOStreams.Out, cwd) + + case "cd": + if len(args) == 0 { + cwd = "/" + } else { + cwd = resolvePath(cwd, args[0]) + } + + case "ls": + path := cwd + if len(args) > 0 { + path = resolvePath(cwd, args[0]) + } + entries, err := client.List(path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("list: %w", err))) + continue + } + for _, e := range entries { + name := e.Name + if e.IsDir { + name += "/" + } + fmt.Fprintln(h.IOStreams.Out, name) + } + + case "cat": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: cat ") + continue + } + path := resolvePath(cwd, args[0]) + reader, err := client.ReadStream(ctx, path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("read: %w", err))) + continue + } + _, err = io.Copy(h.IOStreams.Out, reader) + reader.Close() + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", err) + } + + case "cp": + if len(args) < 2 { + fmt.Fprintln(h.IOStreams.Err, "Usage: cp ") + continue + } + src, dst := resolvePath(cwd, args[0]), resolvePath(cwd, args[1]) + if err := client.Copy(src, dst); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("copy: %w", err))) + } + + case "mkdir": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: mkdir ") + continue + } + path := resolvePath(cwd, args[0]) + if err := client.Mkdir(path); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("mkdir: %w", err))) + } + + case "mv": + if len(args) < 2 { + fmt.Fprintln(h.IOStreams.Err, "Usage: mv ") + continue + } + old, newPath := resolvePath(cwd, args[0]), resolvePath(cwd, args[1]) + if err := client.Rename(old, newPath); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("mv: %w", err))) + } + + case "rm": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: rm ") + continue + } + path := resolvePath(cwd, args[0]) + if err := client.Delete(path); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("rm: %w", err))) + } + + case "sql": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: sql ") + continue + } + query := strings.Join(args, " ") + rows, err := client.SQL(query) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("sql: %w", err))) + continue + } + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + if err := enc.Encode(rows); err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", err) + } + + case "stat": + if len(args) < 1 { + fmt.Fprintln(h.IOStreams.Err, "Usage: stat ") + continue + } + path := resolvePath(cwd, args[0]) + info, err := client.Stat(path) + if err != nil { + fmt.Fprintf(h.IOStreams.Err, "Error: %v\n", suggestInitIfTenantNotFound(fmt.Errorf("stat: %w", err))) + continue + } + fmt.Fprintf(h.IOStreams.Out, "Size: %d, Dir: %v, Rev: %d\n", info.Size, info.IsDir, info.Revision) + + default: + fmt.Fprintf(h.IOStreams.Err, "Unknown command: %s\n", cmd) + } + } + + return nil + }, + } +} + +func resolvePath(cwd, p string) string { + if strings.HasPrefix(p, "/") { + return p + } + return filepath.Join(cwd, p) +} diff --git a/internal/cli/fs/sql.go b/internal/cli/fs/sql.go new file mode 100644 index 00000000..c0ad35c7 --- /dev/null +++ b/internal/cli/fs/sql.go @@ -0,0 +1,73 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/spf13/cobra" +) + +func sqlCmd(h *internal.Helper) *cobra.Command { + var query, file string + var cmd = &cobra.Command{ + Use: "sql", + Short: "Execute a SQL query", + Long: `Execute a SQL query against the FS backend database. + +You can provide the query directly with -q or from a file with -f.`, + Example: ` ticloud fs sql -q "SELECT 1" + ticloud fs sql -f query.sql`, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + if query == "" && file == "" { + return fmt.Errorf("either -q or -f is required") + } + if query != "" && file != "" { + return fmt.Errorf("-q and -f are mutually exclusive") + } + + if file != "" { + data, err := os.ReadFile(file) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + query = string(data) + } + + rows, err := client.SQL(query) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("sql: %w", err)) + } + + enc := json.NewEncoder(h.IOStreams.Out) + enc.SetIndent("", " ") + return enc.Encode(rows) + }, + } + + cmd.Flags().StringVarP(&query, "query", "q", "", "SQL query string") + cmd.Flags().StringVarP(&file, "file", "f", "", "Path to a SQL file") + + return cmd +} diff --git a/internal/cli/fs/stat.go b/internal/cli/fs/stat.go new file mode 100644 index 00000000..7f2a3645 --- /dev/null +++ b/internal/cli/fs/stat.go @@ -0,0 +1,54 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "fmt" + + "github.com/tidbcloud/tidbcloud-cli/internal" + + "github.com/dustin/go-humanize" + "github.com/spf13/cobra" +) + +func statCmd(h *internal.Helper) *cobra.Command { + return &cobra.Command{ + Use: "stat ", + Short: "Display file metadata", + Example: ` ticloud fs stat :/file.txt + ticloud fs stat :/myfolder/`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := newClient(cmd) + if err != nil { + return err + } + + rp := ParseRemotePath(args[0]) + path := rp.Path + + info, err := client.Stat(path) + if err != nil { + return suggestInitIfTenantNotFound(fmt.Errorf("stat %s: %w", path, err)) + } + + fmt.Printf("Path: %s\n", path) + fmt.Printf("Size: %s (%d bytes)\n", humanize.IBytes(uint64(info.Size)), info.Size) + fmt.Printf("Type: %s\n", map[bool]string{true: "directory", false: "file"}[info.IsDir]) + fmt.Printf("Version: %d\n", info.Revision) + return nil + }, + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 847c9b22..4e8fe139 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -24,6 +24,7 @@ import ( "github.com/tidbcloud/tidbcloud-cli/internal" "github.com/tidbcloud/tidbcloud-cli/internal/cli/auth" configCmd "github.com/tidbcloud/tidbcloud-cli/internal/cli/config" + "github.com/tidbcloud/tidbcloud-cli/internal/cli/fs" "github.com/tidbcloud/tidbcloud-cli/internal/cli/project" "github.com/tidbcloud/tidbcloud-cli/internal/cli/serverless" "github.com/tidbcloud/tidbcloud-cli/internal/cli/upgrade" @@ -194,6 +195,7 @@ func RootCmd(h *internal.Helper) *cobra.Command { rootCmd.AddCommand(auth.AuthCmd(h)) rootCmd.AddCommand(configCmd.ConfigCmd(h)) + rootCmd.AddCommand(fs.FSCmd(h)) rootCmd.AddCommand(serverless.Cmd(h)) rootCmd.AddCommand(project.ProjectCmd(h)) rootCmd.AddCommand(version.VersionCmd(h)) diff --git a/internal/config/config.go b/internal/config/config.go index 1baf0012..743c68e2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -28,10 +28,11 @@ const ( cliNameInTiUP = "cloud" HomePath = ".ticloud" - Confirmed = "yes" - OAuthEndpoint = "https://oauth.tidbcloud.com" - ClientID = "wiSBy7f27zWBaBCxS16tDm7DDj2T3POgwFFbefTrgx8FAXKhzaPzv1Uta9NTck2r" - ClientSecret = "ieKdWDao0QFmHbfYXpQJHuYZ9nLdpRptfE9d3W30WettIFLZL66JKewznvVkY39IkEbBQiZK60pjcnm7BN7Lj9uRiCSpIC4n2aI3IUyHlLKtvxZLrrfXugHsC7qhb1Js" + Confirmed = "yes" + OAuthEndpoint = "https://oauth.tidbcloud.com" + DefaultFSEndpoint = "https://fs.tidbapi.com" + ClientID = "wiSBy7f27zWBaBCxS16tDm7DDj2T3POgwFFbefTrgx8FAXKhzaPzv1Uta9NTck2r" + ClientSecret = "ieKdWDao0QFmHbfYXpQJHuYZ9nLdpRptfE9d3W30WettIFLZL66JKewznvVkY39IkEbBQiZK60pjcnm7BN7Lj9uRiCSpIC4n2aI3IUyHlLKtvxZLrrfXugHsC7qhb1Js" ) var ( diff --git a/internal/config/profile.go b/internal/config/profile.go index 8736052c..f72b6b21 100644 --- a/internal/config/profile.go +++ b/internal/config/profile.go @@ -124,6 +124,25 @@ func (p *Profile) GetIAMEndpoint() string { return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.IAMEndpoint)) } +func GetFSEndpoint() string { return activeProfile.GetFSEndpoint() } +func (p *Profile) GetFSEndpoint() string { + endpoint := viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSEndpoint)) + if endpoint == "" { + return DefaultFSEndpoint + } + return endpoint +} + +func GetFSClusterID() string { return activeProfile.GetFSClusterID() } +func (p *Profile) GetFSClusterID() string { + return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSClusterID)) +} + +func GetFSZeroInstanceID() string { return activeProfile.GetFSZeroInstanceID() } +func (p *Profile) GetFSZeroInstanceID() string { + return viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.FSZeroInstanceID)) +} + func GetOAuthEndpoint() (apiUrl string) { return activeProfile.GetOAuthEndpoint() } func (p *Profile) GetOAuthEndpoint() (newApiUrl string) { newApiUrl = viper.GetString(fmt.Sprintf("%s.%s", p.name, prop.OAuthEndpoint)) diff --git a/internal/flag/flag.go b/internal/flag/flag.go index e81a11df..b960be0d 100644 --- a/internal/flag/flag.go +++ b/internal/flag/flag.go @@ -134,6 +134,11 @@ const ( MigrationConfigFile string = "config-file" MigrationMode string = "mode" DryRun string = "dry-run" + + // FS flags + FSEndpoint string = "fs-endpoint" + FSClusterID string = "fs.cluster-id" + FSZeroInstanceID string = "fs.zero-instance-id" ) const OutputHelp = "Output format, one of [\"human\" \"json\"]. For the complete result, please use json format." diff --git a/internal/iostream/iostream.go b/internal/iostream/iostream.go index 7d136b5c..8261321e 100644 --- a/internal/iostream/iostream.go +++ b/internal/iostream/iostream.go @@ -23,6 +23,7 @@ import ( ) type IOStreams struct { + In io.Reader Out io.Writer Err io.Writer CanPrompt bool @@ -32,6 +33,7 @@ func System() *IOStreams { canPrompt := isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) return &IOStreams{ + In: os.Stdin, Out: os.Stdout, Err: os.Stderr, CanPrompt: canPrompt, @@ -40,6 +42,7 @@ func System() *IOStreams { func Test() *IOStreams { return &IOStreams{ + In: &bytes.Buffer{}, Out: &bytes.Buffer{}, Err: &bytes.Buffer{}, CanPrompt: false, diff --git a/internal/prop/property.go b/internal/prop/property.go index 102f23ef..c7c4d47c 100644 --- a/internal/prop/property.go +++ b/internal/prop/property.go @@ -30,6 +30,9 @@ const ( OAuthClientID string = "oauth-client-id" OAuthClientSecret string = "oauth-client-secret" TelemetryEnabled string = "telemetry-enabled" + FSEndpoint string = "fs-endpoint" + FSClusterID string = "fs.cluster-id" + FSZeroInstanceID string = "fs.zero-instance-id" // shall not be set by user TokenExpiredAt string = "token-expired-at" @@ -42,7 +45,7 @@ func GlobalProperties() []string { } func ProfileProperties() []string { - return []string{PublicKey, PrivateKey, ServerlessEndpoint, IAMEndpoint, OAuthEndpoint, OAuthClientID, OAuthClientSecret, TelemetryEnabled} + return []string{PublicKey, PrivateKey, ServerlessEndpoint, IAMEndpoint, OAuthEndpoint, OAuthClientID, OAuthClientSecret, TelemetryEnabled, FSEndpoint, FSClusterID, FSZeroInstanceID} } func ValidateApiUrl(value string) (*url.URL, error) { diff --git a/internal/service/fs/client.go b/internal/service/fs/client.go new file mode 100644 index 00000000..4ba67c53 --- /dev/null +++ b/internal/service/fs/client.go @@ -0,0 +1,1596 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fs provides the filesystem client for TiDB Cloud FS operations. +package fs + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // PartSize is the default S3 multipart part size (8 MiB). + PartSize = 8 * 1024 * 1024 + + // DefaultSmallFileThreshold matches the server's threshold for direct PUT vs multipart. + DefaultSmallFileThreshold = 50_000 // 50,000 bytes + + // Upload concurrency limits + uploadMaxConcurrency = 16 + uploadMaxBufferBytes = 256 * 1024 * 1024 // 256 MB +) + +// Part represents a single part in a multipart upload. +type Part struct { + Number int64 + Size int64 +} + +// CalcParts calculates the parts needed for a file of given size. +func CalcParts(totalSize, partSize int64) []Part { + if totalSize <= 0 { + return nil + } + if partSize <= 0 { + partSize = PartSize + } + parts := make([]Part, 0, (totalSize+partSize-1)/partSize) + var offset int64 + for offset < totalSize { + sz := partSize + if offset+sz > totalSize { + sz = totalSize - offset + } + parts = append(parts, Part{ + Number: int64(len(parts) + 1), + Size: sz, + }) + offset += sz + } + return parts +} + +// Client is the FS HTTP client. +type Client struct { + baseURL string + httpClient *http.Client + clusterID string + zeroInstanceID string + smallFileThreshold int64 // 0 means use DefaultSmallFileThreshold +} + +// GetBaseURL returns the base URL of the client. +func (c *Client) GetBaseURL() string { + return c.baseURL +} + +// GetClusterID returns the associated TiDB Cloud cluster ID. +func (c *Client) GetClusterID() string { + return c.clusterID +} + +// GetZeroInstanceID returns the associated TiDB Zero instance ID. +func (c *Client) GetZeroInstanceID() string { + return c.zeroInstanceID +} + +// ProvisionResult is the response from the FS provision endpoint. +type ProvisionResult struct { + TenantID string `json:"tenant_id"` + APIKey string `json:"api_key"` + Status string `json:"status"` +} + +// Provision initializes the FS tenant for the associated database. +func (c *Client) Provision(ctx context.Context, user, password string) (*ProvisionResult, error) { + body, err := json.Marshal(map[string]string{ + "user": user, + "password": password, + }) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/provision", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusAccepted { + return nil, readError(resp) + } + var result ProvisionResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode provision response: %w", err) + } + return &result, nil +} + +// NewClient creates a new FS client. +// The httpClient should already be configured with authentication (e.g., Bearer or Digest transport). +// clusterID and zeroInstanceID are sent as X-TIDBCLOUD-CLUSTER-ID and X-TIDBCLOUD-ZERO-INSTANCE-ID headers. +func NewClient(baseURL string, httpClient *http.Client, clusterID, zeroInstanceID string) *Client { + if httpClient == nil { + httpClient = &http.Client{} + } + return &Client{ + baseURL: strings.TrimRight(baseURL, "/"), + httpClient: httpClient, + clusterID: clusterID, + zeroInstanceID: zeroInstanceID, + } +} + +// FileInfo represents a file entry from a directory listing. +type FileInfo struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"isDir"` +} + +// StatResult represents file metadata from HEAD. +type StatResult struct { + Size int64 + IsDir bool + Revision int64 +} + +func (c *Client) url(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return c.baseURL + "/v1/fs" + path +} + +// RawPost sends a raw POST request to the specified endpoint. +func (c *Client) RawPost(endpoint string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPost, c.baseURL+endpoint, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + return c.do(req) +} + +func (c *Client) do(req *http.Request) (*http.Response, error) { + return c.doWithClient(c.httpClient, req) +} + +func (c *Client) doWithClient(client *http.Client, req *http.Request) (*http.Response, error) { + // Inject TiDB Cloud native provider headers. Cluster ID takes precedence over Zero instance ID. + if c.clusterID != "" { + req.Header.Set("X-TIDBCLOUD-CLUSTER-ID", c.clusterID) + } else if c.zeroInstanceID != "" { + req.Header.Set("X-TIDBCLOUD-ZERO-INSTANCE-ID", c.zeroInstanceID) + } + return client.Do(req) +} + +// Write uploads data to a remote path. +func (c *Client) Write(path string, data []byte) error { + req, err := http.NewRequest(http.MethodPut, c.url(path), bytes.NewReader(data)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Read downloads a file's content. +func (c *Client) Read(path string) ([]byte, error) { + req, err := http.NewRequest(http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + return io.ReadAll(resp.Body) +} + +// List returns the entries in a directory. +func (c *Client) List(path string) ([]FileInfo, error) { + // Use an explicit value to avoid intermediaries dropping bare "?list". + req, err := http.NewRequest(http.MethodGet, c.url(path)+"?list=1", nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Entries []FileInfo `json:"entries"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return result.Entries, nil +} + +// Stat returns metadata for a path. +func (c *Client) Stat(path string) (*StatResult, error) { + req, err := http.NewRequest(http.MethodHead, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + _ = resp.Body.Close() + if resp.StatusCode == 404 { + return nil, fmt.Errorf("not found: %s", path) + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("HTTP %d", resp.StatusCode) + } + s := &StatResult{ + IsDir: resp.Header.Get("X-FS-IsDir") == "true", + } + if cl := resp.Header.Get("Content-Length"); cl != "" { + s.Size, _ = strconv.ParseInt(cl, 10, 64) + } + if rev := resp.Header.Get("X-FS-Revision"); rev != "" { + s.Revision, _ = strconv.ParseInt(rev, 10, 64) + } + return s, nil +} + +// Delete removes a file or directory. +func (c *Client) Delete(path string) error { + req, err := http.NewRequest(http.MethodDelete, c.url(path), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Copy performs a server-side zero-copy (same file_id, new path). +func (c *Client) Copy(srcPath, dstPath string) error { + req, err := http.NewRequest(http.MethodPost, c.url(dstPath)+"?copy", nil) + if err != nil { + return err + } + req.Header.Set("X-Dat9-Copy-Source", srcPath) + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Rename moves/renames a file or directory (metadata-only). +func (c *Client) Rename(oldPath, newPath string) error { + req, err := http.NewRequest(http.MethodPost, c.url(newPath)+"?rename", nil) + if err != nil { + return err + } + req.Header.Set("X-Dat9-Rename-Source", oldPath) + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// Mkdir creates a directory. +func (c *Client) Mkdir(path string) error { + req, err := http.NewRequest(http.MethodPost, c.url(path)+"?mkdir", nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +func readError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + var errResp struct { + Error string `json:"error"` + } + if json.Unmarshal(body, &errResp) == nil && errResp.Error != "" { + return fmt.Errorf("%s", errResp.Error) + } + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) +} + +// SQL executes a SQL query. +func (c *Client) SQL(query string) ([]map[string]interface{}, error) { + body, err := json.Marshal(map[string]string{"query": query}) + if err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, c.baseURL+"/v1/sql", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var rows []map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil { + return nil, fmt.Errorf("decode: %w", err) + } + return rows, nil +} + +// SearchResult represents a search result. +type SearchResult struct { + Path string `json:"path"` + Name string `json:"name"` + SizeBytes int64 `json:"size_bytes"` + Score *float64 `json:"score,omitempty"` +} + +// Grep searches for a pattern in files. +func (c *Client) Grep(query, pathPrefix string, limit int) ([]SearchResult, error) { + u := c.url(pathPrefix) + "?grep=" + url.QueryEscape(query) + if limit > 0 { + u += "&limit=" + strconv.Itoa(limit) + } + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var results []SearchResult + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, err + } + return results, nil +} + +// Find searches for files matching criteria. +func (c *Client) Find(pathPrefix string, params url.Values) ([]SearchResult, error) { + params.Set("find", "") + u := c.url(pathPrefix) + "?" + params.Encode() + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var results []SearchResult + if err := json.NewDecoder(resp.Body).Decode(&results); err != nil { + return nil, err + } + return results, nil +} + +// ProgressFunc is called after each part upload completes. +type ProgressFunc func(partNumber, totalParts int, bytesUploaded int64) + +// UploadPlan is the server's 202 response for large file uploads. +type UploadPlan struct { + UploadID string `json:"upload_id"` + PartSize int64 `json:"part_size"` + Parts []PartURL `json:"parts"` +} + +// PartURL is a presigned URL for uploading one part. +type PartURL struct { + Number int `json:"number"` + URL string `json:"url"` + Size int64 `json:"size"` + ChecksumSHA256 string `json:"checksum_sha256,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + ExpiresAt string `json:"expires_at"` +} + +// WriteStream uploads data from a reader. For small files (size < threshold), +// it does a direct PUT with body. For large files, it sends a Content-Length-only +// PUT to get a 202 with presigned URLs, then uploads parts concurrently. +func (c *Client) WriteStream(ctx context.Context, path string, r io.Reader, size int64, progress ProgressFunc) error { + threshold := int64(DefaultSmallFileThreshold) + if c.smallFileThreshold > 0 { + threshold = c.smallFileThreshold + } + if size < threshold { + // Small file: direct PUT with body + data, err := io.ReadAll(r) + if err != nil { + return fmt.Errorf("read data: %w", err) + } + return c.Write(path, data) + } + ra, ok := r.(io.ReaderAt) + if !ok { + return fmt.Errorf("large uploads require an io.ReaderAt (seekable source) to compute per-part checksums") + } + checksums, err := computePartChecksumsFromReaderAt(ra, size, PartSize) + if err != nil { + return fmt.Errorf("compute part checksums: %w", err) + } + plan, err := c.initiateUpload(ctx, path, size, checksums) + if err != nil { + return err + } + return c.uploadParts(ctx, plan, ra, progress) +} + +type uploadInitiateRequest struct { + Path string `json:"path"` + TotalSize int64 `json:"total_size"` + PartChecksums []string `json:"part_checksums"` +} + +type uploadResumeRequest struct { + PartChecksums []string `json:"part_checksums"` +} + +func (c *Client) initiateUpload(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, error) { + plan, resp, err := c.initiateUploadByBody(ctx, path, size, checksums) + if err == nil { + return plan, nil + } + if resp != nil { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusMethodNotAllowed { + return c.initiateUploadLegacy(ctx, path, size, checksums) + } + if resp.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(err.Error()), "unknown upload action") { + return c.initiateUploadLegacy(ctx, path, size, checksums) + } + return UploadPlan{}, err + } + return UploadPlan{}, err +} + +func (c *Client) initiateUploadByBody(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, *http.Response, error) { + body, err := json.Marshal(uploadInitiateRequest{Path: path, TotalSize: size, PartChecksums: checksums}) + if err != nil { + return UploadPlan{}, nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+"/v1/uploads/initiate", bytes.NewReader(body)) + if err != nil { + return UploadPlan{}, nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return UploadPlan{}, nil, err + } + if resp.StatusCode != http.StatusAccepted { + return UploadPlan{}, resp, readError(resp) + } + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + _ = resp.Body.Close() + return UploadPlan{}, nil, fmt.Errorf("decode upload plan: %w", err) + } + _ = resp.Body.Close() + return plan, nil, nil +} + +func (c *Client) initiateUploadLegacy(ctx context.Context, path string, size int64, checksums []string) (UploadPlan, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.url(path), http.NoBody) + if err != nil { + return UploadPlan{}, err + } + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("X-FS-Content-Length", fmt.Sprintf("%d", size)) + if len(checksums) > 0 { + req.Header.Set("X-FS-Part-Checksums", strings.Join(checksums, ",")) + } + + resp, err := c.do(req) + if err != nil { + return UploadPlan{}, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusAccepted { + return UploadPlan{}, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return UploadPlan{}, fmt.Errorf("decode upload plan: %w", err) + } + return plan, nil +} + +func uploadParallelism(partSize int64) int { + if partSize <= 0 { + partSize = PartSize + } + byMemory := int(uploadMaxBufferBytes / partSize) + if byMemory < 1 { + byMemory = 1 + } + return min(byMemory, uploadMaxConcurrency) +} + +func checksumParallelism(partSize int64, partCount int) int { + if partSize <= 0 { + partSize = PartSize + } + byMemory := int(uploadMaxBufferBytes / partSize) + if byMemory < 1 { + byMemory = 1 + } + return min(runtime.GOMAXPROCS(0), partCount, byMemory) +} + +func (c *Client) uploadParts(ctx context.Context, plan UploadPlan, ra io.ReaderAt, progress ProgressFunc) error { + stdPartSize := plan.PartSize + if stdPartSize <= 0 && len(plan.Parts) > 0 { + stdPartSize = plan.Parts[0].Size + } + if stdPartSize <= 0 { + stdPartSize = PartSize + } + maxConcurrency := uploadParallelism(stdPartSize) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errCh := make(chan error, 1) + sem := make(chan struct{}, maxConcurrency) + var wg sync.WaitGroup + + for _, part := range plan.Parts { + select { + case err := <-errCh: + cancel() + wg.Wait() + return err + default: + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + wg.Add(1) + go func(p PartURL) { + defer wg.Done() + defer func() { <-sem }() + + data := make([]byte, p.Size) + offset := int64(p.Number-1) * stdPartSize + n, err := ra.ReadAt(data, offset) + if err != nil && err != io.EOF { + select { + case errCh <- fmt.Errorf("read part %d: %w", p.Number, err): + default: + } + cancel() + return + } + if int64(n) != p.Size { + select { + case errCh <- fmt.Errorf("short read for part %d: got %d want %d", p.Number, n, p.Size): + default: + } + cancel() + return + } + + _, err = c.uploadOnePart(ctx, p, data) + if err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + cancel() + return + } + + if progress != nil { + progress(p.Number, len(plan.Parts), int64(len(data))) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) uploadOnePart(ctx context.Context, part PartURL, data []byte) (string, error) { + checksum := part.ChecksumSHA256 + if checksum == "" { + h := sha256.Sum256(data) + checksum = base64.StdEncoding.EncodeToString(h[:]) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) + if err != nil { + return "", err + } + for k, v := range part.Headers { + if strings.EqualFold(k, "host") { + continue + } + req.Header.Set(k, v) + } + req.ContentLength = int64(len(data)) + req.Header.Set("x-amz-checksum-sha256", checksum) + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return "", fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + return resp.Header.Get("ETag"), nil +} + +func (c *Client) completeUpload(ctx context.Context, uploadID string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/complete", nil) + if err != nil { + return err + } + + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// ReadStream reads a file, following 302 redirects for large files. +func (c *Client) ReadStream(ctx context.Context, path string) (io.ReadCloser, error) { + noRedirectClient := *c.httpClient + noRedirectClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.doWithClient(&noRedirectClient, req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect: + _ = resp.Body.Close() + location := resp.Header.Get("Location") + if location == "" { + return nil, fmt.Errorf("302 without Location header") + } + req2, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return nil, err + } + resp2, err := c.httpClient.Do(req2) + if err != nil { + return nil, err + } + if resp2.StatusCode >= 300 { + defer func() { _ = resp2.Body.Close() }() + return nil, readError(resp2) + } + return resp2.Body, nil + + case resp.StatusCode >= 300: + defer func() { _ = resp.Body.Close() }() + return nil, readError(resp) + + default: + return resp.Body, nil + } +} + +// ReadStreamRange reads a byte range from a remote file. +func (c *Client) ReadStreamRange(ctx context.Context, path string, offset, length int64) (io.ReadCloser, error) { + if length <= 0 { + return io.NopCloser(bytes.NewReader(nil)), nil + } + + noRedirectClient := *c.httpClient + noRedirectClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url(path), nil) + if err != nil { + return nil, err + } + resp, err := c.doWithClient(&noRedirectClient, req) + if err != nil { + return nil, err + } + + switch { + case resp.StatusCode == http.StatusFound || resp.StatusCode == http.StatusTemporaryRedirect: + _ = resp.Body.Close() + location := resp.Header.Get("Location") + if location == "" { + return nil, fmt.Errorf("302 without Location header") + } + req2, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil) + if err != nil { + return nil, err + } + req2.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) + resp2, err := c.httpClient.Do(req2) + if err != nil { + return nil, err + } + + switch resp2.StatusCode { + case http.StatusPartialContent: + return resp2.Body, nil + case http.StatusRequestedRangeNotSatisfiable: + defer func() { _ = resp2.Body.Close() }() + return io.NopCloser(bytes.NewReader(nil)), nil + default: + if resp2.StatusCode >= 300 { + defer func() { _ = resp2.Body.Close() }() + return nil, readError(resp2) + } + return c.sliceBody(resp2.Body, offset, length) + } + + case resp.StatusCode >= 300: + defer func() { _ = resp.Body.Close() }() + return nil, readError(resp) + + default: + return c.sliceBody(resp.Body, offset, length) + } +} + +func (c *Client) sliceBody(rc io.ReadCloser, offset, length int64) (io.ReadCloser, error) { + if offset > 0 { + if _, err := io.CopyN(io.Discard, rc, offset); err != nil { + _ = rc.Close() + if err == io.EOF { + return io.NopCloser(strings.NewReader("")), nil + } + return nil, fmt.Errorf("skip to offset: %w", err) + } + } + return &limitedReadCloser{r: io.LimitReader(rc, length), c: rc}, nil +} + +type limitedReadCloser struct { + r io.Reader + c io.Closer +} + +func (l *limitedReadCloser) Read(p []byte) (int, error) { return l.r.Read(p) } +func (l *limitedReadCloser) Close() error { return l.c.Close() } + +// UploadMeta is the server's response for querying active uploads. +type UploadMeta struct { + UploadID string `json:"upload_id"` + PartsTotal int `json:"parts_total"` + Status string `json:"status"` + ExpiresAt string `json:"expires_at"` +} + +// ResumeUpload queries for an incomplete upload and resumes it. +func (c *Client) ResumeUpload(ctx context.Context, path string, r io.ReaderAt, totalSize int64, progress ProgressFunc) error { + meta, err := c.queryUpload(ctx, path) + if err != nil { + return err + } + + checksums, err := computePartChecksumsFromReaderAt(r, totalSize, PartSize) + if err != nil { + return fmt.Errorf("compute part checksums: %w", err) + } + plan, err := c.requestResume(ctx, meta.UploadID, checksums) + if err != nil { + return err + } + + if len(plan.Parts) == 0 { + return c.completeUpload(ctx, plan.UploadID) + } + + if err := c.uploadMissingParts(ctx, *plan, r, meta.PartsTotal, progress); err != nil { + return err + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) queryUpload(ctx context.Context, path string) (*UploadMeta, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.baseURL+"/v1/uploads?path="+path+"&status=UPLOADING", nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + + var envelope struct { + Uploads []UploadMeta `json:"uploads"` + } + if err := json.NewDecoder(resp.Body).Decode(&envelope); err != nil { + return nil, fmt.Errorf("decode upload meta: %w", err) + } + if len(envelope.Uploads) == 0 { + return nil, fmt.Errorf("no active upload for %s", path) + } + return &envelope.Uploads[0], nil +} + +func (c *Client) requestResume(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, error) { + plan, resp, err := c.requestResumeByBody(ctx, uploadID, checksums) + if err == nil { + return plan, nil + } + if resp != nil { + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(err.Error()), "missing x-fs-part-checksums header") { + return c.requestResumeLegacy(ctx, uploadID, checksums) + } + return nil, err + } + return nil, err +} + +func (c *Client) requestResumeByBody(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, *http.Response, error) { + body, err := json.Marshal(uploadResumeRequest{PartChecksums: checksums}) + if err != nil { + return nil, nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/resume", bytes.NewReader(body)) + if err != nil { + return nil, nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, nil, err + } + + if resp.StatusCode == http.StatusGone { + _ = resp.Body.Close() + return nil, nil, fmt.Errorf("upload %s has expired", uploadID) + } + if resp.StatusCode >= 300 { + return nil, resp, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + _ = resp.Body.Close() + return nil, nil, fmt.Errorf("decode resume plan: %w", err) + } + _ = resp.Body.Close() + return &plan, nil, nil +} + +func (c *Client) requestResumeLegacy(ctx context.Context, uploadID string, checksums []string) (*UploadPlan, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/v1/uploads/"+uploadID+"/resume", nil) + if err != nil { + return nil, err + } + if len(checksums) > 0 { + req.Header.Set("X-FS-Part-Checksums", strings.Join(checksums, ",")) + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusGone { + return nil, fmt.Errorf("upload %s has expired", uploadID) + } + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + + var plan UploadPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return nil, fmt.Errorf("decode resume plan: %w", err) + } + return &plan, nil +} + +func (c *Client) uploadMissingParts(ctx context.Context, plan UploadPlan, r io.ReaderAt, totalParts int, progress ProgressFunc) error { + stdPartSize := plan.PartSize + if stdPartSize <= 0 { + stdPartSize = PartSize + } + maxConcurrency := uploadParallelism(stdPartSize) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + sem := make(chan struct{}, maxConcurrency) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + for _, part := range plan.Parts { + select { + case err := <-errCh: + cancel() + wg.Wait() + return err + default: + } + + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + wg.Add(1) + go func(p PartURL) { + defer wg.Done() + defer func() { <-sem }() + + data := make([]byte, p.Size) + offset := int64(p.Number-1) * stdPartSize + n, err := r.ReadAt(data, offset) + if err != nil && err != io.EOF { + select { + case errCh <- fmt.Errorf("read part %d at offset %d: %w", p.Number, offset, err): + default: + } + cancel() + return + } + if int64(n) != p.Size { + select { + case errCh <- fmt.Errorf("short read for part %d at offset %d: got %d want %d", p.Number, offset, n, p.Size): + default: + } + cancel() + return + } + + _, err = c.uploadOnePart(ctx, p, data) + if err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + cancel() + return + } + if progress != nil { + progress(p.Number, totalParts, int64(len(data))) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return nil +} + +func computePartChecksumsFromReaderAt(r io.ReaderAt, totalSize int64, partSize int64) ([]string, error) { + if totalSize <= 0 { + return nil, nil + } + parts := CalcParts(totalSize, partSize) + checksums := make([]string, len(parts)) + workers := checksumParallelism(partSize, len(parts)) + + var wg sync.WaitGroup + var firstErr error + var errOnce sync.Once + partCh := make(chan int, len(parts)) + + for i := range parts { + partCh <- i + } + close(partCh) + + for w := 0; w < workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + buf := make([]byte, partSize) + for i := range partCh { + p := parts[i] + data := buf[:p.Size] + offset := int64(p.Number-1) * partSize + n, err := r.ReadAt(data, offset) + if err != nil && err != io.EOF { + errOnce.Do(func() { firstErr = fmt.Errorf("read part %d: %w", p.Number, err) }) + return + } + if int64(n) != p.Size { + errOnce.Do(func() { + firstErr = fmt.Errorf("short read for part %d: got %d want %d", p.Number, n, p.Size) + }) + return + } + h := sha256.Sum256(data) + checksums[i] = base64.StdEncoding.EncodeToString(h[:]) + } + }() + } + wg.Wait() + + if firstErr != nil { + return nil, firstErr + } + return checksums, nil +} + +// PatchPlan mirrors the server's response for a PATCH request. +type PatchPlan struct { + UploadID string `json:"upload_id"` + PartSize int64 `json:"part_size"` + UploadParts []*PatchPartURL `json:"upload_parts"` + CopiedParts []int `json:"copied_parts"` +} + +// PatchPartURL describes one dirty part the client must upload. +type PatchPartURL struct { + Number int `json:"number"` + URL string `json:"url"` + Size int64 `json:"size"` + Headers map[string]string `json:"headers,omitempty"` + ExpiresAt string `json:"expires_at"` + ReadURL string `json:"read_url,omitempty"` + ReadHeaders map[string]string `json:"read_headers,omitempty"` +} + +// PatchFile performs a partial update of a large file. +func (c *Client) PatchFile(ctx context.Context, path string, newSize int64, dirtyParts []int, readPart func(partNumber int, partSize int64, origData []byte) ([]byte, error), progress ProgressFunc) error { + reqBody, err := json.Marshal(map[string]any{ + "new_size": newSize, + "dirty_parts": dirtyParts, + }) + if err != nil { + return fmt.Errorf("marshal patch request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, c.url(path), bytes.NewReader(reqBody)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusAccepted { + return readError(resp) + } + + var plan PatchPlan + if err := json.NewDecoder(resp.Body).Decode(&plan); err != nil { + return fmt.Errorf("decode patch plan: %w", err) + } + + const maxConcurrency = 4 + sem := make(chan struct{}, maxConcurrency) + errCh := make(chan error, 1) + var wg sync.WaitGroup + + totalParts := len(plan.UploadParts) + len(plan.CopiedParts) + + for _, part := range plan.UploadParts { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + } + + select { + case err := <-errCh: + wg.Wait() + return err + default: + } + + wg.Add(1) + go func(p *PatchPartURL) { + defer wg.Done() + defer func() { <-sem }() + + if err := c.uploadPatchPart(ctx, p, readPart); err != nil { + select { + case errCh <- fmt.Errorf("part %d: %w", p.Number, err): + default: + } + return + } + + if progress != nil { + progress(p.Number, totalParts, p.Size) + } + }(part) + } + + wg.Wait() + + select { + case err := <-errCh: + return err + default: + } + + return c.completeUpload(ctx, plan.UploadID) +} + +func (c *Client) uploadPatchPart(ctx context.Context, part *PatchPartURL, readPart func(int, int64, []byte) ([]byte, error)) error { + var origData []byte + if part.ReadURL != "" { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, part.ReadURL, nil) + if err != nil { + return fmt.Errorf("create read request: %w", err) + } + for k, v := range part.ReadHeaders { + if !strings.EqualFold(k, "host") { + req.Header.Set(k, v) + } + } + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("download original part: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return fmt.Errorf("download original part: HTTP %d", resp.StatusCode) + } + origData, err = io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("read original part body: %w", err) + } + } + + data, err := readPart(part.Number, part.Size, origData) + if err != nil { + return fmt.Errorf("readPart callback: %w", err) + } + + h := sha256.Sum256(data) + checksum := base64.StdEncoding.EncodeToString(h[:]) + + req, err := http.NewRequestWithContext(ctx, http.MethodPut, part.URL, bytes.NewReader(data)) + if err != nil { + return err + } + for k, v := range part.Headers { + if strings.EqualFold(k, "host") { + continue + } + req.Header.Set(k, v) + } + req.ContentLength = int64(len(data)) + req.Header.Set("x-amz-checksum-sha256", checksum) + + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload part: HTTP %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// VaultSecret holds secret metadata returned by the management API. +type VaultSecret struct { + Name string `json:"name"` + SecretType string `json:"secret_type"` + Revision int64 `json:"revision"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// VaultTokenIssueResponse is returned when issuing a scoped capability token. +type VaultTokenIssueResponse struct { + Token string `json:"token"` + TokenID string `json:"token_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +// VaultAuditEvent is an audit event returned by the vault audit API. +type VaultAuditEvent struct { + EventID string `json:"event_id"` + EventType string `json:"event_type"` + TokenID string `json:"token_id,omitempty"` + AgentID string `json:"agent_id,omitempty"` + TaskID string `json:"task_id,omitempty"` + SecretName string `json:"secret_name,omitempty"` + FieldName string `json:"field_name,omitempty"` + Adapter string `json:"adapter,omitempty"` + Detail map[string]any `json:"detail,omitempty"` + Timestamp time.Time `json:"timestamp"` +} + +func (c *Client) vaultURL(path string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + return c.baseURL + "/v1/vault" + path +} + +// CreateVaultSecret creates a new secret via the management API. +func (c *Client) CreateVaultSecret(ctx context.Context, name string, fields map[string]string) (*VaultSecret, error) { + body, err := json.Marshal(map[string]any{ + "name": name, + "fields": fields, + "created_by": "ticloud-cli", + }) + if err != nil { + return nil, fmt.Errorf("marshal secret create request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.vaultURL("/secrets"), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var sec VaultSecret + if err := json.NewDecoder(resp.Body).Decode(&sec); err != nil { + return nil, fmt.Errorf("decode secret create response: %w", err) + } + return &sec, nil +} + +// UpdateVaultSecret rotates a secret via the management API. +func (c *Client) UpdateVaultSecret(ctx context.Context, name string, fields map[string]string) (*VaultSecret, error) { + body, err := json.Marshal(map[string]any{ + "fields": fields, + "updated_by": "ticloud-cli", + }) + if err != nil { + return nil, fmt.Errorf("marshal secret update request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, c.vaultURL("/secrets/"+url.PathEscape(name)), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var sec VaultSecret + if err := json.NewDecoder(resp.Body).Decode(&sec); err != nil { + return nil, fmt.Errorf("decode secret update response: %w", err) + } + return &sec, nil +} + +// DeleteVaultSecret deletes a secret via the management API. +func (c *Client) DeleteVaultSecret(ctx context.Context, name string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.vaultURL("/secrets/"+url.PathEscape(name)), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// ListVaultSecrets lists secret metadata via the management API. +func (c *Client) ListVaultSecrets(ctx context.Context) ([]VaultSecret, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/secrets"), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Secrets []VaultSecret `json:"secrets"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode secret list response: %w", err) + } + if result.Secrets == nil { + result.Secrets = []VaultSecret{} + } + return result.Secrets, nil +} + +// IssueVaultToken issues a scoped capability token via the management API. +func (c *Client) IssueVaultToken(ctx context.Context, agentID, taskID string, scope []string, ttl time.Duration) (*VaultTokenIssueResponse, error) { + ttlSeconds := int(ttl / time.Second) + body, err := json.Marshal(map[string]any{ + "agent_id": agentID, + "task_id": taskID, + "scope": scope, + "ttl_seconds": ttlSeconds, + }) + if err != nil { + return nil, fmt.Errorf("marshal token issue request: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.vaultURL("/tokens"), bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result VaultTokenIssueResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode token issue response: %w", err) + } + return &result, nil +} + +// RevokeVaultToken revokes a capability token via the management API. +func (c *Client) RevokeVaultToken(ctx context.Context, tokenID string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, c.vaultURL("/tokens/"+url.PathEscape(tokenID)), nil) + if err != nil { + return err + } + resp, err := c.do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return readError(resp) + } + return nil +} + +// QueryVaultAudit queries the audit log via the management API. +func (c *Client) QueryVaultAudit(ctx context.Context, secretName string, limit int) ([]VaultAuditEvent, error) { + u, err := url.Parse(c.vaultURL("/audit")) + if err != nil { + return nil, err + } + q := u.Query() + if secretName != "" { + q.Set("secret", secretName) + } + if limit > 0 { + q.Set("limit", fmt.Sprintf("%d", limit)) + } + u.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Events []VaultAuditEvent `json:"events"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode audit response: %w", err) + } + if result.Events == nil { + result.Events = []VaultAuditEvent{} + } + return result.Events, nil +} + +// ListReadableVaultSecrets enumerates secrets visible to the bearer capability token. +func (c *Client) ListReadableVaultSecrets(ctx context.Context) ([]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read"), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result struct { + Secrets []string `json:"secrets"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode readable secret list response: %w", err) + } + if result.Secrets == nil { + result.Secrets = []string{} + } + return result.Secrets, nil +} + +// ReadVaultSecret reads all fields of a secret using the consumption API. +func (c *Client) ReadVaultSecret(ctx context.Context, name string) (map[string]string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read/"+url.PathEscape(name)), nil) + if err != nil { + return nil, err + } + resp, err := c.do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return nil, readError(resp) + } + var result map[string]string + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode secret read response: %w", err) + } + if result == nil { + result = map[string]string{} + } + return result, nil +} + +// ReadVaultSecretField reads a single field via the consumption API. +func (c *Client) ReadVaultSecretField(ctx context.Context, name, field string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.vaultURL("/read/"+url.PathEscape(name)+"/"+url.PathEscape(field)), nil) + if err != nil { + return "", err + } + resp, err := c.do(req) + if err != nil { + return "", err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode >= 300 { + return "", readError(resp) + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("read field response: %w", err) + } + return string(data), nil +} diff --git a/internal/service/fs/client_auth_test.go b/internal/service/fs/client_auth_test.go new file mode 100644 index 00000000..1ae12452 --- /dev/null +++ b/internal/service/fs/client_auth_test.go @@ -0,0 +1,133 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fs + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestClientInjectsZeroInstanceIDHeader(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + instanceID := "test-instance-123" + client := NewClient(server.URL, &http.Client{}, "", instanceID) + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders == nil { + t.Fatal("No request was made to server") + } + + instanceHeader := capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") + if instanceHeader != instanceID { + t.Errorf("X-TIDBCLOUD-ZERO-INSTANCE-ID header = %q, want %q", instanceHeader, instanceID) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != "" { + t.Error("X-TIDBCLOUD-CLUSTER-ID should not be set when zero instance ID is used") + } +} + +func TestClientInjectsClusterIDHeader(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + clusterID := "test-cluster-456" + client := NewClient(server.URL, &http.Client{}, clusterID, "") + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders == nil { + t.Fatal("No request was made to server") + } + + clusterHeader := capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") + if clusterHeader != clusterID { + t.Errorf("X-TIDBCLOUD-CLUSTER-ID header = %q, want %q", clusterHeader, clusterID) + } + + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when cluster ID is used") + } +} + +func TestClientClusterIDWinsOverZeroInstanceID(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + clusterID := "test-cluster-456" + instanceID := "test-instance-123" + client := NewClient(server.URL, &http.Client{}, clusterID, instanceID) + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != clusterID { + t.Errorf("X-TIDBCLOUD-CLUSTER-ID header = %q, want %q", capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID"), clusterID) + } + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when both IDs are provided (cluster wins)") + } +} + +func TestClientNoDBHeadersWhenNotConfigured(t *testing.T) { + var capturedHeaders http.Header + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedHeaders = r.Header + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"entries": []}`)) + })) + defer server.Close() + + client := NewClient(server.URL, &http.Client{}, "", "") + + _, err := client.List("/") + if err != nil { + t.Logf("Expected error: %v", err) + } + + if capturedHeaders.Get("X-TIDBCLOUD-CLUSTER-ID") != "" { + t.Error("X-TIDBCLOUD-CLUSTER-ID should not be set when not configured") + } + if capturedHeaders.Get("X-TIDBCLOUD-ZERO-INSTANCE-ID") != "" { + t.Error("X-TIDBCLOUD-ZERO-INSTANCE-ID should not be set when not configured") + } +} diff --git a/internal/service/fs/fuse/fs.go b/internal/service/fs/fuse/fs.go new file mode 100644 index 00000000..4ec2747e --- /dev/null +++ b/internal/service/fs/fuse/fs.go @@ -0,0 +1,503 @@ +//go:build !windows + +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fuse + +import ( + "context" + "io" + "os" + "path/filepath" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + gofuse "github.com/hanwen/go-fuse/v2/fuse" +) + +// ReadOnlyFS is a read-only FUSE filesystem for TiDB Cloud FS. +type ReadOnlyFS struct { + gofuse.RawFileSystem + + client *fs.Client + opts *MountOptions + inodes *inodeManager + dirs *dirCache + uid uint32 + gid uint32 +} + +// inodeManager manages inode to path mappings. +type inodeManager struct { + mu sync.RWMutex + byInode map[uint64]*inodeEntry + byPath map[string]uint64 + nextIno uint64 +} + +type inodeEntry struct { + Path string + IsDir bool + RefCnt uint64 + Attr *fs.StatResult + AttrTime time.Time +} + +// dirCache caches directory listings. +type dirCache struct { + mu sync.RWMutex + entries map[string]*dirCacheEntry + ttl time.Duration +} + +type dirCacheEntry struct { + Items []fs.FileInfo + Timestamp time.Time +} + +// NewReadOnlyFS creates a new read-only FUSE filesystem. +func NewReadOnlyFS(client *fs.Client, opts *MountOptions) *ReadOnlyFS { + fsys := &ReadOnlyFS{ + RawFileSystem: gofuse.NewDefaultRawFileSystem(), + client: client, + opts: opts, + inodes: &inodeManager{ + byInode: make(map[uint64]*inodeEntry), + byPath: make(map[string]uint64), + nextIno: gofuse.FUSE_ROOT_ID, + }, + dirs: &dirCache{ + entries: make(map[string]*dirCacheEntry), + ttl: 5 * time.Second, + }, + uid: uint32(os.Getuid()), + gid: uint32(os.Getgid()), + } + + // Initialize root inode + fsys.inodes.byPath["/"] = gofuse.FUSE_ROOT_ID + fsys.inodes.byInode[gofuse.FUSE_ROOT_ID] = &inodeEntry{ + Path: "/", + IsDir: true, + RefCnt: 1, + } + + return fsys +} + +// GetAttr implements GetAttr. +func (f *ReadOnlyFS) GetAttr(cancel <-chan struct{}, in *gofuse.GetAttrIn, out *gofuse.AttrOut) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + // Check cache + if entry.Attr != nil && time.Since(entry.AttrTime) < time.Second { + f.fillAttr(entry, out) + return gofuse.OK + } + + // Special-case root directory: backend may not support Stat("/") + if entry.Path == "/" { + entry.IsDir = true + entry.Attr = &fs.StatResult{IsDir: true, Size: 0} + entry.AttrTime = time.Now() + f.fillAttr(entry, out) + return gofuse.OK + } + + // Fetch from server + stat, err := f.client.Stat(entry.Path) + if err != nil { + return gofuse.ENOENT + } + + entry.Attr = stat + entry.AttrTime = time.Now() + entry.IsDir = stat.IsDir + + f.fillAttr(entry, out) + return gofuse.OK +} + +func (f *ReadOnlyFS) fillAttr(entry *inodeEntry, out *gofuse.AttrOut) { + out.Attr.Ino = f.inodes.getIno(entry.Path) + if entry.Attr != nil { + out.Attr.Size = uint64(entry.Attr.Size) + } else { + out.Attr.Size = 0 + } + out.Attr.Mode = 0o755 + if entry.IsDir { + out.Attr.Mode |= syscall.S_IFDIR + out.Attr.Nlink = 2 + } else { + out.Attr.Mode |= syscall.S_IFREG + out.Attr.Nlink = 1 + } + out.Attr.Uid = f.uid + out.Attr.Gid = f.gid + out.SetTimeout(1 * time.Second) +} + +// Lookup implements Lookup. +func (f *ReadOnlyFS) Lookup(cancel <-chan struct{}, header *gofuse.InHeader, name string, out *gofuse.EntryOut) gofuse.Status { + parentIno := header.NodeId + + f.inodes.mu.RLock() + parentEntry, ok := f.inodes.byInode[parentIno] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + path := filepath.Join(parentEntry.Path, name) + + // Get or create inode + ino := f.inodes.GetOrCreate(path) + + f.inodes.mu.RLock() + entry := f.inodes.byInode[ino] + f.inodes.mu.RUnlock() + + // Fetch attr + stat, err := f.client.Stat(path) + if err != nil { + return gofuse.ENOENT + } + + entry.Attr = stat + entry.AttrTime = time.Now() + entry.IsDir = stat.IsDir + + out.NodeId = ino + out.Generation = 1 + out.Attr.Ino = ino + if entry.Attr != nil { + out.Attr.Size = uint64(entry.Attr.Size) + } else { + out.Attr.Size = 0 + } + out.Attr.Mode = 0o755 + if entry.IsDir { + out.Attr.Mode |= syscall.S_IFDIR + } else { + out.Attr.Mode |= syscall.S_IFREG + } + out.Attr.Nlink = 1 + out.Attr.Uid = f.uid + out.Attr.Gid = f.gid + + return gofuse.OK +} + +// Access implements Access. +func (f *ReadOnlyFS) Access(cancel <-chan struct{}, in *gofuse.AccessIn) gofuse.Status { + return gofuse.OK +} + +// OpenDir implements OpenDir. +func (f *ReadOnlyFS) OpenDir(cancel <-chan struct{}, in *gofuse.OpenIn, out *gofuse.OpenOut) gofuse.Status { + out.Fh = in.NodeId + return gofuse.OK +} + +// ReadDir implements ReadDir. +func (f *ReadOnlyFS) ReadDir(cancel <-chan struct{}, in *gofuse.ReadIn, out *gofuse.DirEntryList) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + realEntries, err := f.getDirEntries(entry.Path) + if err != nil { + return gofuse.EIO + } + + dotdotIno := in.NodeId + if entry.Path != "/" { + parentPath := filepath.Dir(entry.Path) + dotdotIno = f.inodes.getIno(parentPath) + if dotdotIno == 0 { + dotdotIno = f.inodes.GetOrCreate(parentPath) + } + } + + type dirItem struct { + name string + ino uint64 + mode uint32 + } + + items := make([]dirItem, 0, 2+len(realEntries)) + items = append(items, dirItem{".", in.NodeId, syscall.S_IFDIR | 0o555}) + items = append(items, dirItem{"..", dotdotIno, syscall.S_IFDIR | 0o555}) + for _, e := range realEntries { + if e.Name == "" || e.Name == ":" { + continue + } + childPath := filepath.Join(entry.Path, e.Name) + childIno := f.inodes.GetOrCreate(childPath) + mode := uint32(0o755) + if e.IsDir { + mode |= syscall.S_IFDIR + } else { + mode |= syscall.S_IFREG + } + items = append(items, dirItem{e.Name, childIno, mode}) + } + + offset := int(in.Offset) + for i, item := range items { + if i < offset { + continue + } + de := gofuse.DirEntry{ + Name: item.name, + Ino: item.ino, + Mode: item.mode, + Off: uint64(i + 1), + } + if !out.AddDirEntry(de) { + break + } + } + + return gofuse.OK +} + +// ReadDirPlus implements ReadDirPlus. +func (f *ReadOnlyFS) ReadDirPlus(cancel <-chan struct{}, in *gofuse.ReadIn, out *gofuse.DirEntryList) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + realEntries, err := f.getDirEntries(entry.Path) + if err != nil { + return gofuse.EIO + } + + dotdotIno := in.NodeId + if entry.Path != "/" { + parentPath := filepath.Dir(entry.Path) + dotdotIno = f.inodes.getIno(parentPath) + if dotdotIno == 0 { + dotdotIno = f.inodes.GetOrCreate(parentPath) + } + } + + type dirItem struct { + name string + ino uint64 + mode uint32 + } + + items := make([]dirItem, 0, 2+len(realEntries)) + items = append(items, dirItem{".", in.NodeId, syscall.S_IFDIR | 0o555}) + items = append(items, dirItem{"..", dotdotIno, syscall.S_IFDIR | 0o555}) + for _, e := range realEntries { + if e.Name == "" || e.Name == ":" { + continue + } + childPath := filepath.Join(entry.Path, e.Name) + childIno := f.inodes.GetOrCreate(childPath) + mode := uint32(0o755) + if e.IsDir { + mode |= syscall.S_IFDIR + } else { + mode |= syscall.S_IFREG + } + items = append(items, dirItem{e.Name, childIno, mode}) + } + + offset := int(in.Offset) + for i, item := range items { + if i < offset { + continue + } + de := gofuse.DirEntry{ + Name: item.name, + Ino: item.ino, + Mode: item.mode, + Off: uint64(i + 1), + } + entryOut := out.AddDirLookupEntry(de) + if entryOut == nil { + break + } + entryOut.NodeId = item.ino + entryOut.Generation = 1 + entryOut.Attr.Ino = item.ino + entryOut.Attr.Size = 0 + if item.mode&syscall.S_IFDIR != 0 { + entryOut.Attr.Mode = syscall.S_IFDIR | 0o755 + } else { + entryOut.Attr.Mode = syscall.S_IFREG | 0o755 + } + if item.mode&syscall.S_IFDIR != 0 { + entryOut.Attr.Nlink = 2 + } else { + entryOut.Attr.Nlink = 1 + } + entryOut.Attr.Uid = f.uid + entryOut.Attr.Gid = f.gid + entryOut.SetEntryTimeout(1 * time.Second) + entryOut.SetAttrTimeout(1 * time.Second) + } + + return gofuse.OK +} + +// Open implements Open. +func (f *ReadOnlyFS) Open(cancel <-chan struct{}, in *gofuse.OpenIn, out *gofuse.OpenOut) gofuse.Status { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return gofuse.ENOENT + } + + if entry.IsDir { + return gofuse.EISDIR + } + + out.Fh = in.NodeId + out.OpenFlags = gofuse.FOPEN_KEEP_CACHE + + return gofuse.OK +} + +// Read implements Read. +func (f *ReadOnlyFS) Read(cancel <-chan struct{}, in *gofuse.ReadIn, buf []byte) (gofuse.ReadResult, gofuse.Status) { + f.inodes.mu.RLock() + entry, ok := f.inodes.byInode[in.NodeId] + f.inodes.mu.RUnlock() + + if !ok { + return nil, gofuse.ENOENT + } + + reader, err := f.client.ReadStreamRange(context.Background(), entry.Path, int64(in.Offset), int64(len(buf))) + if err != nil { + return nil, gofuse.EIO + } + defer reader.Close() + + n, err := reader.Read(buf) + if err != nil && err != io.EOF { + return nil, gofuse.EIO + } + + return gofuse.ReadResultData(buf[:n]), gofuse.OK +} + +// Release implements Release. +func (f *ReadOnlyFS) Release(cancel <-chan struct{}, in *gofuse.ReleaseIn) { + // Nothing to do for read-only +} + +// ReleaseDir implements ReleaseDir. +func (f *ReadOnlyFS) ReleaseDir(in *gofuse.ReleaseIn) { + // Nothing to do +} + +// Forget implements Forget. +func (f *ReadOnlyFS) Forget(nodeID uint64, nlookup uint64) { + f.inodes.Forget(nodeID, nlookup) +} + +// getDirEntries gets directory entries with caching. +func (f *ReadOnlyFS) getDirEntries(path string) ([]fs.FileInfo, error) { + f.dirs.mu.RLock() + cache, ok := f.dirs.entries[path] + f.dirs.mu.RUnlock() + + if ok && time.Since(cache.Timestamp) < f.dirs.ttl { + return cache.Items, nil + } + + entries, err := f.client.List(path) + if err != nil { + return nil, err + } + + f.dirs.mu.Lock() + f.dirs.entries[path] = &dirCacheEntry{ + Items: entries, + Timestamp: time.Now(), + } + f.dirs.mu.Unlock() + + return entries, nil +} + +// inodeManager methods + +func (im *inodeManager) GetOrCreate(path string) uint64 { + im.mu.Lock() + defer im.mu.Unlock() + + if ino, ok := im.byPath[path]; ok { + return ino + } + + ino := atomic.AddUint64(&im.nextIno, 1) + im.byPath[path] = ino + im.byInode[ino] = &inodeEntry{ + Path: path, + RefCnt: 1, + } + return ino +} + +func (im *inodeManager) getIno(path string) uint64 { + im.mu.RLock() + defer im.mu.RUnlock() + return im.byPath[path] +} + +func (im *inodeManager) Forget(ino uint64, nlookup uint64) { + im.mu.Lock() + defer im.mu.Unlock() + + entry, ok := im.byInode[ino] + if !ok { + return + } + + if entry.RefCnt <= nlookup { + delete(im.byInode, ino) + delete(im.byPath, entry.Path) + } else { + entry.RefCnt -= nlookup + } +} diff --git a/internal/service/fs/fuse/mount.go b/internal/service/fs/fuse/mount.go new file mode 100644 index 00000000..a6a933c1 --- /dev/null +++ b/internal/service/fs/fuse/mount.go @@ -0,0 +1,109 @@ +//go:build !windows + +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fuse provides FUSE filesystem implementation for TiDB Cloud FS. +package fuse + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/tidbcloud/tidbcloud-cli/internal/service/fs" + + gofuse "github.com/hanwen/go-fuse/v2/fuse" +) + +// MountOptions configures the FUSE mount. +type MountOptions struct { + Client *fs.Client // FS client (with auth and headers configured) + MountPoint string // Local mount point + ReadOnly bool // Mount as read-only (MVP default) + Debug bool // Enable FUSE debug logging + AllowOther bool // Allow other users to access mount +} + +// Validate checks the mount options. +func (o *MountOptions) Validate() error { + if o.Client == nil { + return fmt.Errorf("FS client is required") + } + if o.MountPoint == "" { + return fmt.Errorf("mount point is required") + } + return nil +} + +// Mount creates and serves a FUSE mount. It blocks until the filesystem +// is unmounted or a signal (SIGINT, SIGTERM) is received. +func Mount(opts *MountOptions) error { + if err := opts.Validate(); err != nil { + return err + } + + // Create mount point if not exists + if err := os.MkdirAll(opts.MountPoint, 0o755); err != nil { + return fmt.Errorf("create mount point: %w", err) + } + + // Test connection + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := opts.Client.List("/"); err != nil { + return fmt.Errorf("cannot reach FS server: %w", err) + } + _ = ctx + + // Create FUSE filesystem + fsys := NewReadOnlyFS(opts.Client, opts) + + // Configure mount options + fuseOpts := &gofuse.MountOptions{ + FsName: "ticloudfs", + Name: "ticloudfs", + MaxReadAhead: 128 * 1024, + Debug: opts.Debug, + AllowOther: opts.AllowOther, + } + if opts.ReadOnly { + fuseOpts.Options = append(fuseOpts.Options, "ro") + } + + // Create FUSE server + server, err := gofuse.NewServer(fsys, opts.MountPoint, fuseOpts) + if err != nil { + return fmt.Errorf("fuse mount: %w", err) + } + + // Handle signals + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + + go func() { + <-sigCh + fmt.Fprintf(os.Stderr, "\nUnmounting %s...\n", opts.MountPoint) + if err := server.Unmount(); err != nil { + fmt.Fprintf(os.Stderr, "Unmount error: %v\n", err) + } + }() + + fmt.Fprintf(os.Stderr, "Mounted on %s (server: %s)\n", opts.MountPoint, opts.Client.GetBaseURL()) + server.Serve() + return nil +}