From fccf9d1455b70a0799ea47565bf1b7291a7c5a7d Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 10:31:53 +0800 Subject: [PATCH 01/11] feat: migrate secrets to SOPS encryption - Add SOPS + Age encryption for Helm chart secrets (datamate, label-studio, milvus) - Remove plaintext passwords from values.yaml files - Replace hardcoded secrets in docker-compose.yml with env vars - Add .env.example template for Docker deployment path - Add scripts/secrets.sh helper for encrypt/decrypt/helm-install - Add docs/SECRETS_SETUP.md setup guide - Update .gitignore: exclude .sops-keys/, allow .env.example and .env.enc Deployment: - K8s/Helm: helm secrets install -f secrets.yaml - Docker: cp .env.example .env && edit && docker compose up --- .gitignore | 9 +- .sops.yaml | 3 + deployment/docker/datamate/.env.example | 29 +++ deployment/docker/datamate/docker-compose.yml | 22 +-- deployment/helm/datamate/secrets.yaml | 26 +++ deployment/helm/datamate/values.yaml | 6 +- deployment/helm/label-studio/secrets.yaml | 19 ++ deployment/helm/label-studio/values.yaml | 6 +- deployment/helm/milvus/secrets.yaml | 18 ++ deployment/helm/milvus/values.yaml | 4 +- docs/SECRETS_SETUP.md | 166 ++++++++++++++++++ scripts/secrets.sh | 82 +++++++++ 12 files changed, 370 insertions(+), 20 deletions(-) create mode 100644 .sops.yaml create mode 100644 deployment/docker/datamate/.env.example create mode 100644 deployment/helm/datamate/secrets.yaml create mode 100644 deployment/helm/label-studio/secrets.yaml create mode 100644 deployment/helm/milvus/secrets.yaml create mode 100644 docs/SECRETS_SETUP.md create mode 100755 scripts/secrets.sh diff --git a/.gitignore b/.gitignore index 9b34b7af4..d78ab5ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,11 @@ Thumbs.db # Milvus **/volumes/ -**/rag_storage/ \ No newline at end of file +**/rag_storage/ +# SOPS encryption keys +.sops-keys/ + +# Environment files - ignore local .env, but allow templates and encrypted files +.env +!.env.example +!.env.enc diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 000000000..efb2f00f7 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example new file mode 100644 index 000000000..90fa6588e --- /dev/null +++ b/deployment/docker/datamate/.env.example @@ -0,0 +1,29 @@ +# DataMate Environment Variables Template +# Copy this file to .env and fill in the values +# cp .env.example .env +# IMPORTANT: Never commit .env to git! It is already in .gitignore. +# +# For K8s/Helm deployment: secrets are managed via SOPS-encrypted secrets.yaml files. +# For Docker deployment: use this .env file (gitignored, local only). + +# Database +DB_PASSWORD=your-secure-password-here + +# JWT Authentication +JWT_SECRET=your-secure-jwt-secret-here +DATAMATE_JWT_ENABLE=false + +# MinIO (for Milvus storage) +MINIO_ACCESS_KEY=your-minio-access-key +MINIO_SECRET_KEY=your-minio-secret-key + +# Label Studio +LABEL_STUDIO_PASSWORD=your-labelstudio-password +LABEL_STUDIO_USER_TOKEN=your-labelstudio-token +LABEL_STUDIO_HOST= + +# Optional: SSL Certificate Password (for encrypted private keys) +CERT_PASS= + +# Optional: Domain for HTTPS +DOMAIN= \ No newline at end of file diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index dc5a8470d..3ab169c0d 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -9,7 +9,7 @@ services: restart: on-failure privileged: true environment: - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset @@ -30,7 +30,7 @@ services: - "18000:18000" environment: - log_level=DEBUG - - pgsql_password=${DB_PASSWORD:-password} + - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} volumes: @@ -52,7 +52,7 @@ services: ports: - '8080:8080' environment: - - JWT_SECRET=default-insecure-key-change-in-production + - JWT_SECRET=${JWT_SECRET:-} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} networks: [ datamate ] @@ -75,7 +75,7 @@ services: restart: on-failure environment: - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=${DB_PASSWORD:-password} + - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} volumes: - postgresql_volume:/var/lib/postgresql/data - database_log_volume:/var/log/datamate/database @@ -93,7 +93,7 @@ services: PG_HOST: "datamate-database" PG_PORT: "5432" PG_USER: "postgres" - PG_PASSWORD: ${DB_PASSWORD:-password} + PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} PG_DATABASE: "datamate" command: - python @@ -213,7 +213,7 @@ services: - DB_PORT=5432 - DB_NAME=labelstudio - DB_USER=postgres - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - AUTH_TYPE=scram-sha-256 - POOL_MODE=transaction - MAX_CLIENT_CONN=100 @@ -241,7 +241,7 @@ services: - DJANGO_DB=default - POSTGRE_NAME=labelstudio - POSTGRE_USER=postgres - - POSTGRE_PASSWORD=${DB_PASSWORD:-password} + - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - POSTGRE_PORT=5432 - POSTGRE_HOST=label-studio-pgbouncer - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-} @@ -249,9 +249,9 @@ services: - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local - USE_USERNAME_FOR_LOGIN=true - LABEL_STUDIO_USERNAME=admin@demo.com - - LABEL_STUDIO_PASSWORD=demoadmin + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true - - LABEL_STUDIO_USER_TOKEN=abc123abc123 + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LOG_LEVEL=DEBUG volumes: - label-studio-data:/label-studio/data:rw @@ -290,8 +290,8 @@ services: container_name: milvus-minio image: minio/minio:RELEASE.2024-12-18T13-15-44Z environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} ports: - "9001:9001" - "9000:9000" diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml new file mode 100644 index 000000000..5717c25f8 --- /dev/null +++ b/deployment/helm/datamate/secrets.yaml @@ -0,0 +1,26 @@ +public: + secrets: + data: + DB_PASSWORD: ENC[AES256_GCM,data:vX907qq/Iu/1v2IWAXYKKN9P2PkFC992GyF1fZT1gmmLdyfWtkARTbmHgqM=,iv:wHWaaQT5m9a34LN9NZiEOxrtwtUnd4y056h6WxlurYc=,tag:7inilRxi9QJY2slkF6P9jg==,type:str] + CERT_PASS: "" + DOMAIN: "" + HOME_PAGE_URL: "" +gateway: + env: + - name: ENC[AES256_GCM,data:yvymNzt7F3oJPA==,iv:JPJOJ+y4dvnXNQYpchL66MRNNrxidMTxyfEgbi30tpA=,tag:mJ9aPHXR61ybkNvpIxqJfw==,type:str] + value: ENC[AES256_GCM,data:om/hHL6wx7Bp2Xud2Te/jsOC8jakLCJOfzeuyZqGxjutK66+v//kjjnkTkY=,iv:WUvXTmTSfEZh4aOMRhtFH1ceRsuwtnm8pv9/qxG8K0U=,tag:pqUJhS+HYtsmBQqKXq5nJw==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYVY2ekE4RWZta3ZIdTVm + eXZTOXViaUpxUGlSNjAyUVk4M2NpZ1lBVWs0ClJMaUVPRXMxbW9OU3F1T0kxSlA0 + VENSZ0o2MW9GTmVQSWwyT0h0cFJoWUUKLS0tIHZGcHZzWTArK01DbVBJc21Ib2ZD + cnpCK01CZkdEaUJVS0NpVUc1NFlHTm8KZcEMm+q1Y4Zb0XJ28soc+cuPluejOs28 + WRB9Jde2Ext6b4LN05ZJ/BkH0LZPYneB7ZhuN+a2o1p2Xc3bjZrO5w== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:22Z" + mac: ENC[AES256_GCM,data:y9aHJfy+QXoNnIUcrdtiubMKAS4JiyAKFE33M3/2REz4D+v/9AKV258c2Q3U7BjGZOaW3wgv0CGbaHWurmYZ5xjwemv6Ev60ozt1Oy3BG9fG29ckcQooeNT/FiavjFJ7Xw72Kyj1mgDf+4cgfbNn5Wdqa9TF15o0vTnBxjC7v00=,iv:7xIH0txZCT/zuBFqrzOcESNice8QAlAzDK7dEqmSn/w=,tag:TPI6at/w+xKLkLvCvgMKHg==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index b19959a70..935de2283 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -39,8 +39,8 @@ public: operator: 1Gi secrets: data: - DB_PASSWORD: "password" - CERT_PASS: "" + DB_PASSWORD: "" # Set via secrets.yaml or --set + CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys DOMAIN: "" HOME_PAGE_URL: "" @@ -166,7 +166,7 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "default-insecure-key-change-in-production" + value: "" # Set via secrets.yaml or --set for production - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED diff --git a/deployment/helm/label-studio/secrets.yaml b/deployment/helm/label-studio/secrets.yaml new file mode 100644 index 000000000..da2071797 --- /dev/null +++ b/deployment/helm/label-studio/secrets.yaml @@ -0,0 +1,19 @@ +env: + POSTGRE_PASSWORD: ENC[AES256_GCM,data:ROFSuT5BkJxGhusq2mGwxuBulcTNQeaVELxP6KimajWkGYBBys9oMJyPXHE=,iv:qqPQMjMUKpBJ76RlscGQ/6ff31e4iRDM0+IJ/q/TLYs=,tag:b5gCVSN/rn7aahqsoowayw==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:niFPf5mPSt4J2CpIhYaS+gUkj3XnwvkUy7VCWPykGd0=,iv:vm1OC2vRdBgHqjXPL/4th9EiSzZg9nGEDOn33axY1y4=,tag:wKGLsepNKasU4ccXv/OWfw==,type:str] + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:dmdU+7/Yd04jMCX5gaj4BpZhudCS2Bwe3FjqW40fX7g=,iv:GZS6oGS8oSBKYMjxBVuVmjEW0Ue/j9dfh50VBC9GvnI=,tag:GBhKPnPNyuaUaiXNfr4JEg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDUGVacVpQSjFwRFQvWllZ + bWlLZitoQ2JLS0pHS2ZQcW91T3RaRVBIU2t3ClVBZkN4N2FmRENPYVo0dU9RWnlD + MVZXNUhjSWFSZWo0Yjg0RTJSSFpGU3MKLS0tIGcvcllDK0FtREphOU9YckhvQlJQ + NE8vVXdqajFHMnBnVHhQSGF3dEdkWnMKTy7oYsfbWxr3U6X1gyS2kKJLxIjy0Zr0 + 1BfP9FAx6CvrveKqXGWwHk8+8034sG0fKO1H1lrBAgzW79eDOU4i+Q== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:39Z" + mac: ENC[AES256_GCM,data:Gni4TWxwzTmjf2+/t64V/SRBttJFuq1qMQ7cZzm2tJu/3pwbFOgcJwDdkoYSRKZrpyngj5EskbWzIgbmv0I9Jts26m9dN4pMr45qxQ1ba2G8eYgx0s2klh7iEFoMVTJmnYCbnl0PuqSWmcHbCxxZdeyo3ja647RlfLarT8QZwvU=,iv:nrDwjR0v96jKSJfQ812fpfrmI6vQEcCzttTVX8mLftU=,tag:AP+JhPFE7CZY5aZX5M8O7w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/label-studio/values.yaml b/deployment/helm/label-studio/values.yaml index 229098ee5..456c6f242 100644 --- a/deployment/helm/label-studio/values.yaml +++ b/deployment/helm/label-studio/values.yaml @@ -41,7 +41,7 @@ env: DJANGO_DB: "default" POSTGRE_NAME: "labelstudio" POSTGRE_USER: "postgres" - POSTGRE_PASSWORD: "password" + POSTGRE_PASSWORD: "" # Set via secrets.yaml or --set POSTGRE_PORT: 5432 POSTGRE_HOST: "datamate-database" LABEL_STUDIO_HOST: "" # can be overridden @@ -49,9 +49,9 @@ env: LOCAL_FILES_DOCUMENT_ROOT: "/label-studio/local" USE_USERNAME_FOR_LOGIN: "true" LABEL_STUDIO_USERNAME: "admin@demo.com" - LABEL_STUDIO_PASSWORD: "demoadmin" + LABEL_STUDIO_PASSWORD: "" # Set via secrets.yaml or --set LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN: "true" - LABEL_STUDIO_USER_TOKEN: "abc123abc123" + LABEL_STUDIO_USER_TOKEN: "" # Set via secrets.yaml or --set LOG_LEVEL: "DEBUG" persistence: diff --git a/deployment/helm/milvus/secrets.yaml b/deployment/helm/milvus/secrets.yaml new file mode 100644 index 000000000..014696cff --- /dev/null +++ b/deployment/helm/milvus/secrets.yaml @@ -0,0 +1,18 @@ +minio: + accessKey: ENC[AES256_GCM,data:57/fzPABN+5+c1tmf9lvR9O5EflmlWPQfgvxTnwHhIo=,iv:Fkyrk5f7md7QQD2fv/XPXLLTxLDUq2mmK2BOi4QcZyI=,tag:AykpxHYoQSTjs840FodsDg==,type:str] + secretKey: ENC[AES256_GCM,data:EgFQZaIInv6Ptn9a+PiXVydwhi8Wh3dO9QkWY9qD86M=,iv:+WWoLUoMJc6cc9a+aLR97O/xvv7f6yfAgFjuKtt62h8=,tag:O6r+/1KhgnVCdxRmzdS8pQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuMlNnRDZJdGRXTklkK0tp + aC9KQ0t2VTExL2tOS3BvZzNRY3o5TnFYa0ZZCkNqY3U1K3g5M2NERDFDR1g3M1hJ + VGcrNDR4Q0tCOWptTzF0MG1QN0pXWFkKLS0tIGhMMUEweUxleWV1Y1RTVmdDQ2k0 + cUZOVC94Qi8xQmNNMFBwWEM3MnhkQ0EK4bTM7Rw3tlaMFInXv1MKlu65iPBxY8Zw + G5h5QK4AOonWoxwrz3H5jFBAZwBsy156wDgBsxvZJB2oCa4wEfC4Wg== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:40Z" + mac: ENC[AES256_GCM,data:N5n9tUIujF8BpI+N4PMzBApaTmpO+Ji/ZTf7Dr/faM9+nGnrL/JcOwAjKwXXGnDbcFi4DFyvo7yx4u68Mx6c4EtIQYs4o9TymGiQAgAUR8STnjBB3pbGjlvG9MFYwKCVuXBclhKpCLDklI/cjkcJjJq4ae2VnHer0wMlSzRZj8I=,iv:L2vlzY9aRAUGBZhgNOl1t3QDZhP8Inffj0TkGXnGbug=,tag:iB3ZKbYAk0D4bc+05q8m6w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/milvus/values.yaml b/deployment/helm/milvus/values.yaml index 584cdcc27..626248e46 100644 --- a/deployment/helm/milvus/values.yaml +++ b/deployment/helm/milvus/values.yaml @@ -634,8 +634,8 @@ minio: image: tag: "RELEASE.2024-12-18T13-15-44Z" pullPolicy: IfNotPresent - accessKey: minioadmin - secretKey: minioadmin + accessKey: "" # Set via secrets.yaml or --set + secretKey: "" # Set via secrets.yaml or --set existingSecret: "" bucketName: "milvus-bucket" rootPath: file diff --git a/docs/SECRETS_SETUP.md b/docs/SECRETS_SETUP.md new file mode 100644 index 000000000..648e419d2 --- /dev/null +++ b/docs/SECRETS_SETUP.md @@ -0,0 +1,166 @@ +# Secrets Management Setup Guide + +This document describes the tools and configuration required for managing encrypted secrets in DataMate. + +## Required Tools + +### Homebrew Packages (macOS) + +| Tool | Version | Purpose | Install Command | +|------|---------|---------|-----------------| +| age | 1.3.1 | Encryption tool for secrets | `brew install age` | +| sops | 3.13.1 | Secrets encryption manager | `brew install sops` | +| helm | 4.1.0+ | Kubernetes package manager | `brew install helm` | + +### Helm Plugins + +| Plugin | Version | Purpose | Install Method | +|--------|---------|---------|----------------| +| helm-secrets | 4.6.0 | Helm integration with sops | Manual clone (see below) | + +**Note:** Helm 4.x has compatibility issues with helm-secrets plugin. Use manual installation: + +```bash +cd /Users/macoo/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +If plugin fails to load, edit `plugin.yaml` to remove the `command:` field and use only `platformCommand:`. + +## Project Configuration Files + +| File | Purpose | Location | +|------|---------|----------| +| `.sops.yaml` | SOPS encryption rules | Project root | +| `.sops-keys/key.txt` | Age encryption key (private) | Project root (gitignored) | +| `secrets.yaml` | Encrypted secrets per chart | `deployment/helm//` | + +## Quick Setup + +### 1. Install Required Tools + +```bash +# macOS (Homebrew) +brew install age sops helm + +# Helm plugin (manual) +cd ~/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +### 2. Generate Age Key (if not exists) + +```bash +mkdir -p .sops-keys +age-keygen -o .sops-keys/key.txt +``` + +**Important:** Backup the key file securely. It cannot be recovered if lost. + +### 3. Update `.sops.yaml` with Your Public Key + +```yaml +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: +``` + +### 4. Create/Encrypt Secrets Files + +```bash +# Set environment variable +export SOPS_AGE_KEY_FILE=.sops-keys/key.txt + +# Encrypt existing file +sops --encrypt --in-place deployment/helm/datamate/secrets.yaml + +# Or create new encrypted file +sops deployment/helm/datamate/secrets.yaml +``` + +## Usage + +### View Decrypted Secrets + +```bash +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml +``` + +### Deploy with Secrets + +```bash +# Using helper script +./scripts/secrets.sh helm-install datamate datamate + +# Manual method +TMP_FILE=$(mktemp) +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml > $TMP_FILE +helm install datamate deployment/helm/datamate -n datamate -f $TMP_FILE +rm $TMP_FILE +``` + +### Docker Compose + +```bash +# Copy and edit .env +cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env + +# Edit with your secrets +vim deployment/docker/datamate/.env + +# Deploy +docker compose -f deployment/docker/datamate/docker-compose.yml up -d +``` + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `SOPS_AGE_KEY_FILE` | Path to age private key | +| `SOPS_CONFIG` | Path to .sops.yaml (optional) | + +## Security Notes + +1. **Never commit `.sops-keys/` directory** - It's in `.gitignore` +2. **Backup your age key** - Store in secure location (password manager, encrypted backup) +3. **Use strong passwords** - Generate with `openssl rand -base64 32` +4. **Rotate keys periodically** - Especially after team changes + +## Troubleshooting + +### Plugin Load Error + +If helm plugin shows "both platformCommand and command are set": +```bash +# Edit plugin.yaml +vim ~/Library/helm/plugins/helm-secrets/plugin.yaml +# Remove the "command:" line, keep only "platformCommand:" +``` + +### Decryption Failed + +```bash +# Check key file exists +ls -la .sops-keys/key.txt + +# Verify key matches encrypted file's recipient +grep "recipient:" deployment/helm/datamate/secrets.yaml +``` + +### Key Rotation + +```bash +# Generate new key +age-keygen -o .sops-keys/key-new.txt + +# Update .sops.yaml with new public key + +# Re-encrypt all secrets +for f in deployment/helm/*/secrets.yaml; do + SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt $f > /tmp/plain.yaml + SOPS_AGE_KEY_FILE=.sops-keys/key-new.txt sops --encrypt /tmp/plain.yaml > $f +done + +# Replace old key +mv .sops-keys/key-new.txt .sops-keys/key.txt +``` \ No newline at end of file diff --git a/scripts/secrets.sh b/scripts/secrets.sh new file mode 100755 index 000000000..c93203449 --- /dev/null +++ b/scripts/secrets.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" +SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" + +if [ ! -f "$SOPS_KEY_FILE" ]; then + echo "Error: SOPS age key file not found at $SOPS_KEY_FILE" + echo "Please generate a key first: age-keygen -o .sops-keys/key.txt" + exit 1 +fi + +export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" + +case "$1" in + encrypt) + if [ -z "$2" ]; then + echo "Usage: $0 encrypt " + exit 1 + fi + sops --encrypt --in-place "$2" + echo "Encrypted: $2" + ;; + decrypt) + if [ -z "$2" ]; then + echo "Usage: $0 decrypt " + exit 1 + fi + sops --decrypt --in-place "$2" + echo "Decrypted: $2" + ;; + view) + if [ -z "$2" ]; then + echo "Usage: $0 view " + exit 1 + fi + sops --decrypt "$2" + ;; + helm-install) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 helm-install [--secrets-file ]" + exit 1 + fi + CHART="$2" + NAMESPACE="$3" + SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" + + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" + rm "$TMP_FILE" + echo "Deployed $CHART to $NAMESPACE" + ;; + helm-upgrade) + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 helm-upgrade [--secrets-file ]" + exit 1 + fi + CHART="$2" + NAMESPACE="$3" + SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" + + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm upgrade "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" + rm "$TMP_FILE" + echo "Upgraded $CHART in $NAMESPACE" + ;; + *) + echo "DataMate SOPS Secret Management" + echo "" + echo "Usage:" + echo " $0 encrypt - Encrypt a secrets file" + echo " $0 decrypt - Decrypt a secrets file" + echo " $0 view - View decrypted secrets (no modification)" + echo " $0 helm-install - Install chart with decrypted secrets" + echo " $0 helm-upgrade - Upgrade chart with decrypted secrets" + echo "" + echo "Environment variables:" + echo " SOPS_KEY_FILE - Path to age key file (default: .sops-keys/key.txt)" + echo " SOPS_CONFIG - Path to .sops.yaml (default: .sops.yaml)" + ;; +esac \ No newline at end of file From 21484dfbeccc18b9e501f9367d580231810b22da Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 10:48:16 +0800 Subject: [PATCH 02/11] feat: update Makefile for SOPS secret management - K8s install: use helm secrets upgrade with -f secrets.yaml for datamate, label-studio, milvus - Docker install: add pre-check for .env file, exit with helpful message if missing - Set SOPS_AGE_KEY_FILE env var for helm secrets decryption --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 859cd0dbb..a72bd672a 100644 --- a/Makefile +++ b/Makefile @@ -254,6 +254,13 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa done; \ exit 1; \ fi + @if [ ! -f deployment/docker/datamate/.env ]; then \ + echo "ERROR: deployment/docker/datamate/.env not found."; \ + echo "Create it from the template:"; \ + echo " cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env"; \ + echo "Then edit it with your actual passwords."; \ + exit 1; \ + fi @if [ "$*" = "label-studio" ]; then \ REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \ elif [ "$*" = "datamate" ]; then \ @@ -326,19 +333,19 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install -f deployment/helm/label-studio/secrets.yaml; \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) -f deployment/helm/datamate/secrets.yaml; \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install; \ + SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install -f deployment/helm/milvus/secrets.yaml; \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi From aefc5cf1caa2ea1b51fab48bb046bed32c9c70d2 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 11:07:38 +0800 Subject: [PATCH 03/11] fix: remove hardcoded JWT secret and unify property naming - JwtUtils.java: remove hardcoded default "datamate-secret-key-for-jwt-token-generation" Change property from jwt.secret to datamate.jwt.secret (aligned with JwtConfig) - application.yml: add datamate.jwt.secret mapping from JWT_SECRET env var - application.yml: remove hardcoded defaults for DB_PASSWORD and REDIS_PASSWORD - docker-compose.yml: add JWT_SECRET env var to datamate-backend service - Helm values.yaml: add JWT_SECRET secretKeyRef to backend env - Helm secrets.yaml: add JWT_SECRET to public.secrets.data --- .../src/main/resources/application.yml | 8 +++++-- .../datamate/common/security/JwtUtils.java | 2 +- deployment/docker/datamate/docker-compose.yml | 1 + deployment/helm/datamate/secrets.yaml | 21 ++++++++++--------- deployment/helm/datamate/values.yaml | 5 +++++ 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/services/main-application/src/main/resources/application.yml b/backend/services/main-application/src/main/resources/application.yml index 64aac288c..bd05cebd5 100644 --- a/backend/services/main-application/src/main/resources/application.yml +++ b/backend/services/main-application/src/main/resources/application.yml @@ -14,7 +14,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 @@ -59,7 +59,7 @@ spring: host: datamate-redis port: 6379 timeout: 2000 - password: ${REDIS_PASSWORD:password} + password: ${REDIS_PASSWORD} lettuce: pool: max-active: 20 @@ -131,6 +131,10 @@ management: # 平台配置 datamate: + # JWT配置 + jwt: + secret: ${JWT_SECRET} + # 通用配置 diff --git a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java index efe4a4b85..5e981daf5 100644 --- a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java +++ b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java @@ -16,7 +16,7 @@ @Component public class JwtUtils { - @Value("${jwt.secret:datamate-secret-key-for-jwt-token-generation}") + @Value("${datamate.jwt.secret}") private String secret; @Value("${jwt.expiration:86400}") // 24小时 diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index 3ab169c0d..b9796e62e 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -10,6 +10,7 @@ services: privileged: true environment: - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} + - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml index 5717c25f8..c688f2b1b 100644 --- a/deployment/helm/datamate/secrets.yaml +++ b/deployment/helm/datamate/secrets.yaml @@ -1,26 +1,27 @@ public: secrets: data: - DB_PASSWORD: ENC[AES256_GCM,data:vX907qq/Iu/1v2IWAXYKKN9P2PkFC992GyF1fZT1gmmLdyfWtkARTbmHgqM=,iv:wHWaaQT5m9a34LN9NZiEOxrtwtUnd4y056h6WxlurYc=,tag:7inilRxi9QJY2slkF6P9jg==,type:str] + DB_PASSWORD: ENC[AES256_GCM,data:OQx8/Cmz248ayvdwVdkNVR1KZJnfy0J/3zicX4TMlvxJDeACcMOsIskmio8=,iv:ZG72YxXBkRKIL7tpaF/NwgU+9D+pl8cWVwDB+JipO3A=,tag:P5TvdWGHUoocVjA23hwjrw==,type:str] + JWT_SECRET: ENC[AES256_GCM,data:dpS/iMHjipR00Ob5xIuA8XpQsx32HdOeQLrvU89vvCCLSRkvPUpP+0DpxT8=,iv:pCsAQPxHhIj/yJ4p7euE5u8L2gR+JlOtTq0GJkSN9AE=,tag:ZqqhvPaxE73yIrOIvKdQwQ==,type:str] CERT_PASS: "" DOMAIN: "" HOME_PAGE_URL: "" gateway: env: - - name: ENC[AES256_GCM,data:yvymNzt7F3oJPA==,iv:JPJOJ+y4dvnXNQYpchL66MRNNrxidMTxyfEgbi30tpA=,tag:mJ9aPHXR61ybkNvpIxqJfw==,type:str] - value: ENC[AES256_GCM,data:om/hHL6wx7Bp2Xud2Te/jsOC8jakLCJOfzeuyZqGxjutK66+v//kjjnkTkY=,iv:WUvXTmTSfEZh4aOMRhtFH1ceRsuwtnm8pv9/qxG8K0U=,tag:pqUJhS+HYtsmBQqKXq5nJw==,type:str] + - name: ENC[AES256_GCM,data:gZSjzeTSIH9mZg==,iv:NIwpYC8X5YSykeLovTZAGamYk1pCcTZ9zORjElMArk0=,tag:BJUww7hxoHIxmgchf2kz5Q==,type:str] + value: ENC[AES256_GCM,data:+iV1mQE+iw5RCOSCfojNIG5RMAH9x+0PBAwNYpEKFvTmUfDvjCEi1ykQW6o=,iv:0mUP/BPYpHB2bvKfQZ15TeaICe3uJdu8EobL8M6DHtg=,tag:9e1QjigvK+cF6eSMZJ3s3A==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCYVY2ekE4RWZta3ZIdTVm - eXZTOXViaUpxUGlSNjAyUVk4M2NpZ1lBVWs0ClJMaUVPRXMxbW9OU3F1T0kxSlA0 - VENSZ0o2MW9GTmVQSWwyT0h0cFJoWUUKLS0tIHZGcHZzWTArK01DbVBJc21Ib2ZD - cnpCK01CZkdEaUJVS0NpVUc1NFlHTm8KZcEMm+q1Y4Zb0XJ28soc+cuPluejOs28 - WRB9Jde2Ext6b4LN05ZJ/BkH0LZPYneB7ZhuN+a2o1p2Xc3bjZrO5w== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNmNZdFJJN2JkaGFVM1lt + K2x5dFNaV3czUVduRlY4aHY2U2JlZVZFbWtvCnlPdHhWcElKS2FQbFJ5cnpMQ0tL + VjhiRjlRK002OEt3UUtSanppZVJCcUEKLS0tIHJVUDF1SU5TZC9qUmlZRGdMZjht + M1I2STNoKzdteXNFTUdVczdsSFY5bUkKblFkFp2073nMZgkpqug9wrrbso14HZAk + ZLzSwQBYtcV1103ri/pGFAyWWqIboAZ3Ki4BIprymRvlX+2XVtqCog== -----END AGE ENCRYPTED FILE----- recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-22T08:19:22Z" - mac: ENC[AES256_GCM,data:y9aHJfy+QXoNnIUcrdtiubMKAS4JiyAKFE33M3/2REz4D+v/9AKV258c2Q3U7BjGZOaW3wgv0CGbaHWurmYZ5xjwemv6Ev60ozt1Oy3BG9fG29ckcQooeNT/FiavjFJ7Xw72Kyj1mgDf+4cgfbNn5Wdqa9TF15o0vTnBxjC7v00=,iv:7xIH0txZCT/zuBFqrzOcESNice8QAlAzDK7dEqmSn/w=,tag:TPI6at/w+xKLkLvCvgMKHg==,type:str] + lastmodified: "2026-05-25T03:07:14Z" + mac: ENC[AES256_GCM,data:iHn7oK3+kUnEZ0f2gYbZF+VloKSYNBcDpFTCP7SEGoXo7RiaGIg3EEj0OfOjlNJq7nLhZorRCq9ewjyhh5K3guW0Ys2Mh8hPrD1FEmw2kkiDzuzQA0qhx6DT7uXTyrhn1QyhvLwWrqMP+8jYjayLfNqMvYWpg5KYFi76MPVi/sE=,iv:SCQ3s/QNL+QILV1517vm3944rvCJU/9I92NGXxGO3tQ=,tag:I8fJVVCT46ygmyuus23+yw==,type:str] unencrypted_suffix: _unencrypted version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 935de2283..50a5fda2a 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -118,6 +118,11 @@ backend: secretKeyRef: name: datamate-conf key: DB_PASSWORD + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.rag.milvus-uri value: "http://milvus:19530" - name: datamate.jwt.enable From 25972b01f5a051f5e46a31fc37f68c64f54cbfcf Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 11:20:07 +0800 Subject: [PATCH 04/11] fix: enforce encrypted private key and restrict permissions - Reject plaintext private keys on startup (exit with error) - Require CERT_PASS when key is encrypted - Set chmod 600 on decrypted key for restrictive access --- scripts/images/frontend/start.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/images/frontend/start.sh b/scripts/images/frontend/start.sh index 6102495fd..d61306fca 100644 --- a/scripts/images/frontend/start.sh +++ b/scripts/images/frontend/start.sh @@ -18,14 +18,17 @@ if [ -f "/cert/server.pem" ]; then fi if [ -f "/cert/server.key" ]; then - # Check if key is encrypted and decrypt if needed - # Supports RSA, EC (Elliptic Curve), PKCS#8, and DSA keys + # Private key MUST be encrypted. Plaintext keys are not accepted. if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then - # Key is encrypted, decrypt using generic pkey command (supports all key types) + if [ -z "$CERT_PASS" ]; then + echo "ERROR: CERT_PASS is required to decrypt the private key" + exit 1 + fi echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin + chmod 600 /etc/nginx/cert/server.key else - # Key is not encrypted, copy directly - cp /cert/server.key /etc/nginx/cert/server.key + echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment." + exit 1 fi chown nginx:nginx /etc/nginx/cert/server.key fi From cdc393a9171d7b53bc3e8979c2004c255269961e Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 15:52:49 +0800 Subject: [PATCH 05/11] fix: remove hardcoded passwords in Python config and exclude dev .env from Docker builds - Create .dockerignore to prevent runtime/datamate-python/.env from being copied into Docker images (it contained localhost:15432 telepresence debug settings) - config.py: remove hardcoded defaults for pgsql_password, mysql_password, label_studio_password, label_studio_user_token - docker-compose.yml: add explicit PGSQL_HOST/PGSQL_PORT for backend-python - Helm values.yaml: add explicit PGSQL_HOST/PGSQL_PORT for backend-python --- .dockerignore | 22 +++++++++++++++++++ deployment/docker/datamate/docker-compose.yml | 2 ++ deployment/helm/datamate/values.yaml | 4 ++++ runtime/datamate-python/app/core/config.py | 10 ++++----- 4 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..27e5b67e7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Environment files (local dev only, never in images) +**/.env +!.env.example + +# Python +**/__pycache__/ +**/*.pyc +**/.venv/ +**/venv/ + +# Logs +**/logs/ +**/*.log + +# IDE +.idea/ +.vscode/ +*.iml + +# Git +.git/ +.gitignore diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index b9796e62e..f9c25ea00 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -31,6 +31,8 @@ services: - "18000:18000" environment: - log_level=DEBUG + - PGSQL_HOST=datamate-database + - PGSQL_PORT=5432 - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index 50a5fda2a..bf53963fc 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -144,6 +144,10 @@ backend: backend-python: env: + - name: PGSQL_HOST + value: "datamate-database" + - name: PGSQL_PORT + value: "5432" - name: pgsql_password valueFrom: secretKeyRef: diff --git a/runtime/datamate-python/app/core/config.py b/runtime/datamate-python/app/core/config.py index 5f6d3ae55..ed49fcf02 100644 --- a/runtime/datamate-python/app/core/config.py +++ b/runtime/datamate-python/app/core/config.py @@ -33,14 +33,14 @@ class Config: pgsql_host: str = "datamate-database" pgsql_port: int = 5432 pgsql_user: str = "postgres" - pgsql_password: str = "password" + pgsql_password: str = "" pgsql_database: str = "datamate" # Database mysql_host: str = "datamate-database" mysql_port: int = 3306 mysql_user: str = "root" - mysql_password: str = "password" + mysql_password: str = "" mysql_database: str = "datamate" database_url: str = "" # Will be overridden by build_database_url() if not provided @@ -63,9 +63,9 @@ def build_database_url(self): # Label Studio label_studio_base_url: str = "http://label-studio:8000" - label_studio_username: Optional[str] = "admin@demo.com" - label_studio_password: Optional[str] = "demoadmin" - label_studio_user_token: Optional[str] = "abc123abc123" # Legacy Token + label_studio_username: Optional[str] = None + label_studio_password: Optional[str] = None + label_studio_user_token: Optional[str] = None # Legacy Token label_studio_local_document_root: str = "/label-studio/local" # Label Studio local file storage path label_studio_file_path_prefix: str = "/data/local-files/?d=" # Label Studio local file serving URL prefix From 1c9b269a2809f66bc66abeba380e4721e2fed1de Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:36:56 +0800 Subject: [PATCH 06/11] refactor: encapsulate SOPS in scripts/secrets.sh, remove helm-secrets dependency - scripts/secrets.sh: add check_tools(), ensure_key() with auto-generation New helm-upgrade command: decrypts secrets.yaml and runs helm upgrade --install - Makefile: K8s install targets now call "bash scripts/secrets.sh helm-upgrade" instead of direct "helm secrets upgrade". No helm-secrets plugin needed. - Users only need sops + age (brew install), key auto-generated on first run. Docker users unaffected - still use .env file. --- Makefile | 6 ++-- scripts/secrets.sh | 79 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index a72bd672a..379654a92 100644 --- a/Makefile +++ b/Makefile @@ -333,19 +333,19 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install -f deployment/helm/label-studio/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade label-studio $(NAMESPACE); \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY) -f deployment/helm/datamate/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade datamate $(NAMESPACE) --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - SOPS_AGE_KEY_FILE="$${SOPS_AGE_KEY_FILE:-$$PWD/.sops-keys/key.txt}" helm secrets upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install -f deployment/helm/milvus/secrets.yaml; \ + bash scripts/secrets.sh helm-upgrade milvus $(NAMESPACE); \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi diff --git a/scripts/secrets.sh b/scripts/secrets.sh index c93203449..a4186e87e 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -1,18 +1,41 @@ #!/bin/bash +# DataMate Secrets Helper +# Manages SOPS-encrypted secrets for Helm deployment. +# Docker users do NOT need this script - use .env file instead. + +set -e SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" -if [ ! -f "$SOPS_KEY_FILE" ]; then - echo "Error: SOPS age key file not found at $SOPS_KEY_FILE" - echo "Please generate a key first: age-keygen -o .sops-keys/key.txt" - exit 1 -fi +# Check required tools +check_tools() { + local missing="" + command -v sops >/dev/null 2>&1 || missing="$missing sops" + command -v age >/dev/null 2>&1 || missing="$missing age" + command -v age-keygen >/dev/null 2>&1 || missing="$missing age-keygen" + if [ -n "$missing" ]; then + echo "Error: Required tools not found:$missing" + echo "Install with: brew install sops age" + exit 1 + fi +} -export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" +# Check or generate age key +ensure_key() { + if [ ! -f "$SOPS_KEY_FILE" ]; then + echo "Age key not found at $SOPS_KEY_FILE" + echo "Generating a new one..." + mkdir -p "$(dirname "$SOPS_KEY_FILE")" + age-keygen -o "$SOPS_KEY_FILE" + echo "Key generated. Keep this file secure and never commit it." + fi + export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" +} case "$1" in encrypt) + check_tools if [ -z "$2" ]; then echo "Usage: $0 encrypt " exit 1 @@ -21,6 +44,7 @@ case "$1" in echo "Encrypted: $2" ;; decrypt) + check_tools if [ -z "$2" ]; then echo "Usage: $0 decrypt " exit 1 @@ -29,25 +53,52 @@ case "$1" in echo "Decrypted: $2" ;; view) + check_tools if [ -z "$2" ]; then echo "Usage: $0 view " exit 1 fi sops --decrypt "$2" ;; - helm-install) + helm-upgrade) + # Usage: scripts/secrets.sh helm-upgrade [extra-helm-args...] + check_tools + ensure_key if [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 helm-install [--secrets-file ]" + echo "Usage: $0 helm-upgrade [extra-helm-args...]" exit 1 fi CHART="$2" NAMESPACE="$3" - SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" - - TMP_FILE=$(mktemp) - sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" - helm install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" - rm "$TMP_FILE" + SECRETS_FILE="deployment/helm/$CHART/secrets.yaml" + shift 3 + + if [ ! -f "$SECRETS_FILE" ]; then + echo "Warning: No secrets file at $SECRETS_FILE, deploying without secrets." + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" "$@" + else + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" "$@" + rm "$TMP_FILE" + echo "Deployed $CHART with decrypted secrets." + fi + ;; + *) + echo "DataMate Secrets Helper" + echo "" + echo "Usage: $0 [args...]" + echo "" + echo "Commands:" + echo " encrypt Encrypt a YAML file in-place" + echo " decrypt Decrypt a YAML file in-place" + echo " view Print decrypted content" + echo " helm-upgrade Decrypt secrets and helm upgrade --install" + echo "" + echo "Docker users: skip this script, use 'cp .env.example .env' instead." + exit 1 + ;; +esac echo "Deployed $CHART to $NAMESPACE" ;; helm-upgrade) From 1cd2316d06e0eb4f08a81ce401103811d060d162 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:39:04 +0800 Subject: [PATCH 07/11] fix: pass Label Studio credentials to backend-python container - docker-compose.yml: add LABEL_STUDIO_USER_TOKEN, LABEL_STUDIO_PASSWORD env vars - Helm values.yaml: add secretKeyRef for both to backend-python env - secrets.yaml: add encrypted LABEL_STUDIO_USER_TOKEN, LABEL_STUDIO_PASSWORD Fixes 500 error "Label Studio API token is required" when creating annotation tasks after config.py defaults were removed. --- deployment/docker/datamate/docker-compose.yml | 2 ++ deployment/helm/datamate/secrets.yaml | 24 ++++++++++--------- deployment/helm/datamate/values.yaml | 10 ++++++++ 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index f9c25ea00..af2b18123 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -36,6 +36,8 @@ services: - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: - dataset_volume:/dataset - flow_volume:/flow diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml index c688f2b1b..25defa1a6 100644 --- a/deployment/helm/datamate/secrets.yaml +++ b/deployment/helm/datamate/secrets.yaml @@ -1,27 +1,29 @@ public: secrets: data: - DB_PASSWORD: ENC[AES256_GCM,data:OQx8/Cmz248ayvdwVdkNVR1KZJnfy0J/3zicX4TMlvxJDeACcMOsIskmio8=,iv:ZG72YxXBkRKIL7tpaF/NwgU+9D+pl8cWVwDB+JipO3A=,tag:P5TvdWGHUoocVjA23hwjrw==,type:str] - JWT_SECRET: ENC[AES256_GCM,data:dpS/iMHjipR00Ob5xIuA8XpQsx32HdOeQLrvU89vvCCLSRkvPUpP+0DpxT8=,iv:pCsAQPxHhIj/yJ4p7euE5u8L2gR+JlOtTq0GJkSN9AE=,tag:ZqqhvPaxE73yIrOIvKdQwQ==,type:str] + DB_PASSWORD: ENC[AES256_GCM,data:F/GtCDyYi4xH8FJWGxojsCdNEDdKQaVIaFBuC5dk6ilLdfGJtk0JDoPRQ14=,iv:mhenVjDgPvX57OaSMnUZGvFJrWjwtqIPJ66PIHqQcKM=,tag:kxxS/w0FUt6N9kHNvSVMOg==,type:str] + JWT_SECRET: ENC[AES256_GCM,data:Pu2cojeOgNSMmxbqevkI+c8Az3QS/A6DgDKv2NeGtYON5YrZFBqxxfV+y2M=,iv:sjVi42/TVmKGzbu9ZxAgU3CGBlHgJ94CEpzAd8eZX0k=,tag:3v/cJNPUZpyFJjU7Wdz+Ow==,type:str] CERT_PASS: "" DOMAIN: "" HOME_PAGE_URL: "" + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:XBjxusCTd5xe2xnx7D8h+AH8rpZ6ZUCyYUws4Zxffr0=,iv:yWwIMpEJzodAhK78SOZXTWZi9GnG387oKeOK+ZORjis=,tag:l6U8sQoUzvHdFit72WI4JQ==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:YR9/z/UthUuwJTaZMqbnwdVNWW60pCrzmUDAOojWo6E=,iv:ojZuiZ5G7l5DF6P+UQaCKkOSXf6+cD7G7iWUfvm3GQc=,tag:iWlohbIDm7o75JfWSfV5oA==,type:str] gateway: env: - - name: ENC[AES256_GCM,data:gZSjzeTSIH9mZg==,iv:NIwpYC8X5YSykeLovTZAGamYk1pCcTZ9zORjElMArk0=,tag:BJUww7hxoHIxmgchf2kz5Q==,type:str] - value: ENC[AES256_GCM,data:+iV1mQE+iw5RCOSCfojNIG5RMAH9x+0PBAwNYpEKFvTmUfDvjCEi1ykQW6o=,iv:0mUP/BPYpHB2bvKfQZ15TeaICe3uJdu8EobL8M6DHtg=,tag:9e1QjigvK+cF6eSMZJ3s3A==,type:str] + - name: ENC[AES256_GCM,data:g6BN4NxwzoB1fQ==,iv:mo8U8lIXN/cexFjE3SGIkapo9cuiUcvnRbazA/hL74U=,tag:uOJz3abAnaGtEUQ+xFoXiA==,type:str] + value: ENC[AES256_GCM,data:f+a/g0gIKXQFdzUa1NFcWHHc0oLvpxUZHl2svGMYEmKAwGFk1FFreJ8/Yik=,iv:jeffKVDeru2BR4m2uTTgZw2NndMgtTb61uzfWMa8H6o=,tag:X0UIUpYg9cN8Z0xXi2KZkQ==,type:str] sops: age: - enc: | -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBqNmNZdFJJN2JkaGFVM1lt - K2x5dFNaV3czUVduRlY4aHY2U2JlZVZFbWtvCnlPdHhWcElKS2FQbFJ5cnpMQ0tL - VjhiRjlRK002OEt3UUtSanppZVJCcUEKLS0tIHJVUDF1SU5TZC9qUmlZRGdMZjht - M1I2STNoKzdteXNFTUdVczdsSFY5bUkKblFkFp2073nMZgkpqug9wrrbso14HZAk - ZLzSwQBYtcV1103ri/pGFAyWWqIboAZ3Ki4BIprymRvlX+2XVtqCog== + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnR2wvVGR3Q3dUUlVGQ3lX + WHNnaUFvdmozTDcwZUpYamhkR0NnTFFrQmpjCmFVY0VnV05IUnQ0YmxFUjlVSFVS + cE5KaEdMWXJRUW1GWlNXVzYzTTNvYVkKLS0tIGRFUXdNSW5RbnAzeUp5cVZ1ZWJ2 + azh3ZzhKb2VYUWZSdHo5NlRldGJQTGsK98hrnUGXdDQKol1duzHyYsd9PLL2eAEC + 4y+AUB5bVjYI3PEt3giBBUYFwIo478FWQ1w8iSNIUwWlhlqEB2hKNg== -----END AGE ENCRYPTED FILE----- recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm - lastmodified: "2026-05-25T03:07:14Z" - mac: ENC[AES256_GCM,data:iHn7oK3+kUnEZ0f2gYbZF+VloKSYNBcDpFTCP7SEGoXo7RiaGIg3EEj0OfOjlNJq7nLhZorRCq9ewjyhh5K3guW0Ys2Mh8hPrD1FEmw2kkiDzuzQA0qhx6DT7uXTyrhn1QyhvLwWrqMP+8jYjayLfNqMvYWpg5KYFi76MPVi/sE=,iv:SCQ3s/QNL+QILV1517vm3944rvCJU/9I92NGXxGO3tQ=,tag:I8fJVVCT46ygmyuus23+yw==,type:str] + lastmodified: "2026-05-25T08:38:49Z" + mac: ENC[AES256_GCM,data:xW7scU3rX48jjA2eHvC7RDj3Fn5P3mPIqwLGpvHf0bsjbQo+huMPfTIafJRF4owum+SiRrASaDkABho6wxLkwiwQ/KlwPHW8QIXCpdgE/OJduQV9MIjAs7uCX+BSVZfVAHhgdR4ftAHhBNF6KrbZ3O9Elw7iZstsvsOoDRy8QPM=,iv:db0m7SeybD9igEPnEsm5M7evLnotq+L2owuSYADQs74=,tag:LyTBdJNskE4CsY//wgHUTA==,type:str] unencrypted_suffix: _unencrypted version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index bf53963fc..e645aa005 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -157,6 +157,16 @@ backend-python: value: *DATAMATE_JWT_ENABLE - name: milvus_uri value: "http://milvus:19530" + - name: LABEL_STUDIO_USER_TOKEN + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_PASSWORD + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_PASSWORD volumes: - *datasetVolume - *flowVolume From 638931b5024b61bb06a66ae39550d42e8f93d970 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:48:33 +0800 Subject: [PATCH 08/11] fix: add LABEL_STUDIO_USERNAME env var for auto-login to Label Studio The loginAnnotationUsingGet flow requires label_studio_username to auto-login to Label Studio. Without it, clicking "edit" on an annotation task redirects to Label Studio login page instead of auto-authenticating. --- deployment/docker/datamate/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index af2b18123..bdbfc4f7f 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -36,6 +36,7 @@ services: - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com} - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: From 659758dffb993184f93605db636de3e504bd8582 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 16:55:19 +0800 Subject: [PATCH 09/11] docs: add LABEL_STUDIO_USERNAME to .env.example --- deployment/docker/datamate/.env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example index 90fa6588e..4a9ad4e84 100644 --- a/deployment/docker/datamate/.env.example +++ b/deployment/docker/datamate/.env.example @@ -18,6 +18,7 @@ MINIO_ACCESS_KEY=your-minio-access-key MINIO_SECRET_KEY=your-minio-secret-key # Label Studio +LABEL_STUDIO_USERNAME=admin@demo.com LABEL_STUDIO_PASSWORD=your-labelstudio-password LABEL_STUDIO_USER_TOKEN=your-labelstudio-token LABEL_STUDIO_HOST= From cdaa8a3ada035bbf0bd6cae7782837f328b37f5c Mon Sep 17 00:00:00 2001 From: MoeexT Date: Mon, 25 May 2026 17:15:59 +0800 Subject: [PATCH 10/11] fix: remove duplicate case blocks in scripts/secrets.sh causing syntax error --- scripts/secrets.sh | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/scripts/secrets.sh b/scripts/secrets.sh index a4186e87e..946d982dd 100755 --- a/scripts/secrets.sh +++ b/scripts/secrets.sh @@ -98,36 +98,4 @@ case "$1" in echo "Docker users: skip this script, use 'cp .env.example .env' instead." exit 1 ;; -esac - echo "Deployed $CHART to $NAMESPACE" - ;; - helm-upgrade) - if [ -z "$2" ] || [ -z "$3" ]; then - echo "Usage: $0 helm-upgrade [--secrets-file ]" - exit 1 - fi - CHART="$2" - NAMESPACE="$3" - SECRETS_FILE="${4:-deployment/helm/$CHART/secrets.yaml}" - - TMP_FILE=$(mktemp) - sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" - helm upgrade "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" - rm "$TMP_FILE" - echo "Upgraded $CHART in $NAMESPACE" - ;; - *) - echo "DataMate SOPS Secret Management" - echo "" - echo "Usage:" - echo " $0 encrypt - Encrypt a secrets file" - echo " $0 decrypt - Decrypt a secrets file" - echo " $0 view - View decrypted secrets (no modification)" - echo " $0 helm-install - Install chart with decrypted secrets" - echo " $0 helm-upgrade - Upgrade chart with decrypted secrets" - echo "" - echo "Environment variables:" - echo " SOPS_KEY_FILE - Path to age key file (default: .sops-keys/key.txt)" - echo " SOPS_CONFIG - Path to .sops.yaml (default: .sops.yaml)" - ;; esac \ No newline at end of file From 7fc4f8aa7f941614fff903585caff7f1e3fa2335 Mon Sep 17 00:00:00 2001 From: MoeexT Date: Tue, 26 May 2026 15:41:52 +0800 Subject: [PATCH 11/11] fix: add LABEL_STUDIO_USERNAME to Helm backend-python env The loginAnnotationUsingGet API requires label_studio_username to auto-login to Label Studio. Docker got this in 638931b, but the Helm values.yaml was missing it, causing K8s deployments to show Label Studio login prompt. --- deployment/helm/datamate/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index e645aa005..978640274 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -162,6 +162,8 @@ backend-python: secretKeyRef: name: datamate-conf key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_USERNAME + value: "admin@demo.com" - name: LABEL_STUDIO_PASSWORD valueFrom: secretKeyRef: