diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..27e5b67e7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,22 @@ +# Environment files (local dev only, never in images) +**/.env +!.env.example + +# Python +**/__pycache__/ +**/*.pyc +**/.venv/ +**/venv/ + +# Logs +**/logs/ +**/*.log + +# IDE +.idea/ +.vscode/ +*.iml + +# Git +.git/ +.gitignore diff --git a/.gitignore b/.gitignore index 9b34b7af4..d78ab5ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,11 @@ Thumbs.db # Milvus **/volumes/ -**/rag_storage/ \ No newline at end of file +**/rag_storage/ +# SOPS encryption keys +.sops-keys/ + +# Environment files - ignore local .env, but allow templates and encrypted files +.env +!.env.example +!.env.enc diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 000000000..efb2f00f7 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,3 @@ +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm diff --git a/Makefile b/Makefile index 859cd0dbb..379654a92 100644 --- a/Makefile +++ b/Makefile @@ -254,6 +254,13 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa done; \ exit 1; \ fi + @if [ ! -f deployment/docker/datamate/.env ]; then \ + echo "ERROR: deployment/docker/datamate/.env not found."; \ + echo "Create it from the template:"; \ + echo " cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env"; \ + echo "Then edit it with your actual passwords."; \ + exit 1; \ + fi @if [ "$*" = "label-studio" ]; then \ REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \ elif [ "$*" = "datamate" ]; then \ @@ -326,19 +333,19 @@ VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru m exit 1; \ fi @if [ "$*" = "label-studio" ]; then \ - helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \ + bash scripts/secrets.sh helm-upgrade label-studio $(NAMESPACE); \ elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \ elif [ "$*" = "mineru-310P" ]; then \ kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \ elif [ "$*" = "datamate" ]; then \ - helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ + bash scripts/secrets.sh helm-upgrade datamate $(NAMESPACE) --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "deer-flow" ]; then \ cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env; \ cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml; \ helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \ elif [ "$*" = "milvus" ]; then \ - helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install; \ + bash scripts/secrets.sh helm-upgrade milvus $(NAMESPACE); \ elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \ kubectl apply -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \ fi diff --git a/backend/services/main-application/src/main/resources/application.yml b/backend/services/main-application/src/main/resources/application.yml index 64aac288c..bd05cebd5 100644 --- a/backend/services/main-application/src/main/resources/application.yml +++ b/backend/services/main-application/src/main/resources/application.yml @@ -14,7 +14,7 @@ spring: driver-class-name: org.postgresql.Driver url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true username: ${DB_USERNAME:postgres} - password: ${DB_PASSWORD:password} + password: ${DB_PASSWORD} hikari: maximum-pool-size: 20 minimum-idle: 5 @@ -59,7 +59,7 @@ spring: host: datamate-redis port: 6379 timeout: 2000 - password: ${REDIS_PASSWORD:password} + password: ${REDIS_PASSWORD} lettuce: pool: max-active: 20 @@ -131,6 +131,10 @@ management: # 平台配置 datamate: + # JWT配置 + jwt: + secret: ${JWT_SECRET} + # 通用配置 diff --git a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java index efe4a4b85..5e981daf5 100644 --- a/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java +++ b/backend/shared/security-common/src/main/java/com/datamate/common/security/JwtUtils.java @@ -16,7 +16,7 @@ @Component public class JwtUtils { - @Value("${jwt.secret:datamate-secret-key-for-jwt-token-generation}") + @Value("${datamate.jwt.secret}") private String secret; @Value("${jwt.expiration:86400}") // 24小时 diff --git a/deployment/docker/datamate/.env.example b/deployment/docker/datamate/.env.example new file mode 100644 index 000000000..4a9ad4e84 --- /dev/null +++ b/deployment/docker/datamate/.env.example @@ -0,0 +1,30 @@ +# DataMate Environment Variables Template +# Copy this file to .env and fill in the values +# cp .env.example .env +# IMPORTANT: Never commit .env to git! It is already in .gitignore. +# +# For K8s/Helm deployment: secrets are managed via SOPS-encrypted secrets.yaml files. +# For Docker deployment: use this .env file (gitignored, local only). + +# Database +DB_PASSWORD=your-secure-password-here + +# JWT Authentication +JWT_SECRET=your-secure-jwt-secret-here +DATAMATE_JWT_ENABLE=false + +# MinIO (for Milvus storage) +MINIO_ACCESS_KEY=your-minio-access-key +MINIO_SECRET_KEY=your-minio-secret-key + +# Label Studio +LABEL_STUDIO_USERNAME=admin@demo.com +LABEL_STUDIO_PASSWORD=your-labelstudio-password +LABEL_STUDIO_USER_TOKEN=your-labelstudio-token +LABEL_STUDIO_HOST= + +# Optional: SSL Certificate Password (for encrypted private keys) +CERT_PASS= + +# Optional: Domain for HTTPS +DOMAIN= \ No newline at end of file diff --git a/deployment/docker/datamate/docker-compose.yml b/deployment/docker/datamate/docker-compose.yml index dc5a8470d..bdbfc4f7f 100644 --- a/deployment/docker/datamate/docker-compose.yml +++ b/deployment/docker/datamate/docker-compose.yml @@ -9,7 +9,8 @@ services: restart: on-failure privileged: true environment: - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} + - JWT_SECRET=${JWT_SECRET:?JWT_SECRET is required. Set in .env file} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} volumes: - dataset_volume:/dataset @@ -30,9 +31,14 @@ services: - "18000:18000" environment: - log_level=DEBUG - - pgsql_password=${DB_PASSWORD:-password} + - PGSQL_HOST=datamate-database + - PGSQL_PORT=5432 + - pgsql_password=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - datamate_jwt_enable=${DATAMATE_JWT_ENABLE:-false} - milvus_uri=${MILVUS_URI:-http://milvus:19530} + - LABEL_STUDIO_USERNAME=${LABEL_STUDIO_USERNAME:-admin@demo.com} + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} volumes: - dataset_volume:/dataset - flow_volume:/flow @@ -52,7 +58,7 @@ services: ports: - '8080:8080' environment: - - JWT_SECRET=default-insecure-key-change-in-production + - JWT_SECRET=${JWT_SECRET:-} - datamate.jwt.enable=${DATAMATE_JWT_ENABLE:-false} networks: [ datamate ] @@ -75,7 +81,7 @@ services: restart: on-failure environment: - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=${DB_PASSWORD:-password} + - POSTGRES_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} volumes: - postgresql_volume:/var/lib/postgresql/data - database_log_volume:/var/log/datamate/database @@ -93,7 +99,7 @@ services: PG_HOST: "datamate-database" PG_PORT: "5432" PG_USER: "postgres" - PG_PASSWORD: ${DB_PASSWORD:-password} + PG_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} PG_DATABASE: "datamate" command: - python @@ -213,7 +219,7 @@ services: - DB_PORT=5432 - DB_NAME=labelstudio - DB_USER=postgres - - DB_PASSWORD=${DB_PASSWORD:-password} + - DB_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - AUTH_TYPE=scram-sha-256 - POOL_MODE=transaction - MAX_CLIENT_CONN=100 @@ -241,7 +247,7 @@ services: - DJANGO_DB=default - POSTGRE_NAME=labelstudio - POSTGRE_USER=postgres - - POSTGRE_PASSWORD=${DB_PASSWORD:-password} + - POSTGRE_PASSWORD=${DB_PASSWORD:?DB_PASSWORD is required. Set in .env file} - POSTGRE_PORT=5432 - POSTGRE_HOST=label-studio-pgbouncer - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-} @@ -249,9 +255,9 @@ services: - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local - USE_USERNAME_FOR_LOGIN=true - LABEL_STUDIO_USERNAME=admin@demo.com - - LABEL_STUDIO_PASSWORD=demoadmin + - LABEL_STUDIO_PASSWORD=${LABEL_STUDIO_PASSWORD:-} - LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN=true - - LABEL_STUDIO_USER_TOKEN=abc123abc123 + - LABEL_STUDIO_USER_TOKEN=${LABEL_STUDIO_USER_TOKEN:-} - LOG_LEVEL=DEBUG volumes: - label-studio-data:/label-studio/data:rw @@ -290,8 +296,8 @@ services: container_name: milvus-minio image: minio/minio:RELEASE.2024-12-18T13-15-44Z environment: - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-} + MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-} ports: - "9001:9001" - "9000:9000" diff --git a/deployment/helm/datamate/secrets.yaml b/deployment/helm/datamate/secrets.yaml new file mode 100644 index 000000000..25defa1a6 --- /dev/null +++ b/deployment/helm/datamate/secrets.yaml @@ -0,0 +1,29 @@ +public: + secrets: + data: + DB_PASSWORD: ENC[AES256_GCM,data:F/GtCDyYi4xH8FJWGxojsCdNEDdKQaVIaFBuC5dk6ilLdfGJtk0JDoPRQ14=,iv:mhenVjDgPvX57OaSMnUZGvFJrWjwtqIPJ66PIHqQcKM=,tag:kxxS/w0FUt6N9kHNvSVMOg==,type:str] + JWT_SECRET: ENC[AES256_GCM,data:Pu2cojeOgNSMmxbqevkI+c8Az3QS/A6DgDKv2NeGtYON5YrZFBqxxfV+y2M=,iv:sjVi42/TVmKGzbu9ZxAgU3CGBlHgJ94CEpzAd8eZX0k=,tag:3v/cJNPUZpyFJjU7Wdz+Ow==,type:str] + CERT_PASS: "" + DOMAIN: "" + HOME_PAGE_URL: "" + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:XBjxusCTd5xe2xnx7D8h+AH8rpZ6ZUCyYUws4Zxffr0=,iv:yWwIMpEJzodAhK78SOZXTWZi9GnG387oKeOK+ZORjis=,tag:l6U8sQoUzvHdFit72WI4JQ==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:YR9/z/UthUuwJTaZMqbnwdVNWW60pCrzmUDAOojWo6E=,iv:ojZuiZ5G7l5DF6P+UQaCKkOSXf6+cD7G7iWUfvm3GQc=,tag:iWlohbIDm7o75JfWSfV5oA==,type:str] +gateway: + env: + - name: ENC[AES256_GCM,data:g6BN4NxwzoB1fQ==,iv:mo8U8lIXN/cexFjE3SGIkapo9cuiUcvnRbazA/hL74U=,tag:uOJz3abAnaGtEUQ+xFoXiA==,type:str] + value: ENC[AES256_GCM,data:f+a/g0gIKXQFdzUa1NFcWHHc0oLvpxUZHl2svGMYEmKAwGFk1FFreJ8/Yik=,iv:jeffKVDeru2BR4m2uTTgZw2NndMgtTb61uzfWMa8H6o=,tag:X0UIUpYg9cN8Z0xXi2KZkQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnR2wvVGR3Q3dUUlVGQ3lX + WHNnaUFvdmozTDcwZUpYamhkR0NnTFFrQmpjCmFVY0VnV05IUnQ0YmxFUjlVSFVS + cE5KaEdMWXJRUW1GWlNXVzYzTTNvYVkKLS0tIGRFUXdNSW5RbnAzeUp5cVZ1ZWJ2 + azh3ZzhKb2VYUWZSdHo5NlRldGJQTGsK98hrnUGXdDQKol1duzHyYsd9PLL2eAEC + 4y+AUB5bVjYI3PEt3giBBUYFwIo478FWQ1w8iSNIUwWlhlqEB2hKNg== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-25T08:38:49Z" + mac: ENC[AES256_GCM,data:xW7scU3rX48jjA2eHvC7RDj3Fn5P3mPIqwLGpvHf0bsjbQo+huMPfTIafJRF4owum+SiRrASaDkABho6wxLkwiwQ/KlwPHW8QIXCpdgE/OJduQV9MIjAs7uCX+BSVZfVAHhgdR4ftAHhBNF6KrbZ3O9Elw7iZstsvsOoDRy8QPM=,iv:db0m7SeybD9igEPnEsm5M7evLnotq+L2owuSYADQs74=,tag:LyTBdJNskE4CsY//wgHUTA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index b19959a70..978640274 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -39,8 +39,8 @@ public: operator: 1Gi secrets: data: - DB_PASSWORD: "password" - CERT_PASS: "" + DB_PASSWORD: "" # Set via secrets.yaml or --set + CERT_PASS: "" # Set via secrets.yaml for encrypted SSL keys DOMAIN: "" HOME_PAGE_URL: "" @@ -118,6 +118,11 @@ backend: secretKeyRef: name: datamate-conf key: DB_PASSWORD + - name: JWT_SECRET + valueFrom: + secretKeyRef: + name: datamate-conf + key: JWT_SECRET - name: datamate.rag.milvus-uri value: "http://milvus:19530" - name: datamate.jwt.enable @@ -139,6 +144,10 @@ backend: backend-python: env: + - name: PGSQL_HOST + value: "datamate-database" + - name: PGSQL_PORT + value: "5432" - name: pgsql_password valueFrom: secretKeyRef: @@ -148,6 +157,18 @@ backend-python: value: *DATAMATE_JWT_ENABLE - name: milvus_uri value: "http://milvus:19530" + - name: LABEL_STUDIO_USER_TOKEN + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_USER_TOKEN + - name: LABEL_STUDIO_USERNAME + value: "admin@demo.com" + - name: LABEL_STUDIO_PASSWORD + valueFrom: + secretKeyRef: + name: datamate-conf + key: LABEL_STUDIO_PASSWORD volumes: - *datasetVolume - *flowVolume @@ -166,7 +187,7 @@ backend-python: gateway: env: - name: JWT_SECRET - value: "default-insecure-key-change-in-production" + value: "" # Set via secrets.yaml or --set for production - name: datamate.jwt.enable value: *DATAMATE_JWT_ENABLE - name: OMS_AUTH_ENABLED diff --git a/deployment/helm/label-studio/secrets.yaml b/deployment/helm/label-studio/secrets.yaml new file mode 100644 index 000000000..da2071797 --- /dev/null +++ b/deployment/helm/label-studio/secrets.yaml @@ -0,0 +1,19 @@ +env: + POSTGRE_PASSWORD: ENC[AES256_GCM,data:ROFSuT5BkJxGhusq2mGwxuBulcTNQeaVELxP6KimajWkGYBBys9oMJyPXHE=,iv:qqPQMjMUKpBJ76RlscGQ/6ff31e4iRDM0+IJ/q/TLYs=,tag:b5gCVSN/rn7aahqsoowayw==,type:str] + LABEL_STUDIO_PASSWORD: ENC[AES256_GCM,data:niFPf5mPSt4J2CpIhYaS+gUkj3XnwvkUy7VCWPykGd0=,iv:vm1OC2vRdBgHqjXPL/4th9EiSzZg9nGEDOn33axY1y4=,tag:wKGLsepNKasU4ccXv/OWfw==,type:str] + LABEL_STUDIO_USER_TOKEN: ENC[AES256_GCM,data:dmdU+7/Yd04jMCX5gaj4BpZhudCS2Bwe3FjqW40fX7g=,iv:GZS6oGS8oSBKYMjxBVuVmjEW0Ue/j9dfh50VBC9GvnI=,tag:GBhKPnPNyuaUaiXNfr4JEg==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDUGVacVpQSjFwRFQvWllZ + bWlLZitoQ2JLS0pHS2ZQcW91T3RaRVBIU2t3ClVBZkN4N2FmRENPYVo0dU9RWnlD + MVZXNUhjSWFSZWo0Yjg0RTJSSFpGU3MKLS0tIGcvcllDK0FtREphOU9YckhvQlJQ + NE8vVXdqajFHMnBnVHhQSGF3dEdkWnMKTy7oYsfbWxr3U6X1gyS2kKJLxIjy0Zr0 + 1BfP9FAx6CvrveKqXGWwHk8+8034sG0fKO1H1lrBAgzW79eDOU4i+Q== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:39Z" + mac: ENC[AES256_GCM,data:Gni4TWxwzTmjf2+/t64V/SRBttJFuq1qMQ7cZzm2tJu/3pwbFOgcJwDdkoYSRKZrpyngj5EskbWzIgbmv0I9Jts26m9dN4pMr45qxQ1ba2G8eYgx0s2klh7iEFoMVTJmnYCbnl0PuqSWmcHbCxxZdeyo3ja647RlfLarT8QZwvU=,iv:nrDwjR0v96jKSJfQ812fpfrmI6vQEcCzttTVX8mLftU=,tag:AP+JhPFE7CZY5aZX5M8O7w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/label-studio/values.yaml b/deployment/helm/label-studio/values.yaml index 229098ee5..456c6f242 100644 --- a/deployment/helm/label-studio/values.yaml +++ b/deployment/helm/label-studio/values.yaml @@ -41,7 +41,7 @@ env: DJANGO_DB: "default" POSTGRE_NAME: "labelstudio" POSTGRE_USER: "postgres" - POSTGRE_PASSWORD: "password" + POSTGRE_PASSWORD: "" # Set via secrets.yaml or --set POSTGRE_PORT: 5432 POSTGRE_HOST: "datamate-database" LABEL_STUDIO_HOST: "" # can be overridden @@ -49,9 +49,9 @@ env: LOCAL_FILES_DOCUMENT_ROOT: "/label-studio/local" USE_USERNAME_FOR_LOGIN: "true" LABEL_STUDIO_USERNAME: "admin@demo.com" - LABEL_STUDIO_PASSWORD: "demoadmin" + LABEL_STUDIO_PASSWORD: "" # Set via secrets.yaml or --set LABEL_STUDIO_ENABLE_LEGACY_API_TOKEN: "true" - LABEL_STUDIO_USER_TOKEN: "abc123abc123" + LABEL_STUDIO_USER_TOKEN: "" # Set via secrets.yaml or --set LOG_LEVEL: "DEBUG" persistence: diff --git a/deployment/helm/milvus/secrets.yaml b/deployment/helm/milvus/secrets.yaml new file mode 100644 index 000000000..014696cff --- /dev/null +++ b/deployment/helm/milvus/secrets.yaml @@ -0,0 +1,18 @@ +minio: + accessKey: ENC[AES256_GCM,data:57/fzPABN+5+c1tmf9lvR9O5EflmlWPQfgvxTnwHhIo=,iv:Fkyrk5f7md7QQD2fv/XPXLLTxLDUq2mmK2BOi4QcZyI=,tag:AykpxHYoQSTjs840FodsDg==,type:str] + secretKey: ENC[AES256_GCM,data:EgFQZaIInv6Ptn9a+PiXVydwhi8Wh3dO9QkWY9qD86M=,iv:+WWoLUoMJc6cc9a+aLR97O/xvv7f6yfAgFjuKtt62h8=,tag:O6r+/1KhgnVCdxRmzdS8pQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuMlNnRDZJdGRXTklkK0tp + aC9KQ0t2VTExL2tOS3BvZzNRY3o5TnFYa0ZZCkNqY3U1K3g5M2NERDFDR1g3M1hJ + VGcrNDR4Q0tCOWptTzF0MG1QN0pXWFkKLS0tIGhMMUEweUxleWV1Y1RTVmdDQ2k0 + cUZOVC94Qi8xQmNNMFBwWEM3MnhkQ0EK4bTM7Rw3tlaMFInXv1MKlu65iPBxY8Zw + G5h5QK4AOonWoxwrz3H5jFBAZwBsy156wDgBsxvZJB2oCa4wEfC4Wg== + -----END AGE ENCRYPTED FILE----- + recipient: age19lksug4t4qtjkak824m0d48d75nyx26p5mgv83yetnupu3w2fq3sx8kxdm + lastmodified: "2026-05-22T08:19:40Z" + mac: ENC[AES256_GCM,data:N5n9tUIujF8BpI+N4PMzBApaTmpO+Ji/ZTf7Dr/faM9+nGnrL/JcOwAjKwXXGnDbcFi4DFyvo7yx4u68Mx6c4EtIQYs4o9TymGiQAgAUR8STnjBB3pbGjlvG9MFYwKCVuXBclhKpCLDklI/cjkcJjJq4ae2VnHer0wMlSzRZj8I=,iv:L2vlzY9aRAUGBZhgNOl1t3QDZhP8Inffj0TkGXnGbug=,tag:iB3ZKbYAk0D4bc+05q8m6w==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/deployment/helm/milvus/values.yaml b/deployment/helm/milvus/values.yaml index 584cdcc27..626248e46 100644 --- a/deployment/helm/milvus/values.yaml +++ b/deployment/helm/milvus/values.yaml @@ -634,8 +634,8 @@ minio: image: tag: "RELEASE.2024-12-18T13-15-44Z" pullPolicy: IfNotPresent - accessKey: minioadmin - secretKey: minioadmin + accessKey: "" # Set via secrets.yaml or --set + secretKey: "" # Set via secrets.yaml or --set existingSecret: "" bucketName: "milvus-bucket" rootPath: file diff --git a/docs/SECRETS_SETUP.md b/docs/SECRETS_SETUP.md new file mode 100644 index 000000000..648e419d2 --- /dev/null +++ b/docs/SECRETS_SETUP.md @@ -0,0 +1,166 @@ +# Secrets Management Setup Guide + +This document describes the tools and configuration required for managing encrypted secrets in DataMate. + +## Required Tools + +### Homebrew Packages (macOS) + +| Tool | Version | Purpose | Install Command | +|------|---------|---------|-----------------| +| age | 1.3.1 | Encryption tool for secrets | `brew install age` | +| sops | 3.13.1 | Secrets encryption manager | `brew install sops` | +| helm | 4.1.0+ | Kubernetes package manager | `brew install helm` | + +### Helm Plugins + +| Plugin | Version | Purpose | Install Method | +|--------|---------|---------|----------------| +| helm-secrets | 4.6.0 | Helm integration with sops | Manual clone (see below) | + +**Note:** Helm 4.x has compatibility issues with helm-secrets plugin. Use manual installation: + +```bash +cd /Users/macoo/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +If plugin fails to load, edit `plugin.yaml` to remove the `command:` field and use only `platformCommand:`. + +## Project Configuration Files + +| File | Purpose | Location | +|------|---------|----------| +| `.sops.yaml` | SOPS encryption rules | Project root | +| `.sops-keys/key.txt` | Age encryption key (private) | Project root (gitignored) | +| `secrets.yaml` | Encrypted secrets per chart | `deployment/helm//` | + +## Quick Setup + +### 1. Install Required Tools + +```bash +# macOS (Homebrew) +brew install age sops helm + +# Helm plugin (manual) +cd ~/Library/helm/plugins +git clone --depth 1 --branch v4.6.0 https://github.com/jkroepke/helm-secrets.git helm-secrets +``` + +### 2. Generate Age Key (if not exists) + +```bash +mkdir -p .sops-keys +age-keygen -o .sops-keys/key.txt +``` + +**Important:** Backup the key file securely. It cannot be recovered if lost. + +### 3. Update `.sops.yaml` with Your Public Key + +```yaml +creation_rules: + - path_regex: deployment/helm/.*\.yaml$ + age: +``` + +### 4. Create/Encrypt Secrets Files + +```bash +# Set environment variable +export SOPS_AGE_KEY_FILE=.sops-keys/key.txt + +# Encrypt existing file +sops --encrypt --in-place deployment/helm/datamate/secrets.yaml + +# Or create new encrypted file +sops deployment/helm/datamate/secrets.yaml +``` + +## Usage + +### View Decrypted Secrets + +```bash +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml +``` + +### Deploy with Secrets + +```bash +# Using helper script +./scripts/secrets.sh helm-install datamate datamate + +# Manual method +TMP_FILE=$(mktemp) +SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt deployment/helm/datamate/secrets.yaml > $TMP_FILE +helm install datamate deployment/helm/datamate -n datamate -f $TMP_FILE +rm $TMP_FILE +``` + +### Docker Compose + +```bash +# Copy and edit .env +cp deployment/docker/datamate/.env.example deployment/docker/datamate/.env + +# Edit with your secrets +vim deployment/docker/datamate/.env + +# Deploy +docker compose -f deployment/docker/datamate/docker-compose.yml up -d +``` + +## Environment Variables + +| Variable | Purpose | +|----------|---------| +| `SOPS_AGE_KEY_FILE` | Path to age private key | +| `SOPS_CONFIG` | Path to .sops.yaml (optional) | + +## Security Notes + +1. **Never commit `.sops-keys/` directory** - It's in `.gitignore` +2. **Backup your age key** - Store in secure location (password manager, encrypted backup) +3. **Use strong passwords** - Generate with `openssl rand -base64 32` +4. **Rotate keys periodically** - Especially after team changes + +## Troubleshooting + +### Plugin Load Error + +If helm plugin shows "both platformCommand and command are set": +```bash +# Edit plugin.yaml +vim ~/Library/helm/plugins/helm-secrets/plugin.yaml +# Remove the "command:" line, keep only "platformCommand:" +``` + +### Decryption Failed + +```bash +# Check key file exists +ls -la .sops-keys/key.txt + +# Verify key matches encrypted file's recipient +grep "recipient:" deployment/helm/datamate/secrets.yaml +``` + +### Key Rotation + +```bash +# Generate new key +age-keygen -o .sops-keys/key-new.txt + +# Update .sops.yaml with new public key + +# Re-encrypt all secrets +for f in deployment/helm/*/secrets.yaml; do + SOPS_AGE_KEY_FILE=.sops-keys/key.txt sops --decrypt $f > /tmp/plain.yaml + SOPS_AGE_KEY_FILE=.sops-keys/key-new.txt sops --encrypt /tmp/plain.yaml > $f +done + +# Replace old key +mv .sops-keys/key-new.txt .sops-keys/key.txt +``` \ No newline at end of file diff --git a/runtime/datamate-python/app/core/config.py b/runtime/datamate-python/app/core/config.py index 5f6d3ae55..ed49fcf02 100644 --- a/runtime/datamate-python/app/core/config.py +++ b/runtime/datamate-python/app/core/config.py @@ -33,14 +33,14 @@ class Config: pgsql_host: str = "datamate-database" pgsql_port: int = 5432 pgsql_user: str = "postgres" - pgsql_password: str = "password" + pgsql_password: str = "" pgsql_database: str = "datamate" # Database mysql_host: str = "datamate-database" mysql_port: int = 3306 mysql_user: str = "root" - mysql_password: str = "password" + mysql_password: str = "" mysql_database: str = "datamate" database_url: str = "" # Will be overridden by build_database_url() if not provided @@ -63,9 +63,9 @@ def build_database_url(self): # Label Studio label_studio_base_url: str = "http://label-studio:8000" - label_studio_username: Optional[str] = "admin@demo.com" - label_studio_password: Optional[str] = "demoadmin" - label_studio_user_token: Optional[str] = "abc123abc123" # Legacy Token + label_studio_username: Optional[str] = None + label_studio_password: Optional[str] = None + label_studio_user_token: Optional[str] = None # Legacy Token label_studio_local_document_root: str = "/label-studio/local" # Label Studio local file storage path label_studio_file_path_prefix: str = "/data/local-files/?d=" # Label Studio local file serving URL prefix diff --git a/scripts/images/frontend/start.sh b/scripts/images/frontend/start.sh index 6102495fd..d61306fca 100644 --- a/scripts/images/frontend/start.sh +++ b/scripts/images/frontend/start.sh @@ -18,14 +18,17 @@ if [ -f "/cert/server.pem" ]; then fi if [ -f "/cert/server.key" ]; then - # Check if key is encrypted and decrypt if needed - # Supports RSA, EC (Elliptic Curve), PKCS#8, and DSA keys + # Private key MUST be encrypted. Plaintext keys are not accepted. if grep -q "ENCRYPTED" /cert/server.key 2>/dev/null; then - # Key is encrypted, decrypt using generic pkey command (supports all key types) + if [ -z "$CERT_PASS" ]; then + echo "ERROR: CERT_PASS is required to decrypt the private key" + exit 1 + fi echo "$CERT_PASS" | openssl pkey -in /cert/server.key -out /etc/nginx/cert/server.key -passin stdin + chmod 600 /etc/nginx/cert/server.key else - # Key is not encrypted, copy directly - cp /cert/server.key /etc/nginx/cert/server.key + echo "ERROR: /cert/server.key is not encrypted. Please encrypt it before deployment." + exit 1 fi chown nginx:nginx /etc/nginx/cert/server.key fi diff --git a/scripts/secrets.sh b/scripts/secrets.sh new file mode 100755 index 000000000..946d982dd --- /dev/null +++ b/scripts/secrets.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# DataMate Secrets Helper +# Manages SOPS-encrypted secrets for Helm deployment. +# Docker users do NOT need this script - use .env file instead. + +set -e + +SOPS_KEY_FILE="${SOPS_KEY_FILE:-$PWD/.sops-keys/key.txt}" +SOPS_CONFIG="${SOPS_CONFIG:-$PWD/.sops.yaml}" + +# Check required tools +check_tools() { + local missing="" + command -v sops >/dev/null 2>&1 || missing="$missing sops" + command -v age >/dev/null 2>&1 || missing="$missing age" + command -v age-keygen >/dev/null 2>&1 || missing="$missing age-keygen" + if [ -n "$missing" ]; then + echo "Error: Required tools not found:$missing" + echo "Install with: brew install sops age" + exit 1 + fi +} + +# Check or generate age key +ensure_key() { + if [ ! -f "$SOPS_KEY_FILE" ]; then + echo "Age key not found at $SOPS_KEY_FILE" + echo "Generating a new one..." + mkdir -p "$(dirname "$SOPS_KEY_FILE")" + age-keygen -o "$SOPS_KEY_FILE" + echo "Key generated. Keep this file secure and never commit it." + fi + export SOPS_AGE_KEY_FILE="$SOPS_KEY_FILE" +} + +case "$1" in + encrypt) + check_tools + if [ -z "$2" ]; then + echo "Usage: $0 encrypt " + exit 1 + fi + sops --encrypt --in-place "$2" + echo "Encrypted: $2" + ;; + decrypt) + check_tools + if [ -z "$2" ]; then + echo "Usage: $0 decrypt " + exit 1 + fi + sops --decrypt --in-place "$2" + echo "Decrypted: $2" + ;; + view) + check_tools + if [ -z "$2" ]; then + echo "Usage: $0 view " + exit 1 + fi + sops --decrypt "$2" + ;; + helm-upgrade) + # Usage: scripts/secrets.sh helm-upgrade [extra-helm-args...] + check_tools + ensure_key + if [ -z "$2" ] || [ -z "$3" ]; then + echo "Usage: $0 helm-upgrade [extra-helm-args...]" + exit 1 + fi + CHART="$2" + NAMESPACE="$3" + SECRETS_FILE="deployment/helm/$CHART/secrets.yaml" + shift 3 + + if [ ! -f "$SECRETS_FILE" ]; then + echo "Warning: No secrets file at $SECRETS_FILE, deploying without secrets." + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" "$@" + else + TMP_FILE=$(mktemp) + sops --decrypt "$SECRETS_FILE" > "$TMP_FILE" + helm upgrade --install "$CHART" "deployment/helm/$CHART" -n "$NAMESPACE" -f "$TMP_FILE" "$@" + rm "$TMP_FILE" + echo "Deployed $CHART with decrypted secrets." + fi + ;; + *) + echo "DataMate Secrets Helper" + echo "" + echo "Usage: $0 [args...]" + echo "" + echo "Commands:" + echo " encrypt Encrypt a YAML file in-place" + echo " decrypt Decrypt a YAML file in-place" + echo " view Print decrypted content" + echo " helm-upgrade Decrypt secrets and helm upgrade --install" + echo "" + echo "Docker users: skip this script, use 'cp .env.example .env' instead." + exit 1 + ;; +esac \ No newline at end of file