diff --git a/.github/actions/e2e-ready/action.yml b/.github/actions/e2e-ready/action.yml index 3c75f388..3d2361a6 100644 --- a/.github/actions/e2e-ready/action.yml +++ b/.github/actions/e2e-ready/action.yml @@ -71,9 +71,7 @@ runs: - name: Use test environment config shell: bash - run: | - cp backend/config.test.toml backend/config.toml - cp backend/secrets.example.toml backend/secrets.toml + run: cp backend/config.test.toml backend/config.toml - name: Pre-pull test runtime images into K3s shell: bash diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index f041b180..f417ab59 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,9 +42,6 @@ jobs: - name: Install backend dependencies run: cd backend && uv sync --frozen --no-dev - - name: Set up config for OpenAPI generation - run: cp backend/secrets.example.toml backend/secrets.toml - - name: Generate OpenAPI spec run: ./deploy.sh openapi diff --git a/.github/workflows/release-deploy.yml b/.github/workflows/release-deploy.yml index d29996b4..63783077 100644 --- a/.github/workflows/release-deploy.yml +++ b/.github/workflows/release-deploy.yml @@ -132,11 +132,12 @@ jobs: REDIS_PASSWORD: ${{ secrets.REDIS_PASSWORD }} MONGO_USER: ${{ secrets.MONGO_USER }} MONGO_PASSWORD: ${{ secrets.MONGO_PASSWORD }} + SECRET_KEY: ${{ secrets.JWT_SECRET_KEY }} with: host: ${{ secrets.DEPLOY_HOST }} username: ${{ secrets.DEPLOY_USER }} key: ${{ secrets.DEPLOY_SSH_KEY }} - envs: GHCR_TOKEN,GHCR_USER,IMAGE_TAG,GRAFANA_ADMIN_USER,GRAFANA_ADMIN_PASSWORD,MAILJET_API_KEY,MAILJET_SECRET_KEY,MAILJET_FROM_ADDRESS,MAILJET_HOST,GRAFANA_ALERT_RECIPIENTS,REDIS_PASSWORD,MONGO_USER,MONGO_PASSWORD + envs: GHCR_TOKEN,GHCR_USER,IMAGE_TAG,GRAFANA_ADMIN_USER,GRAFANA_ADMIN_PASSWORD,MAILJET_API_KEY,MAILJET_SECRET_KEY,MAILJET_FROM_ADDRESS,MAILJET_HOST,GRAFANA_ALERT_RECIPIENTS,REDIS_PASSWORD,MONGO_USER,MONGO_PASSWORD,SECRET_KEY command_timeout: 10m script: | set -e @@ -148,6 +149,10 @@ jobs: export IMAGE_TAG="$IMAGE_TAG" export COMPOSE_PROFILES=observability + export SECRET_KEY="$SECRET_KEY" + export MONGO_USER="$MONGO_USER" + export MONGO_PASSWORD="$MONGO_PASSWORD" + export REDIS_PASSWORD="$REDIS_PASSWORD" export GRAFANA_ROOT_URL="https://grafana.integr8scode.cc/" export GRAFANA_ADMIN_USER="$GRAFANA_ADMIN_USER" export GRAFANA_ADMIN_PASSWORD="$GRAFANA_ADMIN_PASSWORD" @@ -156,9 +161,6 @@ jobs: export MAILJET_FROM_ADDRESS="$MAILJET_FROM_ADDRESS" export GF_SMTP_HOST="$MAILJET_HOST" export GRAFANA_ALERT_RECIPIENTS="$GRAFANA_ALERT_RECIPIENTS" - export REDIS_PASSWORD="$REDIS_PASSWORD" - export MONGO_USER="$MONGO_USER" - export MONGO_PASSWORD="$MONGO_PASSWORD" docker compose pull docker compose up -d --remove-orphans --no-build --wait --wait-timeout 180 diff --git a/.github/workflows/stack-tests.yml b/.github/workflows/stack-tests.yml index d023820b..eee041c3 100644 --- a/.github/workflows/stack-tests.yml +++ b/.github/workflows/stack-tests.yml @@ -54,9 +54,6 @@ jobs: uv python install 3.12 uv sync --frozen --group test --no-dev - - name: Set up config - run: cp backend/secrets.example.toml backend/secrets.toml - - name: Run unit tests timeout-minutes: 5 run: | diff --git a/AGENTS.md b/AGENTS.md index 4a230900..c8df9d9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -438,22 +438,22 @@ class ExecutionService: ## Backend: Settings (TOML — No Env Vars) ```python -# WRONG — never use environment variables +# WRONG — never read config directly from env vars in application code DATABASE_URL = os.getenv("DATABASE_URL") -# CORRECT — inject via DI +# CORRECT — inject via DI (Settings reads env vars for secrets internally) settings: FromDishka[Settings] ``` Config load order (each overrides previous): 1. `config.toml` — base settings (committed) -2. `secrets.toml` — credentials (gitignored, never committed) -3. `config..toml` — per-worker overrides (optional) +2. `config..toml` — per-worker overrides (optional) +3. Environment variables for secrets: `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD` ```python -Settings() # config.toml + secrets.toml -Settings(config_path="config.test.toml") # tests -Settings(override_path="config.saga-orchestrator.toml") # worker +Settings() # config.toml + env vars +Settings(config_path="config.test.toml") # tests + env vars +Settings(override_path="config.saga-orchestrator.toml") # worker + env vars ``` `Settings` uses `model_config = ConfigDict(extra="forbid")` — unknown keys are errors. diff --git a/README.md b/README.md index dfcf8041..60f78dda 100644 --- a/README.md +++ b/README.md @@ -70,11 +70,10 @@ things safe and efficient. You'll get the results back in no time. ```bash git clone https://github.com/HardMax71/Integr8sCode.git cd Integr8sCode -cp backend/secrets.example.toml backend/secrets.toml ./deploy.sh dev ``` -The `secrets.toml` file holds credentials and is gitignored. The example template has working development defaults. +Secrets (MongoDB, Redis, JWT) come from environment variables with dev defaults — zero config needed for local development. For production, set `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD` as env vars. ### Verify diff --git a/backend/app/settings.py b/backend/app/settings.py index a7953175..96f6b930 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -1,5 +1,5 @@ +import os import tomllib -from pathlib import Path from typing import Literal from urllib.parse import quote @@ -11,19 +11,19 @@ class Settings(BaseModel): - """Application settings loaded from TOML configuration files. + """Application settings loaded from TOML + environment variables. - All config is read from TOML — no environment variables, no .env files. + Non-secret config comes from TOML files; secrets come from env vars with dev defaults. Load order (each layer overrides the previous): 1. config_path — base settings (committed to git) - 2. secrets_path — sensitive overrides (gitignored, mounted from K8s Secret in prod) - 3. override_path — per-worker service overrides (TRACING_SERVICE_NAME, etc.) + 2. override_path — per-worker service overrides (TRACING_SERVICE_NAME, etc.) + 3. Environment variables for secrets: SECRET_KEY, MONGO_USER, MONGO_PASSWORD, REDIS_PASSWORD Usage: - Settings() # config.toml + secrets - Settings(config_path="config.test.toml") # test config (has own secrets) - Settings(override_path="config.saga-orchestrator.toml") # base + secrets + worker + Settings() # config.toml + env vars + Settings(config_path="config.test.toml") # test config + env vars + Settings(override_path="config.saga-orchestrator.toml") # base + worker + env vars """ model_config = ConfigDict(extra="forbid") @@ -32,16 +32,19 @@ def __init__( self, config_path: str = "config.toml", override_path: str | None = None, - secrets_path: str = "secrets.toml", ) -> None: with open(config_path, "rb") as f: data = tomllib.load(f) - if Path(secrets_path).is_file(): - with open(secrets_path, "rb") as f: - data |= tomllib.load(f) if override_path: with open(override_path, "rb") as f: data |= tomllib.load(f) + for key, default in ( + ("SECRET_KEY", "CHANGE_ME_min_32_chars_long_!!!!"), + ("MONGO_USER", "root"), + ("MONGO_PASSWORD", "rootpassword"), + ("REDIS_PASSWORD", "redispassword"), + ): + data[key] = os.environ.get(key) or data.get(key) or default if data.get("MONGO_USER") and data.get("MONGO_PASSWORD"): user = quote(data["MONGO_USER"], safe="") password = quote(data["MONGO_PASSWORD"], safe="") @@ -176,7 +179,7 @@ def __init__( DEVELOPMENT_MODE: bool = False SECURE_COOKIES: bool = True - # CORS allowed origins (overridable via config.toml / secrets.toml) + # CORS allowed origins (overridable via config.toml) CORS_ORIGINS: list[str] = Field(default_factory=lambda: [ "https://localhost:5001", "https://127.0.0.1:5001", diff --git a/backend/config.test.toml b/backend/config.test.toml index 4aa46cdd..0c7ff04d 100644 --- a/backend/config.test.toml +++ b/backend/config.test.toml @@ -1,6 +1,6 @@ # Integr8sCode backend test configuration # Differences from config.toml: lower timeouts, faster bcrypt, relaxed rate limits -# Secrets (SECRET_KEY, MONGODB_URL) live in secrets.toml — use secrets.test.toml in CI. +# Secrets come from environment variables with dev defaults (see settings.py). ENVIRONMENT = "test" PROJECT_NAME = "integr8scode" diff --git a/backend/config.toml b/backend/config.toml index 4c35a236..5863cdf1 100644 --- a/backend/config.toml +++ b/backend/config.toml @@ -1,8 +1,8 @@ # --8<-- [start:core] # Integr8sCode backend configuration (development defaults). -# Secrets (SECRET_KEY, MONGO_USER, MONGO_PASSWORD) live in secrets.toml (gitignored). -# Production: mount secrets.toml from a Kubernetes Secret or generate in CI. -# See secrets.example.toml for the required keys. +# Secrets (SECRET_KEY, MONGO_USER, MONGO_PASSWORD, REDIS_PASSWORD) come from +# environment variables with dev defaults — see Settings.__init__() in settings.py. +# Production: set env vars via docker-compose or GitHub Actions secrets. PROJECT_NAME = "integr8scode" DATABASE_NAME = "integr8scode_db" diff --git a/backend/secrets.example.toml b/backend/secrets.example.toml deleted file mode 100644 index eb424c4f..00000000 --- a/backend/secrets.example.toml +++ /dev/null @@ -1,24 +0,0 @@ -# Copy to secrets.toml and fill in real values. -# cp secrets.example.toml secrets.toml -# -# Required keys (no defaults in config.toml): -# SECRET_KEY — JWT signing key, min 32 characters -# MONGO_USER — MongoDB root username -# MONGO_PASSWORD — MongoDB root password -# -# The backend builds MONGODB_URL from MONGO_USER + MONGO_PASSWORD + MONGO_HOST/PORT/DB -# (host/port/db have defaults in config.toml; only credentials live here). -# -# GitHub Actions: create secrets.toml from repository secrets: -# cat > backend/secrets.toml << EOF -# SECRET_KEY = "${{ secrets.JWT_SECRET_KEY }}" -# MONGO_USER = "${{ secrets.MONGO_USER }}" -# MONGO_PASSWORD = "${{ secrets.MONGO_PASSWORD }}" -# EOF -# -# Kubernetes: store as a Secret and mount at /app/secrets.toml - -SECRET_KEY = "CHANGE_ME_min_32_chars_long_!!!!" -MONGO_USER = "root" -MONGO_PASSWORD = "rootpassword" -REDIS_PASSWORD = "redispassword" diff --git a/docker-compose.yaml b/docker-compose.yaml index e796f7b4..0fb58f99 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,3 +1,9 @@ +x-backend-secrets: &backend-secrets + SECRET_KEY: ${SECRET_KEY:-CHANGE_ME_min_32_chars_long_!!!!} + MONGO_USER: ${MONGO_USER:-root} + MONGO_PASSWORD: ${MONGO_PASSWORD:-rootpassword} + REDIS_PASSWORD: ${REDIS_PASSWORD:-redispassword} + services: # Shared base image for all Python backend services base: @@ -113,7 +119,6 @@ services: - ./backend/certs:/app/certs:ro - ./backend/config.test.toml:/app/config.test.toml:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - shared_ca:/shared_ca:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro ports: @@ -123,9 +128,10 @@ services: - app-network container_name: backend environment: - - WEB_CONCURRENCY=${WEB_CONCURRENCY:-2} - - WEB_THREADS=${WEB_THREADS:-4} - - WEB_TIMEOUT=${WEB_TIMEOUT:-60} + <<: *backend-secrets + WEB_CONCURRENCY: ${WEB_CONCURRENCY:-2} + WEB_THREADS: ${WEB_THREADS:-4} + WEB_TIMEOUT: ${WEB_TIMEOUT:-60} extra_hosts: - "host.docker.internal:host-gateway" mem_limit: 768m @@ -301,11 +307,11 @@ services: mongo: condition: service_healthy environment: - - DEFAULT_USER_PASSWORD=${DEFAULT_USER_PASSWORD:-user123} - - ADMIN_USER_PASSWORD=${ADMIN_USER_PASSWORD:-admin123} + <<: *backend-secrets + DEFAULT_USER_PASSWORD: ${DEFAULT_USER_PASSWORD:-user123} + ADMIN_USER_PASSWORD: ${ADMIN_USER_PASSWORD:-admin123} volumes: - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro command: ["python", "-m", "scripts.seed_users"] networks: - app-network @@ -322,12 +328,13 @@ services: condition: service_healthy mongo: condition: service_started + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.k8s-worker.toml:/app/config.k8s-worker.toml:ro networks: - app-network @@ -343,12 +350,13 @@ services: depends_on: kafka: condition: service_healthy + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.pod-monitor.toml:/app/config.pod-monitor.toml:ro networks: - app-network @@ -366,12 +374,13 @@ services: condition: service_healthy mongo: condition: service_started + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/kubeconfig.yaml:/app/kubeconfig.yaml:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.result-processor.toml:/app/config.result-processor.toml:ro networks: - app-network @@ -389,11 +398,12 @@ services: condition: service_healthy mongo: condition: service_started + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.saga-orchestrator.toml:/app/config.saga-orchestrator.toml:ro networks: - app-network @@ -452,11 +462,12 @@ services: condition: service_healthy mongo: condition: service_started + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.event-replay.toml:/app/config.event-replay.toml:ro networks: - app-network @@ -473,11 +484,12 @@ services: condition: service_healthy mongo: condition: service_started + environment: + <<: *backend-secrets volumes: - ./backend/app:/app/app:ro - ./backend/workers:/app/workers:ro - ./backend/config.toml:/app/config.toml:ro - - ./backend/secrets.toml:/app/secrets.toml:ro - ./backend/config.dlq-processor.toml:/app/config.dlq-processor.toml:ro networks: - app-network diff --git a/docs/getting-started.md b/docs/getting-started.md index df9eaae4..f1fb1605 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -11,11 +11,10 @@ The full stack includes a Svelte frontend, FastAPI backend, MongoDB, Redis, Kafk ```bash git clone https://github.com/HardMax71/Integr8sCode.git cd Integr8sCode -cp backend/secrets.example.toml backend/secrets.toml ./deploy.sh dev ``` -The `secrets.toml` file holds credentials (`SECRET_KEY`, `MONGODB_URL`) and is gitignored. The example template ships with working development defaults, so copying it is all you need for local use. +Secrets (JWT key, MongoDB and Redis credentials) are read from environment variables with dev defaults baked in — no config files to copy for local development. Wait for the services to come up. You can watch progress with `docker compose logs -f` in another terminal. When you see the backend responding to health checks, you're ready. diff --git a/docs/operations/cicd.md b/docs/operations/cicd.md index 2588889c..0c0d278f 100644 --- a/docs/operations/cicd.md +++ b/docs/operations/cicd.md @@ -191,7 +191,7 @@ This action finalizes the environment after boot tasks complete: 1. **Finalize k3s** — copies kubeconfig, rewrites the API server address to `host.docker.internal` so containers inside docker compose can reach the k3s API server, creates the `integr8scode` namespace 2. **Start cert-generator** in the background -3. **Copy test config** — uses `config.test.toml` and `secrets.example.toml` +3. **Copy test config** — uses `config.test.toml` (secrets come from env var defaults) 4. **Wait for image pull and infra** — blocks until the background pull completes and checks the exit code from `/tmp/infra-pull.exit`, failing fast if the background process had errors 5. **Start compose stack** with `docker compose up -d --no-build` diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 5969d69e..7d22cd6e 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -167,7 +167,7 @@ docker compose logs backend | Issue | Cause | Solution | |-----------------------|-----------------------------------|---------------------------------------------------| | Unknown topic errors | Kafka not ready or wrong prefix | Check `docker compose logs kafka` | -| MongoDB auth errors | Password mismatch | Verify `secrets.toml` matches compose env vars | +| MongoDB auth errors | Password mismatch | Verify `MONGO_USER`/`MONGO_PASSWORD` env vars match MongoDB init | | Worker crash loop | Config file missing | Ensure `config..toml` exists | ### Kafka topic debugging diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 81c8a6a4..4913dd5d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1,35 +1,34 @@ # Configuration Reference -All backend configuration is loaded from TOML files — no environment variables, no `.env` files. The `Settings` class (`app/settings.py`) reads files in this order, each layer overriding the previous: +Non-secret configuration is loaded from TOML files. Secrets come from environment variables with dev defaults. The `Settings` class (`app/settings.py`) reads in this order, each layer overriding the previous: 1. **`config.toml`** — base settings, committed to git (no secrets) -2. **`secrets.toml`** — sensitive values (`SECRET_KEY`, `MONGODB_URL`), gitignored -3. **per-worker override** — optional TOML file for service-specific settings (e.g. `config.saga-orchestrator.toml`) +2. **per-worker override** — optional TOML file for service-specific settings (e.g. `config.saga-orchestrator.toml`) +3. **Environment variables** for secrets: `SECRET_KEY`, `MONGO_USER`, `MONGO_PASSWORD`, `REDIS_PASSWORD` ```python -# Default — reads config.toml + secrets.toml +# Default — reads config.toml + env vars Settings() -# Tests — reads config.test.toml + secrets.toml +# Tests — reads config.test.toml + env vars Settings(config_path="config.test.toml") -# Worker — reads config.toml + secrets.toml + worker override +# Worker — reads config.toml + worker override + env vars Settings(override_path="config.saga-orchestrator.toml") ``` ## Secrets -Credentials live in `secrets.toml`, which is gitignored. A committed template with development defaults is provided: +Secrets are read from environment variables. Dev defaults are built in so local development requires zero configuration: -```bash -cp backend/secrets.example.toml backend/secrets.toml -``` - -```toml ---8<-- "backend/secrets.example.toml" -``` +| Variable | Description | Dev default | +|----------|-------------|-------------| +| `SECRET_KEY` | JWT signing key (min 32 chars) | `CHANGE_ME_min_32_chars_long_!!!!` | +| `MONGO_USER` | MongoDB root username | `root` | +| `MONGO_PASSWORD` | MongoDB root password | `rootpassword` | +| `REDIS_PASSWORD` | Redis password | `redispassword` | -For production, mount `secrets.toml` from a Kubernetes Secret at `/app/secrets.toml`. In CI, generate it from repository secrets (see the template comments for an example). +Docker Compose passes these via a shared `x-backend-secrets` YAML anchor to all backend services. For production, set the env vars through GitHub Actions secrets or your deployment platform. ## Core @@ -42,10 +41,10 @@ For production, mount `secrets.toml` from a Kubernetes Secret at `/app/secrets.t |-----|-------------|---------| | `PROJECT_NAME` | Application name for logs and metadata | `integr8scode` | | `DATABASE_NAME` | MongoDB database name | `integr8scode_db` | - | `SECRET_KEY` | JWT signing key, min 32 chars. **Lives in `secrets.toml`** | — (required) | + | `SECRET_KEY` | JWT signing key, min 32 chars. **From env var** | `CHANGE_ME_min_32_chars_long_!!!!` | | `ALGORITHM` | JWT signing algorithm | `HS256` | | `ACCESS_TOKEN_EXPIRE_MINUTES` | Token lifetime in minutes | `1440` (24h) | - | `MONGODB_URL` | MongoDB connection string. **Lives in `secrets.toml`** | `mongodb://mongo:27017/integr8scode` | + | `MONGODB_URL` | MongoDB connection string. **Built from `MONGO_USER`/`MONGO_PASSWORD` env vars + `MONGO_HOST`/`MONGO_PORT`/`MONGO_DB` from TOML** | (computed) | ## Kubernetes @@ -164,7 +163,7 @@ For production, mount `secrets.toml` from a Kubernetes Secret at `/app/secrets.t ## Worker overrides -Each worker runs with a small override TOML that sets `TRACING_SERVICE_NAME` and `KAFKA_CONSUMER_GROUP_ID`. These are mounted alongside `config.toml` and `secrets.toml` in Docker Compose: +Each worker runs with a small override TOML that sets `TRACING_SERVICE_NAME` and `KAFKA_CONSUMER_GROUP_ID`. These are mounted alongside `config.toml` in Docker Compose: | File | Service | |------|---------| @@ -183,8 +182,4 @@ Each worker runs with a small override TOML that sets `TRACING_SERVICE_NAME` and Settings(config_path="config.test.toml") ``` -Secrets are still loaded from `secrets.toml`. In CI, the workflow copies the example template: - -```bash -cp backend/secrets.example.toml backend/secrets.toml -``` +Secrets come from environment variables with dev defaults — no extra setup needed in CI or locally.