diff --git a/crates/bin/ampd/src/main.rs b/crates/bin/ampd/src/main.rs index bdee4ed8b..76c6230f6 100644 --- a/crates/bin/ampd/src/main.rs +++ b/crates/bin/ampd/src/main.rs @@ -35,24 +35,24 @@ enum Command { /// If `--config` is not provided, searches for `config.toml` within this directory. #[arg(long, env = "AMP_DIR")] amp_dir: Option, - /// Enable Arrow Flight RPC Server. - #[arg(long, env = "FLIGHT_SERVER")] - flight_server: bool, - /// Enable JSON Lines Server. - #[arg(long, env = "JSONL_SERVER")] - jsonl_server: bool, - /// Enable Admin API Server. - #[arg(long, env = "ADMIN_SERVER")] - admin_server: bool, + /// Enable Arrow Flight RPC Server (env: AMP_FLIGHT_SERVER). + #[arg(long, num_args = 0..=1, default_missing_value = "true")] + flight_server: Option, + /// Enable JSON Lines Server (env: AMP_JSONL_SERVER). + #[arg(long, num_args = 0..=1, default_missing_value = "true")] + jsonl_server: Option, + /// Enable Admin API Server (env: AMP_ADMIN_SERVER). + #[arg(long, num_args = 0..=1, default_missing_value = "true")] + admin_server: Option, }, /// Run query server (Arrow Flight, JSON Lines) Server { - /// Enable Arrow Flight RPC Server. - #[arg(long, env = "FLIGHT_SERVER")] - flight_server: bool, - /// Enable JSON Lines Server. - #[arg(long, env = "JSONL_SERVER")] - jsonl_server: bool, + /// Enable Arrow Flight RPC Server (env: AMP_FLIGHT_SERVER). + #[arg(long, num_args = 0..=1, default_missing_value = "true")] + flight_server: Option, + /// Enable JSON Lines Server (env: AMP_JSONL_SERVER). + #[arg(long, num_args = 0..=1, default_missing_value = "true")] + jsonl_server: Option, }, /// Run a distributed worker node Worker { @@ -107,16 +107,10 @@ async fn main_inner() -> Result<(), Error> { match command { Command::Solo { amp_dir, - mut flight_server, - mut jsonl_server, - mut admin_server, + flight_server, + jsonl_server, + admin_server, } => { - if !flight_server && !jsonl_server && !admin_server { - flight_server = true; - jsonl_server = true; - admin_server = true; - } - // Resolve the amp state directory, creating it if needed. let amp_dir = match amp_dir { Some(dir) => dir, @@ -150,17 +144,21 @@ async fn main_inner() -> Result<(), Error> { Ok(()) } Command::Server { - mut flight_server, - mut jsonl_server, + flight_server: cli_flight_server, + jsonl_server: cli_jsonl_server, } => { - // If neither of the flags are set, enable both servers + let config_path = config_path.as_ref().ok_or(Error::MissingConfigPath)?; + let config = config::load(config_path).map_err(Error::LoadConfig)?; + + // Merge CLI flags with config values (CLI takes precedence when provided) + let mut flight_server = cli_flight_server.unwrap_or(config.flight_server); + let mut jsonl_server = cli_jsonl_server.unwrap_or(config.jsonl_server); + + // If neither flag is set (CLI or config), enable both servers if !flight_server && !jsonl_server { flight_server = true; jsonl_server = true; } - - let config_path = config_path.as_ref().ok_or(Error::MissingConfigPath)?; - let config = config::load(config_path).map_err(Error::LoadConfig)?; let metadata_db_config = amp_config::metadb::load(config_path, None) .ok_or(Error::MissingMetadataDbConfig)?; let addrs = config.server_addrs.clone(); diff --git a/crates/bin/ampd/src/solo_cmd.rs b/crates/bin/ampd/src/solo_cmd.rs index 9bc60d5f2..1f14c117b 100644 --- a/crates/bin/ampd/src/solo_cmd.rs +++ b/crates/bin/ampd/src/solo_cmd.rs @@ -23,9 +23,9 @@ type ServerFuture = Pin, - flight_server: bool, - jsonl_server: bool, - admin_server: bool, + flight_server: Option, + jsonl_server: Option, + admin_server: Option, ) -> Result<(), Error> { // 1. Check if user provided metadata_db.url before starting managed PostgreSQL let effective_config_path = config_path @@ -81,6 +81,18 @@ pub async fn run( ) .map_err(Error::ConfigLoad)?; + // Merge CLI flags with config values (CLI takes precedence when provided) + let mut flight_server = flight_server.unwrap_or(config.flight_server); + let mut jsonl_server = jsonl_server.unwrap_or(config.jsonl_server); + let mut admin_server = admin_server.unwrap_or(config.admin_server); + + // If no servers are explicitly enabled (CLI or config), enable all + if !flight_server && !jsonl_server && !admin_server { + flight_server = true; + jsonl_server = true; + admin_server = true; + } + // 4. Init monitoring let (_providers, meter) = monitoring::init(config.opentelemetry.as_ref()).map_err(Error::MonitoringInit)?; diff --git a/crates/config/src/config_file.rs b/crates/config/src/config_file.rs index 07191a8b3..5e8ab9350 100644 --- a/crates/config/src/config_file.rs +++ b/crates/config/src/config_file.rs @@ -17,7 +17,9 @@ //! | 1 (highest) | `AMP_CONFIG_*` env vars | `merge` — always wins | //! | 2 | TOML file values | `merge` — base configuration | //! | 3 | Deprecated flat keys (`metadata_db_url`, `dataset_defs_dir`) | `join` — fallback only | -//! | 4 (lowest) | Caller-provided `ConfigDefaultsOverride`s | `join` — fallback only | +//! | 4 | Server flag env vars (`AMP_FLIGHT_SERVER`, `AMP_JSONL_SERVER`, `AMP_ADMIN_SERVER`) | `join` — fallback only | +//! | 5 | Deprecated env vars (`FLIGHT_SERVER`, `JSONL_SERVER`, `ADMIN_SERVER`) | `join` — fallback only | +//! | 6 (lowest) | Caller-provided `ConfigDefaultsOverride`s | `join` — fallback only | //! //! ## Environment variables //! @@ -25,7 +27,10 @@ //! separate nested keys. For example, `AMP_CONFIG_METADATA_DB__URL` maps to //! `metadata_db.url` in the config file. -use std::path::{Path, PathBuf}; +use std::{ + env, + path::{Path, PathBuf}, +}; use dump::{ConfigDuration, ParquetConfig}; use figment::{ @@ -34,6 +39,16 @@ use figment::{ }; use monitoring::config::OpenTelemetryConfig; +/// Environment variable migrations for server enable flags. +/// +/// Each tuple contains (new_name, deprecated_name, config_key). The new env var names +/// use the `AMP_*` prefix for consistency with other CLI env vars (AMP_CONFIG, AMP_DIR, AMP_NODE_ID). +const SERVER_FLAG_ENV_VARS: &[(&str, &str, &str)] = &[ + ("AMP_FLIGHT_SERVER", "FLIGHT_SERVER", "flight_server"), + ("AMP_JSONL_SERVER", "JSONL_SERVER", "jsonl_server"), + ("AMP_ADMIN_SERVER", "ADMIN_SERVER", "admin_server"), +]; + /// Default data directory name - stores Parquet files pub const DEFAULT_DATA_DIRNAME: &str = "data"; @@ -128,6 +143,25 @@ pub fn load( figment = figment.join(Serialized::default("manifests_dir", value)); } + // Handle server flag env vars (AMP_FLIGHT_SERVER, etc.) and their deprecated names + // Priority: TOML/AMP_CONFIG_* (already merged above) > AMP_*_SERVER > FLIGHT_SERVER (deprecated) + for (new_env, deprecated_env, config_key) in SERVER_FLAG_ENV_VARS { + // Check new env var first (AMP_FLIGHT_SERVER) + if let Ok(value) = env::var(new_env) { + let bool_value = parse_bool_env(&value); + figment = figment.join(Serialized::default(config_key, bool_value)); + } + // Fall back to deprecated env var (FLIGHT_SERVER) with warning + else if let Ok(value) = env::var(deprecated_env) { + tracing::warn!( + "environment variable `{deprecated_env}` is deprecated; \ + use `{new_env}` instead" + ); + let bool_value = parse_bool_env(&value); + figment = figment.join(Serialized::default(config_key, bool_value)); + } + } + // Caller-provided ConfigDefaultsOverride (lowest priority — fills gaps only) for ConfigDefaultsOverride(provider) in defaults { figment = figment.join(provider); @@ -187,6 +221,17 @@ pub struct ConfigFile { /// Admin API server address (default: "0.0.0.0:1610") pub admin_api_addr: Option, + // Service enable flags (for solo/server modes) + /// Enable Arrow Flight RPC server (default: false, env: AMP_FLIGHT_SERVER) + #[serde(default)] + pub flight_server: bool, + /// Enable JSON Lines server (default: false, env: AMP_JSONL_SERVER) + #[serde(default)] + pub jsonl_server: bool, + /// Enable Admin API server (default: false, env: AMP_ADMIN_SERVER) + #[serde(default)] + pub admin_server: bool, + // Observability pub opentelemetry: Option, @@ -254,3 +299,8 @@ fn default_keep_alive_interval() -> u64 { #[derive(Debug, thiserror::Error)] #[error("Failed to load configuration file")] pub struct LoadConfigFileError(#[source] pub Box); + +/// Parses a string as a boolean. Truthy values: "1", "true", "yes", "on" (case-insensitive). +fn parse_bool_env(value: &str) -> bool { + matches!(value.to_lowercase().as_str(), "1" | "true" | "yes" | "on") +} diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index f88b553d2..c221256d1 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -106,6 +106,9 @@ fn resolve_config( config_path, poll_interval: config_file.poll_interval_secs.into(), keep_alive_interval: config_file.keep_alive_interval, + flight_server: config_file.flight_server, + jsonl_server: config_file.jsonl_server, + admin_server: config_file.admin_server, worker_events, }) } @@ -223,6 +226,12 @@ pub struct Config { pub poll_interval: Duration, /// Keep-alive interval for streaming server (in seconds). pub keep_alive_interval: u64, + /// Enable Arrow Flight RPC server (env: AMP_FLIGHT_SERVER). + pub flight_server: bool, + /// Enable JSON Lines server (env: AMP_JSONL_SERVER). + pub jsonl_server: bool, + /// Enable Admin API server (env: AMP_ADMIN_SERVER). + pub admin_server: bool, /// Worker event streaming configuration. pub worker_events: WorkerEventsConfig, } diff --git a/docs/config.md b/docs/config.md index 5fb6cacc1..beb26523c 100644 --- a/docs/config.md +++ b/docs/config.md @@ -34,6 +34,34 @@ The following optional configuration keys allow you to control the hostname and - `jsonl_addr`: JSON Lines server address (default: `0.0.0.0:1603`) - `admin_api_addr`: Admin API server address (default: `0.0.0.0:1610`) +# Service enable flags + +These boolean flags control which services are started. They can be set via config file, environment variables, or CLI flags. CLI flags take precedence over config/env values. + +| Config Key | Environment Variable | CLI Flag | Description | +|------------|---------------------|----------|-------------| +| `flight_server` | `AMP_FLIGHT_SERVER` | `--flight-server [bool]` | Enable Arrow Flight RPC server | +| `jsonl_server` | `AMP_JSONL_SERVER` | `--jsonl-server [bool]` | Enable JSON Lines server | +| `admin_server` | `AMP_ADMIN_SERVER` | `--admin-server [bool]` | Enable Admin API server (solo mode only) | + +**Default behavior:** If none of these flags are explicitly set (via any method), all applicable services are enabled by default. Setting any flag explicitly means only the flagged services will start. + +**Precedence:** CLI flag > Config file / Environment variable + +**CLI flag values:** `--flight-server` alone enables the server. Use `--flight-server false` to explicitly disable a server that is enabled in the config file. + +**Environment variable values:** Accepts `1`, `true`, `yes`, `on` (case-insensitive) as truthy values. + +### Deprecated environment variables + +The following environment variable names are deprecated and will emit warnings. They continue to work for backward compatibility: + +| Deprecated | Replacement | +|------------|-------------| +| `FLIGHT_SERVER` | `AMP_FLIGHT_SERVER` | +| `JSONL_SERVER` | `AMP_JSONL_SERVER` | +| `ADMIN_SERVER` | `AMP_ADMIN_SERVER` | + ## Logging Simplified control of the logging verbosity level is offered by the `AMP_LOG` env var. It accepts diff --git a/docs/config.sample.toml b/docs/config.sample.toml index 21f31f737..3540c6a57 100644 --- a/docs/config.sample.toml +++ b/docs/config.sample.toml @@ -42,6 +42,16 @@ manifests_dir = "manifests" # jsonl_addr = "0.0.0.0:1603" # JSON Lines server address (default: "0.0.0.0:1603") # admin_api_addr = "0.0.0.0:1610" # Admin API server address (default: "0.0.0.0:1610") +# Service enable flags (for solo/server modes) +# These can also be set via environment variables (AMP_FLIGHT_SERVER, etc.) or CLI flags +# (--flight-server, --flight-server false). +# CLI flags take precedence over config file and environment variable values. +# NOTE: When none of these are explicitly set, all applicable services start by default. +# Setting any flag to true means only the explicitly enabled services will start. +# flight_server = true # Enable Arrow Flight server (env: AMP_FLIGHT_SERVER) +# jsonl_server = true # Enable JSON Lines server (env: AMP_JSONL_SERVER) +# admin_server = true # Enable Admin API server (env: AMP_ADMIN_SERVER, solo mode only) + # ================================ # Table sections # ================================ diff --git a/docs/features/app-ampd-server.md b/docs/features/app-ampd-server.md index 3303fb333..ad304d55c 100644 --- a/docs/features/app-ampd-server.md +++ b/docs/features/app-ampd-server.md @@ -55,6 +55,21 @@ ampd server | `flight_addr` | `0.0.0.0:1602` | Arrow Flight server binding | | `jsonl_addr` | `0.0.0.0:1603` | JSON Lines server binding | +### Service Enable Flags + +Control which servers are started via CLI flags, config file, or environment variables. CLI flags take precedence. + +| Config Key | Environment Variable | CLI Flag | Description | +|------------|---------------------|----------|-------------| +| `flight_server` | `AMP_FLIGHT_SERVER` | `--flight-server [bool]` | Enable Arrow Flight server | +| `jsonl_server` | `AMP_JSONL_SERVER` | `--jsonl-server [bool]` | Enable JSON Lines server | + +**Default behavior:** If no flags are explicitly set, both servers are enabled. + +**CLI flag values:** `--flight-server` alone enables the server. Use `--flight-server false` to explicitly disable. + +**Environment variable values:** Accepts `1`, `true`, `yes`, `on` (case-insensitive). + ### Streaming Settings | Setting | Default | Description | diff --git a/docs/features/app-ampd-solo.md b/docs/features/app-ampd-solo.md index c8c151f2c..ab2fefd56 100644 --- a/docs/features/app-ampd-solo.md +++ b/docs/features/app-ampd-solo.md @@ -55,6 +55,24 @@ Solo mode runs all three ampd services in a single process: query server (Flight Solo mode supports an optional `--config` (or `AMP_CONFIG`) file. If no config is provided, it auto-discovers `/config.toml` and falls back to zero-config defaults. See [app-ampd-solo-ampdir](app-ampd-solo-ampdir.md) for amp dir and config discovery details. +### Service Enable Flags + +Control which services are started via CLI flags, config file, or environment variables. CLI flags take precedence. + +| Config Key | Environment Variable | CLI Flag | Description | +|------------|---------------------|----------|-------------| +| `flight_server` | `AMP_FLIGHT_SERVER` | `--flight-server [bool]` | Enable Arrow Flight server | +| `jsonl_server` | `AMP_JSONL_SERVER` | `--jsonl-server [bool]` | Enable JSON Lines server | +| `admin_server` | `AMP_ADMIN_SERVER` | `--admin-server [bool]` | Enable Admin API server | + +**Default behavior:** If no flags are explicitly set (via any method), all services are enabled. + +**CLI flag values:** `--flight-server` alone enables the server. Use `--flight-server false` to explicitly disable. + +**Environment variable values:** Accepts `1`, `true`, `yes`, `on` (case-insensitive). + +**Deprecated environment variables:** `FLIGHT_SERVER`, `JSONL_SERVER`, `ADMIN_SERVER` still work but emit warnings. Use the `AMP_*` prefixed versions instead. + ## Usage ### Default Behavior diff --git a/docs/opencli-specs/ampd.spec.json b/docs/opencli-specs/ampd.spec.json index 91e56d3ad..86499b578 100644 --- a/docs/opencli-specs/ampd.spec.json +++ b/docs/opencli-specs/ampd.spec.json @@ -90,29 +90,29 @@ }, "server": { "summary": "Start query servers", - "description": "Starts one or more query servers for serving blockchain data through different protocols. By default, both query servers (Arrow Flight and JSON Lines) are started. Servers can be selectively enabled using flags. If no server flags (--flight-server, --jsonl-server) are specified, BOTH servers are enabled by default. When any server flag is specified, only the explicitly enabled servers will start. Default ports: Arrow Flight (1602), JSON Lines (1603). The server runs continuously until terminated (Ctrl+C or kill signal).", + "description": "Starts one or more query servers for serving blockchain data through different protocols. By default, both query servers (Arrow Flight and JSON Lines) are started. Servers can be selectively enabled or disabled using flags. If no server flags (--flight-server, --jsonl-server) are specified, BOTH servers are enabled by default. When any server flag is specified, only the explicitly enabled servers will start. Use --flag false to explicitly disable a server enabled in the config file. Default ports: Arrow Flight (1602), JSON Lines (1603). The server runs continuously until terminated (Ctrl+C or kill signal).", "parameters": [ { "name": "flight-server", "in": "flag", - "description": "Enable Arrow Flight RPC Server. This provides a high-performance binary protocol for querying data (default port 1602). Uses Apache Arrow Flight for efficient data transfer.", + "description": "Enable or disable Arrow Flight RPC Server. Pass without a value or with 'true' to enable, pass 'false' to disable. When omitted, the config file value is used.", "required": false, "schema": { "type": "boolean", - "default": false + "x-optional": true }, - "x-environment": "FLIGHT_SERVER" + "x-environment": "AMP_FLIGHT_SERVER" }, { "name": "jsonl-server", "in": "flag", - "description": "Enable JSON Lines Server. This provides a simple HTTP interface for querying data (default port 1603). Accepts SQL queries via POST requests and returns results in JSON Lines format.", + "description": "Enable or disable JSON Lines Server. Pass without a value or with 'true' to enable, pass 'false' to disable. When omitted, the config file value is used.", "required": false, "schema": { "type": "boolean", - "default": false + "x-optional": true }, - "x-environment": "JSONL_SERVER" + "x-environment": "AMP_JSONL_SERVER" } ], "responses": { @@ -140,40 +140,40 @@ }, "dev": { "summary": "Start development mode with embedded worker and query servers", - "description": "Starts a combined server, controller, and worker in a single process for simplified local development and testing. By default, all three components (Arrow Flight, JSON Lines, and Admin API) are started along with an embedded worker. Components can be selectively enabled using flags. If no flags (--flight-server, --jsonl-server, --admin-server) are specified, ALL THREE are enabled by default. When any flag is specified, only the explicitly enabled components will start. The worker always runs with node ID 'worker'. Default ports: Arrow Flight (1602), JSON Lines (1603), Admin API (1610). Dev mode allows temporary database configuration for quick local testing. The process runs continuously until terminated (Ctrl+C or kill signal).", + "description": "Starts a combined server, controller, and worker in a single process for simplified local development and testing. By default, all three components (Arrow Flight, JSON Lines, and Admin API) are started along with an embedded worker. Components can be selectively enabled or disabled using flags. If no flags (--flight-server, --jsonl-server, --admin-server) are specified, ALL THREE are enabled by default. When any flag is specified, only the explicitly enabled components will start. Use --flag false to explicitly disable a component enabled in the config file. The worker always runs with node ID 'worker'. Default ports: Arrow Flight (1602), JSON Lines (1603), Admin API (1610). Dev mode allows temporary database configuration for quick local testing. The process runs continuously until terminated (Ctrl+C or kill signal).", "parameters": [ { "name": "flight-server", "in": "flag", - "description": "Enable Arrow Flight RPC Server. This provides a high-performance binary protocol for querying data (default port 1602). Uses Apache Arrow Flight for efficient data transfer.", + "description": "Enable or disable Arrow Flight RPC Server. Pass without a value or with 'true' to enable, pass 'false' to disable. When omitted, the config file value is used.", "required": false, "schema": { "type": "boolean", - "default": false + "x-optional": true }, - "x-environment": "FLIGHT_SERVER" + "x-environment": "AMP_FLIGHT_SERVER" }, { "name": "jsonl-server", "in": "flag", - "description": "Enable JSON Lines Server. This provides a simple HTTP interface for querying data (default port 1603). Accepts SQL queries via POST requests and returns results in JSON Lines format.", + "description": "Enable or disable JSON Lines Server. Pass without a value or with 'true' to enable, pass 'false' to disable. When omitted, the config file value is used.", "required": false, "schema": { "type": "boolean", - "default": false + "x-optional": true }, - "x-environment": "JSONL_SERVER" + "x-environment": "AMP_JSONL_SERVER" }, { "name": "admin-server", "in": "flag", - "description": "Enable Admin API Server. This provides management and administrative endpoints (default port 1610) for monitoring and controlling the amp system.", + "description": "Enable or disable Admin API Server. Pass without a value or with 'true' to enable, pass 'false' to disable. When omitted, the config file value is used.", "required": false, "schema": { "type": "boolean", - "default": false + "x-optional": true }, - "x-environment": "ADMIN_SERVER" + "x-environment": "AMP_ADMIN_SERVER" } ], "responses": { diff --git a/docs/upgrading.md b/docs/upgrading.md index 8c8fa931c..ee9d7898c 100644 --- a/docs/upgrading.md +++ b/docs/upgrading.md @@ -116,6 +116,28 @@ psql -c "SELECT * FROM _amp_migrations ORDER BY id;" ## Version-Specific Notes +### Upcoming: Environment Variable Naming Standardization + +#### Deprecated Environment Variables + +The following environment variables have been renamed to use the `AMP_` prefix for consistency: + +| Old Name | New Name | +|----------|----------| +| `FLIGHT_SERVER` | `AMP_FLIGHT_SERVER` | +| `JSONL_SERVER` | `AMP_JSONL_SERVER` | +| `ADMIN_SERVER` | `AMP_ADMIN_SERVER` | + +**Migration**: The old names still work but will emit a deprecation warning at startup. Update your configuration to use the new names. Old names will be removed in a future release. + +**Action required**: Update your environment variables, deployment scripts, and container definitions to use the new `AMP_*` names. + +#### Service Enable Flags Now Configurable in TOML + +The server enable flags (`flight_server`, `jsonl_server`, `admin_server`) can now be set in the TOML config file in addition to environment variables and CLI flags. This allows persistent configuration without relying on environment variables or CLI flags. + +CLI flags accept an optional value (`--flight-server` to enable, `--flight-server false` to disable). When omitted, the config file value is used. See [config.md](config.md#service-enable-flags) for details. + ### Future Releases Check the release notes for each version for specific upgrade instructions and breaking changes.