diff --git a/decisions/0014-storage-clis-for-blobstore-operations.md b/decisions/0014-storage-clis-for-blobstore-operations.md index 42b603ce99..1816efc894 100644 --- a/decisions/0014-storage-clis-for-blobstore-operations.md +++ b/decisions/0014-storage-clis-for-blobstore-operations.md @@ -2,14 +2,21 @@ ## Status -📝 **Accepted** - This ADR defines a shared direction for replacing fog-based blobstore implementations. +✅ **Implemented** - Storage-CLI support is fully implemented for following Iaas Providers with native type names. | Provider | Status | Notes | |--------------|---------------------------|---------------------------------------------------------------------------------------------------------| -| Azure | 🚧 PoC in Progress | [PoC](https://github.com/cloudfoundry/cloud_controller_ng/pull/4397) done with `bosh-azure-storage-cli` | -| AWS | 🧭 Open for Contribution | | -| GCP | 🧭 Open for Contribution | | -| Alibaba Cloud| 🧭 Open for Contribution | | +| Azure | ✅ Implemented | Uses `bosh-azure-storage-cli` with native type `azurebs` | +| AWS | ✅ Implemented | Uses `bosh-s3cli` with native type `s3` | +| GCP | ✅ Implemented | Uses `bosh-gcscli` with native type `gcs` | +| Alibaba Cloud| ✅ Implemented | Uses `bosh-ali-storage-cli` with native type `alioss` | + +**Configuration Migration Status:** +- The `blobstore_provider` field accepts both native storage-cli type names AND legacy fog names +- **Recommended:** Use native storage-cli type names (azurebs, s3, gcs, alioss) +- **Legacy fog names** (AzureRM, AWS, Google, aliyun) still supported for backwards compatibility +- **WebDAV/dav intentionally excluded** until fully supported +- **Timeline:** Legacy fog name support to be removed May 2026 ## Context @@ -46,15 +53,16 @@ Specifically, we will: packages: app_package_directory_key: app-packages blobstore_type: storage-cli + blobstore_provider: azurebs # Native storage-cli type (RECOMMENDED) + # OR: blobstore_provider: AzureRM # Legacy fog name (DEPRECATED) connection_config: azure_storage_access_key: azure_storage_account_name: container_name: app-packages environment: AzureCloud - provider: AzureRM max_package_size: 1610612736 ``` -* Field `provider` will be used to determine the corresponding storage CLI blobstore client class (same approach is used for fog) +* Field `blobstore_provider` will be used to determine the corresponding storage CLI blobstore client class (same approach is used for fog) * The `fog_connection` field will be renamed to `connection_config` to make it independent * Values from `connection_config` are used to generate the corresponding config file for the Bosh storage CLIs * Config generation could be moved away from ccng into capi-release to avoid duplication diff --git a/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb b/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb index b5b5df5801..e9fb97b10a 100644 --- a/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb +++ b/lib/cloud_controller/blobstore/storage_cli/storage_cli_client.rb @@ -17,24 +17,45 @@ class StorageCliClient < BaseClient 'resource_pool' => :storage_cli_config_file_resource_pool }.freeze - PROVIDER_TO_STORAGE_CLI_STORAGETYPE = { + # Native storage-cli type names supported by CC (dav intentionally excluded for now) + STORAGE_CLI_TYPES = %w[azurebs alioss s3 gcs].freeze + + # DEPRECATED: Legacy fog provider names (remove after migration window) + LEGACY_PROVIDER_TO_STORAGE_CLI_TYPE = { 'AzureRM' => 'azurebs', 'aliyun' => 'alioss', 'AWS' => 's3', - 'webdav' => 'dav', 'Google' => 'gcs' + # 'webdav' => 'dav', # intentionally not enabled yet }.freeze - IMPLEMENTED_PROVIDERS = %w[AzureRM aliyun Google AWS].freeze - def initialize(directory_key:, resource_type:, root_dir:, min_size: nil, max_size: nil) raise 'Missing resource_type' if resource_type.nil? config_file_path = config_path_for(resource_type) cfg = fetch_config(resource_type) - @provider = cfg['provider'].to_s - raise BlobstoreError.new("No provider specified in config file: #{File.basename(config_file_path)}") if @provider.empty? - raise "Unimplemented provider: #{@provider}, implemented ones are: #{IMPLEMENTED_PROVIDERS.join(', ')}" unless IMPLEMENTED_PROVIDERS.include?(@provider) + + # Get provider field (can contain either fog name or storage-cli type) + provider = cfg['provider']&.to_s + raise BlobstoreError.new("No provider specified in config file: #{File.basename(config_file_path)}") if provider.nil? || provider.empty? + + # Explicitly block unfinished webdav storage-cli support to avoid confusion and wasted effort on debugging + # unsupported providers. Remove this check when webdav support is added. + raise "provider '#{provider}' is not supported yet" if %w[webdav dav].include?(provider) + + @storage_type = + if STORAGE_CLI_TYPES.include?(provider) + provider + else + # START LEGACY FOG SUPPORT (delete this whole else-branch after migration) + LEGACY_PROVIDER_TO_STORAGE_CLI_TYPE[provider] + # END LEGACY FOG SUPPORT + end + + unless @storage_type + raise "Unknown provider: #{provider}. Supported storage-cli types: #{STORAGE_CLI_TYPES.join(', ')} " \ + "(legacy fog names accepted temporarily: #{LEGACY_PROVIDER_TO_STORAGE_CLI_TYPE.keys.join(', ')})" + end @cli_path = cli_path @config_file = config_file_path @@ -43,7 +64,6 @@ def initialize(directory_key:, resource_type:, root_dir:, min_size: nil, max_siz @root_dir = root_dir @min_size = min_size || 0 @max_size = max_size - @storage_type = PROVIDER_TO_STORAGE_CLI_STORAGETYPE[@provider] end def fetch_config(resource_type) diff --git a/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb b/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb index 6fff02091f..ead5a6e64f 100644 --- a/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb +++ b/spec/unit/lib/cloud_controller/blobstore/storage_cli/storage_cli_client_spec.rb @@ -5,7 +5,9 @@ module CloudController module Blobstore RSpec.describe StorageCliClient do describe 'client init' do - it 'init the correct client when JSON has provider AzureRM' do + # DEPRECATED: Legacy fog provider tests - remove after migration window + # START LEGACY FOG SUPPORT TESTS + it 'init the correct client when JSON has provider AzureRM (legacy fog name)' do droplets_cfg = Tempfile.new(['droplets', '.json']) droplets_cfg.write({ provider: 'AzureRM', account_key: 'bommelkey', @@ -23,7 +25,6 @@ module Blobstore root_dir: 'dummy-root', resource_type: 'droplets' ) - expect(client.instance_variable_get(:@provider)).to eq('AzureRM') expect(client.instance_variable_get(:@storage_type)).to eq('azurebs') expect(client.instance_variable_get(:@resource_type)).to eq('droplets') expect(client.instance_variable_get(:@root_dir)).to eq('dummy-root') @@ -31,8 +32,37 @@ module Blobstore droplets_cfg.close! end + # END LEGACY FOG SUPPORT TESTS - it 'raises an error for an unimplemented provider' do + it 'init the correct client when JSON has provider azurebs (native storage-cli type)' do + droplets_cfg = Tempfile.new(['droplets', '.json']) + droplets_cfg.write({ provider: 'azurebs', + account_key: 'bommelkey', + account_name: 'bommel', + container_name: 'bommelcontainer', + environment: 'BommelCloud' }.to_json) + droplets_cfg.flush + + config_double = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double) + allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path) + + client = StorageCliClient.new( + directory_key: 'dummy-key', + root_dir: 'dummy-root', + resource_type: 'droplets' + ) + expect(client.instance_variable_get(:@storage_type)).to eq('azurebs') + expect(client.instance_variable_get(:@resource_type)).to eq('droplets') + expect(client.instance_variable_get(:@root_dir)).to eq('dummy-root') + expect(client.instance_variable_get(:@directory_key)).to eq('dummy-key') + + droplets_cfg.close! + end + + # DEPRECATED: Legacy fog provider tests - remove after migration window + # START LEGACY FOG SUPPORT TESTS + it 'raises an error for an unknown legacy provider' do droplets_cfg = Tempfile.new(['droplets', '.json']) droplets_cfg.write( { provider: 'UnknownProvider', @@ -49,10 +79,77 @@ module Blobstore expect do StorageCliClient.new(directory_key: 'dummy-key', root_dir: 'dummy-root', resource_type: 'droplets') - end.to raise_error(RuntimeError, 'Unimplemented provider: UnknownProvider, implemented ones are: AzureRM, aliyun, Google, AWS') + end.to raise_error(RuntimeError, /Unknown provider: UnknownProvider/) + + droplets_cfg.close! + end + + it 'blocks webdav/dav provider explicitly' do + droplets_cfg = Tempfile.new(['droplets', '.json']) + droplets_cfg.write( + { provider: 'webdav', + account_key: 'bommelkey' }.to_json + ) + droplets_cfg.flush + + config_double = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double) + allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path) + + expect do + StorageCliClient.new(directory_key: 'dummy-key', root_dir: 'dummy-root', resource_type: 'droplets') + end.to raise_error(RuntimeError, /is not supported yet/) + + droplets_cfg.close! + end + # END LEGACY FOG SUPPORT TESTS + + it 'raises an error for an unknown storage-cli type' do + droplets_cfg = Tempfile.new(['droplets', '.json']) + droplets_cfg.write( + { provider: 'unknown_type', + account_key: 'bommelkey', + account_name: 'bommel', + container_name: 'bommelcontainer', + environment: 'BommelCloud' }.to_json + ) + droplets_cfg.flush + + config_double = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double) + allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path) + + expect do + StorageCliClient.new(directory_key: 'dummy-key', root_dir: 'dummy-root', resource_type: 'droplets') + end.to raise_error(RuntimeError, /Unknown provider: unknown_type/) + + droplets_cfg.close! + end + + # DEPRECATED: Legacy fog provider test - remove after migration window + # START LEGACY FOG SUPPORT TESTS + it 'raises an error when provider is missing' do + droplets_cfg = Tempfile.new(['droplets', '.json']) + droplets_cfg.write( + { account_key: 'bommelkey', + account_name: 'bommel', + container_name: 'bommelcontainer', + environment: 'BommelCloud' }.to_json + ) + droplets_cfg.flush + + config_double = instance_double(VCAP::CloudController::Config) + allow(VCAP::CloudController::Config).to receive(:config).and_return(config_double) + allow(config_double).to receive(:get).with(:storage_cli_config_file_droplets).and_return(droplets_cfg.path) + + expect do + StorageCliClient.new(directory_key: 'dummy-key', root_dir: 'dummy-root', resource_type: 'droplets') + end.to raise_error(BlobstoreError, /No provider specified/) droplets_cfg.close! end + # END LEGACY FOG SUPPORT TESTS + # After removal, change error message expectation to /No storage_type specified/ it 'raise when no resource type' do expect do