diff --git a/jobs/blobstore/spec b/jobs/blobstore/spec index 3d7640d8826..f0c5212d632 100644 --- a/jobs/blobstore/spec +++ b/jobs/blobstore/spec @@ -81,3 +81,6 @@ properties: default: false blobstore.secret: description: Secret used to sign the HMAC pre-signed URLs + blobstore.use_storage_cli: + description: "When true, use storage-cli instead of s3cli for S3 operations" + default: false diff --git a/jobs/director/templates/director.yml.erb b/jobs/director/templates/director.yml.erb index 42aa781702a..e89ebb489a4 100644 --- a/jobs/director/templates/director.yml.erb +++ b/jobs/director/templates/director.yml.erb @@ -288,9 +288,17 @@ params['verify_multidigest_path'] = '/var/vcap/packages/verify_multidigest/bin/v params['version'] = '282.1.2' if p('blobstore.provider') == "s3" - params['blobstore']['provider'] = "s3cli" - params['blobstore']['options']['s3cli_config_path'] = "/var/vcap/data/director/tmp" - params['blobstore']['options']['s3cli_path'] = "/var/vcap/packages/s3cli/bin/s3cli" + if p('blobstore.use_storage_cli', false) + params['blobstore']['provider'] = "storagecli" + params['blobstore']['options']['storage_cli_config_path'] = "/var/vcap/data/director/tmp" + params['blobstore']['options']['storage_cli_path'] = "/var/vcap/packages/storage-cli/bin/storage-cli" + # Essential for storage-cli to know which provider to use + params['blobstore']['options']['storage_provider'] = "s3" + else + params['blobstore']['provider'] = "s3cli" + params['blobstore']['options']['s3cli_config_path'] = "/var/vcap/data/director/tmp" + params['blobstore']['options']['s3cli_path'] = "/var/vcap/packages/s3cli/bin/s3cli" + end end if p('blobstore.provider') == "gcs" @@ -396,4 +404,4 @@ params['agent'] = { } JSON.dump(params) -%> +%> \ No newline at end of file diff --git a/packages/storage-cli/README.md b/packages/storage-cli/README.md new file mode 100644 index 00000000000..f7a68211fe3 --- /dev/null +++ b/packages/storage-cli/README.md @@ -0,0 +1,6 @@ +storage-cli package +============ +This package is used for communicating with s3 blobstores + +The blob compiled from [storage-s3cli](https://github.com/cloudfoundry/storage-cli/tree/main/s3) +and hosted in [GitHub_Repo](https://github.com/cloudfoundry/storage-cli/releases/latest) \ No newline at end of file diff --git a/packages/storage-cli/packaging b/packages/storage-cli/packaging new file mode 100644 index 00000000000..e20dcf5c294 --- /dev/null +++ b/packages/storage-cli/packaging @@ -0,0 +1,5 @@ +set -e + +mkdir -p ${BOSH_INSTALL_TARGET}/bin +mv storage-cli/storage-cli-*-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/storage-cli +chmod +x ${BOSH_INSTALL_TARGET}/bin/storage-cli \ No newline at end of file diff --git a/packages/storage-cli/spec b/packages/storage-cli/spec new file mode 100644 index 00000000000..c4a9f37b0ae --- /dev/null +++ b/packages/storage-cli/spec @@ -0,0 +1,4 @@ +--- +name: storage-cli +files: +- storage-cli/storage-cli-*-linux-amd64 # The binary from github.com/cloudfoundry/storage-cli \ No newline at end of file diff --git a/src/bosh-director/lib/bosh/director/blobstore/s3cli_blobstore_client.rb b/src/bosh-director/lib/bosh/director/blobstore/s3cli_blobstore_client.rb index 059334b32cf..75a32b099c1 100644 --- a/src/bosh-director/lib/bosh/director/blobstore/s3cli_blobstore_client.rb +++ b/src/bosh-director/lib/bosh/director/blobstore/s3cli_blobstore_client.rb @@ -23,9 +23,21 @@ class S3cliBlobstoreClient < Client def initialize(options) super(options) - @s3cli_path = @options.fetch(:s3cli_path) - unless Kernel.system(@s3cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null') - raise BlobstoreError, 'Cannot find s3cli executable. Please specify s3cli_path parameter' + @use_storage_cli = @options.fetch(:use_storage_cli, false) + + if @use_storage_cli + @cli_path = @options.fetch(:storage_cli_path) + @storage_provider = @options.fetch(:storage_provider, 's3') + @base_command = [@cli_path.to_s, '-s', @storage_provider.to_s] + unless Kernel.system(@cli_path.to_s, 'version', out: '/dev/null', err: '/dev/null') + raise BlobstoreError, "Cannot find storage-cli executable at #{@cli_path}" + end + else + @cli_path = @options.fetch(:s3cli_path) + @base_command = [@cli_path.to_s] + unless Kernel.system(@cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null') + raise BlobstoreError, 'Cannot find s3cli executable. Please specify s3cli_path parameter' + end end @s3cli_options = { @@ -54,7 +66,8 @@ def initialize(options) @options[:credentials_source] = 'none' end - @config_file = write_config_file(@s3cli_options, @options.fetch(:s3cli_config_path, nil)) + config_path = @use_storage_cli ? @options.fetch(:storage_cli_config_path, nil) : @options.fetch(:s3cli_config_path, nil) + @config_file = write_config_file(@s3cli_options, config_path) end def redacted_credential_properties_list @@ -82,7 +95,7 @@ def create_file(object_id, file) # @param [File] file file to store the retrived object in def get_file(object_id, file) begin - out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s) + out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s) rescue Exception => e raise BlobstoreError, e.inspect end @@ -96,7 +109,7 @@ def get_file(object_id, file) # @param [String] object_id object id to delete def delete_object(object_id) begin - out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'delete', object_id.to_s) + out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'delete', object_id.to_s) rescue Exception => e raise BlobstoreError, e.inspect end @@ -105,7 +118,7 @@ def delete_object(object_id) def object_exists?(object_id) begin - out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'exists', object_id.to_s) + out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'exists', object_id.to_s) return true if status.exitstatus.zero? return false if status.exitstatus == 3 rescue Exception => e @@ -117,7 +130,7 @@ def object_exists?(object_id) def sign_url(object_id, verb, duration) begin out, err, status = Open3.capture3( - @s3cli_path.to_s, + *@base_command, '-c', @config_file.to_s, 'sign', @@ -144,7 +157,7 @@ def required_credential_properties_list # @return [void] def store_in_s3(path, oid) begin - out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s) + out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s) rescue Exception => e raise BlobstoreError, e.inspect end @@ -156,4 +169,4 @@ def full_oid_path(object_id) end end end -end +end \ No newline at end of file diff --git a/src/bosh-director/spec/unit/bosh/director/blobstore/s3cli_blobstore_client_spec.rb b/src/bosh-director/spec/unit/bosh/director/blobstore/s3cli_blobstore_client_spec.rb index 9c010c2d92f..1e53ccd24d8 100644 --- a/src/bosh-director/spec/unit/bosh/director/blobstore/s3cli_blobstore_client_spec.rb +++ b/src/bosh-director/spec/unit/bosh/director/blobstore/s3cli_blobstore_client_spec.rb @@ -8,6 +8,8 @@ module Bosh::Director::Blobstore allow(Dir).to receive(:tmpdir).and_return(base_dir) allow(SecureRandom).to receive_messages(uuid: 'FAKE_UUID') allow(Kernel).to receive(:system).with('/var/vcap/packages/s3cli/bin/s3cli', '--v', out: '/dev/null', err: '/dev/null').and_return(true) + # Stub for the new storage-cli executable + allow(Kernel).to receive(:system).with('/var/vcap/packages/storage-cli/bin/storage-cli', 'version', out: '/dev/null', err: '/dev/null').and_return(true) end let(:options) do @@ -132,6 +134,46 @@ module Bosh::Director::Blobstore expect(JSON.parse(stored_config_file[0])['swift_temp_url_key']).to eq('the_swift_temp_url_key') end end + + context 'when use_storage_cli is true' do + let(:storage_cli_path) { '/var/vcap/packages/storage-cli/bin/storage-cli' } + let(:storage_options) do + { + use_storage_cli: true, + storage_cli_path: storage_cli_path, + storage_provider: 'gcs', + bucket_name: 'storage-bucket', + } + end + + it 'validates the storage-cli path using "version" instead of "--v"' do + expect(Kernel).to receive(:system).with(storage_cli_path, 'version', out: '/dev/null', err: '/dev/null').and_return(true) + described_class.new(storage_options) + end + + it 'raises error if storage-cli is not found' do + allow(Kernel).to receive(:system).with(storage_cli_path, 'version', out: '/dev/null', err: '/dev/null').and_return(false) + expect { described_class.new(storage_options) }.to raise_error( + Bosh::Director::Blobstore::BlobstoreError, /Cannot find storage-cli executable/ + ) + end + + it 'creates config file using storage_cli_config_path if provided' do + storage_config_path = Dir.mktmpdir + described_class.new(storage_options.merge(storage_cli_config_path: storage_config_path)) + expect(File.exist?(File.join(storage_config_path, 'blobstore-config'))).to eq(true) + end + + it 'uses the -s provider flag in commands' do + allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status]) + cli_client = described_class.new(storage_options) + + expect(Open3).to receive(:capture3).with( + storage_cli_path, '-s', 'gcs', '-c', anything, 'delete', object_id.to_s + ) + cli_client.delete(object_id) + end + end end describe '#delete' do @@ -260,4 +302,4 @@ module Bosh::Director::Blobstore end end end -end +end \ No newline at end of file