Support storage cli

jobs/blobstore/spec

@@ -81,3 +81,6 @@ properties:
     default: false
   blobstore.secret:
     description: Secret used to sign the HMAC pre-signed URLs
+  blobstore.use_storage_cli:
+    description: "When true, use storage-cli instead of s3cli for S3 operations"
+    default: false

jobs/director/templates/director.yml.erb

@@ -288,9 +288,17 @@ params['verify_multidigest_path'] = '/var/vcap/packages/verify_multidigest/bin/v
 params['version'] = '282.1.2'
 
 if p('blobstore.provider') == "s3"
-   params['blobstore']['provider'] = "s3cli"
-   params['blobstore']['options']['s3cli_config_path'] = "/var/vcap/data/director/tmp"
-   params['blobstore']['options']['s3cli_path'] = "/var/vcap/packages/s3cli/bin/s3cli"
+  if p('blobstore.use_storage_cli', false)
+    params['blobstore']['provider'] = "storagecli"
+    params['blobstore']['options']['storage_cli_config_path'] = "/var/vcap/data/director/tmp"
+    params['blobstore']['options']['storage_cli_path'] = "/var/vcap/packages/storage-cli/bin/storage-cli"
+    # Essential for storage-cli to know which provider to use
+    params['blobstore']['options']['storage_provider'] = "s3"
+  else
+    params['blobstore']['provider'] = "s3cli"
+    params['blobstore']['options']['s3cli_config_path'] = "/var/vcap/data/director/tmp"
+    params['blobstore']['options']['s3cli_path'] = "/var/vcap/packages/s3cli/bin/s3cli"
+  end
 end
 
 if p('blobstore.provider') == "gcs"
@@ -396,4 +404,4 @@ params['agent'] = {
 }
 
 JSON.dump(params)
-%>
+%>

packages/storage-cli/README.md

@@ -0,0 +1,6 @@
+storage-cli package
+============
+This package is used for communicating with s3 blobstores
+
+The blob compiled from [storage-s3cli](https://github.com/cloudfoundry/storage-cli/tree/main/s3)
+and hosted in [GitHub_Repo](https://github.com/cloudfoundry/storage-cli/releases/latest)

packages/storage-cli/packaging

@@ -0,0 +1,5 @@
+set -e
+
+mkdir -p ${BOSH_INSTALL_TARGET}/bin
+mv storage-cli/storage-cli-*-linux-amd64 ${BOSH_INSTALL_TARGET}/bin/storage-cli
+chmod +x ${BOSH_INSTALL_TARGET}/bin/storage-cli

packages/storage-cli/spec

@@ -0,0 +1,4 @@
+---
+name: storage-cli
+files:
+- storage-cli/storage-cli-*-linux-amd64 # The binary from github.com/cloudfoundry/storage-cli

src/bosh-director/lib/bosh/director/blobstore/s3cli_blobstore_client.rb

@@ -23,9 +23,21 @@ class S3cliBlobstoreClient < Client
       def initialize(options)
         super(options)
 
-        @s3cli_path = @options.fetch(:s3cli_path)
-        unless Kernel.system(@s3cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null')
-          raise BlobstoreError, 'Cannot find s3cli executable. Please specify s3cli_path parameter'
+        @use_storage_cli = @options.fetch(:use_storage_cli, false)
+
+        if @use_storage_cli
+          @cli_path = @options.fetch(:storage_cli_path)
+          @storage_provider = @options.fetch(:storage_provider, 's3')
+          @base_command = [@cli_path.to_s, '-s', @storage_provider.to_s]
+          unless Kernel.system(@cli_path.to_s, 'version', out: '/dev/null', err: '/dev/null')
+            raise BlobstoreError, "Cannot find storage-cli executable at #{@cli_path}"
+          end
+        else
+          @cli_path = @options.fetch(:s3cli_path)
+          @base_command = [@cli_path.to_s]
+          unless Kernel.system(@cli_path.to_s, '--v', out: '/dev/null', err: '/dev/null')
+            raise BlobstoreError, 'Cannot find s3cli executable. Please specify s3cli_path parameter'
+          end
         end
 
         @s3cli_options = {
@@ -54,7 +66,8 @@ def initialize(options)
           @options[:credentials_source] = 'none'
         end
 
-        @config_file = write_config_file(@s3cli_options, @options.fetch(:s3cli_config_path, nil))
+        config_path = @use_storage_cli ? @options.fetch(:storage_cli_config_path, nil) : @options.fetch(:s3cli_config_path, nil)
+        @config_file = write_config_file(@s3cli_options, config_path)
       end
 
       def redacted_credential_properties_list
@@ -82,7 +95,7 @@ def create_file(object_id, file)
       # @param [File] file file to store the retrived object in
       def get_file(object_id, file)
         begin
-          out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s)
+          out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'get', object_id.to_s, file.path.to_s)
         rescue Exception => e
           raise BlobstoreError, e.inspect
         end
@@ -96,7 +109,7 @@ def get_file(object_id, file)
       # @param [String] object_id object id to delete
       def delete_object(object_id)
         begin
-          out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'delete', object_id.to_s)
+          out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'delete', object_id.to_s)
         rescue Exception => e
           raise BlobstoreError, e.inspect
         end
@@ -105,7 +118,7 @@ def delete_object(object_id)
 
       def object_exists?(object_id)
         begin
-          out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'exists', object_id.to_s)
+          out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'exists', object_id.to_s)
           return true if status.exitstatus.zero?
           return false if status.exitstatus == 3
         rescue Exception => e
@@ -117,7 +130,7 @@ def object_exists?(object_id)
       def sign_url(object_id, verb, duration)
         begin
           out, err, status = Open3.capture3(
-            @s3cli_path.to_s,
+            *@base_command,
             '-c',
             @config_file.to_s,
             'sign',
@@ -144,7 +157,7 @@ def required_credential_properties_list
       # @return [void]
       def store_in_s3(path, oid)
         begin
-          out, err, status = Open3.capture3(@s3cli_path.to_s, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s)
+          out, err, status = Open3.capture3(*@base_command, '-c', @config_file.to_s, 'put', path.to_s, oid.to_s)
         rescue Exception => e
           raise BlobstoreError, e.inspect
         end
@@ -156,4 +169,4 @@ def full_oid_path(object_id)
       end
     end
   end
-end
+end

src/bosh-director/spec/unit/bosh/director/blobstore/s3cli_blobstore_client_spec.rb

@@ -8,6 +8,8 @@ module Bosh::Director::Blobstore
       allow(Dir).to receive(:tmpdir).and_return(base_dir)
       allow(SecureRandom).to receive_messages(uuid: 'FAKE_UUID')
       allow(Kernel).to receive(:system).with('/var/vcap/packages/s3cli/bin/s3cli', '--v', out: '/dev/null', err: '/dev/null').and_return(true)
+      # Stub for the new storage-cli executable
+      allow(Kernel).to receive(:system).with('/var/vcap/packages/storage-cli/bin/storage-cli', 'version', out: '/dev/null', err: '/dev/null').and_return(true)
     end
 
     let(:options) do
@@ -132,6 +134,46 @@ module Bosh::Director::Blobstore
           expect(JSON.parse(stored_config_file[0])['swift_temp_url_key']).to eq('the_swift_temp_url_key')
         end
       end
+
+      context 'when use_storage_cli is true' do
+        let(:storage_cli_path) { '/var/vcap/packages/storage-cli/bin/storage-cli' }
+        let(:storage_options) do
+          {
+            use_storage_cli: true,
+            storage_cli_path: storage_cli_path,
+            storage_provider: 'gcs',
+            bucket_name: 'storage-bucket',
+          }
+        end
+
+        it 'validates the storage-cli path using "version" instead of "--v"' do
+          expect(Kernel).to receive(:system).with(storage_cli_path, 'version', out: '/dev/null', err: '/dev/null').and_return(true)
+          described_class.new(storage_options)
+        end
+
+        it 'raises error if storage-cli is not found' do
+          allow(Kernel).to receive(:system).with(storage_cli_path, 'version', out: '/dev/null', err: '/dev/null').and_return(false)
+          expect { described_class.new(storage_options) }.to raise_error(
+            Bosh::Director::Blobstore::BlobstoreError, /Cannot find storage-cli executable/
+          )
+        end
+
+        it 'creates config file using storage_cli_config_path if provided' do
+          storage_config_path = Dir.mktmpdir
+          described_class.new(storage_options.merge(storage_cli_config_path: storage_config_path))
+          expect(File.exist?(File.join(storage_config_path, 'blobstore-config'))).to eq(true)
+        end
+
+        it 'uses the -s provider flag in commands' do
+          allow(Open3).to receive(:capture3).and_return([nil, nil, success_exit_status])
+          cli_client = described_class.new(storage_options)
+
+          expect(Open3).to receive(:capture3).with(
+            storage_cli_path, '-s', 'gcs', '-c', anything, 'delete', object_id.to_s
+          )
+          cli_client.delete(object_id)
+        end
+      end
     end
 
     describe '#delete' do
@@ -260,4 +302,4 @@ module Bosh::Director::Blobstore
         end
     end
   end
-end
+end