feat(storage): enhance StorageTransport retry logic and parameter serialization

handwritten/storage/src/acl.ts

@@ -528,10 +528,10 @@ class Acl extends AclRoleAccessorMethods {
     if (this.parent instanceof File) {
       const file = this.parent as File;
       const bucket = file.parent;
-      url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`;
     } else if (this.parent instanceof Bucket) {
       const bucket = this.parent as Bucket;
-      url = `${bucket.baseUrl}/${bucket.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}${url}`;
     }
 
     this.storageTransport
@@ -644,14 +644,14 @@ class Acl extends AclRoleAccessorMethods {
       query.userProject = options.userProject;
     }
 
-    let url = `${this.pathPrefix}/${options.entity}`;
+    let url = `${this.pathPrefix}/${encodeURIComponent(options.entity)}`;
     if (this.parent instanceof File) {
       const file = this.parent as File;
       const bucket = file.parent;
-      url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`;
     } else if (this.parent instanceof Bucket) {
       const bucket = this.parent as Bucket;
-      url = `${bucket.baseUrl}/${bucket.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}${url}`;
     }
 
     this.storageTransport
@@ -768,7 +768,7 @@ class Acl extends AclRoleAccessorMethods {
 
     let url = `${this.pathPrefix}`;
     if (options) {
-      url = `${url}/${options.entity}`;
+      url = `${url}/${encodeURIComponent(options.entity)}`;
       if (options.generation) {
         query.generation = options.generation;
       }
@@ -781,10 +781,10 @@ class Acl extends AclRoleAccessorMethods {
     if (this.parent instanceof File) {
       const file = this.parent as File;
       const bucket = file.parent;
-      url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`;
     } else if (this.parent instanceof Bucket) {
       const bucket = this.parent as Bucket;
-      url = `${bucket.baseUrl}/${bucket.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}${url}`;
     }
 
     this.storageTransport
@@ -888,14 +888,14 @@ class Acl extends AclRoleAccessorMethods {
       query.userProject = options.userProject;
     }
 
-    let url = `${this.pathPrefix}/${options.entity}`;
+    let url = `${this.pathPrefix}/${encodeURIComponent(options.entity)}`;
     if (this.parent instanceof File) {
       const file = this.parent as File;
       const bucket = file.parent;
-      url = `${bucket.baseUrl}/${bucket.name}/${file.baseUrl}/${file.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}/o/${encodeURIComponent(file.name)}${url}`;
     } else if (this.parent instanceof Bucket) {
       const bucket = this.parent as Bucket;
-      url = `${bucket.baseUrl}/${bucket.name}${url}`;
+      url = `/storage/v1/b/${bucket.name}${url}`;
     }
 
     this.storageTransport

handwritten/storage/src/bucket.ts

@@ -1272,7 +1272,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
     super({
       storageTransport: storage.storageTransport,
       parent: storage,
-      baseUrl: '/b',
+      baseUrl: '/storage/v1/b',
       id: name,
       createMethod: storage.createBucket.bind(storage),
       methods,
@@ -1736,7 +1736,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
       .makeRequest(
         {
           method: 'POST',
-          url: '/compose',
+          url: `/storage/v1/b/${this.name}/o/${encodeURIComponent(destinationFile.name)}/compose`,
           maxRetries,
           body: JSON.stringify({
             destination: {
@@ -1758,6 +1758,9 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
               return sourceObject;
             }),
           }),
+          headers: {
+            'Content-Type': 'application/json',
+          },
           queryParameters: options as unknown as StorageQueryParameters,
         },
         (err, resp) => {
@@ -2099,6 +2102,9 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
           body: JSON.stringify(convertObjKeysToSnakeCase(body)),
           queryParameters: query as unknown as StorageQueryParameters,
           retry: false,
+          headers: {
+            'Content-Type': 'application/json',
+          },
         },
         (err, data, resp) => {
           if (err) {

handwritten/storage/src/channel.ts

@@ -43,7 +43,7 @@ class Channel extends ServiceObject<Channel, BaseMetadata> {
     const config = {
       parent: storage,
       storageTransport: storage.storageTransport,
-      baseUrl: '/channels',
+      baseUrl: '/storage/v1/channels',
       id: '',
       methods: {},
     };
@@ -89,6 +89,9 @@ class Channel extends ServiceObject<Channel, BaseMetadata> {
           method: 'POST',
           url: `${this.baseUrl}/stop`,
           body: JSON.stringify(this.metadata),
+          headers: {
+            'Content-Type': 'application/json',
+          },
           responseType: 'json',
         },
         (err, data, resp) => {

handwritten/storage/src/file.ts

@@ -73,6 +73,7 @@ import {
   SetMetadataOptions,
 } from './nodejs-common/service-object.js';
 import {
+  Gaxios,
   GaxiosError,
   GaxiosInterceptor,
   GaxiosOptionsPrepared,
@@ -82,7 +83,6 @@ import {
   StorageQueryParameters,
   StorageRequestOptions,
 } from './storage-transport.js';
-import * as gaxios from 'gaxios';
 import mime from 'mime';
 
 export type GetExpirationDateResponse = [Date];
@@ -1405,13 +1405,24 @@ class File extends ServiceObject<File, FileMetadata> {
     }
 
     if (newFile.encryptionKey !== undefined) {
-      this.setEncryptionKey(newFile.encryptionKey!);
+      headers.set('x-goog-encryption-algorithm', 'AES256');
+      headers.set(
+        'x-goog-encryption-key',
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (newFile as any).encryptionKeyBase64 || '',
+      );
+      headers.set(
+        'x-goog-encryption-key-sha256',
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        (newFile as any).encryptionKeyHash || '',
+      );
     } else if (options.destinationKmsKeyName !== undefined) {
       query.destinationKmsKeyName = options.destinationKmsKeyName;
       delete options.destinationKmsKeyName;
     } else if (newFile.kmsKeyName !== undefined) {
       query.destinationKmsKeyName = newFile.kmsKeyName;
     }
+    headers.set('Content-Type', 'application/json');
 
     if (query.destinationKmsKeyName) {
       this.kmsKeyName = query.destinationKmsKeyName;
@@ -1441,7 +1452,7 @@ class File extends ServiceObject<File, FileMetadata> {
       .makeRequest<RewriteResponse>(
         {
           method: 'POST',
-          url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/rewriteTo/b/${
+          url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/rewriteTo/b/${
             destBucket.name
           }/o/${encodeURIComponent(newFile.name)}`,
           queryParameters: query as unknown as StorageQueryParameters,
@@ -1638,6 +1649,8 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const headers = response.headers;
+      const isStoredCompressed =
+        headers.get('x-goog-stored-content-encoding') === 'gzip';
       const isCompressed = headers.get('content-encoding') === 'gzip';
       const hashes: {crc32c?: string; md5?: string} = {};
 
@@ -1651,7 +1664,7 @@ class File extends ServiceObject<File, FileMetadata> {
 
       const transformStreams: Transform[] = [];
 
-      if (shouldRunValidation) {
+      if (shouldRunValidation && !isStoredCompressed) {
         // The x-goog-hash header should be set with a crc32c and md5 hash.
         // ex: headers.set('x-goog-hash', 'crc32c=xxxx,md5=xxxx')
         if (typeof headers.get('x-goog-hash') === 'string') {
@@ -1720,6 +1733,7 @@ class File extends ServiceObject<File, FileMetadata> {
       const headers = {
         'Accept-Encoding': 'gzip',
         'Cache-Control': 'no-store',
+        ...(this.encryptionKeyHeaders || {}),
       } as Headers;
 
       if (rangeRequest) {
@@ -1730,11 +1744,13 @@ class File extends ServiceObject<File, FileMetadata> {
       }
 
       const reqOpts: StorageRequestOptions = {
-        url: `${this.bucket.baseUrl}/${this.bucket.name}${this.baseUrl}/${this.name}`,
+        url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}`,
         headers,
         queryParameters: query as unknown as StorageQueryParameters,
         responseType: 'stream',
-      };
+        decompress: options.decompress,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      } as any;
 
       if (options[GCCL_GCS_CMD_KEY]) {
         reqOpts[GCCL_GCS_CMD_KEY] = options[GCCL_GCS_CMD_KEY];
@@ -2425,6 +2441,18 @@ class File extends ServiceObject<File, FileMetadata> {
     }
   }
 
+  get encryptionKeyHeaders(): Record<string, string> | undefined {
+    if (!this.encryptionKey) {
+      return undefined;
+    }
+
+    return {
+      'x-goog-encryption-algorithm': 'AES256',
+      'x-goog-encryption-key': this.encryptionKey.toString('base64'),
+      'x-goog-encryption-key-sha256': this.encryptionKeyHash!,
+    };
+  }
+
   /**
    * The Storage API allows you to use a custom key for server-side encryption.
    *
@@ -3309,22 +3337,18 @@ class File extends ServiceObject<File, FileMetadata> {
    */
 
   isPublic(callback?: IsPublicCallback): Promise<IsPublicResponse> | void {
-    // Build any custom headers based on the defined interceptors on the parent
-    // storage object and this object
-    const storageInterceptors = this.storage?.interceptors || [];
-    const fileInterceptors = this.interceptors || [];
-    const allInterceptors = storageInterceptors.concat(fileInterceptors);
-
-    for (const curInter of allInterceptors) {
-      gaxios.instance.interceptors.request.add(curInter);
-    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const {callback: cb} = normalize<any, IsPublicCallback>(
+      undefined,
+      callback,
+    );
+    const url = `${this.storage.apiEndpoint}/${this.bucket.name}/${encodeURIComponent(this.name)}`;
 
+    const gaxios = new Gaxios();
     gaxios
       .request({
         method: 'GET',
-        url: `${this.storage.apiEndpoint}/${
-          this.bucket.name
-        }/${encodeURIComponent(this.name)}`,
+        url,
         retryConfig: {
           retry: this.storage.retryOptions.maxRetries,
           noResponseRetries: this.storage.retryOptions.maxRetries,
@@ -3334,12 +3358,17 @@ class File extends ServiceObject<File, FileMetadata> {
           totalTimeout: this.storage.retryOptions.totalTimeout,
         },
       })
-      .then(() => callback!(null, true))
+      .then(() => {
+        cb(null, true);
+      })
       .catch(err => {
-        if (err.status === 403) {
-          callback!(null, false);
+        const status = err.response?.status;
+        // 401 Unauthorized or 403 Forbidden means the object is NOT public.
+        if (status === 401 || status === 403) {
+          cb(null, false);
         } else {
-          callback!(err);
+          // Any other error (like 404) is a real error.
+          cb(err);
         }
       });
   }
@@ -3687,7 +3716,7 @@ class File extends ServiceObject<File, FileMetadata> {
       .makeRequest(
         {
           method: 'POST',
-          url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/moveTo/o/${encodeURIComponent(newFile.name)}`,
+          url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/moveTo/o/${encodeURIComponent(newFile.name)}`,
           queryParameters: query as StorageQueryParameters,
           body: JSON.stringify(options),
         },
@@ -4018,7 +4047,7 @@ class File extends ServiceObject<File, FileMetadata> {
   async restore(options: RestoreOptions): Promise<File> {
     const file = await this.storageTransport.makeRequest<File>({
       method: 'POST',
-      url: `/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/restore`,
+      url: `/storage/v1/b/${this.bucket.name}/o/${encodeURIComponent(this.name)}/restore`,
       queryParameters: options as unknown as StorageQueryParameters,
     });
     return file as File;
@@ -4553,6 +4582,14 @@ class File extends ServiceObject<File, FileMetadata> {
       },
     ];
 
+    const headers: Record<string, string> = {};
+    if (this.encryptionKey) {
+      headers['x-goog-encryption-algorithm'] = 'AES256';
+      headers['x-goog-encryption-key'] = this.encryptionKeyBase64!;
+      headers['x-goog-encryption-key-sha256'] = this.encryptionKeyHash!;
+    }
+    reqOpts.headers = headers;
+
     this.storageTransport
       .makeRequest(reqOpts as StorageRequestOptions, (err, body, resp) => {
         if (err) {

handwritten/storage/src/hmacKey.ts

@@ -354,7 +354,7 @@ export class HmacKey extends ServiceObject<HmacKey, HmacKeyMetadata> {
       storageTransport: storage.storageTransport,
       parent: storage,
       id: accessId,
-      baseUrl: `/projects/${projectId}/hmacKeys`,
+      baseUrl: `/storage/v1/projects/${projectId}/hmacKeys`,
       methods,
     });