feat: allow for object versioining in gcp aws

src/prerna/engine/api/IStorageEngine.java

@@ -63,6 +63,18 @@ public interface IStorageEngine extends IEngine {
 	 */
 	List<Map<String, Object>> listDetails(String path) throws Exception;
 
+	/**
+	 * List all versions of a specific object in storage.
+	 * Only supported by engines with versioning enabled (S3, GCS).
+	 * 
+	 * @param storagePath the path to the object in storage
+	 * @return a list of version details (versionId, lastModified, size, isLatest)
+	 * @throws Exception if listing fails
+	 */
+	default List<Map<String, Object>> listVersions(String storagePath) throws Exception {
+		throw new UnsupportedOperationException("Object versioning is not supported by this storage engine");
+	}
+
 	/**
 	 * 
 	 * @param localPath
@@ -81,13 +93,17 @@ public interface IStorageEngine extends IEngine {
 	void syncStorageToLocal(String storagePath, String localPath) throws Exception;
 
 	/**
+	 * Copy local files to a storage folder path.
+	 * Returns the version identifier if the underlying storage has versioning enabled,
+	 * or null if versioning is not supported/enabled.
 	 * 
-	 * @param localFilePath
-	 * @param storageFolderPath
-	 * @param metadata
-	 * @throws Exception
+	 * @param localFilePath     the local file or folder path(s) to upload
+	 * @param storageFolderPath the destination path in storage
+	 * @param metadata          optional metadata to attach to the uploaded objects
+	 * @return the version identifier (S3 versionId or GCS generation), or null if not available
+	 * @throws Exception if the upload fails
 	 */
-	void copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata) throws Exception;
+	String copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata) throws Exception;
 
 	/**
 	 * 
@@ -141,4 +157,16 @@ default byte[] readBlobToMemory(String storagePath) throws Exception {
 	default void updateBlobMetadata(String storagePath, Map<String, Object> metadata) throws Exception {
 		throw new UnsupportedOperationException("updateBlobMetadata is not supported by this storage engine"); 
 	}
+
+	/**
+	 * Copy a specific version of a file from storage to local.
+	 * 
+	 * @param storageFilePath the path to the file in storage
+	 * @param localFolderPath the local folder to download to
+	 * @param versionId       the version identifier (S3 versionId or GCS generation number)
+	 * @throws Exception if the operation is not supported or fails
+	 */
+	default void copyToLocal(String storageFilePath, String localFolderPath, String versionId) throws Exception {
+		copyToLocal(storageFilePath, localFolderPath);
+	}
 }

src/prerna/engine/impl/storage/AWSNativeBlobStorageEngine.java

@@ -40,11 +40,13 @@
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -68,7 +70,11 @@
 import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
 import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
 import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest;
+import software.amazon.awssdk.services.s3.model.ListObjectVersionsResponse;
+import software.amazon.awssdk.services.s3.model.ObjectVersion;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
 import software.amazon.awssdk.services.s3.model.S3Object;
 
@@ -239,6 +245,39 @@ public List<Map<String, Object>> listDetails(String path) throws Exception {
 		return objectDetails;
 	}
 
+	@Override
+	public List<Map<String, Object>> listVersions(String storagePath) throws Exception {
+		List<Map<String, Object>> versions = new ArrayList<>();
+		String key = normalizeStoragePrefixPath(storagePath);
+		// Remove trailing slash for file keys
+		if (key.endsWith("/")) {
+			key = key.substring(0, key.length() - 1);
+		}
+
+		ListObjectVersionsRequest request = ListObjectVersionsRequest.builder()
+				.bucket(this.bucket)
+				.prefix(key)
+				.build();
+
+		ListObjectVersionsResponse response = this.client.listObjectVersions(request);
+
+		for (ObjectVersion version : response.versions()) {
+			// Only include exact key matches (not prefix matches)
+			if (!version.key().equals(key)) {
+				continue;
+			}
+			Map<String, Object> versionInfo = new LinkedHashMap<>();
+			versionInfo.put("versionId", version.versionId());
+			versionInfo.put("lastModified", version.lastModified() != null ? version.lastModified().toString() : null);
+			versionInfo.put("size", version.size());
+			versionInfo.put("isLatest", version.isLatest());
+			versionInfo.put("key", version.key());
+			versions.add(versionInfo);
+		}
+
+		return versions;
+	}
+
 	@Override
 	public void syncLocalToStorage(String localPath, String storagePath, Map<String, Object> metadata)
 			throws Exception {
@@ -376,11 +415,12 @@ public void syncStorageToLocal(String storagePath, String localPath) throws Exce
 	}
 
 	@Override
-	public void copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
+	public String copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
 			throws Exception {
 		List<Path> paths = parseLocalPaths(localFilePath);
 		List<String> uploadedFiles = new ArrayList<>();
 		List<String> failedFiles = new ArrayList<>();
+		AtomicReference<String> lastVersionId = new AtomicReference<>(null);
 		boolean found = false;
 
 		for (Path path : paths) {
@@ -396,7 +436,11 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 				try (Stream<Path> stream = Files.walk(path)) {
 					stream.filter(Files::isRegularFile).forEach(file -> {
 						try {
-							uploadedFiles.add(uploadFile(path, file, storageFolderPath, metadata));
+							String versionId = uploadFile(path, file, storageFolderPath, metadata);
+							uploadedFiles.add(file.toString());
+							if (versionId != null) {
+								lastVersionId.set(versionId);
+							}
 						} catch (Exception e) {
 							failedFiles.add(file.toString());
 							classLogger.error("Failed to upload file: {}", file, e);
@@ -407,7 +451,11 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 				}
 			} else {
 				try {
-					uploadedFiles.add(uploadFile(path.getParent(), path, storageFolderPath, metadata));
+					String versionId = uploadFile(path.getParent(), path, storageFolderPath, metadata);
+					uploadedFiles.add(path.toString());
+					if (versionId != null) {
+						lastVersionId.set(versionId);
+					}
 					found = true;
 				} catch (Exception e) {
 					failedFiles.add(path.toString());
@@ -416,7 +464,7 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 				}
 			}
 		}
-		// Delete empty blobs from GCS
+		// Delete empty blobs from S3
 		deleteEmptyBlobsFromS3(storageFolderPath);
 		if (uploadedFiles.isEmpty()) {
 			classLogger.info("No files were uploaded.");
@@ -425,6 +473,7 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 		}
 		classLogger.info(found ? "Copy completed successfully for: {}" : "No files found to copy for: {}",
 				storageFolderPath);
+		return lastVersionId.get();
 	}
 
 	@Override
@@ -510,6 +559,31 @@ public void copyToLocal(String storageFilePath, String localFolderPath) throws E
 				storageFilePath);
 	}
 
+	@Override
+	public void copyToLocal(String storageFilePath, String localFolderPath, String versionId) throws Exception {
+		if (versionId != null && !versionId.isEmpty()) {
+			String key = normalizeStoragePrefixPath(storageFilePath);
+			Path localDirectory = Paths.get(localFolderPath);
+			Files.createDirectories(localDirectory);
+
+			String fileName = key.contains("/") ? key.substring(key.lastIndexOf("/") + 1) : key;
+			Path localFilePath = localDirectory.resolve(fileName);
+
+			GetObjectRequest getRequest = GetObjectRequest.builder()
+					.bucket(this.bucket)
+					.key(key)
+					.versionId(versionId)
+					.build();
+
+			try (ResponseInputStream<GetObjectResponse> responseStream = this.client.getObject(getRequest)) {
+				Files.copy(responseStream, localFilePath, StandardCopyOption.REPLACE_EXISTING);
+				classLogger.info("Downloaded versioned file: {} (versionId={})", localFilePath, versionId);
+			}
+		} else {
+			copyToLocal(storageFilePath, localFolderPath);
+		}
+	}
+
 	@Override
 	public void deleteFromStorage(String storagePath) throws Exception {
 		storagePath = Utility.normalizePath(storagePath);
@@ -699,9 +773,11 @@ private String uploadFileToS3(String storagePath, Path filePath, Path basePath,
 			PutObjectRequest putRequest = PutObjectRequest.builder().bucket(this.bucket).key(fileKey).metadata(metaMap)
 					.build();
 
-			this.client.putObject(putRequest, filePath);
+			PutObjectResponse response = this.client.putObject(putRequest, filePath);
 			classLogger.info("Uploaded/Updated file: {}", fileKey);
-			return;
+			if (response.versionId() != null) {
+				classLogger.info("Version ID for {}: {}", fileKey, response.versionId());
+			}
 		}, "Uploading file to S3: " + fileKey);
 
 		return fileKey;
@@ -871,12 +947,16 @@ private String uploadFile(Path rootPath, Path file, String storageFolderPath, Ma
 					.collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())));
 		}
 
+		AtomicReference<String> versionIdRef = new AtomicReference<>(null);
 		retryOperation(() -> {
-			this.client.putObject(putBuilder.build(), file);
+			PutObjectResponse response = this.client.putObject(putBuilder.build(), file);
 			classLogger.info("Uploaded file to S3: {}", fileKey);
+			if (response.versionId() != null) {
+				versionIdRef.set(response.versionId());
+			}
 		}, "Uploading to S3: " + fileKey);
 
-		return fileKey;
+		return versionIdRef.get();
 	}
 
 	private void downloadFile(String key, Path localFilePath) throws IOException {

src/prerna/engine/impl/storage/AbstractRCloneStorageEngine.java

@@ -135,9 +135,10 @@ public void syncStorageToLocal(String storagePath, String localPath) throws IOEx
 	}
 
 	@Override
-	public void copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
+	public String copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
 			throws IOException, InterruptedException {
 		copyToStorage(localFilePath, storageFolderPath, null, metadata);
+		return null;
 	}
 
 	@Override

src/prerna/engine/impl/storage/AzureNativeBlobStorageEngine.java

@@ -303,7 +303,7 @@ public void syncStorageToLocal(String storagePath, String localPath) throws Exce
 	}
 
 	@Override
-	public void copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
+	public String copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
 			throws Exception {
 		// Extract container and blob directory
 		String[] containerAndPath = extractContainerAndPath(storageFolderPath);
@@ -358,6 +358,7 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 		}
 		classLogger.info(found ? "Copy completed successfully for: {}" : "No files found to copy for: {}",
 				storageFolderPath);
+		return null;
 	}
 
 	@Override

src/prerna/engine/impl/storage/DeveloperLocalFileSystemStorageEngine.java

@@ -284,7 +284,7 @@ private void copyIfChanged(Path source, Path destination) throws IOException {
 	}
 
 	@Override
-	public void copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
+	public String copyToStorage(String localFilePath, String storageFolderPath, Map<String, Object> metadata)
 			throws Exception {
 		List<Path> sources = parseLocalPaths(localFilePath);
 		Path destinationFolder = resolveStoragePath(storageFolderPath);
@@ -302,6 +302,7 @@ public void copyToStorage(String localFilePath, String storageFolderPath, Map<St
 						StandardCopyOption.COPY_ATTRIBUTES);
 			}
 		}
+		return null;
 	}
 
 	@Override