Fix Sync of template.properties in Swift

Author: syed

engine/storage/src/org/apache/cloudstack/storage/datastore/ObjectInDataStoreManagerImpl.java

@@ -156,6 +156,7 @@ public DataObject create(DataObject obj, DataStore dataStore) {
                         // template.properties
                         // there
                     }
+
                     ts.setInstallPath(installPath);
                     ts.setState(ObjectInDataStoreStateMachine.State.Allocated);
                     ts = templateDataStoreDao.persist(ts);

services/secondary-storage/server/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java

@@ -597,32 +597,14 @@ protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
                     return answer;
                 }
                 s_logger.debug("starting copy template to swift");
-                DataTO newTemplate = answer.getNewData();
-                File templateFile = getFile(newTemplate.getPath(), ((NfsTO)srcDataStore).getUrl());
-                SwiftTO swift = (SwiftTO)destDataStore;
-                String containterName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
-                String swiftPath = SwiftUtil.putObject(swift, templateFile, containterName, templateFile.getName());
-                //upload template.properties
-                File properties = new File(templateFile.getParent() + File.separator + _tmpltpp);
-                if (properties.exists()) {
-                    SwiftUtil.putObject(swift, properties, containterName, _tmpltpp);
-                }
+                TemplateObjectTO newTemplate = (TemplateObjectTO)answer.getNewData();
+                newTemplate.setDataStore(srcDataStore);
+                CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
+                Answer result = copyFromNfsToSwift(newCpyCmd);
 
-                //clean up template data on staging area
-                try {
-                    DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
-                    execute(deleteCommand);
-                } catch (Exception e) {
-                    s_logger.debug("Failed to clean up staging area:", e);
-                }
+                cleanupStagingNfs(newTemplate);
+                return result;
 
-                TemplateObjectTO template = new TemplateObjectTO();
-                template.setPath(swiftPath);
-                template.setSize(templateFile.length());
-                template.setPhysicalSize(template.getSize());
-                SnapshotObjectTO snapshot = (SnapshotObjectTO)srcData;
-                template.setFormat(snapshot.getVolume().getFormat());
-                return new CopyCmdAnswer(template);
             } else if (destDataStore instanceof S3TO) {
                 //create template on the same data store
                 CopyCmdAnswer answer =
@@ -635,20 +617,29 @@ protected Answer createTemplateFromSnapshot(CopyCommand cmd) {
                 newTemplate.setDataStore(srcDataStore);
                 CopyCommand newCpyCmd = new CopyCommand(newTemplate, destData, cmd.getWait(), cmd.executeInSequence());
                 Answer result = copyFromNfsToS3(newCpyCmd);
-                //clean up template data on staging area
-                try {
-                    DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
-                    execute(deleteCommand);
-                } catch (Exception e) {
-                    s_logger.debug("Failed to clean up staging area:", e);
-                }
+
+                cleanupStagingNfs(newTemplate);
+
                 return result;
             }
         }
         s_logger.debug("Failed to create templat from snapshot");
         return new CopyCmdAnswer("Unsupported prototcol");
     }
 
+    /**
+     * clean up template data on staging area
+     * @param newTemplate: The template on the secondary storage that needs to be cleaned up
+     */
+    private void cleanupStagingNfs(TemplateObjectTO newTemplate) {
+        try {
+            DeleteCommand deleteCommand = new DeleteCommand(newTemplate);
+            execute(deleteCommand);
+        } catch (Exception e) {
+            s_logger.debug("Failed to clean up staging area:", e);
+        }
+    }
+
     protected Answer copyFromNfsToImage(CopyCommand cmd) {
         DataTO destData = cmd.getDestTO();
         DataStoreTO destDataStore = destData.getDataStore();
@@ -765,15 +756,17 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) {
             _storage.create(uniqDir.getAbsolutePath(), "template.properties");
             File metaFile = new File(metaFileName);
             FileWriter writer = new FileWriter(metaFile);
+            long virtualSize = getVirtualSize(file, getTemplateFormat(file.getName()));
             BufferedWriter bufferWriter = new BufferedWriter(writer);
             bufferWriter.write("uniquename=" + cmd.getName());
             bufferWriter.write("\n");
             bufferWriter.write("filename=" + fileName);
             bufferWriter.write("\n");
             bufferWriter.write("size=" + file.length());
+            bufferWriter.write("\n");
+            bufferWriter.write("virtualsize=" + virtualSize);
             bufferWriter.close();
             writer.close();
-
             SwiftUtil.putObject(swiftTO, metaFile, container, "template.properties");
             metaFile.delete();
             uniqDir.delete();
@@ -785,7 +778,7 @@ protected Answer registerTemplateOnSwift(DownloadCommand cmd) {
             }
 
             DownloadAnswer answer =
-                    new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum);
+                    new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, virtualSize, file.length(), md5sum);
             return answer;
         } catch (IOException e) {
             s_logger.debug("Failed to register template into swift", e);
@@ -942,6 +935,93 @@ protected Answer copyFromNfsToS3(CopyCommand cmd) {
         }
     }
 
+    /**
+     * Creates a template.properties for Swift with its correct unique name
+     *
+     * @param swift  The swift object
+     * @param srcFile Source file on the staging NFS
+     * @param containerName Destination container
+     * @return true on successful write
+     */
+    protected boolean swiftUploadMetadataFile(SwiftTO swift, File srcFile, String containerName) throws IOException {
+
+        File uniqDir = _storage.createUniqDir();
+        String metaFileName = uniqDir.getAbsolutePath() + File.separator + "template.properties";
+        _storage.create(uniqDir.getAbsolutePath(), "template.properties");
+
+        String uniqueName = FilenameUtils.getBaseName(srcFile.getName());
+        File metaFile = new File(metaFileName);
+        FileWriter writer = new FileWriter(metaFile);
+        BufferedWriter bufferWriter = new BufferedWriter(writer);
+        bufferWriter.write("uniquename=" + uniqueName);
+        bufferWriter.write("\n");
+        bufferWriter.write("filename=" + srcFile.getName());
+        bufferWriter.write("\n");
+        bufferWriter.write("size=" + srcFile.length());
+        bufferWriter.write("\n");
+        bufferWriter.write("virtualsize=" + getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+        bufferWriter.close();
+        writer.close();
+
+        SwiftUtil.putObject(swift, metaFile, containerName, _tmpltpp);
+        metaFile.delete();
+        uniqDir.delete();
+
+        return true;
+    }
+
+    /**
+     * Copies data from NFS and uploads it into a Swift container
+     *
+     * @param cmd CopyComand
+     * @return CopyCmdAnswer
+     */
+    protected Answer copyFromNfsToSwift(CopyCommand cmd) {
+
+        final DataTO srcData = cmd.getSrcTO();
+        final DataTO destData = cmd.getDestTO();
+
+        DataStoreTO srcDataStore = srcData.getDataStore();
+        NfsTO srcStore = (NfsTO)srcDataStore;
+        DataStoreTO destDataStore = destData.getDataStore();
+        File srcFile = getFile(srcData.getPath(), srcStore.getUrl());
+
+        SwiftTO swift = (SwiftTO)destDataStore;
+
+        try {
+
+            String containerName = SwiftUtil.getContainerName(destData.getObjectType().toString(), destData.getId());
+            String swiftPath = SwiftUtil.putObject(swift, srcFile, containerName, srcFile.getName());
+
+
+            DataTO retObj = null;
+            if (destData.getObjectType() == DataObjectType.TEMPLATE) {
+                swiftUploadMetadataFile(swift, srcFile, containerName);
+                TemplateObjectTO newTemplate = new TemplateObjectTO();
+                newTemplate.setPath(swiftPath);
+                newTemplate.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                newTemplate.setPhysicalSize(srcFile.length());
+                newTemplate.setFormat(getTemplateFormat(srcFile.getName()));
+                retObj = newTemplate;
+            } else if (destData.getObjectType() == DataObjectType.VOLUME) {
+                VolumeObjectTO newVol = new VolumeObjectTO();
+                newVol.setPath(containerName);
+                newVol.setSize(getVirtualSize(srcFile, getTemplateFormat(srcFile.getName())));
+                retObj = newVol;
+            } else if (destData.getObjectType() == DataObjectType.SNAPSHOT) {
+                SnapshotObjectTO newSnapshot = new SnapshotObjectTO();
+                newSnapshot.setPath(containerName);
+                retObj = newSnapshot;
+            }
+
+            return new CopyCmdAnswer(retObj);
+
+        } catch (Exception e) {
+            s_logger.error("failed to upload " + srcData.getPath(), e);
+            return new CopyCmdAnswer("failed to upload " + srcData.getPath() + e.toString());
+        }
+    }
+
     String swiftDownload(SwiftTO swift, String container, String rfilename, String lFullPath) {
         Script command = new Script("/bin/bash", s_logger);
         command.add("-c");