Improved sync

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## Latest
+
+### Changes
+
+- Improved `mapache sync`.
+
 ## v0.2.2
 
 ### Fixes

core/src/backend/cache.rs

@@ -192,7 +192,7 @@ impl StorageBackend for CacheBackend {
         Ok(())
     }
 
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<crate::backend::BackendNode>> {
         self.backend.list_dir(path).await
     }
 

core/src/backend/dry.rs

@@ -1,7 +1,4 @@
-use std::{
-    path::{Path, PathBuf},
-    sync::Arc,
-};
+use std::{path::Path, sync::Arc};
 
 use anyhow::Result;
 use async_trait::async_trait;
@@ -60,7 +57,7 @@ impl StorageBackend for DryBackend {
     }
 
     #[inline]
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<super::BackendNode>> {
         self.backend.list_dir(path).await
     }
 

core/src/backend/limiter.rs

@@ -1,6 +1,6 @@
 use std::{
     io::SeekFrom,
-    path::{Path, PathBuf},
+    path::Path,
     pin::Pin,
     sync::{
         Arc,
@@ -329,7 +329,7 @@ impl StorageBackend for ThrottledBackend {
         self.inner.rename(from, to).await
     }
 
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<crate::backend::BackendNode>> {
         self.inner.list_dir(path).await
     }
 

core/src/backend/localfs.rs

@@ -8,7 +8,7 @@ use async_trait::async_trait;
 use tokio::io::{AsyncReadExt, AsyncSeekExt};
 
 use crate::{
-    backend::{Handle, NodeAttr, WriteContents},
+    backend::{BackendNode, Handle, NodeAttr, WriteContents},
     repository::repo::REPO_TMP_EXTENSION,
 };
 
@@ -305,9 +305,9 @@ impl StorageBackend for LocalFS {
         self.exists_exact(&full_path).await
     }
 
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<BackendNode>> {
         let full_path = self.full_path(path);
-        let mut paths = Vec::new();
+        let mut nodes = Vec::new();
 
         let mut read_dir = tokio::fs::read_dir(&full_path).await.with_context(|| {
             format!("Could not list contents of directory '{}'", path.display())
@@ -319,17 +319,24 @@ impl StorageBackend for LocalFS {
             .with_context(|| format!("Failed while iterating directory '{}'", path.display()))?
         {
             let entry_path = entry.path();
+            let metadata = entry.metadata().await.with_context(|| {
+                format!("Could not get metadata for '{}'", entry_path.display())
+            })?;
 
             // Strip the base_path to keep paths relative to the repo root
             let relative = entry_path
                 .strip_prefix(&self.base_path)
                 .map(Path::to_path_buf)
                 .context("Found entry outside of repository base path")?;
 
-            paths.push(relative);
+            if metadata.is_file() {
+                nodes.push(BackendNode::File(relative, metadata.len()));
+            } else if metadata.is_dir() {
+                nodes.push(BackendNode::Dir(relative));
+            }
         }
 
-        Ok(paths)
+        Ok(nodes)
     }
 
     async fn is_file(&self, path: &Path) -> bool {

core/src/backend/mod.rs

@@ -155,7 +155,7 @@ pub trait StorageBackend: Send + Sync {
     async fn rename(&self, from: &Path, to: &Path) -> Result<()>;
 
     /// Lists the immediate contents of a directory.
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>>;
+    async fn list_dir(&self, path: &Path) -> Result<Vec<BackendNode>>;
 
     /// Recursively creates a directory path.
     async fn create_dir(&self, path: &Path) -> Result<()>;
@@ -364,23 +364,23 @@ impl BackendUrl {
 /// Represents a generic filesystem node within a backend.
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub enum BackendNode {
-    File(PathBuf),
+    File(PathBuf, u64),
     Dir(PathBuf),
 }
 
 impl BackendNode {
     /// Returns the inner path regardless of node type.
     pub fn path(&self) -> &Path {
         match self {
-            BackendNode::File(path) => path,
+            BackendNode::File(path, _) => path,
             BackendNode::Dir(path) => path,
         }
     }
 
     /// Consumes the node and returns the inner path.
     pub fn into_path(self) -> PathBuf {
         match self {
-            BackendNode::File(path) => path,
+            BackendNode::File(path, _) => path,
             BackendNode::Dir(path) => path,
         }
     }
@@ -402,12 +402,15 @@ pub async fn read_backend_dir(
     while let Some(current) = stack.pop() {
         let entries = backend.list_dir(&current).await?;
 
-        for sub_path in entries {
-            if backend.is_file(&sub_path).await {
-                nodes.push(BackendNode::File(sub_path.to_path_buf()));
-            } else if backend.is_dir(&sub_path).await {
-                nodes.push(BackendNode::Dir(sub_path.to_path_buf()));
-                stack.push(sub_path.to_path_buf());
+        for node in entries {
+            match &node {
+                BackendNode::File(_, _) => {
+                    nodes.push(node);
+                }
+                BackendNode::Dir(sub_path) => {
+                    nodes.push(node.clone());
+                    stack.push(sub_path.to_path_buf());
+                }
             }
         }
     }

core/src/backend/s3.rs

@@ -4,7 +4,9 @@ use anyhow::{Context, Result, bail};
 use async_trait::async_trait;
 use s3::{Bucket, Region, creds::Credentials};
 
-use crate::backend::{Handle, NodeAttr, RetryOptions, StorageBackend, WriteContents, retry};
+use crate::backend::{
+    BackendNode, Handle, NodeAttr, RetryOptions, StorageBackend, WriteContents, retry,
+};
 
 /// A storage backend that interacts with S3-compatible APIs.
 pub struct S3Backend {
@@ -201,13 +203,13 @@ impl StorageBackend for S3Backend {
         .await
     }
 
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<BackendNode>> {
         let mut prefix = self.key_from_path(path);
         if !prefix.is_empty() && !prefix.ends_with('/') {
             prefix.push('/');
         }
 
-        let mut paths = Vec::new();
+        let mut nodes = Vec::new();
         let mut continuation_token: Option<String> = None;
 
         loop {
@@ -236,7 +238,7 @@ impl StorageBackend for S3Backend {
                 for p in prefixes {
                     let dir_key = p.prefix.trim_end_matches('/');
                     if let Ok(rel) = self.path_from_key(dir_key) {
-                        paths.push(rel);
+                        nodes.push(BackendNode::Dir(rel));
                     }
                 }
             }
@@ -247,7 +249,7 @@ impl StorageBackend for S3Backend {
                     continue;
                 }
                 if let Ok(rel) = self.path_from_key(&obj.key) {
-                    paths.push(rel);
+                    nodes.push(BackendNode::File(rel, obj.size));
                 }
             }
 
@@ -258,9 +260,9 @@ impl StorageBackend for S3Backend {
             }
         }
 
-        paths.sort();
-        paths.dedup();
-        Ok(paths)
+        nodes.sort();
+        nodes.dedup();
+        Ok(nodes)
     }
 
     async fn is_file(&self, path: &Path) -> bool {

core/src/backend/sftp.rs

@@ -25,7 +25,7 @@ use tokio::{
 
 use crate::{
     backend::{
-        Handle, NodeAttr, RetryOptions, StorageBackend, WriteContents,
+        BackendNode, Handle, NodeAttr, RetryOptions, StorageBackend, WriteContents,
         limiter::{RateLimiter, ThrottledReader, ThrottledWriter},
         retry, set_readonly_mode,
     },
@@ -561,7 +561,7 @@ impl StorageBackend for SftpBackend {
         .await
     }
 
-    async fn list_dir(&self, path: &Path) -> Result<Vec<PathBuf>> {
+    async fn list_dir(&self, path: &Path) -> Result<Vec<BackendNode>> {
         let full = self.full_path(path);
 
         self.retry(|| async {
@@ -579,7 +579,13 @@ impl StorageBackend for SftpBackend {
                 if name == "." || name == ".." {
                     continue;
                 }
-                out.push(path.join(name));
+                let path = path.join(name);
+                let metadata = entry.metadata();
+                if metadata.is_file() {
+                    out.push(BackendNode::File(path, metadata.len()));
+                } else if metadata.is_dir() {
+                    out.push(BackendNode::Dir(path));
+                }
             }
 
             Ok(out)