diff --git a/binaryunit.py b/binaryunit.py index 298c47a..4aa9cf0 100755 --- a/binaryunit.py +++ b/binaryunit.py @@ -16,7 +16,7 @@ def __init__(self, value, unit): def power(self, number): for power in range(len(PREFIXES)): if number < MAX_NUMBER: - return power; + return power number /= BASE diff --git a/context.py b/context.py new file mode 100644 index 0000000..063a19b --- /dev/null +++ b/context.py @@ -0,0 +1,23 @@ +class Context(object): + @property + def drive(self): + raise NotImplementedError() + + @property + def http(self): + raise NotImplementedError() + + def logProgress(self, + path, + start, + bytesUploaded, + bytesTotal, + progress, + end): + raise NotImplementedError() + + def addToBatch(self, request, callback = None): + raise NotImplementedError() + + def addToBatchAndExecute(self, request): + raise NotImplementedError() diff --git a/date.py b/date.py index cca1610..07d45da 100755 --- a/date.py +++ b/date.py @@ -51,3 +51,7 @@ def __str__(self): dateTime = datetime.datetime.utcfromtimestamp(seconds) return dateTime.strftime(DATE_TIME_FORMAT) + + @property + def seconds(self): + return self._seconds diff --git a/driveutils.py b/driveutils.py index db58055..cf74dd9 100755 --- a/driveutils.py +++ b/driveutils.py @@ -11,7 +11,7 @@ TIMEOUT = 60 # seconds -FIELDS = 'id, title, mimeType, createdDate, modifiedDate, md5Checksum, fileSize' +FIELDS = 'id, title, mimeType, createdDate, modifiedDate, md5Checksum, fileSize, description, downloadUrl' DEFAULT_CLIENT_ID = '387402765904.apps.googleusercontent.com' DEFAULT_CLIENT_SECRET = 'WTj0xKbLAFjDqUeT2HGDZHCi' @@ -31,7 +31,8 @@ def drive(saveCredentials = None): http = (credentials(saveCredentials) .authorize(httplib2.Http(timeout = TIMEOUT))) - return apiclient.discovery.build('drive', 'v2', http = http) + return (apiclient.discovery.build('drive', 'v2', http = http), + http) def credentials(save = None): clientId = config.get('clientId') diff --git a/file.py b/file.py index 28206a8..70ffe89 100755 --- a/file.py +++ b/file.py @@ -6,6 +6,8 @@ class File(object): def __init__(self, path, name, folder = None): + # Never instantiate the base class. + assert self.__class__ != File self._path = path self._name = name self._folder = utils.firstNonNone(folder, False) @@ -38,7 +40,6 @@ def md5(self): if self.folder: md5 = hashlib.md5() md5.update(self.name.encode('utf-8')) - return md5.hexdigest() return self.contentMd5 @@ -49,4 +50,9 @@ def contentMd5(self): @property def exists(self): - return False + raise NotImplementedError() + + # Select the first element of the tuple if the file is a local file, or the + # second element of the tuple if the file is a remote file. + def select(self, tuple): + raise NotImplementedError() diff --git a/folder.py b/folder.py index 5f76ea9..ca4ff16 100755 --- a/folder.py +++ b/folder.py @@ -3,17 +3,19 @@ import file import utils -import os - -def empty(file): - return Folder(file) +import posixpath class Folder(object): def __init__(self, file, children = None, duplicate = None): + # Never instantiate the base class. + assert self.__class__ != Folder self._file = file self._children = utils.firstNonNone(children, {}) self._duplicate = utils.firstNonNone(duplicate, []) + def _newFolder(self, file, children = None, duplicate = None): + return self.__class__(file, children, duplicate) + def addChild(self, file): if file.name in self._children: self._duplicate.append(file) @@ -49,12 +51,63 @@ def folders(self): return filter(lambda f: f.folder, self._children.values()) def withoutChildren(self): - return Folder(self._file) + return self._newFolder(self._file) def withoutDuplicate(self): - return Folder(self._file, self._children) + return self._newFolder(self._file, self._children) def createFile(self, name, folder = None): - path = os.path.join(self._file.path, name) - - return file.File(path, name, folder) + raise NotImplementedError() + +import localfolder +import remotefolder + +import urlparse + +class Factory(object): + class VirtualFile(file.File): + def __init__(self): + super(Factory.VirtualFile, self).__init__(None, None, True) + + class VirtualFolder(Folder): + def __init__(self): + super(Factory.VirtualFolder, self).__init__(Factory.VirtualFile()) + + def __init__(self, context): + self.remoteFolderFactory = remotefolder.Factory(context) + self.localFolderFactory = localfolder.Factory(context) + + def createVirtual(self, paths): + virtualFolder = Factory.VirtualFolder() + for path in paths: + (head, tail) = posixpath.split(path) + if tail == '': + pathFolder = self.createFromURL(head) + virtualFolder.addChildren(pathFolder.children.values()) + else: + realFolder = self.createFromURL(head) + # Virtual folder are sources, so the file must already exist. + virtualFolder.addChild(realFolder.children[tail]) + + return virtualFolder + + def createFromURL(self, url): + parsed_url = urlparse.urlparse(url) + if parsed_url.scheme == 'gdrive': + return self.remoteFolderFactory.create(parsed_url.path) + elif not parsed_url.scheme or parsed_url.scheme == 'file': + return self.localFolderFactory.create(parsed_url.path) + raise RuntimeError('Unable to handle url: %s' % file_descriptor) + + def create(self, file_descriptor): + return self._getFactoryForFileDescriptor( + file_descriptor).create(file_descriptor) + + def createEmpty(self, file_descriptor): + return self._getFactoryForFileDescriptor( + file_descriptor).createEmpty(file_descriptor) + + def _getFactoryForFileDescriptor(self, file_descriptor): + assert isinstance(file_descriptor, file.File) + return file_descriptor.select((self.localFolderFactory, + self.remoteFolderFactory)) diff --git a/gdrsync.py b/gdrsync.py index 40c482a..822c9ec 100755 --- a/gdrsync.py +++ b/gdrsync.py @@ -4,23 +4,21 @@ import argparse import os -parser = argparse.ArgumentParser(description = 'Copy files from a local system' - ' to a Google drive repository.') +parser = argparse.ArgumentParser(description = 'Synchronize between a local' + ' system and a Google drive repository.') -parser.add_argument('localPaths', nargs='+', - help = ('local paths. A trailing %s means "copy the contents of this' +parser.add_argument('sourcePaths', nargs='+', + help = ('source paths. A trailing %s means "copy the contents of this' ' directory", as opposed to "copy the directory itself"' % os.path.sep), - metavar = 'LOCAL') -parser.add_argument('remotePath', help = 'remote path', metavar = 'REMOTE') + metavar = 'SOURCE') +parser.add_argument('targetPath', help = 'target path', metavar = 'TARGET') parser.add_argument('-c', action = 'store_true', help = 'skip based on checksum, not mod-time & size', dest = 'checksum') parser.add_argument('-d', action = 'store_true', help = 'delete duplicate and extraneous files from dest dirs', dest = 'delete') -parser.add_argument('-L', action = 'store_true', - help = 'transform symlink into referent file/dir', dest = 'copyLinks') parser.add_argument('-n', action = 'store_true', help = 'perform a trial run with no changes made', dest = 'dryRun') parser.add_argument('-r', action = 'store_true', @@ -46,17 +44,15 @@ logging.getLogger('oauth2client.util').setLevel(logging.ERROR) import binaryunit +import context import driveutils import folder -import localfolder -import remotefolder import requestexecutor +import transfer import utils -import virtuallocalfolder import apiclient.http import errno -import mimetypes import time CHUNKSIZE = 1 * utils.MIB @@ -67,287 +63,240 @@ LOGGER = logging.getLogger(__name__) -class GDRsync(object): +class GDRsync(context.Context): def __init__(self, args): self.args = args - self.drive = driveutils.drive(self.args.saveCredentials) - - self.localFolderFactory = localfolder.Factory() - self.remoteFolderFactory = remotefolder.Factory(self.drive) + (self._drive, self._http) = driveutils.drive(self.args.saveCredentials) + self.batch = None + self.nbBatch = 0 self.copiedFiles = 0 self.copiedSize = 0 self.copiedTime = 0 - self.checkedFiles = 0 self.checkedSize = 0 - self.totalFiles = 0 + self.folderFactory = folder.Factory(self) + + @property + def drive(self): + return self._drive + + @property + def http(self): + return self._http + def sync(self): LOGGER.info('Starting...') - virtualLocalFolder = virtuallocalfolder.Factory().create(self.args.localPaths) - remoteFolder = self.remoteFolderFactory.create(self.args.remotePath) - self._sync(virtualLocalFolder, remoteFolder) + sourceFolder = self.folderFactory.createVirtual(self.args.sourcePaths) + targetFolder = self.folderFactory.createFromURL(self.args.targetPath) + self._sync(sourceFolder, targetFolder) + self._flushBatch() - self.logResult(); + self.logResult() LOGGER.info('End.') - def _sync(self, localFolder, remoteFolder): - remoteFolder = self.trash(localFolder, remoteFolder) + def _sync(self, sourceFolder, targetFolder): + targetFolder = self.trash(sourceFolder, targetFolder) - self.totalFiles += len(localFolder.children) + self.totalFiles += len(sourceFolder.children) - remoteFolder = self.syncFolder(localFolder, remoteFolder) + targetFolder = self.syncFolder(sourceFolder, targetFolder) if not self.args.recursive: return - for localFile in localFolder.folders(): - if (not self.args.copyLinks) and localFile.link: - continue - - remoteFile = remoteFolder.children[localFile.name] + for sourceFile in sourceFolder.folders(): + targetFile = targetFolder.children[sourceFile.name] - self._sync(self.createLocalFolder(localFile), - self.createRemoteFolder(remoteFile)) + self._sync(self.createSourceFolder(sourceFile), + self.createTargetFolder(targetFile)) - def trash(self, localFolder, remoteFolder): + def trash(self, sourceFolder, targetFolder): if not self.args.delete: - return remoteFolder + return targetFolder - remoteFolder = self.trashDuplicate(localFolder, remoteFolder) - remoteFolder = self.trashExtraneous(localFolder, remoteFolder) - remoteFolder = self.trashDifferentType(localFolder, remoteFolder) + targetFolder = self.trashDuplicate(sourceFolder, targetFolder) + targetFolder = self.trashExtraneous(sourceFolder, targetFolder) + targetFolder = self.trashDifferentType(sourceFolder, targetFolder) - return remoteFolder + return targetFolder - def trashDuplicate(self, localFolder, remoteFolder): - for remoteFile in remoteFolder.duplicate: - LOGGER.debug('%s: Duplicate file.', remoteFile.path) + def trashDuplicate(self, sourceFolder, targetFolder): + for targetFile in targetFolder.duplicate: + LOGGER.debug('%s: Duplicate file.', targetFile.path) - remoteFile = self.trashFile(remoteFile) + self.trashFile(targetFile) - return remoteFolder.withoutDuplicate() + return targetFolder.withoutDuplicate() - def trashFile(self, remoteFile): - LOGGER.info('%s: Trashing file...', remoteFile.path) + def trashFile(self, targetFile): + LOGGER.info('%s: Trashing file...', targetFile.path) if self.args.dryRun: - return remoteFile - - def request(): - return (self.drive.files() - .trash(fileId = remoteFile.delegate['id'], - fields = driveutils.FIELDS) - .execute()) - - file = requestexecutor.execute(request) + return targetFile - return remoteFile.withDelegate(file) + transfer.trashFile(self, targetFile) - def trashExtraneous(self, localFolder, remoteFolder): - output = remoteFolder.withoutChildren() - for remoteFile in remoteFolder.children.values(): - if remoteFile.name in localFolder.children: - output.addChild(remoteFile) + def trashExtraneous(self, sourceFolder, targetFolder): + output = targetFolder.withoutChildren() + for targetFile in targetFolder.children.values(): + if targetFile.name in sourceFolder.children: + output.addChild(targetFile) continue - LOGGER.debug('%s: Extraneous file.', remoteFile.path) + LOGGER.debug('%s: Extraneous file.', targetFile.path) - remoteFile = self.trashFile(remoteFile) + self.trashFile(targetFile) return output - def trashDifferentType(self, localFolder, remoteFolder): - output = remoteFolder.withoutChildren() - for remoteFile in remoteFolder.children.values(): - localFile = localFolder.children[remoteFile.name] - if localFile.folder == remoteFile.folder: - output.addChild(remoteFile) + def trashDifferentType(self, sourceFolder, targetFolder): + output = targetFolder.withoutChildren() + for targetFile in targetFolder.children.values(): + sourceFile = sourceFolder.children[targetFile.name] + if sourceFile.folder == targetFile.folder: + output.addChild(targetFile) continue - LOGGER.debug('%s: Different type: %s != %s.', remoteFile.path, - localFile.folder, remoteFile.folder) + LOGGER.debug('%s: Different type: %s != %s.', targetFile.path, + sourceFile.folder, targetFile.folder) - remoteFile = self.trashFile(remoteFile) + self.trashFile(targetFile) return output - def syncFolder(self, localFolder, remoteFolder): - output = (remoteFolder.withoutChildren() - .addChildren(remoteFolder.children.values())) - for localFile in localFolder.children.values(): + def syncFolder(self, sourceFolder, targetFolder): + output = (targetFolder.withoutChildren() + .addChildren(targetFolder.children.values())) + for sourceFile in sourceFolder.children.values(): self.checkedFiles += 1 - self.checkedSize += localFile.size + self.checkedSize += sourceFile.size try: - remoteFile = self.copy(localFile, remoteFolder) - if remoteFile is None: + targetFile = self.copy(sourceFile, targetFolder) + if targetFile is None: continue - output.addChild(remoteFile) + output.addChild(targetFile) except OSError as error: if error.errno != errno.ENOENT: raise - LOGGER.warn('%s: No such file or directory.', localFile.path) + LOGGER.warn('%s: No such file or directory.', sourceFile.path) return output - def copy(self, localFile, remoteFolder): - remoteFile = remoteFolder.children.get(localFile.name) + def copy(self, sourceFile, targetFolder): + targetFile = targetFolder.children.get(sourceFile.name) - fileOperation = self.fileOperation(localFile, remoteFile) + fileOperation = self.fileOperation(sourceFile, targetFile) if fileOperation is None: return None - if remoteFile is None: - remoteFile = remoteFolder.createFile(localFile.name, - localFile.folder) + if targetFile is None: + targetFile = targetFolder.createFile(sourceFile.name, + sourceFile.folder) - return fileOperation(localFile, remoteFile) + return fileOperation(sourceFile, targetFile) - def fileOperation(self, localFile, remoteFile): - if (not self.args.copyLinks) and localFile.link: - LOGGER.info('%s: Skipping non-regular file... (Checked %d/%d files)', - localFile.path, self.checkedFiles, self.totalFiles) - - return None - - if remoteFile is None: - if localFile.folder: + def fileOperation(self, sourceFile, targetFile): + if targetFile is None: + if sourceFile.folder: return self.insertFolder return self.insertFile - if self.args.update and (remoteFile.modified > localFile.modified): + if (self.args.update and + not sourceFile.link and + not sourceFile.folder and + targetFile.modified > sourceFile.modified): LOGGER.debug('%s: Newer destination file: %s < %s.', - remoteFile.path, localFile.modified, remoteFile.modified) + targetFile.path, sourceFile.modified, targetFile.modified) elif self.args.checksum: - fileOperation = self.checkChecksum(localFile, remoteFile) + fileOperation = self.checkChecksum(sourceFile, targetFile) if fileOperation is not None: return fileOperation - fileOperation = self.checkSize(localFile, remoteFile) + fileOperation = self.checkSize(sourceFile, targetFile) if fileOperation is not None: return fileOperation - fileOperation = self.checkModified(localFile, remoteFile) + fileOperation = self.checkMetadataModified(sourceFile, targetFile) if fileOperation is not None: return fileOperation else: - fileOperation = self.checkSize(localFile, remoteFile) + fileOperation = self.checkSize(sourceFile, targetFile) if fileOperation is not None: return fileOperation - fileOperation = self.checkModified(localFile, remoteFile) + fileOperation = self.checkMetadataModified(sourceFile, targetFile) if fileOperation is not None: return fileOperation - LOGGER.debug('%s: Up to date. (Checked %d/%d files)', remoteFile.path, + LOGGER.debug('%s: Up to date. (Checked %d/%d files)', targetFile.path, self.checkedFiles, self.totalFiles) return None - def checkChecksum(self, localFile, remoteFile): - if remoteFile.md5 == localFile.md5: + def checkChecksum(self, sourceFile, targetFile): + if targetFile.md5 == sourceFile.md5: return None - LOGGER.debug('%s: Different checksum: %s != %s.', remoteFile.path, - localFile.md5, remoteFile.md5) + LOGGER.debug('%s: Different checksum: %s != %s.', targetFile.path, + sourceFile.md5, targetFile.md5) return self.updateFile - def checkSize(self, localFile, remoteFile): - if remoteFile.size == localFile.size: + def checkSize(self, sourceFile, targetFile): + if targetFile.size == sourceFile.size: return None - LOGGER.debug('%s: Different size: %d != %d.', remoteFile.path, - localFile.size, remoteFile.size) + LOGGER.debug('%s: Different size: %d != %d.', targetFile.path, + sourceFile.size, targetFile.size) return self.updateFile - def checkModified(self, localFile, remoteFile): - if remoteFile.modified == localFile.modified: + def checkMetadataModified(self, sourceFile, targetFile): + if (targetFile.metadata() == sourceFile.metadata()): return None - fileOperation = self.checkChecksum(localFile, remoteFile) + fileOperation = self.checkChecksum(sourceFile, targetFile) if fileOperation is not None: return fileOperation - LOGGER.debug('%s: Different modified time: %s != %s.', remoteFile.path, - localFile.modified, remoteFile.modified) + LOGGER.debug('%s: Different metadata: %s != %s.', + targetFile.path, sourceFile.metadata(), + targetFile.metadata()) return self.touch - def insertFolder(self, localFile, remoteFile): + def insertFolder(self, sourceFile, targetFile): LOGGER.info('%s: Inserting folder... (Checked %d/%d files)', - remoteFile.path, self.checkedFiles, self.totalFiles) + targetFile.path, self.checkedFiles, self.totalFiles) if self.args.dryRun: - return remoteFile - - body = remoteFile.delegate.copy() - body['modifiedDate'] = str(localFile.modified) - def request(): - return (self.drive.files().insert(body = body, - fields = driveutils.FIELDS).execute()) - - file = requestexecutor.execute(request) + return targetFile - return remoteFile.withDelegate(file) + return transfer.insertFolder(self, sourceFile, targetFile) - def insertFile(self, localFile, remoteFile): + def insertFile(self, sourceFile, targetFile): LOGGER.info('%s: Inserting file... (Checked %d/%d files)', - remoteFile.path, self.checkedFiles, self.totalFiles) + targetFile.path, self.checkedFiles, self.totalFiles) if self.args.dryRun: - return remoteFile - - def createRequest(body, media): - return (self.drive.files().insert(body = body, media_body = media, - fields = driveutils.FIELDS)) - - return self.copyFile(localFile, remoteFile, createRequest) - - def copyFile(self, localFile, remoteFile, createRequest): - body = remoteFile.delegate.copy() - body['modifiedDate'] = str(localFile.modified) - - (mimeType, encoding) = mimetypes.guess_type(localFile.delegate) - if mimeType is None: - mimeType = DEFAULT_MIME_TYPE + return targetFile - resumable = (localFile.size > CHUNKSIZE) - media = apiclient.http.MediaFileUpload(localFile.delegate, - mimetype = mimeType, chunksize = CHUNKSIZE, - resumable = resumable) + return transfer.transferData(self, sourceFile, targetFile) - def request(): - request = createRequest(body, media) - - start = time.time() - if not resumable: - file = request.execute() - self.logProgress(remoteFile.path, start, localFile.size) - - return file - - while True: - (progress, file) = request.next_chunk() - if file is not None: - self.logProgress(remoteFile.path, start, localFile.size) - - return file - - self.logProgress(remoteFile.path, start, - progress.resumable_progress, progress.total_size, - progress.progress(), False) - - file = requestexecutor.execute(request) + def updateFile(self, sourceFile, targetFile): + LOGGER.info('%s: Updating file... (Checked %d/%d files)', + targetFile.path, self.checkedFiles, self.totalFiles) + if self.args.dryRun: + return targetFile - return remoteFile.withDelegate(file) + return transfer.transferData(self, sourceFile, targetFile) def logProgress(self, path, start, bytesUploaded, bytesTotal = None, progress = 1.0, end = True): @@ -386,52 +335,28 @@ def bS(self, bytesUploaded, elapsed): def eta(self, elapsed, bytesUploaded, bytesTotal): if bytesUploaded == 0: return 0 - + bS = bytesUploaded / elapsed finish = bytesTotal / bS return round(finish - elapsed) - def updateFile(self, localFile, remoteFile): - LOGGER.info('%s: Updating file... (Checked %d/%d files)', - remoteFile.path, self.checkedFiles, self.totalFiles) + def touch(self, sourceFile, targetFile): + LOGGER.info('%s: Updating metadata... (Checked %d/%d files)', + targetFile.path, self.checkedFiles, self.totalFiles) if self.args.dryRun: - return remoteFile + return targetFile - def createRequest(body, media): - return (self.drive.files() - .update(fileId = remoteFile.delegate['id'], body = body, - media_body = media, setModifiedDate = True, - fields = driveutils.FIELDS)) + return transfer.touchFile(self, sourceFile, targetFile) - return self.copyFile(localFile, remoteFile, createRequest) + def createSourceFolder(self, sourceFile): + return self.folderFactory.create(sourceFile) - def touch(self, localFile, remoteFile): - LOGGER.info('%s: Updating modified date... (Checked %d/%d files)', - remoteFile.path, self.checkedFiles, self.totalFiles) - if self.args.dryRun: - return remoteFile - - body = {'modifiedDate': str(localFile.modified)} - - def request(): - return (self.drive.files() - .patch(fileId = remoteFile.delegate['id'], body = body, - setModifiedDate = True, fields = driveutils.FIELDS) - .execute()) - - file = requestexecutor.execute(request) - - return remoteFile.withDelegate(file) + def createTargetFolder(self, targetFile): + if self.args.dryRun and (not targetFile.exists): + return self.folderFactory.createEmpty(targetFile) - def createLocalFolder(self, localFile): - return self.localFolderFactory.create(localFile) - - def createRemoteFolder(self, remoteFile): - if self.args.dryRun and (not remoteFile.exists): - return folder.empty(remoteFile) - - return self.remoteFolderFactory.create(remoteFile) + return self.folderFactory.create(targetFile) def logResult(self): copiedSize = binaryunit.BinaryUnit(self.copiedSize, 'B') @@ -446,4 +371,34 @@ def logResult(self): copiedTime, round(bS.value), bS.unit, self.checkedFiles, round(checkedSize.value), checkedSize.unit) + def addToBatch(self, request, callback = None): + if not self.batch: + self.batch = apiclient.http.BatchHttpRequest() + self.batch.add(request, callback) + self.nbBatch += 1 + if self.nbBatch > 500: + self._flushBatch() + + def addToBatchAndExecute(self, request): + # If there is only one request to execute, do not bother setting up a + # batch. + if not self.batch: + return requestexecutor.execute(request.execute) + + results = {} + def callback(id, result, exception): + results['id'] = id + results['result'] = result + results['exception'] = exception + self.addToBatch(request, callback) + self._flushBatch() + return results.get('result') + + def _flushBatch(self): + if not self.batch: + return + requestexecutor.execute(self.batch.execute) + self.batch = None + self.nbBatch = 0 + GDRsync(args).sync() diff --git a/localfile.py b/localfile.py index 0249e0c..1881ce2 100755 --- a/localfile.py +++ b/localfile.py @@ -6,65 +6,92 @@ import date import hashlib import os +import stat MD5_BUFFER_SIZE = 16 * utils.KIB def fromParent(parent, path, folder = None): - return fromParentPath(parent.path, path, folder) + return fromParentPath(parent.context, parent.path, path, folder) -def fromParentPath(parentPath, path, folder = None): +def fromParentPath(context, parentPath, path, folder = None): name = os.path.basename(path) path = os.path.join(parentPath, name) - return LocalFile(path, folder) + return LocalFile(context, path, folder) class LocalFile(file.File): - def __init__(self, path, folder = None): - path = unicode(path) + def __init__(self, context, path, folder = None): name = os.path.basename(path) - folder = utils.firstNonNone(folder, os.path.isdir(path)) + folder = utils.firstNonNone(folder, os.path.isdir(path) and + not os.path.islink(path)) super(LocalFile, self).__init__(path, name, folder) - - @property - def delegate(self): - return self.path + self._content_md5 = None + self.context = context @property def contentSize(self): - if not self.exists: + if not self.exists or self.link or self.folder: return 0 return os.path.getsize(self.path) @property def modified(self): - return date.fromSeconds(os.path.getmtime(self.path)) + file_stat = os.lstat(self.path) + return date.fromSeconds(file_stat.st_mtime) @property def contentMd5(self): - with open(self.path, mode = 'rb') as file: + if self._content_md5 is None: md5 = hashlib.md5() - while True: - data = file.read(MD5_BUFFER_SIZE) - if not data: - break + if self.contentSize > 0: + with open(self.path, mode = 'rb') as file: + while True: + data = file.read(MD5_BUFFER_SIZE) + if not data: + break - md5.update(data) + md5.update(data) - return md5.hexdigest() + self._content_md5 = md5.hexdigest() + return self._content_md5 @property def exists(self): - return os.path.exists(self.path) + return os.path.lexists(self.path) @property def link(self): return os.path.islink(self.path) + def metadata(self, withMd5 = False): + file_stat = os.lstat(self.path) + metadata = { + 'uid': file_stat.st_uid, + 'gid': file_stat.st_gid, + 'fileSize': self.contentSize + } + if self.link: + metadata['target'] = unicode(os.readlink(self.path), encoding='latin-1') + metadata['type'] = 'link' + else: + if not self.folder: + metadata['modifiedDate'] = str(self.modified) + metadata['mode'] = stat.S_IMODE(file_stat.st_mode) + if withMd5: + metadata['cs'] = self.md5 + return metadata + + def select(self, tuple): + return tuple[0] + class Factory(object): - def create(self, path): - if not os.path.exists(path): + def __init__(self, context): + self.context = context + + def create(self, context, path): + if not os.path.lexists(path): raise RuntimeError('%s not found' % path) - return LocalFile(path) + return LocalFile(context, path) diff --git a/localfolder.py b/localfolder.py index b0af5cc..ccdc5e9 100755 --- a/localfolder.py +++ b/localfolder.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import file import folder import localfile @@ -10,15 +11,21 @@ def createFile(self, name, folder = None): return localfile.fromParent(self.file, name, folder) class Factory(object): - def create(self, file): - if not isinstance(file, localfile.LocalFile): - localFileFactory = localfile.Factory() + def __init__(self, context): + self.localFileFactory = localfile.Factory(context) + self.context = context - return self.create(localFileFactory.create(file)) + def createEmpty(self, file): + return LocalFolder(file) - localFolder = folder.Folder(file) - for path in os.listdir(file.delegate): - localFile = localfile.fromParent(file, path) + def create(self, fileinstance): + if not isinstance(fileinstance, file.File): + return self.create( + self.localFileFactory.create(self.context, fileinstance)) + + localFolder = LocalFolder(fileinstance) + for path in os.listdir(fileinstance.path): + localFile = localfile.fromParent(fileinstance, path) localFolder.addChild(localFile) return localFolder diff --git a/remotefile.py b/remotefile.py index 117f96a..aa1c8e3 100755 --- a/remotefile.py +++ b/remotefile.py @@ -3,6 +3,7 @@ import date import driveutils import file +import json import requestexecutor import utils @@ -12,24 +13,46 @@ FILE_ID_QUERY = '(title = \'%(title)s\') and (not trashed)' +def _getFilename(delegate): + title = delegate['title'] + if title.__class__ == unicode: + return title.encode('latin-1', 'xmlcharrefreplace') + return title + def fromParent(parent, delegate): return fromParentPath(parent.path, delegate) def fromParentPath(parentPath, delegate): - path = posixpath.join(parentPath, delegate['title']) + path = posixpath.join(parentPath, _getFilename(delegate)) return RemoteFile(path, delegate) class RemoteFile(file.File): def __init__(self, path, delegate, folder = None): - parent = unicode(path) - name = delegate['title'] + name = _getFilename(delegate) folder = utils.firstNonNone(folder, delegate.get('mimeType') == MIME_FOLDER) super(RemoteFile, self).__init__(path, name, folder) self._delegate = delegate + try: + self._parsedMetadata = json.loads(self._delegate.get('description', + '{}')) + except: + self._parsedMetadata = {} + if self._parsedMetadata.get('modifiedDate') is None: + self._parsedMetadata['modifiedDate'] = self._delegate.get('modifiedDate', + self._delegate.get('createdDate', None)) + if self._parsedMetadata.get('cs') is None: + self._parsedMetadata['cs'] = self._delegate.get('md5Checksum') + if self._parsedMetadata.get('fileSize') is None: + self._parsedMetadata['fileSize'] = int(self._delegate.get('fileSize', 0)) + if self.link or self.folder: + del self._parsedMetadata['modifiedDate'] + self._parsedMetadataWithoutMd5 = self._parsedMetadata.copy() + if 'cs' in self._parsedMetadataWithoutMd5.keys(): + del self._parsedMetadataWithoutMd5['cs'] @property def delegate(self): @@ -37,29 +60,39 @@ def delegate(self): @property def contentSize(self): - return int(self._delegate['fileSize']) + return self.metadata().get('fileSize') @property def modified(self): - modifiedDate = self._delegate.get('modifiedDate', - self._delegate['createdDate']) - + modifiedDate = self.metadata().get('modifiedDate') return date.fromString(modifiedDate) @property def contentMd5(self): - return self._delegate['md5Checksum'] + return self.metadata(True).get('cs') @property def exists(self): return 'id' in self._delegate + @property + def link(self): + return self.metadata(True).get('type', '') == 'link' + + def metadata(self, withMd5 = False): + if withMd5: + return self._parsedMetadata + return self._parsedMetadataWithoutMd5 + def withDelegate(self, delegate): return RemoteFile(self.path, delegate) + def select(self, tuple): + return tuple[1] + class Factory(object): - def __init__(self, drive): - self.drive = drive + def __init__(self, context): + self.context = context def create(self, path): fileId = self.retrieveFileId(path) @@ -67,7 +100,7 @@ def create(self, path): raise RuntimeError('%s not found' % path) def request(): - return (self.drive.files().get(fileId = fileId, + return (self.context.drive.files().get(fileId = fileId, fields = driveutils.FIELDS).execute()) file = requestexecutor.execute(request) @@ -88,8 +121,11 @@ def retrieveFileId(self, path): query = (FILE_ID_QUERY % {'title' : driveutils.escapeQueryParameter(name)}) def request(): - return (self.drive.children().list(folderId = parentId, q = query, - maxResults = 1, fields = 'items(id)') .execute()) + return (self.context.drive.children().list( + folderId = parentId, + q = query, + maxResults = 1, + fields = 'items(id)').execute()) children = requestexecutor.execute(request) for child in children.get('items'): diff --git a/remotefolder.py b/remotefolder.py index 384315c..2ebf33c 100755 --- a/remotefolder.py +++ b/remotefolder.py @@ -6,8 +6,6 @@ import requestexecutor import utils -import os - CHILDREN_QUERY = '(\'%(parents)s\' in parents) and (not trashed)' CHILDREN_FIELDS = 'nextPageToken, items(%s)' % driveutils.FIELDS @@ -24,30 +22,32 @@ def withoutChildren(self): def withoutDuplicate(self): return RemoteFolder(self._file, self._children) - def createFile(self, name, folder = None, mimeType = None): + def createFile(self, name, folder = None): folder = utils.firstNonNone(folder, False) - file = {'title': name} - if mimeType is None: - if folder: - file['mimeType'] = remotefile.MIME_FOLDER - else: - file['mimeType'] = mimeType - + file = {'title': unicode(name, encoding='latin-1')} + if folder: + file['mimeType'] = remotefile.MIME_FOLDER file['parents'] = [{'id': self.file.delegate['id']}] return remotefile.fromParent(self.file, file) def createFolder(self, name): - return self.createFile(name, remotefile.MIME_FOLDER) + return self.createFile(name, True) class Factory(object): - def __init__(self, drive): - self.drive = drive + def __init__(self, context): + self.context = context + + def createEmpty(self, file): + # This is only used for dry run, create a marker id. + if not file.delegate.get('id'): + file.delegate['id'] = '-1' + return RemoteFolder(file) def create(self, file): if not isinstance(file, remotefile.RemoteFile): - remoteFileFactory = remotefile.Factory(self.drive) + remoteFileFactory = remotefile.Factory(self.context) return self.create(remoteFileFactory.create(file)) @@ -57,7 +57,7 @@ def request(): pageToken = None while True: - list = (self.drive.files().list(q = query, + list = (self.context.drive.files().list(q = query, fields = CHILDREN_FIELDS, pageToken = pageToken, maxResults = LIST_MAX_RESULTS)) diff --git a/test/__main__.py b/test/__main__.py new file mode 100644 index 0000000..30ebde3 --- /dev/null +++ b/test/__main__.py @@ -0,0 +1,20 @@ +import os +import sys +import unittest + + +here = os.path.dirname(__file__) +loader = unittest.defaultTestLoader + +def suite(): + suite = unittest.TestSuite() + for fn in os.listdir(here): + if fn.startswith("test") and fn.endswith(".py"): + modname = fn[:-3] + __import__(modname) + module = sys.modules[modname] + suite.addTest(loader.loadTestsFromModule(module)) + return suite + +if __name__ == "__main__": + unittest.main(defaultTest="suite") diff --git a/test/drivemock.py b/test/drivemock.py new file mode 100644 index 0000000..1de6023 --- /dev/null +++ b/test/drivemock.py @@ -0,0 +1,40 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +class DriveMock: + def __init__(self, iteration = 0, has_sub_iteration = True): + self._iteration = iteration + self._has_sub_iteration = has_sub_iteration + + def __getattr__(self, name): + return self + + def __call__(self, *args, **kwargs): + if kwargs.get('maxResults', 0) > 1: + return DriveMock(0, False) + return self + + def __str__(self): + return 'DriveMock(%d, %s)' % (self._iteration, self._has_sub_iteration) + + def __int__(self): + return 1 + + def __iter__(self): + if not self._has_sub_iteration: + return DriveMock(0, False) + return DriveMock(1) + + def __nonzero__(self): + return False + + def next(self): + if self._iteration > 0: + self._iteration -= 1 + return self + raise StopIteration() + + def get(self, *args, **kwargs): + if len(args) == 1 and args[0] == 'nextPageToken': + return None + return self diff --git a/test/prepare-test.sh b/test/prepare-test.sh index bc6b9cb..2e06268 100755 --- a/test/prepare-test.sh +++ b/test/prepare-test.sh @@ -9,10 +9,10 @@ function logAndRun() { eval "${@}" } -logAndRun "${GDRSYNC}" -vv -dLr "${DIR}/remote/" /test +logAndRun "${GDRSYNC}" -vv -dr "${DIR}/remote/" gdrive:///test -find "${DIR}/local" | xargs touch -h -d '1978-08-03 01:23:45.6789987Z' -touch -d '1970-01-01 00:00:00Z' "${DIR}/local/invalidDate" +find "${DIR}/local" | xargs touch -h -t '197808030123.45' +touch -t '197001010000.00' "${DIR}/local/invalidDate" cat <<-EOF Rename using the web interface: diff --git a/test/test.sh b/test/test.sh index 95aec90..7db0538 100755 --- a/test/test.sh +++ b/test/test.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + DIR="$(dirname "${0}")" GDRSYNC="${DIR}/../gdrsync.py" @@ -9,10 +11,10 @@ function logAndRun() { eval "${@}" } -logAndRun "${GDRSYNC}" -vv -n "${DIR}/local/" /test +logAndRun "${GDRSYNC}" -vv -n "${DIR}/local/" gdrive:///test -logAndRun "${GDRSYNC}" -vv -r -n "${DIR}/local/" /test +logAndRun "${GDRSYNC}" -vv -r -n "${DIR}/local/" gdrive:///test -logAndRun "${GDRSYNC}" -vv -u "${DIR}/local/" /test +logAndRun "${GDRSYNC}" -vv -u "${DIR}/local/" gdrive:///test -logAndRun "${GDRSYNC}" -vv -dLr "${DIR}/local/" /test +logAndRun "${GDRSYNC}" -vv -dr "${DIR}/local/" gdrive:///test diff --git a/test/test_folder.py b/test/test_folder.py new file mode 100644 index 0000000..90f792d --- /dev/null +++ b/test/test_folder.py @@ -0,0 +1,30 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import sys +sys.path.append('%s/..' % sys.path[0]) + +import drivemock +import localfolder +import folder +import remotefolder + +import os +import unittest + +class TestFolder(unittest.TestCase): + + def test_folder_factory(self): + os.chdir('%s/..' % sys.path[0]) + drive = drivemock.DriveMock() + context = lambda: None + context.drive = drive + + factory = folder.Factory(context) + self.assertIsInstance(factory.createFromURL('gdrive:///tmp'), remotefolder.RemoteFolder) + self.assertIsInstance(factory.createFromURL('file:///tmp'), localfolder.LocalFolder) + self.assertIsInstance(factory.createFromURL('/tmp'), localfolder.LocalFolder) + self.assertIsInstance(factory.createFromURL('test'), localfolder.LocalFolder) + +if __name__ == '__main__': + unittest.main() diff --git a/transfer.py b/transfer.py new file mode 100644 index 0000000..2c95d28 --- /dev/null +++ b/transfer.py @@ -0,0 +1,214 @@ +import driveutils +import requestexecutor +import utils + +import apiclient.http +import io +import json +import mimetypes +import shutil +import time +import os + +def _openSourceFile(sourceFile, targetFile): + return io.open(sourceFile.path, "rb") + +def _openTargetFile(sourceFile, targetFile): + if targetFile.exists: + mode = "r+b" + else: + mode = "wb" + output = io.open(targetFile.path, mode) + output.seek(0) + return output + +def _metadata(sourceFile): + return json.dumps(sourceFile.metadata(withMd5 = True)) + +def _copyRemoteFile(context, sourceFile, targetFile): + if targetFile.exists: + trashFile(context, targetFile) + body = sourceFile.delegate.copy() + body['parents'] = targetFile.delegate['parents'] + file = context.addToBatchAndExecute(context.drive.files().copy( + fileId = sourceFile.delegate['id'], + body = body, + fields = driveutils.FIELDS)) + return targetFile.withDelegate(file) + +def _uploadLocalFile(context, sourceFile, targetFile): + body = targetFile.delegate.copy() + body['description'] = _metadata(sourceFile) + + (mimeType, _) = mimetypes.guess_type(sourceFile.path) + if mimeType is None: + mimeType = utils.DEFAULT_MIME_TYPE + + resumable = (sourceFile.size > utils.CHUNKSIZE) + if sourceFile.link: + media = None + else: + input =_openSourceFile(sourceFile, targetFile) + media = apiclient.http.MediaIoBaseUpload( + input, + mimetype = mimeType, + chunksize = utils.CHUNKSIZE, + resumable = resumable) + + def request(): + if targetFile.exists: + request = context.drive.files().update( + fileId = targetFile.delegate['id'], + body = body, + media_body = media, + fields = driveutils.FIELDS) + else: + request = context.drive.files().insert( + body = body, + media_body = media, + fields = driveutils.FIELDS) + + start = time.time() + if not resumable: + file = request.execute() + context.logProgress(targetFile.path, start, sourceFile.size) + + return file + + while True: + (progress, file) = request.next_chunk() + if file is not None: + context.logProgress( + targetFile.path, start, sourceFile.size) + + return file + + context.logProgress(targetFile.path, start, + progress.resumable_progress, progress.total_size, + progress.progress(), False) + + file = requestexecutor.execute(request) + + return targetFile.withDelegate(file) + +def _downloadRemoteFile(context, sourceFile, targetFile): + + if sourceFile.link: + os.symlink(sourceFile.metadata()['target'], targetFile.path) + return targetFile + + def request(): + with _openTargetFile(sourceFile, targetFile) as targetFd: + http_request = apiclient.http.HttpRequest( + context.http, + None, + sourceFile.delegate.get('downloadUrl'), + headers = {}) + downloader = apiclient.http.MediaIoBaseDownload( + targetFd, + http_request, + chunksize=utils.CHUNKSIZE) + + start = time.time() + while True: + (progress, done) = downloader.next_chunk() + if done is not None: + context.logProgress(targetFile.path, start, sourceFile.size) + # If the download is a success, truncate the local file in + # case it was longer than the downloaded file. + targetFd.truncate() + return done + + context.logProgress(targetFile.path, start, + progress.resumable_progress, progress.total_size, + progress.progress(), False) + + requestexecutor.execute(request) + return touchFile(context, sourceFile, targetFile) + +def _copyLocalFile(context, sourceFile, targetFile): + if sourceFile.link: + os.symlink(sourceFile.metadata()['target'], targetFile.path) + return targetFile + + with io.open(sourceFile.path, "rb") as sourceFd, \ + io.open(targetFile.path, "wb") as targetFd: + shutil.copyfileobj(sourceFd, targetFd) + + return touchFile(context, sourceFile, targetFile) + +def _trashLocalFile(context, targetFile): + if targetFile.folder: + shutil.rmtree(targetFile.path) + else: + os.unlink(targetFile.path) + +def _trashRemoteFile(context, targetFile): + context.addToBatch(context.drive.files().trash( + fileId = targetFile.delegate['id'], + fields = driveutils.FIELDS)) + +def _touchLocalFile(context, sourceFile, targetFile): + metadata = sourceFile.metadata() + os.lchown(targetFile.path, + metadata.get('uid', -1), + metadata.get('gid', -1)) + if sourceFile.link: + if hasattr(os, 'lchmod'): + os.lchmod(targetFile.path, 0777) + elif metadata.get('mode'): + os.chmod(targetFile.path, metadata.get('mode')) + if not sourceFile.link and not sourceFile.folder: + os.utime(targetFile.path, (sourceFile.modified.seconds, + sourceFile.modified.seconds)) + + return targetFile + +def _touchRemoteFile(context, sourceFile, targetFile): + body = {'description': _metadata(sourceFile)} + context.addToBatch(context.drive.files().patch( + fileId = targetFile.delegate['id'], + body = body, + setModifiedDate = True, + fields = driveutils.FIELDS)) + return targetFile + +def _insertLocalFolder(context, sourceFile, targetFile): + os.makedirs(targetFile.path) + return touchFile(context, sourceFile, targetFile) + +def _insertRemoteFolder(context, sourceFile, targetFile): + body = targetFile.delegate.copy() + body['description'] = _metadata(sourceFile) + file = context.addToBatchAndExecute(context.drive.files().insert( + body = body, + fields = driveutils.FIELDS)) + return targetFile.withDelegate(file) + +def trashFile(context, targetFile): + return targetFile.select((_trashLocalFile, _trashRemoteFile))( + context, + targetFile) + +def transferData(context, sourceFile, targetFile): + actions = ( + (_copyLocalFile, _uploadLocalFile), + (_downloadRemoteFile, _copyRemoteFile) + ) + + return targetFile.select(sourceFile.select(actions))( + context, + sourceFile, + targetFile) + +def touchFile(context, sourceFile, targetFile): + return targetFile.select((_touchLocalFile, _touchRemoteFile))( + context, + sourceFile, + targetFile) + +def insertFolder(context, sourceFile, targetFile): + return targetFile.select((_insertLocalFolder, _insertRemoteFolder))( + context, + sourceFile, + targetFile) diff --git a/utils.py b/utils.py index 2086105..6157fd1 100755 --- a/utils.py +++ b/utils.py @@ -2,6 +2,9 @@ KIB = 0x400 # bytes / kibibyte MIB = KIB * KIB # bytes / mebibyte +CHUNKSIZE = 1 * MIB + +DEFAULT_MIME_TYPE = 'application/octet-stream' def firstNonNone(*arguments): for argument in arguments: diff --git a/virtualfile.py b/virtualfile.py deleted file mode 100755 index ee9d046..0000000 --- a/virtualfile.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/python - -import file - -class VirtualFile(file.File): - def __init__(self, folder = None): - super(VirtualFile, self).__init__(None, None, folder) diff --git a/virtualfolder.py b/virtualfolder.py deleted file mode 100755 index 5f54ac4..0000000 --- a/virtualfolder.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/python - -import folder -import virtualfile - -import os - -class Factory(object): - def create(self, paths): - virtualFolder = folder.Folder(virtualfile.VirtualFile(True)) - for path in paths: - (head, tail) = self.split(path) - if tail == '': - pathFolder = self.folderFactory.create(head) - virtualFolder.addChildren(pathFolder.children.values()) - else: - pathFile = self.fileFactory.create(path) - virtualFolder.addChild(pathFile) - - return virtualFolder - - def split(self, path): - raise NotImplementedError() - - @property - def folderFactory(self): - raise NotImplementedError() - - @property - def fileFactory(self): - raise NotImplementedError() diff --git a/virtuallocalfolder.py b/virtuallocalfolder.py deleted file mode 100755 index 85de1ef..0000000 --- a/virtuallocalfolder.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python - -import localfile -import localfolder -import virtualfolder - -import os - -class Factory(virtualfolder.Factory): - def __init__(self): - self._folderFactory = localfolder.Factory() - self._fileFactory = localfile.Factory() - - def split(self, path): - return os.path.split(path) - - @property - def folderFactory(self): - return self._folderFactory - - @property - def fileFactory(self): - return self._fileFactory diff --git a/virtualremotefolder.py b/virtualremotefolder.py deleted file mode 100755 index 61d4831..0000000 --- a/virtualremotefolder.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python - -import remotefile -import remotefolder -import virtualfolder - -import posixpath - -class Factory(virtualfolder.Factory): - def __init__(self, drive): - self._folderFactory = remotefolder.Factory(drive) - self._fileFactory = remotefile.Factory(drive) - - def split(self, path): - return posixpath.split(path) - - @property - def folderFactory(self): - return self._folderFactory - - @property - def fileFactory(self): - return self._fileFactory