From c319e1984fb0301954b9508e501ee0e4d00b4727 Mon Sep 17 00:00:00 2001 From: Sergey Ermeykin Date: Mon, 19 Aug 2019 22:01:49 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5?= =?UTF-8?q?=D0=BD=20=D0=BC=D0=B5=D1=82=D0=BE=D0=B4=20uploaViaHash,=20?= =?UTF-8?q?=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=BC=D0=B5?= =?UTF-8?q?=D1=82=D0=BE=D0=B4=20upload.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- YaDiskClient/YaDiskClient.py | 128 ++++++++++++++++++++++++++++++++--- 1 file changed, 119 insertions(+), 9 deletions(-) diff --git a/YaDiskClient/YaDiskClient.py b/YaDiskClient/YaDiskClient.py index c593e01..0165ecb 100644 --- a/YaDiskClient/YaDiskClient.py +++ b/YaDiskClient/YaDiskClient.py @@ -4,7 +4,12 @@ from requests import request import xml.etree.ElementTree as ET - +import sys +import hashlib +import os +import http.client +import logging +import base64 class YaDiskException(Exception): """Common exception class for YaDisk. Arg 'code' have code of HTTP Error.""" @@ -42,8 +47,11 @@ class YaDisk(object): login = None password = None - url = "https://webdav.yandex.ru/" + host = "webdav.yandex.ru" + url = "https://" + host + "/" namespaces = {'d': 'DAV:'} + bufSize = 65536 + maxSizeFile = 50*1024**3 # https://yandex.ru/support/disk/uploading.html def __init__(self, login, password): super(YaDisk, self).__init__() @@ -52,11 +60,46 @@ def __init__(self, login, password): if self.login is None or self.password is None: raise YaDiskException(400, "Please, specify login and password for Yandex.Disk account.") - def _sendRequest(self, type, addUrl="/", addHeaders={}, data=None): + def _calcHash(self, file): + md5 = hashlib.md5() + sha256 = hashlib.sha256() + with open(file, 'rb') as f: + while True: + data = f.read(self.bufSize) + if not data: + break + md5.update(data) + sha256.update(data) + return md5.hexdigest(), sha256.hexdigest() + + def _sendRequest(self, type, addUrl="/", addHeaders={}, data=None, runAtContinue = True): headers = {"Accept": "*/*"} headers.update(addHeaders) url = self.url + addUrl - return request(type, url, headers=headers, auth=(self.login, self.password), data=data) + # https://stackoverflow.com/questions/38084993/python-http-client-stuck-on-100-continue + if "Expect" not in headers: + return request(type, url, headers=headers, auth=(self.login, self.password), data=data) + else: + # https://yandex.ru/dev/disk/doc/dg/concepts/quickstart-docpage/ + auth = base64.b64encode(bytes(self.login + ':' + self.password, 'utf-8')).decode('utf-8') + headers.update({ 'Authorization' : 'Basic ' + auth }) + conn = ContinueHTTPSConnection(self.host) + conn.request(type, url, body=None, headers=headers) + response = conn.getresponse() + if (response.status == http.client.CONTINUE) and runAtContinue: + response.read() + conn.send(data) + response = conn.getresponse() + else: + conn.close() + class R: + status_code = 0 + content = "" + resp = R() + resp.status_code = response.status + resp.content = response.msg + return resp + def ls(self, path, offset=None, amount=None): """ @@ -152,13 +195,52 @@ def mv(self, src, dst): if resp.status_code != 201: raise YaDiskException(resp.status_code, resp.content) - def upload(self, file, path): + def upload(self, file, path, calcHash=True): """Upload file.""" - with open(file, "rb") as f: - resp = self._sendRequest("PUT", path, data=f) - if resp.status_code != 201: - raise YaDiskException(resp.status_code, resp.content) + size = os.path.getsize(file) + if size > self.maxSizeFile: + print ("Big file. Aborted.") + return False + + if (calcHash): + md5, sha256 = self._calcHash(file) + with open(file, "rb") as f: + resp = self._sendRequest("PUT", path, data=f, addHeaders={ + "Etag": md5, + "Sha256": sha256, + "Content-Length": str(size), + "Expect": "100-continue" + }) + else: + with open(file, "rb") as f: + resp = self._sendRequest("PUT", path, data=f) + + if resp.status_code != 201: + raise YaDiskException(resp.status_code, resp.content) + else: + return True + + def uploadViaHash(self, md5, sha256, size, path): + """Upload file via hash.""" + + if size > self.maxSizeFile: + print ("Big file. Aborted.") + return False + + resp = self._sendRequest("PUT", path, runAtContinue = False, addHeaders={ + "Etag": md5, + "Sha256": sha256, + "Content-Length": str(size), + "Expect": "100-continue" + }) + + if resp.status_code == 201: + return True + elif resp.status_code == 100: + return False + else: + raise YaDiskException(resp.status_code, resp.content) def download(self, path, file): """Download remote file to disk.""" @@ -222,3 +304,31 @@ def publish_doc(self, path): def hide_doc(self, path): warn('This method was deprecated in favor method "unpublish"', DeprecationWarning, stacklevel=2) return self.unpublish(path) + + +class ContinueHTTPResponse(http.client.HTTPResponse): + def _read_status(self, *args, **kwargs): + version, status, reason = super()._read_status(*args, **kwargs) + if status == 100: + status = 199 + return version, status, reason + + def begin(self, *args, **kwargs): + super().begin(*args, **kwargs) + if self.status == 199: + self.status = 100 + + def _check_close(self, *args, **kwargs): + return super()._check_close(*args, **kwargs) and self.status != 100 + + +class ContinueHTTPSConnection(http.client.HTTPSConnection): + response_class = ContinueHTTPResponse + + def getresponse(self, *args, **kwargs): + logging.debug('running getresponse') + response = super().getresponse(*args, **kwargs) + if response.status == 100: + setattr(self, '_HTTPConnection__state', http.client._CS_REQ_SENT) + setattr(self, '_HTTPConnection__response', None) + return response From 5ae0c6c8810ae658075d50fde9860537118ec068 Mon Sep 17 00:00:00 2001 From: Sergey Ermeykin Date: Mon, 26 Aug 2019 22:22:35 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=9F=D0=BE=D1=87=D0=B8=D0=BD=D0=BA=D0=B0?= =?UTF-8?q?=20=D1=82=D0=B5=D1=81=D1=82=D0=BE=D0=B2.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- YaDiskClient/YaDiskClient.py | 2 +- tests/test_yaDisk.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/YaDiskClient/YaDiskClient.py b/YaDiskClient/YaDiskClient.py index 0165ecb..c894e23 100644 --- a/YaDiskClient/YaDiskClient.py +++ b/YaDiskClient/YaDiskClient.py @@ -184,7 +184,7 @@ def cp(self, src, dst): _check_dst_absolute(dst) resp = self._sendRequest("COPY", src, {'Destination': dst}) - if resp.status_code != 201: + if resp.status_code not in (201, 202): raise YaDiskException(resp.status_code, resp.content) def mv(self, src, dst): diff --git a/tests/test_yaDisk.py b/tests/test_yaDisk.py index da8029a..43e6a26 100644 --- a/tests/test_yaDisk.py +++ b/tests/test_yaDisk.py @@ -18,6 +18,9 @@ class TestYaDisk(unittest.TestCase): remote_folder = None remote_file = None remote_path = None + md5 = 'E678A6380E2EA14F1B104D5F3E64EA70' + sha256 = '0EB25AE2FBFB90C988C0BE1C650D272819D6386A1C5536C59AD46F4AE8BEA760' + size = 647218 @classmethod def setUpClass(cls): @@ -112,3 +115,10 @@ def test_bad_auth(self): YaDisk(None, None) except YaDiskException as e: self.assertTrue(str(e).startswith(str(e.code))) + + def test_upload_via_hash(self): + mp3_file = "{folder}/{file}.png".format(folder=self.remote_folder, file=''.join(random.choice(string.ascii_uppercase) for _ in range(6))) + self.disk.mkdir(self.remote_folder) + result = self.disk.uploadViaHash(self.md5, self.sha256, self.size, mp3_file) + self.assertTrue(result) + self.disk.rm(self.remote_folder) \ No newline at end of file From a57081cd00feb5701b24083faa0889726f22fe42 Mon Sep 17 00:00:00 2001 From: Sergey Ermeykin Date: Mon, 26 Aug 2019 23:03:36 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=A3=D0=B1=D1=80=D0=B0=D0=BD=D0=B0=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6=D0=BA=D0=B0=20=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D1=81=D0=B8=D0=B9=202.7,=203.3.=20=D0=94=D0=BE?= =?UTF-8?q?=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D0=B0=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D0=B4=D0=B5=D1=80=D0=B6=D0=BA=D0=B0=20=D0=B2=D0=B5=D1=80=D1=81?= =?UTF-8?q?=D0=B8=D0=B8=203.7.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .travis.yml | 3 +-- setup.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index de412e7..e1f405b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,11 +4,10 @@ os: - "linux" python: - - "2.7" - - "3.3" - "3.4" - "3.5" - "3.6" + - "3.7" install: - pip install requests - pip install python-coveralls diff --git a/setup.py b/setup.py index b4c9cac..8177123 100644 --- a/setup.py +++ b/setup.py @@ -24,11 +24,10 @@ 'Intended Audience :: System Administrators', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python', 'Topic :: Internet', 'Topic :: Utilities',