From 867d401d7f8fcc703dedaa318d807e10f5bd5e08 Mon Sep 17 00:00:00 2001 From: Yahia Meawad <35457341+SpartorA@users.noreply.github.com> Date: Sun, 22 Aug 2021 01:08:55 +0100 Subject: [PATCH 1/2] Update download_lt.py This is probably because of mod_security or some similar server security feature which blocks known spider/bot user agents (urllib uses something like python urllib/3.3.0, it's easily detected) --- download_lt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/download_lt.py b/download_lt.py index 7a65a9b..da5c8d6 100755 --- a/download_lt.py +++ b/download_lt.py @@ -9,6 +9,7 @@ import subprocess import sys import os +import requests from contextlib import closing from distutils.spawn import find_executable @@ -128,7 +129,7 @@ def download_lt(update=False): return with closing(TemporaryFile()) as t: - with closing(urlopen(url)) as u: + with closing(requests.get(url)) as u: content_len = int(u.headers['Content-Length']) sys.stdout.write( @@ -140,7 +141,7 @@ def download_lt(update=False): chunk_len = content_len // 100 data_len = 0 while True: - data = u.read(chunk_len) + data = u.iter_content(chunk_size=chunk_len, decode_unicode=False) if not data: break data_len += len(data) From c07c66c989d384ddfcb4a6b0fe72e9780ee4bd1d Mon Sep 17 00:00:00 2001 From: Yahia Meawad <35457341+SpartorA@users.noreply.github.com> Date: Sun, 22 Aug 2021 02:45:16 +0100 Subject: [PATCH 2/2] Update download_lt.py --- download_lt.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/download_lt.py b/download_lt.py index da5c8d6..4def5c7 100755 --- a/download_lt.py +++ b/download_lt.py @@ -129,7 +129,7 @@ def download_lt(update=False): return with closing(TemporaryFile()) as t: - with closing(requests.get(url)) as u: + with closing(requests.get(url,stream=True)) as u: content_len = int(u.headers['Content-Length']) sys.stdout.write( @@ -140,10 +140,7 @@ def download_lt(update=False): chunk_len = content_len // 100 data_len = 0 - while True: - data = u.iter_content(chunk_size=chunk_len, decode_unicode=False) - if not data: - break + for data in u.iter_content(chunk_len): data_len += len(data) t.write(data) sys.stdout.write(