diff --git a/install.py b/install.py index 76313f6..38aac7a 100644 --- a/install.py +++ b/install.py @@ -1,7 +1,10 @@ import launch if not launch.is_installed("bs4"): - launch.run_pip("install bs4", "bs4 requirement for GDick links parsing") + launch.run_pip("install bs4", "bs4 requirement for GDisk links parsing") + +if not launch.is_installed("requests-cache"): + launch.run_pip("install requests-cache", "requests-cache requirement disabling download cache") # Make this installation optional only for Firebase storage due to incompatible # protobuf (4.22.0) version dependency of latest (6.1.0) firebase-admin version. But tensorboard requires 3.20.0 diff --git a/scripts/mo/dl/http_downloader.py b/scripts/mo/dl/http_downloader.py index f96404a..de5bd63 100644 --- a/scripts/mo/dl/http_downloader.py +++ b/scripts/mo/dl/http_downloader.py @@ -2,6 +2,7 @@ import threading from urllib.parse import urlparse import requests +import requests_cache from tqdm import tqdm from scripts.mo.dl.downloader import Downloader @@ -41,9 +42,11 @@ class HttpDownloader(Downloader): if api_key: auth_header = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + api_key} - response = requests.get(url, stream=True, headers=auth_header) + with requests_cache.disabled(): + response = requests.get(url, stream=True, headers=auth_header) else: - response = requests.get(url, stream=True) + with requests_cache.disabled(): + response = requests.get(url, stream=True) total_size = int(response.headers.get('content-length', 0))