Improve special-files download script

- Now download files via chunks, to make the download of
 large files possible on low bandwiths.
- Show some kind of download progress
- Move default download sources to text files (out of python code)
- Brush up documentation

Signed-off-by: Fon E. Noel NFEBE <[email protected]>

Author: nfebe

README.md

@@ -114,11 +114,11 @@ Run `./upload-test.py test-tree/challenging-names --only=414 --remote-dir=test-4
 ##### Large uploads
 ###### Uploads
 
-To test for large file uploads. A couple of large files are needed and can be downloaded via:
+To test large file (`400MB` +) uploads, a couple of large files are required. Some ready-made test files can be downloaded via:
 
-`./special-files-downloader --large`
+`./special-files-downloader.py --large`
 
-If you have you own large files you would like to test with, you can list the links to those files in a text file like so:
+If you have your own large files or other kinds of files you would like to run tests with, you can list the links to those files in a text file like so:
 
 'my_files.txt'
 ```
@@ -127,16 +127,16 @@ https://link.com/to/file_2.extension
 https://link.com/to/file_3.extension
 ```
 
-and then run `./special-files-downloader --my-sources my_files.txt` 
+and then run `./special-files-downloader.py --my-source my_files.txt`
 
-- *You can specify as many paths as you want*
-- *You can name the the source text file anything you want but pass the right name and path to `--my-sources`*
+- *You can specify as many paths as you want inside the file*
+- *You can name the the source text file anything you want but pass the right name and path to `--my-source`*
 
-**You don't need to download any files if you already have some special files on your computer, simply copy such files into one of these directories `test-tree/special-files/large`, `test-tree/special-files/zips`, `test-tree/special-files/mixed` or `test-tree/special-files/custom`**
+**You don't need to download any files if you already have some special files on your computer, simply copy such files into one of these directories `test-tree/special-files/`, `test-tree/special-files/large`, `test-tree/special-files/zips`,  or `test-tree/special-files/custom`**
 
 Once the files are on disk:
 
-Run `./upload-test.py test-tree/special-files --remote-dir=special-files --log-file=special-files-log.txt --remote=prod --archive-path="/archives/QA (0a21-0000)/My Files/"`
+Run `./upload-test.py test-tree/special-files/large --remote-dir=large-files --log-file=large-files-log.txt --remote=prod --archive-path="/archives/QA (0a21-0000)/My Files/"`
 
 ### What file types and scenarios are left out?
 

source_large_files.txt

@@ -0,0 +1 @@
+https://www.quintic.com/software/sample_videos/Cricket%20Bowling%20150fps%201200.avi

source_zip_files.txt

@@ -2,64 +2,110 @@
 import os
 import sys
 import argparse
-import requests
 from urllib.parse import urlparse
+import requests
 
-SPECIAL_FILES_PATH = "test-tree/special-files/"
-LARGE_FILE_URIS = [
-    "https://www.quintic.com/software/sample_videos/Cricket%20Bowling%20150fps%201200.avi"
-]
+SPECIAL_FILES_ROOT = "test-tree/special-files/"
+LARGE_FILES_PATH = SPECIAL_FILES_ROOT + "large/"
+ZIP_FILES_PATH = SPECIAL_FILES_ROOT + "zips/"
+CUSTOM_FILES_PATH = SPECIAL_FILES_ROOT + "custom/"
+CHUNK_SIZE = 1024 * 1024
+LARGE_FILE_URLS = []
 ZIP_FILE_URLS = []
 
 
 def parse_cli():
+    """Prepare parser"""
     parser = argparse.ArgumentParser(
         prog="special-files-downloader", description="Download special test files"
     )
 
-    parser.add_argument("--large", help=f"Download large files for testing")
-    parser.add_argument("--zip", help=f"Download zip files for testing")
     parser.add_argument(
-        "--my-sources", help=f"Download files from links listed in text file path"
+        "--large", help="Download large files for testing", action="store_true"
+    )
+    parser.add_argument(
+        "--zip", help="Download zip files for testing", action="store_true"
+    )
+    parser.add_argument(
+        "--my-source", help="Download files from links listed in text file path"
     )
     parser.add_argument(
-        "--all",
-        help="Download all earmarked special files.",
+        "--all", help="Download all earmarked special files.", action="store_true"
     )
 
     return parser
 
 
-def download_file_from_url(uri, path):
-    response = requests.get(uri, allow_redirects=False)
-    fname = os.path.basename(urlparse(uri).path)
-    if response.status_code == 200:
-        with open(path + fname, "wb") as file:
-            file.write(response.content)
+def check_paths():
+    """Ensure special-file folders required in test-tree are present"""
+    if not os.path.exists(SPECIAL_FILES_ROOT):
+        os.makedirs(SPECIAL_FILES_ROOT)
+    if not os.path.exists(LARGE_FILES_PATH):
+        os.makedirs(LARGE_FILES_PATH)
+    if not os.path.exists(ZIP_FILES_PATH):
+        os.makedirs(ZIP_FILES_PATH)
+    if not os.path.exists(CUSTOM_FILES_PATH):
+        os.makedirs(CUSTOM_FILES_PATH)
+
+
+def get_file_urls():
+    """Get links to default special-files required for testing"""
+    global LARGE_FILE_URLS
+    global ZIP_FILE_URLS
+    large_files_handle = open("source_large_files.txt", "r", encoding="utf-8")
+    large_files = large_files_handle.readlines()
+    large_files_handle.close()
+    LARGE_FILE_URLS = map(lambda x: x.strip(), large_files)
+    zip_files_handle = open("source_zip_files.txt", "r", encoding="utf-8")
+    zip_files = zip_files_handle.readlines()
+    zip_files_handle.close()
+    ZIP_FILE_URLS = map(lambda x: x.strip(), zip_files)
+
+
+def download_file_from_url(url, path):
+    """Download file in url to path"""
+    fname = os.path.basename(urlparse(url).path)
+    print(f"\nDownloading {fname}\n")
+    underline = "=" * len(fname)
+    underline = underline + "============"
+    print(underline + "\n")
+    size = 0
+    with requests.get(url, stream=True) as response:
+        if response.status_code == 200:
+            with open(path + fname, "wb") as file:
+                for chunk in response.iter_content(chunk_size=CHUNK_SIZE):
+                    file.write(chunk)
+                    size = size + (CHUNK_SIZE)
+                    print(f"Downloaded {size} bytes of {fname} ...")
 
 
 def main():
+    "Script entry point"
+    check_paths()
+    get_file_urls()
     parser = parse_cli()
     if len(sys.argv) == 1:
         parser.print_help()
-        print("No downloads done....")
+        print(
+            "\n========================\n| No downloads done... |\n========================\n"
+        )
     args = parser.parse_args()
 
     if args.large:
-        for uri in LARGE_FILE_URIS:
-            download_file_from_url(uri, SPECIAL_FILES_PATH + "large/")
+        for url in LARGE_FILE_URLS:
+            download_file_from_url(url, LARGE_FILES_PATH)
     if args.zip:
-        for uri in ZIP_FILE_URLS:
-            download_file_from_url(uri, SPECIAL_FILES_PATH + "zips/")
+        for url in ZIP_FILE_URLS:
+            download_file_from_url(url, ZIP_FILE_URLS)
     if args.all:
-        ALL = ZIP_FILE_URLS + LARGE_FILE_URIS
-        for uri in ALL:
-            download_file_from_url(uri, SPECIAL_FILES_PATH + "mixed/")
-    if args.my_sources:
-        sources = open(args.my_sources, "r")
-        sources = sources.readlines()
-        for source in sources:
-            download_file_from_url(source, SPECIAL_FILES_PATH + "custom/")
+        ALL_URLS = ZIP_FILE_URLS + LARGE_FILE_URLS
+        for url in ALL_URLS:
+            download_file_from_url(url, SPECIAL_FILES_ROOT)
+    if args.my_source:
+        source = open(args.my_source, "r")
+        source = source.readlines()
+        for url in source:
+            download_file_from_url(url.strip(), CUSTOM_FILES_PATH)
 
 
 if __name__ == "__main__":