Added headers to requests.get calls

Author: lpla

Diff for: bitextor/bitextor_wget.py

@@ -132,14 +132,20 @@ def run(url, out_path, time_limit, agent, filetypes, warcfilename, wait):
 
     if '//' not in args.url:
         args.url = '%s%s' % ('http://', args.url)
+    headers = requests.utils.default_headers()
 
+    headers.update(
+        {
+            'User-Agent': 'Mozilla/5.0 (compatible; Bitextor/8 +https://github.com/bitextor/bitextor)',
+        }
+    )
     connection_error, fixed_url = check_connection(args.url)
 
     if not connection_error:
         args.url = fixed_url
 
         try:
-            robots = requests.get(args.url + "/robots.txt", timeout=15).text.split("\n")
+            robots = requests.get(args.url + "/robots.txt", timeout=15, headers=headers).text.split("\n")
             for line in robots:
                 if "Crawl-delay" in line:
                     try:

Diff for: bitextor/utils/common.py

@@ -155,10 +155,17 @@ def check_lengths(file_path_from, file_path_to, throw=True):
 def check_connection(url):
     connection_error = False
     connection = None
+    headers = requests.utils.default_headers()
+
+    headers.update(
+        {
+            'User-Agent': 'Mozilla/5.0 (compatible; Bitextor/8 +https://github.com/bitextor/bitextor)',
+        }
+    )
 
     for check in range(2):
         try:
-            connection = requests.get(url, timeout=15)
+            connection = requests.get(url, timeout=15, headers=headers)
         except requests.exceptions.ConnectTimeout:
             if check:
                 connection_error = True