[WIP] adding crawler functions to testing

autowebcompat/crawler.py

@@ -0,0 +1,19 @@
+from selenium.common.exceptions import NoAlertPresentException
+from selenium.common.exceptions import NoSuchWindowException
+
+
+def close_all_windows_except_first(driver):
+    windows = driver.window_handles
+
+    for window in windows[1:]:
+        driver.switch_to_window(window)
+        driver.close()
+
+    while True:
+        try:
+            alert = driver.switch_to_alert()
+            alert.dismiss()
+        except (NoAlertPresentException, NoSuchWindowException):
+            break
+
+    driver.switch_to_window(windows[0])

autowebcompat/driver.py

@@ -0,0 +1,41 @@
+import os
+
+from selenium import webdriver
+
+from autowebcompat.utils import get_browser_bin
+
+
+class Driver:
+
+    def __init__(self):
+        chrome_bin, nightly_bin = get_browser_bin()
+
+        os.environ['PATH'] += os.pathsep + os.path.abspath('tools')
+        os.environ['MOZ_HEADLESS'] = '1'
+        os.environ['MOZ_HEADLESS_WIDTH'] = '412'
+        os.environ['MOZ_HEADLESS_HEIGHT'] = '808'
+        firefox_profile = webdriver.FirefoxProfile()
+        firefox_profile.set_preference('general.useragent.override',
+                                       'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0')
+        chrome_options = webdriver.ChromeOptions()
+        chrome_options.binary_location = chrome_bin
+        chrome_options.add_argument('--no-sandbox')
+        chrome_options.add_argument('--headless')
+        chrome_options.add_argument('--window-size=412,732')
+        chrome_options.add_argument(
+            '--user-agent=Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/M4B30Z) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36')
+
+        self.firefox_driver = webdriver.Firefox(firefox_profile=firefox_profile, firefox_binary=nightly_bin)
+        self.chrome_driver = webdriver.Chrome(chrome_options=chrome_options)
+
+    def quit(self):
+        self.firefox_driver.quit()
+        self.chrome_driver.quit()
+
+    @property
+    def firefox(self):
+        return self.firefox_driver
+
+    @property
+    def chrome(self):
+        return self.chrome_driver

autowebcompat/utils.py

@@ -236,6 +236,21 @@ def write_bounding_boxes(bounding_boxes, file_name):
         print(json.dumps(bounding_boxes), file=f)
 
 
+def get_browser_bin():
+    if sys.platform.startswith('linux'):
+        chrome_bin = 'tools/chrome-linux/chrome'
+        nightly_bin = 'tools/nightly/firefox-bin'
+    elif sys.platform.startswith('darwin'):
+        chrome_bin = 'tools/chrome.app/Contents/MacOS/chrome'
+        nightly_bin = 'tools/Nightly.app/Contents/MacOS/firefox'
+    elif sys.platform.startswith('win32'):
+        chrome_bin = 'tools\\Google\\Chrome\\Application\\chrome.exe'
+        nightly_bin = 'tools\\Nightly\\firefox.exe'
+    else:
+        chrome_bin = nightly_bin = None
+    return chrome_bin, nightly_bin
+
+
 def get_all_model_summary(model, model_summary):
     line = []
     model.summary(print_fn=lambda x: line.append(x + '\n'))

collect.py

@@ -3,33 +3,23 @@
 import json
 import os
 import random
-import sys
 import time
 import traceback
 
 from PIL import Image
 from lxml import etree
-from selenium import webdriver
-from selenium.common.exceptions import NoAlertPresentException
 from selenium.common.exceptions import NoSuchElementException
-from selenium.common.exceptions import NoSuchWindowException
 from selenium.common.exceptions import TimeoutException
 
 from autowebcompat import utils
+from autowebcompat.crawler import close_all_windows_except_first
+from autowebcompat.driver import Driver
+from autowebcompat.utils import get_browser_bin
 
 MAX_THREADS = 5
 MAX_INTERACTION_DEPTH = 7
 
-if sys.platform.startswith('linux'):
-    chrome_bin = 'tools/chrome-linux/chrome'
-    nightly_bin = 'tools/nightly/firefox-bin'
-elif sys.platform.startswith('darwin'):
-    chrome_bin = 'tools/chrome.app/Contents/MacOS/chrome'
-    nightly_bin = 'tools/Nightly.app/Contents/MacOS/firefox'
-elif sys.platform.startswith('win32'):
-    chrome_bin = 'tools\\Google\\Chrome\\Application\\chrome.exe'
-    nightly_bin = 'tools\\Nightly\\firefox.exe'
-
+chrome_bin, nightly_bin = get_browser_bin()
 
 utils.mkdir('data')
 
@@ -74,23 +64,6 @@ def wait_loaded(driver):
         print('Continuing...')
 
 
-def close_all_windows_except_first(driver):
-    windows = driver.window_handles
-
-    for window in windows[1:]:
-        driver.switch_to_window(window)
-        driver.close()
-
-    while True:
-        try:
-            alert = driver.switch_to_alert()
-            alert.dismiss()
-        except (NoAlertPresentException, NoSuchWindowException):
-            break
-
-    driver.switch_to_window(windows[0])
-
-
 def get_element_properties(driver, child):
     child_properties = driver.execute_script("""
       let elem_properties = {
@@ -399,29 +372,9 @@ def run_tests(firefox_driver, chrome_driver, bugs):
     chrome_driver.quit()
 
 
-os.environ['PATH'] += os.pathsep + os.path.abspath('tools')
-os.environ['MOZ_HEADLESS'] = '1'
-os.environ['MOZ_HEADLESS_WIDTH'] = '412'
-os.environ['MOZ_HEADLESS_HEIGHT'] = '808'
-firefox_profile = webdriver.FirefoxProfile()
-firefox_profile.set_preference('general.useragent.override', 'Mozilla/5.0 (Android 6.0.1; Mobile; rv:54.0) Gecko/54.0 Firefox/54.0')
-firefox_profile.set_preference('intl.accept_languages', 'it')
-firefox_profile.set_preference('media.volume_scale', '0.0')
-chrome_options = webdriver.ChromeOptions()
-chrome_options.binary_location = chrome_bin
-chrome_options.add_argument('--no-sandbox')
-chrome_options.add_argument('--headless')
-chrome_options.add_argument('--hide-scrollbars')
-chrome_options.add_argument('--window-size=412,732')
-chrome_options.add_argument('--user-agent=Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5 Build/M4B30Z) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36')
-chrome_options.add_argument('--lang=it')
-chrome_options.add_argument('--mute-audio')
-
-
 def main(bugs):
-    firefox_driver = webdriver.Firefox(firefox_profile=firefox_profile, firefox_binary=nightly_bin)
-    chrome_driver = webdriver.Chrome(chrome_options=chrome_options)
-    run_tests(firefox_driver, chrome_driver, bugs)
+    driver = Driver()
+    run_tests(driver.firefox, driver.chrome, bugs)
 
 
 if __name__ == '__main__':

tests/test_crawler.py

@@ -0,0 +1,24 @@
+from autowebcompat import crawler
+from autowebcompat.driver import Driver
+
+
+class TestCrawler:
+
+    def setup_method(self):
+        self.driver = Driver()
+
+    def teardown_method(self):
+        pass
+
+    def test_close_all_windows_except_first(self):
+        self.driver.firefox.execute_script("window.open('https://twitter.com')")
+        self.driver.firefox.execute_script("window.open('https://google.com')")
+        self.driver.chrome.execute_script("window.open('https://twitter.com')")
+        self.driver.chrome.execute_script("window.open('https://google.com')")
+        assert len(self.driver.firefox.window_handles) == 3, 'More than 3 windows are open'
+
+        crawler.close_all_windows_except_first(self.driver.firefox)
+        crawler.close_all_windows_except_first(self.driver.chrome)
+
+        assert len(self.driver.firefox.window_handles) == 1
+        assert len(self.driver.chrome.window_handles) == 1

tests/test_utils.py

@@ -137,6 +137,14 @@ def test_to_categorical_label():
     assert categorical_label == 1
 
 
+def test_get_browser_bin():
+    chrome_bin, nightly_bin = utils.get_browser_bin()
+    assert isinstance(chrome_bin, str)
+    assert isinstance(nightly_bin, str)
+    assert os.path.exists(os.path.join(os.path.abspath('./'), chrome_bin))
+    assert os.path.exists(os.path.join(os.path.abspath('./'), nightly_bin))
+
+
 def test_create_file_name():
     assert(utils.create_file_name(bug_id='1661', browser='chrome', seq_no=None) == '1661_chrome')
     assert(utils.create_file_name(bug_id='1661', browser='chrome', width='20', height='10', seq_no=None) == '1661_H_20_V_10_chrome')