Skip to content

Port Crawler Improvements #258

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Port Crawler Improvements #258

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
99ce1b8
Port Crawler Improvements
Shashi456 Aug 15, 2018
d75af2c
Indentation errors
Shashi456 Aug 27, 2018
c13f5a8
Merge branch 'master' into coll
Shashi456 Aug 27, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
213 changes: 120 additions & 93 deletions collect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,14 @@
from PIL import Image
from lxml import etree
from selenium import webdriver
from selenium.common.exceptions import ElementNotInteractableException
from selenium.common.exceptions import InvalidSelectorException
from selenium.common.exceptions import NoAlertPresentException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoSuchWindowException
from selenium.common.exceptions import StaleElementReferenceException
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import WebDriverException

from autowebcompat import utils

Expand Down Expand Up @@ -129,108 +133,131 @@ def was_visited(current_path, visited_paths, elem_properties):


def do_something(driver, visited_paths, current_path, elem_properties=None, xpath=None):
elem = None

body = driver.find_elements_by_tag_name('body')
assert len(body) == 1
body = body[0]

buttons = body.find_elements_by_tag_name('button')
links = body.find_elements_by_tag_name('a')
inputs = body.find_elements_by_tag_name('input')
selects = body.find_elements_by_tag_name('select')
children = buttons + links + inputs + selects

if elem_properties is None and xpath is None:
random.shuffle(children)
children_to_ignore = [] # list of elements with same properties to ignore

for child in children:
if child in children_to_ignore:
continue

# Get all the properties of the child.
elem_properties = get_element_properties(driver, child)
already_clicked_elems = set()
not_clickable_elems = set()
while True:
elem = None

# If the element is not displayed or is disabled, the user can't interact with it. Skip
# non-displayed/disabled elements, since we're trying to mimic a real user.
if not child.is_displayed() or not child.is_enabled():
continue
body = driver.find_elements_by_tag_name('body')
assert len(body) == 1
body = body[0]

if was_visited(current_path, visited_paths, elem_properties):
continue
buttons = body.find_elements_by_tag_name('button')
links = body.find_elements_by_tag_name('a')
inputs = body.find_elements_by_tag_name('input')
selects = body.find_elements_by_tag_name('select')
children = buttons + links + inputs + selects

elem = child
if elem_properties is None and xpath is None:
random.shuffle(children)
children_to_ignore = [] # list of elements with same properties to ignore

# We mark the current path as visited
elems = get_elements_with_properties(driver, elem_properties, children)
if len(elems) == 1:
elem = child
xpath = driver.execute_script(get_xpath_script, elem)
break
else:
children_to_ignore.extend(elems)
else:
if 'id' in elem_properties['attributes'].keys():
elem_id = elem_properties['attributes']['id']
elem = driver.find_element_by_id(elem_id)
if xpath is None:
xpath = driver.execute_script(get_xpath_script, elem)
elif xpath is not None:
try:
elem = driver.find_element_by_xpath(xpath)
except NoSuchElementException:
elems = get_elements_with_properties(driver, elem_properties, children)
assert len(elems) == 1
elem = elems[0]
xpath = driver.execute_script(get_xpath_script, elem)
if set(children) - already_clicked_elems > not_clickable_elems:
children = list(set(children) - already_clicked_elems)
for child in children:
if child in children_to_ignore or child in not_clickable_elems:
continue

# Get all the properties of the child.
elem_properties = get_element_properties(driver, child)

# If the element is not displayed or is disabled, the user can't interact with it. Skip
# non-displayed/disabled elements, since we're trying to mimic a real user.
if not child.is_displayed() or not child.is_enabled():
continue

if was_visited(current_path, visited_paths, elem_properties):
continue

elem = child

# We mark the current path as visited
elems = get_elements_with_properties(driver, elem_properties, children)
if len(elems) == 1:
elem = child
xpath = driver.execute_script(get_xpath_script, elem)
break
else:
children_to_ignore.extend(elems)
except (ElementNotInteractableException, StaleElementReferenceException, InvalidSelectorException, WebDriverException):
# Ignore frequent exceptions.
traceback.print_exc()
not_clickable_elems.add(elem)
close_all_windows_except_first(driver)
else:
elems = get_elements_with_properties(driver, elem_properties, children)
assert len(elems) == 1
elem = elems[0]
xpath = driver.execute_script(get_xpath_script, elem)

if elem is None:
return None

driver.execute_script('arguments[0].scrollIntoView();', elem)

if elem.tag_name in ['button', 'a']:
elem.click()
elif elem.tag_name == 'input':
input_type = elem.get_attribute('type')
if input_type == 'url':
elem.send_keys('http://www.mozilla.org/')
elif input_type == 'text':
elem.send_keys('marco')
elif input_type == 'email':
elem.send_keys('[email protected]')
elif input_type == 'password':
elem.send_keys('aMildlyComplexPasswordIn2017')
elif input_type == 'checkbox':
elem.click()
elif input_type == 'number':
elem.send_keys('3')
elif input_type == 'radio':
elem.click()
elif input_type == 'search':
elem.clear()
elem.send_keys('quick search')
elif input_type == 'submit':
try:
if 'id' in elem_properties['attributes'].keys():
elem_id = elem_properties['attributes']['id']
elem = driver.find_element_by_id(elem_id)
if xpath is None:
xpath = driver.execute_script(get_xpath_script, elem)
elif xpath is not None:
try:
elem = driver.find_element_by_xpath(xpath)
except NoSuchElementException:
elems = get_elements_with_properties(driver, elem_properties, children)
assert len(elems) == 1
elem = elems[0]
xpath = driver.execute_script(get_xpath_script, elem)
else:
elems = get_elements_with_properties(driver, elem_properties, children)
assert len(elems) == 1
elem = elems[0]
xpath = driver.execute_script(get_xpath_script, elem)
except (ElementNotInteractableException, StaleElementReferenceException, InvalidSelectorException, WebDriverException):
# Ignore frequent exceptions.
traceback.print_exc()
not_clickable_elems.add(elem)
close_all_windows_except_first(driver)

if elem is None:
return None

driver.execute_script('arguments[0].scrollIntoView();', elem)

if elem.tag_name in ['button', 'a']:
elem.click()
elif input_type == 'color':
driver.execute_script("arguments[0].value = '#ff0000'", elem)
else:
raise Exception('Unsupported input type: %s' % input_type)
elif elem.tag_name == 'select':
for option in elem.find_elements_by_tag_name('option'):
if option.text != '':
option.click()
break
elif elem.tag_name == 'input':
input_type = elem.get_attribute('type')
if input_type == 'url':
elem.send_keys('http://www.mozilla.org/')
elif input_type == 'text':
elem.send_keys('marco')
elif input_type == 'email':
elem.send_keys('[email protected]')
elif input_type == 'password':
elem.send_keys('aMildlyComplexPasswordIn2017')
elif input_type == 'checkbox':
elem.click()
elif input_type == 'number':
elem.send_keys('3')
elif input_type == 'radio':
elem.click()
elif input_type == 'tel':
elem.send_keys('1234567890')
elif input_type == 'date':
elem.send_keys('20000101')
elif input_type == 'search':
elem.clear()
elem.send_keys('quick search')
elif input_type in ['submit', 'reset', 'button']:
elem.click()
elif input_type == 'color':
driver.execute_script("arguments[0].value = '#ff0000'", elem)
else:
raise Exception('Unsupported input type: %s' % input_type)
elif elem.tag_name == 'select':
for option in elem.find_elements_by_tag_name('option'):
if option.text != '':
option.click()
break

already_clicked_elems.add(elem)

close_all_windows_except_first(driver)
close_all_windows_except_first(driver)

return elem_properties, xpath
return elem_properties, xpath


def screenshot(driver, bug_id, browser, seq_no):
Expand Down