Skip to content

Commit 99ce1b8

Browse files
Shashi456Shashi456
committed
Port Crawler Improvements
1 parent 5a41b9d commit 99ce1b8

File tree

1 file changed

+120
-97
lines changed

1 file changed

+120
-97
lines changed

‎collect.py‎

Lines changed: 120 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,19 @@
1010
from PIL import Image
1111
from lxml import etree
1212
from selenium import webdriver
13+
from selenium.common.exceptions import ElementNotInteractableException
14+
from selenium.common.exceptions import InvalidSelectorException
1315
from selenium.common.exceptions import NoAlertPresentException
1416
from selenium.common.exceptions import NoSuchElementException
1517
from selenium.common.exceptions import NoSuchWindowException
18+
from selenium.common.exceptions import StaleElementReferenceException
1619
from selenium.common.exceptions import TimeoutException
20+
from selenium.common.exceptions import WebDriverException
1721

1822
from autowebcompat import utils
1923

24+
already_clicked_elems = set()
25+
2026
MAX_THREADS = 5
2127
MAX_INTERACTION_DEPTH = 7
2228

@@ -129,108 +135,124 @@ def was_visited(current_path, visited_paths, elem_properties):
129135

130136

131137
def do_something(driver, visited_paths, current_path, elem_properties=None, xpath=None):
132-
elem = None
133-
134-
body = driver.find_elements_by_tag_name('body')
135-
assert len(body) == 1
136-
body = body[0]
137-
138-
buttons = body.find_elements_by_tag_name('button')
139-
links = body.find_elements_by_tag_name('a')
140-
inputs = body.find_elements_by_tag_name('input')
141-
selects = body.find_elements_by_tag_name('select')
142-
children = buttons + links + inputs + selects
143-
144-
if elem_properties is None and xpath is None:
145-
random.shuffle(children)
146-
children_to_ignore = [] # list of elements with same properties to ignore
147-
148-
for child in children:
149-
if child in children_to_ignore:
150-
continue
151-
152-
# Get all the properties of the child.
153-
elem_properties = get_element_properties(driver, child)
138+
not_clickable_elems = set()
139+
while True:
140+
elem = None
154141

155-
# If the element is not displayed or is disabled, the user can't interact with it. Skip
156-
# non-displayed/disabled elements, since we're trying to mimic a real user.
157-
if not child.is_displayed() or not child.is_enabled():
158-
continue
142+
body = driver.find_elements_by_tag_name('body')
143+
assert len(body) == 1
144+
body = body[0]
159145

160-
if was_visited(current_path, visited_paths, elem_properties):
161-
continue
146+
buttons = body.find_elements_by_tag_name('button')
147+
links = body.find_elements_by_tag_name('a')
148+
inputs = body.find_elements_by_tag_name('input')
149+
selects = body.find_elements_by_tag_name('select')
150+
children = buttons + links + inputs + selects
162151

163-
elem = child
152+
if elem_properties is None and xpath is None:
153+
random.shuffle(children)
154+
children_to_ignore = [] # list of elements with same properties to ignore
164155

165-
# We mark the current path as visited
166-
elems = get_elements_with_properties(driver, elem_properties, children)
167-
if len(elems) == 1:
168-
elem = child
169-
xpath = driver.execute_script(get_xpath_script, elem)
170-
break
171-
else:
172-
children_to_ignore.extend(elems)
173-
else:
174-
if 'id' in elem_properties['attributes'].keys():
175-
elem_id = elem_properties['attributes']['id']
176-
elem = driver.find_element_by_id(elem_id)
177-
if xpath is None:
178-
xpath = driver.execute_script(get_xpath_script, elem)
179-
elif xpath is not None:
180156
try:
181-
elem = driver.find_element_by_xpath(xpath)
182-
except NoSuchElementException:
183-
elems = get_elements_with_properties(driver, elem_properties, children)
184-
assert len(elems) == 1
185-
elem = elems[0]
186-
xpath = driver.execute_script(get_xpath_script, elem)
187-
else:
188-
elems = get_elements_with_properties(driver, elem_properties, children)
189-
assert len(elems) == 1
190-
elem = elems[0]
191-
xpath = driver.execute_script(get_xpath_script, elem)
192-
193-
if elem is None:
194-
return None
195-
196-
driver.execute_script('arguments[0].scrollIntoView();', elem)
197-
198-
if elem.tag_name in ['button', 'a']:
199-
elem.click()
200-
elif elem.tag_name == 'input':
201-
input_type = elem.get_attribute('type')
202-
if input_type == 'url':
203-
elem.send_keys('http://www.mozilla.org/')
204-
elif input_type == 'text':
205-
elem.send_keys('marco')
206-
elif input_type == 'email':
207-
elem.send_keys('[email protected]')
208-
elif input_type == 'password':
209-
elem.send_keys('aMildlyComplexPasswordIn2017')
210-
elif input_type == 'checkbox':
211-
elem.click()
212-
elif input_type == 'number':
213-
elem.send_keys('3')
214-
elif input_type == 'radio':
215-
elem.click()
216-
elif input_type == 'search':
217-
elem.clear()
218-
elem.send_keys('quick search')
219-
elif input_type == 'submit':
220-
elem.click()
221-
elif input_type == 'color':
222-
driver.execute_script("arguments[0].value = '#ff0000'", elem)
223-
else:
224-
raise Exception('Unsupported input type: %s' % input_type)
225-
elif elem.tag_name == 'select':
226-
for option in elem.find_elements_by_tag_name('option'):
227-
if option.text != '':
228-
option.click()
229-
break
230-
231-
close_all_windows_except_first(driver)
232-
233-
return elem_properties, xpath
157+
if set(children) - already_clicked_elems > not_clickable_elems:
158+
children = list(set(children) - already_clicked_elems)
159+
for child in children:
160+
if child in children_to_ignore or child in not_clickable_elems:
161+
continue
162+
163+
# Get all the properties of the child.
164+
elem_properties = get_element_properties(driver, child)
165+
166+
# If the element is not displayed or is disabled, the user can't interact with it. Skip
167+
# non-displayed/disabled elements, since we're trying to mimic a real user.
168+
if not child.is_displayed() or not child.is_enabled():
169+
continue
170+
171+
if was_visited(current_path, visited_paths, elem_properties):
172+
continue
173+
174+
elem = child
175+
176+
# We mark the current path as visited
177+
elems = get_elements_with_properties(driver, elem_properties, children)
178+
if len(elems) == 1:
179+
elem = child
180+
xpath = driver.execute_script(get_xpath_script, elem)
181+
break
182+
else:
183+
children_to_ignore.extend(elems)
184+
else:
185+
if 'id' in elem_properties['attributes'].keys():
186+
elem_id = elem_properties['attributes']['id']
187+
elem = driver.find_element_by_id(elem_id)
188+
if xpath is None:
189+
xpath = driver.execute_script(get_xpath_script, elem)
190+
elif xpath is not None:
191+
try:
192+
elem = driver.find_element_by_xpath(xpath)
193+
except NoSuchElementException:
194+
elems = get_elements_with_properties(driver, elem_properties, children)
195+
assert len(elems) == 1
196+
elem = elems[0]
197+
xpath = driver.execute_script(get_xpath_script, elem)
198+
else:
199+
elems = get_elements_with_properties(driver, elem_properties, children)
200+
assert len(elems) == 1
201+
elem = elems[0]
202+
xpath = driver.execute_script(get_xpath_script, elem)
203+
204+
if elem is None:
205+
return None
206+
207+
driver.execute_script('arguments[0].scrollIntoView();', elem)
208+
209+
if elem.tag_name in ['button', 'a']:
210+
elem.click()
211+
elif elem.tag_name == 'input':
212+
input_type = elem.get_attribute('type')
213+
if input_type == 'url':
214+
elem.send_keys('http://www.mozilla.org/')
215+
elif input_type == 'text':
216+
elem.send_keys('marco')
217+
elif input_type == 'email':
218+
elem.send_keys('[email protected]')
219+
elif input_type == 'password':
220+
elem.send_keys('aMildlyComplexPasswordIn2017')
221+
elif input_type == 'checkbox':
222+
elem.click()
223+
elif input_type == 'number':
224+
elem.send_keys('3')
225+
elif input_type == 'radio':
226+
elem.click()
227+
elif input_type == 'tel':
228+
elem.send_keys('1234567890')
229+
elif input_type == 'date':
230+
elem.send_keys('20000101')
231+
elif input_type == 'search':
232+
elem.clear()
233+
elem.send_keys('quick search')
234+
elif input_type in ['submit', 'reset', 'button']:
235+
elem.click()
236+
elif input_type == 'color':
237+
driver.execute_script("arguments[0].value = '#ff0000'", elem)
238+
else:
239+
raise Exception('Unsupported input type: %s' % input_type)
240+
elif elem.tag_name == 'select':
241+
for option in elem.find_elements_by_tag_name('option'):
242+
if option.text != '':
243+
option.click()
244+
break
245+
246+
already_clicked_elems.add(elem)
247+
248+
close_all_windows_except_first(driver)
249+
250+
return elem_properties, xpath
251+
252+
except(ElementNotInteractableException, StaleElementReferenceException, InvalidSelectorException, WebDriverException):
253+
traceback.print_exc()
254+
not_clickable_elems.add(elem)
255+
close_all_windows_except_first(driver)
234256

235257

236258
def screenshot(driver, bug_id, browser, seq_no):
@@ -421,6 +443,7 @@ def run_tests(firefox_driver, chrome_driver, bugs):
421443
def main(bugs):
422444
firefox_driver = webdriver.Firefox(firefox_profile=firefox_profile, firefox_binary=nightly_bin)
423445
chrome_driver = webdriver.Chrome(chrome_options=chrome_options)
446+
already_clicked_elems.clear()
424447
run_tests(firefox_driver, chrome_driver, bugs)
425448

426449

0 commit comments

Comments
 (0)