diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5d18a4b..45abf0f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -60,7 +60,7 @@ jobs: - name: Install python test dependencies run: | python --version - python -m pip install mock robotframework opencv-python eel . + python -m pip install mock robotframework opencv-python scikit-image eel . - name: Run tests run: | diff --git a/.gitignore b/.gitignore index df4c17f..b5bd1bf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,5 @@ log.html /bin /include /lib +*egg-info +test_* diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..4351a7e --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.8.7 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..1446bf0 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,41 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Aktuelle Datei", + "type": "python", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/src/ImageHorizonLibrary", + "justMyCode": false + }, + { + "name": "libdoc", + "type": "python", + "request": "launch", + "module": "robot.libdoc", + "args": [ + "-P", + "src", + "ImageHorizonLibrary", + "doc\\ImageHorizonLibrary.html" + ], + "console": "integratedTerminal", + "justMyCode": false + }, + { + "type": "robotframework-lsp", + "name": "Robot Framework: Launch .robot file", + "request": "launch", + "cwd": "${workspaceFolder}", + "target": "${file}", + "terminal": "integrated", + "env": {}, + "args": [] + } + ] +} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7374028 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "python.testing.unittestArgs": [ + "-v", + "-s", + "./tests/utest", + "-p", + "test_*.py" + ], + "python.testing.pytestEnabled": false, + "python.testing.unittestEnabled": true +} \ No newline at end of file diff --git a/README.rst b/README.rst index 6ee54fe..3917f87 100644 --- a/README.rst +++ b/README.rst @@ -3,11 +3,14 @@ ImageHorizonLibrary =================== This Robot Framework library provides the facilities to automate GUIs based on -image recognition similar to Sikuli. This library wraps pyautogui_ to achieve -this. +image recognition similar to Sikuli, but without any Java dependeny (100% Python). -For non pixel perfect matches, there is a feature called `confidence level` -that comes with a dependency OpenCV (python package: `opencv-python`). +There are two different recognition strategies: *default* (using pyautogui_) and *edge* (using skimage_). + +For non pixel perfect matches, there is a feature called "confidence level" that +allows to define the percentage of pixels which *must* match. + +In the *default* strategy, confidence comes with a dependency to OpenCV (python package: `opencv-python`). This functionality is optional - you are not required to install `opencv-python` package if you do not use confidence level. @@ -35,7 +38,6 @@ Prerequisites - `Python 3.x` - pip_ for easy installation -- pyautogui_ and `it's prerequisites`_ - `Robot Framework`_ On Ubuntu, you need to take `special measures`_ to make the screenshot @@ -149,6 +151,7 @@ To regenerate documentation (`doc/ImageHorizonLibrary.html`), use this command: .. _Python 3.x: http://python.org .. _pip: https://pypi.python.org/pypi/pip .. _pyautogui: https://github.com/asweigart/pyautogui +.. _skimage: https://scikit-image.org/docs/dev/auto_examples/features_detection/plot_template.html .. _it's prerequisites: https://pyautogui.readthedocs.org/en/latest/install.html .. _Robot Framework: http://robotframework.org .. _double all coordinates: https://github.com/asweigart/pyautogui/issues/33 diff --git a/_requirements.txt b/_requirements.txt new file mode 100644 index 0000000..34ad71f --- /dev/null +++ b/_requirements.txt @@ -0,0 +1,8 @@ +robotframework +mock +pyautogui +tk +opencv-python +scikit-image +# pyqt5 is only needed to use skimage.viewer.ImageViewer(refImage) while debugging +# pyqt5 \ No newline at end of file diff --git a/build/lib/ImageHorizonLibrary/__init__.py b/build/lib/ImageHorizonLibrary/__init__.py new file mode 100644 index 0000000..7d84d2a --- /dev/null +++ b/build/lib/ImageHorizonLibrary/__init__.py @@ -0,0 +1,408 @@ +# -*- coding: utf-8 -*- +from collections import OrderedDict +from contextlib import contextmanager +import inspect + +from .errors import * # import errors before checking dependencies! + +try: + import pyautogui as ag +except ImportError: + raise ImageHorizonLibraryError('There is something wrong with pyautogui or ' + 'it is not installed.') + +try: + from robot.api import logger as LOGGER + from robot.libraries.BuiltIn import BuiltIn +except ImportError: + raise ImageHorizonLibraryError('There is something wrong with ' + 'Robot Framework or it is not installed.') + +try: + from tkinter import Tk as TK +except ImportError: + raise ImageHorizonLibraryError('There is either something wrong with ' + 'Tkinter or you are running this on Java, ' + 'which is not a supported platform. Please ' + 'use Python and verify that Tkinter works.') + +try: + import skimage as sk +except ImportError: + raise ImageHorizonLibraryError('There is either something wrong with skimage ' + '(scikit-image) or it is not installed.') + +from . import utils +from .interaction import * +from .recognition import * +from .version import VERSION + +__version__ = VERSION + +class ImageHorizonLibrary(_Keyboard, _Mouse, _OperatingSystem, _RecognizeImages, _Screenshot): + '''A cross-platform Robot Framework library for GUI automation. + + *Key features*: + - Automates *keyboard and mouse actions* on the screen (based on [https://pyautogui.readthedocs.org|pyautogui]). + - The regions to execute these actions on (buttons, sliders, input fields etc.) are determined by `reference images` which the library detects on the screen - independently of the OS or the application type. + - Two different image `recognition strategies`: `default` (fast and reliable of predictable screen content), and `edge` (to facilitate the recognition of unpredictable pixel deviations) + - The library can also take screenshots in case of failure or by intention. + + = Image Recognition = + + == Reference images == + ``reference_image`` parameter can be either a single file or a folder. + If ``reference_image`` is a folder, image recognition is tried separately + for each image in that folder, in alphabetical order until a match is found. + + For ease of use, reference image names are automatically normalized + according to the following rules: + + - The name is lower cased: ``MYPICTURE`` and ``mYPiCtUrE`` become + ``mypicture`` + + - All spaces are converted to underscore ``_``: ``my picture`` becomes + ``my_picture`` + + - If the image name does not end in ``.png``, it will be added: + ``mypicture`` becomes ``mypicture.png`` + + - Path to _reference folder_ is prepended. This option must be given when + `importing` the library. + + Using good names for reference images is evident from easy-to-read test + data: + + | `Import Library` | ImageHorizonLibrary | reference_folder=images | | + | `Click Image` | popup Window title | | # Path is images/popup_window_title.png | + | `Click Image` | button Login Without User Credentials | | # Path is images/button_login_without_user_credentials.png | + + == Recognition strategies == + Basically, image recognition works by searching a reference image on the + another image (a screnshot of the current desktop). + If there is a region with 100% matching pixels of the reference image, this + area represents a match. + + By default, the reference image must be an exakt sub-image of the screenshot. + This works flawlessly in most cases. + + But problems can arise when: + + - the application's GUI uses transpareny effects + - the screen resolution/the window size changes + - font aliasing is used for dynamic text + - compression algorithms in RDP/Citrix cause invisible artifacts + - ...and in many more situations. + + In those situations, a certain amount of the pixels do not match. + + To solve this, ImageHorizon comes with a parameter ``confidence level``. This is a decimal value + between 0 and 1 (inclusive) and defines how many percent of the reference image pixels + must match the found region's imag. It is set to 1.0 = 100% by default. + + Confidence level can be set during `library importing` and re-adjusted during the test + case with the keyword `Set Confidence`. + + === Default image detection strategy === + + If imported without any strategy argument, the library uses [https://pyautogui.readthedocs.org|pyautogui] + under the hood to recognize images on the screen. + This is the perfect choice to start writing tests. + + To use `confidence level in mode` ``default`` the + [https://pypi.org/project/opencv-python|opencv-python] Python package + must be installed separately: + + | $ pip install opencv-python + + After installation, the library will automatically use OpenCV for confidence + levels lower than 1.0. + + === The "edge" image detection strategy === + + The default image recognition reaches its limitations when the area to + match contains a *disproportionate amount of unpredictable pixels*. + + The idea for this strategy came from a problem in real life: a web application + showing a topographical map (loaded from a 3rd party provider), with a layer of + interstate highways as black lines. For some reasons, the pixels of topographic + areas between the highway lines (which are the vast majority) showed a slight + deviation in brightness - invisible for the naked eye, but enough to make the test failing. + + The abstract and simplified example for this is a horizontal black line of 1px width in a + matrix of 10x10 white pixels. To neglect a (slight) brightness deviation of the white pixels, + you would need a confidence level of 0.1 which allows 90% of the pixels to be + different. This is insanse and leads to inpredictable results. + + That's why ``edge`` was implemented as an alternative recognition strategy. + The key here lies in the approach to *reduce both images* (reference and screenshot + image) *to the essential characteristics* and then *compare _those_ images*. + + "Essential characteristics" of an image are those areas where neighbouring pixels show a + sharp change of brightness, better known as "edges". [https://en.wikipedia.org/wiki/Edge_detection|Edge detection] + is the process of finding the edges in an image, done by [https://scikit-image.org/|scikit-image] in this library. + + As a brief digression, edge detection is a multi-step process: + + - apply a [https://en.wikipedia.org/wiki/Gaussian_filter|Gaussian filter] (blurs the image to remove noise; intensity set by parameter `sigma`) + - apply a [https://en.wikipedia.org/wiki/Sobel_operator|Sobel filter] (remove non-max pixels, get a 1 pixel edge curve) + - separate weak edges from strong ones with [https://en.wikipedia.org/wiki/Canny_edge_detector#Edge_tracking_by_hysteresis|hysteresis] + - apply the `template_matching` routine to get a [https://en.wikipedia.org/wiki/Cross-correlation|cross correlation] matrix of values from -1 (no correlation) to +1 (perfect correlation). + - Filter out only those coordinates with values greater than the ``confidence`` level, take the max + + The keyword `Debug Image` opens a debugger UI where confidence level, Gaussian sigma and low/high thresholds can be tested and adjusted to individual needs. + + Edge detection costs some extra CPU time; you should always first try + to use the ``default`` strategy and only selectively switch to ``edge`` + when a confidence level below 0.9 is not sufficient to detect images reliably anymore: + + | # use with defaults + | Set Strategy edge + | # use with custom parameters + | Set Strategy edge edge_sigma=2.0 edge_low_threshold=0.1 edge_high_threshold=0.3 confidence=0.8 + + To use strategy ``edge``, the [https://scikit-image.org|scikit-image] Python package must be installed separately: + + | $ pip install scikit-image + + = Performance = + + Locating images on screen, especially if screen resolution is large and + reference image is also large, might take considerable time, regardless + of the strategy. + It is therefore advisable to save the returned coordinates if you are + manipulating the same context many times in the row: + + | `Wait For` | label Name | | + | `Click To The Left Of Image` | label Name | 200 | + + In the above example, same image is located twice. Below is an example how + we can leverage the returned location: + + | ${location}= | `Wait For` | label Name | + | `Click To The Left Of` | ${location} | 200 | + ''' + + ROBOT_LIBRARY_SCOPE = 'TEST SUITE' + ROBOT_LIBRARY_VERSION = VERSION + + def __init__(self, reference_folder=None, screenshot_folder=None, + keyword_on_failure='ImageHorizonLibrary.Take A Screenshot', + confidence=None, strategy='default', + edge_sigma=2.0, edge_low_threshold=0.1, edge_high_threshold=0.3): + '''ImageHorizonLibrary can be imported with several options. + + ``reference_folder`` is path to the folder where all reference images + are stored. It must be a _valid absolute path_. As the library + is suite-specific (ie. new instance is created for every suite), + different suites can have different folders for it's reference images. + + ``screenshot_folder`` is path to the folder where screenshots are + saved. If not given, screenshots are saved to the current working + directory. + + ``keyword_on_failure`` is the keyword to be run, when location-related + keywords fail. If you wish to not take screenshots, use for example + `BuiltIn.No Operation`. Keyword must however be a valid keyword. + + ``strategy`` sets the way how images are detected on the screen. See also + keyword `Set Strategy` to change the strategy during the test. Parameters: + - ``default`` - (Default) + - ``edge`` - Advanced image recognition options with canny edge detection + + The ``edge`` strategy allows these additional parameters: + - ``edge_sigma`` - Gaussian blur intensity + - ``edge_low_threshold`` - low pixel gradient threshold + - ``edge_high_threshold`` - high pixel gradient threshold + ''' + + # _RecognizeImages.set_strategy(self, strategy) + self.reference_folder = reference_folder + self.screenshot_folder = screenshot_folder + self.keyword_on_failure = keyword_on_failure + self.open_applications = OrderedDict() + self.screenshot_counter = 1 + self.is_windows = utils.is_windows() + self.is_mac = utils.is_mac() + self.is_linux = utils.is_linux() + self.has_retina = utils.has_retina() + self.has_cv = utils.has_cv() + self.has_skimage = utils.has_skimage() + self.confidence = confidence + self.initial_confidence = confidence + self._class_bases = inspect.getmro(self.__class__) + self.set_strategy(strategy, self.confidence) + self.edge_sigma = edge_sigma + self.edge_low_threshold = edge_low_threshold + self.edge_high_threshold = edge_high_threshold + + + + def set_strategy(self, strategy, edge_sigma=2.0, edge_low_threshold=0.1, edge_high_threshold=0.3, confidence=None): + '''Changes the way how images are detected on the screen. This can also be done globally during `Importing`. + Strategies: + - ``default`` + - ``edge`` - Advanced image recognition options with canny edge detection + + The ``edge`` strategy allows these additional parameters: + - ``edge_sigma`` - Gaussian blur intensity + - ``edge_low_threshold`` - low pixel gradient threshold + - ``edge_high_threshold`` - high pixel gradient threshold + + Both strategies can optionally be initialized with a new confidence.''' + + self.strategy = strategy + if strategy == 'default': + self.strategy_instance = _StrategyPyautogui(self) + elif strategy == 'edge': + self.strategy_instance = _StrategySkimage(self) + self.edge_sigma = edge_sigma + self.edge_low_threshold = edge_low_threshold + self.edge_high_threshold = edge_high_threshold + else: + raise StrategyException('Invalid strategy: "%s"' % strategy) + + if not confidence is None: + self.set_confidence(confidence) + + # Linking protected _try_locate to the strategy's method + self._try_locate = self.strategy_instance._try_locate + + def _get_location(self, direction, location, offset): + x, y = location + offset = int(offset) + if direction == 'left': + x = x - offset + if direction == 'up': + y = y - offset + if direction == 'right': + x = x + offset + if direction == 'down': + y = y + offset + return x, y + + def _click_to_the_direction_of(self, direction, location, offset, + clicks, button, interval): + x, y = self._get_location(direction, location, offset) + try: + clicks = int(clicks) + except ValueError: + raise MouseException('Invalid argument "%s" for `clicks`') + if button not in ['left', 'middle', 'right']: + raise MouseException('Invalid button "%s" for `button`') + try: + interval = float(interval) + except ValueError: + raise MouseException('Invalid argument "%s" for `interval`') + + LOGGER.info('Clicking %d time(s) at (%d, %d) with ' + '%s mouse button at interval %f' % (clicks, x, y, + button, interval)) + ag.click(x, y, clicks=clicks, button=button, interval=interval) + + def _convert_to_valid_special_key(self, key): + key = str(key).lower() + if key.startswith('key.'): + key = key.split('key.', 1)[1] + elif len(key) > 1: + return None + if key in ag.KEYBOARD_KEYS: + return key + return None + + def _validate_keys(self, keys): + valid_keys = [] + for key in keys: + valid_key = self._convert_to_valid_special_key(key) + if not valid_key: + raise KeyboardException('Invalid keyboard key "%s", valid ' + 'keyboard keys are:\n%r' % + (key, ', '.join(ag.KEYBOARD_KEYS))) + valid_keys.append(valid_key) + return valid_keys + + def _press(self, *keys, **options): + keys = self._validate_keys(keys) + ag.hotkey(*keys, **options) + + @contextmanager + def _tk(self): + tk = TK() + yield tk.clipboard_get() + tk.destroy() + + def copy(self): + '''Executes ``Ctrl+C`` on Windows and Linux, ``⌘+C`` on OS X and + returns the content of the clipboard.''' + key = 'Key.command' if self.is_mac else 'Key.ctrl' + self._press(key, 'c') + return self.get_clipboard_content() + + def get_clipboard_content(self): + '''Returns what is currently copied in the system clipboard.''' + with self._tk() as clipboard_content: + return clipboard_content + + def pause(self): + '''Shows a dialog that must be dismissed with manually clicking. + + This is mainly for when you are developing the test case and want to + stop the test execution. + + It should probably not be used otherwise. + ''' + ag.alert(text='Test execution paused.', title='Pause', + button='Continue') + + def _run_on_failure(self): + if not self.keyword_on_failure: + return + try: + BuiltIn().run_keyword(self.keyword_on_failure) + except Exception as e: + LOGGER.debug(e) + LOGGER.warn('Failed to take a screenshot. ' + 'Is Robot Framework running?') + + def set_reference_folder(self, reference_folder_path): + '''Sets where all reference images are stored. + + See `library importing` for format of the reference folder path. + ''' + self.reference_folder = reference_folder_path + + def set_screenshot_folder(self, screenshot_folder_path): + '''Sets the folder where screenshots are saved to. + + See `library importing` for more specific information. + ''' + self.screenshot_folder = screenshot_folder_path + + def reset_confidence(self): + '''Resets the confidence level to the library default. + If no confidence was given during import, this is None.''' + LOGGER.info('Resetting confidence level to {}.'.format(self.initial_confidence)) + self.confidence = self.initial_confidence + + def set_confidence(self, new_confidence): + '''Sets the accuracy when finding images. + + ``new_confidence`` is a decimal number between 0 and 1 inclusive. + + See `Confidence level` about additional dependencies that needs to be + installed before this keyword has any effect. + ''' + if new_confidence is not None: + try: + new_confidence = float(new_confidence) + if not 1 >= new_confidence >= 0: + LOGGER.warn('Unable to set confidence to {}. Value ' + 'must be between 0 and 1, inclusive.' + .format(new_confidence)) + else: + self.confidence = new_confidence + except TypeError as err: + LOGGER.warn("Can't set confidence to {}".format(new_confidence)) + else: + self.confidence = None + diff --git a/build/lib/ImageHorizonLibrary/errors.py b/build/lib/ImageHorizonLibrary/errors.py new file mode 100644 index 0000000..ea318da --- /dev/null +++ b/build/lib/ImageHorizonLibrary/errors.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +class ImageHorizonLibraryError(ImportError): + pass + + +class ImageNotFoundException(Exception): + def __init__(self, image_name): + self.image_name = image_name + + def __str__(self): + return 'Reference image "%s" was not found on screen' % self.image_name + + +class InvalidImageException(Exception): + pass + + +class KeyboardException(Exception): + pass + + +class MouseException(Exception): + pass + + +class OSException(Exception): + pass + + +class ReferenceFolderException(Exception): + pass + + +class ScreenshotFolderException(Exception): + pass + +class StrategyException(Exception): + pass \ No newline at end of file diff --git a/build/lib/ImageHorizonLibrary/interaction/__init__.py b/build/lib/ImageHorizonLibrary/interaction/__init__.py new file mode 100644 index 0000000..ddac8ea --- /dev/null +++ b/build/lib/ImageHorizonLibrary/interaction/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from ._keyboard import _Keyboard +from ._mouse import _Mouse +from ._operating_system import _OperatingSystem + +__all__ = [ + '_Keyboard', + '_Mouse', + '_OperatingSystem' +] diff --git a/build/lib/ImageHorizonLibrary/interaction/_keyboard.py b/build/lib/ImageHorizonLibrary/interaction/_keyboard.py new file mode 100644 index 0000000..5ec3db7 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/interaction/_keyboard.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +import pyautogui as ag + + +class _Keyboard(object): + def press_combination(self, *keys): + '''Press given keyboard keys. + + All keyboard keys must be prefixed with ``Key.``. + + Keyboard keys are case-insensitive: + + | Press Combination | KEY.ALT | key.f4 |  + | Press Combination | kEy.EnD | | + + [https://pyautogui.readthedocs.org/en/latest/keyboard.html#keyboard-keys| + See valid keyboard keys here]. + ''' + self._press(*keys) + + def type(self, *keys_or_text): + '''Type text and keyboard keys. + + See valid keyboard keys in `Press Combination`. + + Examples: + + | Type | separated | Key.ENTER | by linebreak | + | Type | Submit this with enter | Key.enter | | + | Type | key.windows | notepad | Key.enter | + ''' + for key_or_text in keys_or_text: + key = self._convert_to_valid_special_key(key_or_text) + if key: + ag.press(key) + else: + ag.typewrite(key_or_text) + + + def type_with_keys_down(self, text, *keys): + '''Press keyboard keys down, then write given text, then release the + keyboard keys. + + See valid keyboard keys in `Press Combination`. + + Examples: + + | Type with keys down | write this in caps | Key.Shift | + ''' + valid_keys = self._validate_keys(keys) + for key in valid_keys: + ag.keyDown(key) + ag.typewrite(text) + for key in valid_keys: + ag.keyUp(key) diff --git a/build/lib/ImageHorizonLibrary/interaction/_mouse.py b/build/lib/ImageHorizonLibrary/interaction/_mouse.py new file mode 100644 index 0000000..5e0c93c --- /dev/null +++ b/build/lib/ImageHorizonLibrary/interaction/_mouse.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +import pyautogui as ag + +from ..errors import MouseException + + +class _Mouse(object): + + def _click_to_the_direction_of(self, direction, location, offset, + clicks, button, interval): + raise NotImplementedError('This is defined in the main class.') + + def click_to_the_above_of(self, location, offset, clicks=1, + button='left', interval=0.0): + '''Clicks above of given location by given offset. + + ``location`` can be any Python sequence type (tuple, list, etc.) that + represents coordinates on the screen ie. have an x-value and y-value. + Locating-related keywords return location you can use with this + keyword. + + ``offset`` is the number of pixels from the specified ``location``. + + ``clicks`` is how many times the mouse button is clicked. + + See `Click` for documentation for valid buttons. + + Example: + + | ${image location}= | Locate | my image | | + | Click To The Above Of | ${image location} | 50 | | + | @{coordinates}= | Create List | ${600} | ${500} | + | Click To The Above Of | ${coordinates} | 100 | | + ''' + self._click_to_the_direction_of('up', location, offset, + clicks, button, interval) + + def click_to_the_below_of(self, location, offset, clicks=1, + button='left', interval=0.0): + '''Clicks below of given location by given offset. + + See argument documentation in `Click To The Above Of`. + ''' + self._click_to_the_direction_of('down', location, offset, + clicks, button, interval) + + def click_to_the_left_of(self, location, offset, clicks=1, + button='left', interval=0.0): + '''Clicks left of given location by given offset. + + See argument documentation in `Click To The Above Of`. + ''' + self._click_to_the_direction_of('left', location, offset, + clicks, button, interval) + + def click_to_the_right_of(self, location, offset, clicks=1, + button='left', interval=0.0): + '''Clicks right of given location by given offset. + + See argument documentation in `Click To The Above Of`. + ''' + self._click_to_the_direction_of('right', location, offset, + clicks, button, interval) + + def move_to(self, *coordinates): + '''Moves the mouse pointer to an absolute coordinates. + + ``coordinates`` can either be a Python sequence type with two values + (eg. ``(x, y)``) or separate values ``x`` and ``y``: + + | Move To | 25 | 150 | | + | @{coordinates}= | Create List | 25 | 150 | + | Move To | ${coordinates} | | | + | ${coords}= | Evaluate | (25, 150) | | + | Move To | ${coords} | | | + + + X grows from left to right and Y grows from top to bottom, which means + that top left corner of the screen is (0, 0) + ''' + if len(coordinates) > 2 or (len(coordinates) == 1 and + type(coordinates[0]) not in (list, tuple)): + raise MouseException('Invalid number of coordinates. Please give ' + 'either (x, y) or x, y.') + if len(coordinates) == 2: + coordinates = (coordinates[0], coordinates[1]) + else: + coordinates = coordinates[0] + try: + coordinates = [int(coord) for coord in coordinates] + except ValueError: + raise MouseException('Coordinates %s are not integers' % + (coordinates,)) + ag.moveTo(*coordinates) + + def mouse_down(self, button='left'): + '''Presses specidied mouse button down''' + ag.mouseDown(button=button) + + def mouse_up(self, button='left'): + '''Releases specified mouse button''' + ag.mouseUp(button=button) + + def click(self, button='left'): + '''Clicks with the specified mouse button. + + Valid buttons are ``left``, ``right`` or ``middle``. + ''' + ag.click(button=button) + + def double_click(self, button='left', interval=0.0): + '''Double clicks with the specified mouse button. + + See documentation of ``button`` in `Click`. + + ``interval`` specifies the time between clicks and should be + floating point number. + ''' + ag.doubleClick(button=button, interval=float(interval)) + + def triple_click(self, button='left', interval=0.0): + '''Triple clicks with the specified mouse button. + + See documentation of ``button`` in `Click`. + + See documentation of ``interval`` in `Double Click`. + ''' + ag.tripleClick(button=button, interval=float(interval)) diff --git a/build/lib/ImageHorizonLibrary/interaction/_operating_system.py b/build/lib/ImageHorizonLibrary/interaction/_operating_system.py new file mode 100644 index 0000000..37c7307 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/interaction/_operating_system.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +import shlex +import subprocess + +from ..errors import OSException + + +class _OperatingSystem(object): + + def launch_application(self, app, alias=None): + '''Launches an application. + + Executes the string argument ``app`` as a separate process with + Python's + ``[https://docs.python.org/2/library/subprocess.html|subprocess]`` + module. It should therefore be the exact command you would use to + launch the application from command line. + + On Windows, if you are using relative or absolute paths in ``app``, + enclose the command with double quotes: + + | Launch Application | "C:\\my folder\\myprogram.exe" | # Needs quotes | + | Launch Application | myprogram.exe | # No need for quotes | + + Returns automatically generated alias which can be used with `Terminate + Application`. + + Automatically generated alias can be overridden by providing ``alias`` + yourself. + ''' + if not alias: + alias = str(len(self.open_applications)) + process = subprocess.Popen(shlex.split(app)) + self.open_applications[alias] = process + return alias + + def terminate_application(self, alias=None): + '''Terminates the process launched with `Launch Application` with + given ``alias``. + + If no ``alias`` is given, terminates the last process that was + launched. + ''' + if alias and alias not in self.open_applications: + raise OSException('Invalid alias "%s".' % alias) + process = self.open_applications.pop(alias, None) + if not process: + try: + _, process = self.open_applications.popitem() + except KeyError: + raise OSException('`Terminate Application` called without ' + '`Launch Application` called first.') + process.terminate() diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py new file mode 100644 index 0000000..b38eb0f --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py @@ -0,0 +1,9 @@ +from tkinter import * +from .image_debugger_controller import UILocatorController + + +class ImageDebugger: + + def __init__(self, image_horizon_instance): + app = UILocatorController(image_horizon_instance) + app.main() diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py new file mode 100644 index 0000000..a174512 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py @@ -0,0 +1,172 @@ +from .image_debugger_model import UILocatorModel +from .image_debugger_view import UILocatorView +from .template_matching_strategies import Pyautogui, Skimage +from .image_manipulation import ImageContainer, ImageFormat +from pathlib import Path +import os, glob +import pyperclip +import numpy as np +import webbrowser + + +class UILocatorController: + + def __init__(self, image_horizon_instance): + self.image_container = ImageContainer() + self.model = UILocatorModel() + self.image_horizon_instance = image_horizon_instance + self.view = UILocatorView(self, self.image_container, self.image_horizon_instance) + + def main(self): + self.init_view() + self.view.main() + + def init_view(self): + self.view.ref_dir_path.set(self.image_horizon_instance.reference_folder) + self.view.scale_conf_lvl_ag.set(0.99) + self.view.scale_sigma_skimage.set(1.0) + self.view.scale_low_thres_skimage.set(0.1) + self.view.scale_high_thres_skimage.set(0.3) + self.view.scale_conf_lvl_skimage.set(0.99) + self.view.matches_found.set("None") + self.view.btn_edge_detec_debugger["state"] = "disabled" + self.view.btn_run_pyautogui["state"] = "disabled" + self.view.btn_run_skimage["state"] = "disabled" + self.view.btn_copy_strategy_snippet["state"] = "disabled" + self.view.hint_msg.set("Ready") + self.view.processing_done = False + + def help(self): + webbrowser.open("https://eficode.github.io/robotframework-imagehorizonlibrary/doc/ImageHorizonLibrary.html") + + def load_needle_image_names(self, combobox=None): + os.chdir(self.image_horizon_instance.reference_folder) + list_needle_images_names = [] + for needle_img_name in glob.glob("*.png"): + list_needle_images_names.append(needle_img_name) + if combobox: + self.combobox = combobox + self.combobox['values']=list_needle_images_names + self.combobox.set('__ __ __ Select a reference image __ __ __') + + def reset_images(self): + self.view.canvas_ref_img.itemconfig( + self.view.ref_img, + image=self.view.needle_img_blank, + ) + self.view.canvas_desktop_img.itemconfig( + self.view.desktop_img, + image=self.view.haystack_img_blank, + ) + + def reset_results(self): + self.view.btn_copy_strategy_snippet["state"] = "disabled" + self.view.btn_edge_detec_debugger["state"] = "disabled" + self.view.matches_found.set("None") + self.view.label_matches_found.config(fg='black') + self.view.set_strategy_snippet.set("") + + def refresh(self): + self.load_needle_image_names() + self.view.btn_run_pyautogui["state"] = "disabled" + self.view.btn_run_skimage["state"] = "disabled" + self.reset_results() + self.reset_images() + self.view._ready() + self.view.processing_done=False + + def copy_to_clipboard(self): + pyperclip.copy(self.strategy_snippet) + + def on_select(self, event): + ref_image = Path(self.view.combobox_needle_img_name.get()) + self.image_container.save_to_img_container(img=ref_image.__str__()) + self.needle_img = self.image_container.get_needle_image(ImageFormat.IMAGETK) + self.view.canvas_ref_img.itemconfig( + self.view.ref_img, + image=self.needle_img, + ) + + self.view.btn_run_pyautogui["state"] = "normal" + self.view.btn_run_skimage["state"] = "normal" + + def _take_screenshot(self): + # Minimize the GUI Debugger window before taking the screenshot + self.view.wm_state('iconic') + screenshot = self.model.capture_desktop() + self.view.wm_state('normal') + return screenshot + + def on_click_run_default_strategy(self): + self.image_horizon_instance.set_strategy('default') + self.image_horizon_instance.confidence = float(self.view.scale_conf_lvl_ag.get()) + self.image_container.save_to_img_container(self._take_screenshot(), is_haystack_img=True) + + matcher = Pyautogui(self.image_container, self.image_horizon_instance) + self.coord = matcher.find_num_of_matches() + matcher.highlight_matches() + + self.haystack_image = self.image_container.get_haystack_image(format=ImageFormat.IMAGETK) + self.view.canvas_desktop_img.itemconfig(self.view.desktop_img, image=self.haystack_image) + + num_of_matches_found = len(self.coord) + self.view.matches_found.set(num_of_matches_found) + font_color = self.model.change_color_of_label(num_of_matches_found) + self.view.label_matches_found.config(fg=font_color) + + self.strategy_snippet = f"Set Strategy default confidence={self.image_horizon_instance.confidence}" + self.view.set_strategy_snippet.set(self.strategy_snippet) + self.view.btn_copy_strategy_snippet["state"] = "normal" + self.view.processing_done = True + + def on_click_run_edge_detec_strategy(self): + self.image_horizon_instance.set_strategy('edge') + self.image_horizon_instance.edge_low_threshold = float(self.view.scale_low_thres_skimage.get()) + self.image_horizon_instance.edge_high_threshold = float(self.view.scale_high_thres_skimage.get()) + if self.image_horizon_instance.edge_high_threshold < self.image_horizon_instance.edge_low_threshold: + self.reset_results() + self.reset_images() + self.view._threshold_error() + self.view.processing_done=False + return + + self.image_horizon_instance.confidence = float(self.view.scale_conf_lvl_skimage.get()) + self.image_horizon_instance.edge_sigma = float(self.view.scale_sigma_skimage.get()) + self.image_container._haystack_image_orig_size = self._take_screenshot() + + + matcher = Skimage(self.image_container, self.image_horizon_instance) + self.coord = matcher.find_num_of_matches() + matcher.highlight_matches() + + self.haystack_image = self.image_container.get_haystack_image(format=ImageFormat.IMAGETK) + self.view.canvas_desktop_img.itemconfig(self.view.desktop_img, image=self.haystack_image) + + num_of_matches_found = len(self.coord) + max_peak = round(np.amax(self.image_horizon_instance.peakmap), 2) + if max_peak < 0.75: + result_msg = f"{num_of_matches_found} / max peak value below 0.75" + else: + result_msg = f"{num_of_matches_found} / {max_peak}" + + self.view.matches_found.set(result_msg) + font_color = self.model.change_color_of_label(num_of_matches_found) + self.view.label_matches_found.config(fg=font_color) + self.view.btn_edge_detec_debugger["state"] = "normal" + + self.strategy_snippet = f"Set Strategy edge edge_sigma={self.image_horizon_instance.edge_sigma} edge_low_threshold={self.image_horizon_instance.edge_low_threshold} edge_high_threshold={self.image_horizon_instance.edge_high_threshold} confidence={self.image_horizon_instance.confidence}" + self.view.set_strategy_snippet.set(self.strategy_snippet) + self.view.btn_copy_strategy_snippet["state"] = "normal" + self.view.processing_done = True + + def on_click_plot_results_skimage(self): + title = f"{self.view.matches_found.get()} matches (confidence: {self.image_horizon_instance.confidence})" + self.model.plot_result( + self.image_container.get_needle_image(ImageFormat.NUMPYARRAY), + self.image_container.get_haystack_image_orig_size(ImageFormat.NUMPYARRAY), + self.image_horizon_instance.needle_edge, + self.image_horizon_instance.haystack_edge, + self.image_horizon_instance.peakmap, + title, + self.coord + ) diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py new file mode 100644 index 0000000..0887553 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py @@ -0,0 +1,53 @@ +import pyautogui as ag +import matplotlib.pyplot as mplt + + +class UILocatorModel(): + def capture_desktop(self): + return ag.screenshot() + + def change_color_of_label(self, num_of_matches_found) -> str: + if(num_of_matches_found == 1): + return 'green' + else: + return 'red' + + def plot_result(self, needle_img, haystack_img, needle_img_edges, haystack_img_edges, peakmap, title, coord): + fig, axs = mplt.subplots(2, 3) + sp_haystack_img = axs[0, 0] + sp_needle_img = axs[0, 1] + sp_invisible = axs[0, 2] + sp_haystack_img_edges = axs[1, 0] + sp_needle_img_edges = axs[1, 1] + sp_peakmap = axs[1, 2] + + # ROW 1 ================================ + sp_needle_img.set_title('Needle') + sp_needle_img.imshow(needle_img, cmap=mplt.cm.gray) + sp_haystack_img.set_title('Haystack') + sp_haystack_img.imshow(haystack_img, cmap=mplt.cm.gray) + sp_haystack_img.sharex(sp_haystack_img_edges) + sp_haystack_img.sharey(sp_haystack_img_edges) + + sp_invisible.set_visible(False) + + # ROW 2 ================================ + sp_needle_img_edges.set_title('Needle (edge d.)') + sp_needle_img_edges.imshow(needle_img_edges, cmap=mplt.cm.gray) + + sp_haystack_img_edges.set_title('Haystack (edge d.)') + sp_haystack_img_edges.imshow(haystack_img_edges, cmap=mplt.cm.gray) + + sp_peakmap.set_title('Peakmap') + sp_peakmap.imshow(peakmap) + sp_peakmap.sharex(sp_haystack_img_edges) + sp_peakmap.sharey(sp_haystack_img_edges) + + for loc in coord: + rect = mplt.Rectangle((loc[0], loc[1]), loc[2], loc[3], edgecolor='r', facecolor='none') + sp_haystack_img_edges.add_patch(rect) + + sp_peakmap.autoscale(False) + fig.suptitle(title, fontsize=14, fontweight='bold') + + mplt.show() \ No newline at end of file diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py new file mode 100644 index 0000000..0df4460 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py @@ -0,0 +1,201 @@ +import tkinter as tk +from tkinter import ttk +from tkinter import * +from PIL import ImageTk, Image +from .image_manipulation import ImageFormat + +class UILocatorView(Tk): + def __init__(self, controller, image_container, image_horizon_instance): + super().__init__() + self.title("ImageHorizonLibrary - Debugger") + self.resizable(False, False) + self.controller = controller + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + self.def_save_loc = tk.StringVar() + self.ref_img_loc = tk.StringVar() + self.needle_img_blank = ImageTk.PhotoImage(Image.new('RGB', (30, 30), color='white')) + self.haystack_img_blank = ImageTk.PhotoImage(Image.new('RGB', (384, 216), color='white')) + self._create_view() + + def main(self): + self.mainloop() + + def _create_view(self): + self._menu() + self._frame_main() + self._frame_load_reference_image() + self._frame_computation_params() + self._frame_results() + self._frame_image_viewer() + self._status_bar() + + def _menu(self): + # ***************** Menu ******************* # + menu = Menu(self) + self.config(menu=menu) + menu.add_command(label="Online Help", command=self.controller.help) + + def _frame_main(self): + # *************** Frame Main *************** # + self.frame_main = Frame(self) + self.frame_main.pack(side=TOP, padx=2, pady=2) + + def _frame_load_reference_image(self): + # ************ Frame Select image ************ # + frame_import_ref_image = LabelFrame(self.frame_main, text="Select reference image (.png)", padx=2, pady=2) + frame_import_ref_image.pack(side=TOP, padx=2, pady=2, fill=X) + Label(frame_import_ref_image, text="Reference directory path:").pack(side=TOP, anchor=W) + self.ref_dir_path = StringVar(frame_import_ref_image) + self.label_ref_dir_path = Label(frame_import_ref_image, textvariable=self.ref_dir_path, fg='green') + self.label_ref_dir_path.pack(side=TOP, anchor=W) + + self.combobox_needle_img_name = ttk.Combobox(frame_import_ref_image, width=70, state="readonly") + self.controller.load_needle_image_names(self.combobox_needle_img_name) + self.combobox_needle_img_name.pack(pady=(0,0), side=LEFT, anchor=W) + self.combobox_needle_img_name.bind("<>", self.controller.on_select) + self.btn_copy_strategy_snippet = Button(frame_import_ref_image, text='Refresh', command=self.controller.refresh) + self.btn_copy_strategy_snippet.pack(side=RIGHT, padx=(2, 0), anchor=E) + + def _frame_computation_params(self): + # ************ Frame Computation ************ # + frame_computation = LabelFrame(self.frame_main, text="Computation", padx=2, pady=2) + frame_computation.pack(side=TOP, padx=2, pady=2, fill=X) + + frame_computation.columnconfigure(0, weight=1) + frame_computation.columnconfigure(1, weight=1) + + # ************ Frame default strategy (pyAutogui) ************* # + self.frame_default_strat = LabelFrame(frame_computation, text="Default strategy", padx=2, pady=2) + self.frame_default_strat.grid(row=0, column=0, padx=2, pady=2, sticky="WNS") + + self.frame_default_strat.columnconfigure(0, weight=3) + self.frame_default_strat.columnconfigure(1, weight=1) + + self.frame_default_strat.bind("", self._default_strategy_config_enter) + self.frame_default_strat.bind("", self._clear_statusbar) + + Label(self.frame_default_strat, text="Confidence factor").grid(row=0, column=0, sticky=SW) + self.scale_conf_lvl_ag = Scale(self.frame_default_strat, from_=0.75, to=1.0, resolution=0.01, orient=HORIZONTAL) + self.scale_conf_lvl_ag.grid(padx=(2, 0), row=0, column=1, sticky=E) + + self.btn_run_pyautogui = Button(frame_computation, text='Detect reference image', command=self.controller.on_click_run_default_strategy) + self.btn_run_pyautogui.grid(row=1, column=0, padx=2, sticky=W) + + # ************ Frame edge detection strategy (edge) ************ # + self.frame_edge_detec_strat = LabelFrame(frame_computation, text="Edge detection strategy", padx=2, pady=2) + self.frame_edge_detec_strat.grid(row=0, column=1, padx=2, pady=2, sticky=W) + + self.frame_edge_detec_strat.columnconfigure(0, weight=3) + self.frame_edge_detec_strat.columnconfigure(1, weight=1) + + self.frame_edge_detec_strat.bind("", self._edge_detec_strategy_config_enter) + self.frame_edge_detec_strat.bind("", self._clear_statusbar) + + Label(self.frame_edge_detec_strat, text="Gaussian width (sigma)").grid(row=0, column=0, sticky=SW) + self.scale_sigma_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=5.0, resolution=0.01, orient=HORIZONTAL) + self.scale_sigma_skimage.grid(padx=(2, 0), row=0, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Lower hysteresis threshold").grid(row=1, column=0, sticky=SW) + self.scale_low_thres_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=10.0, resolution=0.01, orient=HORIZONTAL) + self.scale_low_thres_skimage.grid(padx=(2, 0), row=1, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Higher hysteresis threshold").grid(row=2, column=0, sticky=SW) + self.scale_high_thres_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=10.0, resolution=0.01, orient=HORIZONTAL) + self.scale_high_thres_skimage.grid(padx=(2, 0), row=2, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Confidence factor").grid(row=3, column=0, sticky=SW) + self.scale_conf_lvl_skimage = Scale(self.frame_edge_detec_strat, from_=0.75, to=1.0, resolution=0.01, orient=HORIZONTAL) + self.scale_conf_lvl_skimage.grid(padx=(2, 0), row=3, column=1, sticky=E) + + self.btn_run_skimage = Button(frame_computation, text='Detect reference image', command=self.controller.on_click_run_edge_detec_strategy) + self.btn_run_skimage.grid(row=1, column=1, padx=2, sticky=W) + self.btn_edge_detec_debugger = Button(frame_computation, text='Edge detection debugger', command=self.controller.on_click_plot_results_skimage) + self.btn_edge_detec_debugger.grid(row=2, column=1, padx=2, sticky=W) + + def _frame_results(self): + # ************ Frame Results ************ # + frame_results = LabelFrame(self.frame_main, text="Results", padx=2, pady=2) + frame_results.pack(side=TOP, padx=2, pady=2, fill=X) + + frame_results_details = Frame(frame_results) + frame_results_details.pack(side=TOP, fill=X) + + Label(frame_results_details, text="Matches found / (Max peak value):").grid(pady=(2, 0), row=0, column=0, sticky=W) + self.matches_found = StringVar(frame_results_details) + self.label_matches_found = Label(frame_results_details, textvariable=self.matches_found) + self.label_matches_found.grid(pady=(2, 0), row=0, column=1, sticky=W) + + frame_snippet = Frame(frame_results) + frame_snippet.pack(side=TOP, fill=X) + Label(frame_snippet, text="Keyword to use this strategy:").pack(pady=(2, 0), side=TOP, anchor=W) + self.set_strategy_snippet = StringVar(frame_snippet) + self.label_strategy_snippet = Entry(frame_snippet, textvariable=self.set_strategy_snippet, width=75) + self.label_strategy_snippet.pack(pady=(0, 0), side=LEFT, anchor=W) + self.btn_copy_strategy_snippet = Button(frame_snippet, text='Copy', command=self.controller.copy_to_clipboard) + self.btn_copy_strategy_snippet.pack(side=RIGHT, padx=(2, 0), anchor=E) + + def _frame_image_viewer(self): + # ************ Image viewer ************ # + self.frame_image_viewer = LabelFrame(self.frame_main, text="Image viewer", padx=2, pady=2) + self.frame_image_viewer.pack(side=TOP, padx=2, pady=2, fill=X) + + self.canvas_desktop_img = Canvas(self.frame_image_viewer, width=384, height=216) + self.canvas_desktop_img.grid(row=3, column=0, sticky="WSEN") + self.desktop_img = self.canvas_desktop_img.create_image(384/2, 216/2, image=self.haystack_img_blank) + self.canvas_desktop_img.bind("", self._img_viewer_click) + self.canvas_desktop_img.bind("", self._img_viewer_hover_enter) + self.canvas_desktop_img.bind("", self._clear_statusbar) + + self.canvas_ref_img = Canvas(self.frame_image_viewer, width=1, height=1) + self.canvas_ref_img.grid(row=3, column=1, sticky="WSEN") + self.ref_img = self.canvas_ref_img.create_image(47.5, 216/2, image=self.needle_img_blank) + + Label(self.frame_image_viewer, text="Desktop").grid(pady=(0, 0), row=4, column=0, sticky="WSEN") + Label(self.frame_image_viewer, text="Reference Image").grid(pady=(0, 0), row=4, column=1, sticky="WSEN") + + + # ************* Status bar *************** # + def _status_bar(self): + self.frame_statusBar = Frame(self) + self.frame_statusBar.pack(side=TOP, fill=X, expand=True) + self.hint_msg = StringVar() + self.label_statusBar = Label(self.frame_statusBar, textvariable=self.hint_msg, bd=1, relief=SUNKEN, anchor=W) + self.label_statusBar.pack(side=BOTTOM, fill=X, expand=True) + + + # ************** Bindings *************** # + + def _img_viewer_click(self, event=None): + if (self.processing_done): + self.haystack_img = self.image_container.get_haystack_image_orig_size(format=ImageFormat.IMAGETK) + img_viewer_window = Toplevel(self) + img_viewer_window.title("Image Viewer") + label_img_viewer = Label(img_viewer_window, image=self.haystack_img) + label_img_viewer.pack() + + # **** Bindings for hints (statusbar) **** # + + def _img_viewer_hover_enter(self, event=None): + if (self.processing_done): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Click to view in full screen") + + def _default_strategy_config_enter(self, event=None): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Configure default strategy (default) parameters") + + def _edge_detec_strategy_config_enter(self, event=None): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Configure edge detection strategy (edge) parameters") + + def _threshold_error(self): + self.label_statusBar.config(fg="RED") + self.hint_msg.set("Higher threshold value must be greater than the lower threshold value!") + + def _ready(self): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Ready") + + def _clear_statusbar(self, event=None): + self.hint_msg.set("") diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py new file mode 100644 index 0000000..3de4012 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py @@ -0,0 +1,58 @@ +from PIL import Image, ImageTk +from enum import Enum, unique +import numpy as np + + +@unique +class ImageFormat(Enum): + PILIMG = 0 + NUMPYARRAY = 1 + IMAGETK = 2 # ImageTk PhotoImage + PATHSTR = 3 + +class ImageContainer: + """Store and retrieve haystack and needle images of different formats.""" + def save_to_img_container(self, img, is_haystack_img=False): + """Save haystack and needle images. + + Args: + - ``img (str/PIL.Image)``: Path (str) or object (PIL.Image) to haystack/needle image. + - ``is_haystack_img (bool, optional)``: If to be saved img is a haystack image. + """ + if isinstance(img, str): + _PIL_img = Image.open(img) + else: + _PIL_img = img + + if is_haystack_img: + self._haystack_image_orig_size = _PIL_img + self._haystack_image = _PIL_img.resize((384, 216), Image.ANTIALIAS) + else: + self._needle_image = {'Path': img, 'Obj': _PIL_img} + + def get_haystack_image(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._haystack_image + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._haystack_image) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._haystack_image) + + def get_haystack_image_orig_size(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._haystack_image_orig_size + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._haystack_image_orig_size) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._haystack_image_orig_size) + + def get_needle_image(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._needle_image['Obj'] + elif format == ImageFormat.PATHSTR: + return self._needle_image['Path'] + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._needle_image['Obj']) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._needle_image['Obj']) + diff --git a/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py new file mode 100644 index 0000000..ea052db --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py @@ -0,0 +1,44 @@ +from PIL import ImageDraw +from abc import ABC, abstractmethod +from .image_manipulation import ImageFormat, ImageContainer + + +class TemplateMatchingStrategy(ABC): + @abstractmethod + def find_num_of_matches(self) -> int: pass + + # Match place is highlighted with a red rectangle + def highlight_matches(self): + haystack_img = self.image_container.get_haystack_image_orig_size(ImageFormat.PILIMG) + draw = ImageDraw.Draw(haystack_img, "RGBA") + + for loc in self.coord: + draw.rectangle([loc[0], loc[1], loc[0]+loc[2], loc[1]+loc[3]], fill=(256, 0, 0, 127)) + draw.rectangle([loc[0], loc[1], loc[0]+loc[2], loc[1]+loc[3]], outline=(256, 0, 0, 127), width=3) + + self.image_container.save_to_img_container(img=haystack_img, is_haystack_img=True) + +class Pyautogui(TemplateMatchingStrategy): + def __init__(self, image_container: ImageContainer, image_horizon_instance): + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + + def find_num_of_matches(self): + self.coord = list(self.image_horizon_instance._locate_all( + self.image_container.get_needle_image(ImageFormat.PATHSTR), + self.image_container.get_haystack_image_orig_size(ImageFormat.PILIMG) + )) + return self.coord + + +class Skimage(TemplateMatchingStrategy): + def __init__(self, image_container: ImageContainer, image_horizon_instance): + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + + def find_num_of_matches(self): + self.coord = list(self.image_horizon_instance._locate_all( + self.image_container.get_needle_image(ImageFormat.PATHSTR), + self.image_container.get_haystack_image_orig_size(ImageFormat.NUMPYARRAY) + )) + return self.coord diff --git a/build/lib/ImageHorizonLibrary/recognition/__init__.py b/build/lib/ImageHorizonLibrary/recognition/__init__.py new file mode 100644 index 0000000..8f078d8 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +from ._recognize_images import _RecognizeImages, _StrategyPyautogui, _StrategySkimage +from ._screenshot import _Screenshot + +__all__ = [ + '_RecognizeImages', + '_StrategyPyautogui', + '_StrategySkimage', + '_Screenshot' +] diff --git a/build/lib/ImageHorizonLibrary/recognition/_recognize_images.py b/build/lib/ImageHorizonLibrary/recognition/_recognize_images.py new file mode 100644 index 0000000..d6aea0d --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/_recognize_images.py @@ -0,0 +1,431 @@ +# -*- coding: utf-8 -*- +from os import listdir +from os.path import abspath, isdir, isfile, join as path_join +from time import time +from contextlib import contextmanager + +import pyautogui as ag +from robot.api import logger as LOGGER + +from skimage.feature import match_template, peak_local_max, canny +from skimage.color import rgb2gray +from skimage.io import imread, imsave + +import numpy as np + + +from ..errors import ImageNotFoundException, InvalidImageException +from ..errors import ReferenceFolderException + +class _RecognizeImages(object): + + def _normalize(self, path): + if (not self.reference_folder or + not isinstance(self.reference_folder, str) or + not isdir(self.reference_folder)): + raise ReferenceFolderException('Reference folder is invalid: ' + '"%s"' % self.reference_folder) + if (not path or not isinstance(path, str)): + raise InvalidImageException('"%s" is invalid image name.' % path) + path = str(path.lower().replace(' ', '_')) + path = abspath(path_join(self.reference_folder, path)) + if not path.endswith('.png') and not isdir(path): + path += '.png' + if not isfile(path) and not isdir(path): + raise InvalidImageException('Image path not found: "%s".' % path) + return path + + def click_image(self, reference_image): + '''Finds the reference image on screen and clicks it once. + + ``reference_image`` is automatically normalized as described in the + `Reference image names`. + ''' + center_location = self.locate(reference_image) + LOGGER.info('Clicking image "%s" in position %s' % (reference_image, + center_location)) + ag.click(center_location) + return center_location + + def _click_to_the_direction_of(self, direction, location, offset, + clicks, button, interval): + raise NotImplementedError('This is defined in the main class.') + + def _locate_and_click_direction(self, direction, reference_image, offset, + clicks, button, interval): + location = self.locate(reference_image) + self._click_to_the_direction_of(direction, location, offset, clicks, + button, interval) + + def click_to_the_above_of_image(self, reference_image, offset, clicks=1, + button='left', interval=0.0): + '''Clicks above of reference image by given offset. + + See `Reference image names` for documentation for ``reference_image``. + + ``offset`` is the number of pixels from the center of the reference + image. + + ``clicks`` and ``button`` are documented in `Click To The Above Of`. + ''' + self._locate_and_click_direction('up', reference_image, offset, + clicks, button, interval) + + def click_to_the_below_of_image(self, reference_image, offset, clicks=1, + button='left', interval=0.0): + '''Clicks below of reference image by given offset. + + See argument documentation in `Click To The Above Of Image`. + ''' + self._locate_and_click_direction('down', reference_image, offset, + clicks, button, interval) + + def click_to_the_left_of_image(self, reference_image, offset, clicks=1, + button='left', interval=0.0): + '''Clicks left of reference image by given offset. + + See argument documentation in `Click To The Above Of Image`. + ''' + self._locate_and_click_direction('left', reference_image, offset, + clicks, button, interval) + + def click_to_the_right_of_image(self, reference_image, offset, clicks=1, + button='left', interval=0.0): + '''Clicks right of reference image by given offset. + + See argument documentation in `Click To The Above Of Image`. + ''' + self._locate_and_click_direction('right', reference_image, offset, + clicks, button, interval) + + def copy_from_the_above_of(self, reference_image, offset): + '''Clicks three times above of reference image by given offset and + copies. + + See `Reference image names` for documentation for ``reference_image``. + + See `Click To The Above Of Image` for documentation for ``offset``. + + Copy is done by pressing ``Ctrl+C`` on Windows and Linux and ``⌘+C`` + on OS X. + ''' + self._locate_and_click_direction('up', reference_image, offset, + clicks=3, button='left', interval=0.0) + return self.copy() + + def copy_from_the_below_of(self, reference_image, offset): + '''Clicks three times below of reference image by given offset and + copies. + + See argument documentation in `Copy From The Above Of`. + ''' + self._locate_and_click_direction('down', reference_image, offset, + clicks=3, button='left', interval=0.0) + return self.copy() + + def copy_from_the_left_of(self, reference_image, offset): + '''Clicks three times left of reference image by given offset and + copies. + + See argument documentation in `Copy From The Above Of`. + ''' + self._locate_and_click_direction('left', reference_image, offset, + clicks=3, button='left', interval=0.0) + return self.copy() + + def copy_from_the_right_of(self, reference_image, offset): + '''Clicks three times right of reference image by given offset and + copies. + + See argument documentation in `Copy From The Above Of`. + ''' + self._locate_and_click_direction('right', reference_image, offset, + clicks=3, button='left', interval=0.0) + return self.copy() + + @contextmanager + def _suppress_keyword_on_failure(self): + keyword = self.keyword_on_failure + self.keyword_on_failure = None + yield None + self.keyword_on_failure = keyword + + def _get_reference_images(self, reference_image): + '''Return an absolute path for the given reference imge. + Return as a list of those if reference_image is a folder. + ''' + is_dir = False + try: + if isdir(self._normalize(reference_image)): + is_dir = True + except InvalidImageException: + pass + is_file = False + try: + if isfile(self._normalize(reference_image)): + is_file = True + except InvalidImageException: + pass + reference_image = self._normalize(reference_image) + + reference_images = [] + if is_file: + reference_images = [reference_image] + elif is_dir: + for f in listdir(self._normalize(reference_image)): + if not isfile(self._normalize(path_join(reference_image, f))): + raise InvalidImageException( + self._normalize(reference_image)) + reference_images.append(path_join(reference_image, f)) + return reference_images + + def _locate(self, reference_image, log_it=True): + reference_images = self._get_reference_images(reference_image) + + location = None + for ref_image in reference_images: + location = self._try_locate(ref_image) + if location != None: + break + + if location is None: + if log_it: + LOGGER.info('Image "%s" was not found ' + 'on screen. (strategy: %s)' % (reference_image, self.strategy)) + self._run_on_failure() + raise ImageNotFoundException(reference_image) + + center_point = ag.center(location) + x = center_point.x + y = center_point.y + if self.has_retina: + x = x / 2 + y = y / 2 + if log_it: + LOGGER.info('Image "%s" found at %r (strategy: %s)' % (reference_image, (x,y), self.strategy)) + return (x, y) + + def _locate_all(self, reference_image, haystack_image=None): + '''Tries to locate all occurrences of the reference image on the screen + or on the haystack image, if given. + Returns a list of location tuples (finds 0..n)''' + reference_images = self._get_reference_images(reference_image) + if len(reference_images) > 1: + raise InvalidImageException( + f'Locating ALL occurences of MANY files ({", ".join(reference_images)}) is not supported.') + locations = self._try_locate(reference_images[0], locate_all=True, haystack_image=haystack_image) + return locations + + def does_exist(self, reference_image): + '''Returns ``True`` if reference image was found on screen or + ``False`` otherwise. Never fails. + + See `Reference image names` for documentation for ``reference_image``. + ''' + with self._suppress_keyword_on_failure(): + try: + return bool(self._locate(reference_image, log_it=True)) + except ImageNotFoundException: + return False + + def locate(self, reference_image): + '''Locate image on screen. + + Fails if image is not found on screen. + + Returns Python tuple ``(x, y)`` of the coordinates. + ''' + return self._locate(reference_image) + + def wait_for(self, reference_image, timeout=10): + '''Tries to locate given image from the screen for given time. + + Fail if the image is not found on the screen after ``timeout`` has + expired. + + See `Reference images` for further documentation. + + ``timeout`` is given in seconds. + + Returns Python tuple ``(x, y)`` of the coordinates. + ''' + stop_time = time() + int(timeout) + location = None + with self._suppress_keyword_on_failure(): + while time() < stop_time: + try: + location = self._locate(reference_image, log_it=True) + break + except ImageNotFoundException: + pass + if location is None: + self._run_on_failure() + raise ImageNotFoundException(self._normalize(reference_image)) + LOGGER.info('Image "%s" found at %r' % (reference_image, location)) + return location + + def debug_image(self): + '''Halts the test execution and opens the image debugger UI. + + Whenever you encounter problems with the recognition accuracy of a reference image, + you should place this keyword just before the line in question. Example: + + | Debug Image + | Wait For hard_to_find_button + + The test will halt at this position and open the debugger UI. Use it as follows: + + - Select the reference image (`hard_to_find_button`) + - Click the button "Detect reference image" for the strategy you want to test (default/edge). The GUI hides itself while it takes the screenshot of the current application. + - The Image Viewer at the botton shows the screenshot with all regions where the reference image was found. + - "Matches Found": More than one match means that either `conficence` is set too low or that the reference image is visible multiple times. If the latter is the case, you should first detect a unique UI element and use relative keywords like `Click To The Right Of`. + - "Max peak value" (only `edge`) gives feedback about the detection accuracy of the best match and is measured as a float number between 0 and 1. A peak value above _confidence_ results in a match. + - "Edge detection debugger" (only `edge`) opens another window where both the reference and screenshot images are shown before and after the edge detection and is very helpful to loearn how the sigma and low/high threshold parameters lead to different results. + - The field "Keyword to use this strategy" shows how to set the strategy to the current settings. Just copy the line and paste it into the test: + + | Set Strategy edge edge_sigma=2.0 edge_low_threshold=0.1 edge_high_threshold=0.3 + | Wait For hard_to_find_button + + The purpose of this keyword is *solely for debugging purposes*; don't + use it in production!''' + from .ImageDebugger import ImageDebugger + debug_app = ImageDebugger(self) + + +class _StrategyPyautogui(): + + def __init__(self, image_horizon_instance): + self.ih_instance = image_horizon_instance + + def _try_locate(self, ref_image, haystack_image=None, locate_all=False): + '''Tries to locate the reference image on the screen or the haystack_image. + Return values: + - locate_all=False: None or 1 location tuple (finds max 1) + - locate_all=True: None or list of location tuples (finds 0..n) + (GUI Debugger mode)''' + + ih = self.ih_instance + location = None + if haystack_image is None: + haystack_image = ag.screenshot() + + if locate_all: + locate_func = ag.locateAll + else: + locate_func = ag.locate #Copy below,take screenshots + + with ih._suppress_keyword_on_failure(): + try: + if ih.has_cv and ih.confidence: + location_res = locate_func(ref_image, + haystack_image, + confidence=ih.confidence) + else: + if ih.confidence: + LOGGER.warn("Can't set confidence because you don't " + "have OpenCV (python-opencv) installed " + "or a confidence level was not given.") + location_res = locate_func(ref_image, haystack_image) + except ImageNotFoundException as ex: + LOGGER.info(ex) + pass + if locate_all: + # convert the generator fo Box objects to a list of tuples + location = [tuple(box) for box in location_res] + else: + # Single Box + location = location_res + return location + + + +class _StrategySkimage(): + _SKIMAGE_DEFAULT_CONFIDENCE = 0.99 + + def __init__(self, image_horizon_instance): + self.ih_instance = image_horizon_instance + + def _try_locate(self, ref_image, haystack_image=None, locate_all=False): + '''Tries to locate the reference image on the screen or the provided haystack_image. + Return values: + - locate_all=False: None or 1 location tuple (finds max 1) + - locate_all=True: None or list of location tuples (finds 0..n) + (GUI Debugger mode)''' + + ih = self.ih_instance + confidence = ih.confidence or self._SKIMAGE_DEFAULT_CONFIDENCE + with ih._suppress_keyword_on_failure(): + needle_img = imread(ref_image, as_gray=True) + needle_img_name = ref_image.split("\\")[-1].split(".")[0] + #haystack_img_height, needle_img_width = needle_img.shape + needle_img_height, needle_img_width = needle_img.shape + if haystack_image is None: + haystack_img_gray = rgb2gray(np.array(ag.screenshot())) + else: + haystack_img_gray = rgb2gray(haystack_image) + + # Canny edge detection on both images + ih.needle_edge = self.detect_edges(needle_img) + ih.haystack_edge = self.detect_edges(haystack_img_gray) + + # peakmap is a "heatmap" of matching coordinates + ih.peakmap = match_template(ih.haystack_edge, ih.needle_edge, pad_input=True) + + # For debugging purposes + debug = False + if debug: + imsave(needle_img_name + "needle.png", needle_img) + imsave(needle_img_name + "needle_edge.png", ih.needle_edge) + imsave(needle_img_name + "haystack.png", haystack_img_gray) + imsave(needle_img_name + "haystack_edge.png", ih.haystack_edge) + imsave(needle_img_name + "peakmap.png", ih.peakmap) + + if locate_all: + # https://stackoverflow.com/questions/48732991/search-for-all-templates-using-scikit-image + peaks = peak_local_max(ih.peakmap,threshold_rel=confidence) + peak_coords = zip(peaks[:,1], peaks[:,0]) + location = [] + for i, pk in enumerate(peak_coords): + x = pk[0] + y = pk[1] + # higest peak level + peak = ih.peakmap[y][x] + if peak > confidence: + loc = (x-needle_img_width/2, y-needle_img_height/2, needle_img_width, needle_img_height) + location.append(loc) + + else: + # translate highest index in peakmap from linear (memory) into + # an index of a matrix with the peakmaps dimensions + ij = np.unravel_index(np.argmax(ih.peakmap), ih.peakmap.shape) + # Extract coordinates of the highest peak; xy is the coordinate + # where the CENTER of the reference image matched. + x, y = ij[::-1] + # higest peak level + peak = ih.peakmap[y][x] + if peak > confidence: + # tuple of xy (topleft) and width/height + location = (x-needle_img_width/2, y-needle_img_height/2, needle_img_width, needle_img_height) + else: + location = None + # TODO: Also return peak level + return location + + def _detect_edges(self, img, sigma, low, high): + edge_img = canny( + image=img, + sigma=sigma, + low_threshold=low, + high_threshold=high, + ) + return edge_img + + def detect_edges(self, img): + '''Apply edge detection on a given image''' + return self._detect_edges( + img, + self.ih_instance.edge_sigma, + self.ih_instance.edge_low_threshold, + self.ih_instance.edge_low_threshold + ) + diff --git a/build/lib/ImageHorizonLibrary/recognition/_screenshot.py b/build/lib/ImageHorizonLibrary/recognition/_screenshot.py new file mode 100644 index 0000000..3738b67 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/recognition/_screenshot.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +from os.path import abspath, join as path_join +from random import choice +from string import ascii_lowercase + +import pyautogui as ag +from robot.libraries.BuiltIn import BuiltIn, RobotNotRunningError +from robot.api import logger as LOGGER + +from ..errors import ScreenshotFolderException + + +class _Screenshot(object): + def _make_up_filename(self): + try: + path = BuiltIn().get_variable_value('${SUITE NAME}') + path = '%s-screenshot' % path.replace(' ', '') + except RobotNotRunningError: + LOGGER.info('Could not get suite name, using ' + 'default naming scheme') + path = 'ImageHorizon-screenshot' + path = '%s-%d.png' % (path, self.screenshot_counter) + self.screenshot_counter += 1 + return path + + def take_a_screenshot(self): + '''Takes a screenshot of the screen. + + This keyword is run on failure if it is not overwritten when + `importing` the library. + + Screenshots are saved to the current working directory or in the + ``screenshot_folder`` if such is defined during `importing`. + + The file name for the screenshot is the current suite name with a + running integer appended. If this keyword is used outside of Robot + Framework execution, file name is this library's name with running + integer appended. + ''' + target_dir = self.screenshot_folder if self.screenshot_folder else '' + if not isinstance(target_dir, str): + raise ScreenshotFolderException('Screenshot folder is invalid: ' + '"%s"' % target_dir) + path = self._make_up_filename() + path = abspath(path_join(target_dir, path)) + LOGGER.info('Screenshot taken: {0}
'.format(path), html=True) + ag.screenshot(path) diff --git a/build/lib/ImageHorizonLibrary/utils.py b/build/lib/ImageHorizonLibrary/utils.py new file mode 100644 index 0000000..e312ff6 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/utils.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +from platform import platform, architecture +from subprocess import call + + +PLATFORM = platform() +ARCHITECTURE = architecture() + + +def is_windows(): + return PLATFORM.lower().startswith('windows') + + +def is_mac(): + return PLATFORM.lower().startswith('darwin') + + +def is_linux(): + return PLATFORM.lower().startswith('linux') + + +def is_java(): + return PLATFORM.lower().startswith('java') + +def has_retina(): + if is_mac(): + # Will return 0 if there is a retina display + return call("system_profiler SPDisplaysDataType | grep 'Retina'", shell=True) == 0 + return False + +def has_cv(): + has_cv = True + try: + import cv2 + except ModuleNotFoundError as err: + has_cv = False + return has_cv + +def has_skimage(): + has_skimage = True + try: + import skimage + except ModuleNotFoundError as err: + has_skimage = False + return has_skimage diff --git a/build/lib/ImageHorizonLibrary/version.py b/build/lib/ImageHorizonLibrary/version.py new file mode 100644 index 0000000..c2635d7 --- /dev/null +++ b/build/lib/ImageHorizonLibrary/version.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +VERSION = '1.1' diff --git a/doc/ImageHorizonLibrary.html b/doc/ImageHorizonLibrary.html index 2d9dce9..8c93351 100644 --- a/doc/ImageHorizonLibrary.html +++ b/doc/ImageHorizonLibrary.html @@ -1,921 +1,1687 @@ - - - - - - - - - - - - - - - - - - - - - - - -
-

Opening library documentation failed

-
    -
  • Verify that you have JavaScript enabled in your browser.
    • -
  • Make sure you are using a modern enough browser. If using Internet Explorer, version 8 or newer is required.
    • -
  • Check are there messages in your browser's JavaScript error log. Please report the problem if you suspect you have encountered a bug.
    • -
-
- - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + +
+

Opening library documentation failed

+
    +
  • Verify that you have JavaScript enabled in your browser.
    • +
  • Make sure you are using a modern enough browser. If using Internet Explorer, version 11 is required.
    • +
  • Check are there messages in your browser's JavaScript error log. Please report the problem if you suspect you have encountered a bug.
    • +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/setup.py b/setup.py index 8b46395..c77b586 100644 --- a/setup.py +++ b/setup.py @@ -31,12 +31,24 @@ license='MIT', install_requires=[ 'robotframework>=2.8', - 'pyautogui>=0.9.30' + # Version 10 is not fully compatible (https://stackoverflow.com/questions/76616042/attributeerror-module-pil-image-has-no-attribute-antialias) + 'matplotlib', + 'Pillow==9.5.0', + 'PyScreeze==0.1.29', + 'pyautogui>=0.9.30', + #'scikit-image@ file:// + #'scikit-image==0.22.0 @ https://files.pythonhosted.org/packages/ce/d0/a3f60c9f57ed295b3076e4acdb29a37bbd8823452562ab2ad51b03d6f377/scikit_image-0.22.0-cp311-cp311-win_amd64.whl', + # scikit-image 0.19 can't be used yet - regression bug + # (reference images get an unexplainable white 1px border) + #'scikit-image==0.18.3', + 'scikit-image==0.22.0', + #'matplotlib==3.4.3' ], packages=[ 'ImageHorizonLibrary', 'ImageHorizonLibrary.interaction', 'ImageHorizonLibrary.recognition', + 'ImageHorizonLibrary.recognition.ImageDebugger' ], package_dir={'': 'src'}, keywords=KEYWORDS, diff --git a/src/ImageHorizonLibrary/__init__.py b/src/ImageHorizonLibrary/__init__.py index 200b49d..7d84d2a 100644 --- a/src/ImageHorizonLibrary/__init__.py +++ b/src/ImageHorizonLibrary/__init__.py @@ -1,13 +1,14 @@ # -*- coding: utf-8 -*- from collections import OrderedDict from contextlib import contextmanager +import inspect from .errors import * # import errors before checking dependencies! try: import pyautogui as ag except ImportError: - raise ImageHorizonLibraryError('There is something wrong pyautogui or ' + raise ImageHorizonLibraryError('There is something wrong with pyautogui or ' 'it is not installed.') try: @@ -25,6 +26,12 @@ 'which is not a supported platform. Please ' 'use Python and verify that Tkinter works.') +try: + import skimage as sk +except ImportError: + raise ImageHorizonLibraryError('There is either something wrong with skimage ' + '(scikit-image) or it is not installed.') + from . import utils from .interaction import * from .recognition import * @@ -32,42 +39,19 @@ __version__ = VERSION - -class ImageHorizonLibrary(_Keyboard, - _Mouse, - _OperatingSystem, - _RecognizeImages, - _Screenshot): +class ImageHorizonLibrary(_Keyboard, _Mouse, _OperatingSystem, _RecognizeImages, _Screenshot): '''A cross-platform Robot Framework library for GUI automation. - ImageHorizonLibrary provides keyboard and mouse actions as well as - facilities to recognize images on screen. It can also take screenshots in - case of failure or otherwise. - + *Key features*: + - Automates *keyboard and mouse actions* on the screen (based on [https://pyautogui.readthedocs.org|pyautogui]). + - The regions to execute these actions on (buttons, sliders, input fields etc.) are determined by `reference images` which the library detects on the screen - independently of the OS or the application type. + - Two different image `recognition strategies`: `default` (fast and reliable of predictable screen content), and `edge` (to facilitate the recognition of unpredictable pixel deviations) + - The library can also take screenshots in case of failure or by intention. - This library is built on top of - [https://pyautogui.readthedocs.org|pyautogui]. + = Image Recognition = - == Confidence Level == - By default, image recognition searches images with pixel-perfect matching. - This is in many scenarios too precise, as changing desktop background, - transpareny in the reference images, slightly changing resolutions, and - myriad of factors might throw the algorithm off. In these cases, it is - advised to adjust the precision manually. - - This ability to adjust can be enabled by installing - [https://pypi.org/project/opencv-python|opencv-python] Python package - separately: - - | $ pip install opencv-python - - After installation, the library will use OpenCV, which enables setting the - precision during `library importing` and during the test case with keyword - `Set Confidence`. - - - = Reference image names = - ``reference_image`` parameter can be either a single file, or a folder. + == Reference images == + ``reference_image`` parameter can be either a single file or a folder. If ``reference_image`` is a folder, image recognition is tried separately for each image in that folder, in alphabetical order until a match is found. @@ -93,11 +77,100 @@ class ImageHorizonLibrary(_Keyboard, | `Click Image` | popup Window title | | # Path is images/popup_window_title.png | | `Click Image` | button Login Without User Credentials | | # Path is images/button_login_without_user_credentials.png | + == Recognition strategies == + Basically, image recognition works by searching a reference image on the + another image (a screnshot of the current desktop). + If there is a region with 100% matching pixels of the reference image, this + area represents a match. + + By default, the reference image must be an exakt sub-image of the screenshot. + This works flawlessly in most cases. + + But problems can arise when: + + - the application's GUI uses transpareny effects + - the screen resolution/the window size changes + - font aliasing is used for dynamic text + - compression algorithms in RDP/Citrix cause invisible artifacts + - ...and in many more situations. + + In those situations, a certain amount of the pixels do not match. + + To solve this, ImageHorizon comes with a parameter ``confidence level``. This is a decimal value + between 0 and 1 (inclusive) and defines how many percent of the reference image pixels + must match the found region's imag. It is set to 1.0 = 100% by default. + + Confidence level can be set during `library importing` and re-adjusted during the test + case with the keyword `Set Confidence`. + + === Default image detection strategy === + + If imported without any strategy argument, the library uses [https://pyautogui.readthedocs.org|pyautogui] + under the hood to recognize images on the screen. + This is the perfect choice to start writing tests. + + To use `confidence level in mode` ``default`` the + [https://pypi.org/project/opencv-python|opencv-python] Python package + must be installed separately: + + | $ pip install opencv-python + + After installation, the library will automatically use OpenCV for confidence + levels lower than 1.0. + + === The "edge" image detection strategy === + + The default image recognition reaches its limitations when the area to + match contains a *disproportionate amount of unpredictable pixels*. + + The idea for this strategy came from a problem in real life: a web application + showing a topographical map (loaded from a 3rd party provider), with a layer of + interstate highways as black lines. For some reasons, the pixels of topographic + areas between the highway lines (which are the vast majority) showed a slight + deviation in brightness - invisible for the naked eye, but enough to make the test failing. + + The abstract and simplified example for this is a horizontal black line of 1px width in a + matrix of 10x10 white pixels. To neglect a (slight) brightness deviation of the white pixels, + you would need a confidence level of 0.1 which allows 90% of the pixels to be + different. This is insanse and leads to inpredictable results. + + That's why ``edge`` was implemented as an alternative recognition strategy. + The key here lies in the approach to *reduce both images* (reference and screenshot + image) *to the essential characteristics* and then *compare _those_ images*. + + "Essential characteristics" of an image are those areas where neighbouring pixels show a + sharp change of brightness, better known as "edges". [https://en.wikipedia.org/wiki/Edge_detection|Edge detection] + is the process of finding the edges in an image, done by [https://scikit-image.org/|scikit-image] in this library. + + As a brief digression, edge detection is a multi-step process: + + - apply a [https://en.wikipedia.org/wiki/Gaussian_filter|Gaussian filter] (blurs the image to remove noise; intensity set by parameter `sigma`) + - apply a [https://en.wikipedia.org/wiki/Sobel_operator|Sobel filter] (remove non-max pixels, get a 1 pixel edge curve) + - separate weak edges from strong ones with [https://en.wikipedia.org/wiki/Canny_edge_detector#Edge_tracking_by_hysteresis|hysteresis] + - apply the `template_matching` routine to get a [https://en.wikipedia.org/wiki/Cross-correlation|cross correlation] matrix of values from -1 (no correlation) to +1 (perfect correlation). + - Filter out only those coordinates with values greater than the ``confidence`` level, take the max + + The keyword `Debug Image` opens a debugger UI where confidence level, Gaussian sigma and low/high thresholds can be tested and adjusted to individual needs. + + Edge detection costs some extra CPU time; you should always first try + to use the ``default`` strategy and only selectively switch to ``edge`` + when a confidence level below 0.9 is not sufficient to detect images reliably anymore: + + | # use with defaults + | Set Strategy edge + | # use with custom parameters + | Set Strategy edge edge_sigma=2.0 edge_low_threshold=0.1 edge_high_threshold=0.3 confidence=0.8 + + To use strategy ``edge``, the [https://scikit-image.org|scikit-image] Python package must be installed separately: + + | $ pip install scikit-image + = Performance = Locating images on screen, especially if screen resolution is large and - reference image is also large, might take considerable time. It is - therefore advisable to save the returned coordinates if you are + reference image is also large, might take considerable time, regardless + of the strategy. + It is therefore advisable to save the returned coordinates if you are manipulating the same context many times in the row: | `Wait For` | label Name | | @@ -108,16 +181,17 @@ class ImageHorizonLibrary(_Keyboard, | ${location}= | `Wait For` | label Name | | `Click To The Left Of` | ${location} | 200 | - ''' + ''' ROBOT_LIBRARY_SCOPE = 'TEST SUITE' ROBOT_LIBRARY_VERSION = VERSION def __init__(self, reference_folder=None, screenshot_folder=None, keyword_on_failure='ImageHorizonLibrary.Take A Screenshot', - confidence=None): + confidence=None, strategy='default', + edge_sigma=2.0, edge_low_threshold=0.1, edge_high_threshold=0.3): '''ImageHorizonLibrary can be imported with several options. - + ``reference_folder`` is path to the folder where all reference images are stored. It must be a _valid absolute path_. As the library is suite-specific (ie. new instance is created for every suite), @@ -130,13 +204,19 @@ def __init__(self, reference_folder=None, screenshot_folder=None, ``keyword_on_failure`` is the keyword to be run, when location-related keywords fail. If you wish to not take screenshots, use for example `BuiltIn.No Operation`. Keyword must however be a valid keyword. - - ``confidence`` provides a tolerance for the ``reference_image``. - It can be used if python-opencv is installed and - is given as number between 0 and 1. Not used - by default. + + ``strategy`` sets the way how images are detected on the screen. See also + keyword `Set Strategy` to change the strategy during the test. Parameters: + - ``default`` - (Default) + - ``edge`` - Advanced image recognition options with canny edge detection + + The ``edge`` strategy allows these additional parameters: + - ``edge_sigma`` - Gaussian blur intensity + - ``edge_low_threshold`` - low pixel gradient threshold + - ``edge_high_threshold`` - high pixel gradient threshold ''' - + + # _RecognizeImages.set_strategy(self, strategy) self.reference_folder = reference_folder self.screenshot_folder = screenshot_folder self.keyword_on_failure = keyword_on_failure @@ -147,8 +227,47 @@ def __init__(self, reference_folder=None, screenshot_folder=None, self.is_linux = utils.is_linux() self.has_retina = utils.has_retina() self.has_cv = utils.has_cv() + self.has_skimage = utils.has_skimage() self.confidence = confidence - + self.initial_confidence = confidence + self._class_bases = inspect.getmro(self.__class__) + self.set_strategy(strategy, self.confidence) + self.edge_sigma = edge_sigma + self.edge_low_threshold = edge_low_threshold + self.edge_high_threshold = edge_high_threshold + + + + def set_strategy(self, strategy, edge_sigma=2.0, edge_low_threshold=0.1, edge_high_threshold=0.3, confidence=None): + '''Changes the way how images are detected on the screen. This can also be done globally during `Importing`. + Strategies: + - ``default`` + - ``edge`` - Advanced image recognition options with canny edge detection + + The ``edge`` strategy allows these additional parameters: + - ``edge_sigma`` - Gaussian blur intensity + - ``edge_low_threshold`` - low pixel gradient threshold + - ``edge_high_threshold`` - high pixel gradient threshold + + Both strategies can optionally be initialized with a new confidence.''' + + self.strategy = strategy + if strategy == 'default': + self.strategy_instance = _StrategyPyautogui(self) + elif strategy == 'edge': + self.strategy_instance = _StrategySkimage(self) + self.edge_sigma = edge_sigma + self.edge_low_threshold = edge_low_threshold + self.edge_high_threshold = edge_high_threshold + else: + raise StrategyException('Invalid strategy: "%s"' % strategy) + + if not confidence is None: + self.set_confidence(confidence) + + # Linking protected _try_locate to the strategy's method + self._try_locate = self.strategy_instance._try_locate + def _get_location(self, direction, location, offset): x, y = location offset = int(offset) @@ -259,6 +378,12 @@ def set_screenshot_folder(self, screenshot_folder_path): ''' self.screenshot_folder = screenshot_folder_path + def reset_confidence(self): + '''Resets the confidence level to the library default. + If no confidence was given during import, this is None.''' + LOGGER.info('Resetting confidence level to {}.'.format(self.initial_confidence)) + self.confidence = self.initial_confidence + def set_confidence(self, new_confidence): '''Sets the accuracy when finding images. diff --git a/src/ImageHorizonLibrary/errors.py b/src/ImageHorizonLibrary/errors.py index 36d4bed..ea318da 100644 --- a/src/ImageHorizonLibrary/errors.py +++ b/src/ImageHorizonLibrary/errors.py @@ -33,3 +33,6 @@ class ReferenceFolderException(Exception): class ScreenshotFolderException(Exception): pass + +class StrategyException(Exception): + pass \ No newline at end of file diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py new file mode 100644 index 0000000..b38eb0f --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/__init__.py @@ -0,0 +1,9 @@ +from tkinter import * +from .image_debugger_controller import UILocatorController + + +class ImageDebugger: + + def __init__(self, image_horizon_instance): + app = UILocatorController(image_horizon_instance) + app.main() diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py new file mode 100644 index 0000000..a174512 --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_controller.py @@ -0,0 +1,172 @@ +from .image_debugger_model import UILocatorModel +from .image_debugger_view import UILocatorView +from .template_matching_strategies import Pyautogui, Skimage +from .image_manipulation import ImageContainer, ImageFormat +from pathlib import Path +import os, glob +import pyperclip +import numpy as np +import webbrowser + + +class UILocatorController: + + def __init__(self, image_horizon_instance): + self.image_container = ImageContainer() + self.model = UILocatorModel() + self.image_horizon_instance = image_horizon_instance + self.view = UILocatorView(self, self.image_container, self.image_horizon_instance) + + def main(self): + self.init_view() + self.view.main() + + def init_view(self): + self.view.ref_dir_path.set(self.image_horizon_instance.reference_folder) + self.view.scale_conf_lvl_ag.set(0.99) + self.view.scale_sigma_skimage.set(1.0) + self.view.scale_low_thres_skimage.set(0.1) + self.view.scale_high_thres_skimage.set(0.3) + self.view.scale_conf_lvl_skimage.set(0.99) + self.view.matches_found.set("None") + self.view.btn_edge_detec_debugger["state"] = "disabled" + self.view.btn_run_pyautogui["state"] = "disabled" + self.view.btn_run_skimage["state"] = "disabled" + self.view.btn_copy_strategy_snippet["state"] = "disabled" + self.view.hint_msg.set("Ready") + self.view.processing_done = False + + def help(self): + webbrowser.open("https://eficode.github.io/robotframework-imagehorizonlibrary/doc/ImageHorizonLibrary.html") + + def load_needle_image_names(self, combobox=None): + os.chdir(self.image_horizon_instance.reference_folder) + list_needle_images_names = [] + for needle_img_name in glob.glob("*.png"): + list_needle_images_names.append(needle_img_name) + if combobox: + self.combobox = combobox + self.combobox['values']=list_needle_images_names + self.combobox.set('__ __ __ Select a reference image __ __ __') + + def reset_images(self): + self.view.canvas_ref_img.itemconfig( + self.view.ref_img, + image=self.view.needle_img_blank, + ) + self.view.canvas_desktop_img.itemconfig( + self.view.desktop_img, + image=self.view.haystack_img_blank, + ) + + def reset_results(self): + self.view.btn_copy_strategy_snippet["state"] = "disabled" + self.view.btn_edge_detec_debugger["state"] = "disabled" + self.view.matches_found.set("None") + self.view.label_matches_found.config(fg='black') + self.view.set_strategy_snippet.set("") + + def refresh(self): + self.load_needle_image_names() + self.view.btn_run_pyautogui["state"] = "disabled" + self.view.btn_run_skimage["state"] = "disabled" + self.reset_results() + self.reset_images() + self.view._ready() + self.view.processing_done=False + + def copy_to_clipboard(self): + pyperclip.copy(self.strategy_snippet) + + def on_select(self, event): + ref_image = Path(self.view.combobox_needle_img_name.get()) + self.image_container.save_to_img_container(img=ref_image.__str__()) + self.needle_img = self.image_container.get_needle_image(ImageFormat.IMAGETK) + self.view.canvas_ref_img.itemconfig( + self.view.ref_img, + image=self.needle_img, + ) + + self.view.btn_run_pyautogui["state"] = "normal" + self.view.btn_run_skimage["state"] = "normal" + + def _take_screenshot(self): + # Minimize the GUI Debugger window before taking the screenshot + self.view.wm_state('iconic') + screenshot = self.model.capture_desktop() + self.view.wm_state('normal') + return screenshot + + def on_click_run_default_strategy(self): + self.image_horizon_instance.set_strategy('default') + self.image_horizon_instance.confidence = float(self.view.scale_conf_lvl_ag.get()) + self.image_container.save_to_img_container(self._take_screenshot(), is_haystack_img=True) + + matcher = Pyautogui(self.image_container, self.image_horizon_instance) + self.coord = matcher.find_num_of_matches() + matcher.highlight_matches() + + self.haystack_image = self.image_container.get_haystack_image(format=ImageFormat.IMAGETK) + self.view.canvas_desktop_img.itemconfig(self.view.desktop_img, image=self.haystack_image) + + num_of_matches_found = len(self.coord) + self.view.matches_found.set(num_of_matches_found) + font_color = self.model.change_color_of_label(num_of_matches_found) + self.view.label_matches_found.config(fg=font_color) + + self.strategy_snippet = f"Set Strategy default confidence={self.image_horizon_instance.confidence}" + self.view.set_strategy_snippet.set(self.strategy_snippet) + self.view.btn_copy_strategy_snippet["state"] = "normal" + self.view.processing_done = True + + def on_click_run_edge_detec_strategy(self): + self.image_horizon_instance.set_strategy('edge') + self.image_horizon_instance.edge_low_threshold = float(self.view.scale_low_thres_skimage.get()) + self.image_horizon_instance.edge_high_threshold = float(self.view.scale_high_thres_skimage.get()) + if self.image_horizon_instance.edge_high_threshold < self.image_horizon_instance.edge_low_threshold: + self.reset_results() + self.reset_images() + self.view._threshold_error() + self.view.processing_done=False + return + + self.image_horizon_instance.confidence = float(self.view.scale_conf_lvl_skimage.get()) + self.image_horizon_instance.edge_sigma = float(self.view.scale_sigma_skimage.get()) + self.image_container._haystack_image_orig_size = self._take_screenshot() + + + matcher = Skimage(self.image_container, self.image_horizon_instance) + self.coord = matcher.find_num_of_matches() + matcher.highlight_matches() + + self.haystack_image = self.image_container.get_haystack_image(format=ImageFormat.IMAGETK) + self.view.canvas_desktop_img.itemconfig(self.view.desktop_img, image=self.haystack_image) + + num_of_matches_found = len(self.coord) + max_peak = round(np.amax(self.image_horizon_instance.peakmap), 2) + if max_peak < 0.75: + result_msg = f"{num_of_matches_found} / max peak value below 0.75" + else: + result_msg = f"{num_of_matches_found} / {max_peak}" + + self.view.matches_found.set(result_msg) + font_color = self.model.change_color_of_label(num_of_matches_found) + self.view.label_matches_found.config(fg=font_color) + self.view.btn_edge_detec_debugger["state"] = "normal" + + self.strategy_snippet = f"Set Strategy edge edge_sigma={self.image_horizon_instance.edge_sigma} edge_low_threshold={self.image_horizon_instance.edge_low_threshold} edge_high_threshold={self.image_horizon_instance.edge_high_threshold} confidence={self.image_horizon_instance.confidence}" + self.view.set_strategy_snippet.set(self.strategy_snippet) + self.view.btn_copy_strategy_snippet["state"] = "normal" + self.view.processing_done = True + + def on_click_plot_results_skimage(self): + title = f"{self.view.matches_found.get()} matches (confidence: {self.image_horizon_instance.confidence})" + self.model.plot_result( + self.image_container.get_needle_image(ImageFormat.NUMPYARRAY), + self.image_container.get_haystack_image_orig_size(ImageFormat.NUMPYARRAY), + self.image_horizon_instance.needle_edge, + self.image_horizon_instance.haystack_edge, + self.image_horizon_instance.peakmap, + title, + self.coord + ) diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py new file mode 100644 index 0000000..0887553 --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_model.py @@ -0,0 +1,53 @@ +import pyautogui as ag +import matplotlib.pyplot as mplt + + +class UILocatorModel(): + def capture_desktop(self): + return ag.screenshot() + + def change_color_of_label(self, num_of_matches_found) -> str: + if(num_of_matches_found == 1): + return 'green' + else: + return 'red' + + def plot_result(self, needle_img, haystack_img, needle_img_edges, haystack_img_edges, peakmap, title, coord): + fig, axs = mplt.subplots(2, 3) + sp_haystack_img = axs[0, 0] + sp_needle_img = axs[0, 1] + sp_invisible = axs[0, 2] + sp_haystack_img_edges = axs[1, 0] + sp_needle_img_edges = axs[1, 1] + sp_peakmap = axs[1, 2] + + # ROW 1 ================================ + sp_needle_img.set_title('Needle') + sp_needle_img.imshow(needle_img, cmap=mplt.cm.gray) + sp_haystack_img.set_title('Haystack') + sp_haystack_img.imshow(haystack_img, cmap=mplt.cm.gray) + sp_haystack_img.sharex(sp_haystack_img_edges) + sp_haystack_img.sharey(sp_haystack_img_edges) + + sp_invisible.set_visible(False) + + # ROW 2 ================================ + sp_needle_img_edges.set_title('Needle (edge d.)') + sp_needle_img_edges.imshow(needle_img_edges, cmap=mplt.cm.gray) + + sp_haystack_img_edges.set_title('Haystack (edge d.)') + sp_haystack_img_edges.imshow(haystack_img_edges, cmap=mplt.cm.gray) + + sp_peakmap.set_title('Peakmap') + sp_peakmap.imshow(peakmap) + sp_peakmap.sharex(sp_haystack_img_edges) + sp_peakmap.sharey(sp_haystack_img_edges) + + for loc in coord: + rect = mplt.Rectangle((loc[0], loc[1]), loc[2], loc[3], edgecolor='r', facecolor='none') + sp_haystack_img_edges.add_patch(rect) + + sp_peakmap.autoscale(False) + fig.suptitle(title, fontsize=14, fontweight='bold') + + mplt.show() \ No newline at end of file diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py new file mode 100644 index 0000000..0df4460 --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_debugger_view.py @@ -0,0 +1,201 @@ +import tkinter as tk +from tkinter import ttk +from tkinter import * +from PIL import ImageTk, Image +from .image_manipulation import ImageFormat + +class UILocatorView(Tk): + def __init__(self, controller, image_container, image_horizon_instance): + super().__init__() + self.title("ImageHorizonLibrary - Debugger") + self.resizable(False, False) + self.controller = controller + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + self.def_save_loc = tk.StringVar() + self.ref_img_loc = tk.StringVar() + self.needle_img_blank = ImageTk.PhotoImage(Image.new('RGB', (30, 30), color='white')) + self.haystack_img_blank = ImageTk.PhotoImage(Image.new('RGB', (384, 216), color='white')) + self._create_view() + + def main(self): + self.mainloop() + + def _create_view(self): + self._menu() + self._frame_main() + self._frame_load_reference_image() + self._frame_computation_params() + self._frame_results() + self._frame_image_viewer() + self._status_bar() + + def _menu(self): + # ***************** Menu ******************* # + menu = Menu(self) + self.config(menu=menu) + menu.add_command(label="Online Help", command=self.controller.help) + + def _frame_main(self): + # *************** Frame Main *************** # + self.frame_main = Frame(self) + self.frame_main.pack(side=TOP, padx=2, pady=2) + + def _frame_load_reference_image(self): + # ************ Frame Select image ************ # + frame_import_ref_image = LabelFrame(self.frame_main, text="Select reference image (.png)", padx=2, pady=2) + frame_import_ref_image.pack(side=TOP, padx=2, pady=2, fill=X) + Label(frame_import_ref_image, text="Reference directory path:").pack(side=TOP, anchor=W) + self.ref_dir_path = StringVar(frame_import_ref_image) + self.label_ref_dir_path = Label(frame_import_ref_image, textvariable=self.ref_dir_path, fg='green') + self.label_ref_dir_path.pack(side=TOP, anchor=W) + + self.combobox_needle_img_name = ttk.Combobox(frame_import_ref_image, width=70, state="readonly") + self.controller.load_needle_image_names(self.combobox_needle_img_name) + self.combobox_needle_img_name.pack(pady=(0,0), side=LEFT, anchor=W) + self.combobox_needle_img_name.bind("<>", self.controller.on_select) + self.btn_copy_strategy_snippet = Button(frame_import_ref_image, text='Refresh', command=self.controller.refresh) + self.btn_copy_strategy_snippet.pack(side=RIGHT, padx=(2, 0), anchor=E) + + def _frame_computation_params(self): + # ************ Frame Computation ************ # + frame_computation = LabelFrame(self.frame_main, text="Computation", padx=2, pady=2) + frame_computation.pack(side=TOP, padx=2, pady=2, fill=X) + + frame_computation.columnconfigure(0, weight=1) + frame_computation.columnconfigure(1, weight=1) + + # ************ Frame default strategy (pyAutogui) ************* # + self.frame_default_strat = LabelFrame(frame_computation, text="Default strategy", padx=2, pady=2) + self.frame_default_strat.grid(row=0, column=0, padx=2, pady=2, sticky="WNS") + + self.frame_default_strat.columnconfigure(0, weight=3) + self.frame_default_strat.columnconfigure(1, weight=1) + + self.frame_default_strat.bind("", self._default_strategy_config_enter) + self.frame_default_strat.bind("", self._clear_statusbar) + + Label(self.frame_default_strat, text="Confidence factor").grid(row=0, column=0, sticky=SW) + self.scale_conf_lvl_ag = Scale(self.frame_default_strat, from_=0.75, to=1.0, resolution=0.01, orient=HORIZONTAL) + self.scale_conf_lvl_ag.grid(padx=(2, 0), row=0, column=1, sticky=E) + + self.btn_run_pyautogui = Button(frame_computation, text='Detect reference image', command=self.controller.on_click_run_default_strategy) + self.btn_run_pyautogui.grid(row=1, column=0, padx=2, sticky=W) + + # ************ Frame edge detection strategy (edge) ************ # + self.frame_edge_detec_strat = LabelFrame(frame_computation, text="Edge detection strategy", padx=2, pady=2) + self.frame_edge_detec_strat.grid(row=0, column=1, padx=2, pady=2, sticky=W) + + self.frame_edge_detec_strat.columnconfigure(0, weight=3) + self.frame_edge_detec_strat.columnconfigure(1, weight=1) + + self.frame_edge_detec_strat.bind("", self._edge_detec_strategy_config_enter) + self.frame_edge_detec_strat.bind("", self._clear_statusbar) + + Label(self.frame_edge_detec_strat, text="Gaussian width (sigma)").grid(row=0, column=0, sticky=SW) + self.scale_sigma_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=5.0, resolution=0.01, orient=HORIZONTAL) + self.scale_sigma_skimage.grid(padx=(2, 0), row=0, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Lower hysteresis threshold").grid(row=1, column=0, sticky=SW) + self.scale_low_thres_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=10.0, resolution=0.01, orient=HORIZONTAL) + self.scale_low_thres_skimage.grid(padx=(2, 0), row=1, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Higher hysteresis threshold").grid(row=2, column=0, sticky=SW) + self.scale_high_thres_skimage = Scale(self.frame_edge_detec_strat, from_=0.0, to=10.0, resolution=0.01, orient=HORIZONTAL) + self.scale_high_thres_skimage.grid(padx=(2, 0), row=2, column=1, sticky=E) + + Label(self.frame_edge_detec_strat, text="Confidence factor").grid(row=3, column=0, sticky=SW) + self.scale_conf_lvl_skimage = Scale(self.frame_edge_detec_strat, from_=0.75, to=1.0, resolution=0.01, orient=HORIZONTAL) + self.scale_conf_lvl_skimage.grid(padx=(2, 0), row=3, column=1, sticky=E) + + self.btn_run_skimage = Button(frame_computation, text='Detect reference image', command=self.controller.on_click_run_edge_detec_strategy) + self.btn_run_skimage.grid(row=1, column=1, padx=2, sticky=W) + self.btn_edge_detec_debugger = Button(frame_computation, text='Edge detection debugger', command=self.controller.on_click_plot_results_skimage) + self.btn_edge_detec_debugger.grid(row=2, column=1, padx=2, sticky=W) + + def _frame_results(self): + # ************ Frame Results ************ # + frame_results = LabelFrame(self.frame_main, text="Results", padx=2, pady=2) + frame_results.pack(side=TOP, padx=2, pady=2, fill=X) + + frame_results_details = Frame(frame_results) + frame_results_details.pack(side=TOP, fill=X) + + Label(frame_results_details, text="Matches found / (Max peak value):").grid(pady=(2, 0), row=0, column=0, sticky=W) + self.matches_found = StringVar(frame_results_details) + self.label_matches_found = Label(frame_results_details, textvariable=self.matches_found) + self.label_matches_found.grid(pady=(2, 0), row=0, column=1, sticky=W) + + frame_snippet = Frame(frame_results) + frame_snippet.pack(side=TOP, fill=X) + Label(frame_snippet, text="Keyword to use this strategy:").pack(pady=(2, 0), side=TOP, anchor=W) + self.set_strategy_snippet = StringVar(frame_snippet) + self.label_strategy_snippet = Entry(frame_snippet, textvariable=self.set_strategy_snippet, width=75) + self.label_strategy_snippet.pack(pady=(0, 0), side=LEFT, anchor=W) + self.btn_copy_strategy_snippet = Button(frame_snippet, text='Copy', command=self.controller.copy_to_clipboard) + self.btn_copy_strategy_snippet.pack(side=RIGHT, padx=(2, 0), anchor=E) + + def _frame_image_viewer(self): + # ************ Image viewer ************ # + self.frame_image_viewer = LabelFrame(self.frame_main, text="Image viewer", padx=2, pady=2) + self.frame_image_viewer.pack(side=TOP, padx=2, pady=2, fill=X) + + self.canvas_desktop_img = Canvas(self.frame_image_viewer, width=384, height=216) + self.canvas_desktop_img.grid(row=3, column=0, sticky="WSEN") + self.desktop_img = self.canvas_desktop_img.create_image(384/2, 216/2, image=self.haystack_img_blank) + self.canvas_desktop_img.bind("", self._img_viewer_click) + self.canvas_desktop_img.bind("", self._img_viewer_hover_enter) + self.canvas_desktop_img.bind("", self._clear_statusbar) + + self.canvas_ref_img = Canvas(self.frame_image_viewer, width=1, height=1) + self.canvas_ref_img.grid(row=3, column=1, sticky="WSEN") + self.ref_img = self.canvas_ref_img.create_image(47.5, 216/2, image=self.needle_img_blank) + + Label(self.frame_image_viewer, text="Desktop").grid(pady=(0, 0), row=4, column=0, sticky="WSEN") + Label(self.frame_image_viewer, text="Reference Image").grid(pady=(0, 0), row=4, column=1, sticky="WSEN") + + + # ************* Status bar *************** # + def _status_bar(self): + self.frame_statusBar = Frame(self) + self.frame_statusBar.pack(side=TOP, fill=X, expand=True) + self.hint_msg = StringVar() + self.label_statusBar = Label(self.frame_statusBar, textvariable=self.hint_msg, bd=1, relief=SUNKEN, anchor=W) + self.label_statusBar.pack(side=BOTTOM, fill=X, expand=True) + + + # ************** Bindings *************** # + + def _img_viewer_click(self, event=None): + if (self.processing_done): + self.haystack_img = self.image_container.get_haystack_image_orig_size(format=ImageFormat.IMAGETK) + img_viewer_window = Toplevel(self) + img_viewer_window.title("Image Viewer") + label_img_viewer = Label(img_viewer_window, image=self.haystack_img) + label_img_viewer.pack() + + # **** Bindings for hints (statusbar) **** # + + def _img_viewer_hover_enter(self, event=None): + if (self.processing_done): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Click to view in full screen") + + def _default_strategy_config_enter(self, event=None): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Configure default strategy (default) parameters") + + def _edge_detec_strategy_config_enter(self, event=None): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Configure edge detection strategy (edge) parameters") + + def _threshold_error(self): + self.label_statusBar.config(fg="RED") + self.hint_msg.set("Higher threshold value must be greater than the lower threshold value!") + + def _ready(self): + self.label_statusBar.config(fg="BLACK") + self.hint_msg.set("Ready") + + def _clear_statusbar(self, event=None): + self.hint_msg.set("") diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py new file mode 100644 index 0000000..3de4012 --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/image_manipulation.py @@ -0,0 +1,58 @@ +from PIL import Image, ImageTk +from enum import Enum, unique +import numpy as np + + +@unique +class ImageFormat(Enum): + PILIMG = 0 + NUMPYARRAY = 1 + IMAGETK = 2 # ImageTk PhotoImage + PATHSTR = 3 + +class ImageContainer: + """Store and retrieve haystack and needle images of different formats.""" + def save_to_img_container(self, img, is_haystack_img=False): + """Save haystack and needle images. + + Args: + - ``img (str/PIL.Image)``: Path (str) or object (PIL.Image) to haystack/needle image. + - ``is_haystack_img (bool, optional)``: If to be saved img is a haystack image. + """ + if isinstance(img, str): + _PIL_img = Image.open(img) + else: + _PIL_img = img + + if is_haystack_img: + self._haystack_image_orig_size = _PIL_img + self._haystack_image = _PIL_img.resize((384, 216), Image.ANTIALIAS) + else: + self._needle_image = {'Path': img, 'Obj': _PIL_img} + + def get_haystack_image(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._haystack_image + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._haystack_image) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._haystack_image) + + def get_haystack_image_orig_size(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._haystack_image_orig_size + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._haystack_image_orig_size) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._haystack_image_orig_size) + + def get_needle_image(self, format: ImageFormat): + if format == ImageFormat.PILIMG: + return self._needle_image['Obj'] + elif format == ImageFormat.PATHSTR: + return self._needle_image['Path'] + elif format == ImageFormat.NUMPYARRAY: + return np.array(self._needle_image['Obj']) + elif format == ImageFormat.IMAGETK: + return ImageTk.PhotoImage(self._needle_image['Obj']) + diff --git a/src/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py b/src/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py new file mode 100644 index 0000000..ea052db --- /dev/null +++ b/src/ImageHorizonLibrary/recognition/ImageDebugger/template_matching_strategies.py @@ -0,0 +1,44 @@ +from PIL import ImageDraw +from abc import ABC, abstractmethod +from .image_manipulation import ImageFormat, ImageContainer + + +class TemplateMatchingStrategy(ABC): + @abstractmethod + def find_num_of_matches(self) -> int: pass + + # Match place is highlighted with a red rectangle + def highlight_matches(self): + haystack_img = self.image_container.get_haystack_image_orig_size(ImageFormat.PILIMG) + draw = ImageDraw.Draw(haystack_img, "RGBA") + + for loc in self.coord: + draw.rectangle([loc[0], loc[1], loc[0]+loc[2], loc[1]+loc[3]], fill=(256, 0, 0, 127)) + draw.rectangle([loc[0], loc[1], loc[0]+loc[2], loc[1]+loc[3]], outline=(256, 0, 0, 127), width=3) + + self.image_container.save_to_img_container(img=haystack_img, is_haystack_img=True) + +class Pyautogui(TemplateMatchingStrategy): + def __init__(self, image_container: ImageContainer, image_horizon_instance): + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + + def find_num_of_matches(self): + self.coord = list(self.image_horizon_instance._locate_all( + self.image_container.get_needle_image(ImageFormat.PATHSTR), + self.image_container.get_haystack_image_orig_size(ImageFormat.PILIMG) + )) + return self.coord + + +class Skimage(TemplateMatchingStrategy): + def __init__(self, image_container: ImageContainer, image_horizon_instance): + self.image_container = image_container + self.image_horizon_instance = image_horizon_instance + + def find_num_of_matches(self): + self.coord = list(self.image_horizon_instance._locate_all( + self.image_container.get_needle_image(ImageFormat.PATHSTR), + self.image_container.get_haystack_image_orig_size(ImageFormat.NUMPYARRAY) + )) + return self.coord diff --git a/src/ImageHorizonLibrary/recognition/__init__.py b/src/ImageHorizonLibrary/recognition/__init__.py index a894318..8f078d8 100644 --- a/src/ImageHorizonLibrary/recognition/__init__.py +++ b/src/ImageHorizonLibrary/recognition/__init__.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- -from ._recognize_images import _RecognizeImages +from ._recognize_images import _RecognizeImages, _StrategyPyautogui, _StrategySkimage from ._screenshot import _Screenshot __all__ = [ '_RecognizeImages', + '_StrategyPyautogui', + '_StrategySkimage', '_Screenshot' ] diff --git a/src/ImageHorizonLibrary/recognition/_recognize_images.py b/src/ImageHorizonLibrary/recognition/_recognize_images.py index b2ed9d5..d6aea0d 100644 --- a/src/ImageHorizonLibrary/recognition/_recognize_images.py +++ b/src/ImageHorizonLibrary/recognition/_recognize_images.py @@ -7,12 +7,19 @@ import pyautogui as ag from robot.api import logger as LOGGER +from skimage.feature import match_template, peak_local_max, canny +from skimage.color import rgb2gray +from skimage.io import imread, imsave + +import numpy as np + + from ..errors import ImageNotFoundException, InvalidImageException from ..errors import ReferenceFolderException - + class _RecognizeImages(object): - def __normalize(self, path): + def _normalize(self, path): if (not self.reference_folder or not isinstance(self.reference_folder, str) or not isdir(self.reference_folder)): @@ -143,71 +150,72 @@ def _suppress_keyword_on_failure(self): yield None self.keyword_on_failure = keyword - def _locate(self, reference_image, log_it=True): + def _get_reference_images(self, reference_image): + '''Return an absolute path for the given reference imge. + Return as a list of those if reference_image is a folder. + ''' is_dir = False try: - if isdir(self.__normalize(reference_image)): + if isdir(self._normalize(reference_image)): is_dir = True except InvalidImageException: pass is_file = False try: - if isfile(self.__normalize(reference_image)): + if isfile(self._normalize(reference_image)): is_file = True except InvalidImageException: pass - reference_image = self.__normalize(reference_image) + reference_image = self._normalize(reference_image) reference_images = [] if is_file: reference_images = [reference_image] elif is_dir: - for f in listdir(self.__normalize(reference_image)): - if not isfile(self.__normalize(path_join(reference_image, f))): + for f in listdir(self._normalize(reference_image)): + if not isfile(self._normalize(path_join(reference_image, f))): raise InvalidImageException( - self.__normalize(reference_image)) + self._normalize(reference_image)) reference_images.append(path_join(reference_image, f)) + return reference_images - def try_locate(ref_image): - location = None - with self._suppress_keyword_on_failure(): - try: - if self.has_cv and self.confidence: - location = ag.locateOnScreen(ref_image, - confidence=self.confidence) - else: - if self.confidence: - LOGGER.warn("Can't set confidence because you don't " - "have OpenCV (python-opencv) installed " - "or a confidence level was not given.") - location = ag.locateOnScreen(ref_image) - except ImageNotFoundException as ex: - LOGGER.info(ex) - pass - return location + def _locate(self, reference_image, log_it=True): + reference_images = self._get_reference_images(reference_image) location = None for ref_image in reference_images: - location = try_locate(ref_image) + location = self._try_locate(ref_image) if location != None: break if location is None: if log_it: LOGGER.info('Image "%s" was not found ' - 'on screen.' % reference_image) + 'on screen. (strategy: %s)' % (reference_image, self.strategy)) self._run_on_failure() raise ImageNotFoundException(reference_image) - if log_it: - LOGGER.info('Image "%s" found at %r' % (reference_image, location)) + center_point = ag.center(location) x = center_point.x y = center_point.y if self.has_retina: x = x / 2 y = y / 2 + if log_it: + LOGGER.info('Image "%s" found at %r (strategy: %s)' % (reference_image, (x,y), self.strategy)) return (x, y) + def _locate_all(self, reference_image, haystack_image=None): + '''Tries to locate all occurrences of the reference image on the screen + or on the haystack image, if given. + Returns a list of location tuples (finds 0..n)''' + reference_images = self._get_reference_images(reference_image) + if len(reference_images) > 1: + raise InvalidImageException( + f'Locating ALL occurences of MANY files ({", ".join(reference_images)}) is not supported.') + locations = self._try_locate(reference_images[0], locate_all=True, haystack_image=haystack_image) + return locations + def does_exist(self, reference_image): '''Returns ``True`` if reference image was found on screen or ``False`` otherwise. Never fails. @@ -216,7 +224,7 @@ def does_exist(self, reference_image): ''' with self._suppress_keyword_on_failure(): try: - return bool(self._locate(reference_image, log_it=False)) + return bool(self._locate(reference_image, log_it=True)) except ImageNotFoundException: return False @@ -235,7 +243,7 @@ def wait_for(self, reference_image, timeout=10): Fail if the image is not found on the screen after ``timeout`` has expired. - See `Reference image names` for documentation for ``reference_image``. + See `Reference images` for further documentation. ``timeout`` is given in seconds. @@ -246,12 +254,178 @@ def wait_for(self, reference_image, timeout=10): with self._suppress_keyword_on_failure(): while time() < stop_time: try: - location = self._locate(reference_image, log_it=False) + location = self._locate(reference_image, log_it=True) break except ImageNotFoundException: pass if location is None: self._run_on_failure() - raise ImageNotFoundException(self.__normalize(reference_image)) + raise ImageNotFoundException(self._normalize(reference_image)) LOGGER.info('Image "%s" found at %r' % (reference_image, location)) return location + + def debug_image(self): + '''Halts the test execution and opens the image debugger UI. + + Whenever you encounter problems with the recognition accuracy of a reference image, + you should place this keyword just before the line in question. Example: + + | Debug Image + | Wait For hard_to_find_button + + The test will halt at this position and open the debugger UI. Use it as follows: + + - Select the reference image (`hard_to_find_button`) + - Click the button "Detect reference image" for the strategy you want to test (default/edge). The GUI hides itself while it takes the screenshot of the current application. + - The Image Viewer at the botton shows the screenshot with all regions where the reference image was found. + - "Matches Found": More than one match means that either `conficence` is set too low or that the reference image is visible multiple times. If the latter is the case, you should first detect a unique UI element and use relative keywords like `Click To The Right Of`. + - "Max peak value" (only `edge`) gives feedback about the detection accuracy of the best match and is measured as a float number between 0 and 1. A peak value above _confidence_ results in a match. + - "Edge detection debugger" (only `edge`) opens another window where both the reference and screenshot images are shown before and after the edge detection and is very helpful to loearn how the sigma and low/high threshold parameters lead to different results. + - The field "Keyword to use this strategy" shows how to set the strategy to the current settings. Just copy the line and paste it into the test: + + | Set Strategy edge edge_sigma=2.0 edge_low_threshold=0.1 edge_high_threshold=0.3 + | Wait For hard_to_find_button + + The purpose of this keyword is *solely for debugging purposes*; don't + use it in production!''' + from .ImageDebugger import ImageDebugger + debug_app = ImageDebugger(self) + + +class _StrategyPyautogui(): + + def __init__(self, image_horizon_instance): + self.ih_instance = image_horizon_instance + + def _try_locate(self, ref_image, haystack_image=None, locate_all=False): + '''Tries to locate the reference image on the screen or the haystack_image. + Return values: + - locate_all=False: None or 1 location tuple (finds max 1) + - locate_all=True: None or list of location tuples (finds 0..n) + (GUI Debugger mode)''' + + ih = self.ih_instance + location = None + if haystack_image is None: + haystack_image = ag.screenshot() + + if locate_all: + locate_func = ag.locateAll + else: + locate_func = ag.locate #Copy below,take screenshots + + with ih._suppress_keyword_on_failure(): + try: + if ih.has_cv and ih.confidence: + location_res = locate_func(ref_image, + haystack_image, + confidence=ih.confidence) + else: + if ih.confidence: + LOGGER.warn("Can't set confidence because you don't " + "have OpenCV (python-opencv) installed " + "or a confidence level was not given.") + location_res = locate_func(ref_image, haystack_image) + except ImageNotFoundException as ex: + LOGGER.info(ex) + pass + if locate_all: + # convert the generator fo Box objects to a list of tuples + location = [tuple(box) for box in location_res] + else: + # Single Box + location = location_res + return location + + + +class _StrategySkimage(): + _SKIMAGE_DEFAULT_CONFIDENCE = 0.99 + + def __init__(self, image_horizon_instance): + self.ih_instance = image_horizon_instance + + def _try_locate(self, ref_image, haystack_image=None, locate_all=False): + '''Tries to locate the reference image on the screen or the provided haystack_image. + Return values: + - locate_all=False: None or 1 location tuple (finds max 1) + - locate_all=True: None or list of location tuples (finds 0..n) + (GUI Debugger mode)''' + + ih = self.ih_instance + confidence = ih.confidence or self._SKIMAGE_DEFAULT_CONFIDENCE + with ih._suppress_keyword_on_failure(): + needle_img = imread(ref_image, as_gray=True) + needle_img_name = ref_image.split("\\")[-1].split(".")[0] + #haystack_img_height, needle_img_width = needle_img.shape + needle_img_height, needle_img_width = needle_img.shape + if haystack_image is None: + haystack_img_gray = rgb2gray(np.array(ag.screenshot())) + else: + haystack_img_gray = rgb2gray(haystack_image) + + # Canny edge detection on both images + ih.needle_edge = self.detect_edges(needle_img) + ih.haystack_edge = self.detect_edges(haystack_img_gray) + + # peakmap is a "heatmap" of matching coordinates + ih.peakmap = match_template(ih.haystack_edge, ih.needle_edge, pad_input=True) + + # For debugging purposes + debug = False + if debug: + imsave(needle_img_name + "needle.png", needle_img) + imsave(needle_img_name + "needle_edge.png", ih.needle_edge) + imsave(needle_img_name + "haystack.png", haystack_img_gray) + imsave(needle_img_name + "haystack_edge.png", ih.haystack_edge) + imsave(needle_img_name + "peakmap.png", ih.peakmap) + + if locate_all: + # https://stackoverflow.com/questions/48732991/search-for-all-templates-using-scikit-image + peaks = peak_local_max(ih.peakmap,threshold_rel=confidence) + peak_coords = zip(peaks[:,1], peaks[:,0]) + location = [] + for i, pk in enumerate(peak_coords): + x = pk[0] + y = pk[1] + # higest peak level + peak = ih.peakmap[y][x] + if peak > confidence: + loc = (x-needle_img_width/2, y-needle_img_height/2, needle_img_width, needle_img_height) + location.append(loc) + + else: + # translate highest index in peakmap from linear (memory) into + # an index of a matrix with the peakmaps dimensions + ij = np.unravel_index(np.argmax(ih.peakmap), ih.peakmap.shape) + # Extract coordinates of the highest peak; xy is the coordinate + # where the CENTER of the reference image matched. + x, y = ij[::-1] + # higest peak level + peak = ih.peakmap[y][x] + if peak > confidence: + # tuple of xy (topleft) and width/height + location = (x-needle_img_width/2, y-needle_img_height/2, needle_img_width, needle_img_height) + else: + location = None + # TODO: Also return peak level + return location + + def _detect_edges(self, img, sigma, low, high): + edge_img = canny( + image=img, + sigma=sigma, + low_threshold=low, + high_threshold=high, + ) + return edge_img + + def detect_edges(self, img): + '''Apply edge detection on a given image''' + return self._detect_edges( + img, + self.ih_instance.edge_sigma, + self.ih_instance.edge_low_threshold, + self.ih_instance.edge_low_threshold + ) + diff --git a/src/ImageHorizonLibrary/utils.py b/src/ImageHorizonLibrary/utils.py index 556547d..e312ff6 100644 --- a/src/ImageHorizonLibrary/utils.py +++ b/src/ImageHorizonLibrary/utils.py @@ -35,3 +35,11 @@ def has_cv(): except ModuleNotFoundError as err: has_cv = False return has_cv + +def has_skimage(): + has_skimage = True + try: + import skimage + except ModuleNotFoundError as err: + has_skimage = False + return has_skimage diff --git a/src/ImageHorizonLibrary/version.py b/src/ImageHorizonLibrary/version.py index 583e59c..c2635d7 100644 --- a/src/ImageHorizonLibrary/version.py +++ b/src/ImageHorizonLibrary/version.py @@ -1,2 +1,2 @@ # -*- coding: utf-8 -*- -VERSION = '1.1-devel' +VERSION = '1.1' diff --git a/test_listener.py b/test_listener.py new file mode 100644 index 0000000..2288d17 --- /dev/null +++ b/test_listener.py @@ -0,0 +1,7 @@ +from robot.api import TestSuite + +suite = TestSuite() +suite.resource.imports.library('ImageHorizonLibrary', args=['reference_folder=C:/Users/simon_meggle/Documents/images']) +test = suite.tests.create('Image Debugger Test') +test.body.create_keyword('Debug Image') +result = suite.run(report=None,log=None) \ No newline at end of file diff --git a/test_prototype.py b/test_prototype.py new file mode 100644 index 0000000..ccd0780 --- /dev/null +++ b/test_prototype.py @@ -0,0 +1,143 @@ +import numpy as np +import matplotlib.pyplot as plt + +from skimage import data +from skimage.feature import match_template +from skimage.feature import peak_local_max +import skimage.viewer +from skimage.color import rgb2gray +import pyautogui as ag +import sys + +screen = data.coins() +coin = screen[170:220, 75:130] + + + +#coin = skimage.io.imread('..\\..\\images\\text.png', as_gray=True) +#coin = skimage.io.imread('..\\..\\images\\win_changed.png', as_gray=True) + +#skimage.viewer.ImageViewer(image).show() +# --- +# result = match_template(image, coin,pad_input=True) #added the pad_input bool +# peak_min_distance = min(coin.shape) +# peak_min_distance = 1 + +# peaks = peak_local_max(result,min_distance=peak_min_distance,threshold_abs=1) +# # produce a plot equivalent to the one in the docs +# plt.imshow(result) +# # highlight matched regions (plural) +# plt.plot(peaks[:,1], peaks[:,0], 'o', markeredgecolor='r', markerfacecolor='none', markersize=10) +# plt.show() +# --- + + +def detect_edges(img, sigma, low, high): + edge_img = skimage.feature.canny( + image=img, + sigma=sigma, + low_threshold=low, + high_threshold=high, + ) + return edge_img + + +def plot_result(what, where, what_edges, where_edges, peakmap, title, locations): + + fig, axs = plt.subplots(2, 3) + sp_what = axs[0, 0] + sp_where = axs[0, 1] + sp_invisible = axs[0, 2] + sp_what_edges = axs[1, 0] + sp_where_edges = axs[1, 1] + sp_peakmap = axs[1, 2] + + # ROW 1 ================================ + sp_what.imshow(what, cmap=plt.cm.gray) + sp_what.set_title('what') + sp_where.imshow(where, cmap=plt.cm.gray) + + sp_where.set_title('where') + sp_where.sharex(sp_where_edges) + sp_where.sharey(sp_where_edges) + + sp_invisible.set_visible(False) + + # ROW 2 ================================ + sp_what_edges.imshow(what_edges, cmap=plt.cm.gray) + sp_what_edges.set_title('what_edge') + + sp_where_edges.imshow(where_edges, cmap=plt.cm.gray) + sp_where_edges.set_title('where_edge') + + sp_peakmap.imshow(peakmap) + sp_peakmap.set_title('peakmap') + sp_peakmap.sharex(sp_where_edges) + sp_peakmap.sharey(sp_where_edges) + + for loc in locations: + # highlight matched region + #hwhat, wwhat = what_edges.shape + # TODO: + #rect = plt.Rectangle((x_peak-int(wwhat/2), y_peak-int(hwhat/2)), wwhat, hwhat, edgecolor='r', facecolor='none') + rect = plt.Rectangle((loc[0], loc[1]), loc[2], loc[3], edgecolor='r', facecolor='none') + sp_where_edges.add_patch(rect) + + + sp_peakmap.autoscale(False) + fig.suptitle(title, fontsize=14, fontweight='bold') + + plt.show() + pass + + + + + +def try_locate(what_name, sigma=2.0, low=0.1, high=0.3, confidence=0.99, locate_all=False): + locations = None + what = skimage.io.imread('..\\..\\images\\' + what_name, as_gray=True) + what_h, what_w = what.shape + where = rgb2gray(np.array(ag.screenshot())) + what_edge = detect_edges(what, sigma, low, high) + where_edges = detect_edges(where, sigma, low, high) + peakmap = match_template(where_edges, what_edge) + + if locate_all: + # https://stackoverflow.com/questions/48732991/search-for-all-templates-using-scikit-image + # peaks = peak_local_max(peakmap) + peaks = peak_local_max(peakmap,threshold_rel=confidence) + peak_coords = zip(peaks[:,1], peaks[:,0]) + locations = [] + for i, pk in enumerate(peak_coords): + locations.append((pk[0], pk[1], what_w, what_h)) + pass + title = f"{len(locations)} matches with confidence level > {confidence}" + else: + ij = np.unravel_index(np.argmax(peakmap), peakmap.shape) + x, y = ij[::-1] + peak = peakmap[y][x] + if peak > confidence: + locations = [(x, y, what_w, what_h)] + matched = peak > confidence + title = f"Match = {str(matched)} (peak at {peak} > {confidence})" + +# plot_result(what, where, peakmap, title, x,y) + plot_result(what, where, what_edge, where_edges, peakmap, title, locations) + + +#image = skimage.io.imread('..\\..\\images\\screen.png', as_gray=True) + +#try_locate('ok.png', sigma=1.4, confidence=0.8, locate_all=True) + +try_locate('3.png', locate_all=True) +#try_locate('3.png', sigma=1.0, confidence=0.8, locate_all=True) +#compare('win.png') +#cProfile.run('compare()') + +# confidence: 0.9999999999999 +# win auf screen: 0.9999999999999992 +# win auf screenshot: 0.9999999999999974 +# win_ch auf screen: 0.9999999419440094 +# win_ch auf screenshot: 0.9999999419440035 + diff --git a/tests/utest/test_recognize_images.py b/tests/utest/test_recognize_images.py index 182d015..1d81b92 100644 --- a/tests/utest/test_recognize_images.py +++ b/tests/utest/test_recognize_images.py @@ -3,7 +3,7 @@ from unittest import TestCase from os.path import abspath, dirname, join as path_join -from mock import call, MagicMock, patch +from mock import call, MagicMock, patch, ANY CURDIR = abspath(dirname(__file__)) TESTIMG_DIR = path_join(CURDIR, 'reference_images') @@ -28,7 +28,8 @@ def test_find_with_confidence(self): self.lib.has_cv = True self.lib.locate('mY_PiCtURe') expected_path = path_join(CURDIR, 'symbolic_link', 'my_picture.png') - self.mock.locateOnScreen.assert_called_once_with(expected_path, confidence=0.5) + # haystack image can be anything + self.mock.locate.assert_called_once_with(expected_path, ANY, confidence=0.5) self.mock.reset_mock() def test_find_with_confidence_no_opencv(self): @@ -37,7 +38,7 @@ def test_find_with_confidence_no_opencv(self): self.lib.has_cv = False self.lib.locate('mY_PiCtURe') expected_path = path_join(CURDIR, 'symbolic_link', 'my_picture.png') - self.mock.locateOnScreen.assert_called_once_with(expected_path) + self.mock.locate.assert_called_once_with(expected_path, ANY) self.mock.reset_mock() def test_click_image(self): @@ -115,7 +116,7 @@ def test_wait_for_negative_path(self): def _verify_path_works(self, image_name, expected): self.lib.locate(image_name) expected_path = path_join(TESTIMG_DIR, expected) - self.mock.locateOnScreen.assert_called_once_with(expected_path) + self.mock.locate.assert_called_once_with(expected_path, ANY) self.mock.reset_mock() def test_locate(self): @@ -125,7 +126,7 @@ def test_locate(self): 'mY_PiCtURe'): self._verify_path_works(image_name, 'my_picture.png') - self.mock.locateOnScreen.return_value = None + self.mock.locate.return_value = None run_on_failure = MagicMock() with self.assertRaises(InvalidImageException), \ patch.object(self.lib, '_run_on_failure', run_on_failure): @@ -143,14 +144,14 @@ def test_locate_with_valid_reference_folder(self): self.lib.reference_folder = path_join(CURDIR, 'symbolic_link') self.lib.locate('mY_PiCtURe') expected_path = path_join(CURDIR, 'symbolic_link', 'my_picture.png') - self.mock.locateOnScreen.assert_called_once_with(expected_path) + self.mock.locate.assert_called_once_with(expected_path, ANY) self.mock.reset_mock() self.lib.reference_folder = path_join(CURDIR, 'rëförence_imägës') self.lib.locate('mŸ PäKSÖR') expected_path = path_join(CURDIR, 'rëförence_imägës', 'mÿ_päksör.png') - self.mock.locateOnScreen.assert_called_once_with(expected_path) + self.mock.locate.assert_called_once_with(expected_path, ANY) self.mock.reset_mock() def test_locate_with_invalid_reference_folder(self):