From a250137ae77ff9931b4f31ee5f3b03d8a8ad197f Mon Sep 17 00:00:00 2001 From: Vanessa Sochat Date: Tue, 7 May 2019 18:17:54 -0400 Subject: [PATCH] updating watcher task functions to be cleaner, inspired by other PR Signed-off-by: Vanessa Sochat --- .github/CODE_OF_CONDUCT.md | 59 ++++++++++++++ .github/CONTRIBUTING.md | 87 ++------------------ .gitignore | 1 + MANIFEST.in | 4 +- docs/_docs/getting-started/index.md | 7 +- docs/_docs/install/index.md | 26 +++++- docs/_docs/watcher-tasks/psutils.md | 2 +- docs/_docs/watcher-tasks/urls.md | 2 +- setup.py | 37 +++++---- watchme/client/__init__.py | 8 +- watchme/client/add.py | 4 +- watchme/client/create.py | 1 - watchme/client/export.py | 4 +- watchme/client/ls.py | 21 +++-- watchme/command/__init__.py | 3 +- watchme/command/create.py | 1 + watchme/command/utils.py | 13 ++- watchme/tasks/__init__.py | 69 ++++++++++++++++ watchme/version.py | 12 ++- watchme/watchers/README.md | 8 +- watchme/watchers/__init__.py | 121 +++++++++------------------- watchme/watchers/settings.py | 34 +++++++- watchme/watchers/urls/helpers.py | 1 + 23 files changed, 315 insertions(+), 210 deletions(-) create mode 100644 .github/CODE_OF_CONDUCT.md diff --git a/.github/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..a366695 --- /dev/null +++ b/.github/CODE_OF_CONDUCT.md @@ -0,0 +1,59 @@ +# WatchMe Code of Conduct v1.0 + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + + * Using welcoming and inclusive language + * Being respectful of differing viewpoints and experiences + * Gracefully accepting constructive criticism + * Focusing on what is best for the community + * Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + + * The use of sexualized language or imagery and unwelcome sexual attention or advances + * Trolling, insulting/derogatory comments, and personal or political attacks + * Public or private harassment + * Publishing others' private information, such as a physical or electronic address, without explicit permission + * Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting @vsoch directly. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. @vsoch is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +## Thanks + +This code of conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org/), version 1.4, +available at http://contributor-covenant.org/version/1/4. diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 0303edd..feac720 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -2,14 +2,12 @@ This code is licensed under the MPL 2.0 [LICENSE](LICENSE). # Contributing -When contributing to the SIF Python Client, it is important to properly communicate the +When contributing to the WatchMe Python Client, it is important to properly communicate the gist of the contribution. If it is a simple code or editorial fix, simply explaining this within the GitHub Pull Request (PR) will suffice. But if this is a larger fix or Enhancement, it should be first discussed with the project -leader or developers. - -Please note we have a code of conduct, described below. Please follow it in -all your interactions with the project members and users. +leader or developers. Please also note that we have a [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) +that should be followed for all interactions with the project members and users. ## Pull Request Process @@ -33,78 +31,7 @@ all your interactions with the project members and users. done by the project lead, @vsoch (or approved by her). -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, gender identity and expression, level of experience, -nationality, personal appearance, race, religion, or sexual identity and -orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -### Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project leader (@vsoch). All -complaints will be reviewed and investigated and will result in a response -that is deemed necessary and appropriate to the circumstances. The project -team is obligated to maintain confidentiality with regard to the reporter of -an incident. Further details of specific enforcement policies may be posted -separately. - -Project maintainers, contributors and users who do not follow or enforce the -Code of Conduct in good faith may face temporary or permanent repercussions -with their involvement in the project as determined by the project's leader(s). - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at [http://contributor-covenant.org/version/1/4][version] - -[homepage]: http://contributor-covenant.org -[version]: http://contributor-covenant.org/version/1/4/ +When you contribute to the project, you agree to add code under the provided +licensing terms, and also we ask that you add your name to the [AUTHORS](AUTHORS.md) +file. Any contribution in the way of documentation, features, or bug fixes is +greatly appreciated. diff --git a/.gitignore b/.gitignore index f4cbdd5..86cd0ef 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ watchme.egg-info/ dist build _site +.eggs __pycache__ diff --git a/MANIFEST.in b/MANIFEST.in index a92bced..574fc21 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,5 +3,5 @@ recursive-include watchme * recursive-exclude * __pycache__ recursive-exclude * *.pyc recursive-exclude * *.pyo -recursive-exclude .docs -recursive-exclude docs +recursive-exclude .docs * +recursive-exclude docs * diff --git a/docs/_docs/getting-started/index.md b/docs/_docs/getting-started/index.md index 4fa7cd5..3f2fce9 100644 --- a/docs/_docs/getting-started/index.md +++ b/docs/_docs/getting-started/index.md @@ -172,7 +172,7 @@ The configuration commands will vary based on the kind of task you want to add, and here is a quick example of adding a task to watch a url (the default task): ```bash -$ watchme add watcher task-singularity-release url@https://github.com/sylabs/singularity/releases +$ watchme add-task watcher task-singularity-release url@https://github.com/sylabs/singularity/releases [task-singularity-release] url = https://github.com/sylabs/singularity/releases active = true @@ -207,7 +207,7 @@ The task is active by default (after you set up its schedule) and you can disabl this with --active false: ```bash -$ watchme add watcher task-singularity-release url@https://github.com/sylabs/singularity/releases --active false +$ watchme add-task watcher task-singularity-release url@https://github.com/sylabs/singularity/releases --active false ``` The reason we save these parameters in the repo is that if you put it under version @@ -639,5 +639,4 @@ $ watchme export system task-cpu vanessa-thinkpad-t460s_vanessa.json --json ## Licenses -This code is licensed under the Affero GPL, version 3.0 or later [LICENSE](LICENSE). -The SIF Header format is licesed by [Sylabs](https://github.com/sylabs/sif/blob/master/pkg/sif/sif.go). +This code is licensed under the Mozilla, version 2.0 or later [LICENSE](LICENSE). diff --git a/docs/_docs/install/index.md b/docs/_docs/install/index.md index b72f63b..2c44741 100644 --- a/docs/_docs/install/index.md +++ b/docs/_docs/install/index.md @@ -10,7 +10,8 @@ order: 1 The only dependency for watchme is to have [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [crontab](https://www.digitalocean.com/community/tutorials/how-to-use-cron-to-automate-tasks-on-a-vps) on your system. Git is used for version control of the pages you are watching, and crontab is -used for scheduling your watches. +used for scheduling your watches. If you want to install a custom watcher type, +see [installing extras](#installing-extras) below. ## Install @@ -62,3 +63,26 @@ actions: If you have any questions or issues, please [open an issue]({{ site.repo }}/issues). + +## Installing Extras + +If you want to install all of watchme's exporters and watchers: + +```bash +$ pip install watchme[all] +``` + +To install all watchers only: + +```bash +$ pip install watchme[watchers] +``` + +or a specific watcher task group: + +```bash +$ pip install watchme[watcher-urls-dynamic] +$ pip install watchme[watcher-psutils] +``` + +To see all of the choices, see [here](https://github.com/vsoch/watchme/blob/master/setup.py#L109) in the setup file. diff --git a/docs/_docs/watcher-tasks/psutils.md b/docs/_docs/watcher-tasks/psutils.md index 7bcadbe..0fe9ef4 100644 --- a/docs/_docs/watcher-tasks/psutils.md +++ b/docs/_docs/watcher-tasks/psutils.md @@ -11,7 +11,7 @@ basic python environment. If your python installation doesn't have the `psutil` module, install as follows: ```bash -pip install watchme[psutils] +pip install watchme[watcher-psutils] ``` Next, create a watcher for your tasks to live under: diff --git a/docs/_docs/watcher-tasks/urls.md b/docs/_docs/watcher-tasks/urls.md index 2cf607f..c7daf30 100644 --- a/docs/_docs/watcher-tasks/urls.md +++ b/docs/_docs/watcher-tasks/urls.md @@ -194,7 +194,7 @@ identified by a class or id) on a page. For this purpose, you can use the functi packages to do this: ```bash -$ pip install watchme[urls-dynamic] +$ pip install watchme[watcher-urls-dynamic] ``` This task will watch for changes based on a selection from a page. For example, diff --git a/setup.py b/setup.py index b45d663..40f514b 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,15 @@ from setuptools import setup, find_packages -import codecs import os ################################################################################ # HELPER FUNCTIONS ############################################################# ################################################################################ + def get_lookup(): - '''get version by way of singularity.version, returns a + '''get version by way of singularity.version, returns a lookup dictionary with several global variables without needing to import singularity ''' @@ -28,12 +28,13 @@ def get_lookup(): exec(filey.read(), lookup) return lookup + # Read in requirements def get_requirements(lookup=None, key="INSTALL_REQUIRES"): '''get_requirements reads in requirements and versions from the lookup obtained with get_lookup''' - if lookup == None: + if lookup is None: lookup = get_lookup() install_requires = [] @@ -41,19 +42,19 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): module_name = module[0] module_meta = module[1] if "exact_version" in module_meta: - dependency = "%s==%s" %(module_name,module_meta['exact_version']) + dependency = "%s==%s" % (module_name, module_meta['exact_version']) elif "min_version" in module_meta: - if module_meta['min_version'] == None: + min_version = module_meta['min_version'] + if min_version is None: dependency = module_name else: - dependency = "%s>=%s" %(module_name,module_meta['min_version']) + dependency = "%s>=%s" % (module_name, min_version) install_requires.append(dependency) return install_requires - # Make sure everything is relative to setup.py -install_path = os.path.dirname(os.path.abspath(__file__)) +install_path = os.path.dirname(os.path.abspath(__file__)) os.chdir(install_path) # Get version information from the lookup @@ -76,8 +77,13 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): if __name__ == "__main__": + # Install all exporters and/or watchers INSTALL_REQUIRES = get_requirements(lookup) - URLS_DYNAMIC = get_requirements(lookup,'INSTALL_URLS_DYNAMIC') + INSTALL_ALL = get_requirements(lookup, 'INSTALL_ALL') + WATCHERS = get_requirements(lookup, 'INSTALL_WATCHERS') + + # Watchers + URLS_DYNAMIC = get_requirements(lookup, 'INSTALL_URLS_DYNAMIC') PSUTILS = get_requirements(lookup, 'INSTALL_PSUTILS') setup(name=NAME, @@ -86,7 +92,7 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): author_email=AUTHOR_EMAIL, maintainer=AUTHOR, maintainer_email=AUTHOR_EMAIL, - packages=find_packages(), + packages=find_packages(), include_package_data=True, zip_safe=False, url=PACKAGE_URL, @@ -96,11 +102,12 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): keywords=KEYWORDS, setup_requires=["pytest-runner"], tests_require=["pytest"], - install_requires = INSTALL_REQUIRES, + install_requires=INSTALL_REQUIRES, extras_require={ - 'all': [INSTALL_REQUIRES], - 'urls-dynamic': [URLS_DYNAMIC], - 'psutils': [PSUTILS] + 'all': [INSTALL_ALL], + 'watchers': [WATCHERS], + 'watcher-urls-dynamic': [URLS_DYNAMIC], + 'watcher-psutils': [PSUTILS] }, classifiers=[ 'Intended Audience :: Science/Research', @@ -113,4 +120,4 @@ def get_requirements(lookup=None, key="INSTALL_REQUIRES"): 'Programming Language :: Python :: 3', ], - entry_points = {'console_scripts': [ 'watchme=watchme.client:main' ] }) + entry_points={'console_scripts': ['watchme=watchme.client:main']}) diff --git a/watchme/client/__init__.py b/watchme/client/__init__.py index ae926a8..c570206 100644 --- a/watchme/client/__init__.py +++ b/watchme/client/__init__.py @@ -112,7 +112,7 @@ def get_parser(): # add - add = subparsers.add_parser("add", + add = subparsers.add_parser("add-task", help="add a task to a watcher.") add.add_argument('watcher', nargs=1, @@ -151,6 +151,10 @@ def get_parser(): ls = subparsers.add_parser("list", help="list all watchers at a base") + ls.add_argument('--watchers', dest="watchers", + help="list watchers available", + default=False, action='store_true') + # protect and freeze protect = subparsers.add_parser("protect", @@ -305,7 +309,7 @@ def help(return_code=0): sys.exit(0) if args.command == "activate": from .activate import main - elif args.command == "add": from .add import main + elif args.command == "add-task": from .add import main elif args.command == "edit": from .edit import main elif args.command == "export": from .export import main elif args.command == "create": from .create import main diff --git a/watchme/client/add.py b/watchme/client/add.py index 0199534..6edd619 100644 --- a/watchme/client/add.py +++ b/watchme/client/add.py @@ -12,14 +12,14 @@ from watchme.logger import bot def main(args, extra): - '''activate one or more watchers + '''add a task for a watcher ''' # Required - will print help if not provided name = args.watcher[0] task = args.task[0] if not task.startswith('task'): - example = 'watchme add watcher task-cpu func@cpu_task type@psutils' + example = 'watchme add-task watcher task-cpu func@cpu_task type@psutils' bot.exit('Task name must start with "task", e.g., %s' % example) # Exit if the user doesn't provide any parameters diff --git a/watchme/client/create.py b/watchme/client/create.py index cf7e73b..3cec159 100644 --- a/watchme/client/create.py +++ b/watchme/client/create.py @@ -20,4 +20,3 @@ def main(args, extra): for watcher in watchers: create_watcher(watcher) - diff --git a/watchme/client/export.py b/watchme/client/export.py index 5d9b894..93418f8 100644 --- a/watchme/client/export.py +++ b/watchme/client/export.py @@ -15,7 +15,7 @@ import os def main(args, extra): - '''activate one or more watchers + '''export temporal data for a watcher ''' # Required - will print help if not provided name = args.watcher[0] @@ -23,7 +23,7 @@ def main(args, extra): filename = args.filename[0] if not task.startswith('task'): - example = 'watchme add watcher task-reddit url@https://www.reddit.com' + example = 'watchme export watcher task-reddit result.txt' bot.exit('Task name must start with "task", e.g., %s' % example) # Use the output file, or a temporary file diff --git a/watchme/client/ls.py b/watchme/client/ls.py index 0ba0c01..75494c1 100644 --- a/watchme/client/ls.py +++ b/watchme/client/ls.py @@ -10,16 +10,25 @@ from watchme.command import ( get_watchers, - list_watcher + list_watcher, + list_watcher_types ) from watchme.logger import bot def main(args, extra): '''list installed watchers - ''' - if extra == None: - get_watchers(args.base) + ''' + if args.watchers == True: + list_watcher_types() + + # Otherwise, we are listing installed watchers and tasks else: - for watcher in extra: - list_watcher(watcher, args.base) + # If no watchers provided, list the watchers + if extra == None: + get_watchers(args.base) + + # Otherwise, list the tasks of the watcher + else: + for watcher in extra: + list_watcher(watcher, args.base) diff --git a/watchme/command/__init__.py b/watchme/command/__init__.py index d8a8160..78853e3 100644 --- a/watchme/command/__init__.py +++ b/watchme/command/__init__.py @@ -26,5 +26,6 @@ ) from .utils import ( get_watchers, - list_watcher + list_watcher, + list_watcher_types ) diff --git a/watchme/command/create.py b/watchme/command/create.py index 18c8efa..101f6d8 100644 --- a/watchme/command/create.py +++ b/watchme/command/create.py @@ -27,6 +27,7 @@ def create_watcher(name=None, watcher_type=None, base=None): name: the watcher to create, uses default or WATCHME_WATCHER watcher_type: the type of watcher to create. defaults to WATCHER_DEFAULT_TYPE + base: The watcher base to use (defaults to $HOME/.watchme) ''' if name == None: name = WATCHME_WATCHER diff --git a/watchme/command/utils.py b/watchme/command/utils.py index f729dde..022440f 100644 --- a/watchme/command/utils.py +++ b/watchme/command/utils.py @@ -8,7 +8,11 @@ ''' -from watchme.defaults import WATCHME_BASE_DIR +from watchme.defaults import ( + WATCHME_BASE_DIR, + WATCHME_TASK_TYPES +) + from watchme.utils import ( get_tmpdir, run_command ) from watchme.logger import bot @@ -55,6 +59,13 @@ def list_watcher(watcher, base=None): else: bot.exit('%s does not exist.' % base) +def list_watcher_types(): + '''list the exporter options provided by watchme + ''' + bot.custom(prefix="watchme:", message="watcher task types", color="CYAN") + bot.info('\n '.join(WATCHME_TASK_TYPES)) + + def clone_watcher(repo, base=None, name=None): '''clone a watcher from Github (or other version control with git) meaning that we clone to a temporary folder, and then move diff --git a/watchme/tasks/__init__.py b/watchme/tasks/__init__.py index 43e5ec6..d2b8240 100644 --- a/watchme/tasks/__init__.py +++ b/watchme/tasks/__init__.py @@ -107,6 +107,75 @@ def run(self): bot.error('Cannot find function.') +# Save Entrypoint + + def write_results(self, result, repo): + '''an entrypoint function for a general task. By default, we parse + results based on the result type. Any particular subclass of the + TaskBase can modify or extend these functions. + + Parameters + ========== + result: the result object to parse + repo: the repo base (watcher.repo) + ''' + files = [] + + # Case 1. The result is a list + if isinstance(result, list): + # Get rid of Nones, if the user accidentally added + result = [r for r in result if r] + + if len(result) == 0: + bot.error('%s returned empty list of results.' % name) + + # json output is specified + elif self.params.get('save_as') == 'json': + bot.debug('Saving single list as one json...') + files.append(self._save_json(result, repo)) + + elif self.params.get('save_as') == 'json': + bot.debug('Saving single list as multiple json...') + files += self._save_json_list(result, repo) + + # Otherwise, sniff for list of paths + elif os.path.exists(result[0]): + bot.debug('Found list of paths...') + files += self._save_files_list(result, repo) + + # Finally, assume just writing text to file + else: + bot.debug('Saving content from list to file...') + files += self._save_text_list(result, repo) + + # Case 2. The result is a string + elif isinstance(result, str): + + # if it's a path to a file, just save to repository + if os.path.exists(result): + files.append(self._save_file(result, repo)) + + # Otherwise, it's a string that needs to be saved to file + else: + files.append(self._save_text(result, repo)) + + # Case 3. The result is a dictionary + elif isinstance(result, dict): + files.append(self._save_json(result,repo)) + + elif result == None: + bot.error('Result for task %s is None' % self.name) + + elif hasattr(self, '_write_results'): + return self._write_results(result) + + else: + bot.error('Unsupported result format %s' % type(result)) + + # Get rid of None results (don't check excessively for None above) + files = [f for f in files if f] + return files + # Saving diff --git a/watchme/version.py b/watchme/version.py index 030bc84..51f2069 100644 --- a/watchme/version.py +++ b/watchme/version.py @@ -6,13 +6,13 @@ # with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -__version__ = "0.0.16" +__version__ = "0.0.17" AUTHOR = 'Vanessa Sochat' AUTHOR_EMAIL = 'vsochat@stanford.edu' NAME = 'watchme' PACKAGE_URL = "http://www.github.com/vsoch/watchme" -KEYWORDS = 'web, changes, cron' -DESCRIPTION = "client to watch for webpage changes, and track over time" +KEYWORDS = 'web, changes, cron, reproducible, version-control' +DESCRIPTION = "reproducible monitoring client with exporters" LICENSE = "LICENSE" INSTALL_REQUIRES = ( @@ -31,6 +31,12 @@ ('psutil', {'min_version': '5.4.3'}), ) +# Install all watchers and exporters INSTALL_ALL = (INSTALL_REQUIRES + INSTALL_PSUTILS + INSTALL_URLS_DYNAMIC) + +# Install all watchers +INSTALL_WATCHERS = (INSTALL_REQUIRES + + INSTALL_PSUTILS + + INSTALL_URLS_DYNAMIC) diff --git a/watchme/watchers/README.md b/watchme/watchers/README.md index 5ed474a..5ecd717 100644 --- a/watchme/watchers/README.md +++ b/watchme/watchers/README.md @@ -5,4 +5,10 @@ Each of these is a Watcher that the user can request. - [urls](urls) to watch for changes in websites (default) - [psutils](psutils) to get basic system statistics -More watchers will be added as the library is developed. +## Watcher Base + +The watcher base is defined in the [init](__init__.py) file here. + + - [schedules](schedule.py) to interact with cronjobs + - [exporters](data.py) (extras) and the default export of temporal data + - [settings](settings.py) meaning add/remove from the watchme.cfg diff --git a/watchme/watchers/__init__.py b/watchme/watchers/__init__.py index 8945f92..72d8d0c 100644 --- a/watchme/watchers/__init__.py +++ b/watchme/watchers/__init__.py @@ -8,7 +8,7 @@ ''' -from watchme.logger import ( bot, RobotNamer ) +from watchme.logger import (bot, RobotNamer) from watchme.version import __version__ from watchme.defaults import ( WATCHME_BASE_DIR, @@ -32,12 +32,14 @@ from .settings import ( get_setting, - set_setting, get_section, + has_setting, + has_section, print_section, print_add_task, remove_setting, - remove_section + remove_section, + set_setting ) from .schedule import ( @@ -114,7 +116,7 @@ def _set_base(self, base=None, create=False): # If the watcher doesn't exist and we need to create: if not os.path.exists(self.repo) or not os.path.exists(self.configfile): if create is True: - create_watcher(self.name) + create_watcher(self.name) else: bot.exit('Watcher %s does not exist. Use watchme create.' % self.name) @@ -139,7 +141,7 @@ def edit_task(self, name, action, key, value=None): ''' if not self.has_task(name): - bot.exit('%s is not a task defined by %s' %(name, self.name)) + bot.exit('%s is not a task defined by %s' % (name, self.name)) if action not in ['update', 'add', 'remove']: bot.exit('Action must be update, add, or remove') @@ -149,7 +151,7 @@ def edit_task(self, name, action, key, value=None): # Add, and it doesn't exist so it's okay if action == "add" and key not in self.config[name]: - bot.info('Adding %s:%s to %s' %(key, value, name)) + bot.info('Adding %s:%s to %s' % (key, value, name)) self.set_setting(name, key, value) # Already exists, encourage user to update @@ -158,7 +160,7 @@ def edit_task(self, name, action, key, value=None): # Update, and it's a valid choice elif action == 'update' and key in self.config[name]: - bot.info('Updating %s to %s in %s' %(key, value, name)) + bot.info('Updating %s to %s in %s' % (key, value, name)) self.set_setting(name, key, value) # Update, and it's not a valid choice @@ -167,7 +169,7 @@ def edit_task(self, name, action, key, value=None): # Remove, and it's a valid choice elif action == "remove" and key in self.config[name]: - bot.info('Removing %s' % key ) + bot.info('Removing %s' % key) del self.config[name][key] # Remove, and it's not a valid choice @@ -198,7 +200,7 @@ def load_config(self): it will be written with a default active status set to false. ''' if not hasattr(self, 'config'): - + # Load the configuration file if it exists (will exit if not found) if self.configfile != None: self.config = read_config(self.configfile) @@ -224,7 +226,7 @@ def _get_params_dict(self, pairs): for pair in pairs: if "@" not in pair: bot.exit('incorrectly formatted param, must be key@value') - key,value = pair.split('@', 1) + key, value = pair.split('@', 1) key = key.lower() # All tasks are not allowed to have default params @@ -249,8 +251,8 @@ def add_task(self, task, task_type, params, force=False, active="true"): params: list of parameters to be validated (key@value) force: if task already exists, overwrite active: add the task as active (default "true") - ''' + # Check again, in case user calling from client if not task.startswith('task'): bot.exit('Task name must start with "task" (e.g., task-reddit)') @@ -322,7 +324,8 @@ def _add_task(self, task, force=False, active='true'): git_add(self.repo, task.name) # Commit changes - git_commit(repo=self.repo, task=self.name, message="ADD task %s" % task.name) + git_commit(repo=self.repo, task=self.name, + message="ADD task %s" % task.name) # Delete @@ -344,7 +347,7 @@ def delete(self): bot.info('Removing watcher %s' % self.name) shutil.rmtree(repo) else: - bot.exit("%s:%s doesn't exist" %(self.name, repo)) + bot.exit("%s:%s doesn't exist" % (self.name, repo)) def remove_task(self, task): @@ -552,7 +555,7 @@ def has_task(self, name): return False - def get_task(self, name, save=False): + def get_task(self, name): '''get a particular task, based on the name. This is where each type of class should check the "type" parameter from the config, and import the correct Task class. @@ -560,7 +563,6 @@ def get_task(self, name, save=False): Parameters ========== name: the name of the task to load - save: if saving, will be True ''' self.load_config() @@ -586,12 +588,12 @@ def get_task(self, name, save=False): bot.exit('Type %s not properly set up in get_task' % task_type) # if not valid, will return None - task = Task(name, params, _save=save) + task = Task(name, params) return task - def _task_selected(self, task, regexp=None): + def _task_selected(self, task, regexp=None, active=True): '''check if a task is active and (if defined) passes user provided task names or regular expressions. @@ -599,6 +601,7 @@ def _task_selected(self, task, regexp=None): ========== task: the task object to check regexp: an optional regular expression (or name) to check + active: a task is selected if it's active (default True) ''' selected = True @@ -607,8 +610,8 @@ def _task_selected(self, task, regexp=None): selected = False # Is the task not active (undefined is active)? - active = task.params.get('active', 'true') - if active == "false": + is_active = task.params.get('active', 'true') + if is_active == "false" and active == True: bot.info('Task %s is not active.' % task) selected = False @@ -621,7 +624,7 @@ def _task_selected(self, task, regexp=None): return selected - def get_tasks(self, regexp=None): + def get_tasks(self, regexp=None, quiet=False, active=True): '''get the tasks for a watcher, possibly matching a regular expression. A list of dictionaries is returned, each holding the parameters for a task. "uri" will hold the task (folder) name, active @@ -630,6 +633,8 @@ def get_tasks(self, regexp=None): ========== regexp: if supplied, the user wants to run only tasks that match a particular pattern + quiet: If quiet, don't print the number of tasks found + active: only return active tasks (default True) ''' self.load_config() @@ -641,10 +646,11 @@ def get_tasks(self, regexp=None): # Check that the task should be run, and is valid if task != None: - if self._task_selected(task, regexp) and task.valid: + if self._task_selected(task, regexp, active) and task.valid: tasks.append(task) - bot.info('Found %s contender tasks.' % len(tasks)) + if quiet == False: + bot.info('Found %s contender tasks.' % len(tasks)) return tasks @@ -686,7 +692,7 @@ def run_tasks(self, queue, parallel=True, show_progress=True): else: bot.info('Running %s' % prefix) results[task.name] = task.run() - progress+=1 + progress += 1 return results @@ -770,83 +776,30 @@ def finish_runs(self, results): ''' for name, result in results.items(): task_folder = os.path.join(self.repo, name) - task = self.get_task(name, save=True) - - # Files to be added via Git after - files = [] + task = self.get_task(name) # Ensure that the task folder exists if not os.path.exists(task_folder): mkdir_p(task_folder) git_add(self.repo, task_folder) - # Case 1. The result is a list - if isinstance(result, list): - - # Get rid of Nones, if the user accidentally added - result = [r for r in result if r] - - if len(result) == 0: - bot.error('%s returned empty list of results.' % name) - - # json output is specified - elif task.params.get('save_as') == 'json': - bot.debug('Saving single list as one json...') - files.append(task._save_json(result, self.repo)) - - elif task.params.get('save_as') == 'json': - bot.debug('Saving single list as multiple json...') - files += task._save_json_list(result, self.repo) - - # Otherwise, sniff for list of paths - elif os.path.exists(result[0]): - bot.debug('Found list of paths...') - files += task._save_files_list(result, self.repo) - - # Finally, assume just writing text to file - else: - bot.debug('Saving content from list to file...') - files += task._save_text_list(result, self.repo) - - # Case 2. The result is a string - elif isinstance(result, str): - - # if it's a path to a file, just save to repository - if os.path.exists(result): - files.append(task._save_file(result, self.repo)) - - # Otherwise, it's a string that needs to be saved to file - else: - files.append(task._save_text(result, self.repo)) - - # Case 3. The result is a dictionary - elif isinstance(result, dict): - files.append(task._save_json(result, self.repo)) - - elif result == None: - bot.error('Result for task %s is None' % name) - - else: - bot.error('Unsupported result format %s' % type(result)) - - # Get rid of None results (don't check excessively for None above) - files = [f for f in files if f] + # Files to be added to the repo via git after + files = task.write_results(result, self.repo) # Add files to git, and commit files.append(write_timestamp(repo=self.repo, task=name)) git_add(repo=self.repo, files=files) - git_commit(repo=self.repo, - task=self.name, + git_commit(repo=self.repo, + task=self.name, message="ADD results %s" % name) - # Identification def __repr__(self): - return "[watcher|%s]" %self.name + return "[watcher|%s]" % self.name def __str__(self): - return "[watcher|%s]" %self.name + return "[watcher|%s]" % self.name # Settings @@ -854,6 +807,8 @@ def __str__(self): Watcher.remove_setting = remove_setting Watcher.get_setting = get_setting Watcher.get_section = get_section +Watcher.has_setting = has_setting +Watcher.has_section = has_section Watcher.set_setting = set_setting Watcher.remove_section = remove_section Watcher.print_section = print_section diff --git a/watchme/watchers/settings.py b/watchme/watchers/settings.py index 35d4c1e..8aa5963 100644 --- a/watchme/watchers/settings.py +++ b/watchme/watchers/settings.py @@ -96,10 +96,11 @@ def print_section(self, section): else: bot.exit('%s is not a valid section.' % section) -def get_setting(self, section, name, default=None): - '''return a setting from the environment (first priority) and then - secrets (second priority) if one can be found. If not, return None. +def get_setting(self, section, name, default=None): + '''return a setting from the config, if defined. Otherwise return + default (None or set by user) + Parameters ========== section: the section in the config, defaults to self.name @@ -119,6 +120,31 @@ def get_setting(self, section, name, default=None): return setting +def has_setting(self, section, name): + '''return a boolean if a config has a setting (or not) + Parameters + ========== + section: the section in the config, defaults to self.name + name: they key (index) of the setting to look up + ''' + self.load_config() + + exists = False + if section in self.config: + if name.lower() in self.config[section]: + exists = True + + return exists + + +def has_section(self, section): + '''return a boolean if a config has a section (e.g., a task or exporter) + Parameters + ========== + section: the section in the config + ''' + self.load_config() + return section in self.config def set_setting(self, section, key, value): '''set a key value pair in a section, if the section exists. Returns @@ -145,6 +171,6 @@ def get_section(self, name): ''' section = None self.load_config() - if name in self.config.sections(): + if name in self.config: section = self.config[name] return section diff --git a/watchme/watchers/urls/helpers.py b/watchme/watchers/urls/helpers.py index c4194f2..2a55ae3 100644 --- a/watchme/watchers/urls/helpers.py +++ b/watchme/watchers/urls/helpers.py @@ -11,6 +11,7 @@ import os import tempfile import requests +import re # Helper functions