serpextract/update_list.py at master · Parsely/serpextract · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""Update the search_engines.pickle list contained within the package.
Use this before deploying an update"""

import os
from urllib.request import urlopen

try:
    import ujson as json
except ImportError:
    import json
from ruamel.yaml import YAML


_here = lambda *paths: os.path.join(os.path.dirname(os.path.abspath(__file__)), *paths)


def main():
    filename = _here("serpextract", "search_engines.json")
    print("Updating search engine parser definitions.")

    url = urlopen(
        "https://raw.githubusercontent.com/matomo-org/searchengine-and-social-list/master/SearchEngines.yml"
    )
    yaml = YAML(typ="safe", pure=True)
    matomo_engines = yaml.load(url)

    # Hard-code Mojeek entries in expected format
    # see: https://github.com/Parsely/serpextract/pull/23
    matomo_engines["Mojeek"] = [
        {
            "backlink": "search?q={k}",
            "hiddenkeyword": ["/^$/", "/", "/\\/search(\\?.*)?/", "/\\/url\\?.*/"],
            "params": ["q"],
            "urls": ["mojeek.com", "www.mojeek.com"],
        }
    ]

    with open(filename, "w") as json_file:
        json.dump(matomo_engines, json_file, indent=2, sort_keys=True)

    print(
        "Saved {} search engine parser definitions to {}.".format(
            len(matomo_engines), filename
        )
    )


if __name__ == "__main__":
    main()