crawler-buddy/pyproject.toml at main · rumca-js/crawler-buddy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# if poetry update takes forever
# export PYTHON_KEYRING_BACKEND=keyring.backends.fail.Keyring

[tool.poetry]
name = "crawler-buddy"
version = "6.2.11"
description = "Internet Crawler"
authors = ["Iwan Grozny <renegat@renegat0x0.ddns.net>"]
license = "GPL3"
readme = "README.md"

[tool.poetry.dependencies]
python = "^3.10"
python-dateutil = "^2.8.2"
yt-dlp = {extras = ["default"], version = "^2026.8.19"}
beautifulsoup4 = "^4.15.0"
lxml = "^5.3.0"
pytz = "^2024.2"
psutil="*"
flask="^3.1.3"
url-cleaner="*"
webtoolkit="^0.1.70"
sqlalchemy="^2.0.51"

# Enable and crawlers as needed

curl-cffi="^0.16.2"

# [Requests]
requests = "^2.34.2"

# [Selenium]
# do not update selenium - it needs to match browser version
selenium="4.38.0"
selenium-wire="*"
# chardet = "^5.2.0" # does not work with some RSS sources
# psycopg2 = "*"
pyvirtualdisplay = "^3.0"
undetected-chromedriver = "^3.5.5"

# [Crawlee]
crawlee="^1.9.3"
playwright="^1.62.0"

# [Stealth requests]
stealth-requests = "^2.0.6"

# [Httpx]
httpx = "^0.28.1"

httpmorph="^0.2.8"
httpcloak="1.6.11"
# botasaurus = "*"
#
# [scrapy]
scrapy = "2.13.3"

[tool.poetry.group.dev.dependencies]
black = "^25.9.0"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"