-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathplugin.py
More file actions
4136 lines (3517 loc) · 169 KB
/
Copy pathplugin.py
File metadata and controls
4136 lines (3517 loc) · 169 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""推特转发插件(MaiBot / FxTwitter)。
功能概览
--------
1. 按配置的间隔(默认 10 分钟)轮询订阅推主的最新推文;
2. 发现新推文后,把正文 + 配图推送到订阅它的聊天流;
3. 提供一组斜杠命令,在聊天里直接完成订阅管理与参数调整。
数据来源
--------
* 主接口:``https://api.fxtwitter.com/2/profile/<handle>/statuses``(FxEmbed v2 JSON)
* 兜底接口:``https://fxtwitter.com/<handle>/feed.atom.xml``(FxEmbed Atom feed)
* 图片托管在 ``pbs.twimg.com``,国内直连不通,默认走本地代理下载后再发送。
状态保存在 ``data/plugins/polarbear.twitter-forwarder/state.json``,
WebUI 配置页只是"基线配置",聊天里的命令改动作为运行时覆盖保存在状态文件中。
"""
from __future__ import annotations
import asyncio
import base64
import contextlib
import difflib
import html as html_module
import importlib.util
import ipaddress
import json
import logging
import os
import random
import re
import shlex
import shutil
import socket
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass, field as dataclass_field
from datetime import datetime
from pathlib import Path
from typing import Any, Iterable, Optional
from urllib.parse import urljoin, urlparse
try: # aiohttp 由宿主环境提供;缺失时插件降级为不可用而不是加载失败
import aiohttp
except Exception: # pragma: no cover - 仅在缺依赖的环境触发
aiohttp = None # type: ignore[assignment]
try: # 链接正文提取用,缺失时退化成正则清洗
from bs4 import BeautifulSoup # type: ignore
except Exception: # pragma: no cover
BeautifulSoup = None # type: ignore[assignment]
try:
import trafilatura # type: ignore
except Exception: # pragma: no cover
trafilatura = None # type: ignore[assignment]
from maibot_sdk import Command, Field, HomeCard, MaiBotPlugin, PluginConfigBase, Tool
from maibot_sdk.types import ToolParameterInfo, ToolParamType
LOGGER = logging.getLogger(__name__)
def available_extractors() -> list[str]:
"""当前环境里可用的「链接正文提取器」。
``trafilatura`` / ``bs4`` / ``readability`` 都是**可选增强**,不在宿主依赖基线里,
manifest 也不声明它们(避免装不上时把插件一起卡住)。三个都缺时仍然能跑:
提取会退化成正则清洗 + ``og:description``,Steam 公告走 RSS 不受影响。
"""
names: list[str] = []
if trafilatura is not None:
names.append("trafilatura")
if BeautifulSoup is not None:
names.append("bs4")
try:
if importlib.util.find_spec("readability") is not None:
names.append("readability")
except (ImportError, ValueError): # pragma: no cover - 环境异常时忽略
pass
return names
PLUGIN_ID = "polarbear.twitter-forwarder"
USER_AGENT = "MaiBot-TwitterForwarder/1.0 (+https://github.com/MaiM-with-u/maibot)"
HANDLE_RE = re.compile(r"^[A-Za-z0-9_]{1,15}$")
CANDIDATE_RE = re.compile(r"^[A-Za-z0-9_]{4,15}$")
HANDLE_MAX_LENGTH = 15
TWEET_ID_RE = re.compile(r"/status(?:es)?/(\d+)")
ATOM_NS = "{http://www.w3.org/2005/Atom}"
RESERVED_PATH_SEGMENTS = {"i", "web", "home", "intent", "search", "hashtag", "explore", "status", "statuses", "compose"}
SUPPORTED_HOSTS = (
"x.com",
"twitter.com",
"mobile.twitter.com",
"fxtwitter.com",
"fixupx.com",
"vxtwitter.com",
"nitter.net",
)
MAX_SEEN_IDS = 300
MAX_HANDLES_PER_MESSAGE = 5
MAX_NODES_PER_FORWARD = 20
MAX_TRANSLATION_CACHE = 300
MAX_LINK_CACHE = 120
STATE_VERSION = 1
# 目标语言描述 → 语言代码前缀,用于判断"这条推文本来就是目标语言"
LANG_TARGET_KEYS: dict[str, tuple[str, ...]] = {
"zh": ("中文", "汉语", "简体", "繁體", "繁体", "chinese", "zh"),
"en": ("英文", "英语", "english", "en"),
"ja": ("日文", "日语", "日本語", "japanese", "ja"),
"ko": ("韩文", "韩语", "한국", "korean", "ko"),
"ru": ("俄文", "俄语", "russian", "ru"),
"fr": ("法文", "法语", "french", "fr"),
"de": ("德文", "德语", "german", "de"),
"es": ("西班牙", "spanish", "es"),
}
# 斜杠命令的公共前缀:同时接受半角 "/" 和全角 "/",并容忍开头的 @机器人
_LEAD = r"^\s*(?:@\S+\s+)?[//]"
# ---------------------------------------------------------------------------
# 配置模型
# ---------------------------------------------------------------------------
class PluginSection(PluginConfigBase):
"""插件基础配置。"""
__ui_label__ = "插件"
__ui_icon__ = "package"
__ui_order__ = 0
enabled: bool = Field(default=True, description="是否启用插件")
config_version: str = Field(default="1.0.0", description="配置版本")
class PollSection(PluginConfigBase):
"""轮询相关配置。"""
__ui_label__ = "轮询"
__ui_icon__ = "refresh-cw"
__ui_order__ = 1
interval_minutes: int = Field(default=10, description="轮询间隔(分钟),聊天里可用 /tw_interval 临时覆盖")
initial_delay_seconds: int = Field(default=30, description="插件加载后首次轮询的延迟(秒)")
request_timeout_seconds: int = Field(default=20, description="单次 HTTP 请求超时(秒)")
retry_times: int = Field(default=2, description="单次请求失败后的重试次数")
fetch_count: int = Field(default=20, description="每个推主每轮拉取的条目数(1-50)")
max_tweets_per_poll: int = Field(default=3, description="每个推主每轮最多推送几条新推文")
max_concurrency: int = Field(default=3, description="同时轮询的推主数量上限")
class TwitterSection(PluginConfigBase):
"""数据源与网络配置。"""
__ui_label__ = "数据源"
__ui_icon__ = "globe"
__ui_order__ = 2
api_base: str = Field(default="https://api.fxtwitter.com", description="FxTwitter API 地址,一般不用改")
feed_base: str = Field(default="https://fxtwitter.com", description="FxTwitter 站点地址,用于 Atom 兜底")
enable_feed_fallback: bool = Field(default=True, description="v2 接口失败时回退到 Atom feed 拉取")
proxy: str = Field(
default="http://127.0.0.1:7890",
description="下载图片用的 HTTP 代理;pbs.twimg.com 国内直连不通,留空表示不用代理",
)
use_proxy_for_api: bool = Field(default=False, description="调用 fxtwitter 接口时也走代理(一般不需要)")
user_agent: str = Field(default=USER_AGENT, description="请求使用的 User-Agent")
class PushSection(PluginConfigBase):
"""推送行为配置。"""
__ui_label__ = "推送"
__ui_icon__ = "send"
__ui_order__ = 3
extra_streams: list[str] = Field(
default_factory=list,
description="额外固定推送目标:填写聊天流 session_id,所有订阅都会同步推送到这些流",
)
include_reposts: bool = Field(default=True, description="是否推送转推(博主转发别人的推文)")
include_replies: bool = Field(default=False, description="是否推送回复(博主回复别人)")
push_latest_on_subscribe: bool = Field(default=True, description="新增订阅时立刻把该推主最新一条推过来")
skip_sensitive: bool = Field(
default=True,
description="是否跳过被接口标记为「可能敏感」的推文(默认跳过,避免群里出现不宜内容)",
)
class MediaSection(PluginConfigBase):
"""媒体下载配置。"""
__ui_label__ = "媒体"
__ui_icon__ = "image"
__ui_order__ = 4
download_images: bool = Field(default=True, description="是否下载并发送推文配图")
max_images: int = Field(default=4, description="单条推文最多发送几张图")
image_quality: str = Field(default="medium", description="图片尺寸:orig / large / medium / small")
max_image_mb: float = Field(default=3.0, description="单张图片大小上限(MB),超限则跳过")
max_total_mb: float = Field(default=6.0, description="单条推文图片总大小上限(MB),避免一次性发太多图")
media_timeout_seconds: int = Field(default=30, description="单张图片下载超时(秒)")
video_mode: str = Field(
default="auto",
description="视频处理:auto=尽量把原视频下载发过来;thumbnail=只发封面+链接;link=只发链接",
)
inline_video_mb: float = Field(
default=10.0,
description="以内联 base64 发送的视频大小上限(MB)。受插件 IPC 16MB 帧限制,请不要超过 11",
)
max_video_mb: float = Field(
default=300.0,
description="视频下载大小上限(MB),超过则回退为封面+链接",
)
video_timeout_seconds: int = Field(default=180, description="视频下载超时(秒)")
big_video_docker_route: bool = Field(
default=True,
description="大于内联上限的视频:下载后用 docker 拷进 SnowLuma 容器,以容器内路径发送(需要当前用户能执行 docker)",
)
docker_container: str = Field(default="snowluma", description="大视频路由的目标容器名")
video_keep_seconds: int = Field(
default=180,
description="发完之后容器内视频文件再保留多少秒才删(大视频上传慢,删太早会让 QQ 那边收不到;0=立刻删)",
)
class DisplaySection(PluginConfigBase):
"""消息排版配置。"""
__ui_label__ = "排版"
__ui_icon__ = "layout"
__ui_order__ = 5
use_forward: bool = Field(
default=True,
description="合并转发总开关:关掉之后任何路径都不会用合并转发,全部走普通图文",
)
forward_for_poll: bool = Field(
default=False,
description=(
"自己轮询到的新推文是否用合并转发(默认 false:按普通图文逐条发,"
"最旧的先发,聊天里从上往下就是时间顺序)"
),
)
forward_for_test: bool = Field(
default=True,
description="/tw_test 预览是否用合并转发(默认 true:一条聊天记录里每条推文一个节点,最新在最上面)",
)
batch_forward: bool = Field(
default=True,
description="一次有多个新推文时,合并成一条聊天记录(合并转发),每条推文占其中一个节点",
)
batch_max_mb: float = Field(
default=7.0,
description="单条聊天记录里图片的总大小上限(MB),超过会自动拆成多条,避免超出插件 IPC 帧限制",
)
video_in_forward: bool = Field(
default=True,
description="合并转发时,把视频也放进它自己的聊天记录节点里(关掉、或走普通图文时视频单独成条发)",
)
forward_nickname: str = Field(default="推特转发", description="合并转发节点里显示的昵称")
show_author: bool = Field(default=True, description="是否显示作者与时间")
header_divider: str = Field(
default="────────────────",
description="作者行下面的分隔线;留空表示不画",
)
hide_expanded_url: bool = Field(
default=True,
description="已经抓出正文的链接,从推文正文里去掉,避免和下面的链接内容重复",
)
show_link: bool = Field(default=True, description="是否附带原推链接")
show_stats: bool = Field(default=True, description="是否显示点赞/转发/浏览数据")
max_text_chars: int = Field(default=600, description="正文最大字符数,0 表示不截断")
class TranslationSection(PluginConfigBase):
"""自动翻译配置。"""
__ui_label__ = "翻译"
__ui_icon__ = "languages"
__ui_order__ = 6
enabled: bool = Field(default=True, description="是否自动翻译推文正文")
model_task: str = Field(
default="replyer",
description="用哪个模型任务做翻译(默认 replyer=回复模型;也可填 utils / planner 等)",
)
target_lang: str = Field(default="简体中文", description="翻译目标语言")
translate_quote: bool = Field(default=True, description="是否一并翻译引用的推文")
skip_if_target_lang: bool = Field(
default=True,
description="推文本来就是目标语言时跳过翻译(按接口给的 lang 判断,拿不到时按汉字比例判断)",
)
max_chars: int = Field(default=1200, description="超过这个长度的推文不翻译(省 token),0 表示不限制")
max_concurrency: int = Field(default=3, description="同时翻译的条数上限")
timeout_seconds: int = Field(default=45, description="单条翻译超时(秒)")
prompt: str = Field(
default=(
"你是专业的推文翻译助手。把用户给出的推文翻译成{target_lang}。\n"
"要求:\n"
"1. 只输出译文,不要输出原文、不要解释、不要加任何前缀或标记;\n"
"2. 保留原文的换行、语气和 emoji;\n"
"3. 人名、产品名、账号名、话题标签保持原文;\n"
"4. 如果原文已经是{target_lang},原样输出。"
),
description="翻译提示词,{target_lang} 会被替换成目标语言",
)
class LinkSection(PluginConfigBase):
"""推文里链接的内容抓取。"""
__ui_label__ = "链接内容"
__ui_icon__ = "link"
__ui_order__ = 7
enabled: bool = Field(default=True, description="是否把推文里链接的正文也抓出来一起发")
max_links: int = Field(default=1, description="每条推文最多展开几个链接")
max_chars: int = Field(default=3000, description="抓到的正文最多保留多少字(0 表示不限制)")
translate_chunk_chars: int = Field(
default=1200,
description="长正文分段翻译时每段的字符数,太小会丢上下文,太大可能撞上模型输出上限",
)
timeout_seconds: int = Field(default=15, description="单次抓取超时(秒);直连失败会用代理再试一次")
max_page_mb: float = Field(default=2.0, description="页面下载大小上限(MB)")
proxy: str = Field(default="", description="抓取用的 HTTP 代理;留空表示直连(失败会自动改用 twitter.proxy 重试)")
fallback_proxy: bool = Field(default=True, description="直连抓不到时,用 [twitter] 的代理再试一次")
allow_private_hosts: bool = Field(
default=False,
description=(
"是否允许抓取指向本机/内网的链接(SSRF 防护开关)。默认关闭:"
"推文里的外链如果解析到 127.0.0.1、10.x、192.168.x、169.254.x 等地址会被直接丢弃"
),
)
translate: bool = Field(default=True, description="抓到的正文是否也翻译(复用 [translation] 的目标语言)")
link_language: str = Field(
default="schinese",
description="Steam 新闻这类支持语言的站点优先取哪种语言(schinese/english/japanese/koreana…)",
)
class CommandSection(PluginConfigBase):
"""聊天命令权限配置。"""
__ui_label__ = "命令"
__ui_icon__ = "terminal"
__ui_order__ = 6
admin_only: bool = Field(default=False, description="true 时所有命令都只有管理员能使用")
cross_chat_admin_only: bool = Field(
default=True,
description=(
"跨聊天命令是否只允许管理员/本地操作员:"
"/tw_all、/tw_del、/tw_reset、/tw_check、/tw_interval。默认开启"
),
)
admins: list[str] = Field(
default_factory=list,
description='管理员 QQ 号列表,例如 ["123456789"];机器人的本地操作员始终放行',
)
class TwitterForwarderConfig(PluginConfigBase):
"""推特转发插件配置。"""
plugin: PluginSection = Field(default_factory=PluginSection)
poll: PollSection = Field(default_factory=PollSection)
twitter: TwitterSection = Field(default_factory=TwitterSection)
push: PushSection = Field(default_factory=PushSection)
media: MediaSection = Field(default_factory=MediaSection)
display: DisplaySection = Field(default_factory=DisplaySection)
translation: TranslationSection = Field(default_factory=TranslationSection)
link: LinkSection = Field(default_factory=LinkSection)
command: CommandSection = Field(default_factory=CommandSection)
# ---------------------------------------------------------------------------
# 数据模型
# ---------------------------------------------------------------------------
@dataclass
class TweetMedia:
"""推文里的一段媒体。"""
kind: str # photo / video / gif
url: str # 主资源地址(视频为 mp4)
thumbnail_url: str = ""
width: int = 0
height: int = 0
duration: float = 0.0 # 秒
formats: list = dataclass_field(default_factory=list) # [(url, bitrate), ...],码率按 bps
def best_variant_candidates(self) -> list[tuple[str, int]]:
"""按码率从高到低返回 (url, bitrate) 候选列表,去重。"""
candidates: list[tuple[str, int]] = []
seen: set[str] = set()
if self.url:
candidates.append((self.url, 0))
seen.add(self.url)
for url, bitrate in sorted(self.formats or [], key=lambda item: -int(item[1] or 0)):
if url and url not in seen:
candidates.append((url, int(bitrate or 0)))
seen.add(url)
return candidates
@dataclass
class Tweet:
"""归一化后的推文。"""
id: str
url: str
text: str
created_ts: float
author_name: str
author_screen_name: str
is_repost: bool = False
reposted_by_name: str = ""
reposted_by_screen_name: str = ""
is_reply: bool = False
sensitive: bool = False
lang: str = ""
quote_author: str = ""
quote_text: str = ""
translation: str = ""
quote_translation: str = ""
link_preview: str = ""
expanded_links: list[str] = dataclass_field(default_factory=list)
media: list[TweetMedia] = dataclass_field(default_factory=list)
likes: int = 0
replies: int = 0
reposts: int = 0
views: int = 0
source: str = "api"
@property
def has_video(self) -> bool:
"""是否包含视频。"""
return any(item.kind in {"video", "gif"} for item in self.media)
def video_media(self) -> list[TweetMedia]:
"""返回推文里的视频媒体(video/gif)。"""
return [item for item in self.media if item.kind in {"video", "gif"}]
def image_sources(self) -> list[str]:
"""返回可下载为图片的候选地址列表。"""
urls: list[str] = []
for item in self.media:
if item.kind == "photo":
urls.append(item.url)
elif item.thumbnail_url:
urls.append(item.thumbnail_url)
return [url for url in urls if url]
@dataclass
class VideoPayload:
"""已经准备好、可以塞进消息里的视频段内容。"""
data: dict[str, Any] # {"binary_data_base64": ...} 或 {"file": "/app/data/twvideo/x.mp4"}
size: int = 0 # 计入批次预算的内联字节数(容器路径引用算 0)
url: str = ""
container: str = ""
container_path: str = ""
host_path: Optional[Path] = None
@property
def is_inline(self) -> bool:
"""是否走内联 base64。"""
return bool(self.data.get("binary_data_base64"))
@dataclass
class PollResult:
"""单个推主的轮询结果。"""
handle: str
pushed: int = 0
fetched: int = 0
error: str = ""
@property
def ok(self) -> bool:
"""本次轮询是否成功。"""
return not self.error
class FxTwitterError(RuntimeError):
"""FxTwitter 调用失败。"""
# ---------------------------------------------------------------------------
# 工具函数
# ---------------------------------------------------------------------------
def normalize_handle(raw: str) -> Optional[str]:
"""把用户输入归一化为小写的推主 handle。
支持 ``@name``、``name``、``https://x.com/name``、``x.com/name/status/123`` 等形式。
Args:
raw: 用户原始输入。
Returns:
Optional[str]: 归一化后的 handle;无法识别时返回 ``None``。
"""
text = (raw or "").strip().replace("@", "@")
if not text:
return None
lowered = text.lower()
if "/" in text or lowered.startswith(("http://", "https://", "www.")):
candidate = text if "://" in text else f"https://{text}"
parsed = urlparse(candidate)
host = (parsed.netloc or "").lower().split(":")[0].removeprefix("www.")
known_host = host in SUPPORTED_HOSTS or host.endswith(tuple(f".{item}" for item in SUPPORTED_HOSTS))
if not known_host:
return None
for segment in (parsed.path or "").split("/"):
normalized_segment = segment.strip()
if not normalized_segment or normalized_segment.lower() in RESERVED_PATH_SEGMENTS:
continue
text = normalized_segment
break
else:
return None
text = text.lstrip("@").strip()
text = text.split("/")[0].split("?")[0].strip()
if not text or text.lower() in RESERVED_PATH_SEGMENTS:
return None
# 纯数字是 Twitter 的用户 ID,不是 handle
if text.isdigit():
return None
if not HANDLE_RE.match(text):
return None
return text.lower()
def parse_handle_list(raw: str) -> list[str]:
"""把一段文本里的多个 handle 解析出来并去重。"""
handles, _ = parse_handle_input(raw)
return handles
def parse_handle_input(raw: str) -> tuple[list[str], list[str]]:
"""解析用户输入,返回可用 handle 与每一条无法识别输入的原因。
Args:
raw: 命令后面跟的原始参数文本。
Returns:
tuple[list[str], list[str]]: ``(可用 handle 列表, 问题说明列表)``。
"""
handles: list[str] = []
problems: list[str] = []
for chunk in re.split(r"[\s,,、;;]+", raw or ""):
token = chunk.strip()
if not token:
continue
handle = normalize_handle(token)
if handle:
if handle not in handles:
handles.append(handle)
else:
problems.append(describe_handle_problem(token))
return handles, problems
def describe_handle_problem(raw: str) -> str:
"""说明某个输入为什么不能当作推主用户名,方便直接回给用户。"""
text = (raw or "").strip().replace("@", "@").strip()
if not text:
return "空输入"
stripped = text.lstrip("@").strip()
if not stripped:
return f"{text}:只有 @ 没有用户名"
lowered = stripped.lower()
if "/" in stripped or lowered.startswith(("http://", "https://", "www.")):
return f"{text}:这不是 x.com / twitter.com 的推主链接"
if stripped.isdigit():
return f"{text}:纯数字是 X 的用户 ID,不是用户名"
if lowered in RESERVED_PATH_SEGMENTS:
return f"{text}:这是 X 的保留路径(如 /home、/i),不是用户名"
if len(stripped) > HANDLE_MAX_LENGTH:
return f"{text}:共 {len(stripped)} 个字符,超过 X 用户名的 {HANDLE_MAX_LENGTH} 字符上限"
invalid_chars = sorted({char for char in stripped if not re.match(r"[A-Za-z0-9_]", char)})
if invalid_chars:
shown = " ".join(invalid_chars[:5])
return f"{text}:含有用户名不能使用的字符「{shown}」"
return f"{text}:无法识别的用户名"
def handle_repair_candidates(raw: str) -> list[str]:
"""给超长/多打字的输入猜几个候选用户名(逐个删掉一个下划线等)。
只在解析失败时用来做"你是不是想订阅 @xxx"的提示,候选还要再经过接口校验。
"""
base = (raw or "").strip().lstrip("@").strip().lower()
if not base:
return []
candidates: list[str] = []
for index, char in enumerate(base):
if char != "_":
continue
candidate = base[:index] + base[index + 1 :]
if candidate and candidate not in candidates:
candidates.append(candidate)
for candidate in (base.replace("_", ""), base.strip("_")):
if candidate and candidate not in candidates:
candidates.append(candidate)
# 多打了首尾字符的情况
if len(base) > HANDLE_MAX_LENGTH:
for candidate in (base[1:], base[:-1]):
if candidate and candidate not in candidates:
candidates.append(candidate)
return [candidate for candidate in candidates if CANDIDATE_RE.match(candidate)]
def format_count(value: int) -> str:
"""把数字格式化成中文习惯的紧凑写法。"""
try:
number = int(value)
except (TypeError, ValueError):
return "0"
if number >= 100_000_000:
return f"{number / 100_000_000:.1f}亿".replace(".0亿", "亿")
if number >= 10_000:
return f"{number / 10_000:.1f}万".replace(".0万", "万")
return str(number)
def format_time(timestamp: float) -> str:
"""把时间戳格式化为本地时间字符串。"""
if not timestamp:
return "未知"
try:
return datetime.fromtimestamp(float(timestamp)).strftime("%Y-%m-%d %H:%M")
except (OverflowError, OSError, ValueError):
return "未知"
def format_ago(timestamp: float) -> str:
"""把时间戳格式化为"多久之前"。"""
if not timestamp:
return "从未"
delta = max(0.0, time.time() - float(timestamp))
if delta < 60:
return f"{int(delta)} 秒前"
if delta < 3600:
return f"{int(delta // 60)} 分钟前"
if delta < 86400:
return f"{delta / 3600:.1f} 小时前"
return f"{delta / 86400:.1f} 天前"
def truncate_text(text: str, limit: int) -> str:
"""按字符数截断文本。"""
normalized = (text or "").strip()
if limit <= 0 or len(normalized) <= limit:
return normalized
return normalized[: max(1, limit - 1)].rstrip() + "…"
def truncate_at_paragraph(text: str, limit: int) -> str:
"""截断长文:尽量切在段落/换行处,避免半句话被砍掉。"""
normalized = (text or "").strip()
if limit <= 0 or len(normalized) <= limit:
return normalized
window = normalized[:limit]
cut = window.rfind("\n")
if cut >= int(limit * 0.6):
return window[:cut].rstrip() + "\n…(内容较长,已截断)"
return window.rstrip() + "…(内容较长,已截断)"
def split_text_chunks(text: str, limit: int) -> list[str]:
"""按段落把长文切成若干块(单段超长时再按句子/空格切),供分段翻译用。"""
normalized = (text or "").strip()
if not normalized:
return []
if limit <= 0 or len(normalized) <= limit:
return [normalized]
chunks: list[str] = []
current = ""
for paragraph in re.split(r"\n+", normalized):
paragraph = paragraph.strip()
if not paragraph:
continue
if len(current) + len(paragraph) + 1 <= limit:
current = f"{current}\n{paragraph}" if current else paragraph
continue
if current:
chunks.append(current)
current = ""
# 单段本身就超长 → 继续按句子/空格切
while len(paragraph) > limit:
window = paragraph[:limit]
cut = max(window.rfind("。"), window.rfind("!"), window.rfind("?"), window.rfind(". "), window.rfind(" "))
if cut < int(limit * 0.4):
cut = limit
chunks.append(paragraph[:cut].strip())
paragraph = paragraph[cut:].strip()
current = paragraph
if current:
chunks.append(current)
return [chunk for chunk in chunks if chunk]
def protect_invalid_tags(raw_html: str) -> str:
"""把 ``<Part 1>`` 这类"不是合法标签"的尖括号内容转义成文本。
RSS / 网页里的正文本身可能带尖括号(例如官方的 ``<Part 1>``),
直接交给 HTML 解析器会被当成未知标签丢掉,导致译文里出现断句。
这里只放行已知标签(以及 ``my-widget`` 这种带连字符的自定义元素),
其余尖括号内容一律转义成文本。
"""
def replace(match: re.Match[str]) -> str:
inner = match.group(1).strip()
if not inner or inner.startswith(("/", "!", "?")):
return match.group(0)
name = re.split(r"[\s/>]", inner, maxsplit=1)[0]
if not name or name.lower() in KNOWN_HTML_TAGS or "-" in name:
return match.group(0)
return f"<{inner}>"
return re.sub(r"<([^<>]{1,80})>", replace, raw_html or "")
def strip_html(raw: str) -> str:
"""把 Atom feed 里的 HTML 片段转成纯文本。"""
text = protect_invalid_tags(raw or "")
text = re.sub(r"(?i)<br\s*/?>", "\n", text)
text = re.sub(r"(?i)</p\s*>", "\n\n", text)
text = re.sub(r"(?i)<blockquote[^>]*>", "\n> ", text)
text = re.sub(r"(?i)</blockquote\s*>", "\n", text)
text = re.sub(r"<[^>]+>", "", text)
text = (
text.replace(" ", " ")
.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace(""", '"')
.replace("'", "'")
)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def resize_twimg_url(url: str, quality: str) -> str:
"""把 pbs.twimg.com 的图片地址改写成指定尺寸。"""
normalized = (url or "").strip()
if not normalized or "pbs.twimg.com" not in normalized:
return normalized
if quality not in {"orig", "large", "medium", "small"}:
return normalized
base = normalized.split("?", maxsplit=1)[0]
return f"{base}?name={quality}"
def parse_tweet_id(url: str) -> str:
"""从推文链接里取出推文 ID。"""
match = TWEET_ID_RE.search(url or "")
return match.group(1) if match else ""
def match_target_lang_code(target_lang: str) -> str:
"""把目标语言描述映射成语言代码前缀(zh / en / ja …),识别不了返回空串。"""
target = (target_lang or "").strip().lower()
if not target:
return ""
for code, keywords in LANG_TARGET_KEYS.items():
if any(keyword in target for keyword in keywords):
return code
return ""
def looks_like_language(text: str, code: str) -> bool:
"""粗略判断文本是否已经是某种语言(接口没给 lang 时的兜底)。"""
if not text or not code:
return False
if code == "zh":
cjk = sum(1 for char in text if "\u4e00" <= char <= "\u9fff")
return cjk >= 8 and cjk / max(1, len(text)) >= 0.3
if code == "ko":
return sum(1 for char in text if "\uac00" <= char <= "\ud7af") >= 8
if code == "ja":
return sum(1 for char in text if "\u3040" <= char <= "\u30ff") >= 8
return False
def clean_translation(text: str) -> str:
"""清理模型输出里可能带的代码块、包裹引号和「译文:」前缀。"""
cleaned = (text or "").strip()
if cleaned.startswith("```"):
cleaned = re.sub(r"^```[a-zA-Z]*\s*", "", cleaned)
cleaned = re.sub(r"\s*```$", "", cleaned).strip()
if len(cleaned) >= 2 and cleaned[0] == cleaned[-1] and cleaned[0] in {'"', "'", "“", "「", "『"}:
cleaned = cleaned[1:-1].strip()
cleaned = re.sub(r"^(译文|翻译|Translation)\s*[::]\s*", "", cleaned, flags=re.IGNORECASE)
return cleaned.strip()
# --- 链接内容抓取 ---------------------------------------------------------
URL_RE = re.compile(r"https?://[^\s<>\"'))】\]]+", re.IGNORECASE)
STEAM_NEWS_RE = re.compile(r"^https?://store\.steampowered\.com/news/app/(\d+)/view/(\d+)", re.IGNORECASE)
SKIP_LINK_HOSTS = (
"x.com",
"twitter.com",
"t.co",
"twimg.com",
"fxtwitter.com",
"fixupx.com",
"vxtwitter.com",
"nitter.net",
"t.me",
)
SKIP_LINK_SUFFIXES = (
".jpg",
".jpeg",
".png",
".gif",
".webp",
".bmp",
".mp4",
".mov",
".webm",
".mp3",
".m4a",
".zip",
".rar",
".7z",
".exe",
".apk",
)
# HTML 解析时放行的标签名;不在名单里又没有连字符的尖括号内容会被当成普通文本
KNOWN_HTML_TAGS = frozenset(
{
"a", "abbr", "address", "article", "aside", "audio", "b", "bdi", "bdo", "blockquote", "body", "br",
"button", "caption", "center", "cite", "code", "col", "colgroup", "dd", "del", "details", "dfn", "div",
"dl", "dt", "em", "embed", "fieldset", "figcaption", "figure", "font", "footer", "form", "h1", "h2",
"h3", "h4", "h5", "h6", "head", "header", "hr", "html", "i", "iframe", "img", "input", "ins", "kbd",
"label", "legend", "li", "link", "main", "mark", "meta", "nav", "noscript", "ol", "option", "p", "picture",
"pre", "q", "s", "samp", "script", "section", "select", "small", "source", "span", "strike", "strong",
"style", "sub", "summary", "sup", "table", "tbody", "td", "textarea", "tfoot", "th", "thead", "time",
"title", "tr", "track", "u", "ul", "var", "video", "wbr",
}
)
def extract_links(text: str) -> list[str]:
"""从推文正文里挑出值得展开的链接(去掉推文自身、媒体和图片直链)。"""
links: list[str] = []
for raw in URL_RE.findall(text or ""):
url = raw.rstrip(".,;:!?、。,;:!?")
host = (urlparse(url).netloc or "").lower().split(":")[0]
if not host:
continue
normalized_host = host.removeprefix("www.")
if any(normalized_host == item or normalized_host.endswith(f".{item}") for item in SKIP_LINK_HOSTS):
continue
if url.lower().split("?")[0].endswith(SKIP_LINK_SUFFIXES):
continue
if url not in links:
links.append(url)
return links
def html_to_text(raw_html: str) -> str:
"""把 HTML 片段转成纯文本(优先 bs4,缺失时退回正则清洗)。"""
text = ""
if BeautifulSoup is not None:
try:
soup = BeautifulSoup(protect_invalid_tags(raw_html), "lxml")
for tag in soup(["script", "style", "noscript", "iframe"]):
tag.decompose()
text = soup.get_text("\n", strip=True)
except Exception:
text = ""
if not text:
text = strip_html(raw_html)
text = re.sub(r"[ \t\u00a0]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def extract_page_title(raw_html: str) -> str:
"""取页面标题:优先 og:title,其次 <title>。"""
patterns = (
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\'](.*?)["\']',
r'<meta[^>]+name=["\']twitter:title["\'][^>]+content=["\'](.*?)["\']',
r"<title[^>]*>(.*?)</title>",
)
for pattern in patterns:
match = re.search(pattern, raw_html or "", re.S | re.I)
if match:
title = html_module.unescape(re.sub(r"\s+", " ", match.group(1))).strip()
if title:
return title
return ""
def extract_article_text(raw_html: str) -> str:
"""从网页里抽正文:trafilatura → readability → 元描述,逐级兜底并过滤导航垃圾。"""
# trafilatura 在解析失败时会往 WARNING 刷日志,提取期间临时压掉
noisy_loggers = [logging.getLogger("trafilatura"), logging.getLogger("justext")]
previous_levels = [(item, item.level) for item in noisy_loggers]
try:
for item in noisy_loggers:
item.setLevel(logging.ERROR)
if trafilatura is not None:
try:
extracted = trafilatura.extract(
raw_html,
include_comments=False,
include_tables=False,
favor_precision=True,
)
if extracted and len(extracted.strip()) >= 80 and not looks_like_boilerplate(extracted):
return re.sub(r"\n{3,}", "\n\n", extracted.strip())
except Exception:
pass
try:
from readability import Document # type: ignore
summary = Document(raw_html).summary()
text = html_to_text(summary)
if len(text) >= 80 and not looks_like_boilerplate(text):
return text
except Exception:
pass
finally:
for item, level in previous_levels:
item.setLevel(level)
# 全都失败时,宁可给一句元描述,也不要一屏导航栏
description = extract_meta_description(raw_html)
if description:
return description
fallback = html_to_text(raw_html)
return "" if looks_like_boilerplate(fallback) else fallback
def extract_meta_description(raw_html: str) -> str:
"""取页面描述:og:description / name=description。"""
patterns = (
r'<meta[^>]+property=["\']og:description["\'][^>]+content=["\'](.*?)["\']',
r'<meta[^>]+name=["\']description["\'][^>]+content=["\'](.*?)["\']',
r'<meta[^>]+name=["\']twitter:description["\'][^>]+content=["\'](.*?)["\']',
)