Skip to content

Commit 07a89b9

Browse files
lovitclaude
andcommitted
refactor(normalizer): emoticon_normalize deprecated 처리 및 HangleEmojiNormalizer로 통합
- emoticon_normalize(): DeprecationWarning 발생 후 HangleEmojiNormalizer로 위임 - 기존 구현의 버그 문서화: 완성형 음절 뒤 자음에서 종성 불일치 시 음절 묵소 삭제 (예: 'ㅋ크ㅋ' → 'ㅋㅋ', HangleEmojiNormalizer는 'ㅋ크ㅋ'로 올바르게 유지) - soynlp.hangle.compose/decompose import 제거 (normalizer 모듈에서 미사용) - test_emoticon_normalize_deprecated 테스트 추가 Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent ff1adb5 commit 07a89b9

2 files changed

Lines changed: 41 additions & 38 deletions

File tree

soynlp/normalizer/normalizer.py

Lines changed: 18 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@
22
import os
33
import re
44
import unicodedata
5+
import warnings
56
from collections.abc import Callable
67
from glob import glob
78

89
from tqdm import tqdm
910

10-
from soynlp.hangle import compose, decompose
11-
1211
logger = logging.getLogger(__name__)
1312

1413
_doublespace_pattern = re.compile(r"\s+")
@@ -58,42 +57,23 @@ def repeat_normalize(sent: str, num_repeats: int = 2) -> str:
5857

5958

6059
def emoticon_normalize(sent: str, num_repeats: int = 2) -> str:
61-
if not sent:
62-
return sent
63-
64-
def _char_type(idx: int) -> int:
65-
if 12593 <= idx <= 12622:
66-
return 0 # Jaum
67-
elif 12623 <= idx <= 12643:
68-
return 1 # Moum
69-
elif 44032 <= idx <= 55203:
70-
return 2 # Complete
71-
return -1
72-
73-
idxs = [_char_type(ord(c)) for c in sent]
74-
sent_ = []
75-
last_idx = len(idxs) - 1
76-
for i, (idx, c) in enumerate(zip(idxs, sent)):
77-
if (0 < i < last_idx) and (idxs[i - 1] == 0 and idx == 2 and idxs[i + 1] == 1):
78-
cho, jung, jong = decompose(c) # type: ignore[misc]
79-
if (cho == sent[i - 1]) and (jung == sent[i + 1]) and (jong == " "):
80-
sent_.append(cho)
81-
sent_.append(jung)
82-
else:
83-
sent_.append(c)
84-
elif (i < last_idx) and (idx == 2) and (idxs[i + 1] == 0):
85-
cho, jung, jong = decompose(c) # type: ignore[misc]
86-
if jong == sent[i + 1]:
87-
sent_.append(compose(cho, jung, " "))
88-
sent_.append(jong)
89-
elif (i > 0) and (idx == 2 and idxs[i - 1] == 0):
90-
cho, jung, jong = decompose(c) # type: ignore[misc]
91-
if cho == sent[i - 1]:
92-
sent_.append(cho)
93-
sent_.append(jung)
94-
else:
95-
sent_.append(c)
96-
return repeat_normalize("".join(sent_), num_repeats)
60+
"""한국어 자모-음절 혼합 이모티콘을 정규화한다.
61+
62+
.. deprecated::
63+
`emoticon_normalize`는 deprecated입니다.
64+
`HangleEmojiNormalizer`와 `RepeatCharacterNormalizer`를 조합하여 사용하세요.
65+
66+
기존 구현은 완성형 음절 뒤에 자음이 오면서 종성 조건이 불일치할 때
67+
해당 음절을 묵소 삭제하는 버그가 있었습니다. (예: 'ㅋ크ㅋ' → 'ㅋㅋ')
68+
`HangleEmojiNormalizer`는 이 케이스를 올바르게 처리합니다.
69+
"""
70+
warnings.warn(
71+
"`emoticon_normalize` is deprecated. Use `HangleEmojiNormalizer` and `RepeatCharacterNormalizer` instead.",
72+
DeprecationWarning,
73+
stacklevel=2,
74+
)
75+
normalized = HangleEmojiNormalizer().normalize(sent)
76+
return repeat_normalize(normalized, num_repeats)
9777

9878

9979
def only_hangle(sent: str) -> str:

tests/unit/test_normalizer.py

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
1+
import warnings
2+
13
from soynlp.normalizer.normalizer import (
24
HangleEmojiNormalizer,
35
PaddingSpacetoWordsNormalizer,
46
PassCharacterNormalizer,
57
RemoveLongspaceNormalizer,
68
RepeatCharacterNormalizer,
79
TextNormalizer,
10+
emoticon_normalize,
811
text_normalizer,
912
)
1013

@@ -89,6 +92,26 @@ def test_normalizer_builder():
8992
)
9093

9194

95+
def test_emoticon_normalize_deprecated():
96+
"""emoticon_normalize는 deprecated이며 HangleEmojiNormalizer와 동일 결과를 반환한다."""
97+
s = "어머나 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ쿠ㅜㅜㅜㅜㅜ이런게 있으면 어떻게 떼어내냐 ㅋㅋㅋㅋㅋ쿠ㅜㅜㅜㅜㅜ 하하"
98+
with warnings.catch_warnings(record=True) as w:
99+
warnings.simplefilter("always")
100+
result = emoticon_normalize(s, num_repeats=2)
101+
assert len(w) == 1
102+
assert issubclass(w[0].category, DeprecationWarning)
103+
assert "deprecated" in str(w[0].message).lower()
104+
105+
expected = RepeatCharacterNormalizer(max_repeat=2)(HangleEmojiNormalizer()(s))
106+
assert result == expected
107+
108+
# 기존 구현의 버그: 'ㅋ크ㅋ' → 'ㅋㅋ' (크가 묵소 삭제됨)
109+
# HangleEmojiNormalizer는 이를 올바르게 처리: 'ㅋ크ㅋ' → 'ㅋ크ㅋ' (변경 없음)
110+
with warnings.catch_warnings(record=True):
111+
warnings.simplefilter("always")
112+
assert emoticon_normalize("ㅋ크ㅋ", num_repeats=0) == "ㅋ크ㅋ"
113+
114+
92115
def test_default_text_normalizer():
93116
assert (
94117
text_normalizer("어머나 ㅋㅋㅋㅋㅋㅋㅋㅋㅋㅋ쿠ㅜㅜㅜㅜㅜ이런게 있으면 어떻게 떼어내냐 ㅋㅋㅋㅋㅋ쿠ㅜㅜㅜㅜㅜ 하하")

0 commit comments

Comments
 (0)