10.1 敏感詞過濾系統#

from collections import deque
import re

class AC:
    def __init__(self):
        self.next = [{}]; self.fail = [0]; self.out = [[]]

    def add(self, word, tag=None):
        s = 0
        for ch in word:
            if ch not in self.next[s]:
                self.next.append({}); self.fail.append(0); self.out.append([])
                self.next[s][ch] = len(self.next) - 1
            s = self.next[s][ch]
        self.out[s].append(tag or word)

    def build(self):
        q = deque()
        for ch, v in self.next[0].items():
            q.append(v)
        while q:
            u = q.popleft()
            for ch, v in self.next[u].items():
                f = self.fail[u]
                while f and ch not in self.next[f]:
                    f = self.fail[f]
                self.fail[v] = self.next[f].get(ch, 0) \
                    if self.next[f].get(ch) != v else 0
                self.out[v] += self.out[self.fail[v]]
                q.append(v)

    def find(self, text):
        s, hits = 0, []
        for i, ch in enumerate(text):
            while s and ch not in self.next[s]:
                s = self.fail[s]
            s = self.next[s].get(ch, 0)
            for w in self.out[s]:
                hits.append((i - len(w) + 1, i + 1, w))
        return hits

# ---- 規格化：全形轉半形、去零寬與干擾符 ----
ZERO = dict.fromkeys(map(ord, "\u200b\u200c\u200d\ufeff\u00ad"), None)
def normalize(t):
    t = t.translate(ZERO)
    t = "".join(chr(ord(c) - 0xFEE0) if 0xFF01 <= ord(c) <= 0xFF5E else c
                for c in t)
    return re.sub(r"[\s.·*]+", "", t)

# ---- 拼音首碼鍵（極簡字典；實務用 pypinyin）----
PY = {"賭": "d", "博": "b", "博": "b", "愽": "b", "毒": "d", "品": "p"}
def pinyin_key(t):
    return "".join(PY.get(c, c) for c in t)

blocklist = ["賭博", "毒品", "詐騙"]
ac = AC()
for w in blocklist:
    ac.add(normalize(w))
    ac.add(pinyin_key(normalize(w)), tag="pinyin:" + w)   # 容錯層
ac.build()

def filter_text(text):
    norm = normalize(text)
    hits = ac.find(norm) + ac.find(pinyin_key(norm))   # 原文＋拼音容錯層
    real = [h for h in hits if not h[2].startswith("pinyin:")]
    fuzzy = [h for h in hits if h[2].startswith("pinyin:")]
    # 最長匹配遮蔽
    mask = list(norm)
    for lo, hi, w in sorted(real, key=lambda h: -(h[1] - h[0])):
        for i in range(lo, hi):
            mask[i] = "＊"
    verdict = "阻擋" if real else ("可疑→複審" if fuzzy else "通過")
    return verdict, "".join(mask)

for t in ["一起去賭博嗎", "毒.品交易", "賭愽一晚", "我去圖書館看書",
          "全形：賭博！", "線上詐騙很多"]:
    v, m = filter_text(t)
    print(f"{t} -> {v}  規格化遮蔽: {m}")