119 lines
3.5 KiB
Python
119 lines
3.5 KiB
Python
"""Pure-Python string metrics used when the optional ``jellyfish`` wheel is absent.
|
|
|
|
The production project may still use jellyfish as an accelerator. These
|
|
implementations keep the business matcher deterministic and testable in
|
|
restricted/offline environments (including CPython 3.13 builders without
|
|
access to PyPI).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
import unicodedata
|
|
|
|
|
|
def _norm(value: str) -> str:
|
|
value = unicodedata.normalize("NFKD", str(value).lower())
|
|
value = "".join(ch for ch in value if not unicodedata.combining(ch))
|
|
return re.sub(r"[^a-z0-9]+", "", value)
|
|
|
|
|
|
def levenshtein_distance(a: str, b: str) -> int:
|
|
a, b = str(a), str(b)
|
|
if a == b:
|
|
return 0
|
|
if not a:
|
|
return len(b)
|
|
if not b:
|
|
return len(a)
|
|
if len(a) > len(b):
|
|
a, b = b, a
|
|
previous = list(range(len(a) + 1))
|
|
for i, cb in enumerate(b, 1):
|
|
current = [i]
|
|
for j, ca in enumerate(a, 1):
|
|
current.append(min(
|
|
current[-1] + 1,
|
|
previous[j] + 1,
|
|
previous[j - 1] + (ca != cb),
|
|
))
|
|
previous = current
|
|
return previous[-1]
|
|
|
|
|
|
def jaro_similarity(a: str, b: str) -> float:
|
|
a, b = str(a), str(b)
|
|
if a == b:
|
|
return 1.0
|
|
if not a or not b:
|
|
return 0.0
|
|
match_distance = max(len(a), len(b)) // 2 - 1
|
|
match_distance = max(match_distance, 0)
|
|
a_match = [False] * len(a)
|
|
b_match = [False] * len(b)
|
|
matches = 0
|
|
for i, ca in enumerate(a):
|
|
lo = max(0, i - match_distance)
|
|
hi = min(i + match_distance + 1, len(b))
|
|
for j in range(lo, hi):
|
|
if b_match[j] or ca != b[j]:
|
|
continue
|
|
a_match[i] = True
|
|
b_match[j] = True
|
|
matches += 1
|
|
break
|
|
if matches == 0:
|
|
return 0.0
|
|
a_chars = [a[i] for i in range(len(a)) if a_match[i]]
|
|
b_chars = [b[j] for j in range(len(b)) if b_match[j]]
|
|
transpositions = sum(x != y for x, y in zip(a_chars, b_chars)) / 2.0
|
|
m = float(matches)
|
|
return (m / len(a) + m / len(b) + (m - transpositions) / m) / 3.0
|
|
|
|
|
|
def jaro_winkler_similarity(a: str, b: str, *, scaling: float = 0.1) -> float:
|
|
jaro = jaro_similarity(a, b)
|
|
prefix = 0
|
|
for ca, cb in zip(str(a), str(b)):
|
|
if ca != cb or prefix == 4:
|
|
break
|
|
prefix += 1
|
|
return jaro + prefix * scaling * (1.0 - jaro)
|
|
|
|
|
|
def metaphone(value: str) -> str:
|
|
"""Small deterministic phonetic key tuned for Portuguese ASR item names.
|
|
|
|
It intentionally mirrors the role (not the implementation) of Jellyfish's
|
|
Metaphone: collapse spelling variants so the matcher can rank transcript
|
|
errors. The key is conservative and is only one signal alongside
|
|
Jaro-Winkler/token similarity.
|
|
"""
|
|
s = _norm(value)
|
|
if not s:
|
|
return ""
|
|
replacements = (
|
|
("sch", "x"), ("sh", "x"), ("ch", "x"), ("ph", "f"),
|
|
("th", "t"), ("nh", "n"), ("lh", "l"), ("qu", "k"),
|
|
("gu", "g"), ("ck", "k"),
|
|
)
|
|
for old, new in replacements:
|
|
s = s.replace(old, new)
|
|
trans = str.maketrans({
|
|
"c": "k", "q": "k", "k": "k",
|
|
"g": "j", "j": "j",
|
|
"v": "f", "f": "f",
|
|
"z": "s", "s": "s", "x": "x",
|
|
"d": "t", "t": "t",
|
|
"b": "p", "p": "p",
|
|
"y": "i", "w": "u",
|
|
})
|
|
s = s.translate(trans)
|
|
first = s[0]
|
|
tail = "".join(ch for ch in s[1:] if ch not in "aeiou")
|
|
code = first + tail
|
|
code = re.sub(r"(.)\1+", r"\1", code)
|
|
return code.upper()
|
|
|
|
|
|
__all__ = ["jaro_winkler_similarity", "levenshtein_distance", "metaphone"]
|