Coverage for baibu/submissions/text.py: 100%

1"""Text normalisation and rule-based redaction. 

2 

3These rules run on every submission before any model sees it, so a 

4deployment without a model still gets consistent text and basic redaction. 

5""" 

6 

7import hashlib 

8import re 

9import unicodedata 

10from dataclasses import dataclass 

11from dataclasses import field 

12 

13_LINE_BREAKS = re.compile(r"\r\n?|[\u2028\u2029]") 

14_SPACES = re.compile(r"[^\S\n]+") 

15_BLANK_LINES = re.compile(r"\n{3,}") 

16_WHITESPACE = re.compile(r"\s+") 

17 

18_EMAIL = re.compile(r"(?<![\w.+-])[\w.+-]+@[\w-]+(?:\.[\w-]+)+") 

19_URL = re.compile(r"\b(?:https?://|www\.)\S+", re.IGNORECASE) 

20# Nine or more digits, optionally with a leading + and separators. Shorter 

21# runs (years, dates, prices) are left alone. 

22_PHONE = re.compile(r"(?<![\w+])\+?\d(?:[\s().-]{0,2}\d){8,}(?!\w)") 

23 

24REDACTIONS = (("email", _EMAIL, "[email]"), ("link", _URL, "[link]"), ("phone", _PHONE, "[phone]")) 

25 

26 

27def normalise(text: str) -> str: 

28 """NFC, Unix line breaks, no control characters, tidy spacing.""" 

29 text = unicodedata.normalize("NFC", text or "") 

30 text = _LINE_BREAKS.sub("\n", text) 

31 text = "".join(ch for ch in text if ch in "\n\t" or unicodedata.category(ch) != "Cc") 

32 lines = [_SPACES.sub(" ", line).strip() for line in text.split("\n")] 

33 return _BLANK_LINES.sub("\n\n", "\n".join(lines)).strip() 

34 

35 

36def content_hash(text: str) -> str: 

37 """Hash that ignores differences in spacing, for duplicate detection.""" 

38 collapsed = _WHITESPACE.sub(" ", normalise(text)).strip() 

39 return hashlib.sha256(collapsed.encode()).hexdigest() 

40 

41 

42def word_count(text: str) -> int: 

43 return len(text.split()) 

44 

45 

46@dataclass 

47class Redacted: 

48 text: str 

49 counts: dict[str, int] = field(default_factory=dict) 

50 

51 @property 

52 def total(self) -> int: 

53 return sum(self.counts.values()) 

54 

55 

56def redact(text: str) -> Redacted: 

57 """Replace email addresses, links and phone numbers with placeholders.""" 

58 counts = {} 

59 for kind, pattern, placeholder in REDACTIONS: 

60 text, n = pattern.subn(placeholder, text) 

61 if n: 

62 counts[kind] = n 

63 return Redacted(text=text, counts=counts)