Coverage for baibu/localization/catalogues.py: 100%

1"""Working with gettext catalogues (.po/.pot/.mo) through polib. 

2 

3Pure functions: nothing here touches the database or file storage. 

4""" 

5 

6import gettext 

7import hashlib 

8import re 

9from dataclasses import dataclass 

10from pathlib import Path 

11 

12import django 

13import polib 

14from django.utils.translation import gettext as _ 

15from django.utils.translation import to_locale 

16 

17DEFAULT_PLURAL_FORMS = "nplurals=2; plural=(n != 1);" 

18MAX_PLURALS = 6 

19PLURAL_FORMS_RE = re.compile(r"^\s*nplurals\s*=\s*(?P<n>\d+)\s*;\s*plural\s*=\s*(?P<expr>[^;]+?)\s*;?\s*$") 

20# printf-style conversions as Python's % operator understands them. 

21PRINTF_RE = re.compile(r"%(?:\((?P<name>[^)]*)\))?[#0\- +]*(?:\*|\d+)?(?:\.(?:\*|\d+))?[hlL]?(?P<type>.)", re.DOTALL) 

22PRINTF_TYPES = set("diouxXeEfFgGcrsa%") 

23 

24 

25class CatalogueError(ValueError): 

26 """A catalogue or a value for it is not valid.""" 

27 

28 

29@dataclass(frozen=True) 

30class Stats: 

31 total: int 

32 translated: int 

33 fuzzy: int 

34 

35 @property 

36 def untranslated(self) -> int: 

37 return self.total - self.translated - self.fuzzy 

38 

39 @property 

40 def percent(self) -> int: 

41 return round(100 * self.translated / self.total) if self.total else 100 

42 

43 

44def parse(data: bytes | str) -> polib.POFile: 

45 text = data.decode("utf-8") if isinstance(data, bytes) else data 

46 if not text.strip(): 

47 return polib.POFile() 

48 try: 

49 return polib.pofile(text) 

50 except (OSError, ValueError) as exc: 

51 msg = _("Not a valid PO file: %(error)s") % {"error": exc} 

52 raise CatalogueError(msg) from exc 

53 

54 

55def dump(po: polib.POFile) -> bytes: 

56 return str(po).encode("utf-8") 

57 

58 

59def compile_mo(po: polib.POFile) -> bytes: 

60 """Compile to the binary .mo format. Only translated, non-fuzzy entries are included.""" 

61 return po.to_binary() 

62 

63 

64def entry_key(entry: polib.POEntry) -> str: 

65 """A stable identifier for an entry (context, message and plural).""" 

66 raw = "\x04".join((entry.msgctxt or "", entry.msgid, entry.msgid_plural or "")) 

67 return hashlib.sha256(raw.encode()).hexdigest()[:16] 

68 

69 

70def active_entries(po: polib.POFile) -> list[polib.POEntry]: 

71 return [entry for entry in po if not entry.obsolete] 

72 

73 

74def entry_state(entry: polib.POEntry) -> str: 

75 if entry.fuzzy: 

76 return "fuzzy" 

77 return "translated" if entry.translated() else "untranslated" 

78 

79 

80def stats(po: polib.POFile) -> Stats: 

81 entries = active_entries(po) 

82 return Stats( 

83 total=len(entries), 

84 translated=sum(1 for entry in entries if entry.translated()), 

85 fuzzy=sum(1 for entry in entries if entry.fuzzy), 

86 ) 

87 

88 

89def content_hash(po: polib.POFile) -> str: 

90 """Hash of the entries (not the header), to tell whether a template changed.""" 

91 digest = hashlib.sha256() 

92 for entry in po: 

93 digest.update(repr((entry.msgctxt, entry.msgid, entry.msgid_plural, entry.occurrences)).encode()) 

94 digest.update(repr((sorted(entry.flags), entry.comment)).encode()) 

95 return digest.hexdigest() 

96 

97 

98# Plural forms ------------------------------------------------------------------ 

99 

100 

101def parse_plural_forms(value: str) -> tuple[int, str]: 

102 """Validate a ``Plural-Forms`` header and return ``(nplurals, expression)``.""" 

103 match = PLURAL_FORMS_RE.match(value or "") 

104 if not match: 

105 msg = _("Plural forms must look like “nplurals=2; plural=(n != 1);”.") 

106 raise CatalogueError(msg) 

107 count = int(match["n"]) 

108 if not 1 <= count <= MAX_PLURALS: 

109 msg = _("The number of plural forms must be between 1 and %(max)d.") % {"max": MAX_PLURALS} 

110 raise CatalogueError(msg) 

111 expression = match["expr"] 

112 try: 

113 gettext.c2py(expression) 

114 except (ValueError, RecursionError) as exc: 

115 msg = _("The plural expression is not valid: %(error)s") % {"error": exc} 

116 raise CatalogueError(msg) from exc 

117 return count, expression 

118 

119 

120def plural_forms(po: polib.POFile) -> str: 

121 return po.metadata.get("Plural-Forms") or DEFAULT_PLURAL_FORMS 

122 

123 

124def plural_count(po: polib.POFile) -> int: 

125 try: 

126 return parse_plural_forms(plural_forms(po))[0] 

127 except CatalogueError: 

128 return 2 

129 

130 

131def plural_examples(value: str, *, limit: int = 3) -> list[list[int]]: 

132 """For each plural form, a few numbers that use it (to label the fields).""" 

133 count, expression = parse_plural_forms(value) 

134 function = gettext.c2py(expression) 

135 examples: list[list[int]] = [[] for _index in range(count)] 

136 for number in range(1000): 

137 index = function(number) 

138 if 0 <= index < count and len(examples[index]) < limit: 

139 examples[index].append(number) 

140 return examples 

141 

142 

143def known_plural_forms(language_code: str) -> str | None: 

144 """The plural forms Django's own catalogue uses for the language, if it has one.""" 

145 path = Path(django.__file__).parent / "conf" / "locale" / to_locale(language_code) / "LC_MESSAGES" / "django.po" 

146 if not path.exists(): 

147 return None 

148 value = polib.pofile(str(path)).metadata.get("Plural-Forms") 

149 try: 

150 parse_plural_forms(value) 

151 except CatalogueError: 

152 return None 

153 return value 

154 

155 

156def normalise_plurals(po: polib.POFile) -> None: 

157 """Give every plural entry exactly one translation slot per plural form.""" 

158 count = plural_count(po) 

159 for entry in po: 

160 if not entry.msgid_plural: 

161 continue 

162 entry.msgstr_plural = {index: entry.msgstr_plural.get(index, "") for index in range(count)} 

163 

164 

165def set_header(po: polib.POFile, *, language_code: str, plural_forms_value: str) -> None: 

166 po.metadata.update( 

167 { 

168 "Project-Id-Version": po.metadata.get("Project-Id-Version") or "interface", 

169 "Language": to_locale(language_code), 

170 "MIME-Version": "1.0", 

171 "Content-Type": "text/plain; charset=UTF-8", 

172 "Content-Transfer-Encoding": "8bit", 

173 "Plural-Forms": plural_forms_value, 

174 } 

175 ) 

176 normalise_plurals(po) 

177 

178 

179# Building and merging ------------------------------------------------------------ 

180 

181 

182def new_catalogue(template: polib.POFile, *, language_code: str, base: polib.POFile | None = None) -> polib.POFile: 

183 """A catalogue for ``language_code`` with the template's entries. 

184 

185 Translations in ``base`` (for example a deployment's existing .po file) 

186 are kept where the message still exists. 

187 """ 

188 po = base if base is not None else polib.POFile() 

189 po.merge(template) 

190 value = (base.metadata.get("Plural-Forms") if base is not None else None) or known_plural_forms(language_code) 

191 try: 

192 parse_plural_forms(value) 

193 except CatalogueError: 

194 value = DEFAULT_PLURAL_FORMS 

195 set_header(po, language_code=language_code, plural_forms_value=value) 

196 for entry in active_entries(po): 

197 if entry.translated() and placeholder_problem(entry, entry.msgstr, *entry.msgstr_plural.values()): 

198 entry.fuzzy = True 

199 return po 

200 

201 

202def merge_template(po: polib.POFile, template: polib.POFile) -> polib.POFile: 

203 """Bring a catalogue up to date with a new template, keeping translations. 

204 

205 Messages no longer in the code become obsolete (kept in the file, never 

206 shown or compiled), so their translations return if the message does. 

207 """ 

208 po.merge(template) 

209 normalise_plurals(po) 

210 return po 

211 

212 

213def find_entry(po: polib.POFile, key: str) -> polib.POEntry | None: 

214 for entry in active_entries(po): 

215 if entry_key(entry) == key: 

216 return entry 

217 return None 

218 

219 

220# Validation ----------------------------------------------------------------------- 

221 

222 

223def _placeholders(text: str) -> tuple[set[str], int, bool]: 

224 """Named placeholders, count of unnamed ones, and whether a stray % was found.""" 

225 named: set[str] = set() 

226 unnamed = 0 

227 stray = False 

228 for match in PRINTF_RE.finditer(text): 

229 if match["type"] not in PRINTF_TYPES: 

230 stray = True 

231 elif match["type"] == "%": 

232 stray = stray or match.group(0) != "%%" 

233 elif match["name"] is not None: 

234 named.add(match["name"]) 

235 else: 

236 unnamed += 1 

237 if text.count("%") > sum(m.group(0).count("%") for m in PRINTF_RE.finditer(text)): 

238 stray = True 

239 return named, unnamed, stray 

240 

241 

242def placeholder_problem(entry: polib.POEntry, *translations: str) -> str | None: 

243 """Return a message if a translation would break the format string, else ``None``. 

244 

245 Only entries flagged ``python-format`` are checked: their translation may 

246 use the same named placeholders (``%(name)s``) as the original, in any 

247 order, and must keep the same number of unnamed ones (``%s``). 

248 """ 

249 if "python-format" not in entry.flags: 

250 return None 

251 sources = [entry.msgid] + ([entry.msgid_plural] if entry.msgid_plural else []) 

252 allowed: set[str] = set() 

253 unnamed_counts: set[int] = set() 

254 for source in sources: 

255 named, unnamed, _stray = _placeholders(source) 

256 allowed |= named 

257 unnamed_counts.add(unnamed) 

258 for text in translations: 

259 if not text: 

260 continue 

261 named, unnamed, stray = _placeholders(text) 

262 if stray: 

263 return _("Write a percent sign as %(sign)s.") % {"sign": "%%"} 

264 if unknown := named - allowed: 

265 names = ", ".join(f"%({name})s" for name in sorted(unknown)) 

266 return _("Unknown placeholder: %(names)s. Use only the placeholders in the original.") % {"names": names} 

267 if unnamed not in unnamed_counts: 

268 return _("Keep the same number of %(placeholder)s placeholders as the original.") % {"placeholder": "%s"} 

269 return None