Coverage for baibu/localization/runtime.py: 100%

1"""Loading published translations into running processes. 

2 

3Published catalogues live in file storage, which every web and worker 

4process can reach. Each process copies the published ``.mo`` files into 

5``LOCALIZATION_PUBLISHED_DIR`` (listed first in ``LOCALE_PATHS``) and resets 

6Django's translation caches. A version marker in the cache (the id of the 

7newest publication) tells a process when to do this again; it checks at most 

8every ``LOCALIZATION_SYNC_SECONDS``, before a request is handled or a Celery 

9task runs. 

10""" 

11 

12import gettext 

13import logging 

14import os 

15import tempfile 

16import threading 

17import time 

18from pathlib import Path 

19 

20from django.conf import settings 

21from django.core.cache import cache 

22from django.utils.translation import to_locale 

23from django.utils.translation import trans_real 

24 

25from baibu.core import storage 

26 

27logger = logging.getLogger(__name__) 

28 

29VERSION_CACHE_KEY = "localization:published-version" 

30NOTHING_PUBLISHED = "none" 

31MO_NAME = "django.mo" 

32 

33_lock = threading.Lock() 

34_state = {"loaded": None, "checked_at": 0.0} 

35 

36 

37def published_dir() -> Path: 

38 return Path(settings.LOCALIZATION_PUBLISHED_DIR) 

39 

40 

41def published_version() -> str: 

42 """The id of the newest publication, from the cache or the database.""" 

43 version = cache.get(VERSION_CACHE_KEY) 

44 if version is None: 

45 from .models import TranslationPublication 

46 

47 newest = TranslationPublication.objects.order_by("-published_at", "-id").values_list("id", flat=True).first() 

48 version = str(newest) if newest else NOTHING_PUBLISHED 

49 cache.set(VERSION_CACHE_KEY, version, timeout=None) 

50 return version 

51 

52 

53def announce(version: str) -> None: 

54 """Tell every process that a new publication exists.""" 

55 cache.set(VERSION_CACHE_KEY, version, timeout=None) 

56 

57 

58def reset_translation_caches() -> None: 

59 """Make Django read the catalogue files again. 

60 

61 Clears what Django's own autoreloader clears when a .mo file changes, 

62 plus the language-availability caches, so a language whose first 

63 catalogue was just published becomes available. The active translation 

64 of other threads is left alone; each request activates its language 

65 again through LocaleMiddleware. 

66 """ 

67 gettext._translations.clear() 

68 trans_real._translations = {} 

69 trans_real._default = None 

70 trans_real.check_for_language.cache_clear() 

71 trans_real.get_languages.cache_clear() 

72 trans_real.get_supported_language_variant.cache_clear() 

73 

74 

75def _write_atomic(path: Path, data: bytes) -> None: 

76 path.parent.mkdir(parents=True, exist_ok=True) 

77 handle, temporary = tempfile.mkstemp(dir=path.parent, prefix=".tmp-", suffix=".mo") 

78 try: 

79 with os.fdopen(handle, "wb") as file: 

80 file.write(data) 

81 Path(temporary).replace(path) 

82 except BaseException: 

83 Path(temporary).unlink(missing_ok=True) 

84 raise 

85 

86 

87def materialise() -> bool: 

88 """Copy the newest publication of each language into the published directory. 

89 

90 Returns ``False`` if a file could not be read from storage (the next 

91 check tries again). 

92 """ 

93 from .models import TranslationPublication 

94 from .services import translatable_languages 

95 

96 codes = [code for code, _name in translatable_languages()] 

97 newest = ( 

98 TranslationPublication.objects.filter(language_code__in=codes) 

99 .order_by("language_code", "-version") 

100 .distinct("language_code") 

101 ) 

102 root = published_dir() 

103 wanted: set[Path] = set() 

104 complete = True 

105 for publication in newest: 

106 target = root / to_locale(publication.language_code) / "LC_MESSAGES" / MO_NAME 

107 wanted.add(target) 

108 data = storage.read_bytes(publication.mo_key) 

109 if data is None: 

110 logger.error("Published translations %s are missing from storage (%s)", publication, publication.mo_key) 

111 complete = False 

112 continue 

113 if not target.exists() or target.read_bytes() != data: 

114 _write_atomic(target, data) 

115 if root.is_dir(): 

116 for stale in root.glob(f"*/LC_MESSAGES/{MO_NAME}"): 

117 if stale not in wanted: 

118 stale.unlink(missing_ok=True) 

119 return complete 

120 

121 

122def sync(*, force: bool = False) -> bool: 

123 """Load newly published translations into this process. Returns whether it reloaded.""" 

124 with _lock: 

125 _state["checked_at"] = time.monotonic() 

126 version = published_version() 

127 if not force and version == _state["loaded"]: 

128 return False 

129 complete = materialise() 

130 reset_translation_caches() 

131 _state["loaded"] = version if complete else None 

132 logger.info("Loaded published translations (version %s)", version) 

133 return True 

134 

135 

136def sync_if_due() -> None: 

137 """Called before each request and task; cheap unless a check is due.""" 

138 if not settings.LOCALIZATION_SYNC: 

139 return 

140 checked_at = _state["checked_at"] 

141 if checked_at and time.monotonic() - checked_at < settings.LOCALIZATION_SYNC_SECONDS: 

142 return 

143 try: 

144 sync() 

145 except Exception: 

146 # Translations must never take a page or a task down. 

147 logger.exception("Could not load published translations") 

148 

149 

150def sync_before_task(**kwargs) -> None: 

151 sync_if_due() 

152 

153 

154def forget() -> None: 

155 """Forget what this process loaded (used by tests).""" 

156 _state["loaded"] = None 

157 _state["checked_at"] = 0.0