3939FENCE_RE = __import__ ("re" ).compile (r"^(`{3,}|~{3,})" )
4040COMMENT_RE = __import__ ("re" ).compile (r"^<!--.*-->$" )
4141
42+ # MkDocs Material extensions that should be recognized but whose
43+ # translatable content is extracted as separate paragraphs.
44+ # - !!! type ["title"] → admonition (note, warning, tip, etc.)
45+ # - ??? ["title"] → collapsible/details
46+ # - === "tab label" → content tabs
47+ ADMONITION_RE = __import__ ("re" ).compile (r'^!!!\s+\w+(\s+"[^"]*")?\s*$' )
48+ DETAILS_RE = __import__ ("re" ).compile (r'^\?\?\?(\s+"[^"]*")?\s*$' )
49+ TAB_RE = __import__ ("re" ).compile (r'^===\s+"[^"]*"\s*$' )
50+
4251# Characters that indicate a line is purely structural (not translatable):
4352# - Markdown headings and list markers
4453# - Table separators
@@ -57,6 +66,11 @@ def _is_translatable_paragraph(paragraph: str) -> bool:
5766 if COMMENT_RE .match (text ):
5867 return False
5968
69+ # Skip MkDocs Material structural directives that have no
70+ # translatable text content (e.g. bare "!!! note").
71+ if ADMONITION_RE .match (text ) and '"' not in text :
72+ return False
73+
6074 # Count characters that typically appear in natural language.
6175 alpha = sum (1 for c in text if c .isalpha ())
6276 if alpha == 0 :
@@ -191,9 +205,26 @@ def _relative_to_source(path: Path) -> str:
191205 return str (path .relative_to (SOURCE_DIR ))
192206
193207
194- def process_file (source_path : Path , dry_run : bool = False ) -> bool :
208+ def _write_po_from_paragraphs (po_path : Path , rel : str , paragraphs : list [str ], dry_run : bool = False ) -> bool :
209+ """Write a fresh .po file from extracted paragraphs, discarding any existing translations."""
210+ header = _po_header (str (rel ))
211+ body_entries = "\n \n " .join (f'msgid "{ p .replace (chr (34 ), chr (92 ) + chr (34 ))} \n msgstr ""' for p in paragraphs )
212+ if dry_run :
213+ print (f" [DRY-RUN] Would force-regenerate: { po_path } { len (paragraphs )} )
214+ return True
215+ po_path .parent .mkdir (parents = True , exist_ok = True )
216+ po_path .write_text (header + body_entries + "\n " , encoding = "utf-8" )
217+ print (f" Force-regenerated: { po_path } { len (paragraphs )} )
218+ return True
219+
220+
221+ def process_file (source_path : Path , dry_run : bool = False , force : bool = False ) -> bool :
195222 """Create or update the .po file for *source_path*.
196223
224+ If *force* is True, the entire .po file is regenerated from the source
225+ markdown, discarding any existing translations. Otherwise only new
226+ paragraphs are appended and existing translations are preserved.
227+
197228 Returns True if the .po file was created or modified and contains
198229 at least one untranslated entry.
199230 """
@@ -209,41 +240,101 @@ def process_file(source_path: Path, dry_run: bool = False) -> bool:
209240 if not paragraphs :
210241 return False
211242
243+ if force and po_path .exists ():
244+ return _write_po_from_paragraphs (po_path , str (rel ), paragraphs , dry_run )
245+
212246 if not po_path .exists ():
213- # Create new .po file.
214- header = _po_header (str (rel ))
215- body_entries = "\n \n " .join (f'msgid "{ p .replace (chr (34 ), chr (92 ) + chr (34 ))} \n msgstr ""' for p in paragraphs )
216- if dry_run :
217- print (f" [DRY-RUN] Would create: { po_path } )
218- return True
219- po_path .parent .mkdir (parents = True , exist_ok = True )
220- po_path .write_text (header + body_entries + "\n " , encoding = "utf-8" )
221- print (f" Created: { po_path } { len (paragraphs )} )
222- return True
247+ return _write_po_from_paragraphs (po_path , str (rel ), paragraphs , dry_run )
223248
224- # Update existing .po file .
249+ # Incremental update: detect new, removed, and modified paragraphs .
225250 po = pofile (str (po_path ))
226- existing_msgids = {entry .msgid for entry in po if entry .msgid }
251+ entries_by_msgid : dict [str , POEntry ] = {}
252+ for entry in po :
253+ if entry .msgid :
254+ entries_by_msgid [entry .msgid ] = entry
227255
228256 new_count = 0
257+ modified_count = 0
258+ removed_count = 0
259+
229260 for para in paragraphs :
230- if para not in existing_msgids :
261+ if para in entries_by_msgid :
262+ continue
263+ # Check if this is a modification of an existing paragraph
264+ # (similar msgid that got updated in the source).
265+ matched = _find_similar_entry (para , entries_by_msgid )
266+ if matched is not None :
267+ old_entry = entries_by_msgid .pop (matched )
268+ new_entry = _build_po_entry (para )
269+ new_entry .msgstr = "" # force re-translation for modified paragraph
270+ po .append (new_entry )
271+ modified_count += 1
272+ else :
231273 po .append (_build_po_entry (para ))
232- existing_msgids .add (para )
233274 new_count += 1
234275
235- if new_count == 0 :
276+ # Mark paragraphs that no longer exist in source as obsolete.
277+ # Any entry still in entries_by_msgid is not in the new paragraphs list.
278+ for old_entry in entries_by_msgid .values ():
279+ old_entry .obsolete = True
280+ removed_count += 1
281+
282+ change_count = new_count + modified_count + removed_count
283+ if change_count == 0 :
236284 return _has_empty_msgstr (po )
237285
238286 if dry_run :
239- print (f" [DRY-RUN] Would add { new_count } { po_path } )
287+ parts = []
288+ if new_count :
289+ parts .append (f"+{ new_count } )
290+ if modified_count :
291+ parts .append (f"~{ modified_count } )
292+ if removed_count :
293+ parts .append (f"-{ removed_count } )
294+ print (f" [DRY-RUN] Would update: { po_path } { ', ' .join (parts )} )
240295 return True
241296
242297 po .save (str (po_path ))
243- print (f" Updated: { po_path } { new_count } )
298+ parts = []
299+ if new_count :
300+ parts .append (f"+{ new_count } )
301+ if modified_count :
302+ parts .append (f"~{ modified_count } )
303+ if removed_count :
304+ parts .append (f"-{ removed_count } )
305+ print (f" Updated: { po_path } { ', ' .join (parts )} )
244306 return True
245307
246308
309+ def _find_similar_entry (new_para : str , entries : dict [str , POEntry ]) -> str | None :
310+ """Check if *new_para* is likely a modified version of an existing entry.
311+
312+ Returns the msgid of the matching entry, or None.
313+ Uses a simple heuristic: the first non-trivial line of each paragraph
314+ must be identical, which handles cases where a paragraph was edited
315+ by adding/removing lines in the middle or end.
316+ """
317+ new_first = _first_significant_line (new_para )
318+ if not new_first :
319+ return None
320+ for msgid in entries :
321+ if _first_significant_line (msgid ) == new_first :
322+ return msgid
323+ return None
324+
325+
326+ def _first_significant_line (text : str ) -> str :
327+ """Return the first non-empty, non-link-reference line of *text*."""
328+ for line in text .split ("\n " ):
329+ stripped = line .strip ()
330+ if not stripped :
331+ continue
332+ if stripped .startswith ("[" ) and stripped .endswith (")" ) and "](" in stripped :
333+ continue
334+ return stripped
335+ return ""
336+
337+
247338def _has_empty_msgstr (po : POFile ) -> bool :
248339 """Return True if *po* contains at least one entry with an empty msgstr."""
249340 return any (entry .msgid and not entry .msgstr for entry in po if not entry .obsolete )
@@ -261,6 +352,11 @@ def main() -> int:
261352 action = "store_true" ,
262353 help = "Do not write .po files, just report changes." ,
263354 )
355+ parser .add_argument (
356+ "--force" ,
357+ action = "store_true" ,
358+ help = "Force full regeneration of ALL .po files, discarding existing translations." ,
359+ )
264360 args = parser .parse_args ()
265361
266362 # Collect English markdown files.
@@ -276,10 +372,13 @@ def main() -> int:
276372
277373 print (f"Scanning { len (en_files )} )
278374
375+ if args .force :
376+ print ("--force enabled: regenerating ALL .po files from scratch" )
377+
279378 needs_translation : list [str ] = []
280379 for source_path in en_files :
281380 try :
282- if process_file (source_path , dry_run = args .dry_run ):
381+ if process_file (source_path , dry_run = args .dry_run , force = args . force ):
283382 rel = _relative_to_source (source_path )
284383 po_rel = str (LOCALE_DIR / Path (rel ).with_suffix (".po" ))
285384 needs_translation .append (po_rel )
0 commit comments