Reduce PO diff noise by filtering source references

Author: methane

.github/workflows/update.yml

@@ -60,8 +60,8 @@ jobs:
       - name: Wrap catalog message files
         run: |
           powrap --modified
-      - name: Ignore timestamp-only changes
-        run: python scripts/translations.py restore-timestamp-only .
+      - name: Normalize PO references and ignore timestamp-only changes
+        run: python scripts/translations.py restore-timestamp-only --references .
       - name: Commit and push changes
         if: github.repository == 'python/python-docs-ja'
         run: |
@@ -108,7 +108,7 @@ jobs:
       - name: Install dependencies
         run: |
           sudo apt-get update -y && sudo apt-get install gettext -y
-          pip install pomerge powrap
+          pip install pomerge powrap 'polib>=1.2.0'
 
       - name: Merge overwriting on stable release branch
         run: |
@@ -117,8 +117,8 @@ jobs:
       - name: Wrap catalog message files
         run: |
           powrap --modified -C "${{ matrix.branch }}"
-      - name: Ignore timestamp-only changes
-        run: python "${{ env.CPYTHON_BRANCH }}/scripts/translations.py" restore-timestamp-only "${{ matrix.branch }}"
+      - name: Normalize PO references and ignore timestamp-only changes
+        run: python "${{ env.CPYTHON_BRANCH }}/scripts/translations.py" restore-timestamp-only --references "${{ matrix.branch }}"
 
       - name: Commit and push changes
         if: github.repository == 'python/python-docs-ja'

scripts/translations.py

@@ -27,33 +27,95 @@ def without_po_timestamps(content):
     return content[:header.start()] + normalized + content[header.end():]
 
 
-def restore_timestamp_only_changes(root):
+def po_references(content):
+    """Locate reference comments without reserializing the PO file."""
+    import polib
+
+    lines = content.splitlines(keepends=True)
+    encoding = polib.detect_encoding(content)
+    entries = polib.pofile(content.decode(encoding), encoding=encoding)
+    references = {}
+    for index, entry in enumerate(entries):
+        if entry.obsolete:
+            continue
+        end = entries[index + 1].linenum - 1 if index + 1 < len(entries) else len(lines)
+        positions = [
+            pos for pos in range(entry.linenum - 1, end)
+            if lines[pos].startswith(b"#:")
+        ]
+        key = (entry.msgctxt, entry.msgid, entry.msgid_plural)
+        references[key] = (entry, positions)
+    return lines, references
+
+
+def normalize_po_references(original, incoming):
+    """Drop references for new/changed messages; ignore line-number updates."""
+    old_lines, old_references = po_references(original)
+    lines, references = po_references(incoming)
+    for key, (entry, positions) in references.items():
+        if not positions:
+            continue
+        old_entry, old_positions = old_references.get(key, (None, []))
+        if old_entry is None:
+            changed = True
+        else:
+            changed = (entry.msgstr, entry.msgstr_plural) != (old_entry.msgstr, old_entry.msgstr_plural)
+        if changed or (old_entry is not None and not old_positions):
+            for pos in positions:
+                lines[pos] = b''
+            continue
+        # Keep added/removed references and file renames for unchanged translations.
+        if not old_positions or (
+            [path for path, _ in entry.occurrences]
+            != [path for path, _ in old_entry.occurrences]
+        ):
+            continue
+        lines[positions[0]] = b"".join(old_lines[pos] for pos in old_positions)
+        for pos in positions[1:]:
+            lines[pos] = b""
+    return b"".join(lines)
+
+
+def restore_timestamp_only_changes(root, *, normalize_references=False):
     """Restore PO files whose unstaged changes only affect header dates.
 
-    Run after wrapping, so formatting differences in downloaded or merged files
-    do not prevent timestamp-only changes from being recognized.
+    Optionally drop references for new/changed messages and restore other line numbers.
+    Run after wrapping so formatting differences do not mask date-only changes.
     """
     result = subprocess.run(
         ["git", "-C", str(root), "diff", "--name-only", "-z", "--", "*.po"],
         check=True, stdout=subprocess.PIPE,
     )
-    for name in result.stdout.split(b"\0"):
+    names = result.stdout.split(b"\0")
+    untracked = set()
+    if normalize_references:
+        untracked = set(subprocess.run(
+            ["git", "-C", str(root), "ls-files", "--others", "--exclude-standard",
+             "-z", "--", "*.po"], check=True, stdout=subprocess.PIPE,
+        ).stdout.split(b"\0"))
+        names.extend(sorted(untracked))
+    for name in names:
         if not name:
             continue
         path = os.fsdecode(name)
         target = root / path
         if not target.is_file():
             continue
-        original = subprocess.run(
+        original = b"" if name in untracked else subprocess.run(
             ["git", "-C", str(root), "show", f":{path}"],
             check=True, stdout=subprocess.PIPE,
         ).stdout
-        if (
-            without_po_timestamps(target.read_bytes())
-            == without_po_timestamps(original)
-        ):
-            target.write_bytes(original)
+        incoming = target.read_bytes()
+        restored = incoming
+        if normalize_references:
+            restored = normalize_po_references(original, restored)
+            if restored != incoming:
+                print(f"Normalized PO references: {path}")
+        if without_po_timestamps(restored) == without_po_timestamps(original):
+            restored = original
             print(f"Ignored timestamp-only change: {path}")
+        if restored != incoming:
+            target.write_bytes(restored)
 
 
 def active_catalogs(config_path):
@@ -183,14 +245,18 @@ def main():
     merge_parser.add_argument("destination", type=Path)
     restore_parser = commands.add_parser("restore-timestamp-only")
     restore_parser.add_argument("root", type=Path)
+    restore_parser.add_argument(
+        "--references", action="store_true",
+        help="drop references for new/changed messages and ignore other line-number updates",
+    )
     args = parser.parse_args()
     if args.command == "sync":
         sync(args.source, args.destination, args.language,
              skip_missing_resources=args.skip_missing_resources)
     elif args.command == "merge":
         merge(args.source, args.destination)
     else:
-        restore_timestamp_only_changes(args.root)
+        restore_timestamp_only_changes(args.root, normalize_references=args.references)
 
 
 if __name__ == "__main__":

tests/test_translations.py

@@ -150,6 +150,114 @@ def test_restore_command_handles_multiple_files_and_explicit_root(self):
         self.assertEqual((self.destination / "translated.po").read_bytes(), translated)
         self.assertEqual((self.destination / "library/with space.po").read_bytes(), DATED_PO)
 
+    def test_normalize_references_and_dates_before_staging(self):
+        for newline in (b"\n", b"\r\n"):
+            with self.subTest(newline=newline):
+                before = DATED_PO.replace(b"\n", newline)
+                changed = before.replace(b"example.rst:10", b"example.rst:200")
+                changed = changed.replace(b"2026-09-29", b"2026-10-24")
+                for name in ("unchanged.po", "translated.po", "deleted.po"):
+                    (self.destination / name).write_bytes(before)
+                self.track()
+                (self.destination / "unchanged.po").write_bytes(changed)
+                (self.destination / "translated.po").write_bytes(
+                    changed.replace('翻訳'.encode(), '新しい訳'.encode())
+                )
+                (self.destination / "deleted.po").unlink()
+                new = self.destination / ("new-crlf.po" if newline == b"\r\n" else "new-lf.po")
+                new.write_bytes(changed)
+                subprocess.run(
+                    [sys.executable, translations.__file__, "restore-timestamp-only",
+                     "--references", str(self.destination)],
+                    check=True, stdout=subprocess.PIPE, cwd=self.source,
+                )
+                self.assertEqual((self.destination / "unchanged.po").read_bytes(), before)
+                expected = before.replace('翻訳'.encode(), '新しい訳'.encode())
+                expected = expected.replace(b"2026-09-29", b"2026-10-24")
+                expected = expected.replace(b"#: example.rst:10" + newline, b"")
+                self.assertEqual((self.destination / "translated.po").read_bytes(), expected)
+                self.assertEqual(new.read_bytes(), changed.replace(b"#: example.rst:200" + newline, b""))
+                self.assertFalse((self.destination / "deleted.po").exists())
+                self.track()
+                staged = subprocess.check_output(
+                    ["git", "-C", str(self.destination), "show", ":translated.po"]
+                )
+                self.assertEqual(staged, expected)
+
+    def test_normalize_references_preserves_other_entry_changes(self):
+        changed = DATED_PO.replace(b"example.rst:10", b"example.rst:200")
+        for old, new, action in [
+            (b'msgid "example"', b'msgid "new source"', "drop"),
+            (b'msgid "example"', b'msgctxt "new context"\nmsgid "example"', "drop"),
+            (b"example.rst:200", b"renamed.rst:200", "keep"),
+            (b"example.rst:200", b"example.rst:200 other.rst:30", "keep"),
+            (b'#: example.rst:200\n', b'', "keep"),
+            (b'msgid "example"', b'#, fuzzy\nmsgid "example"', "restore"),
+            (b'msgid "example"', b'#. new comment\nmsgid "example"', "restore"),
+            ('msgstr "翻訳"'.encode(), 'msgstr "新しい訳"'.encode(), "drop"),
+        ]:
+            with self.subTest(change=new):
+                incoming = changed.replace(old, new)
+                expected = incoming
+                if action == "drop":
+                    expected = incoming.replace(b"#: example.rst:200\n", b"")
+                elif action == "restore":
+                    expected = incoming.replace(b"example.rst:200", b"example.rst:10")
+                self.assertEqual(translations.normalize_po_references(DATED_PO, incoming),
+                                 expected)
+
+    def test_restore_line_numbers_matches_context_plural_and_reordered_entries(self):
+        header = DATED_PO.split(b"#: example.rst:10")[0]
+        first = (b'#: one.rst:10 two.rst:20\nmsgctxt "first"\n'
+                 b'msgid "same"\nmsgstr "first translation"\n\n')
+        second = (b'#: one.rst:30\nmsgctxt "second"\n'
+                  b'msgid "same"\nmsgid_plural "others"\nmsgstr[0] "plural"\n\n')
+        added = b'#: new.rst:100\nmsgid "new"\nmsgstr ""\n\n'
+        obsolete = b'#~ msgid "old"\n#~ msgstr "old translation"\n'
+        updated_first = first.replace(b"one.rst:10 two.rst:20", b"one.rst:100\n#: two.rst:200")
+        updated_first = updated_first.replace(b'msgid "same"', b'msgid ""\n"sa"\n"me"')
+        updated_second = second.replace(b"one.rst:30", b"one.rst:300")
+        incoming = header + added + updated_second + updated_first + obsolete
+        expected_first = updated_first.replace(b"one.rst:100\n#: two.rst:200", b"one.rst:10 two.rst:20")
+        self.assertEqual(
+            translations.normalize_po_references(header + first + second + obsolete, incoming),
+            header + added.replace(b"#: new.rst:100\n", b"") + second + expected_first + obsolete,
+        )
+        # A new plural message with a translation has its references removed.
+        updated_second = updated_second.replace(b'"others"', b'"new plural"')
+        self.assertEqual(
+            translations.normalize_po_references(header + second, header + updated_second),
+            header + updated_second.replace(b"#: one.rst:300\n", b""),
+        )
+
+    def test_untranslated_source_changes_drop_references(self):
+        before = DATED_PO.replace('msgstr "翻訳"'.encode(), b'msgstr ""')
+        for original, incoming in [
+            (b"", before),
+            (before, before.replace(b'msgid "example"', b'msgid "new source"')),
+            (before, before.replace(b'msgid "example"',
+                                    b'msgctxt "new context"\nmsgid "example"')),
+            (before, before.replace(b'msgid "example"\nmsgstr ""\n',
+                                    b'msgid "example"\nmsgid_plural "examples"\nmsgstr[0] ""\n')),
+        ]:
+            with self.subTest(incoming=incoming):
+                expected = incoming.replace(b"#: example.rst:10\n", b"")
+                normalized = translations.normalize_po_references(original, incoming)
+                self.assertEqual(normalized, expected)
+                self.assertEqual(translations.normalize_po_references(normalized, incoming), expected)
+
+    def test_removed_references_do_not_return_on_next_download(self):
+        updated = DATED_PO.replace('翻訳'.encode(), '新しい訳'.encode())
+        updated = updated.replace(b"#: example.rst:10\n", b"#: example.rst:200\n#: other.rst:300\n")
+        expected = updated.replace(b"#: example.rst:200\n#: other.rst:300\n", b"")
+        normalized = translations.normalize_po_references(DATED_PO, updated)
+        self.assertEqual(normalized, expected)
+        self.assertEqual(translations.normalize_po_references(normalized, updated), expected)
+        # Translation text resembling a reference is not a comment.
+        literal = updated.replace('新しい訳'.encode(), b"#: example.rst:200")
+        self.assertIn(b'msgstr "#: example.rst:200"',
+                      translations.normalize_po_references(normalized, literal))
+
     def test_sync_removes_old_path_and_preserves_untracked_files(self):
         catalogs = ["builtins/functions.po", "library/builtins.po", "bugs.po"]
         write_config(self.source, catalogs)