Skip to content

Commit beef127

Browse files
committed
Preserve Transifex config when skipping unavailable resources
1 parent b80c757 commit beef127

3 files changed

Lines changed: 83 additions & 26 deletions

File tree

‎scripts/translations.py‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -110,32 +110,25 @@ def sync(source, destination, language, *, skip_missing_resources=False):
110110
catalogs = active_catalogs(config)
111111
downloaded = source / language / "LC_MESSAGES"
112112
skipped = set()
113-
filtered_config = configparser.ConfigParser(interpolation=None)
114113
if skip_missing_resources:
115-
filtered_config.read(config, encoding="utf-8")
114+
resources = configparser.ConfigParser(interpolation=None)
115+
resources.read(config, encoding="utf-8")
116116
# Check every missing download before modifying the repository. Missing
117117
# translations of existing resources (or API errors) must still fail.
118-
for section in filtered_config.sections():
118+
for section in resources.sections():
119119
if section == "main":
120120
continue
121121
path = Path(PurePosixPath(
122-
filtered_config[section]["source_file"]
122+
resources[section]["source_file"]
123123
).relative_to("pot").with_suffix(".po"))
124124
if not (downloaded / path).is_file() and not resource_exists(section):
125125
skipped.add(path)
126-
if not (destination / path).is_file():
127-
filtered_config.remove_section(section)
128126
print(f"Skipped unavailable Transifex resource: {section} ({path})")
129127
incoming = [path for path in catalogs if path not in skipped]
130128
require_catalogs(downloaded, incoming)
131129
if not incoming:
132130
raise ValueError("No PO catalogs downloaded; refusing to sync")
133-
# Keep existing translations of temporarily unavailable resources. New
134-
# resources without a PO must not enter the config used by the merge job.
135-
catalogs = [
136-
path for path in catalogs
137-
if path not in skipped or (destination / path).is_file()
138-
]
131+
# Keep existing translations of temporarily unavailable resources.
139132
obsolete = set(tracked_catalogs(destination)) - set(catalogs)
140133

141134
for path in incoming:
@@ -146,16 +139,17 @@ def sync(source, destination, language, *, skip_missing_resources=False):
146139
(destination / path).unlink(missing_ok=True)
147140
print(f"Removed obsolete catalog: {path}")
148141
(destination / ".tx").mkdir(exist_ok=True)
149-
if skipped:
150-
with (destination / ".tx/config").open("w", encoding="utf-8") as stream:
151-
filtered_config.write(stream)
152-
else:
153-
shutil.copyfile(config, destination / ".tx/config")
142+
shutil.copyfile(config, destination / ".tx/config")
154143

155144

156145
def merge(source, destination):
157-
# Ignore stale PO files even if a source checkout still contains them.
158-
catalogs = active_catalogs(source / ".tx/config")
146+
# The config includes new resources whose translations are not available
147+
# yet. Merge only tracked catalogs, ignoring stale files outside the config.
148+
catalogs = sorted(
149+
set(active_catalogs(source / ".tx/config")) & set(tracked_catalogs(source))
150+
)
151+
if not catalogs:
152+
raise ValueError(f"No source catalogs in {source}")
159153
targets = tracked_catalogs(destination)
160154
if not targets:
161155
raise ValueError(f"No target catalogs in {destination}")

‎scripts/update.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ sphinx-intl create-txconfig
3232
sphinx-intl update-txconfig-resources -p pot -d . --transifex-organization-name python-doc --transifex-project-name python-newest
3333

3434
# Pull translations into the temporary LANGUAGE/LC_MESSAGES directory.
35-
tx pull -l "${LANGUAGE}" -t --use-git-timestamps
35+
if ! tx pull -l "${LANGUAGE}" -t --use-git-timestamps; then
36+
echo "tx pull failed; validating downloaded catalogs before continuing." >&2
37+
fi
3638

3739
# Validate the download before copying catalogs or removing obsolete files.
3840
# Newly generated POTs may not have a Transifex resource yet.

‎tests/test_translations.py‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import json
2+
import os
23
from pathlib import Path
34
import shutil
45
import subprocess
@@ -196,6 +197,10 @@ def test_incomplete_download_does_not_change_repository(self):
196197
def test_sync_skips_unregistered_resource_and_picks_it_up_later(self):
197198
catalogs = ["bugs.po", "c-api/pyexpat.po"]
198199
write_config(self.source, catalogs)
200+
config = self.source / ".tx/config"
201+
original = config.read_bytes().replace(b"host =", b"host =")
202+
original = b"# Preserve tx formatting\n" + original
203+
config.write_bytes(original)
199204
download = self.source / "ja/LC_MESSAGES"
200205
write_po(download, "bugs.po", "新しい訳")
201206
obsolete = write_po(self.destination, "removed.po", "古い訳")
@@ -208,11 +213,9 @@ def test_sync_skips_unregistered_resource_and_picks_it_up_later(self):
208213
self.assertFalse(obsolete.exists())
209214
self.assertFalse((self.destination / "c-api/pyexpat.po").exists())
210215
self.assertEqual(translations.active_catalogs(self.destination / ".tx/config"),
211-
[Path("bugs.po")])
212-
translations.require_catalogs(
213-
self.destination, translations.active_catalogs(self.destination / ".tx/config")
214-
)
215-
# The generated source config remains complete for subsequent runs.
216+
[Path(path) for path in catalogs])
217+
self.assertEqual((self.destination / ".tx/config").read_bytes(), original)
218+
# Both configs remain complete for subsequent runs.
216219
self.assertEqual(translations.active_catalogs(self.source / ".tx/config"),
217220
[Path(path) for path in catalogs])
218221
write_po(download, "c-api/pyexpat.po", "追加された訳")
@@ -222,6 +225,7 @@ def test_sync_skips_unregistered_resource_and_picks_it_up_later(self):
222225
exists.assert_not_called()
223226
self.assertEqual((download / "c-api/pyexpat.po").read_bytes(),
224227
(self.destination / "c-api/pyexpat.po").read_bytes())
228+
self.assertEqual((self.destination / ".tx/config").read_bytes(), original)
225229

226230
def test_sync_preserves_existing_po_for_unavailable_resource(self):
227231
catalogs = ["bugs.po", "c-api/pyexpat.po"]
@@ -236,6 +240,8 @@ def test_sync_preserves_existing_po_for_unavailable_resource(self):
236240
self.assertEqual(old.read_bytes(), before)
237241
self.assertEqual(translations.active_catalogs(self.destination / ".tx/config"),
238242
[Path(path) for path in catalogs])
243+
self.assertEqual((self.destination / ".tx/config").read_bytes(),
244+
(self.source / ".tx/config").read_bytes())
239245

240246
def test_skip_does_not_hide_incomplete_download_or_api_failure(self):
241247
write_config(self.source, ["bugs.po", "new.po", "missing.po"])
@@ -305,11 +311,66 @@ def test_config_cannot_escape_catalog_directory(self):
305311
with self.assertRaises(ValueError):
306312
translations.active_catalogs(self.source / ".tx/config")
307313

314+
def test_update_validates_after_pull_failure(self):
315+
scripts = self.source / "scripts"
316+
scripts.mkdir()
317+
update = scripts / "update.sh"
318+
shutil.copyfile(Path(translations.__file__).with_name("update.sh"), update)
319+
doc = self.source / "cpython/Doc"
320+
doc.mkdir(parents=True)
321+
(doc / "conf.py").touch()
322+
executables = self.root / "bin"
323+
executables.mkdir()
324+
validation_log = self.root / "validation.log"
325+
commands = {
326+
"sphinx-build": "exit 0",
327+
"sphinx-intl": "exit 0",
328+
"tx": "exit 1",
329+
"python": 'printf "%s\\n" "$@" > "$VALIDATION_LOG"\n'
330+
'exit "$VALIDATION_STATUS"',
331+
}
332+
for name, body in commands.items():
333+
executable = executables / name
334+
executable.write_text("#!/bin/sh\n" + body + "\n", encoding="utf-8")
335+
executable.chmod(0o755)
336+
for status in (0, 42):
337+
with self.subTest(validation_status=status):
338+
validation_log.unlink(missing_ok=True)
339+
result = subprocess.run(
340+
["sh", str(update)], capture_output=True, text=True,
341+
env={**os.environ, "PATH": f"{executables}:{os.environ['PATH']}",
342+
"VALIDATION_LOG": str(validation_log),
343+
"VALIDATION_STATUS": str(status)},
344+
)
345+
self.assertEqual(result.returncode, status, result.stderr)
346+
self.assertIn("tx pull failed", result.stderr)
347+
arguments = validation_log.read_text().splitlines()
348+
self.assertEqual(arguments[1], "sync")
349+
self.assertIn("--skip-missing-resources", arguments)
350+
351+
def test_merge_refuses_empty_source_and_missing_tracked_po(self):
352+
subprocess.run(["git", "init", "--quiet", str(self.source)], check=True)
353+
write_config(self.source, ["new.po", "bugs.po"])
354+
write_po(self.destination, "bugs.po", "既存の訳")
355+
self.track()
356+
with mock.patch.object(translations.subprocess, "run",
357+
wraps=subprocess.run) as run:
358+
with self.assertRaisesRegex(ValueError, "No source catalogs"):
359+
translations.merge(self.source, self.destination)
360+
missing = write_po(self.source, "bugs.po", "新しい訳")
361+
subprocess.run(["git", "-C", str(self.source), "add", "."], check=True)
362+
missing.unlink()
363+
with self.assertRaises(FileNotFoundError):
364+
translations.merge(self.source, self.destination)
365+
self.assertFalse(any(call.args[0][0] == "pomerge" for call in run.call_args_list))
366+
308367
@unittest.skipUnless(shutil.which("pomerge"), "pomerge is required")
309368
def test_merge_uses_current_translation_across_renamed_paths(self):
310-
write_config(self.source, ["builtins/functions.po"])
369+
subprocess.run(["git", "init", "--quiet", str(self.source)], check=True)
370+
write_config(self.source, ["builtins/functions.po", "c-api/pyexpat.po"])
311371
write_po(self.source, "builtins/functions.po", "新しい訳")
312372
write_po(self.source, "library/functions.po", "古い訳")
373+
subprocess.run(["git", "-C", str(self.source), "add", "."], check=True)
313374
old = write_po(self.destination, "library/functions.po", "以前の訳")
314375
self.track()
315376

0 commit comments

Comments
 (0)