Skip to content

Commit 14ca665

Browse files
committed
Skip unavailable Transifex resources during translation updates
1 parent c03859a commit 14ca665

3 files changed

Lines changed: 163 additions & 6 deletions

File tree

‎scripts/translations.py‎

Lines changed: 59 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@
77
import re
88
import shutil
99
import subprocess
10+
from urllib.error import HTTPError
11+
from urllib.parse import quote
12+
from urllib.request import Request, urlopen
1013

1114

1215
def without_po_timestamps(content):
@@ -86,22 +89,68 @@ def require_catalogs(root, catalogs):
8689
raise FileNotFoundError(f"Missing PO catalogs in {root}: {', '.join(missing)}")
8790

8891

89-
def sync(source, destination, language):
92+
def resource_exists(resource):
93+
"""Only a resource-level 404 permits skipping an absent download."""
94+
request = Request(
95+
"https://rest.api.transifex.com/resources/" + quote(resource, safe=":"),
96+
headers={"Authorization": f"Bearer {os.environ['TX_TOKEN']}"},
97+
)
98+
try:
99+
with urlopen(request, timeout=30):
100+
return True
101+
except HTTPError as error:
102+
error.close()
103+
if error.code == 404:
104+
return False
105+
raise
106+
107+
108+
def sync(source, destination, language, *, skip_missing_resources=False):
90109
config = source / ".tx/config"
91110
catalogs = active_catalogs(config)
92111
downloaded = source / language / "LC_MESSAGES"
93-
require_catalogs(downloaded, catalogs)
112+
skipped = set()
113+
filtered_config = configparser.ConfigParser(interpolation=None)
114+
if skip_missing_resources:
115+
filtered_config.read(config, encoding="utf-8")
116+
# Check every missing download before modifying the repository. Missing
117+
# translations of existing resources (or API errors) must still fail.
118+
for section in filtered_config.sections():
119+
if section == "main":
120+
continue
121+
path = Path(PurePosixPath(
122+
filtered_config[section]["source_file"]
123+
).relative_to("pot").with_suffix(".po"))
124+
if not (downloaded / path).is_file() and not resource_exists(section):
125+
skipped.add(path)
126+
if not (destination / path).is_file():
127+
filtered_config.remove_section(section)
128+
print(f"Skipped unavailable Transifex resource: {section} ({path})")
129+
incoming = [path for path in catalogs if path not in skipped]
130+
require_catalogs(downloaded, incoming)
131+
if not incoming:
132+
raise ValueError("No PO catalogs downloaded; refusing to sync")
133+
# Keep existing translations of temporarily unavailable resources. New
134+
# resources without a PO must not enter the config used by the merge job.
135+
catalogs = [
136+
path for path in catalogs
137+
if path not in skipped or (destination / path).is_file()
138+
]
94139
obsolete = set(tracked_catalogs(destination)) - set(catalogs)
95140

96-
for path in catalogs:
141+
for path in incoming:
97142
target = destination / path
98143
target.parent.mkdir(parents=True, exist_ok=True)
99144
shutil.copyfile(downloaded / path, target)
100145
for path in sorted(obsolete):
101146
(destination / path).unlink(missing_ok=True)
102147
print(f"Removed obsolete catalog: {path}")
103148
(destination / ".tx").mkdir(exist_ok=True)
104-
shutil.copyfile(config, destination / ".tx/config")
149+
if skipped:
150+
with (destination / ".tx/config").open("w", encoding="utf-8") as stream:
151+
filtered_config.write(stream)
152+
else:
153+
shutil.copyfile(config, destination / ".tx/config")
105154

106155

107156
def merge(source, destination):
@@ -131,14 +180,19 @@ def main():
131180
sync_parser.add_argument("source", type=Path)
132181
sync_parser.add_argument("destination", type=Path)
133182
sync_parser.add_argument("--language", default="ja")
183+
sync_parser.add_argument(
184+
"--skip-missing-resources", action="store_true",
185+
help="skip missing downloads only for resources returning 404",
186+
)
134187
merge_parser = commands.add_parser("merge")
135188
merge_parser.add_argument("source", type=Path)
136189
merge_parser.add_argument("destination", type=Path)
137190
restore_parser = commands.add_parser("restore-timestamp-only")
138191
restore_parser.add_argument("root", type=Path)
139192
args = parser.parse_args()
140193
if args.command == "sync":
141-
sync(args.source, args.destination, args.language)
194+
sync(args.source, args.destination, args.language,
195+
skip_missing_resources=args.skip_missing_resources)
142196
elif args.command == "merge":
143197
merge(args.source, args.destination)
144198
else:

‎scripts/update.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,6 @@ sphinx-intl update-txconfig-resources -p pot -d . --transifex-organization-name
3535
tx pull -l "${LANGUAGE}" -t --use-git-timestamps
3636

3737
# Validate the download before copying catalogs or removing obsolete files.
38-
python "${ROOTDIR}/scripts/translations.py" sync "${WORKDIR}" "${ROOTDIR}" --language "${LANGUAGE}"
38+
# Newly generated POTs may not have a Transifex resource yet.
39+
python "${ROOTDIR}/scripts/translations.py" sync "${WORKDIR}" "${ROOTDIR}" \
40+
--language "${LANGUAGE}" --skip-missing-resources

‎tests/test_translations.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import sys
66
import tempfile
77
import unittest
8+
from unittest import mock
9+
from urllib.error import HTTPError, URLError
810

911
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
1012
import translations
@@ -191,6 +193,105 @@ def test_incomplete_download_does_not_change_repository(self):
191193
self.assertEqual(config_before, (self.destination / ".tx/config").read_bytes())
192194
self.assertFalse((self.destination / "builtins/functions.po").exists())
193195

196+
def test_sync_skips_unregistered_resource_and_picks_it_up_later(self):
197+
catalogs = ["bugs.po", "c-api/pyexpat.po"]
198+
write_config(self.source, catalogs)
199+
download = self.source / "ja/LC_MESSAGES"
200+
write_po(download, "bugs.po", "新しい訳")
201+
obsolete = write_po(self.destination, "removed.po", "古い訳")
202+
self.track()
203+
204+
with mock.patch.object(translations, "resource_exists", return_value=False) as exists:
205+
translations.sync(self.source, self.destination, "ja",
206+
skip_missing_resources=True)
207+
exists.assert_called_once_with("resource1")
208+
self.assertFalse(obsolete.exists())
209+
self.assertFalse((self.destination / "c-api/pyexpat.po").exists())
210+
self.assertEqual(translations.active_catalogs(self.destination / ".tx/config"),
211+
[Path("bugs.po")])
212+
translations.require_catalogs(
213+
self.destination, translations.active_catalogs(self.destination / ".tx/config")
214+
)
215+
# The generated source config remains complete for subsequent runs.
216+
self.assertEqual(translations.active_catalogs(self.source / ".tx/config"),
217+
[Path(path) for path in catalogs])
218+
write_po(download, "c-api/pyexpat.po", "追加された訳")
219+
with mock.patch.object(translations, "resource_exists") as exists:
220+
translations.sync(self.source, self.destination, "ja",
221+
skip_missing_resources=True)
222+
exists.assert_not_called()
223+
self.assertEqual((download / "c-api/pyexpat.po").read_bytes(),
224+
(self.destination / "c-api/pyexpat.po").read_bytes())
225+
226+
def test_sync_preserves_existing_po_for_unavailable_resource(self):
227+
catalogs = ["bugs.po", "c-api/pyexpat.po"]
228+
write_config(self.source, catalogs)
229+
write_po(self.source / "ja/LC_MESSAGES", "bugs.po", "新しい訳")
230+
old = write_po(self.destination, "c-api/pyexpat.po", "既存の訳")
231+
self.track()
232+
before = old.read_bytes()
233+
with mock.patch.object(translations, "resource_exists", return_value=False):
234+
translations.sync(self.source, self.destination, "ja",
235+
skip_missing_resources=True)
236+
self.assertEqual(old.read_bytes(), before)
237+
self.assertEqual(translations.active_catalogs(self.destination / ".tx/config"),
238+
[Path(path) for path in catalogs])
239+
240+
def test_skip_does_not_hide_incomplete_download_or_api_failure(self):
241+
write_config(self.source, ["bugs.po", "new.po", "missing.po"])
242+
write_po(self.source / "ja/LC_MESSAGES", "bugs.po", "新しい訳")
243+
write_config(self.destination, ["old.po"])
244+
old = write_po(self.destination, "old.po", "既存の訳")
245+
self.track()
246+
before = old.read_bytes()
247+
config_before = (self.destination / ".tx/config").read_bytes()
248+
for outcome, error in [(True, FileNotFoundError),
249+
(URLError("offline"), URLError)]:
250+
with self.subTest(outcome=outcome):
251+
with mock.patch.object(translations, "resource_exists",
252+
side_effect=[False, outcome]):
253+
with self.assertRaises(error):
254+
translations.sync(self.source, self.destination, "ja",
255+
skip_missing_resources=True)
256+
self.assertEqual(old.read_bytes(), before)
257+
self.assertEqual((self.destination / ".tx/config").read_bytes(),
258+
config_before)
259+
self.assertFalse((self.destination / "bugs.po").exists())
260+
261+
def test_skip_refuses_empty_download(self):
262+
write_config(self.source, ["new.po"])
263+
old = write_po(self.destination, "old.po", "既存の訳")
264+
self.track()
265+
with mock.patch.object(translations, "resource_exists", return_value=False):
266+
with self.assertRaisesRegex(ValueError, "No PO catalogs downloaded"):
267+
translations.sync(self.source, self.destination, "ja",
268+
skip_missing_resources=True)
269+
self.assertTrue(old.exists())
270+
self.assertFalse((self.destination / ".tx/config").exists())
271+
272+
def test_resource_exists_only_skips_404(self):
273+
resource = "o:python-doc:p:python-newest:r:c-api--pyexpat"
274+
with mock.patch.dict(translations.os.environ, {"TX_TOKEN": "test-token"}):
275+
with mock.patch.object(translations, "urlopen") as open_url:
276+
self.assertTrue(translations.resource_exists(resource))
277+
request = open_url.call_args.args[0]
278+
self.assertEqual(request.full_url,
279+
"https://rest.api.transifex.com/resources/" + resource)
280+
self.assertEqual(request.get_header("Authorization"), "Bearer test-token")
281+
self.assertEqual(open_url.call_args.kwargs, {"timeout": 30})
282+
for code in [404, 401, 403, 429, 500]:
283+
with self.subTest(code=code):
284+
error = HTTPError("https://rest.api.transifex.com/", code, "error", {}, None)
285+
with mock.patch.object(translations, "urlopen", side_effect=error):
286+
if code == 404:
287+
self.assertFalse(translations.resource_exists(resource))
288+
else:
289+
with self.assertRaises(HTTPError):
290+
translations.resource_exists(resource)
291+
with mock.patch.object(translations, "urlopen", side_effect=TimeoutError):
292+
with self.assertRaises(TimeoutError):
293+
translations.resource_exists(resource)
294+
194295
def test_empty_config_does_not_delete_catalogs(self):
195296
write_config(self.source, [])
196297
old = write_po(self.destination, "library/functions.po", "古い訳")

0 commit comments

Comments
 (0)