From 51a3f243b1980fdd22bd83bd4e3fb21093a7d101 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 30 Jul 2026 15:35:49 -0700 Subject: [PATCH 1/2] Fix Crowdin upload configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dest` only works under preserve_hierarchy: true, so the previous flat-plus-dest config never validated and every upload failed on it. Turn hierarchy on and dest every entry back to the project root, leaving the Crowdin layout flat so existing files keep their paths and their file IDs. A CSV dest must end in a literal `.csv`: the CLI extension-tests the unresolved dest string, and on a miss drops the scheme silently, uploading a file that parses to nothing. File plugin catalogs under optional_plugins/ — they are not a translation priority. Install the workspace packages in CI, without which plugin message extraction is skipped and those CSVs never reach Crowdin. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014Bd4b7a59DK4bhQpaQmRV1 --- .github/workflows/i18n-download.yml | 5 +- .github/workflows/i18n-upload.yml | 5 +- Makefile | 8 ++-- crowdin.yml | 74 ++++++++++++++++++----------- 4 files changed, 60 insertions(+), 32 deletions(-) diff --git a/.github/workflows/i18n-download.yml b/.github/workflows/i18n-download.yml index 46e3c4e1a7c..27efa086791 100644 --- a/.github/workflows/i18n-download.yml +++ b/.github/workflows/i18n-download.yml @@ -37,7 +37,10 @@ jobs: - name: Install gettext run: sudo apt-get update && sudo apt-get install -y gettext - name: Install Python dependencies - run: uv sync + # --all-packages installs the python_packages/* members; without them the + # Makefile's importability guard skips their message-file generation and + # their *-messages.json are never regenerated from the download. + run: uv sync --all-packages - name: Install JavaScript dependencies run: pnpm install --frozen-lockfile - name: Download source fonts diff --git a/.github/workflows/i18n-upload.yml b/.github/workflows/i18n-upload.yml index f41be1987f3..7064421a0ee 100644 --- a/.github/workflows/i18n-upload.yml +++ b/.github/workflows/i18n-upload.yml @@ -36,7 +36,10 @@ jobs: - name: Install gettext run: sudo apt-get update && sudo apt-get install -y gettext - name: Install Python dependencies - run: uv sync + # --all-packages installs the python_packages/* members; without them the + # Makefile's importability guard skips their message extraction and their + # CSVs never reach Crowdin. + run: uv sync --all-packages - name: Install JavaScript dependencies run: pnpm install --frozen-lockfile - name: Extract and upload strings to Crowdin diff --git a/Makefile b/Makefile index 68db3586a1e..f457370d52e 100644 --- a/Makefile +++ b/Makefile @@ -254,10 +254,12 @@ i18n-extract-frontend: # buildConfig.js via the *installed* package (webpack_json.py files(module)), so # skip any plugin whose package is not importable here — e.g. `make dist`, which # installs only the root project, not the python_packages/* members. Where the - # members are installed (uv sync --all-packages) they extract normally. + # members are installed (uv sync --all-packages) they extract normally. The guard + # must run under uv: a bare `python` is the system interpreter, which never sees + # the members, so every plugin would be skipped however the project was synced. @for dir in $(I18N_PLUGIN_FRONTEND_DIRS); do \ module=$$(basename $$dir); \ - if ! python -c "import $$module" >/dev/null 2>&1; then \ + if ! uv run python -c "import $$module" >/dev/null 2>&1; then \ echo "Skipping frontend messages for $$module (package not installed)"; \ continue; \ fi; \ @@ -313,7 +315,7 @@ i18n-download-translations: i18n-extract-frontend # package is not importable here (same reason as i18n-extract-frontend). @for dir in $(I18N_PLUGIN_FRONTEND_DIRS); do \ module=$$(basename $$dir); \ - if ! python -c "import $$module" >/dev/null 2>&1; then \ + if ! uv run python -c "import $$module" >/dev/null 2>&1; then \ echo "Skipping message files for $$module (package not installed)"; \ continue; \ fi; \ diff --git a/crowdin.yml b/crowdin.yml index 597c65479c1..0c355df4e70 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -10,9 +10,26 @@ "base_path": "." "base_url": "https://api.crowdin.com" # -# Defines whether to preserve the original directory structure in the Crowdin project +# Defines whether to preserve the original directory structure in the Crowdin project. +# Must be true, but NOT to get the source hierarchy: every entry below carries a +# `dest`, and those dests define the Crowdin layout — flat at the project root for +# what translators work on, plus an optional_plugins/ folder for the de-prioritised +# plugin catalogs. It is true only because the CLI rejects `dest` otherwise; flipping +# it to false would drop every dest and reinstate the source paths. # -"preserve_hierarchy": false +# `dest` is also what keeps same-named files distinct (each plugin's django.po), which +# preserve_hierarchy: false could not do at all. +# +# Crowdin keys files by their project path, so editing a `dest` below creates a new +# file and leaves the old one holding its translations — move it in the Crowdin UI +# instead, which preserves the file ID. +# +# A `dest` for a CSV MUST end in a literal `.csv` (not `%original_file_name%`): +# UploadSourcesAction.isSpreadsheet() extension-tests the UNRESOLVED dest string, and +# on a miss silently drops `scheme`/`first_line_contains_header`, so the file uploads +# clean but parses to nothing. `type: csv` does not help — isSpreadsheet ignores it. +# +"preserve_hierarchy": true # # Files configuration. # See https://support.crowdin.com/developer/configuration-file/ for all available options @@ -22,45 +39,48 @@ # files: [ # CSV translation files - {"source": "/kolibri/locale/en/LC_MESSAGES/*.csv", "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": &language_mapping {"locale_with_underscore": { + {"source": "/kolibri/locale/en/LC_MESSAGES/*.csv", "dest": "/%file_name%.csv", "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": &language_mapping {"locale_with_underscore": { # Generated mapping from crowdin_code to Django locale code # Generated by build_tools/i18n/generate_mapping.py "ach": "ach_UG", "ar": "ar", "bg": "bg_BG", "bn": "bn_BD", "de": "de", "el": "el", "en": "en", "es-ES": "es_ES", "fa": "fa", "fr": "fr_FR", "fv": "ff_CM", "gu-IN": "gu_IN", "ha": "ha", "hi": "hi_IN", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es_419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt_BR", "pt-mz": "pt_MZ", "sw-TZ": "sw_TZ", "te": "te", "uk": "uk", "ur-PK": "ur_PK", "vi": "vi", "yo": "yo", "zh-CN": "zh_Hans"}}}, - # Frontend i18n for ALL python_packages/* plugins (issue #14970) — one entry. - # Plugin frontend CSVs are namespace-prefixed (e.g. - # kolibri_sentry_plugin-messages.csv), so their basenames are globally unique - # under preserve_hierarchy: false. The `**` in source and translation carries - # each plugin's / subpath through, so downloads land back in the - # correct plugin locale/. A new frontend plugin needs NO change here. (Ignore - # node_modules so the recursive `**` can't pick up any vendored CSVs.) - {"source": "/python_packages/**/locale/en/LC_MESSAGES/*.csv", "ignore": ["/python_packages/**/node_modules/**"], "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/python_packages/**/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": *language_mapping}, + # Frontend i18n for ALL python_packages/* plugins (issue #14970) — one entry, which + # rests on plugin frontend CSVs being namespace-prefixed (kolibri_sentry_plugin- + # messages.csv): the basenames are globally unique, so one flattening `dest` cannot + # collide the way the per-plugin django.po below would. + # `dest` files them under optional_plugins/ in the Crowdin project: they are not a + # translation priority, so they stay out of the top-level file list. The `**` in + # source and translation carries each plugin's / subpath through, so + # downloads land back in the correct plugin locale/ regardless. A new frontend + # plugin needs NO change here. (Ignore node_modules so the recursive `**` can't + # pick up any vendored CSVs.) + {"source": "/python_packages/**/locale/en/LC_MESSAGES/*.csv", "dest": "/optional_plugins/%file_name%.csv", "ignore": ["/python_packages/**/node_modules/**"], "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/python_packages/**/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": *language_mapping}, # PO translation files - {"source": "/kolibri/locale/en/LC_MESSAGES/*.po", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "ignore": ["/kolibri/locale/en/LC_MESSAGES/*.json", "/kolibri/locale/en/LC_MESSAGES/*.mo", "/kolibri/locale/en/LC_MESSAGES/README.md"], "languages_mapping": *language_mapping}, - # Plugin backend PO — one entry PER plugin, unavoidably (unlike the frontend - # CSVs above). Django makemessages always emits a file named django.po, so - # under preserve_hierarchy: false every plugin's django.po flattens to the same - # Crowdin file and collides (with each other and with kolibri's). Keeping them - # distinct needs a unique `dest` per file, which is inherently one explicit - # entry per plugin: a wildcard `dest` disambiguated by directory would require - # preserve_hierarchy: true globally, restructuring kolibri's own catalogs on - # Crowdin (out of scope). `dest` renames only in the Crowdin project; the - # `translation` template still writes downloads to the real local django.po, so - # compile / message-file steps are unaffected. A plugin gaining its first - # backend string needs a new entry mirroring this one. - {"source": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/en/LC_MESSAGES/django.po", "dest": "/kolibri_oidc_client_plugin.django.po", "update_option": "update_as_unapproved", "translation": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/%locale_with_underscore%/LC_MESSAGES/django.po", "languages_mapping": *language_mapping}, + {"source": "/kolibri/locale/en/LC_MESSAGES/*.po", "dest": "/%file_name%.po", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "ignore": ["/kolibri/locale/en/LC_MESSAGES/*.json", "/kolibri/locale/en/LC_MESSAGES/*.mo", "/kolibri/locale/en/LC_MESSAGES/README.md"], "languages_mapping": *language_mapping}, + # Plugin backend PO — one entry PER plugin, unlike the frontend CSVs above, and + # likewise filed under optional_plugins/. Django makemessages emits no domain but + # `django`, so every plugin's catalog is named django.po and they would collide + # with each other inside the folder. No placeholder can separate them either: + # `%file_name%` is the source basename, which is `django` for every plugin, and + # `%original_path%` is the whole relative source directory, which would rebuild the + # python_packages///locale/en/LC_MESSAGES hierarchy under + # optional_plugins/. So a literal `dest` per plugin it is. + # `dest` renames only in the Crowdin project; the `translation` template still + # writes downloads to the real local django.po, so compile / message-file steps are + # unaffected. A plugin gaining its first backend string needs a new entry mirroring + # this one. + {"source": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/en/LC_MESSAGES/django.po", "dest": "/optional_plugins/kolibri_oidc_client_plugin.django.po", "update_option": "update_as_unapproved", "translation": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/%locale_with_underscore%/LC_MESSAGES/django.po", "languages_mapping": *language_mapping}, # desktop-app (issue #15073) — two i18n subsystems with different locale-code # conventions. The wxPython app reuses the shared underscore anchor (its folders # are Django/gettext codes like es_ES). The Inno Setup installer needs its own # inline hyphen-lowercase %locale% map because installer/translations/definitions.py # keys the locale definitions by hyphen-lowercase codes (es-es, zh-hans). Both # maps derive from the same language_info.json via generate_mapping.py. - {"source": "/platforms/desktop-app/src/kolibri_app/locales/en/LC_MESSAGES/wxapp.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/src/kolibri_app/locales/%locale_with_underscore%/LC_MESSAGES/wxapp.po", "languages_mapping": *language_mapping}, {"source": "/platforms/desktop-app/installer/translations/locale/en/messages.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/installer/translations/locale/%locale%/messages.po", "languages_mapping": {"locale": {"ach": "ach-ug", "ar": "ar", "bg": "bg-bg", "bn": "bn-bd", "de": "de", "el": "el", "en": "en", "es-ES": "es-es", "fa": "fa", "fr": "fr-fr", "fv": "ff-cm", "gu-IN": "gu-in", "ha": "ha", "hi": "hi-in", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es-419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-br", "pt-mz": "pt-mz", "sw-TZ": "sw-tz", "te": "te", "uk": "uk", "ur-PK": "ur-pk", "vi": "vi", "yo": "yo", "zh-CN": "zh-hans"}}}, + {"source": "/platforms/desktop-app/src/kolibri_app/locales/en/LC_MESSAGES/wxapp.po", "dest": "/desktop_app.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/src/kolibri_app/locales/%locale_with_underscore%/LC_MESSAGES/wxapp.po", "languages_mapping": *language_mapping}, {"source": "/platforms/desktop-app/installer/translations/locale/en/messages.po", "dest": "/desktop_installer.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/installer/translations/locale/%locale%/messages.po", "languages_mapping": {"locale": {"ach": "ach-ug", "ar": "ar", "bg": "bg-bg", "bn": "bn-bd", "de": "de", "el": "el", "en": "en", "es-ES": "es-es", "fa": "fa", "fr": "fr-fr", "fv": "ff-cm", "gu-IN": "gu-in", "ha": "ha", "hi": "hi-in", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es-419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-br", "pt-mz": "pt-mz", "sw-TZ": "sw-tz", "te": "te", "uk": "uk", "ur-PK": "ur-pk", "vi": "vi", "yo": "yo", "zh-CN": "zh-hans"}}}, # Android installer UI strings (issue #15074). type: android forces Crowdin's # Android-resource parser (not .xml auto-detect), which honours # translatable="false" (so task_worker_process is not uploaded) and # Android-encodes downloads. %android_code% expands to the values- # qualifier; the android_code mapping (generated by # build_tools/i18n/generate_mapping.py) overrides Crowdin's defaults so folders - # match Kolibri's language set. dest keeps the flattened basename unique on - # Crowdin under preserve_hierarchy: false. + # match Kolibri's language set. {"source": "/platforms/android/app/src/main/res/values/strings.xml", "type": "android", "dest": "/android_strings.xml", "translation": "/platforms/android/app/src/main/res/values-%android_code%/strings.xml", "update_option": "update_as_unapproved", "languages_mapping": {"android_code": {"ach": "ach-rUG", "ar": "ar", "bg": "bg-rBG", "bn": "bn-rBD", "de": "de", "el": "el", "en": "en", "es-ES": "es-rES", "fa": "fa", "fr": "fr-rFR", "fv": "ff-rCM", "gu-IN": "gu-rIN", "ha": "ha", "hi": "hi-rIN", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "b+es+419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-rBR", "pt-mz": "pt-rMZ", "sw-TZ": "sw-rTZ", "te": "te", "uk": "uk", "ur-PK": "ur-rPK", "vi": "vi", "yo": "yo", "zh-CN": "zh"}}}] From 271483742838c2595fbf04b878e2e0ab691daec6 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 30 Jul 2026 16:13:31 -0700 Subject: [PATCH 2/2] Prune unsupported languages from every synced locale tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cleanup_unsupported_languages only walked kolibri/locale, so the plugin, desktop-app and Android catalogs kept whatever extra languages Crowdin delivered — a languages_mapping renames codes, it does not filter. For Android that meant unsupported values- directories shipping in the APK. Find the trees by shape rather than by name, so a new plugin or platform app is covered without an edit here. Plain locale directories prune against the union of the gettext and installer code sets, since a path does not reveal which convention its tree follows; res/ prunes values- and leaves the other resource directories alone. Invoke via -m, as the mapping helpers are now imported as a package. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014Bd4b7a59DK4bhQpaQmRV1 --- Makefile | 2 +- .../i18n/cleanup_unsupported_languages.py | 148 +++++++++++-- test/test_crowdin_config.py | 43 ++++ ...test_i18n_cleanup_unsupported_languages.py | 197 ++++++++++++++++++ 4 files changed, 369 insertions(+), 21 deletions(-) create mode 100644 test/test_crowdin_config.py create mode 100644 test/test_i18n_cleanup_unsupported_languages.py diff --git a/Makefile b/Makefile index f457370d52e..4cd980c7ad8 100644 --- a/Makefile +++ b/Makefile @@ -305,7 +305,7 @@ i18n-download-translations: i18n-extract-frontend fi @echo "✅ Translation files downloaded successfully" rm -f kolibri/locale/.crowdin-download-marker - uv run python build_tools/i18n/cleanup_unsupported_languages.py + uv run python -m build_tools.i18n.cleanup_unsupported_languages pnpm exec kolibri-i18n code-gen --output-dir ./packages/kolibri/utils/internal $(MAKE) i18n-django-compilemessages pnpm exec kolibri-i18n create-message-files --pluginFile ./build_tools/build_plugins.txt --ignore '**/node_modules/!(kolibri-common)/**','**/static/**' diff --git a/build_tools/i18n/cleanup_unsupported_languages.py b/build_tools/i18n/cleanup_unsupported_languages.py index 30d30ba7439..7eb434a0440 100644 --- a/build_tools/i18n/cleanup_unsupported_languages.py +++ b/build_tools/i18n/cleanup_unsupported_languages.py @@ -4,41 +4,149 @@ This script should be run after downloading translations from Crowdin to clean up any languages that were downloaded but are not officially supported by Kolibri. +A `languages_mapping` in crowdin.yml only renames codes, it does not filter, so +Crowdin delivers every language enabled on the project into every tree we sync. """ +import glob import logging import os +import re import shutil -from utils import available_languages -from utils import LOCALE_DATA_FOLDER -from utils import to_locale +from build_tools.i18n.generate_mapping import get_android_language_mapping +from build_tools.i18n.generate_mapping import get_installer_language_mapping +from build_tools.i18n.generate_mapping import get_language_mapping +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -def cleanup_unsupported_languages(): +# `values-` prefixes every Android resource qualifier, not just locales, so match the +# shape of a locale one — `fr`, `fr-rFR` or BCP-47 `b+es+419` — and leave the rest +# (values-night, values-v21, values-sw600dp) alone. `car` and `tv` are the UI-mode +# qualifiers short enough to also be shaped like a language code (both are real ISO 639 +# codes), so they are denied outright: in a res/ directory they are never a locale. +ANDROID_LOCALE_QUALIFIER = re.compile( + r"^values-(?!(?:car|tv)$)(b\+[a-z]{2,3}(?:\+[A-Za-z0-9]+)*|[a-z]{2,3}(?:-r[A-Z]{2})?)$" +) + +# Non-hidden build output that recursive discovery would otherwise walk into: p4a +# unpacks a full CPython under android_root/, and an unpacked Kolibri tar or a +# PyInstaller/sdist staging tree holds catalogs we do not own. glob already skips +# dot-prefixed directories, so .venv/ needs no entry. +EXCLUDED_DIR_NAMES = frozenset( + { + "node_modules", + "site-packages", + "android_root", + "build", + "dist", + "tar", + } +) + + +def _supported_locale_names(): + """Every locale directory name any of our sync targets may legitimately use. + + A directory's path does not reveal which convention its tree follows — the + desktop-app installer keys folders by hyphen-lowercase intl codes (es-es) while + everything gettext-based uses Django codes (es_ES) — so plain locale directories + are pruned against the union. Only unsupported *languages* are being removed + here; a code in the wrong convention for its tree is a different problem. """ - Remove locale directories that are not in our supported languages list. + return set(get_language_mapping().values()) | set( + get_installer_language_mapping().values() + ) + + +def prune_locale_dirs(directory, supported_names, pattern=None): + """Remove immediate subdirectories of `directory` for unsupported languages. + + `pattern` is a compiled regex whose first group extracts the locale name from a + subdirectory name; a name it does not match is not a locale directory and is left + alone. Without one, the subdirectory name is the locale name. Returns the sorted + names of what was removed. """ - # Get the set of supported locale directory names - supported_locales = set() - for lang_obj in available_languages(): - intl_code = lang_obj["intl_code"] - locale_dir = to_locale(intl_code) - supported_locales.add(locale_dir) - - # Check all directories in the locale folder - removed_locales = [] - for item in os.listdir(LOCALE_DATA_FOLDER): - item_path = os.path.join(LOCALE_DATA_FOLDER, item) + if not os.path.isdir(directory): + return [] + + removed = [] + for item in sorted(os.listdir(directory)): + item_path = os.path.join(directory, item) - # Skip if not a directory or if it's a special file if not os.path.isdir(item_path): continue - # If this directory is not in our supported list, remove it - if item not in supported_locales: + if pattern is None: + locale_name = item + else: + match = pattern.match(item) + if match is None: + continue + locale_name = match.group(1) + + if locale_name not in supported_names: shutil.rmtree(item_path) - removed_locales.append(item) + removed.append(item) + + return removed + + +def _is_excluded(path, repo_root): + # Relative to the root: a checkout under ~/build/kolibri must not exclude everything. + parts = os.path.relpath(path, repo_root).split(os.sep) + return any( + part in EXCLUDED_DIR_NAMES or part.endswith(".egg-info") for part in parts + ) + + +def _locale_roots(repo_root): + """Directories named locale/ or locales/, holding one subdirectory per language.""" + yield os.path.join(repo_root, "kolibri", "locale") + + for top in ("python_packages", "platforms"): + for name in ("locale", "locales"): + for path in glob.glob( + os.path.join(repo_root, top, "**", name), recursive=True + ): + # Vendored dependencies and staged build output ship their own + # catalogs; not ours to prune. + if _is_excluded(path, repo_root): + continue + yield path + + +def _android_res_roots(repo_root): + """Android res/ directories, identified by their untranslated values/ folder.""" + for path in glob.glob( + os.path.join(repo_root, "platforms", "**", "res", "values"), recursive=True + ): + if _is_excluded(path, repo_root): + continue + yield os.path.dirname(path) + + +def cleanup_targets(repo_root=REPO_ROOT): + """Yield (directory, supported_names, pattern) for every tree Crowdin writes to.""" + supported = _supported_locale_names() + for directory in _locale_roots(repo_root): + yield directory, supported, None + + android_supported = set(get_android_language_mapping().values()) + for directory in _android_res_roots(repo_root): + yield directory, android_supported, ANDROID_LOCALE_QUALIFIER + + +def cleanup_unsupported_languages(repo_root=REPO_ROOT): + """ + Remove locale directories that are not in our supported languages list. + """ + removed_locales = [] + for directory, supported_names, pattern in cleanup_targets(repo_root): + for name in prune_locale_dirs(directory, supported_names, pattern): + removed_locales.append( + os.path.join(os.path.relpath(directory, repo_root), name) + ) if removed_locales: logging.info( diff --git a/test/test_crowdin_config.py b/test/test_crowdin_config.py new file mode 100644 index 00000000000..f32f7cbd8d3 --- /dev/null +++ b/test/test_crowdin_config.py @@ -0,0 +1,43 @@ +"""Project-wide invariants of crowdin.yml. + +Both of these fail silently rather than loudly, which is why they are asserted here +rather than left to the comments in the file: a CSV `dest` that loses its literal +`.csv` uploads clean and parses to nothing, and a missing `dest` puts a file on a +path no translation is attached to. +""" + +import os + +import pytest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _config(): + # PyYAML rides in transitively via drf-yasg (gated to python_version >= + # '3.8'); skip on the 3.6/3.7 no-uv CI where it isn't installed. + yaml = pytest.importorskip("yaml") + with open(os.path.join(REPO_ROOT, "crowdin.yml")) as f: + return yaml.safe_load(f) + + +def test_hierarchy_is_preserved(): + # The CLI silently drops every `dest` without this. + assert _config()["preserve_hierarchy"] is True + + +def test_every_entry_has_a_dest(): + assert [e for e in _config()["files"] if not e.get("dest")] == [] + + +def test_dest_extension_matches_source(): + # UploadSourcesAction.isSpreadsheet() extension-tests the unresolved `dest`, so a + # CSV whose dest does not end in a literal `.csv` loses its scheme. + for entry in _config()["files"]: + source_ext = os.path.splitext(entry["source"])[1] + assert os.path.splitext(entry["dest"])[1] == source_ext, entry["source"] + + +def test_dests_are_unique(): + dests = [e["dest"] for e in _config()["files"]] + assert sorted(dests) == sorted(set(dests)) diff --git a/test/test_i18n_cleanup_unsupported_languages.py b/test/test_i18n_cleanup_unsupported_languages.py new file mode 100644 index 00000000000..e06196ef885 --- /dev/null +++ b/test/test_i18n_cleanup_unsupported_languages.py @@ -0,0 +1,197 @@ +import os + +import pytest + +from build_tools.i18n.cleanup_unsupported_languages import ANDROID_LOCALE_QUALIFIER +from build_tools.i18n.cleanup_unsupported_languages import cleanup_targets +from build_tools.i18n.cleanup_unsupported_languages import prune_locale_dirs + + +def _path(root, relpath): + # The module builds its paths with os.path.join/glob, so a forward-slash literal + # would not compare equal to them on Windows. + return os.path.join(root, *relpath.split("/")) + + +def _make_dirs(root, *relpaths): + for relpath in relpaths: + os.makedirs(_path(root, relpath)) + + +def test_prune_removes_unsupported_and_keeps_supported(tmp_path): + locale_dir = str(tmp_path) + _make_dirs(locale_dir, "en", "fr_FR", "tr_TR", "kaa") + + removed = prune_locale_dirs(locale_dir, {"en", "fr_FR"}) + + assert removed == ["kaa", "tr_TR"] + assert sorted(os.listdir(locale_dir)) == ["en", "fr_FR"] + + +def test_prune_ignores_files(tmp_path): + locale_dir = str(tmp_path) + _make_dirs(locale_dir, "fr_FR") + open(os.path.join(locale_dir, "language_info.json"), "w").close() + + assert prune_locale_dirs(locale_dir, {"fr_FR"}) == [] + assert os.path.exists(os.path.join(locale_dir, "language_info.json")) + + +def test_prune_with_pattern_only_touches_locale_qualifiers(tmp_path): + # values- prefixes every Android resource qualifier, not just locales. Pruning + # must reach the unsupported locale and nothing else — a deleted values-night or + # values-v21 would ride out unnoticed in the auto-generated translation PR. + res_dir = str(tmp_path) + _make_dirs( + res_dir, + "values", + "values-fr-rFR", + "values-b+es+419", + "values-tr-rTR", + "values-night", + "values-v21", + "values-sw600dp", + "values-land", + # UI-mode qualifiers that are also ISO 639 codes, so shape alone can't tell + # them from a locale. + "values-car", + "values-tv", + "drawable", + "xml", + ) + + removed = prune_locale_dirs( + res_dir, {"fr-rFR", "b+es+419"}, pattern=ANDROID_LOCALE_QUALIFIER + ) + + assert removed == ["values-tr-rTR"] + assert sorted(os.listdir(res_dir)) == [ + "drawable", + "values", + "values-b+es+419", + "values-car", + "values-fr-rFR", + "values-land", + "values-night", + "values-sw600dp", + "values-tv", + "values-v21", + "xml", + ] + + +def test_prune_missing_directory_is_noop(tmp_path): + assert prune_locale_dirs(os.path.join(str(tmp_path), "nope"), {"en"}) == [] + + +@pytest.fixture +def repo_root(tmp_path): + # Deliberately NOT the plugins/platforms this repo currently ships: discovery + # has to work for whatever is added next, not for a hardcoded inventory. + root = str(tmp_path) + _make_dirs( + root, + "kolibri/locale", + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale", + "python_packages/kolibri-notyet-plugin/kolibri_notyet_plugin/locale", + "platforms/handheld-app/src/handheld_app/locales", + "platforms/handheld-app/installer/translations/locale", + "platforms/pocket-os/app/src/main/res/values", + ) + return root + + +def _target_dirs(repo_root): + return {directory for directory, _, _ in cleanup_targets(repo_root)} + + +def _target_for(repo_root, relpath): + return next( + (supported, pattern) + for directory, supported, pattern in cleanup_targets(repo_root) + if directory == _path(repo_root, relpath) + ) + + +def test_cleanup_targets_discovers_trees_by_shape_not_by_name(repo_root): + targets = _target_dirs(repo_root) + + for expected in ( + "kolibri/locale", + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale", + "python_packages/kolibri-notyet-plugin/kolibri_notyet_plugin/locale", + "platforms/handheld-app/src/handheld_app/locales", + "platforms/handheld-app/installer/translations/locale", + "platforms/pocket-os/app/src/main/res", + ): + assert _path(repo_root, expected) in targets + + +@pytest.mark.parametrize( + "relpath", + [ + "python_packages/kolibri-imaginary-plugin/node_modules/dep/locale", + # p4a unpacks a full CPython under android_root/, so Django's own catalogs + # sit here in any checkout where the APK has been built. + "platforms/android/android_root/python-install/lib/python3.9/" + "site-packages/django/conf/locale", + "platforms/handheld-app/build/lib/handheld_app/locales", + "platforms/android/tar/extracted/kolibri-0.19/kolibri/locale", + ], +) +def test_untracked_build_output_is_left_alone(repo_root, relpath): + _make_dirs(repo_root, relpath) + targets = _target_dirs(repo_root) + + assert _path(repo_root, relpath) not in targets + # Guard the assertion above against passing vacuously: whatever shape paths + # compare in on this platform, a real root is still discovered. + assert _path(repo_root, "kolibri/locale") in targets + + +def test_excluded_names_only_match_below_the_repo_root(tmp_path): + # A checkout can itself sit under an excluded name (~/build/kolibri, /srv/dist/kolibri). + # Matching the whole path there would drop every glob-discovered root — silently, + # and for exactly the trees this script exists to prune. + root = _path(str(tmp_path), "build/kolibri") + plugin_locale = ( + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale" + ) + _make_dirs(root, plugin_locale, "platforms/pocket-os/app/src/main/res/values") + + targets = _target_dirs(root) + + assert _path(root, plugin_locale) in targets + assert _path(root, "platforms/pocket-os/app/src/main/res") in targets + + +def test_staged_android_res_is_left_alone(repo_root): + _make_dirs(repo_root, "platforms/android/build/intermediates/res/values") + targets = _target_dirs(repo_root) + + assert _path(repo_root, "platforms/android/build/intermediates/res") not in targets + assert _path(repo_root, "platforms/pocket-os/app/src/main/res") in targets + + +def test_plain_locale_dirs_accept_any_configured_convention(repo_root): + # A path's shape does not reveal which locale-code convention its tree uses + # (gettext es_ES vs the installer's es-es), so plain locale dirs accept both. + supported, pattern = _target_for(repo_root, "kolibri/locale") + + assert pattern is None + assert {"es_ES", "es-es"} <= supported + + +def test_android_res_prunes_values_qualifiers(repo_root): + supported, pattern = _target_for(repo_root, "platforms/pocket-os/app/src/main/res") + + assert pattern is ANDROID_LOCALE_QUALIFIER + assert {"es-rES", "b+es+419"} <= supported + + +def test_supported_sets_are_not_everything(repo_root): + # Canary on the mapping generators: `kaa` is one of the languages the Crowdin + # download delivered beyond Kolibri's set, so a supported set that contains it is + # a set that would prune nothing. + for _, supported, _ in cleanup_targets(repo_root): + assert "kaa" not in supported