diff --git a/.github/workflows/i18n-download.yml b/.github/workflows/i18n-download.yml index 46e3c4e1a7c..27efa086791 100644 --- a/.github/workflows/i18n-download.yml +++ b/.github/workflows/i18n-download.yml @@ -37,7 +37,10 @@ jobs: - name: Install gettext run: sudo apt-get update && sudo apt-get install -y gettext - name: Install Python dependencies - run: uv sync + # --all-packages installs the python_packages/* members; without them the + # Makefile's importability guard skips their message-file generation and + # their *-messages.json are never regenerated from the download. + run: uv sync --all-packages - name: Install JavaScript dependencies run: pnpm install --frozen-lockfile - name: Download source fonts diff --git a/.github/workflows/i18n-upload.yml b/.github/workflows/i18n-upload.yml index f41be1987f3..7064421a0ee 100644 --- a/.github/workflows/i18n-upload.yml +++ b/.github/workflows/i18n-upload.yml @@ -36,7 +36,10 @@ jobs: - name: Install gettext run: sudo apt-get update && sudo apt-get install -y gettext - name: Install Python dependencies - run: uv sync + # --all-packages installs the python_packages/* members; without them the + # Makefile's importability guard skips their message extraction and their + # CSVs never reach Crowdin. + run: uv sync --all-packages - name: Install JavaScript dependencies run: pnpm install --frozen-lockfile - name: Extract and upload strings to Crowdin diff --git a/Makefile b/Makefile index 68db3586a1e..4cd980c7ad8 100644 --- a/Makefile +++ b/Makefile @@ -254,10 +254,12 @@ i18n-extract-frontend: # buildConfig.js via the *installed* package (webpack_json.py files(module)), so # skip any plugin whose package is not importable here — e.g. `make dist`, which # installs only the root project, not the python_packages/* members. Where the - # members are installed (uv sync --all-packages) they extract normally. + # members are installed (uv sync --all-packages) they extract normally. The guard + # must run under uv: a bare `python` is the system interpreter, which never sees + # the members, so every plugin would be skipped however the project was synced. @for dir in $(I18N_PLUGIN_FRONTEND_DIRS); do \ module=$$(basename $$dir); \ - if ! python -c "import $$module" >/dev/null 2>&1; then \ + if ! uv run python -c "import $$module" >/dev/null 2>&1; then \ echo "Skipping frontend messages for $$module (package not installed)"; \ continue; \ fi; \ @@ -303,7 +305,7 @@ i18n-download-translations: i18n-extract-frontend fi @echo "✅ Translation files downloaded successfully" rm -f kolibri/locale/.crowdin-download-marker - uv run python build_tools/i18n/cleanup_unsupported_languages.py + uv run python -m build_tools.i18n.cleanup_unsupported_languages pnpm exec kolibri-i18n code-gen --output-dir ./packages/kolibri/utils/internal $(MAKE) i18n-django-compilemessages pnpm exec kolibri-i18n create-message-files --pluginFile ./build_tools/build_plugins.txt --ignore '**/node_modules/!(kolibri-common)/**','**/static/**' @@ -313,7 +315,7 @@ i18n-download-translations: i18n-extract-frontend # package is not importable here (same reason as i18n-extract-frontend). @for dir in $(I18N_PLUGIN_FRONTEND_DIRS); do \ module=$$(basename $$dir); \ - if ! python -c "import $$module" >/dev/null 2>&1; then \ + if ! uv run python -c "import $$module" >/dev/null 2>&1; then \ echo "Skipping message files for $$module (package not installed)"; \ continue; \ fi; \ diff --git a/build_tools/i18n/cleanup_unsupported_languages.py b/build_tools/i18n/cleanup_unsupported_languages.py index 30d30ba7439..7eb434a0440 100644 --- a/build_tools/i18n/cleanup_unsupported_languages.py +++ b/build_tools/i18n/cleanup_unsupported_languages.py @@ -4,41 +4,149 @@ This script should be run after downloading translations from Crowdin to clean up any languages that were downloaded but are not officially supported by Kolibri. +A `languages_mapping` in crowdin.yml only renames codes, it does not filter, so +Crowdin delivers every language enabled on the project into every tree we sync. """ +import glob import logging import os +import re import shutil -from utils import available_languages -from utils import LOCALE_DATA_FOLDER -from utils import to_locale +from build_tools.i18n.generate_mapping import get_android_language_mapping +from build_tools.i18n.generate_mapping import get_installer_language_mapping +from build_tools.i18n.generate_mapping import get_language_mapping +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) -def cleanup_unsupported_languages(): +# `values-` prefixes every Android resource qualifier, not just locales, so match the +# shape of a locale one — `fr`, `fr-rFR` or BCP-47 `b+es+419` — and leave the rest +# (values-night, values-v21, values-sw600dp) alone. `car` and `tv` are the UI-mode +# qualifiers short enough to also be shaped like a language code (both are real ISO 639 +# codes), so they are denied outright: in a res/ directory they are never a locale. +ANDROID_LOCALE_QUALIFIER = re.compile( + r"^values-(?!(?:car|tv)$)(b\+[a-z]{2,3}(?:\+[A-Za-z0-9]+)*|[a-z]{2,3}(?:-r[A-Z]{2})?)$" +) + +# Non-hidden build output that recursive discovery would otherwise walk into: p4a +# unpacks a full CPython under android_root/, and an unpacked Kolibri tar or a +# PyInstaller/sdist staging tree holds catalogs we do not own. glob already skips +# dot-prefixed directories, so .venv/ needs no entry. +EXCLUDED_DIR_NAMES = frozenset( + { + "node_modules", + "site-packages", + "android_root", + "build", + "dist", + "tar", + } +) + + +def _supported_locale_names(): + """Every locale directory name any of our sync targets may legitimately use. + + A directory's path does not reveal which convention its tree follows — the + desktop-app installer keys folders by hyphen-lowercase intl codes (es-es) while + everything gettext-based uses Django codes (es_ES) — so plain locale directories + are pruned against the union. Only unsupported *languages* are being removed + here; a code in the wrong convention for its tree is a different problem. """ - Remove locale directories that are not in our supported languages list. + return set(get_language_mapping().values()) | set( + get_installer_language_mapping().values() + ) + + +def prune_locale_dirs(directory, supported_names, pattern=None): + """Remove immediate subdirectories of `directory` for unsupported languages. + + `pattern` is a compiled regex whose first group extracts the locale name from a + subdirectory name; a name it does not match is not a locale directory and is left + alone. Without one, the subdirectory name is the locale name. Returns the sorted + names of what was removed. """ - # Get the set of supported locale directory names - supported_locales = set() - for lang_obj in available_languages(): - intl_code = lang_obj["intl_code"] - locale_dir = to_locale(intl_code) - supported_locales.add(locale_dir) - - # Check all directories in the locale folder - removed_locales = [] - for item in os.listdir(LOCALE_DATA_FOLDER): - item_path = os.path.join(LOCALE_DATA_FOLDER, item) + if not os.path.isdir(directory): + return [] + + removed = [] + for item in sorted(os.listdir(directory)): + item_path = os.path.join(directory, item) - # Skip if not a directory or if it's a special file if not os.path.isdir(item_path): continue - # If this directory is not in our supported list, remove it - if item not in supported_locales: + if pattern is None: + locale_name = item + else: + match = pattern.match(item) + if match is None: + continue + locale_name = match.group(1) + + if locale_name not in supported_names: shutil.rmtree(item_path) - removed_locales.append(item) + removed.append(item) + + return removed + + +def _is_excluded(path, repo_root): + # Relative to the root: a checkout under ~/build/kolibri must not exclude everything. + parts = os.path.relpath(path, repo_root).split(os.sep) + return any( + part in EXCLUDED_DIR_NAMES or part.endswith(".egg-info") for part in parts + ) + + +def _locale_roots(repo_root): + """Directories named locale/ or locales/, holding one subdirectory per language.""" + yield os.path.join(repo_root, "kolibri", "locale") + + for top in ("python_packages", "platforms"): + for name in ("locale", "locales"): + for path in glob.glob( + os.path.join(repo_root, top, "**", name), recursive=True + ): + # Vendored dependencies and staged build output ship their own + # catalogs; not ours to prune. + if _is_excluded(path, repo_root): + continue + yield path + + +def _android_res_roots(repo_root): + """Android res/ directories, identified by their untranslated values/ folder.""" + for path in glob.glob( + os.path.join(repo_root, "platforms", "**", "res", "values"), recursive=True + ): + if _is_excluded(path, repo_root): + continue + yield os.path.dirname(path) + + +def cleanup_targets(repo_root=REPO_ROOT): + """Yield (directory, supported_names, pattern) for every tree Crowdin writes to.""" + supported = _supported_locale_names() + for directory in _locale_roots(repo_root): + yield directory, supported, None + + android_supported = set(get_android_language_mapping().values()) + for directory in _android_res_roots(repo_root): + yield directory, android_supported, ANDROID_LOCALE_QUALIFIER + + +def cleanup_unsupported_languages(repo_root=REPO_ROOT): + """ + Remove locale directories that are not in our supported languages list. + """ + removed_locales = [] + for directory, supported_names, pattern in cleanup_targets(repo_root): + for name in prune_locale_dirs(directory, supported_names, pattern): + removed_locales.append( + os.path.join(os.path.relpath(directory, repo_root), name) + ) if removed_locales: logging.info( diff --git a/crowdin.yml b/crowdin.yml index 597c65479c1..0c355df4e70 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -10,9 +10,26 @@ "base_path": "." "base_url": "https://api.crowdin.com" # -# Defines whether to preserve the original directory structure in the Crowdin project +# Defines whether to preserve the original directory structure in the Crowdin project. +# Must be true, but NOT to get the source hierarchy: every entry below carries a +# `dest`, and those dests define the Crowdin layout — flat at the project root for +# what translators work on, plus an optional_plugins/ folder for the de-prioritised +# plugin catalogs. It is true only because the CLI rejects `dest` otherwise; flipping +# it to false would drop every dest and reinstate the source paths. # -"preserve_hierarchy": false +# `dest` is also what keeps same-named files distinct (each plugin's django.po), which +# preserve_hierarchy: false could not do at all. +# +# Crowdin keys files by their project path, so editing a `dest` below creates a new +# file and leaves the old one holding its translations — move it in the Crowdin UI +# instead, which preserves the file ID. +# +# A `dest` for a CSV MUST end in a literal `.csv` (not `%original_file_name%`): +# UploadSourcesAction.isSpreadsheet() extension-tests the UNRESOLVED dest string, and +# on a miss silently drops `scheme`/`first_line_contains_header`, so the file uploads +# clean but parses to nothing. `type: csv` does not help — isSpreadsheet ignores it. +# +"preserve_hierarchy": true # # Files configuration. # See https://support.crowdin.com/developer/configuration-file/ for all available options @@ -22,45 +39,48 @@ # files: [ # CSV translation files - {"source": "/kolibri/locale/en/LC_MESSAGES/*.csv", "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": &language_mapping {"locale_with_underscore": { + {"source": "/kolibri/locale/en/LC_MESSAGES/*.csv", "dest": "/%file_name%.csv", "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": &language_mapping {"locale_with_underscore": { # Generated mapping from crowdin_code to Django locale code # Generated by build_tools/i18n/generate_mapping.py "ach": "ach_UG", "ar": "ar", "bg": "bg_BG", "bn": "bn_BD", "de": "de", "el": "el", "en": "en", "es-ES": "es_ES", "fa": "fa", "fr": "fr_FR", "fv": "ff_CM", "gu-IN": "gu_IN", "ha": "ha", "hi": "hi_IN", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es_419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt_BR", "pt-mz": "pt_MZ", "sw-TZ": "sw_TZ", "te": "te", "uk": "uk", "ur-PK": "ur_PK", "vi": "vi", "yo": "yo", "zh-CN": "zh_Hans"}}}, - # Frontend i18n for ALL python_packages/* plugins (issue #14970) — one entry. - # Plugin frontend CSVs are namespace-prefixed (e.g. - # kolibri_sentry_plugin-messages.csv), so their basenames are globally unique - # under preserve_hierarchy: false. The `**` in source and translation carries - # each plugin's / subpath through, so downloads land back in the - # correct plugin locale/. A new frontend plugin needs NO change here. (Ignore - # node_modules so the recursive `**` can't pick up any vendored CSVs.) - {"source": "/python_packages/**/locale/en/LC_MESSAGES/*.csv", "ignore": ["/python_packages/**/node_modules/**"], "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/python_packages/**/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": *language_mapping}, + # Frontend i18n for ALL python_packages/* plugins (issue #14970) — one entry, which + # rests on plugin frontend CSVs being namespace-prefixed (kolibri_sentry_plugin- + # messages.csv): the basenames are globally unique, so one flattening `dest` cannot + # collide the way the per-plugin django.po below would. + # `dest` files them under optional_plugins/ in the Crowdin project: they are not a + # translation priority, so they stay out of the top-level file list. The `**` in + # source and translation carries each plugin's / subpath through, so + # downloads land back in the correct plugin locale/ regardless. A new frontend + # plugin needs NO change here. (Ignore node_modules so the recursive `**` can't + # pick up any vendored CSVs.) + {"source": "/python_packages/**/locale/en/LC_MESSAGES/*.csv", "dest": "/optional_plugins/%file_name%.csv", "ignore": ["/python_packages/**/node_modules/**"], "first_line_contains_header": true, "scheme": "identifier,source_phrase,context,translation", "update_option": "update_as_unapproved", "translation": "/python_packages/**/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "languages_mapping": *language_mapping}, # PO translation files - {"source": "/kolibri/locale/en/LC_MESSAGES/*.po", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "ignore": ["/kolibri/locale/en/LC_MESSAGES/*.json", "/kolibri/locale/en/LC_MESSAGES/*.mo", "/kolibri/locale/en/LC_MESSAGES/README.md"], "languages_mapping": *language_mapping}, - # Plugin backend PO — one entry PER plugin, unavoidably (unlike the frontend - # CSVs above). Django makemessages always emits a file named django.po, so - # under preserve_hierarchy: false every plugin's django.po flattens to the same - # Crowdin file and collides (with each other and with kolibri's). Keeping them - # distinct needs a unique `dest` per file, which is inherently one explicit - # entry per plugin: a wildcard `dest` disambiguated by directory would require - # preserve_hierarchy: true globally, restructuring kolibri's own catalogs on - # Crowdin (out of scope). `dest` renames only in the Crowdin project; the - # `translation` template still writes downloads to the real local django.po, so - # compile / message-file steps are unaffected. A plugin gaining its first - # backend string needs a new entry mirroring this one. - {"source": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/en/LC_MESSAGES/django.po", "dest": "/kolibri_oidc_client_plugin.django.po", "update_option": "update_as_unapproved", "translation": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/%locale_with_underscore%/LC_MESSAGES/django.po", "languages_mapping": *language_mapping}, + {"source": "/kolibri/locale/en/LC_MESSAGES/*.po", "dest": "/%file_name%.po", "update_option": "update_as_unapproved", "translation": "/kolibri/locale/%locale_with_underscore%/LC_MESSAGES/%original_file_name%", "ignore": ["/kolibri/locale/en/LC_MESSAGES/*.json", "/kolibri/locale/en/LC_MESSAGES/*.mo", "/kolibri/locale/en/LC_MESSAGES/README.md"], "languages_mapping": *language_mapping}, + # Plugin backend PO — one entry PER plugin, unlike the frontend CSVs above, and + # likewise filed under optional_plugins/. Django makemessages emits no domain but + # `django`, so every plugin's catalog is named django.po and they would collide + # with each other inside the folder. No placeholder can separate them either: + # `%file_name%` is the source basename, which is `django` for every plugin, and + # `%original_path%` is the whole relative source directory, which would rebuild the + # python_packages///locale/en/LC_MESSAGES hierarchy under + # optional_plugins/. So a literal `dest` per plugin it is. + # `dest` renames only in the Crowdin project; the `translation` template still + # writes downloads to the real local django.po, so compile / message-file steps are + # unaffected. A plugin gaining its first backend string needs a new entry mirroring + # this one. + {"source": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/en/LC_MESSAGES/django.po", "dest": "/optional_plugins/kolibri_oidc_client_plugin.django.po", "update_option": "update_as_unapproved", "translation": "/python_packages/kolibri-oidc-client-plugin/kolibri_oidc_client_plugin/locale/%locale_with_underscore%/LC_MESSAGES/django.po", "languages_mapping": *language_mapping}, # desktop-app (issue #15073) — two i18n subsystems with different locale-code # conventions. The wxPython app reuses the shared underscore anchor (its folders # are Django/gettext codes like es_ES). The Inno Setup installer needs its own # inline hyphen-lowercase %locale% map because installer/translations/definitions.py # keys the locale definitions by hyphen-lowercase codes (es-es, zh-hans). Both # maps derive from the same language_info.json via generate_mapping.py. - {"source": "/platforms/desktop-app/src/kolibri_app/locales/en/LC_MESSAGES/wxapp.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/src/kolibri_app/locales/%locale_with_underscore%/LC_MESSAGES/wxapp.po", "languages_mapping": *language_mapping}, {"source": "/platforms/desktop-app/installer/translations/locale/en/messages.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/installer/translations/locale/%locale%/messages.po", "languages_mapping": {"locale": {"ach": "ach-ug", "ar": "ar", "bg": "bg-bg", "bn": "bn-bd", "de": "de", "el": "el", "en": "en", "es-ES": "es-es", "fa": "fa", "fr": "fr-fr", "fv": "ff-cm", "gu-IN": "gu-in", "ha": "ha", "hi": "hi-in", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es-419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-br", "pt-mz": "pt-mz", "sw-TZ": "sw-tz", "te": "te", "uk": "uk", "ur-PK": "ur-pk", "vi": "vi", "yo": "yo", "zh-CN": "zh-hans"}}}, + {"source": "/platforms/desktop-app/src/kolibri_app/locales/en/LC_MESSAGES/wxapp.po", "dest": "/desktop_app.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/src/kolibri_app/locales/%locale_with_underscore%/LC_MESSAGES/wxapp.po", "languages_mapping": *language_mapping}, {"source": "/platforms/desktop-app/installer/translations/locale/en/messages.po", "dest": "/desktop_installer.po", "update_option": "update_as_unapproved", "translation": "/platforms/desktop-app/installer/translations/locale/%locale%/messages.po", "languages_mapping": {"locale": {"ach": "ach-ug", "ar": "ar", "bg": "bg-bg", "bn": "bn-bd", "de": "de", "el": "el", "en": "en", "es-ES": "es-es", "fa": "fa", "fr": "fr-fr", "fv": "ff-cm", "gu-IN": "gu-in", "ha": "ha", "hi": "hi-in", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "es-419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-br", "pt-mz": "pt-mz", "sw-TZ": "sw-tz", "te": "te", "uk": "uk", "ur-PK": "ur-pk", "vi": "vi", "yo": "yo", "zh-CN": "zh-hans"}}}, # Android installer UI strings (issue #15074). type: android forces Crowdin's # Android-resource parser (not .xml auto-detect), which honours # translatable="false" (so task_worker_process is not uploaded) and # Android-encodes downloads. %android_code% expands to the values- # qualifier; the android_code mapping (generated by # build_tools/i18n/generate_mapping.py) overrides Crowdin's defaults so folders - # match Kolibri's language set. dest keeps the flattened basename unique on - # Crowdin under preserve_hierarchy: false. + # match Kolibri's language set. {"source": "/platforms/android/app/src/main/res/values/strings.xml", "type": "android", "dest": "/android_strings.xml", "translation": "/platforms/android/app/src/main/res/values-%android_code%/strings.xml", "update_option": "update_as_unapproved", "languages_mapping": {"android_code": {"ach": "ach-rUG", "ar": "ar", "bg": "bg-rBG", "bn": "bn-rBD", "de": "de", "el": "el", "en": "en", "es-ES": "es-rES", "fa": "fa", "fr": "fr-rFR", "fv": "ff-rCM", "gu-IN": "gu-rIN", "ha": "ha", "hi": "hi-rIN", "ht": "ht", "id": "id", "it": "it", "ka": "ka", "km": "km", "ko": "ko", "la": "b+es+419", "mr": "mr", "my": "my", "ny": "ny", "pa-IN": "pa", "pt-BR": "pt-rBR", "pt-mz": "pt-rMZ", "sw-TZ": "sw-rTZ", "te": "te", "uk": "uk", "ur-PK": "ur-rPK", "vi": "vi", "yo": "yo", "zh-CN": "zh"}}}] diff --git a/test/test_crowdin_config.py b/test/test_crowdin_config.py new file mode 100644 index 00000000000..f32f7cbd8d3 --- /dev/null +++ b/test/test_crowdin_config.py @@ -0,0 +1,43 @@ +"""Project-wide invariants of crowdin.yml. + +Both of these fail silently rather than loudly, which is why they are asserted here +rather than left to the comments in the file: a CSV `dest` that loses its literal +`.csv` uploads clean and parses to nothing, and a missing `dest` puts a file on a +path no translation is attached to. +""" + +import os + +import pytest + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _config(): + # PyYAML rides in transitively via drf-yasg (gated to python_version >= + # '3.8'); skip on the 3.6/3.7 no-uv CI where it isn't installed. + yaml = pytest.importorskip("yaml") + with open(os.path.join(REPO_ROOT, "crowdin.yml")) as f: + return yaml.safe_load(f) + + +def test_hierarchy_is_preserved(): + # The CLI silently drops every `dest` without this. + assert _config()["preserve_hierarchy"] is True + + +def test_every_entry_has_a_dest(): + assert [e for e in _config()["files"] if not e.get("dest")] == [] + + +def test_dest_extension_matches_source(): + # UploadSourcesAction.isSpreadsheet() extension-tests the unresolved `dest`, so a + # CSV whose dest does not end in a literal `.csv` loses its scheme. + for entry in _config()["files"]: + source_ext = os.path.splitext(entry["source"])[1] + assert os.path.splitext(entry["dest"])[1] == source_ext, entry["source"] + + +def test_dests_are_unique(): + dests = [e["dest"] for e in _config()["files"]] + assert sorted(dests) == sorted(set(dests)) diff --git a/test/test_i18n_cleanup_unsupported_languages.py b/test/test_i18n_cleanup_unsupported_languages.py new file mode 100644 index 00000000000..e06196ef885 --- /dev/null +++ b/test/test_i18n_cleanup_unsupported_languages.py @@ -0,0 +1,197 @@ +import os + +import pytest + +from build_tools.i18n.cleanup_unsupported_languages import ANDROID_LOCALE_QUALIFIER +from build_tools.i18n.cleanup_unsupported_languages import cleanup_targets +from build_tools.i18n.cleanup_unsupported_languages import prune_locale_dirs + + +def _path(root, relpath): + # The module builds its paths with os.path.join/glob, so a forward-slash literal + # would not compare equal to them on Windows. + return os.path.join(root, *relpath.split("/")) + + +def _make_dirs(root, *relpaths): + for relpath in relpaths: + os.makedirs(_path(root, relpath)) + + +def test_prune_removes_unsupported_and_keeps_supported(tmp_path): + locale_dir = str(tmp_path) + _make_dirs(locale_dir, "en", "fr_FR", "tr_TR", "kaa") + + removed = prune_locale_dirs(locale_dir, {"en", "fr_FR"}) + + assert removed == ["kaa", "tr_TR"] + assert sorted(os.listdir(locale_dir)) == ["en", "fr_FR"] + + +def test_prune_ignores_files(tmp_path): + locale_dir = str(tmp_path) + _make_dirs(locale_dir, "fr_FR") + open(os.path.join(locale_dir, "language_info.json"), "w").close() + + assert prune_locale_dirs(locale_dir, {"fr_FR"}) == [] + assert os.path.exists(os.path.join(locale_dir, "language_info.json")) + + +def test_prune_with_pattern_only_touches_locale_qualifiers(tmp_path): + # values- prefixes every Android resource qualifier, not just locales. Pruning + # must reach the unsupported locale and nothing else — a deleted values-night or + # values-v21 would ride out unnoticed in the auto-generated translation PR. + res_dir = str(tmp_path) + _make_dirs( + res_dir, + "values", + "values-fr-rFR", + "values-b+es+419", + "values-tr-rTR", + "values-night", + "values-v21", + "values-sw600dp", + "values-land", + # UI-mode qualifiers that are also ISO 639 codes, so shape alone can't tell + # them from a locale. + "values-car", + "values-tv", + "drawable", + "xml", + ) + + removed = prune_locale_dirs( + res_dir, {"fr-rFR", "b+es+419"}, pattern=ANDROID_LOCALE_QUALIFIER + ) + + assert removed == ["values-tr-rTR"] + assert sorted(os.listdir(res_dir)) == [ + "drawable", + "values", + "values-b+es+419", + "values-car", + "values-fr-rFR", + "values-land", + "values-night", + "values-sw600dp", + "values-tv", + "values-v21", + "xml", + ] + + +def test_prune_missing_directory_is_noop(tmp_path): + assert prune_locale_dirs(os.path.join(str(tmp_path), "nope"), {"en"}) == [] + + +@pytest.fixture +def repo_root(tmp_path): + # Deliberately NOT the plugins/platforms this repo currently ships: discovery + # has to work for whatever is added next, not for a hardcoded inventory. + root = str(tmp_path) + _make_dirs( + root, + "kolibri/locale", + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale", + "python_packages/kolibri-notyet-plugin/kolibri_notyet_plugin/locale", + "platforms/handheld-app/src/handheld_app/locales", + "platforms/handheld-app/installer/translations/locale", + "platforms/pocket-os/app/src/main/res/values", + ) + return root + + +def _target_dirs(repo_root): + return {directory for directory, _, _ in cleanup_targets(repo_root)} + + +def _target_for(repo_root, relpath): + return next( + (supported, pattern) + for directory, supported, pattern in cleanup_targets(repo_root) + if directory == _path(repo_root, relpath) + ) + + +def test_cleanup_targets_discovers_trees_by_shape_not_by_name(repo_root): + targets = _target_dirs(repo_root) + + for expected in ( + "kolibri/locale", + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale", + "python_packages/kolibri-notyet-plugin/kolibri_notyet_plugin/locale", + "platforms/handheld-app/src/handheld_app/locales", + "platforms/handheld-app/installer/translations/locale", + "platforms/pocket-os/app/src/main/res", + ): + assert _path(repo_root, expected) in targets + + +@pytest.mark.parametrize( + "relpath", + [ + "python_packages/kolibri-imaginary-plugin/node_modules/dep/locale", + # p4a unpacks a full CPython under android_root/, so Django's own catalogs + # sit here in any checkout where the APK has been built. + "platforms/android/android_root/python-install/lib/python3.9/" + "site-packages/django/conf/locale", + "platforms/handheld-app/build/lib/handheld_app/locales", + "platforms/android/tar/extracted/kolibri-0.19/kolibri/locale", + ], +) +def test_untracked_build_output_is_left_alone(repo_root, relpath): + _make_dirs(repo_root, relpath) + targets = _target_dirs(repo_root) + + assert _path(repo_root, relpath) not in targets + # Guard the assertion above against passing vacuously: whatever shape paths + # compare in on this platform, a real root is still discovered. + assert _path(repo_root, "kolibri/locale") in targets + + +def test_excluded_names_only_match_below_the_repo_root(tmp_path): + # A checkout can itself sit under an excluded name (~/build/kolibri, /srv/dist/kolibri). + # Matching the whole path there would drop every glob-discovered root — silently, + # and for exactly the trees this script exists to prune. + root = _path(str(tmp_path), "build/kolibri") + plugin_locale = ( + "python_packages/kolibri-imaginary-plugin/kolibri_imaginary_plugin/locale" + ) + _make_dirs(root, plugin_locale, "platforms/pocket-os/app/src/main/res/values") + + targets = _target_dirs(root) + + assert _path(root, plugin_locale) in targets + assert _path(root, "platforms/pocket-os/app/src/main/res") in targets + + +def test_staged_android_res_is_left_alone(repo_root): + _make_dirs(repo_root, "platforms/android/build/intermediates/res/values") + targets = _target_dirs(repo_root) + + assert _path(repo_root, "platforms/android/build/intermediates/res") not in targets + assert _path(repo_root, "platforms/pocket-os/app/src/main/res") in targets + + +def test_plain_locale_dirs_accept_any_configured_convention(repo_root): + # A path's shape does not reveal which locale-code convention its tree uses + # (gettext es_ES vs the installer's es-es), so plain locale dirs accept both. + supported, pattern = _target_for(repo_root, "kolibri/locale") + + assert pattern is None + assert {"es_ES", "es-es"} <= supported + + +def test_android_res_prunes_values_qualifiers(repo_root): + supported, pattern = _target_for(repo_root, "platforms/pocket-os/app/src/main/res") + + assert pattern is ANDROID_LOCALE_QUALIFIER + assert {"es-rES", "b+es+419"} <= supported + + +def test_supported_sets_are_not_everything(repo_root): + # Canary on the mapping generators: `kaa` is one of the languages the Crowdin + # download delivered beyond Kolibri's set, so a supported set that contains it is + # a set that would prune nothing. + for _, supported, _ in cleanup_targets(repo_root): + assert "kaa" not in supported