From ed07da8be5513ac74aabb1c934a4545aaae4f5a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wolf-Martell=20Montw=C3=A9?= Date: Mon, 12 Feb 2024 18:26:37 +0100 Subject: [PATCH] Add translation cli --- cli/translation-cli/README.md | 23 ++++++++ cli/translation-cli/build.gradle.kts | 17 ++++++ .../cli/translation/LanguageCodeLoader.kt | 37 ++++++++++++ .../net/thunderbird/cli/translation/Main.kt | 3 + .../ResourceConfigurationsFormatter.kt | 17 ++++++ .../SupportedLanguagesFormatter.kt | 15 +++++ .../cli/translation/TranslationCli.kt | 51 ++++++++++++++++ .../cli/translation/net/Language.kt | 9 +++ .../translation/net/TranslationResponse.kt | 18 ++++++ .../cli/translation/net/WeblateClient.kt | 58 +++++++++++++++++++ .../cli/translation/net/WeblateConfig.kt | 26 +++++++++ gradle/libs.versions.toml | 5 ++ scripts/translation | 3 + settings.gradle.kts | 1 + 14 files changed, 283 insertions(+) create mode 100644 cli/translation-cli/README.md create mode 100644 cli/translation-cli/build.gradle.kts create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/LanguageCodeLoader.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/Main.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/ResourceConfigurationsFormatter.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/SupportedLanguagesFormatter.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/TranslationCli.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/Language.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/TranslationResponse.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateClient.kt create mode 100644 cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateConfig.kt create mode 100755 scripts/translation diff --git a/cli/translation-cli/README.md b/cli/translation-cli/README.md new file mode 100644 index 0000000000..0e09800399 --- /dev/null +++ b/cli/translation-cli/README.md @@ -0,0 +1,23 @@ +# Translation CLI + +This is a command line interface that will check the [weblate](https://hosted.weblate.org/projects/tb-android/#languages) translation state for all languages and print out the ones that are above a certain threshold. + +## Usage + +To use this script you need to have a weblate token. You can get it by logging in to weblate and going to your profile settings. + +You can run the script with the following command: + +```bash +./scripts/translation --token [--threshold 70] +``` + +It will print out the languages that are above the threshold. The default threshold is 70. You can change it by passing the `--threshold` argument. + +If you want a code example, you can pass the `--print-all` argument. It will print out example code for easier integration into the project. + +```bash +./scripts/translation --token --print-all +``` + +You could use this output to update the `resourceConfigurations` variable in the `app-k9mail/build.gradle.kts` file and the `supported_languages` in the `arrays_general_settings_values.xml` file. diff --git a/cli/translation-cli/build.gradle.kts b/cli/translation-cli/build.gradle.kts new file mode 100644 index 0000000000..af2f91e72b --- /dev/null +++ b/cli/translation-cli/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + id(ThunderbirdPlugins.App.jvm) +} + +version = "unspecified" + +application { + mainClass.set("net.thunderbird.cli.translation.MainKt") +} + +dependencies { + implementation(libs.clikt) + implementation(platform(libs.http4k.bom)) + implementation(libs.http4k.core) + implementation(libs.http4k.client.okhttp) + implementation(libs.http4k.format.moshi) +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/LanguageCodeLoader.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/LanguageCodeLoader.kt new file mode 100644 index 0000000000..6b9721cec8 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/LanguageCodeLoader.kt @@ -0,0 +1,37 @@ +package net.thunderbird.cli.translation + +import net.thunderbird.cli.translation.net.Language +import net.thunderbird.cli.translation.net.Translation +import net.thunderbird.cli.translation.net.WeblateClient + +class LanguageCodeLoader( + private val client: WeblateClient = WeblateClient(), +) { + fun loadCurrentAndroidLanguageCodes(token: String, threshold: Double): List { + val languages = client.loadLanguages(token) + val translations = client.loadTranslations(token) + val languageCodeLookup = createLanguageCodeLookup(translations) + + return filterAndMapLanguages(languages, threshold, languageCodeLookup) + } + + private fun createLanguageCodeLookup(translations: List): Map { + return translations.associate { it.language.code to it.languageCode } + } + + private fun filterAndMapLanguages( + languages: List, + threshold: Double, + languageCodeLookup: Map, + ): List { + return languages.filter { it.translatedPercent >= threshold } + .map { + languageCodeLookup[it.code] ?: throw IllegalArgumentException("Language code ${it.code} is not mapped") + }.map { fixLanguageCodeFormat(it) } + .sorted() + } + + private fun fixLanguageCodeFormat(languageCode: String): String { + return if (languageCode.contains("-r")) languageCode.replace("-r", "_") else languageCode + } +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/Main.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/Main.kt new file mode 100644 index 0000000000..963c993f49 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/Main.kt @@ -0,0 +1,3 @@ +package net.thunderbird.cli.translation + +fun main(args: Array) = TranslationCli().main(args) diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/ResourceConfigurationsFormatter.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/ResourceConfigurationsFormatter.kt new file mode 100644 index 0000000000..ff87447945 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/ResourceConfigurationsFormatter.kt @@ -0,0 +1,17 @@ +package net.thunderbird.cli.translation + +class ResourceConfigurationsFormatter { + fun format(languageCodes: List) = buildString { + appendLine("android {") + appendLine(" defaultConfig {") + appendLine(" resourceConfigurations.addAll(") + appendLine(" listOf(") + languageCodes.forEach { code -> + appendLine(" \"$code\",") + } + appendLine(" ),") + appendLine(" )") + appendLine(" }") + appendLine("}") + }.trim() +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/SupportedLanguagesFormatter.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/SupportedLanguagesFormatter.kt new file mode 100644 index 0000000000..9bbed9aa84 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/SupportedLanguagesFormatter.kt @@ -0,0 +1,15 @@ +package net.thunderbird.cli.translation + +class SupportedLanguagesFormatter { + fun format(languageCodes: List) = buildString { + appendLine("") + appendLine("") + appendLine(" ") + appendLine(" ") + languageCodes.forEach { + appendLine(" $it") + } + appendLine(" ") + appendLine("") + }.trim() +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/TranslationCli.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/TranslationCli.kt new file mode 100644 index 0000000000..921371e2d7 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/TranslationCli.kt @@ -0,0 +1,51 @@ +package net.thunderbird.cli.translation + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.flag +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.double + +const val TRANSLATED_THRESHOLD = 70.0 + +class TranslationCli( + private val languageCodeLoader: LanguageCodeLoader = LanguageCodeLoader(), + private val configurationsFormatter: ResourceConfigurationsFormatter = ResourceConfigurationsFormatter(), + private val supportedLanguagesFormatter: SupportedLanguagesFormatter = SupportedLanguagesFormatter(), +) : CliktCommand( + name = "translation", + help = "Translation CLI", +) { + private val token: String by option( + help = "Weblate API token", + ).required() + + private val threshold: Double by option( + help = "Threshold for translation completion", + ).double().default(TRANSLATED_THRESHOLD) + + private val printAll: Boolean by option( + help = "Print code example", + ).flag() + + override fun run() { + val languageCodes = languageCodeLoader.loadCurrentAndroidLanguageCodes(token, threshold) + val size = languageCodes.size + + echo("\nLanguages that are translated above the threshold of ($threshold%): $size") + echo("--------------------------------------------------------------") + echo(languageCodes.joinToString(", ")) + if (printAll) { + echo("--------------------------------------------------------------") + echo(configurationsFormatter.format(languageCodes)) + echo("--------------------------------------------------------------") + echo("--------------------------------------------------------------") + echo(supportedLanguagesFormatter.format(languageCodes)) + echo("--------------------------------------------------------------") + echo("Please read docs/translating.md for more information on how to update language values.") + echo("--------------------------------------------------------------") + } + echo() + } +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/Language.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/Language.kt new file mode 100644 index 0000000000..c975e1ee8f --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/Language.kt @@ -0,0 +1,9 @@ +package net.thunderbird.cli.translation.net + +import com.squareup.moshi.Json + +data class Language( + val code: String, + @Json(name = "translated_percent") + val translatedPercent: Double, +) diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/TranslationResponse.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/TranslationResponse.kt new file mode 100644 index 0000000000..f63f45233d --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/TranslationResponse.kt @@ -0,0 +1,18 @@ +package net.thunderbird.cli.translation.net + +import com.squareup.moshi.Json + +data class TranslationResponse( + val next: String?, + val results: List, +) + +data class Translation( + @Json(name = "language_code") + val languageCode: String, + val language: TranslationLanguage, +) + +data class TranslationLanguage( + val code: String, +) diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateClient.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateClient.kt new file mode 100644 index 0000000000..4f722254f4 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateClient.kt @@ -0,0 +1,58 @@ +package net.thunderbird.cli.translation.net + +import org.http4k.client.OkHttp +import org.http4k.core.Body +import org.http4k.core.HttpHandler +import org.http4k.core.Method +import org.http4k.core.Request +import org.http4k.core.Uri +import org.http4k.format.Moshi.auto + +class WeblateClient( + private val client: HttpHandler = OkHttp(), + private val config: WeblateConfig = WeblateConfig(), +) { + fun loadLanguages(token: String): List { + val request = Request(Method.GET, Uri.of(config.projectsLanguagesUrl())) + .headers(config.getDefaultHeaders(token)) + + val response = client(request) + val lens = Body.auto>().toLens() + + return lens(response).toList() + } + + fun loadTranslations(token: String): List { + val translations = mutableListOf() + var page = 1 + var hasNextPage = true + + while (hasNextPage) { + val translationPage = loadTranslationPage(token, page) + translations.addAll(translationPage.results) + + hasNextPage = translationPage.next != null + page++ + } + + return translations + } + + private fun loadTranslationPage(token: String, page: Int): TranslationResponse { + val request = Request(Method.GET, Uri.of(config.componentsTranslationsUrl(page))) + .headers(config.getDefaultHeaders(token)) + + val response = client(request) + val lens = Body.auto().toLens() + + return lens(response) + } + + private companion object { + private fun WeblateConfig.projectsLanguagesUrl() = + "${baseUrl}projects/$projectName/languages/" + + private fun WeblateConfig.componentsTranslationsUrl(page: Int) = + "${baseUrl}components/$projectName/$defaultComponent/translations/?page=$page" + } +} diff --git a/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateConfig.kt b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateConfig.kt new file mode 100644 index 0000000000..8109431ef2 --- /dev/null +++ b/cli/translation-cli/src/main/kotlin/net/thunderbird/cli/translation/net/WeblateConfig.kt @@ -0,0 +1,26 @@ +package net.thunderbird.cli.translation.net + +/** + * Configuration for Weblate API + * + * @property baseUrl Base URL of the Weblate API + * @property projectName Name of the Weblate project + * @property defaultComponent Default component to use for translations + */ +data class WeblateConfig( + val baseUrl: String = "https://hosted.weblate.org/api/", + val projectName: String = "tb-android", + val defaultComponent: String = "app-strings", + private val defaultHeaders: Map = mapOf( + "Accept" to "application/json", + "Authorization" to "Token $PLACEHOLDER_TOKEN", + ), +) { + fun getDefaultHeaders(token: String): List> = + defaultHeaders.mapValues { it.value.replace(PLACEHOLDER_TOKEN, token) } + .map { (key, value) -> key to value } + + private companion object { + const val PLACEHOLDER_TOKEN = "{weblate_token}" + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2ecc23b55d..384617b4f6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -58,6 +58,7 @@ fastAdapter = "5.7.0" forkhandlesBom = "2.12.2.0" glide = "4.16.0" gradle = "8.5" +http4kBom = "5.13.7.0" icu4j = "72.1" javaDiffUtils = "4.12" jcipAnnotations = "1.0" @@ -177,6 +178,10 @@ forkhandles-bom = { module = "dev.forkhandles:forkhandles-bom", version.ref = "f forkhandles-fabrikate4k = { module = "dev.forkhandles:fabrikate4k" } glide = { module = "com.github.bumptech.glide:glide", version.ref = "glide" } glide-compiler = { module = "com.github.bumptech.glide:compiler", version.ref = "glide" } +http4k-bom = { module = "org.http4k:http4k-bom", version.ref = "http4kBom" } +http4k-core = { module = "org.http4k:http4k-core" } +http4k-client-okhttp = { module = "org.http4k:http4k-client-okhttp" } +http4k-format-moshi = { module = "org.http4k:http4k-format-moshi" } icu4j-charset = { module = "com.ibm.icu:icu4j-charset", version.ref = "icu4j" } jetbrains-annotations = { module = "org.jetbrains:annotations", version.ref = "jetbrainsAnnotations" } jcip-annotations = { module = "net.jcip:jcip-annotations", version.ref = "jcipAnnotations" } diff --git a/scripts/translation b/scripts/translation new file mode 100755 index 0000000000..bcc2971efb --- /dev/null +++ b/scripts/translation @@ -0,0 +1,3 @@ +#!/bin/sh + +./gradlew --quiet ":cli:translation-cli:installDist" < /dev/null && ./cli/translation-cli/build/install/translation-cli/bin/translation-cli "$@" diff --git a/settings.gradle.kts b/settings.gradle.kts index 20877997f2..2db6d467e6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -108,4 +108,5 @@ include(":plugins:openpgp-api-lib:openpgp-api") include( ":cli:autodiscovery-cli", ":cli:html-cleaner-cli", + ":cli:translation-cli", )