diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..c4b17d79 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use_flake diff --git a/.gitignore b/.gitignore index 929a619e..4670104f 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,7 @@ crowdin.properties # C++ .cxx/ + +# Nix stuff +.direnv/ +result diff --git a/LANGUAGEPACKS.md b/LANGUAGEPACKS.md new file mode 100644 index 00000000..960db309 --- /dev/null +++ b/LANGUAGEPACKS.md @@ -0,0 +1,67 @@ +# Language Packs + +## Languages + +- [Summary](#summary) +- [Chinese / 中文](#chinese--中文) + +## Summary + +Stub. + +This page should describe how language packs work, how to import them, and point to the location of downloadable +language packs. + +The homepage of default language packs included in FlorisBoard should link to this page. + +## Chinese / 中文 + +TODO: translate into native language + +### Default barebones Chinese shape-based pack + +默认中文形码语言包 / 預設中文形碼語言包 / Default barebones Chinese shape-based language pack which are always available. + +Please download the [full Chinese language pack](#full-chinese-shape-based-pack) to access input methods +such as wubi, boshiami, and other versions of cangjie. + +This pack is released under the same license as Florisboard. + +TODO: translate into native language + +### Full Chinese shape-based pack + +Fcitx5 中文形码语言包 / Fcitx5 中文形碼語言包 / Chinese shape-based language pack based on fcitx5-table-extra. + +This pack is released under a separate license. Please visit: [TODO: add link to release page](https://) + +The following input methods are included in this language pack: + +- 中文 (中国) [T9笔画] / Chinese (China) [T9] +- 中文 (中国) [五笔98] / Chinese (China) [WUBI98] +- 中文 (中国) [五笔98-拼音混打] / Chinese (China) [WUBI98PINYIN] +- 中文 (中国) [五笔98-单字] / Chinese (China) [WUBI98SINGLE] +- 中文 (中国) [五笔-大字库] / Chinese (China) [WUBILARGE] +- 中文 (中国) [郑码] / Chinese (China) [ZHENGMA] +- 中文 (中国) [郑码-大字库] / Chinese (China) [ZHENGMALARGE] +- 中文 (中国) [郑码-拼音混打] / Chinese (China) [ZHENGMAPINYIN] +- 中文 (香港) [廣東拼音] / Chinese (Hong Kong) [CANTONESE] +- 中文 (香港) [港式廣東話] / Chinese (Hong Kong) [CANTONHK] +- 中文 (香港) [輕鬆-大字庫] / Chinese (Hong Kong) [EASYLARGE] +- 中文 (香港) [粤語拼音-表格] / Chinese (Hong Kong) [JYUTPINGTABLE] +- 中文 (香港) [速成三代] / Chinese (Hong Kong) [QUICK3] +- 中文 (香港) [經典速成] / Chinese (Hong Kong) [QUICKCLASSIC] +- 中文 (香港) [筆順五碼] / Chinese (Hong Kong) [STROKE5] +- 中文 (台灣) [行列] / Chinese (Taiwan) [ARRAY30] +- 中文 (台灣) [行列-大字库] / Chinese (Taiwan) [ARRAY30LARGE] +- 中文 (台灣) [嘸蝦米] / Chinese (Taiwan) [BOSHIAMY] +- 中文 (台灣) [倉頡三代] / Chinese (Taiwan) [CANGJIE3] +- 中文 (台灣) [倉頡五代] / Chinese (Taiwan) [CANGJIE5] +- 中文 (台灣) [倉頡-大字庫] / Chinese (Taiwan) [CANGJIELARGE] +- 中文 (台灣) [速成五代] / Chinese (Taiwan) [QUICK5] +- 中文 (台灣) [快倉六] / Chinese (Taiwan) [SCJ6] +- 中文 (台灣) [吳語注音] / Chinese (Taiwan) [WU] + +Third-party license: [https://github.com/fcitx/fcitx5-table-extra/blob/master/LICENSES/GPL-3.0-or-later.txt] + +TODO: translate into native language diff --git a/app/src/main/assets/ime/keyboard/org.florisboard.localization/extension.json b/app/src/main/assets/ime/keyboard/org.florisboard.localization/extension.json index faf6f59f..a605c716 100644 --- a/app/src/main/assets/ime/keyboard/org.florisboard.localization/extension.json +++ b/app/src/main/assets/ime/keyboard/org.florisboard.localization/extension.json @@ -646,6 +646,21 @@ "preferred": { "characters": "org.florisboard.layouts:urdu_phonetic" } + }, + { + "languageTag": "zh-CN-zhengma", + "composer": "org.florisboard.composers:appender", + "nlpProviders": { + "spelling": "org.florisboard.nlp.providers.han.shape", + "suggestion": "org.florisboard.nlp.providers.han.shape" + }, + "currencySet": "org.florisboard.currencysets:yen", + "popupMapping": "org.florisboard.localization:en", + "preferred": { + "characters": "org.florisboard.layouts:qwerty", + "symbols": "org.florisboard.layouts:cjk", + "symbols2": "org.florisboard.layouts:cjk" + } }, { "languageTag": "bn-BD", diff --git a/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/extension.json b/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/extension.json new file mode 100644 index 00000000..be1776a2 --- /dev/null +++ b/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/extension.json @@ -0,0 +1,32 @@ +{ + "$": "ime.extension.languagepack", + "meta": { + "id": "org.florisboard.hanshapebasedbasicpack", + "version": "0.0.0", + "title": "Default barebones Chinese shape-based pack", + "description": "默认中文形码语言包 / 預設中文形碼語言包 / Default barebones Chinese shape-based language pack which are always available.\n请访问下面的主页链接下载更多输入法 / 請訪問下面的主頁鏈接下載更多輸入法 / Please visit the homepage linked below to download more input methods.", + "maintainers": [ "patrickgold ", "waelwindows", "moonbeamcelery" ], + "homepage": "https://github.com/florisboard/florisboard/blob/master/LANGUAGEPACKS.md#default-barebones-chinese-shape-based-pack", + "license": "apache-2.0" + }, + "items": [ + { + "id": "zh_CN_zhengma", + "label": "中文 (中国) [郑码] / Chinese (China) [ZHENGMA]", + "authors": [ "waelwindows", "moonbeamcelery" ], + "hanShapeBasedKeyCode": "abcdefghijklmnopqrstuvwxyz" + }, + { + "id": "zh_TW_boshiamy", + "label": "中文 (台灣) [嘸蝦米] / Chinese (Taiwan) [BOSHIAMY]", + "authors": [ "waelwindows", "moonbeamcelery" ], + "hanShapeBasedKeyCode": ",.'abcdefghijklmnopqrstuvwxyz[]" + }, + { + "id": "zh_TW_cangjielarge", + "label": "中文 (台灣) [倉頡-大字庫] / Chinese (Taiwan) [CANGJIELARGE]", + "authors": [ "waelwindows", "moonbeamcelery" ], + "hanShapeBasedKeyCode": "abcdefghijklmnopqrstuvwxyz&" + } + ] +} diff --git a/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/han.sqlite3 b/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/han.sqlite3 new file mode 100644 index 00000000..dc30f810 Binary files /dev/null and b/app/src/main/assets/ime/languagepack/org.florisboard.hanshapebasedbasicpack/han.sqlite3 differ diff --git a/app/src/main/assets/ime/languagepack/org.florisboard.languagepack/extension.json b/app/src/main/assets/ime/languagepack/org.florisboard.languagepack/extension.json new file mode 100644 index 00000000..519b4347 --- /dev/null +++ b/app/src/main/assets/ime/languagepack/org.florisboard.languagepack/extension.json @@ -0,0 +1,23 @@ +{ + "$": "ime.extension.languagepack", + "meta": { + "id": "org.florisboard.languagepack", + "version": "0.0.0", + "title": "Default language pack", + "description": "Default language pack which are always available.", + "maintainers": [ "patrickgold " ], + "license": "apache-2.0" + }, + "items": [ + { + "id": "de_DE_neobone", + "label": "Deutsch (Deutschland) [NEOBONE] / German (Germany) [NEOBONE]", + "authors": [ "patrickgold" ] + }, + { + "id": "ja_JP_jis", + "label": "日本語 (日本) [JIS] / Japanese (Japan) [JIS]", + "authors": [ "patrickgold" ] + } + ] +} diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/Routes.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/Routes.kt index 65ecff61..41d90c69 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/Routes.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/Routes.kt @@ -44,6 +44,8 @@ import dev.patrickgold.florisboard.app.settings.dictionary.UserDictionaryType import dev.patrickgold.florisboard.app.settings.gestures.GesturesScreen import dev.patrickgold.florisboard.app.settings.keyboard.InputFeedbackScreen import dev.patrickgold.florisboard.app.settings.keyboard.KeyboardScreen +import dev.patrickgold.florisboard.app.settings.localization.LanguagePackManagerScreen +import dev.patrickgold.florisboard.app.settings.localization.LanguagePackManagerScreenAction import dev.patrickgold.florisboard.app.settings.localization.LocalizationScreen import dev.patrickgold.florisboard.app.settings.localization.SelectLocaleScreen import dev.patrickgold.florisboard.app.settings.localization.SubtypeEditorScreen @@ -67,6 +69,9 @@ object Routes { const val Localization = "settings/localization" const val SelectLocale = "settings/localization/select-locale" + const val LanguagePackManager = "settings/localization/language-pack-manage/{action}" + fun LanguagePackManager(action: LanguagePackManagerScreenAction) = + LanguagePackManager.curlyFormat("action" to action.id) const val SubtypeAdd = "settings/localization/subtype/add" const val SubtypeEdit = "settings/localization/subtype/edit/{id}" fun SubtypeEdit(id: Long) = SubtypeEdit.curlyFormat("id" to id) @@ -147,6 +152,12 @@ object Routes { composable(Settings.Localization) { LocalizationScreen() } composable(Settings.SelectLocale) { SelectLocaleScreen() } + composable(Settings.LanguagePackManager) { navBackStack -> + val action = navBackStack.arguments?.getString("action")?.let { actionId -> + LanguagePackManagerScreenAction.values().firstOrNull { it.id == actionId } + } + LanguagePackManagerScreen(action) + } composable(Settings.SubtypeAdd) { SubtypeEditorScreen(null) } composable(Settings.SubtypeEdit) { navBackStack -> val id = navBackStack.arguments?.getString("id")?.toLongOrNull() diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionComponentView.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionComponentView.kt index 9f4dac3a..bac828a0 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionComponentView.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionComponentView.kt @@ -37,6 +37,7 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import dev.patrickgold.florisboard.R +import dev.patrickgold.florisboard.ime.nlp.LanguagePackComponent import dev.patrickgold.florisboard.ime.theme.ThemeExtensionComponent import dev.patrickgold.florisboard.lib.compose.FlorisIconButton import dev.patrickgold.florisboard.lib.compose.FlorisOutlinedBox @@ -96,6 +97,22 @@ fun ExtensionComponentView( color = LocalContentColor.current.copy(alpha = LocalContentAlpha.current), ) } + is LanguagePackComponent -> { + val text = remember( + component.authors, component.locale, component.hanShapeBasedKeyCode, + ) { + buildString { + appendLine("authors = ${component.authors}") + appendLine("locale = ${component.locale.localeTag()}") + appendLine("hanShapeBasedKeyCode = ${component.hanShapeBasedKeyCode}") + } + } + Text( + text = text, + style = MaterialTheme.typography.body2, + color = LocalContentColor.current.copy(alpha = LocalContentAlpha.current), + ) + } else -> { } } } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionImportScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionImportScreen.kt index bc4df4a9..6fcfb6a4 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionImportScreen.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionImportScreen.kt @@ -48,6 +48,7 @@ import dev.patrickgold.florisboard.app.LocalNavController import dev.patrickgold.florisboard.cacheManager import dev.patrickgold.florisboard.extensionManager import dev.patrickgold.florisboard.ime.keyboard.KeyboardExtension +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension import dev.patrickgold.florisboard.ime.theme.ThemeExtension import dev.patrickgold.florisboard.lib.NATIVE_NULLPTR import dev.patrickgold.florisboard.lib.android.showLongToast @@ -60,6 +61,7 @@ import dev.patrickgold.florisboard.lib.compose.FlorisScreen import dev.patrickgold.florisboard.lib.compose.defaultFlorisOutlinedBox import dev.patrickgold.florisboard.lib.compose.florisHorizontalScroll import dev.patrickgold.florisboard.lib.compose.stringRes +import dev.patrickgold.florisboard.lib.devtools.flogDebug import dev.patrickgold.florisboard.lib.io.FileRegistry import dev.patrickgold.florisboard.lib.kotlin.resultOk @@ -82,6 +84,11 @@ enum class ExtensionImportScreenType( id = "ext-theme", titleResId = R.string.ext__import__ext_theme, supportedFiles = listOf(FileRegistry.FlexExtension), + ), + EXT_LANGUAGEPACK( + id = "ext-languagepack", + titleResId = R.string.ext__import__ext_languagepack, + supportedFiles = listOf(FileRegistry.FlexExtension), ); } @@ -174,6 +181,9 @@ fun ExtensionImportScreen(type: ExtensionImportScreenType, initUuid: String?) = ExtensionImportScreenType.EXT_THEME -> { ext.takeIf { it is ThemeExtension }?.let { extensionManager.import(it) } } + ExtensionImportScreenType.EXT_LANGUAGEPACK -> { + ext.takeIf { it is LanguagePackExtension }?.let { extensionManager.import(it) } + } } } }.onSuccess { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionViewScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionViewScreen.kt index bac794a9..4e3e1229 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionViewScreen.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/ext/ExtensionViewScreen.kt @@ -46,6 +46,7 @@ import dev.patrickgold.florisboard.R import dev.patrickgold.florisboard.app.LocalNavController import dev.patrickgold.florisboard.app.Routes import dev.patrickgold.florisboard.extensionManager +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension import dev.patrickgold.florisboard.ime.theme.ThemeExtension import dev.patrickgold.florisboard.ime.theme.ThemeExtensionComponentImpl import dev.patrickgold.florisboard.lib.android.showLongToast @@ -174,6 +175,18 @@ private fun ViewScreen(ext: Extension) = FlorisScreen { ) } } + is LanguagePackExtension -> { + ExtensionComponentListView( + title = stringRes(R.string.ext__meta__components_language_pack), + components = ext.items, + ) { component -> + ExtensionComponentView( + modifier = Modifier.defaultFlorisOutlinedBox(), + meta = ext.meta, + component = component, + ) + } + } else -> { // Render nothing } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LanguagePackManagerScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LanguagePackManagerScreen.kt new file mode 100644 index 00000000..234f14a6 --- /dev/null +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LanguagePackManagerScreen.kt @@ -0,0 +1,216 @@ +/* + * Copyright (C) 2021 Patrick Goldinger + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.patrickgold.florisboard.app.settings.localization + +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.IntrinsicSize +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.rememberScrollState +import androidx.compose.material.ButtonDefaults +import androidx.compose.material.Icon +import androidx.compose.material.LocalContentColor +import androidx.compose.material.MaterialTheme +import androidx.compose.material.RadioButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.key +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.painterResource +import androidx.compose.ui.unit.dp +import dev.patrickgold.florisboard.R +import dev.patrickgold.florisboard.app.LocalNavController +import dev.patrickgold.florisboard.app.Routes +import dev.patrickgold.florisboard.app.ext.ExtensionImportScreenType +import dev.patrickgold.florisboard.app.florisPreferenceModel +import dev.patrickgold.florisboard.extensionManager +import dev.patrickgold.florisboard.ime.nlp.LanguagePackComponent +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension +import dev.patrickgold.florisboard.lib.android.showLongToast +import dev.patrickgold.florisboard.lib.compose.FlorisConfirmDeleteDialog +import dev.patrickgold.florisboard.lib.compose.FlorisOutlinedBox +import dev.patrickgold.florisboard.lib.compose.FlorisScreen +import dev.patrickgold.florisboard.lib.compose.FlorisTextButton +import dev.patrickgold.florisboard.lib.compose.defaultFlorisOutlinedBox +import dev.patrickgold.florisboard.lib.compose.rippleClickable +import dev.patrickgold.florisboard.lib.compose.stringRes +import dev.patrickgold.florisboard.lib.ext.Extension +import dev.patrickgold.florisboard.lib.ext.ExtensionComponentName +import dev.patrickgold.florisboard.lib.observeAsNonNullState +import dev.patrickgold.jetpref.datastore.model.observeAsState +import dev.patrickgold.jetpref.datastore.ui.ExperimentalJetPrefDatastoreUi +import dev.patrickgold.jetpref.datastore.ui.Preference +import dev.patrickgold.jetpref.material.ui.JetPrefListItem + +enum class LanguagePackManagerScreenAction(val id: String) { + MANAGE("manage-installed-language-packs"); +} + +// TODO: this file is based on ThemeManagerScreen.kt and can arguably be merged. +@OptIn(ExperimentalJetPrefDatastoreUi::class) +@Composable +fun LanguagePackManagerScreen(action: LanguagePackManagerScreenAction?) = FlorisScreen { + title = stringRes(when (action) { + LanguagePackManagerScreenAction.MANAGE -> R.string.settings__localization__language_pack_title + else -> error("LanguagePack manager screen action must not be null") + }) + + val prefs by florisPreferenceModel() + val navController = LocalNavController.current + val context = LocalContext.current + val extensionManager by context.extensionManager() + + val indexedLanguagePackExtensions by extensionManager.languagePacks.observeAsNonNullState() + val selectedManagerLanguagePackId = remember { mutableStateOf(null) } + val extGroupedLanguagePacks = remember(indexedLanguagePackExtensions) { + buildMap> { + for (ext in indexedLanguagePackExtensions) { + put(ext.meta.id, ext.items) + } + }.mapValues { (_, configs) -> configs.sortedBy { it.label } } + } + + fun getLanguagePackIdPref(): Nothing = TODO("Not implemented yet") + + fun setLanguagePack(extId: String, componentId: String) { + val extComponentName = ExtensionComponentName(extId, componentId) + when (action) { + LanguagePackManagerScreenAction.MANAGE -> { + selectedManagerLanguagePackId.value = extComponentName + } + } + } + + val activeLanguagePackId by when (action) { + LanguagePackManagerScreenAction.MANAGE -> selectedManagerLanguagePackId + } + var languagePackExtToDelete by remember { mutableStateOf(null) } + + content { + val grayColor = LocalContentColor.current.copy(alpha = 0.56f) + if (action == LanguagePackManagerScreenAction.MANAGE) { + FlorisOutlinedBox( + modifier = Modifier.defaultFlorisOutlinedBox(), + ) { + this@content.Preference( + onClick = { navController.navigate( + Routes.Ext.Import(ExtensionImportScreenType.EXT_LANGUAGEPACK, null) + ) }, + iconId = R.drawable.ic_input, + title = stringRes(R.string.action__import), + ) + } + } + for ((extensionId, configs) in extGroupedLanguagePacks) key(extensionId) { + val ext = extensionManager.getExtensionById(extensionId)!! + FlorisOutlinedBox( + modifier = Modifier.defaultFlorisOutlinedBox(), + title = ext.meta.title, + onTitleClick = { navController.navigate(Routes.Ext.View(extensionId)) }, + subtitle = extensionId, + onSubtitleClick = { navController.navigate(Routes.Ext.View(extensionId)) }, + ) { + Column( + // Allowing horizontal scroll to fit translations in descriptions. + Modifier.horizontalScroll(rememberScrollState()).width(intrinsicSize = IntrinsicSize.Max), + ) { + for (config in configs) key(extensionId, config.id) { + JetPrefListItem( + modifier = Modifier.rippleClickable { + setLanguagePack(extensionId, config.id) + }, +// icon = { +// RadioButton( +// selected = activeLanguagePackId?.extensionId == extensionId && +// activeLanguagePackId?.componentId == config.id, +// onClick = null, +// ) +// }, + text = config.label, +// trailing = { +// Icon( +// modifier = Modifier.size(ButtonDefaults.IconSize), +// painter = painterResource(if (config.isNightLanguagePack) { +// R.drawable.ic_dark_mode +// } else { +// R.drawable.ic_light_mode +// }), +// contentDescription = null, +// tint = grayColor, +// ) +// }, + ) + } + } + if (action == LanguagePackManagerScreenAction.MANAGE && extensionManager.canDelete(ext)) { + Row( + modifier = Modifier + .fillMaxWidth() + .padding(horizontal = 6.dp), + ) { + FlorisTextButton( + onClick = { + languagePackExtToDelete = ext + }, + icon = painterResource(R.drawable.ic_delete), + text = stringRes(R.string.action__delete), + colors = ButtonDefaults.textButtonColors( + contentColor = MaterialTheme.colors.error, + ), + ) + Spacer(modifier = Modifier.weight(1f)) +// FlorisTextButton( +// onClick = { +// navController.navigate(Routes.Ext.Edit(ext.meta.id)) +// }, +// icon = painterResource(R.drawable.ic_edit), +// text = stringRes(R.string.action__edit), +// ) + } + } + } + } + + if (languagePackExtToDelete != null) { + FlorisConfirmDeleteDialog( + onConfirm = { + runCatching { + extensionManager.delete(languagePackExtToDelete!!) + }.onFailure { error -> + context.showLongToast( + R.string.error__snackbar_message, + "error_message" to error.localizedMessage, + ) + } + languagePackExtToDelete = null + }, + onDismiss = { languagePackExtToDelete = null }, + what = languagePackExtToDelete!!.meta.title, + ) + } + } +} diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LocalizationScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LocalizationScreen.kt index 1997dd10..1468d95a 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LocalizationScreen.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/LocalizationScreen.kt @@ -16,6 +16,8 @@ package dev.patrickgold.florisboard.app.settings.localization +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.padding import androidx.compose.material.ExtendedFloatingActionButton import androidx.compose.material.Icon @@ -30,12 +32,22 @@ import androidx.compose.ui.unit.dp import dev.patrickgold.florisboard.R import dev.patrickgold.florisboard.app.LocalNavController import dev.patrickgold.florisboard.app.Routes +import dev.patrickgold.florisboard.app.settings.advanced.Restore +import dev.patrickgold.florisboard.app.settings.theme.ThemeManagerScreenAction +import dev.patrickgold.florisboard.cacheManager import dev.patrickgold.florisboard.ime.core.DisplayLanguageNamesIn import dev.patrickgold.florisboard.ime.keyboard.LayoutType +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension +import dev.patrickgold.florisboard.ime.nlp.han.HanShapeBasedLanguageProvider import dev.patrickgold.florisboard.keyboardManager +import dev.patrickgold.florisboard.lib.android.readToFile +import dev.patrickgold.florisboard.lib.android.showLongToast import dev.patrickgold.florisboard.lib.compose.FlorisScreen import dev.patrickgold.florisboard.lib.compose.FlorisWarningCard import dev.patrickgold.florisboard.lib.compose.stringRes +import dev.patrickgold.florisboard.lib.io.ZipUtils +import dev.patrickgold.florisboard.lib.io.parentDir +import dev.patrickgold.florisboard.lib.io.subFile import dev.patrickgold.florisboard.lib.observeAsNonNullState import dev.patrickgold.florisboard.subtypeManager import dev.patrickgold.jetpref.datastore.model.observeAsState @@ -53,6 +65,7 @@ fun LocalizationScreen() = FlorisScreen { val context = LocalContext.current val keyboardManager by context.keyboardManager() val subtypeManager by context.subtypeManager() + val cacheManager by context.cacheManager() floatingActionButton { ExtendedFloatingActionButton( @@ -67,12 +80,21 @@ fun LocalizationScreen() = FlorisScreen { ) } + content { ListPreference( prefs.localization.displayLanguageNamesIn, title = stringRes(R.string.settings__localization__display_language_names_in__label), entries = DisplayLanguageNamesIn.listEntries(), ) + Preference( +// iconId = R.drawable.ic_edit, + title = stringRes(R.string.settings__localization__language_pack_title), + summary = stringRes(R.string.settings__localization__language_pack_summary), + onClick = { + navController.navigate(Routes.Settings.LanguagePackManager(LanguagePackManagerScreenAction.MANAGE)) + }, + ) PreferenceGroup(title = stringRes(R.string.settings__localization__group_subtypes__label)) { val subtypes by subtypeManager.subtypesFlow.collectAsState() if (subtypes.isEmpty()) { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SelectLocaleScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SelectLocaleScreen.kt index cf8690dd..bb860bd5 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SelectLocaleScreen.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SelectLocaleScreen.kt @@ -38,6 +38,7 @@ import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.RectangleShape +import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.painterResource import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.unit.dp @@ -64,14 +65,15 @@ fun SelectLocaleScreen() = FlorisScreen { val displayLanguageNamesIn by prefs.localization.displayLanguageNamesIn.observeAsState() var searchTermValue by remember { mutableStateOf(TextFieldValue()) } - val systemLocales = remember(displayLanguageNamesIn) { - FlorisLocale.installedSystemLocales().sortedBy { locale -> + val context = LocalContext.current + val systemLocales = + FlorisLocale.extendedAvailableLocales(context).sortedBy { locale -> when (displayLanguageNamesIn) { DisplayLanguageNamesIn.SYSTEM_LOCALE -> locale.displayName() DisplayLanguageNamesIn.NATIVE_LOCALE -> locale.displayName(locale) }.lowercase() } - } + val filteredSystemLocales = remember(searchTermValue) { if (searchTermValue.text.isBlank()) { systemLocales @@ -80,6 +82,7 @@ fun SelectLocaleScreen() = FlorisScreen { systemLocales.filter { locale -> locale.displayName().lowercase().contains(term) || locale.displayName(locale).lowercase().contains(term) || + locale.displayName(FlorisLocale.ENGLISH).lowercase().contains(term) || locale.languageTag().lowercase().startsWith(term) || locale.localeTag().lowercase().startsWith(term) } diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SubtypeEditorScreen.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SubtypeEditorScreen.kt index 9d968a52..6424f7b3 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SubtypeEditorScreen.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/app/settings/localization/SubtypeEditorScreen.kt @@ -63,6 +63,9 @@ import dev.patrickgold.florisboard.ime.core.SubtypePreset import dev.patrickgold.florisboard.ime.keyboard.LayoutArrangementComponent import dev.patrickgold.florisboard.ime.keyboard.LayoutType import dev.patrickgold.florisboard.ime.keyboard.extCorePopupMapping +import dev.patrickgold.florisboard.ime.text.key.KeyCode +import dev.patrickgold.florisboard.ime.nlp.han.HanShapeBasedLanguageProvider +import dev.patrickgold.florisboard.ime.nlp.latin.LatinLanguageProvider import dev.patrickgold.florisboard.keyboardManager import dev.patrickgold.florisboard.lib.FlorisLocale import dev.patrickgold.florisboard.lib.compose.FlorisButtonBar @@ -136,6 +139,7 @@ private class SubtypeEditorState(init: Subtype?) { primaryLocale.value = subtype.primaryLocale secondaryLocales.value = subtype.secondaryLocales composer.value = subtype.composer + nlpProviders.value = subtype.nlpProviders currencySet.value = subtype.currencySet punctuationRule.value = subtype.punctuationRule popupMapping.value = subtype.popupMapping @@ -201,6 +205,7 @@ fun SubtypeEditorScreen(id: Long?) = FlorisScreen { var currencySet by subtypeEditor.currencySet var popupMapping by subtypeEditor.popupMapping var layoutMap by subtypeEditor.layoutMap + var nlpProviders by subtypeEditor.nlpProviders var showSubtypePresetsDialog by rememberSaveable { mutableStateOf(false) } var showSelectAsError by rememberSaveable { mutableStateOf(false) } @@ -371,6 +376,38 @@ fun SubtypeEditorScreen(id: Long?) = FlorisScreen { SubtypeGroupSpacer() + SubtypeProperty(stringRes(R.string.settings__localization__subtype_suggestion_provider)) { + // TODO: Put this map somewhere more formal (another KeyboardExtension field?) + // optionally use a string resource below + val nlpProviderMappings = mapOf( + LatinLanguageProvider.ProviderId to "Latin", + HanShapeBasedLanguageProvider.ProviderId to "Chinese shape-based" + ) + + val nlpProviderMappingIds = remember(nlpProviderMappings) { + SelectListKeys + nlpProviderMappings.keys + } + val nlpProviderMappingLabels = remember(nlpProviderMappings) { + selectListValues + nlpProviderMappings.values.map { it } + } + var expanded by remember { mutableStateOf(false) } + val selectedIndex = nlpProviderMappingIds.indexOf(nlpProviders.suggestion).coerceAtLeast(0) + FlorisDropdownMenu( + items = nlpProviderMappingLabels, + expanded = expanded, + selectedIndex = selectedIndex, + isError = showSelectAsError && selectedIndex == 0, + onSelectItem = { nlpProviders = SubtypeNlpProviderMap( + suggestion = nlpProviderMappingIds[it] as String, + spelling = nlpProviderMappingIds[it] as String + ) }, + onExpandRequest = { expanded = true }, + onDismissRequest = { expanded = false }, + ) + } + + SubtypeGroupSpacer() + SubtypeProperty(stringRes(R.string.settings__localization__subtype_symbols_layout)) { val layoutType = LayoutType.SYMBOLS SubtypeLayoutDropdown( diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/core/Subtype.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/core/Subtype.kt index e162d93b..94830fe0 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/core/Subtype.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/core/Subtype.kt @@ -24,6 +24,7 @@ import dev.patrickgold.florisboard.ime.keyboard.extCoreLayout import dev.patrickgold.florisboard.ime.keyboard.extCorePopupMapping import dev.patrickgold.florisboard.ime.keyboard.extCorePunctuationRule import dev.patrickgold.florisboard.ime.nlp.latin.LatinLanguageProvider +import dev.patrickgold.florisboard.ime.nlp.han.HanShapeBasedLanguageProvider import dev.patrickgold.florisboard.lib.FlorisLocale import dev.patrickgold.florisboard.lib.ext.ExtensionComponentName import kotlinx.serialization.SerialName diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/AbstractEditorInstance.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/AbstractEditorInstance.kt index c18f9761..c60ce12d 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/AbstractEditorInstance.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/AbstractEditorInstance.kt @@ -31,6 +31,7 @@ import dev.patrickgold.florisboard.ime.text.composing.Composer import dev.patrickgold.florisboard.keyboardManager import dev.patrickgold.florisboard.lib.ext.ExtensionComponentName import dev.patrickgold.florisboard.lib.kotlin.guardedByLock +import dev.patrickgold.florisboard.nlpManager import dev.patrickgold.florisboard.subtypeManager import kotlinx.coroutines.MainScope import kotlinx.coroutines.flow.MutableStateFlow @@ -53,6 +54,7 @@ abstract class AbstractEditorInstance(context: Context) { private val keyboardManager by context.keyboardManager() private val subtypeManager by context.subtypeManager() + private val nlpManager by context.nlpManager() private val scope = MainScope() protected val breakIterators = BreakIteratorGroup() @@ -274,17 +276,7 @@ abstract class AbstractEditorInstance(context: Context) { } private suspend fun determineLocalComposing(textBeforeSelection: CharSequence): EditorRange { - return breakIterators.word(subtypeManager.activeSubtype.primaryLocale) { - it.setText(textBeforeSelection.toString()) - val end = it.last() - val isWord = it.ruleStatus != BreakIterator.WORD_NONE - if (isWord) { - val start = it.previous() - EditorRange(start, end) - } else { - EditorRange.Unspecified - } - } + return nlpManager.determineLocalComposing(textBeforeSelection, breakIterators) } private fun InputConnection.setComposingRegion(composing: EditorRange) { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/EditorInstance.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/EditorInstance.kt index cadea9fd..bd1e9728 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/EditorInstance.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/editor/EditorInstance.kt @@ -42,6 +42,7 @@ import dev.patrickgold.florisboard.lib.android.AndroidVersion import dev.patrickgold.florisboard.lib.android.showShortToast import dev.patrickgold.florisboard.lib.ext.ExtensionComponentName import dev.patrickgold.florisboard.nlpManager +import dev.patrickgold.florisboard.subtypeManager import kotlinx.coroutines.runBlocking import java.util.concurrent.atomic.AtomicInteger @@ -54,6 +55,7 @@ class EditorInstance(context: Context) : AbstractEditorInstance(context) { private val appContext by context.appContext() private val clipboardManager by context.clipboardManager() private val keyboardManager by context.keyboardManager() + private val subtypeManager by context.subtypeManager() private val nlpManager by context.nlpManager() private val activeState get() = keyboardManager.activeState @@ -512,15 +514,16 @@ class EditorInstance(context: Context) : AbstractEditorInstance(context) { } private fun PhantomSpaceState.determine(text: String, forceActive: Boolean = false): Boolean { - val content = activeContent - val selection = content.selection - if (!(isActive || forceActive) || selection.isNotValid || selection.start <= 0 || text.isEmpty()) return false - val textBefore = content.getTextBeforeCursor(1) - val punctuationRule = nlpManager.getActivePunctuationRule() - return textBefore.isNotEmpty() && - (punctuationRule.symbolsPrecedingPhantomSpace.contains(textBefore[textBefore.length - 1]) || - textBefore[textBefore.length - 1].isLetterOrDigit()) && - (punctuationRule.symbolsFollowingPhantomSpace.contains(text[0]) || text[0].isLetterOrDigit()) + val content = activeContent + val selection = content.selection + if (!(isActive || forceActive) || selection.isNotValid || selection.start <= 0 || text.isEmpty()) return false + val textBefore = content.getTextBeforeCursor(1) + val punctuationRule = nlpManager.getActivePunctuationRule() + if (!subtypeManager.activeSubtype.primaryLocale.supportsAutoSpace) return false; + return textBefore.isNotEmpty() && + (punctuationRule.symbolsPrecedingPhantomSpace.contains(textBefore[textBefore.length - 1]) || + textBefore[textBefore.length - 1].isLetterOrDigit()) && + (punctuationRule.symbolsFollowingPhantomSpace.contains(text[0]) || text[0].isLetterOrDigit()) } class AutoSpaceState { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/keyboard/KeyboardManager.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/keyboard/KeyboardManager.kt index 92047cc0..b0407c7e 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/keyboard/KeyboardManager.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/keyboard/KeyboardManager.kt @@ -502,7 +502,8 @@ class KeyboardManager(context: Context) : InputKeyEventReceiver { * enabled by the user. */ private fun handleSpace(data: KeyData) { - nlpManager.getAutoCommitCandidate()?.let { commitCandidate(it) } + val candidate = nlpManager.getAutoCommitCandidate() + candidate?.let { commitCandidate(it) } if (prefs.keyboard.spaceBarSwitchesToCharacters.get()) { when (activeState.keyboardMode) { KeyboardMode.NUMERIC_ADVANCED, @@ -523,7 +524,10 @@ class KeyboardManager(context: Context) : InputKeyEventReceiver { } } } - editorInstance.commitText(KeyCode.SPACE.toChar().toString()) + if (!subtypeManager.activeSubtype.primaryLocale.supportsAutoSpace && + candidate != null) { /* Do nothing */ } else { + editorInstance.commitText(KeyCode.SPACE.toChar().toString()) + } } /** diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/LanguagePackExtension.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/LanguagePackExtension.kt new file mode 100644 index 00000000..20ec7e39 --- /dev/null +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/LanguagePackExtension.kt @@ -0,0 +1,92 @@ +package dev.patrickgold.florisboard.ime.nlp + +import android.content.Context +import android.database.sqlite.SQLiteDatabase +import android.database.sqlite.SQLiteException +import dev.patrickgold.florisboard.appContext +import dev.patrickgold.florisboard.assetManager +import dev.patrickgold.florisboard.lib.FlorisLocale +import dev.patrickgold.florisboard.lib.android.copy +import dev.patrickgold.florisboard.lib.devtools.flogDebug +import dev.patrickgold.florisboard.lib.devtools.flogError +import dev.patrickgold.florisboard.lib.ext.Extension +import dev.patrickgold.florisboard.lib.ext.ExtensionComponent +import dev.patrickgold.florisboard.lib.ext.ExtensionComponentName +import dev.patrickgold.florisboard.lib.ext.ExtensionEditor +import dev.patrickgold.florisboard.lib.ext.ExtensionMeta +import dev.patrickgold.florisboard.lib.io.FlorisRef +import dev.patrickgold.florisboard.lib.io.FsDir +import dev.patrickgold.florisboard.lib.io.parentDir +import dev.patrickgold.florisboard.lib.io.subFile +import dev.patrickgold.florisboard.lib.kotlin.tryOrNull +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder + +@Serializable +class LanguagePackComponent( + override val id: String, + override val label: String, + override val authors: List, + val locale: FlorisLocale = FlorisLocale.fromTag(id), + val hanShapeBasedKeyCode: String = "abcdefghijklmnopqrstuvwxyz", +) : ExtensionComponent { + @Transient var parent: LanguagePackExtension? = null + + @SerialName("hanShapeBasedTable") + private val _hanShapeBasedTable: String? = null // Allows overriding the sqlite3 table to query in the json + val hanShapeBasedTable + get() = _hanShapeBasedTable ?: locale.variant +} + +@SerialName(LanguagePackExtension.SERIAL_TYPE) +@Serializable +class LanguagePackExtension( // FIXME: how to make this support multiple types of language packs, and selectively load? + override val meta: ExtensionMeta, + override val dependencies: List? = null, + val items: List = listOf(), + val hanShapeBasedSQLite: String = "han.sqlite3", +) : Extension() { + + override fun components(): List = items + + override fun edit(): ExtensionEditor { + TODO("LOL LMAO") + } + + companion object { + const val SERIAL_TYPE = "ime.extension.languagepack" + } + + override fun serialType() = SERIAL_TYPE + + @Transient var hanShapeBasedSQLiteDatabase: SQLiteDatabase = SQLiteDatabase.create(null) + + override fun onAfterLoad(context: Context, cacheDir: FsDir) { + // FIXME: this is loading language packs of all subtypes when they load. + super.onAfterLoad(context, cacheDir) + + val databasePath = workingDir?.subFile(hanShapeBasedSQLite)?.path + if (databasePath == null) { + flogError { "Han shape-based language pack not found or loaded" } + } else try { + // TODO: use lock on database? + hanShapeBasedSQLiteDatabase.takeIf { it.isOpen }?.close() + hanShapeBasedSQLiteDatabase = + SQLiteDatabase.openDatabase(databasePath, null, SQLiteDatabase.OPEN_READONLY); + } catch (e: SQLiteException) { + flogError { "SQLiteException in openDatabase: path=$databasePath, error='${e}'" } + } + } + + override fun onBeforeUnload(context: Context, cacheDir: FsDir) { + super.onBeforeUnload(context, cacheDir) + hanShapeBasedSQLiteDatabase.takeIf { it.isOpen }?.close() + } +} diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt index 558d4c8f..397e28a9 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt @@ -17,6 +17,7 @@ package dev.patrickgold.florisboard.ime.nlp import android.content.Context +import android.icu.text.BreakIterator import android.os.Build import android.os.SystemClock import android.util.LruCache @@ -32,7 +33,9 @@ import dev.patrickgold.florisboard.editorInstance import dev.patrickgold.florisboard.ime.clipboard.provider.ItemType import dev.patrickgold.florisboard.ime.core.Subtype import dev.patrickgold.florisboard.ime.editor.EditorContent +import dev.patrickgold.florisboard.ime.editor.EditorRange import dev.patrickgold.florisboard.ime.nlp.latin.LatinLanguageProvider +import dev.patrickgold.florisboard.ime.nlp.han.HanShapeBasedLanguageProvider import dev.patrickgold.florisboard.keyboardManager import dev.patrickgold.florisboard.lib.devtools.flogError import dev.patrickgold.florisboard.lib.kotlin.collectLatestIn @@ -65,6 +68,7 @@ class NlpManager(context: Context) { private val providers = guardedByLock { mapOf( LatinLanguageProvider.ProviderId to ProviderInstanceWrapper(LatinLanguageProvider(context)), + HanShapeBasedLanguageProvider.ProviderId to ProviderInstanceWrapper(HanShapeBasedLanguageProvider(context)), ) } @@ -170,6 +174,12 @@ class NlpManager(context: Context) { ) } + suspend fun determineLocalComposing(textBeforeSelection: CharSequence, breakIterators: BreakIteratorGroup): EditorRange { + return getSuggestionProvider(subtypeManager.activeSubtype).determineLocalComposing( + subtypeManager.activeSubtype, textBeforeSelection, breakIterators + ) + } + fun suggest(subtype: Subtype, content: EditorContent) { val reqTime = SystemClock.uptimeMillis() scope.launch { diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpProviders.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpProviders.kt index 81f0cea1..642a12c0 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpProviders.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpProviders.kt @@ -16,8 +16,10 @@ package dev.patrickgold.florisboard.ime.nlp +import android.icu.text.BreakIterator import dev.patrickgold.florisboard.ime.core.Subtype import dev.patrickgold.florisboard.ime.editor.EditorContent +import dev.patrickgold.florisboard.ime.editor.EditorRange /** * Base interface for any NLP provider implementation. NLP providers maintain their own internal state and only receive @@ -193,6 +195,33 @@ interface SuggestionProvider : NlpProvider { * exist, 0.0 should be returned. */ suspend fun getFrequencyForWord(subtype: Subtype, word: String): Double + + /** + * When initializing composing text given a new context, the suggestion engine determines the composing range. + * The default behavior gets the last word according to the current subtype's primaryLocale. + * @param subtype The current subtype used to determine word or character boundary. + * @param textBeforeSelection The text whose end we want to compose. + * @param breakIterators cache of BreakIterator(s) to determine boundary. + * + * @return EditorRange indicating composing range. + */ + suspend fun determineLocalComposing( + subtype: Subtype, + textBeforeSelection: CharSequence, + breakIterators: BreakIteratorGroup + ): EditorRange { + return breakIterators.word(subtype.primaryLocale) { + it.setText(textBeforeSelection.toString()) + val end = it.last() + val isWord = it.ruleStatus != BreakIterator.WORD_NONE + if (isWord) { + val start = it.previous() + EditorRange(start, end) + } else { + EditorRange.Unspecified + } + } + } } /** diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/han/HanShapeBasedLanguageProvider.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/han/HanShapeBasedLanguageProvider.kt new file mode 100644 index 00000000..140646b3 --- /dev/null +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/han/HanShapeBasedLanguageProvider.kt @@ -0,0 +1,297 @@ +/* + * Copyright (C) 2022 Patrick Goldinger + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.patrickgold.florisboard.ime.nlp.han + +import android.content.Context +import android.database.sqlite.SQLiteException +import android.icu.text.BreakIterator +import dev.patrickgold.florisboard.appContext +import dev.patrickgold.florisboard.extensionManager +import dev.patrickgold.florisboard.ime.core.Subtype +import dev.patrickgold.florisboard.ime.editor.EditorContent +import dev.patrickgold.florisboard.ime.editor.EditorRange +import dev.patrickgold.florisboard.ime.nlp.BreakIteratorGroup +import dev.patrickgold.florisboard.ime.nlp.LanguagePackComponent +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension +import dev.patrickgold.florisboard.ime.nlp.SpellingProvider +import dev.patrickgold.florisboard.ime.nlp.SpellingResult +import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate +import dev.patrickgold.florisboard.ime.nlp.SuggestionProvider +import dev.patrickgold.florisboard.ime.nlp.WordSuggestionCandidate +import dev.patrickgold.florisboard.lib.devtools.flogDebug +import dev.patrickgold.florisboard.lib.devtools.flogError +import dev.patrickgold.florisboard.subtypeManager +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext + +class HanShapeBasedLanguageProvider(val context: Context) : SpellingProvider, SuggestionProvider { + companion object { + // Default user ID used for all subtypes, unless otherwise specified. + // See `ime/core/Subtype.kt` Line 210 and 211 for the default usage + const val ProviderId = "org.florisboard.nlp.providers.han.shape" + + const val DB_PATH = "han.sqlite3"; + } + + + private val appContext by context.appContext() + + private val maxFreqBySubType = mutableMapOf(); + private val extensionManager by context.extensionManager() + private val subtypeManager by context.subtypeManager() + private val allLanguagePacks: List + // Assume other types of extensions do not extend LanguagePackExtension + get() = extensionManager.languagePacks.value ?: listOf() + private var __connectedActiveLanguagePacks: Set = setOf() // FIXME: hack for not able to observe extensionManager.languagePacks and subtypeManager.subtypes + private var languagePackItems: Map = mapOf() // init in refreshLanguagePacks() + private var keyCode: Map> = mapOf() // init in refreshLanguagePacks() + private val activeLanguagePacks // language packs referenced in subtypes + get() = buildSet { + val locales = subtypeManager.subtypes.map { it.primaryLocale.localeTag() }.toSet() + for (languagePack in allLanguagePacks) { + // FIXME: skip checking language pack type because it always is for now + if (languagePack.items.any { it.locale.localeTag() in locales }) { + add(languagePack) + } + } + } + private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) // same as NlpManager's preload() + + override val providerId = ProviderId + + init { +// // FIXME: observeForever only callable on the main thread. +// extensionManager.languagePacks.observeForever { refreshLanguagePacks() } + } + + private fun refreshLanguagePacks() { + scope.launch { create() } + } + + override suspend fun create() { + // Here we initialize our provider, set up all things which are not language dependent. + // Refresh language pack parsing + + // build index of available language packs + languagePackItems = buildMap { + for (languagePack in allLanguagePacks) { + // FIXME: skip checking language pack type because it always is for now +// if (languagePack is HanShapeBasedLanguagePackExtensionImpl) + for (languagePackItem in languagePack.items) { + put(languagePackItem.locale.localeTag(), languagePackItem) + // FIXME: how to put this in deserialization? + languagePackItem.parent = languagePack + } + } + }.toMap() + keyCode = buildMap { + languagePackItems.forEach { (tag, languagePackItem) -> + put(tag, languagePackItem.hanShapeBasedKeyCode.toSet()) + } + put("default", "abcdefghijklmnopqrstuvwxyz".toSet()) + }.toMap() + + // Load all actively used language packs. + val activeLanguagePacks = activeLanguagePacks + for (activeLanguagePack in activeLanguagePacks) { + if (!activeLanguagePack.isLoaded()) { + // populates activeLanguagePack.hanShapeBasedSQLiteDatabase + // FIXME: every time this is copied over to cache. + activeLanguagePack.load(context) + } + } + __connectedActiveLanguagePacks = activeLanguagePacks + } + + override suspend fun preload(subtype: Subtype) = withContext(Dispatchers.IO) { + // Here we have the chance to preload dictionaries and prepare a neural network for a specific language. + // Is kept in sync with the active keyboard subtype of the user, however a new preload does not necessary mean + // the previous language is not needed anymore (e.g. if the user constantly switches between two subtypes) + + // To read a file from the APK assets the following methods can be used: + // appContext.assets.open() + // appContext.assets.reader() + // appContext.assets.bufferedReader() + // appContext.assets.readText() + // To copy an APK file/dir to the file system cache (appContext.cacheDir), the following methods are available: + // appContext.assets.copy() + // appContext.assets.copyRecursively() + + // The subtype we get here contains a lot of data, however we are only interested in subtype.primaryLocale and + // subtype.secondaryLocales. + } + + override suspend fun spell( + subtype: Subtype, + word: String, + precedingWords: List, + followingWords: List, + maxSuggestionCount: Int, + allowPossiblyOffensive: Boolean, + isPrivateSession: Boolean, + ): SpellingResult { + return when (word.lowercase()) { + // Use typo for typing errors + "typo" -> SpellingResult.typo(arrayOf("typo1", "typo2", "typo3")) + // Use grammar error if the algorithm can detect this. On Android 11 and lower grammar errors are visually + // marked as typos due to a lack of support + "gerror" -> SpellingResult.grammarError(arrayOf("grammar1", "grammar2", "grammar3")) + // Use valid word for valid input + else -> SpellingResult.validWord() + } + } + + override suspend fun suggest( + subtype: Subtype, + content: EditorContent, + maxCandidateCount: Int, + allowPossiblyOffensive: Boolean, + isPrivateSession: Boolean, + ): List { + if (__connectedActiveLanguagePacks != activeLanguagePacks) { + // FIXME: hack for not able to observe extensionManager.languagePacks + refreshLanguagePacks() + } + if (content.composingText.isEmpty()) { + return emptyList(); + } + val (languagePackItem, languagePackExtension) = getLanguagePack(subtype) ?: return emptyList(); + val layout: String = languagePackItem.hanShapeBasedTable + try { + val database = languagePackExtension.hanShapeBasedSQLiteDatabase + val cur = database.query(layout, arrayOf ( "code", "text" ), "code LIKE ? || '%'", arrayOf(content.composingText), "", "", "code ASC, weight DESC", "$maxCandidateCount"); + cur.moveToFirst(); + val rowCount = cur.getCount(); + flogDebug { "Query was '${content.composingText}'" } + val suggestions = buildList { + for (n in 0 until rowCount) { + val code = cur.getString(0); + val word = cur.getString(1); + cur.moveToNext(); + add(WordSuggestionCandidate( + text = "$word", + secondaryText = code, + confidence = 0.5, + isEligibleForAutoCommit = n == 0, + // We set ourselves as the source provider so we can get notify events for our candidate + sourceProvider = this@HanShapeBasedLanguageProvider, + )) + } + } + return suggestions + } catch (e: IllegalStateException) { + flogError { "Invalid layout '${layout}' not found" } + return emptyList(); + } catch (e: SQLiteException) { + flogError { "SQLiteException: layout=$layout, composing=${content.composingText}, error='${e}'" } + return emptyList(); + } + } + + override suspend fun notifySuggestionAccepted(subtype: Subtype, candidate: SuggestionCandidate) { + // We can use flogDebug, flogInfo, flogWarning and flogError for debug logging, which is a wrapper for Logcat + flogDebug { candidate.toString() } + } + + override suspend fun notifySuggestionReverted(subtype: Subtype, candidate: SuggestionCandidate) { + flogDebug { candidate.toString() } + } + + override suspend fun removeSuggestion(subtype: Subtype, candidate: SuggestionCandidate): Boolean { + flogDebug { candidate.toString() } + return false + } + + fun getLanguagePack(subtype: Subtype): Pair? { + val languagePackItem = languagePackItems[subtype.primaryLocale.localeTag()] + val languagePackExtension = languagePackItem?.parent + if (languagePackItem == null || languagePackExtension == null) { + flogError { "Could not read language pack item / extension" } + return null; + } + return Pair(languagePackItem, languagePackExtension) + } + + override suspend fun getListOfWords(subtype: Subtype): List { + val (languagePackItem, languagePackExtension) = getLanguagePack(subtype) ?: return emptyList(); + val layout: String = languagePackItem.hanShapeBasedTable + try { + val database = languagePackExtension.hanShapeBasedSQLiteDatabase + val cur = database.query(layout, arrayOf ( "text" ), "", arrayOf(), "", "", "weight DESC, code ASC", ""); + cur.moveToFirst(); + val rowCount = cur.getCount(); + val suggestions = buildList { + for (n in 0 until rowCount) { + val word = cur.getString(0); + cur.moveToNext(); + add(word) + } + } + flogDebug { "Read ${suggestions.size} words for ${subtype.primaryLocale.localeTag()}" } + return suggestions; + } catch (e: SQLiteException) { + flogError { "Encountered an SQL error: ${e}" } + return emptyList(); + } + } + + override suspend fun getFrequencyForWord(subtype: Subtype, word: String): Double { + val (languagePackItem, languagePackExtension) = getLanguagePack(subtype) ?: return 0.0; + val layout: String = languagePackItem.hanShapeBasedTable + try { + val database = languagePackExtension.hanShapeBasedSQLiteDatabase + val cur = database.query(layout, arrayOf ( "weight" ), "code = ?", arrayOf(word), "", "", "", ""); + cur.moveToFirst(); + return try { cur.getDouble(0) } catch (e: Exception) { 0.0 }; + } catch (e: SQLiteException) { + return 0.0; + } + } + + override suspend fun destroy() { + // Here we have the chance to de-allocate memory and finish our work. However this might never be called if + // the app process is killed (which will most likely always be the case). + } + + override suspend fun determineLocalComposing(subtype: Subtype, textBeforeSelection: CharSequence, breakIterators: BreakIteratorGroup): EditorRange { + return breakIterators.character(subtype.primaryLocale) { + it.setText(textBeforeSelection.toString()) + val end = it.last() + var start = end + var next = it.previous() + val keyCodeLocale = keyCode[subtype.primaryLocale.localeTag()]?: keyCode["default"]?: emptySet() + while (next != BreakIterator.DONE) { + val sub = textBeforeSelection.substring(next, start) + if (! sub.all { char -> char in keyCodeLocale }) + break + start = next + next = it.previous() + } + if (start != end) { + flogDebug { "Determined $start - $end as composing: ${textBeforeSelection.substring(start, end)}" } + EditorRange(start, end) + } else { + flogDebug { "Determined Unspecified as composing" } + EditorRange.Unspecified + } + } + } + +} diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/lib/FlorisLocale.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/lib/FlorisLocale.kt index e5a3ace8..815325b2 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/lib/FlorisLocale.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/lib/FlorisLocale.kt @@ -16,6 +16,9 @@ package dev.patrickgold.florisboard.lib +import android.content.Context +import dev.patrickgold.florisboard.extensionManager +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension import dev.patrickgold.florisboard.lib.kotlin.titlecase import kotlinx.serialization.KSerializer import kotlinx.serialization.Serializable @@ -125,6 +128,31 @@ class FlorisLocale private constructor(val base: Locale) { * @see java.util.Locale.getAvailableLocales */ fun installedSystemLocales(): List = Locale.getAvailableLocales().map { from(it) } + + /** + * Returns a list of all installed locales and custom locales. + * + */ + fun extendedAvailableLocales(context: Context): List { + val systemLocales = installedSystemLocales() + val extensionManager by context.extensionManager() + val systemLocalesSet = buildSet { + for (locale in systemLocales) { + add(locale.localeTag()) + } + }.toSet() + val extraLocales = buildList { + for (languagePackExtension in extensionManager.languagePacks.value ?: listOf()) { + for (languagePackItem in languagePackExtension.items) { + val locale = languagePackItem.locale + if (from(locale.language, locale.country).localeTag() in systemLocalesSet) { + add(locale.localeTag()) + } + } + } + }.toSet() + return systemLocales + extraLocales.map { fromTag(it) } + } } /** @@ -195,6 +223,16 @@ class FlorisLocale private constructor(val base: Locale) { else -> true } + /** + * Returns true if suggestions in this language should have spaces added after, false otherwise. + * TODO: this is absolutely not exhaustive and hard-coded, find solution based on ICU or system + */ + val supportsAutoSpace: Boolean + get() = when (language) { + "zh", "ko", "jp", "th" -> false + else -> true + } + /** * Generates the language tag for this locale in the format `xx`, * `xx-YY` or `xx-YY-zzz` and returns it as a string. diff --git a/app/src/main/kotlin/dev/patrickgold/florisboard/lib/ext/ExtensionManager.kt b/app/src/main/kotlin/dev/patrickgold/florisboard/lib/ext/ExtensionManager.kt index fc9b725c..d2caf230 100644 --- a/app/src/main/kotlin/dev/patrickgold/florisboard/lib/ext/ExtensionManager.kt +++ b/app/src/main/kotlin/dev/patrickgold/florisboard/lib/ext/ExtensionManager.kt @@ -23,6 +23,7 @@ import androidx.lifecycle.LiveData import dev.patrickgold.florisboard.appContext import dev.patrickgold.florisboard.assetManager import dev.patrickgold.florisboard.ime.keyboard.KeyboardExtension +import dev.patrickgold.florisboard.ime.nlp.LanguagePackExtension import dev.patrickgold.florisboard.ime.text.composing.Appender import dev.patrickgold.florisboard.ime.text.composing.Composer import dev.patrickgold.florisboard.ime.text.composing.HangulUnicode @@ -62,6 +63,7 @@ val ExtensionJsonConfig = Json { polymorphic(Extension::class) { subclass(KeyboardExtension::class, KeyboardExtension.serializer()) subclass(ThemeExtension::class, ThemeExtension.serializer()) + subclass(LanguagePackExtension::class, LanguagePackExtension.serializer()) } polymorphic(Composer::class) { subclass(Appender::class, Appender.serializer()) @@ -77,6 +79,7 @@ class ExtensionManager(context: Context) { companion object { const val IME_KEYBOARD_PATH = "ime/keyboard" const val IME_THEME_PATH = "ime/theme" + const val IME_LANGUAGEPACK_PATH = "ime/languagepack" private const val FILE_OBSERVER_MASK = FileObserver.CLOSE_WRITE or FileObserver.DELETE or FileObserver.MOVED_FROM or FileObserver.MOVED_TO @@ -88,10 +91,12 @@ class ExtensionManager(context: Context) { val keyboardExtensions = ExtensionIndex(KeyboardExtension.serializer(), IME_KEYBOARD_PATH) val themes = ExtensionIndex(ThemeExtension.serializer(), IME_THEME_PATH) + val languagePacks = ExtensionIndex(LanguagePackExtension.serializer(), IME_LANGUAGEPACK_PATH) fun init() { keyboardExtensions.init() themes.init() + languagePacks.init() } fun import(ext: Extension) { @@ -100,6 +105,7 @@ class ExtensionManager(context: Context) { val relGroupPath = when (ext) { is KeyboardExtension -> IME_KEYBOARD_PATH is ThemeExtension -> IME_THEME_PATH + is LanguagePackExtension -> IME_LANGUAGEPACK_PATH else -> error("Unknown extension type") } ext.sourceRef = FlorisRef.internal(relGroupPath).subRef(extFileName) @@ -125,6 +131,7 @@ class ExtensionManager(context: Context) { fun getExtensionById(id: String): Extension? { keyboardExtensions.value?.find { it.meta.id == id }?.let { return it } themes.value?.find { it.meta.id == id }?.let { return it } + languagePacks.value?.find { it.meta.id == id }?.let { return it } return null } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 92dc9fe3..76fd3415 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -101,10 +101,13 @@ Display language names in Subtypes Add subtype + Manage installed language packs + Experimental: manage extensions that add support for specific languages (shape-based Chinese input for now) Edit subtype Primary language Popup mapping Characters layout + Suggestion engine Symbols primary layout Symbols secondary layout Composer @@ -599,6 +602,7 @@ Authors Bundled components Bundled themes + Bundled language packs This extension archive does not contain any bundled components. Description Homepage @@ -638,6 +642,7 @@ Import extension Import keyboard extension Import theme extension + Import language pack extension File can not be imported. Reason: Unsupported or unrecognized file type. Unable to replace or update default extension packages provided with the app core assets. Consider updating the app itself if you intend to use a newer version of a core extension package. diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..c4fcf3da --- /dev/null +++ b/flake.lock @@ -0,0 +1,42 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1673387755, + "narHash": "sha256-7y2wuml3bnR2jRgSpDTdeKx9HeAuGW7rYpk4G+Ibeuk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "ce1aa29621356706746c53e2d480da7c68f6c972", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..d9ea5255 --- /dev/null +++ b/flake.nix @@ -0,0 +1,81 @@ +{ + description = "florisboard build environment"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { + self, + nixpkgs, + flake-utils, + }: let + supportedSystems = with flake-utils.lib.system; [x86_64-linux x86_64-darwin aarch64-darwin]; + forAllSystems = f: nixpkgs.lib.genAttrs supportedSystems (system: f system); + android = { + versions = { + tools = "26.1.1"; + platformTools = "33.0.3"; + buildTools = "31.0.0"; + ndk = "22.1.7171670"; + cmake = "3.18.1"; + emulator = "31.3.9"; + }; + platforms = ["32"]; + }; + in + flake-utils.lib.eachSystem supportedSystems + ( + system: let + pkgs = import nixpkgs { + inherit system; + config = { + android_sdk.accept_license = true; # accept all of the sdk licenses + allowUnfree = true; # needed to get android stuff to compile + }; + }; + # Make the android enviornment we specify + android-composition = pkgs.androidenv.composeAndroidPackages { + toolsVersion = android.versions.tools; + platformToolsVersion = android.versions.platformTools; + buildToolsVersions = [android.versions.buildTools]; + platformVersions = android.platforms; + cmakeVersions = [android.versions.cmake]; + includeNDK = true; + ndkVersions = [android.versions.ndk]; + includeEmulator = true; + emulatorVersion = android.versions.emulator; + }; + android-sdk = + (pkgs.androidenv.composeAndroidPackages { + toolsVersion = android.versions.tools; + platformToolsVersion = android.versions.platformTools; + buildToolsVersions = [android.versions.buildTools]; + platformVersions = android.platforms; + cmakeVersions = [android.versions.cmake]; + includeNDK = true; + ndkVersions = [android.versions.ndk]; + }) + .androidsdk; + in rec { + devShells.default = pkgs.mkShell rec { + ANDROID_SDK_ROOT = "${android-sdk}/libexec/android-sdk"; + ANDROID_NDK_ROOT = "${ANDROID_SDK_ROOT}/ndk-bundle"; + GRADLE_OPTS = "-Dorg.gradle.project.android.aapt2FromMavenOverride=${ANDROID_SDK_ROOT}/build-tools/${android.versions.buildTools}/aapt2"; + JAVA_HOME = "${pkgs.jdk8.home}"; + nativeBulidInputs = with pkgs; [ + android-sdk + jdk8 + clang + kotlin-language-server + ]; + + # Use the same cmakeVersion here + shellHook = '' + export PATH="$(echo "$ANDROID_SDK_ROOT/cmake/${android.versions.cmake}".*/bin):$PATH" + ''; + }; + } + ); +} diff --git a/utils/convert_fcitx5_sqlite.py b/utils/convert_fcitx5_sqlite.py new file mode 100644 index 00000000..497b967b --- /dev/null +++ b/utils/convert_fcitx5_sqlite.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 + +# Execute in this folder to convert to sqlite: +# python3 convert_fcitx5_sqlite.py +# Or for a subset of tables only: +# python3 convert_fcitx5_sqlite.py cangjie-large.txt quick-classic.txt wubi-large.txt zhengma.txt +# https://github.com/fcitx/fcitx5-table-extra/tree/master/tables +# The tables are in public domain per their README. + +import os +import sys +import re +import json +import glob +import sqlite3 +import collections + + +def put_table(database, schema, table): + length, table = table['LengthReal'], table['Data'] + assert re.fullmatch('[a-zA-Z0-9_]+', schema) is not None + columns = len(table[0]) + assert all(len(x) == columns for x in table) + with sqlite3.connect(database) as con: + cur = con.cursor() + if columns == 3: + cur.execute(f'create table {schema}(code VARCHAR({length}), text TEXT, weight INT)') + cur.executemany(f'insert into {schema} values(?, ?, ?)', table) + elif columns == 4: + # hard-coded 5-long stem + length_stem = max(len(x[3]) for x in table if x[3] is not None) + cur.execute(f'create table {schema}(code VARCHAR({length}), text TEXT, weight INT, stem VARCHAR({length_stem}))') + cur.executemany(f'insert into {schema} values(?, ?, ?, ?)', table) + else: + raise ValueError(f'Number of columns ({columns}) not supported') + + +fcitx_fields_translate = { + '组词规则': 'Rule', + '数据': 'Data', + '提示': 'Prompt', + '拼音长度': 'PinyinLength', + '键码': 'KeyCode', + '拼音': 'Pinyin', + '码长': 'Length', + '构词': 'ConstructPhrase', +} + + +def parse_fcitx_table(table): + with open(table, 'rt') as f: + lines = [line.strip('\n') for line in f.readlines()] + parsed = dict() + field_now = '' + for idx, line in enumerate(lines): + if '\ufeff' in line: + line = line.replace('\ufeff', '') + if not line or line.startswith(';'): + continue + if line.startswith('[') and line.endswith(']'): + # starting a table + field_now = line[1:-1] + field_now = fcitx_fields_translate.get(field_now, field_now) + table_now = parsed[field_now] = [] + else: + if field_now: + # appending to a table + if field_now == 'Data': + # Parse first ' ' or '\t' as splitting point. + # Assume ' ' and '\t' may be in the text. + split = len(line) + for x in ' \t': + try: + split = min(split, line.index(x)) + except ValueError: + pass + if split == len(line): + print(f'Throwing away row with one column:') + print(repr(line)) + line = None + else: + line = (line[:split], line[split+1:]) + # elif field_now == 'Rule': + else: + line = line.split('=') + assert len(line) == 2 + # else: + # raise ValueError(f'Table field {field_now} not recognized') + if line is not None: + table_now.append(line) + else: + # parsing other settings + assert '=' in line, f'{table} has line without "=":\n{line}' + split = line.index('=') + field = line[:split] + field = fcitx_fields_translate.get(field, field) + parsed[field] = line[split+1:] + return parsed + + +def clean_fcitx_table(table): + # process Data with special field. + out = dict(table) + + # compute actual KeyCode used. + keycode_real = set() + for x in out['Data']: + keycode_real |= set(x[0]) + + # Prompt: just add to word list and KeyCode. + if 'Prompt' in out and out['Prompt'] in keycode_real: + out['KeyCode'] += out['Prompt'] + # Pinyin: just add to word list and KeyCode. + if 'Pinyin' in out and out['Pinyin'] in keycode_real: + out['KeyCode'] += out['Pinyin'] + # ConstructPhrase: add to "stem" column. (for zhengma_large) + if 'ConstructPhrase' in out and out['ConstructPhrase'] in keycode_real: + conchar = out['ConstructPhrase'] + # separate constructing and non-constructing parts of the table + table_noncon = [x for x in out['Data'] if conchar not in x[0]] + table_con = [(x[0][1:], x[1]) for x in out['Data'] if conchar in x[0]] + # do a join on text + dict_con = {x[1]: x[0] for x in table_con} + assert len(table_con) == len(dict_con), \ + 'ConstructPhrase entries not unique' + assert all(not conchar in x for x in dict_con.values()), \ + 'ConstructPhrase appearing after starts' + out['Data'] = [(x[0], x[1], dict_con.get(x[1], None)) + for x in table_noncon] + + # Weight: just use order. + counter = collections.Counter(x[0] for x in out['Data']) + for idx, x in enumerate(out['Data']): + weight = counter[x[0]] + counter.subtract((x[0],)) + x = x[:2] + (weight,) + x[2:] + out['Data'][idx] = x + assert not len(list(counter.elements())) + + # compute KeyCodeReal one more time after trimming table + keycode_real = set() + for x in out['Data']: + keycode_real |= set(x[0]) + out['KeyCodeReal'] = keycode_real + + # actual seek length + out['LengthReal'] = max(len(x[0]) for x in out['Data']) + return out + + +# Loading +tables = dict() +file_list = sys.argv[1:] if len(sys.argv) > 1 else glob.glob('[a-z]*.txt') +assert all(x.endswith('.txt') for x in file_list) +for x in file_list: + print(f'Processing {x}...') + schema = x[:-4].replace('-', '').replace('_', '') + tables[schema] = parse_fcitx_table(x) + conf = parse_fcitx_table(x[:-4] + '.conf.in') + conf = {k: {x[0]: x[1] for x in v} for k, v in conf.items()} + tables[schema]['.conf.in'] = conf + tables[schema]['FlorisLocale'] = f"{conf['InputMethod']['LangCode']}_{schema}" + +# Fixing +if 'wubi98_pinyin' in tables: + tables['wubi98pinyin']['KeyCode'] += 'z' + keycode = set(tables['wubi98pinyin']['KeyCode']) | set(tables['wubi98pinyin']['Pinyin']) + for idx, x in enumerate(tables['wubi98pinyin']['Data']): + if not all(ch in keycode for ch in x[0]): + x = list(x) + x[0] = ''.join(ch for ch in x[0] if ch in keycode) + tables['wubi98pinyin']['Data'][idx] = tuple(x) +if 'easylarge' in tables: + tables['easylarge']['KeyCode'] += '|' + +# Cleaning +for schema, table in tables.items(): + print(f'Cleaning {schema}, with {len(table["Data"])} items...', end='') + tables[schema] = clean_fcitx_table(table) + print(f' Done, with {len(tables[schema]["Data"])} items.') + +# Analysis +if True: + for schema, table in tables.items(): + print(f'Analyzing {schema}... LengthReal = {table["LengthReal"]}') + specials = ["Prompt", "Pinyin", "ConstructPhrase"] + for field in specials: + if field in table: + has = [x for x in table['Data'] if table[field] in x[0]] + if has: + print(f'There are {len(has)}/{len(table["Data"])} with {field}={table[field]}') + keycode = set(table['KeyCode']) + keycode_real = set(table['KeyCodeReal']) + if keycode != keycode_real: + print(f'KeyCode mismatch:') + print(f'Claimed not used: ' + ''.join(sorted(keycode - keycode_real))) + print(f'Exists unclaimed: ' + ''.join(sorted(keycode_real - keycode))) + +# Writing +language_pack = [dict(id=table['FlorisLocale'], hanShapeBasedKeyCode=table['KeyCode']) for schema, table in tables.items()] +with open('./extension-draft.json', 'wt') as f: + json.dump({'$': 'ime.extension.languagepack', 'items': sorted(language_pack, key=lambda x: x['id'])}, f, indent=2) +database = './han.sqlite3' +if os.path.exists(database): + os.remove(database) +for schema, table in tables.items(): + put_table(database, schema, table) + # put_table(database, table['FlorisLocale'], table) +print({schema: table['KeyCode'] for schema, table in tables.items()}) + +# Final display +with sqlite3.connect(database) as con: + cur = con.cursor() + # for schema in ['zh_CN_zhengmapinyin', 'zh_CN_zhengmalarge', 'zh_CN_wubilarge', 'zh_CN_wubi98', 'zh_TW_cangjie5', 'zh_HK_stroke5']: + for schema in ['zhengmapinyin', 'zhengmalarge', 'wubilarge', 'wubi98', 'cangjie5', 'stroke5']: + if schema not in tables: continue + cur.execute(f'select * from {schema} order by length(code) desc') + print(cur.fetchmany(10)) +