diff --git a/app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp b/app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp index e210d6e9..1cab5798 100644 --- a/app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp +++ b/app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp @@ -36,6 +36,7 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeDis jobject thiz, jlong native_ptr) { auto *suggestionList = reinterpret_cast(native_ptr); + suggestionList->clear(); delete suggestionList; } @@ -102,3 +103,19 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSiz auto *suggestionList = reinterpret_cast(native_ptr); return suggestionList->size(); } + +extern "C" +JNIEXPORT jboolean JNICALL +Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeGetIsPrimaryTokenAutoInsert( + JNIEnv *env, jobject thiz, jlong native_ptr) { + auto *suggestionList = reinterpret_cast(native_ptr); + return suggestionList->isPrimaryTokenAutoInsert; +} + +extern "C" +JNIEXPORT void JNICALL +Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSetIsPrimaryTokenAutoInsert( + JNIEnv *env, jobject thiz, jlong native_ptr, jboolean v) { + auto *suggestionList = reinterpret_cast(native_ptr); + suggestionList->isPrimaryTokenAutoInsert = v; +} diff --git a/app/src/main/cpp/ime/nlp/nlp.h b/app/src/main/cpp/ime/nlp/nlp.h index 34a9b259..1247f332 100644 --- a/app/src/main/cpp/ime/nlp/nlp.h +++ b/app/src/main/cpp/ime/nlp/nlp.h @@ -22,10 +22,9 @@ namespace ime::nlp { typedef std::string word_t; -typedef int16_t freq_t; +typedef uint16_t freq_t; -const freq_t FREQ_MIN = 0x00; -const freq_t FREQ_MAX = 0xFF; +const freq_t FREQ_VALUE_MASK = 0xFF; const freq_t FREQ_POSSIBLY_OFFENSIVE = 0x01; } // namespace ime::nlp diff --git a/app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp b/app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp index 461c098e..be678677 100644 --- a/app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp +++ b/app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp @@ -21,7 +21,7 @@ using namespace ime::nlp; SuggestionList::SuggestionList(size_t _maxSize) : - maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize]) + maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize]), isPrimaryTokenAutoInsert(false) { // Initialize the internal array to null pointers for (size_t n = 0; n < maxSize; n++) { @@ -66,6 +66,7 @@ void SuggestionList::clear() { internalArray[n] = nullptr; } internalSize = 0; + isPrimaryTokenAutoInsert = false; } bool SuggestionList::contains(WeightedToken &element) { diff --git a/app/src/main/cpp/ime/nlp/staged_suggestion_list.h b/app/src/main/cpp/ime/nlp/staged_suggestion_list.h index 3592ca68..60fffa4d 100644 --- a/app/src/main/cpp/ime/nlp/staged_suggestion_list.h +++ b/app/src/main/cpp/ime/nlp/staged_suggestion_list.h @@ -37,6 +37,8 @@ public: bool isEmpty() const; size_t size() const; + bool isPrimaryTokenAutoInsert; + private: WeightedToken** internalArray; size_t internalSize; diff --git a/app/src/main/cpp/ime/nlp/token.cpp b/app/src/main/cpp/ime/nlp/token.cpp index f645eb3f..ec01704f 100644 --- a/app/src/main/cpp/ime/nlp/token.cpp +++ b/app/src/main/cpp/ime/nlp/token.cpp @@ -22,36 +22,36 @@ using namespace ime::nlp; Token::Token(word_t _data) : data(std::move(_data)) {} -bool operator==(const Token &t1, const Token &t2) { +bool ime::nlp::operator==(const Token &t1, const Token &t2) { return t1.data == t2.data; } -bool operator!=(const Token &t1, const Token &t2) { +bool ime::nlp::operator!=(const Token &t1, const Token &t2) { return t1.data != t2.data; } WeightedToken::WeightedToken(word_t _data, freq_t _freq) : Token(std::move(_data)), freq(_freq) {} -bool operator==(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator==(const WeightedToken &t1, const WeightedToken &t2) { return t1.data == t2.data && t1.freq == t2.freq; } -bool operator!=(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator!=(const WeightedToken &t1, const WeightedToken &t2) { return t1.data != t2.data || t1.freq != t2.freq; } -bool operator<(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator<(const WeightedToken &t1, const WeightedToken &t2) { return t1.freq < t2.freq; } -bool operator<=(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator<=(const WeightedToken &t1, const WeightedToken &t2) { return t1.freq <= t2.freq; } -bool operator>(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator>(const WeightedToken &t1, const WeightedToken &t2) { return t1.freq > t2.freq; } -bool operator>=(const WeightedToken &t1, const WeightedToken &t2) { +bool ime::nlp::operator>=(const WeightedToken &t1, const WeightedToken &t2) { return t1.freq >= t2.freq; } diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt index c896379f..4035d849 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt @@ -17,40 +17,35 @@ package dev.patrickgold.florisboard.ime.dictionary import dev.patrickgold.florisboard.ime.extension.Asset -import dev.patrickgold.florisboard.ime.nlp.LanguageModel -import dev.patrickgold.florisboard.ime.nlp.MutableLanguageModel -import dev.patrickgold.florisboard.ime.nlp.Token -import dev.patrickgold.florisboard.ime.nlp.WeightedToken +import dev.patrickgold.florisboard.ime.nlp.SuggestionList +import dev.patrickgold.florisboard.ime.nlp.Word /** * Standardized dictionary interface for interacting with dictionaries. */ -interface Dictionary> : Asset { - val languageModel: LanguageModel - +interface Dictionary : Asset { /** * Gets token predictions based on the given [precedingTokens] and the [currentToken]. The * length of the returned list is limited to [maxSuggestionCount]. Note that the returned list * may at any time give back less items than [maxSuggestionCount] indicates. */ fun getTokenPredictions( - precedingTokens: List>, - currentToken: Token?, + precedingTokens: List, + currentToken: Word?, maxSuggestionCount: Int, - allowPossiblyOffensive: Boolean - ): List> + allowPossiblyOffensive: Boolean, + destSuggestionList: SuggestionList + ) fun getDate(): Long fun getVersion(): Int } -interface MutableDictionary> : Dictionary { - override val languageModel: MutableLanguageModel - +interface MutableDictionary : Dictionary { fun trainTokenPredictions( - precedingTokens: List>, - lastToken: Token + precedingTokens: List, + lastToken: Word ) fun setDate(date: Int) diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt index ca45e90f..a76ecee9 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt @@ -19,18 +19,27 @@ package dev.patrickgold.florisboard.ime.dictionary import android.content.Context import androidx.room.Room import dev.patrickgold.florisboard.ime.core.Preferences +import dev.patrickgold.florisboard.ime.core.Subtype import dev.patrickgold.florisboard.ime.extension.AssetRef +import dev.patrickgold.florisboard.ime.nlp.SuggestionList +import dev.patrickgold.florisboard.ime.nlp.Word +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.Dispatchers import timber.log.Timber import java.lang.ref.WeakReference +import java.util.* /** * TODO: document */ -class DictionaryManager private constructor(context: Context) { +class DictionaryManager private constructor( + context: Context, + private val defaultDispatcher: CoroutineDispatcher = Dispatchers.Default +) { private val applicationContext: WeakReference = WeakReference(context.applicationContext ?: context) private val prefs get() = Preferences.default() - private val dictionaryCache: MutableMap> = mutableMapOf() + private val dictionaryCache: MutableMap = mutableMapOf() private var florisUserDictionaryDatabase: FlorisUserDictionaryDatabase? = null private var systemUserDictionaryDatabase: SystemUserDictionaryDatabase? = null @@ -56,25 +65,54 @@ class DictionaryManager private constructor(context: Context) { } } - fun loadDictionary(ref: AssetRef): Result> { - dictionaryCache[ref.toString()]?.let { - return Result.success(it) + inline fun suggest( + currentWord: Word, + preceidingWords: List, + subtype: Subtype, + allowPossiblyOffensive: Boolean, + maxSuggestionCount: Int, + block: (suggestions: SuggestionList) -> Unit + ) { + val suggestions = SuggestionList.new(maxSuggestionCount) + queryUserDictionary(currentWord, subtype.locale, suggestions) + block(suggestions) + suggestions.dispose() + } + + fun prepareDictionaries(subtype: Subtype) { + // TODO: Implement this + } + + fun queryUserDictionary(word: Word, locale: Locale, destSuggestionList: SuggestionList) { + val florisDao = florisUserDictionaryDao() + val systemDao = systemUserDictionaryDao() + if (florisDao == null && systemDao == null) { + return } - if (ref.path.endsWith(".flict")) { - // Assume this is a Flictionary - applicationContext.get()?.let { - Flictionary.load(it, ref).onSuccess { flict -> - dictionaryCache[ref.toString()] = flict - return Result.success(flict) - }.onFailure { err -> - Timber.i(err) - return Result.failure(err) + if (prefs.dictionary.enableFlorisUserDictionary) { + florisDao?.query(word, locale)?.let { + for (entry in it) { + destSuggestionList.add(entry.word, entry.freq) + } + } + florisDao?.queryShortcut(word, locale)?.let { + for (entry in it) { + destSuggestionList.add(entry.word, entry.freq) + } + } + } + if (prefs.dictionary.enableSystemUserDictionary) { + systemDao?.query(word, locale)?.let { + for (entry in it) { + destSuggestionList.add(entry.word, entry.freq) + } + } + systemDao?.queryShortcut(word, locale)?.let { + for (entry in it) { + destSuggestionList.add(entry.word, entry.freq) } } - } else { - return Result.failure(Exception("Unable to determine supported type for given AssetRef!")) } - return Result.failure(Exception("If this message is ever thrown, something is completely broken...")) } @Synchronized diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt index 1bb586d1..a4a0bc97 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt @@ -30,15 +30,15 @@ import kotlin.jvm.Throws * This class accepts binary dictionary files of the type "flict" as defined in here: * https://github.com/florisboard/dictionary-tools/blob/main/flictionary.md */ +/** class Flictionary private constructor( override val name: String, override val label: String, override val authors: List, private val date: Long, private val version: Int, - private val headerStr: String, - override val languageModel: LanguageModel -) : Dictionary { + private val headerStr: String +) : Dictionary { companion object { private const val VERSION_0 = 0x0 @@ -427,3 +427,4 @@ fun InputStream.readNext(b: ByteArray, off: Int, len: Int): Int { } return lenRead } +*/ diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/FlorisLanguageModel.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/FlorisLanguageModel.kt deleted file mode 100644 index 7bf299b5..00000000 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/FlorisLanguageModel.kt +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (C) 2021 Patrick Goldinger - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.patrickgold.florisboard.ime.nlp - -/** - * Represents the root node to a n-gram tree. - */ -open class NgramTree( - sameOrderChildren: MutableList = mutableListOf(), - higherOrderChildren: MutableList = mutableListOf() -) : NgramNode(0, '?', -1, sameOrderChildren, higherOrderChildren) - -/** - * A node of a n-gram tree, which holds the character it represents, the corresponding frequency, - * a pre-computed string representing all parent characters and the current one as well as child - * nodes, one for the same order n-gram nodes and one for the higher order n-gram nodes. - */ -open class NgramNode( - val order: Int, - val char: Char, - val freq: Int, - val sameOrderChildren: MutableList = mutableListOf(), - val higherOrderChildren: MutableList = mutableListOf() -) { - companion object { - const val FREQ_CHARACTER = -1 - const val FREQ_WORD_MIN = 0 - const val FREQ_WORD_MAX = 255 - const val FREQ_WORD_FILLER = -2 - const val FREQ_IS_POSSIBLY_OFFENSIVE = 0 - } - - val isCharacter: Boolean - get() = freq == FREQ_CHARACTER - - val isWord: Boolean - get() = freq in FREQ_WORD_MIN..FREQ_WORD_MAX - - val isWordFiller: Boolean - get() = freq == FREQ_WORD_FILLER - - val isPossiblyOffensive: Boolean - get() = freq == FREQ_IS_POSSIBLY_OFFENSIVE - - fun findWord(word: String): NgramNode? { - var currentNode = this - for ((pos, char) in word.withIndex()) { - val childNode = if (pos == 0) { - currentNode.higherOrderChildren.find { it.char == char } - } else { - currentNode.sameOrderChildren.find { it.char == char } - } - if (childNode != null) { - currentNode = childNode - } else { - return null - } - } - return if (currentNode.isWord || currentNode.isWordFiller) { - currentNode - } else { - null - } - } - - /** - * This function allows to search for a given [input] word with a given [maxEditDistance] and - * adds all matches in the trie to the [list]. - */ - fun listSimilarWords( - input: String, - list: SuggestionList, - word: StringBuilder, - allowPossiblyOffensive: Boolean, - maxEditDistance: Int, - deletionCost: Int = 0, - insertionCost: Int = 0, - substitutionCost: Int = 0, - pos: Int = -1 - ) { - if (pos > -1) { - word.append(char) - } - val costSum = deletionCost + insertionCost + substitutionCost - if (pos > -1 && (pos + 1 == input.length) && isWord && ((isPossiblyOffensive && allowPossiblyOffensive) - || !isPossiblyOffensive)) { - // Using shift right instead of divide by 2^(costSum) as it is mathematically the - // same but faster. - list.add(word.toString(), freq shr costSum) - } - if (pos <= -1) { - for (childNode in higherOrderChildren) { - childNode.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, 0, 0, 0, 0 - ) - } - } else if (maxEditDistance == costSum) { - if (pos + 1 < input.length) { - sameOrderChildren.find { it.char == input[pos + 1] }?.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, - deletionCost, insertionCost, substitutionCost, pos + 1 - ) - } - } else { - // Delete - if (pos + 2 < input.length) { - sameOrderChildren.find { it.char == input[pos + 2] }?.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, - deletionCost + 1, insertionCost, substitutionCost, pos + 2 - ) - } - for (childNode in sameOrderChildren) { - if (pos + 1 < input.length && childNode.char == input[pos + 1]) { - childNode.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, - deletionCost, insertionCost, substitutionCost, pos + 1 - ) - } else { - // Insert - childNode.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, - deletionCost, insertionCost + 1, substitutionCost, pos - ) - if (pos + 1 < input.length) { - // Substitute - childNode.listSimilarWords( - input, list, word, allowPossiblyOffensive, maxEditDistance, - deletionCost, insertionCost, substitutionCost + 1, pos + 1 - ) - } - } - } - } - if (pos > -1) { - word.deleteAt(word.lastIndex) - } - } - - fun listAllSameOrderWords(list: SuggestionList, word: StringBuilder, allowPossiblyOffensive: Boolean) { - word.append(char) - if (isWord && ((isPossiblyOffensive && allowPossiblyOffensive) || !isPossiblyOffensive)) { - if (list.canAdd(freq)) { - list.add(word.toString(), freq) - } - } - for (childNode in sameOrderChildren) { - childNode.listAllSameOrderWords(list, word, allowPossiblyOffensive) - } - word.deleteAt(word.lastIndex) - } -} - -open class FlorisLanguageModel( - initTreeObj: NgramTree? = null -) : LanguageModel { - protected val ngramTree: NgramTree = initTreeObj ?: NgramTree() - - override fun getNgram(vararg tokens: String): Ngram { - val ngramOut = getNgramOrNull(*tokens) - if (ngramOut != null) { - return ngramOut - } else { - throw NullPointerException("No n-gram found matching the given tokens: $tokens") - } - } - - override fun getNgram(ngram: Ngram): Ngram { - val ngramOut = getNgramOrNull(ngram) - if (ngramOut != null) { - return ngramOut - } else { - throw NullPointerException("No n-gram found matching the given ngram: $ngram") - } - } - - override fun getNgramOrNull(vararg tokens: String): Ngram? { - var currentNode: NgramNode = ngramTree - for (token in tokens) { - val childNode = currentNode.findWord(token) - if (childNode != null) { - currentNode = childNode - } else { - return null - } - } - return Ngram(tokens.toList().map { Token(it) }, currentNode.freq) - } - - override fun getNgramOrNull(ngram: Ngram): Ngram? { - return getNgramOrNull(*ngram.tokens.toStringList().toTypedArray()) - } - - override fun hasNgram(ngram: Ngram, doMatchFreq: Boolean): Boolean { - val result = getNgramOrNull(ngram) - return if (result != null) { - if (doMatchFreq) { - ngram.freq == result.freq - } else { - true - } - } else { - false - } - } - - override fun matchAllNgrams( - ngram: Ngram, - maxEditDistance: Int, - maxTokenCount: Int, - allowPossiblyOffensive: Boolean - ): List> { - val ngramList = mutableListOf>() - var currentNode: NgramNode = ngramTree - for ((t, token) in ngram.tokens.withIndex()) { - val word = token.data - if (t + 1 >= ngram.tokens.size) { - if (word.isNotEmpty()) { - // The last word is not complete, so find all possible words and sort - val splitWord = mutableListOf() - var splitNode: NgramNode? = currentNode - for ((pos, char) in word.withIndex()) { - val node = if (pos == 0) { - splitNode?.higherOrderChildren?.find { it.char == char } - } else { - splitNode?.sameOrderChildren?.find { it.char == char } - } - splitWord.add(char) - splitNode = node - if (node == null) { - break - } - } - if (splitNode != null) { - // Input thus far is valid - val wordNodes = SuggestionList.new(maxTokenCount) - val strBuilder = StringBuilder().append(word.substring(0, word.length - 1)) - splitNode.listAllSameOrderWords(wordNodes, strBuilder, allowPossiblyOffensive) - ngramList.addAll(wordNodes) - } - if (ngramList.size < maxTokenCount) { - val wordNodes = SuggestionList.new(maxTokenCount) - val strBuilder = StringBuilder() - currentNode.listSimilarWords(word, wordNodes, strBuilder, allowPossiblyOffensive, maxEditDistance) - ngramList.addAll(wordNodes) - } - } - } else { - val node = currentNode.findWord(word) - if (node == null) { - return ngramList - } else { - currentNode = node - } - } - } - return ngramList - } - - fun toFlorisMutableLanguageModel(): FlorisMutableLanguageModel = FlorisMutableLanguageModel(ngramTree) -} - -open class FlorisMutableLanguageModel( - initTreeObj: NgramTree? = null -) : MutableLanguageModel, FlorisLanguageModel(initTreeObj) { - override fun deleteNgram(ngram: Ngram) { - TODO("Not yet implemented") - } - - override fun insertNgram(ngram: Ngram) { - TODO("Not yet implemented") - } - - override fun updateNgram(ngram: Ngram) { - TODO("Not yet implemented") - } - - fun toFlorisLanguageModel(): FlorisLanguageModel = FlorisLanguageModel(ngramTree) -} diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/LanguageModel.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/LanguageModel.kt deleted file mode 100644 index 0f9a053c..00000000 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/LanguageModel.kt +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2021 Patrick Goldinger - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.patrickgold.florisboard.ime.nlp - -/** - * Abstract interface for a language model. Can house any n-grams with a minimum order of one. - */ -interface LanguageModel> { - /** - * Tries to get the n-gram for the passed [tokens]. Throws a NPE if no match could be found. - */ - @Throws(NullPointerException::class) - fun getNgram(vararg tokens: T): Ngram - - /** - * Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while - * searching. Throws a NPE if no match could be found. - */ - @Throws(NullPointerException::class) - fun getNgram(ngram: Ngram): Ngram - - /** - * Tries to get the n-gram for the passed [tokens]. Returns null if no match could be found. - */ - fun getNgramOrNull(vararg tokens: T): Ngram? - - /** - * Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while - * searching. Returns null if no match could be found. - */ - fun getNgramOrNull(ngram: Ngram): Ngram? - - /** - * Checks if a given [ngram] exists within this model. If [doMatchFreq] is set to true, the - * frequency is also matched. - */ - fun hasNgram(ngram: Ngram, doMatchFreq: Boolean = false): Boolean - - /** - * Matches all n-grams which match the given [ngram], whereas the last item in the n-gram is - * is used to search for predictions. - */ - fun matchAllNgrams( - ngram: Ngram, - maxEditDistance: Int, - maxTokenCount: Int, - allowPossiblyOffensive: Boolean - ): List> -} - -/** - * Mutable version of [LanguageModel]. - */ -interface MutableLanguageModel> : LanguageModel { - fun deleteNgram(ngram: Ngram) - - fun insertNgram(ngram: Ngram) - - fun updateNgram(ngram: Ngram) -} diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt index 7a4f8bf2..237302bd 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt @@ -36,6 +36,8 @@ value class SuggestionList private constructor( external fun nativeClear(nativePtr: NativePtr) external fun nativeContains(nativePtr: NativePtr, element: Word): Boolean external fun nativeGetOrNull(nativePtr: NativePtr, index: Int): Word? + external fun nativeGetIsPrimaryTokenAutoInsert(nativePtr: NativePtr): Boolean + external fun nativeSetIsPrimaryTokenAutoInsert(nativePtr: NativePtr, v: Boolean) external fun nativeSize(nativePtr: NativePtr): Int } @@ -75,6 +77,9 @@ value class SuggestionList private constructor( override fun isEmpty(): Boolean = size <= 0 + val isPrimaryTokenAutoInsert: Boolean + get() = nativeGetIsPrimaryTokenAutoInsert(_nativePtr) + override fun iterator(): Iterator { return SuggestionListIterator(this) } diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/TextProcessor.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/TextProcessor.kt deleted file mode 100644 index 28d665ac..00000000 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/nlp/TextProcessor.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2021 Patrick Goldinger - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package dev.patrickgold.florisboard.ime.nlp - -class TextProcessor { - data class Word( - val word: String, - val isPossiblyOffensive: Boolean = false - ) -} diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt index 06e0862c..1703fa0b 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt @@ -27,13 +27,10 @@ import dev.patrickgold.florisboard.R import dev.patrickgold.florisboard.debug.* import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem import dev.patrickgold.florisboard.ime.core.* -import dev.patrickgold.florisboard.ime.dictionary.Dictionary import dev.patrickgold.florisboard.ime.dictionary.DictionaryManager import dev.patrickgold.florisboard.ime.extension.AssetManager import dev.patrickgold.florisboard.ime.extension.AssetRef import dev.patrickgold.florisboard.ime.extension.AssetSource -import dev.patrickgold.florisboard.ime.nlp.Token -import dev.patrickgold.florisboard.ime.nlp.toStringList import dev.patrickgold.florisboard.ime.text.gestures.GlideTypingManager import dev.patrickgold.florisboard.ime.text.gestures.SwipeAction import dev.patrickgold.florisboard.ime.text.key.* @@ -42,7 +39,6 @@ import dev.patrickgold.florisboard.ime.text.layout.LayoutManager import dev.patrickgold.florisboard.ime.text.smartbar.SmartbarView import kotlinx.coroutines.* import org.json.JSONArray -import java.util.* import kotlin.math.roundToLong /** @@ -74,8 +70,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In lateinit var textKeyboardIconSet: TextKeyboardIconSet private set private var textViewGroup: LinearLayout? = null - private val dictionaryManager: DictionaryManager = DictionaryManager.default() - private var activeDictionary: Dictionary? = null + private val dictionaryManager: DictionaryManager get() = DictionaryManager.default() val inputEventDispatcher: InputEventDispatcher = InputEventDispatcher.new( repeatableKeyCodes = intArrayOf( KeyCode.ARROW_DOWN, @@ -420,11 +415,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In override fun onSubtypeChanged(newSubtype: Subtype) { launch { if (activeEditorInstance.isComposingEnabled) { - withContext(Dispatchers.IO) { - dictionaryManager.loadDictionary(AssetRef(AssetSource.Assets, "ime/dict/en.flict")).let { - activeDictionary = it.getOrDefault(null) - } - } + dictionaryManager.prepareDictionaries(newSubtype) } if (prefs.glide.enabled) { GlideTypingManager.getInstance().setWordData(newSubtype) @@ -447,28 +438,24 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In if (activeEditorInstance.isComposingEnabled && !inputEventDispatcher.isPressed(KeyCode.DELETE) && !isGlidePostEffect) { if (activeEditorInstance.shouldReevaluateComposingSuggestions) { activeEditorInstance.shouldReevaluateComposingSuggestions = false - activeDictionary?.let { - launch(Dispatchers.Default) { - val startTime = System.nanoTime() - val suggestions = queryUserDictionary( - activeEditorInstance.cachedInput.currentWord.text, - florisboard.activeSubtype.locale - ).toMutableList() - suggestions.addAll(it.getTokenPredictions( - precedingTokens = listOf(), - currentToken = Token(activeEditorInstance.cachedInput.currentWord.text), - maxSuggestionCount = 16, - allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive - ).toStringList()) - if (BuildConfig.DEBUG) { - val elapsed = (System.nanoTime() - startTime) / 1000.0 - flogInfo { "sugg fetch time: $elapsed us" } - } + launch(Dispatchers.Default) { + val startTime = System.nanoTime() + dictionaryManager.suggest( + currentWord = activeEditorInstance.cachedInput.currentWord.text, + preceidingWords = listOf(), + subtype = florisboard.activeSubtype, + allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive, + maxSuggestionCount = 16 + ) { suggestions -> withContext(Dispatchers.Main) { smartbarView?.setCandidateSuggestionWords(startTime, suggestions) smartbarView?.updateCandidateSuggestionCapsState() } } + if (BuildConfig.DEBUG) { + val elapsed = (System.nanoTime() - startTime) / 1000.0 + flogInfo { "sugg fetch time: $elapsed us" } + } } } else { smartbarView?.setCandidateSuggestionWords(System.nanoTime(), null) @@ -480,40 +467,6 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In smartbarView?.onPrimaryClipChanged() } - private fun queryUserDictionary(word: String, locale: Locale): Set { - val florisDao = dictionaryManager.florisUserDictionaryDao() - val systemDao = dictionaryManager.systemUserDictionaryDao() - if (florisDao == null && systemDao == null) { - return setOf() - } - val retList = mutableSetOf() - if (prefs.dictionary.enableFlorisUserDictionary) { - florisDao?.query(word, locale)?.let { - for (entry in it) { - retList.add(entry.word) - } - } - florisDao?.queryShortcut(word, locale)?.let { - for (entry in it) { - retList.add(entry.word) - } - } - } - if (prefs.dictionary.enableSystemUserDictionary) { - systemDao?.query(word, locale)?.let { - for (entry in it) { - retList.add(entry.word) - } - } - systemDao?.queryShortcut(word, locale)?.let { - for (entry in it) { - retList.add(entry.word) - } - } - } - return retList - } - /** * Updates the current caps state according to the [EditorInstance.cursorCapsMode], while * respecting [capsLock] property and the correction.autoCapitalization preference. diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt index 4175ae55..70dbabe4 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt @@ -105,10 +105,12 @@ class GlideTypingManager : GlideTypingGesture.Listener, CoroutineScope by MainSc textInputManager.isGlidePostEffect = true textInputManager.smartbarView?.setCandidateSuggestionWords( time, - suggestions.subList( + // FIXME + /*suggestions.subList( 1.coerceAtMost(min(commit.compareTo(false), suggestions.size)), maxSuggestionsToShow.coerceAtMost(suggestions.size) - ).map { textInputManager.fixCase(it) } + ).map { textInputManager.fixCase(it) }*/ + null ) textInputManager.smartbarView?.updateCandidateSuggestionCapsState() if (commit && suggestions.isNotEmpty()) { diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt index d99b0e87..c078d33e 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt @@ -36,6 +36,7 @@ import androidx.core.graphics.ColorUtils import dev.patrickgold.florisboard.R import dev.patrickgold.florisboard.ime.clip.FlorisClipboardManager import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem +import dev.patrickgold.florisboard.ime.nlp.SuggestionList import dev.patrickgold.florisboard.ime.theme.Theme import dev.patrickgold.florisboard.ime.theme.ThemeManager import dev.patrickgold.florisboard.ime.theme.ThemeValue @@ -100,7 +101,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener { themeManager = ThemeManager.defaultOrNull() themeManager?.registerOnThemeUpdatedListener(this) florisClipboardManager = FlorisClipboardManager.getInstanceOrNull() - updateCandidates(candidates) + recomputeCandidates() } override fun onDetachedFromWindow() { @@ -113,7 +114,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener { velocityTracker = null } - fun updateCandidates(newCandidates: List?) { + fun updateCandidates(newCandidates: SuggestionList?) { candidates.clear() if (newCandidates != null) { candidates.addAll(newCandidates) diff --git a/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt b/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt index 3df02009..8e9e7897 100644 --- a/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt +++ b/app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt @@ -28,6 +28,7 @@ import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem import dev.patrickgold.florisboard.ime.core.FlorisBoard import dev.patrickgold.florisboard.ime.core.Preferences import dev.patrickgold.florisboard.ime.core.Subtype +import dev.patrickgold.florisboard.ime.nlp.SuggestionList import dev.patrickgold.florisboard.ime.text.key.KeyVariation import dev.patrickgold.florisboard.ime.text.keyboard.KeyboardMode import dev.patrickgold.florisboard.ime.theme.Theme @@ -275,7 +276,7 @@ class SmartbarView : ConstraintLayout, ThemeManager.OnThemeUpdatedListener { } } - fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: List?) { + fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: SuggestionList?) { if (suggestionInitDate > lastSuggestionInitDate) { lastSuggestionInitDate = suggestionInitDate binding.candidates.updateCandidates(suggestions)