Finalize base implementation for SuggestionList

app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp

@@ -36,6 +36,7 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeDis
         jobject thiz,
         jlong native_ptr) {
     auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
+    suggestionList->clear();
     delete suggestionList;
 }
 
@@ -102,3 +103,19 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSiz
     auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
     return suggestionList->size();
 }
+
+extern "C"
+JNIEXPORT jboolean JNICALL
+Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeGetIsPrimaryTokenAutoInsert(
+        JNIEnv *env, jobject thiz, jlong native_ptr) {
+    auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
+    return suggestionList->isPrimaryTokenAutoInsert;
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSetIsPrimaryTokenAutoInsert(
+        JNIEnv *env, jobject thiz, jlong native_ptr, jboolean v) {
+    auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
+    suggestionList->isPrimaryTokenAutoInsert = v;
+}

app/src/main/cpp/ime/nlp/nlp.h

@@ -22,10 +22,9 @@
 namespace ime::nlp {
 
 typedef std::string word_t;
-typedef int16_t freq_t;
+typedef uint16_t freq_t;
 
-const freq_t FREQ_MIN =                 0x00;
+const freq_t FREQ_VALUE_MASK =          0xFF;
-const freq_t FREQ_MAX =                 0xFF;
 const freq_t FREQ_POSSIBLY_OFFENSIVE =  0x01;
 
 } // namespace ime::nlp

app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp

@@ -21,7 +21,7 @@
 using namespace ime::nlp;
 
 SuggestionList::SuggestionList(size_t _maxSize) :
-    maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize])
+    maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize]), isPrimaryTokenAutoInsert(false)
 {
     // Initialize the internal array to null pointers
     for (size_t n = 0; n < maxSize; n++) {
@@ -66,6 +66,7 @@ void SuggestionList::clear() {
         internalArray[n] = nullptr;
     }
     internalSize = 0;
+    isPrimaryTokenAutoInsert = false;
 }
 
 bool SuggestionList::contains(WeightedToken &element) {

app/src/main/cpp/ime/nlp/staged_suggestion_list.h

@@ -37,6 +37,8 @@ public:
     bool isEmpty() const;
     size_t size() const;
 
+    bool isPrimaryTokenAutoInsert;
+
 private:
     WeightedToken** internalArray;
     size_t internalSize;

app/src/main/cpp/ime/nlp/token.cpp

@@ -22,36 +22,36 @@ using namespace ime::nlp;
 
 Token::Token(word_t _data) : data(std::move(_data)) {}
 
-bool operator==(const Token &t1, const Token &t2) {
+bool ime::nlp::operator==(const Token &t1, const Token &t2) {
     return t1.data == t2.data;
 }
 
-bool operator!=(const Token &t1, const Token &t2) {
+bool ime::nlp::operator!=(const Token &t1, const Token &t2) {
     return t1.data != t2.data;
 }
 
 WeightedToken::WeightedToken(word_t _data, freq_t _freq) : Token(std::move(_data)), freq(_freq) {}
 
-bool operator==(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator==(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.data == t2.data && t1.freq == t2.freq;
 }
 
-bool operator!=(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator!=(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.data != t2.data || t1.freq != t2.freq;
 }
 
-bool operator<(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator<(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.freq < t2.freq;
 }
 
-bool operator<=(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator<=(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.freq <= t2.freq;
 }
 
-bool operator>(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator>(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.freq > t2.freq;
 }
 
-bool operator>=(const WeightedToken &t1, const WeightedToken &t2) {
+bool ime::nlp::operator>=(const WeightedToken &t1, const WeightedToken &t2) {
     return t1.freq >= t2.freq;
 }

app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt

@@ -17,40 +17,35 @@
 package dev.patrickgold.florisboard.ime.dictionary
 
 import dev.patrickgold.florisboard.ime.extension.Asset
-import dev.patrickgold.florisboard.ime.nlp.LanguageModel
+import dev.patrickgold.florisboard.ime.nlp.SuggestionList
-import dev.patrickgold.florisboard.ime.nlp.MutableLanguageModel
+import dev.patrickgold.florisboard.ime.nlp.Word
-import dev.patrickgold.florisboard.ime.nlp.Token
-import dev.patrickgold.florisboard.ime.nlp.WeightedToken
 
 /**
  * Standardized dictionary interface for interacting with dictionaries.
  */
-interface Dictionary<T : Any, F : Comparable<F>> : Asset {
+interface Dictionary : Asset {
-    val languageModel: LanguageModel<T, F>
-
     /**
      * Gets token predictions based on the given [precedingTokens] and the [currentToken]. The
      * length of the returned list is limited to [maxSuggestionCount]. Note that the returned list
      * may at any time give back less items than [maxSuggestionCount] indicates.
      */
     fun getTokenPredictions(
-        precedingTokens: List<Token<T>>,
+        precedingTokens: List<Word>,
-        currentToken: Token<T>?,
+        currentToken: Word?,
         maxSuggestionCount: Int,
-        allowPossiblyOffensive: Boolean
+        allowPossiblyOffensive: Boolean,
-    ): List<WeightedToken<T, F>>
+        destSuggestionList: SuggestionList
+    )
 
     fun getDate(): Long
 
     fun getVersion(): Int
 }
 
-interface MutableDictionary<T : Any, F : Comparable<F>> : Dictionary<T, F> {
+interface MutableDictionary : Dictionary {
-    override val languageModel: MutableLanguageModel<T, F>
-
     fun trainTokenPredictions(
-        precedingTokens: List<Token<T>>,
+        precedingTokens: List<Word>,
-        lastToken: Token<T>
+        lastToken: Word
     )
 
     fun setDate(date: Int)

app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt

@@ -19,18 +19,27 @@ package dev.patrickgold.florisboard.ime.dictionary
 import android.content.Context
 import androidx.room.Room
 import dev.patrickgold.florisboard.ime.core.Preferences
+import dev.patrickgold.florisboard.ime.core.Subtype
 import dev.patrickgold.florisboard.ime.extension.AssetRef
+import dev.patrickgold.florisboard.ime.nlp.SuggestionList
+import dev.patrickgold.florisboard.ime.nlp.Word
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.Dispatchers
 import timber.log.Timber
 import java.lang.ref.WeakReference
+import java.util.*
 
 /**
  * TODO: document
  */
-class DictionaryManager private constructor(context: Context) {
+class DictionaryManager private constructor(
+    context: Context,
+    private val defaultDispatcher: CoroutineDispatcher = Dispatchers.Default
+) {
     private val applicationContext: WeakReference<Context> = WeakReference(context.applicationContext ?: context)
     private val prefs get() = Preferences.default()
 
-    private val dictionaryCache: MutableMap<String, Dictionary<String, Int>> = mutableMapOf()
+    private val dictionaryCache: MutableMap<String, Dictionary> = mutableMapOf()
 
     private var florisUserDictionaryDatabase: FlorisUserDictionaryDatabase? = null
     private var systemUserDictionaryDatabase: SystemUserDictionaryDatabase? = null
@@ -56,25 +65,54 @@ class DictionaryManager private constructor(context: Context) {
         }
     }
 
-    fun loadDictionary(ref: AssetRef): Result<Dictionary<String, Int>> {
+    inline fun suggest(
-        dictionaryCache[ref.toString()]?.let {
+        currentWord: Word,
-            return Result.success(it)
+        preceidingWords: List<Word>,
+        subtype: Subtype,
+        allowPossiblyOffensive: Boolean,
+        maxSuggestionCount: Int,
+        block: (suggestions: SuggestionList) -> Unit
+    ) {
+        val suggestions = SuggestionList.new(maxSuggestionCount)
+        queryUserDictionary(currentWord, subtype.locale, suggestions)
+        block(suggestions)
+        suggestions.dispose()
     }
-        if (ref.path.endsWith(".flict")) {
+
-            // Assume this is a Flictionary
+    fun prepareDictionaries(subtype: Subtype) {
-            applicationContext.get()?.let {
+        // TODO: Implement this
-                Flictionary.load(it, ref).onSuccess { flict ->
+    }
-                    dictionaryCache[ref.toString()] = flict
+
-                    return Result.success(flict)
+    fun queryUserDictionary(word: Word, locale: Locale, destSuggestionList: SuggestionList) {
-                }.onFailure { err ->
+        val florisDao = florisUserDictionaryDao()
-                    Timber.i(err)
+        val systemDao = systemUserDictionaryDao()
-                    return Result.failure(err)
+        if (florisDao == null && systemDao == null) {
+            return
+        }
+        if (prefs.dictionary.enableFlorisUserDictionary) {
+            florisDao?.query(word, locale)?.let {
+                for (entry in it) {
+                    destSuggestionList.add(entry.word, entry.freq)
+                }
+            }
+            florisDao?.queryShortcut(word, locale)?.let {
+                for (entry in it) {
+                    destSuggestionList.add(entry.word, entry.freq)
+                }
+            }
+        }
+        if (prefs.dictionary.enableSystemUserDictionary) {
+            systemDao?.query(word, locale)?.let {
+                for (entry in it) {
+                    destSuggestionList.add(entry.word, entry.freq)
+                }
+            }
+            systemDao?.queryShortcut(word, locale)?.let {
+                for (entry in it) {
+                    destSuggestionList.add(entry.word, entry.freq)
                 }
             }
-        } else {
-            return Result.failure(Exception("Unable to determine supported type for given AssetRef!"))
         }
-        return Result.failure(Exception("If this message is ever thrown, something is completely broken..."))
     }
 
     @Synchronized

app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt

@@ -30,15 +30,15 @@ import kotlin.jvm.Throws
  * This class accepts binary dictionary files of the type "flict" as defined in here:
  * https://github.com/florisboard/dictionary-tools/blob/main/flictionary.md
  */
+/**
 class Flictionary private constructor(
     override val name: String,
     override val label: String,
     override val authors: List<String>,
     private val date: Long,
     private val version: Int,
-    private val headerStr: String,
+    private val headerStr: String
-    override val languageModel: LanguageModel<String, Int>
+) : Dictionary {
-) : Dictionary<String, Int> {
     companion object {
         private const val VERSION_0 =                           0x0
 
@@ -427,3 +427,4 @@ fun InputStream.readNext(b: ByteArray, off: Int, len: Int): Int {
     }
     return lenRead
 }
+*/

app/src/main/java/dev/patrickgold/florisboard/ime/nlp/FlorisLanguageModel.kt

@@ -1,292 +0,0 @@
-/*
- * Copyright (C) 2021 Patrick Goldinger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package dev.patrickgold.florisboard.ime.nlp
-
-/**
- * Represents the root node to a n-gram tree.
- */
-open class NgramTree(
-    sameOrderChildren: MutableList<NgramNode> = mutableListOf(),
-    higherOrderChildren: MutableList<NgramNode> = mutableListOf()
-) : NgramNode(0, '?', -1, sameOrderChildren, higherOrderChildren)
-
-/**
- * A node of a n-gram tree, which holds the character it represents, the corresponding frequency,
- * a pre-computed string representing all parent characters and the current one as well as child
- * nodes, one for the same order n-gram nodes and one for the higher order n-gram nodes.
- */
-open class NgramNode(
-    val order: Int,
-    val char: Char,
-    val freq: Int,
-    val sameOrderChildren: MutableList<NgramNode> = mutableListOf(),
-    val higherOrderChildren: MutableList<NgramNode> = mutableListOf()
-) {
-    companion object {
-        const val FREQ_CHARACTER = -1
-        const val FREQ_WORD_MIN = 0
-        const val FREQ_WORD_MAX = 255
-        const val FREQ_WORD_FILLER = -2
-        const val FREQ_IS_POSSIBLY_OFFENSIVE = 0
-    }
-
-    val isCharacter: Boolean
-        get() = freq == FREQ_CHARACTER
-
-    val isWord: Boolean
-        get() = freq in FREQ_WORD_MIN..FREQ_WORD_MAX
-
-    val isWordFiller: Boolean
-        get() = freq == FREQ_WORD_FILLER
-
-    val isPossiblyOffensive: Boolean
-        get() = freq == FREQ_IS_POSSIBLY_OFFENSIVE
-
-    fun findWord(word: String): NgramNode? {
-        var currentNode = this
-        for ((pos, char) in word.withIndex()) {
-            val childNode = if (pos == 0) {
-                currentNode.higherOrderChildren.find { it.char == char }
-            } else {
-                currentNode.sameOrderChildren.find { it.char == char }
-            }
-            if (childNode != null) {
-                currentNode = childNode
-            } else {
-                return null
-            }
-        }
-        return if (currentNode.isWord || currentNode.isWordFiller) {
-            currentNode
-        } else {
-            null
-        }
-    }
-
-    /**
-     * This function allows to search for a given [input] word with a given [maxEditDistance] and
-     * adds all matches in the trie to the [list].
-     */
-    fun listSimilarWords(
-        input: String,
-        list: SuggestionList,
-        word: StringBuilder,
-        allowPossiblyOffensive: Boolean,
-        maxEditDistance: Int,
-        deletionCost: Int = 0,
-        insertionCost: Int = 0,
-        substitutionCost: Int = 0,
-        pos: Int = -1
-    ) {
-        if (pos > -1) {
-            word.append(char)
-        }
-        val costSum = deletionCost + insertionCost + substitutionCost
-        if (pos > -1 && (pos + 1 == input.length) && isWord && ((isPossiblyOffensive && allowPossiblyOffensive)
-            || !isPossiblyOffensive)) {
-            // Using shift right instead of divide by 2^(costSum) as it is mathematically the
-            // same but faster.
-            list.add(word.toString(), freq shr costSum)
-        }
-        if (pos <= -1) {
-            for (childNode in higherOrderChildren) {
-                childNode.listSimilarWords(
-                    input, list, word, allowPossiblyOffensive, maxEditDistance, 0, 0, 0, 0
-                )
-            }
-        } else if (maxEditDistance == costSum) {
-            if (pos + 1 < input.length) {
-                sameOrderChildren.find { it.char == input[pos + 1] }?.listSimilarWords(
-                    input, list, word, allowPossiblyOffensive, maxEditDistance,
-                    deletionCost, insertionCost, substitutionCost, pos + 1
-                )
-            }
-        } else {
-            // Delete
-            if (pos + 2 < input.length) {
-                sameOrderChildren.find { it.char == input[pos + 2] }?.listSimilarWords(
-                    input, list, word, allowPossiblyOffensive, maxEditDistance,
-                    deletionCost + 1, insertionCost, substitutionCost, pos + 2
-                )
-            }
-            for (childNode in sameOrderChildren) {
-                if (pos + 1 < input.length && childNode.char == input[pos + 1]) {
-                    childNode.listSimilarWords(
-                        input, list, word, allowPossiblyOffensive, maxEditDistance,
-                        deletionCost, insertionCost, substitutionCost, pos + 1
-                    )
-                } else  {
-                    // Insert
-                    childNode.listSimilarWords(
-                        input, list, word, allowPossiblyOffensive, maxEditDistance,
-                        deletionCost, insertionCost + 1, substitutionCost, pos
-                    )
-                    if (pos + 1 < input.length) {
-                        // Substitute
-                        childNode.listSimilarWords(
-                            input, list, word, allowPossiblyOffensive, maxEditDistance,
-                            deletionCost, insertionCost, substitutionCost + 1, pos + 1
-                        )
-                    }
-                }
-            }
-        }
-        if (pos > -1) {
-            word.deleteAt(word.lastIndex)
-        }
-    }
-
-    fun listAllSameOrderWords(list: SuggestionList, word: StringBuilder, allowPossiblyOffensive: Boolean) {
-        word.append(char)
-        if (isWord && ((isPossiblyOffensive && allowPossiblyOffensive) || !isPossiblyOffensive)) {
-            if (list.canAdd(freq)) {
-                list.add(word.toString(), freq)
-            }
-        }
-        for (childNode in sameOrderChildren) {
-            childNode.listAllSameOrderWords(list, word, allowPossiblyOffensive)
-        }
-        word.deleteAt(word.lastIndex)
-    }
-}
-
-open class FlorisLanguageModel(
-    initTreeObj: NgramTree? = null
-) : LanguageModel<String, Int> {
-    protected val ngramTree: NgramTree = initTreeObj ?: NgramTree()
-
-    override fun getNgram(vararg tokens: String): Ngram<String, Int> {
-        val ngramOut = getNgramOrNull(*tokens)
-        if (ngramOut != null) {
-            return ngramOut
-        } else {
-            throw NullPointerException("No n-gram found matching the given tokens: $tokens")
-        }
-    }
-
-    override fun getNgram(ngram: Ngram<String, Int>): Ngram<String, Int> {
-        val ngramOut = getNgramOrNull(ngram)
-        if (ngramOut != null) {
-            return ngramOut
-        } else {
-            throw NullPointerException("No n-gram found matching the given ngram: $ngram")
-        }
-    }
-
-    override fun getNgramOrNull(vararg tokens: String): Ngram<String, Int>? {
-        var currentNode: NgramNode = ngramTree
-        for (token in tokens) {
-            val childNode = currentNode.findWord(token)
-            if (childNode != null) {
-                currentNode = childNode
-            } else {
-                return null
-            }
-        }
-        return Ngram(tokens.toList().map { Token(it) }, currentNode.freq)
-    }
-
-    override fun getNgramOrNull(ngram: Ngram<String, Int>): Ngram<String, Int>? {
-        return getNgramOrNull(*ngram.tokens.toStringList().toTypedArray())
-    }
-
-    override fun hasNgram(ngram: Ngram<String, Int>, doMatchFreq: Boolean): Boolean {
-        val result = getNgramOrNull(ngram)
-        return if (result != null) {
-            if (doMatchFreq) {
-                ngram.freq == result.freq
-            } else {
-                true
-            }
-        } else {
-            false
-        }
-    }
-
-    override fun matchAllNgrams(
-        ngram: Ngram<String, Int>,
-        maxEditDistance: Int,
-        maxTokenCount: Int,
-        allowPossiblyOffensive: Boolean
-    ): List<WeightedToken<String, Int>> {
-        val ngramList = mutableListOf<WeightedToken<String, Int>>()
-        var currentNode: NgramNode = ngramTree
-        for ((t, token) in ngram.tokens.withIndex()) {
-            val word = token.data
-            if (t + 1 >= ngram.tokens.size) {
-                if (word.isNotEmpty()) {
-                    // The last word is not complete, so find all possible words and sort
-                    val splitWord = mutableListOf<Char>()
-                    var splitNode: NgramNode? = currentNode
-                    for ((pos, char) in word.withIndex()) {
-                        val node = if (pos == 0) {
-                            splitNode?.higherOrderChildren?.find { it.char == char }
-                        } else {
-                            splitNode?.sameOrderChildren?.find { it.char == char }
-                        }
-                        splitWord.add(char)
-                        splitNode = node
-                        if (node == null) {
-                            break
-                        }
-                    }
-                    if (splitNode != null) {
-                        // Input thus far is valid
-                        val wordNodes = SuggestionList.new(maxTokenCount)
-                        val strBuilder = StringBuilder().append(word.substring(0, word.length - 1))
-                        splitNode.listAllSameOrderWords(wordNodes, strBuilder, allowPossiblyOffensive)
-                        ngramList.addAll(wordNodes)
-                    }
-                    if (ngramList.size < maxTokenCount) {
-                        val wordNodes = SuggestionList.new(maxTokenCount)
-                        val strBuilder = StringBuilder()
-                        currentNode.listSimilarWords(word, wordNodes, strBuilder, allowPossiblyOffensive, maxEditDistance)
-                        ngramList.addAll(wordNodes)
-                    }
-                }
-            } else {
-                val node = currentNode.findWord(word)
-                if (node == null) {
-                    return ngramList
-                } else {
-                    currentNode = node
-                }
-            }
-        }
-        return ngramList
-    }
-
-    fun toFlorisMutableLanguageModel(): FlorisMutableLanguageModel = FlorisMutableLanguageModel(ngramTree)
-}
-
-open class FlorisMutableLanguageModel(
-    initTreeObj: NgramTree? = null
-) : MutableLanguageModel<String, Int>, FlorisLanguageModel(initTreeObj) {
-    override fun deleteNgram(ngram: Ngram<String, Int>) {
-        TODO("Not yet implemented")
-    }
-
-    override fun insertNgram(ngram: Ngram<String, Int>) {
-        TODO("Not yet implemented")
-    }
-
-    override fun updateNgram(ngram: Ngram<String, Int>) {
-        TODO("Not yet implemented")
-    }
-
-    fun toFlorisLanguageModel(): FlorisLanguageModel = FlorisLanguageModel(ngramTree)
-}

app/src/main/java/dev/patrickgold/florisboard/ime/nlp/LanguageModel.kt

@@ -1,74 +0,0 @@
-/*
- * Copyright (C) 2021 Patrick Goldinger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package dev.patrickgold.florisboard.ime.nlp
-
-/**
- * Abstract interface for a language model. Can house any n-grams with a minimum order of one.
- */
-interface LanguageModel<T : Any, F : Comparable<F>> {
-    /**
-     * Tries to get the n-gram for the passed [tokens]. Throws a NPE if no match could be found.
-     */
-    @Throws(NullPointerException::class)
-    fun getNgram(vararg tokens: T): Ngram<T, F>
-
-    /**
-     * Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while
-     * searching. Throws a NPE if no match could be found.
-     */
-    @Throws(NullPointerException::class)
-    fun getNgram(ngram: Ngram<T, F>): Ngram<T, F>
-
-    /**
-     * Tries to get the n-gram for the passed [tokens]. Returns null if no match could be found.
-     */
-    fun getNgramOrNull(vararg tokens: T): Ngram<T, F>?
-
-    /**
-     * Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while
-     * searching. Returns null if no match could be found.
-     */
-    fun getNgramOrNull(ngram: Ngram<T, F>): Ngram<T, F>?
-
-    /**
-     * Checks if a given [ngram] exists within this model. If [doMatchFreq] is set to true, the
-     * frequency is also matched.
-     */
-    fun hasNgram(ngram: Ngram<T, F>, doMatchFreq: Boolean = false): Boolean
-
-    /**
-     * Matches all n-grams which match the given [ngram], whereas the last item in the n-gram is
-     * is used to search for predictions.
-     */
-    fun matchAllNgrams(
-        ngram: Ngram<T, F>,
-        maxEditDistance: Int,
-        maxTokenCount: Int,
-        allowPossiblyOffensive: Boolean
-    ): List<WeightedToken<T, F>>
-}
-
-/**
- * Mutable version of [LanguageModel].
- */
-interface MutableLanguageModel<T : Any, F : Comparable<F>> : LanguageModel<T, F> {
-    fun deleteNgram(ngram: Ngram<T, F>)
-
-    fun insertNgram(ngram: Ngram<T, F>)
-
-    fun updateNgram(ngram: Ngram<T, F>)
-}

app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt

@@ -36,6 +36,8 @@ value class SuggestionList private constructor(
         external fun nativeClear(nativePtr: NativePtr)
         external fun nativeContains(nativePtr: NativePtr, element: Word): Boolean
         external fun nativeGetOrNull(nativePtr: NativePtr, index: Int): Word?
+        external fun nativeGetIsPrimaryTokenAutoInsert(nativePtr: NativePtr): Boolean
+        external fun nativeSetIsPrimaryTokenAutoInsert(nativePtr: NativePtr, v: Boolean)
         external fun nativeSize(nativePtr: NativePtr): Int
     }
 
@@ -75,6 +77,9 @@ value class SuggestionList private constructor(
 
     override fun isEmpty(): Boolean = size <= 0
 
+    val isPrimaryTokenAutoInsert: Boolean
+        get() = nativeGetIsPrimaryTokenAutoInsert(_nativePtr)
+
     override fun iterator(): Iterator<Word> {
         return SuggestionListIterator(this)
     }

app/src/main/java/dev/patrickgold/florisboard/ime/nlp/TextProcessor.kt

@@ -1,24 +0,0 @@
-/*
- * Copyright (C) 2021 Patrick Goldinger
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package dev.patrickgold.florisboard.ime.nlp
-
-class TextProcessor {
-    data class Word(
-        val word: String,
-        val isPossiblyOffensive: Boolean = false
-    )
-}

app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt

@@ -27,13 +27,10 @@ import dev.patrickgold.florisboard.R
 import dev.patrickgold.florisboard.debug.*
 import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
 import dev.patrickgold.florisboard.ime.core.*
-import dev.patrickgold.florisboard.ime.dictionary.Dictionary
 import dev.patrickgold.florisboard.ime.dictionary.DictionaryManager
 import dev.patrickgold.florisboard.ime.extension.AssetManager
 import dev.patrickgold.florisboard.ime.extension.AssetRef
 import dev.patrickgold.florisboard.ime.extension.AssetSource
-import dev.patrickgold.florisboard.ime.nlp.Token
-import dev.patrickgold.florisboard.ime.nlp.toStringList
 import dev.patrickgold.florisboard.ime.text.gestures.GlideTypingManager
 import dev.patrickgold.florisboard.ime.text.gestures.SwipeAction
 import dev.patrickgold.florisboard.ime.text.key.*
@@ -42,7 +39,6 @@ import dev.patrickgold.florisboard.ime.text.layout.LayoutManager
 import dev.patrickgold.florisboard.ime.text.smartbar.SmartbarView
 import kotlinx.coroutines.*
 import org.json.JSONArray
-import java.util.*
 import kotlin.math.roundToLong
 
 /**
@@ -74,8 +70,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
     lateinit var textKeyboardIconSet: TextKeyboardIconSet
         private set
     private var textViewGroup: LinearLayout? = null
-    private val dictionaryManager: DictionaryManager = DictionaryManager.default()
+    private val dictionaryManager: DictionaryManager get() = DictionaryManager.default()
-    private var activeDictionary: Dictionary<String, Int>? = null
     val inputEventDispatcher: InputEventDispatcher = InputEventDispatcher.new(
         repeatableKeyCodes = intArrayOf(
             KeyCode.ARROW_DOWN,
@@ -420,11 +415,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
     override fun onSubtypeChanged(newSubtype: Subtype) {
         launch {
             if (activeEditorInstance.isComposingEnabled) {
-                withContext(Dispatchers.IO) {
+                dictionaryManager.prepareDictionaries(newSubtype)
-                    dictionaryManager.loadDictionary(AssetRef(AssetSource.Assets, "ime/dict/en.flict")).let {
-                        activeDictionary = it.getOrDefault(null)
-                    }
-                }
             }
             if (prefs.glide.enabled) {
                 GlideTypingManager.getInstance().setWordData(newSubtype)
@@ -447,28 +438,24 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
         if (activeEditorInstance.isComposingEnabled && !inputEventDispatcher.isPressed(KeyCode.DELETE) && !isGlidePostEffect) {
             if (activeEditorInstance.shouldReevaluateComposingSuggestions) {
                 activeEditorInstance.shouldReevaluateComposingSuggestions = false
-                activeDictionary?.let {
                 launch(Dispatchers.Default) {
                     val startTime = System.nanoTime()
-                        val suggestions = queryUserDictionary(
+                    dictionaryManager.suggest(
-                            activeEditorInstance.cachedInput.currentWord.text,
+                        currentWord = activeEditorInstance.cachedInput.currentWord.text,
-                            florisboard.activeSubtype.locale
+                        preceidingWords = listOf(),
-                        ).toMutableList()
+                        subtype = florisboard.activeSubtype,
-                        suggestions.addAll(it.getTokenPredictions(
+                        allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive,
-                            precedingTokens = listOf(),
+                        maxSuggestionCount = 16
-                            currentToken = Token(activeEditorInstance.cachedInput.currentWord.text),
+                    ) { suggestions ->
-                            maxSuggestionCount = 16,
-                            allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive
-                        ).toStringList())
-                        if (BuildConfig.DEBUG) {
-                            val elapsed = (System.nanoTime() - startTime) / 1000.0
-                            flogInfo { "sugg fetch time: $elapsed us" }
-                        }
                         withContext(Dispatchers.Main) {
                             smartbarView?.setCandidateSuggestionWords(startTime, suggestions)
                             smartbarView?.updateCandidateSuggestionCapsState()
                         }
                     }
+                    if (BuildConfig.DEBUG) {
+                        val elapsed = (System.nanoTime() - startTime) / 1000.0
+                        flogInfo { "sugg fetch time: $elapsed us" }
+                    }
                 }
             } else {
                 smartbarView?.setCandidateSuggestionWords(System.nanoTime(), null)
@@ -480,40 +467,6 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
         smartbarView?.onPrimaryClipChanged()
     }
 
-    private fun queryUserDictionary(word: String, locale: Locale): Set<String> {
-        val florisDao = dictionaryManager.florisUserDictionaryDao()
-        val systemDao = dictionaryManager.systemUserDictionaryDao()
-        if (florisDao == null && systemDao == null) {
-            return setOf()
-        }
-        val retList = mutableSetOf<String>()
-        if (prefs.dictionary.enableFlorisUserDictionary) {
-            florisDao?.query(word, locale)?.let {
-                for (entry in it) {
-                    retList.add(entry.word)
-                }
-            }
-            florisDao?.queryShortcut(word, locale)?.let {
-                for (entry in it) {
-                    retList.add(entry.word)
-                }
-            }
-        }
-        if (prefs.dictionary.enableSystemUserDictionary) {
-            systemDao?.query(word, locale)?.let {
-                for (entry in it) {
-                    retList.add(entry.word)
-                }
-            }
-            systemDao?.queryShortcut(word, locale)?.let {
-                for (entry in it) {
-                    retList.add(entry.word)
-                }
-            }
-        }
-        return retList
-    }
-
     /**
      * Updates the current caps state according to the [EditorInstance.cursorCapsMode], while
      * respecting [capsLock] property and the correction.autoCapitalization preference.

app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt

@@ -105,10 +105,12 @@ class GlideTypingManager : GlideTypingGesture.Listener, CoroutineScope by MainSc
                 textInputManager.isGlidePostEffect = true
                 textInputManager.smartbarView?.setCandidateSuggestionWords(
                     time,
-                    suggestions.subList(
+                    // FIXME
+                    /*suggestions.subList(
                         1.coerceAtMost(min(commit.compareTo(false), suggestions.size)),
                         maxSuggestionsToShow.coerceAtMost(suggestions.size)
-                    ).map { textInputManager.fixCase(it) }
+                    ).map { textInputManager.fixCase(it) }*/
+                    null
                 )
                 textInputManager.smartbarView?.updateCandidateSuggestionCapsState()
                 if (commit && suggestions.isNotEmpty()) {

app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt

@@ -36,6 +36,7 @@ import androidx.core.graphics.ColorUtils
 import dev.patrickgold.florisboard.R
 import dev.patrickgold.florisboard.ime.clip.FlorisClipboardManager
 import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
+import dev.patrickgold.florisboard.ime.nlp.SuggestionList
 import dev.patrickgold.florisboard.ime.theme.Theme
 import dev.patrickgold.florisboard.ime.theme.ThemeManager
 import dev.patrickgold.florisboard.ime.theme.ThemeValue
@@ -100,7 +101,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener {
         themeManager = ThemeManager.defaultOrNull()
         themeManager?.registerOnThemeUpdatedListener(this)
         florisClipboardManager = FlorisClipboardManager.getInstanceOrNull()
-        updateCandidates(candidates)
+        recomputeCandidates()
     }
 
     override fun onDetachedFromWindow() {
@@ -113,7 +114,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener {
         velocityTracker = null
     }
 
-    fun updateCandidates(newCandidates: List<String>?) {
+    fun updateCandidates(newCandidates: SuggestionList?) {
         candidates.clear()
         if (newCandidates != null) {
             candidates.addAll(newCandidates)

app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt

@@ -28,6 +28,7 @@ import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
 import dev.patrickgold.florisboard.ime.core.FlorisBoard
 import dev.patrickgold.florisboard.ime.core.Preferences
 import dev.patrickgold.florisboard.ime.core.Subtype
+import dev.patrickgold.florisboard.ime.nlp.SuggestionList
 import dev.patrickgold.florisboard.ime.text.key.KeyVariation
 import dev.patrickgold.florisboard.ime.text.keyboard.KeyboardMode
 import dev.patrickgold.florisboard.ime.theme.Theme
@@ -275,7 +276,7 @@ class SmartbarView : ConstraintLayout, ThemeManager.OnThemeUpdatedListener {
         }
     }
 
-    fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: List<String>?) {
+    fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: SuggestionList?) {
         if (suggestionInitDate > lastSuggestionInitDate) {
             lastSuggestionInitDate = suggestionInitDate
             binding.candidates.updateCandidates(suggestions)