mirror/florisboard
0
0
Fork 0
mirror of https://github.com/florisboard/florisboard.git synced 2024-09-19 11:32:18 +02:00
Code

Finalize base implementation for SuggestionList

Browse Source
This commit is contained in:
Patrick Goldinger 2021-05-12 19:29:21 +02:00
parent b5b89fde4f
commit bcad0af35e
16 changed files with 130 additions and 505 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
app/src/main/cpp/dev_patrickgold_florisboard_ime_nlp_SuggestionList.cpp
View File

@ -36,6 +36,7 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeDis
jobject thiz,
jlong native_ptr) {
auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
suggestionList->clear();
delete suggestionList;
}
@ -102,3 +103,19 @@ Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSiz
auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
return suggestionList->size();
}
extern "C"
JNIEXPORT jboolean JNICALL
Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeGetIsPrimaryTokenAutoInsert(
JNIEnv *env, jobject thiz, jlong native_ptr) {
auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
return suggestionList->isPrimaryTokenAutoInsert;
}
extern "C"
JNIEXPORT void JNICALL
Java_dev_patrickgold_florisboard_ime_nlp_SuggestionList_00024Companion_nativeSetIsPrimaryTokenAutoInsert(
JNIEnv *env, jobject thiz, jlong native_ptr, jboolean v) {
auto *suggestionList = reinterpret_cast<SuggestionList *>(native_ptr);
suggestionList->isPrimaryTokenAutoInsert = v;
}

5
app/src/main/cpp/ime/nlp/nlp.h
View File

@ -22,10 +22,9 @@
namespace ime::nlp {
typedef std::string word_t;
typedef int16_t freq_t;
typedef uint16_t freq_t;
const freq_t FREQ_MIN = 0x00;
const freq_t FREQ_MAX = 0xFF;
const freq_t FREQ_VALUE_MASK = 0xFF;
const freq_t FREQ_POSSIBLY_OFFENSIVE = 0x01;
} // namespace ime::nlp

3
app/src/main/cpp/ime/nlp/staged_suggestion_list.cpp
View File

@ -21,7 +21,7 @@
using namespace ime::nlp;
SuggestionList::SuggestionList(size_t _maxSize) :
maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize])
maxSize(_maxSize), internalSize(0), internalArray(new WeightedToken*[_maxSize]), isPrimaryTokenAutoInsert(false)
{
// Initialize the internal array to null pointers
for (size_t n = 0; n < maxSize; n++) {
@ -66,6 +66,7 @@ void SuggestionList::clear() {
internalArray[n] = nullptr;
}
internalSize = 0;
isPrimaryTokenAutoInsert = false;
}
bool SuggestionList::contains(WeightedToken &element) {

2
app/src/main/cpp/ime/nlp/staged_suggestion_list.h
View File

@ -37,6 +37,8 @@ public:
bool isEmpty() const;
size_t size() const;
bool isPrimaryTokenAutoInsert;
private:
WeightedToken** internalArray;
size_t internalSize;

16
app/src/main/cpp/ime/nlp/token.cpp
View File

@ -22,36 +22,36 @@ using namespace ime::nlp;
Token::Token(word_t _data) : data(std::move(_data)) {}
bool operator==(const Token &t1, const Token &t2) {
bool ime::nlp::operator==(const Token &t1, const Token &t2) {
return t1.data == t2.data;
}
bool operator!=(const Token &t1, const Token &t2) {
bool ime::nlp::operator!=(const Token &t1, const Token &t2) {
return t1.data != t2.data;
}
WeightedToken::WeightedToken(word_t _data, freq_t _freq) : Token(std::move(_data)), freq(_freq) {}
bool operator==(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator==(const WeightedToken &t1, const WeightedToken &t2) {
return t1.data == t2.data && t1.freq == t2.freq;
}
bool operator!=(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator!=(const WeightedToken &t1, const WeightedToken &t2) {
return t1.data != t2.data || t1.freq != t2.freq;
}
bool operator<(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator<(const WeightedToken &t1, const WeightedToken &t2) {
return t1.freq < t2.freq;
}
bool operator<=(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator<=(const WeightedToken &t1, const WeightedToken &t2) {
return t1.freq <= t2.freq;
}
bool operator>(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator>(const WeightedToken &t1, const WeightedToken &t2) {
return t1.freq > t2.freq;
}
bool operator>=(const WeightedToken &t1, const WeightedToken &t2) {
bool ime::nlp::operator>=(const WeightedToken &t1, const WeightedToken &t2) {
return t1.freq >= t2.freq;
}

27
app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Dictionary.kt
View File

@ -17,40 +17,35 @@
package dev.patrickgold.florisboard.ime.dictionary
import dev.patrickgold.florisboard.ime.extension.Asset
import dev.patrickgold.florisboard.ime.nlp.LanguageModel
import dev.patrickgold.florisboard.ime.nlp.MutableLanguageModel
import dev.patrickgold.florisboard.ime.nlp.Token
import dev.patrickgold.florisboard.ime.nlp.WeightedToken
import dev.patrickgold.florisboard.ime.nlp.SuggestionList
import dev.patrickgold.florisboard.ime.nlp.Word
/**
* Standardized dictionary interface for interacting with dictionaries.
*/
interface Dictionary<T : Any, F : Comparable<F>> : Asset {
val languageModel: LanguageModel<T, F>
interface Dictionary : Asset {
/**
* Gets token predictions based on the given [precedingTokens] and the [currentToken]. The
* length of the returned list is limited to [maxSuggestionCount]. Note that the returned list
* may at any time give back less items than [maxSuggestionCount] indicates.
*/
fun getTokenPredictions(
precedingTokens: List<Token<T>>,
currentToken: Token<T>?,
precedingTokens: List<Word>,
currentToken: Word?,
maxSuggestionCount: Int,
allowPossiblyOffensive: Boolean
): List<WeightedToken<T, F>>
allowPossiblyOffensive: Boolean,
destSuggestionList: SuggestionList
)
fun getDate(): Long
fun getVersion(): Int
}
interface MutableDictionary<T : Any, F : Comparable<F>> : Dictionary<T, F> {
override val languageModel: MutableLanguageModel<T, F>
interface MutableDictionary : Dictionary {
fun trainTokenPredictions(
precedingTokens: List<Token<T>>,
lastToken: Token<T>
precedingTokens: List<Word>,
lastToken: Word
)
fun setDate(date: Int)

72
app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt
View File

@ -19,18 +19,27 @@ package dev.patrickgold.florisboard.ime.dictionary
import android.content.Context
import androidx.room.Room
import dev.patrickgold.florisboard.ime.core.Preferences
import dev.patrickgold.florisboard.ime.core.Subtype
import dev.patrickgold.florisboard.ime.extension.AssetRef
import dev.patrickgold.florisboard.ime.nlp.SuggestionList
import dev.patrickgold.florisboard.ime.nlp.Word
import kotlinx.coroutines.CoroutineDispatcher
import kotlinx.coroutines.Dispatchers
import timber.log.Timber
import java.lang.ref.WeakReference
import java.util.*
/**
* TODO: document
*/
class DictionaryManager private constructor(context: Context) {
class DictionaryManager private constructor(
context: Context,
private val defaultDispatcher: CoroutineDispatcher = Dispatchers.Default
) {
private val applicationContext: WeakReference<Context> = WeakReference(context.applicationContext ?: context)
private val prefs get() = Preferences.default()
private val dictionaryCache: MutableMap<String, Dictionary<String, Int>> = mutableMapOf()
private val dictionaryCache: MutableMap<String, Dictionary> = mutableMapOf()
private var florisUserDictionaryDatabase: FlorisUserDictionaryDatabase? = null
private var systemUserDictionaryDatabase: SystemUserDictionaryDatabase? = null
@ -56,25 +65,54 @@ class DictionaryManager private constructor(context: Context) {
}
}
fun loadDictionary(ref: AssetRef): Result<Dictionary<String, Int>> {
dictionaryCache[ref.toString()]?.let {
return Result.success(it)
inline fun suggest(
currentWord: Word,
preceidingWords: List<Word>,
subtype: Subtype,
allowPossiblyOffensive: Boolean,
maxSuggestionCount: Int,
block: (suggestions: SuggestionList) -> Unit
) {
val suggestions = SuggestionList.new(maxSuggestionCount)
queryUserDictionary(currentWord, subtype.locale, suggestions)
block(suggestions)
suggestions.dispose()
}
fun prepareDictionaries(subtype: Subtype) {
// TODO: Implement this
}
fun queryUserDictionary(word: Word, locale: Locale, destSuggestionList: SuggestionList) {
val florisDao = florisUserDictionaryDao()
val systemDao = systemUserDictionaryDao()
if (florisDao == null && systemDao == null) {
return
}
if (ref.path.endsWith(".flict")) {
// Assume this is a Flictionary
applicationContext.get()?.let {
Flictionary.load(it, ref).onSuccess { flict ->
dictionaryCache[ref.toString()] = flict
return Result.success(flict)
}.onFailure { err ->
Timber.i(err)
return Result.failure(err)
if (prefs.dictionary.enableFlorisUserDictionary) {
florisDao?.query(word, locale)?.let {
for (entry in it) {
destSuggestionList.add(entry.word, entry.freq)
}
}
florisDao?.queryShortcut(word, locale)?.let {
for (entry in it) {
destSuggestionList.add(entry.word, entry.freq)
}
}
}
if (prefs.dictionary.enableSystemUserDictionary) {
systemDao?.query(word, locale)?.let {
for (entry in it) {
destSuggestionList.add(entry.word, entry.freq)
}
}
systemDao?.queryShortcut(word, locale)?.let {
for (entry in it) {
destSuggestionList.add(entry.word, entry.freq)
}
}
} else {
return Result.failure(Exception("Unable to determine supported type for given AssetRef!"))
}
return Result.failure(Exception("If this message is ever thrown, something is completely broken..."))
}
@Synchronized

7
app/src/main/java/dev/patrickgold/florisboard/ime/dictionary/Flictionary.kt
View File

@ -30,15 +30,15 @@ import kotlin.jvm.Throws
* This class accepts binary dictionary files of the type "flict" as defined in here:
* https://github.com/florisboard/dictionary-tools/blob/main/flictionary.md
*/
/**
class Flictionary private constructor(
override val name: String,
override val label: String,
override val authors: List<String>,
private val date: Long,
private val version: Int,
private val headerStr: String,
override val languageModel: LanguageModel<String, Int>
) : Dictionary<String, Int> {
private val headerStr: String
) : Dictionary {
companion object {
private const val VERSION_0 = 0x0
@ -427,3 +427,4 @@ fun InputStream.readNext(b: ByteArray, off: Int, len: Int): Int {
}
return lenRead
}
*/

292
app/src/main/java/dev/patrickgold/florisboard/ime/nlp/FlorisLanguageModel.kt
View File

@ -1,292 +0,0 @@
/*
* Copyright (C) 2021 Patrick Goldinger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dev.patrickgold.florisboard.ime.nlp
/**
* Represents the root node to a n-gram tree.
*/
open class NgramTree(
sameOrderChildren: MutableList<NgramNode> = mutableListOf(),
higherOrderChildren: MutableList<NgramNode> = mutableListOf()
) : NgramNode(0, '?', -1, sameOrderChildren, higherOrderChildren)
/**
* A node of a n-gram tree, which holds the character it represents, the corresponding frequency,
* a pre-computed string representing all parent characters and the current one as well as child
* nodes, one for the same order n-gram nodes and one for the higher order n-gram nodes.
*/
open class NgramNode(
val order: Int,
val char: Char,
val freq: Int,
val sameOrderChildren: MutableList<NgramNode> = mutableListOf(),
val higherOrderChildren: MutableList<NgramNode> = mutableListOf()
) {
companion object {
const val FREQ_CHARACTER = -1
const val FREQ_WORD_MIN = 0
const val FREQ_WORD_MAX = 255
const val FREQ_WORD_FILLER = -2
const val FREQ_IS_POSSIBLY_OFFENSIVE = 0
}
val isCharacter: Boolean
get() = freq == FREQ_CHARACTER
val isWord: Boolean
get() = freq in FREQ_WORD_MIN..FREQ_WORD_MAX
val isWordFiller: Boolean
get() = freq == FREQ_WORD_FILLER
val isPossiblyOffensive: Boolean
get() = freq == FREQ_IS_POSSIBLY_OFFENSIVE
fun findWord(word: String): NgramNode? {
var currentNode = this
for ((pos, char) in word.withIndex()) {
val childNode = if (pos == 0) {
currentNode.higherOrderChildren.find { it.char == char }
} else {
currentNode.sameOrderChildren.find { it.char == char }
}
if (childNode != null) {
currentNode = childNode
} else {
return null
}
}
return if (currentNode.isWord || currentNode.isWordFiller) {
currentNode
} else {
null
}
}
/**
* This function allows to search for a given [input] word with a given [maxEditDistance] and
* adds all matches in the trie to the [list].
*/
fun listSimilarWords(
input: String,
list: SuggestionList,
word: StringBuilder,
allowPossiblyOffensive: Boolean,
maxEditDistance: Int,
deletionCost: Int = 0,
insertionCost: Int = 0,
substitutionCost: Int = 0,
pos: Int = -1
) {
if (pos > -1) {
word.append(char)
}
val costSum = deletionCost + insertionCost + substitutionCost
if (pos > -1 && (pos + 1 == input.length) && isWord && ((isPossiblyOffensive && allowPossiblyOffensive)
|| !isPossiblyOffensive)) {
// Using shift right instead of divide by 2^(costSum) as it is mathematically the
// same but faster.
list.add(word.toString(), freq shr costSum)
}
if (pos <= -1) {
for (childNode in higherOrderChildren) {
childNode.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance, 0, 0, 0, 0
)
}
} else if (maxEditDistance == costSum) {
if (pos + 1 < input.length) {
sameOrderChildren.find { it.char == input[pos + 1] }?.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance,
deletionCost, insertionCost, substitutionCost, pos + 1
)
}
} else {
// Delete
if (pos + 2 < input.length) {
sameOrderChildren.find { it.char == input[pos + 2] }?.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance,
deletionCost + 1, insertionCost, substitutionCost, pos + 2
)
}
for (childNode in sameOrderChildren) {
if (pos + 1 < input.length && childNode.char == input[pos + 1]) {
childNode.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance,
deletionCost, insertionCost, substitutionCost, pos + 1
)
} else {
// Insert
childNode.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance,
deletionCost, insertionCost + 1, substitutionCost, pos
)
if (pos + 1 < input.length) {
// Substitute
childNode.listSimilarWords(
input, list, word, allowPossiblyOffensive, maxEditDistance,
deletionCost, insertionCost, substitutionCost + 1, pos + 1
)
}
}
}
}
if (pos > -1) {
word.deleteAt(word.lastIndex)
}
}
fun listAllSameOrderWords(list: SuggestionList, word: StringBuilder, allowPossiblyOffensive: Boolean) {
word.append(char)
if (isWord && ((isPossiblyOffensive && allowPossiblyOffensive) || !isPossiblyOffensive)) {
if (list.canAdd(freq)) {
list.add(word.toString(), freq)
}
}
for (childNode in sameOrderChildren) {
childNode.listAllSameOrderWords(list, word, allowPossiblyOffensive)
}
word.deleteAt(word.lastIndex)
}
}
open class FlorisLanguageModel(
initTreeObj: NgramTree? = null
) : LanguageModel<String, Int> {
protected val ngramTree: NgramTree = initTreeObj ?: NgramTree()
override fun getNgram(vararg tokens: String): Ngram<String, Int> {
val ngramOut = getNgramOrNull(*tokens)
if (ngramOut != null) {
return ngramOut
} else {
throw NullPointerException("No n-gram found matching the given tokens: $tokens")
}
}
override fun getNgram(ngram: Ngram<String, Int>): Ngram<String, Int> {
val ngramOut = getNgramOrNull(ngram)
if (ngramOut != null) {
return ngramOut
} else {
throw NullPointerException("No n-gram found matching the given ngram: $ngram")
}
}
override fun getNgramOrNull(vararg tokens: String): Ngram<String, Int>? {
var currentNode: NgramNode = ngramTree
for (token in tokens) {
val childNode = currentNode.findWord(token)
if (childNode != null) {
currentNode = childNode
} else {
return null
}
}
return Ngram(tokens.toList().map { Token(it) }, currentNode.freq)
}
override fun getNgramOrNull(ngram: Ngram<String, Int>): Ngram<String, Int>? {
return getNgramOrNull(*ngram.tokens.toStringList().toTypedArray())
}
override fun hasNgram(ngram: Ngram<String, Int>, doMatchFreq: Boolean): Boolean {
val result = getNgramOrNull(ngram)
return if (result != null) {
if (doMatchFreq) {
ngram.freq == result.freq
} else {
true
}
} else {
false
}
}
override fun matchAllNgrams(
ngram: Ngram<String, Int>,
maxEditDistance: Int,
maxTokenCount: Int,
allowPossiblyOffensive: Boolean
): List<WeightedToken<String, Int>> {
val ngramList = mutableListOf<WeightedToken<String, Int>>()
var currentNode: NgramNode = ngramTree
for ((t, token) in ngram.tokens.withIndex()) {
val word = token.data
if (t + 1 >= ngram.tokens.size) {
if (word.isNotEmpty()) {
// The last word is not complete, so find all possible words and sort
val splitWord = mutableListOf<Char>()
var splitNode: NgramNode? = currentNode
for ((pos, char) in word.withIndex()) {
val node = if (pos == 0) {
splitNode?.higherOrderChildren?.find { it.char == char }
} else {
splitNode?.sameOrderChildren?.find { it.char == char }
}
splitWord.add(char)
splitNode = node
if (node == null) {
break
}
}
if (splitNode != null) {
// Input thus far is valid
val wordNodes = SuggestionList.new(maxTokenCount)
val strBuilder = StringBuilder().append(word.substring(0, word.length - 1))
splitNode.listAllSameOrderWords(wordNodes, strBuilder, allowPossiblyOffensive)
ngramList.addAll(wordNodes)
}
if (ngramList.size < maxTokenCount) {
val wordNodes = SuggestionList.new(maxTokenCount)
val strBuilder = StringBuilder()
currentNode.listSimilarWords(word, wordNodes, strBuilder, allowPossiblyOffensive, maxEditDistance)
ngramList.addAll(wordNodes)
}
}
} else {
val node = currentNode.findWord(word)
if (node == null) {
return ngramList
} else {
currentNode = node
}
}
}
return ngramList
}
fun toFlorisMutableLanguageModel(): FlorisMutableLanguageModel = FlorisMutableLanguageModel(ngramTree)
}
open class FlorisMutableLanguageModel(
initTreeObj: NgramTree? = null
) : MutableLanguageModel<String, Int>, FlorisLanguageModel(initTreeObj) {
override fun deleteNgram(ngram: Ngram<String, Int>) {
TODO("Not yet implemented")
}
override fun insertNgram(ngram: Ngram<String, Int>) {
TODO("Not yet implemented")
}
override fun updateNgram(ngram: Ngram<String, Int>) {
TODO("Not yet implemented")
}
fun toFlorisLanguageModel(): FlorisLanguageModel = FlorisLanguageModel(ngramTree)
}

74
app/src/main/java/dev/patrickgold/florisboard/ime/nlp/LanguageModel.kt
View File

@ -1,74 +0,0 @@
/*
* Copyright (C) 2021 Patrick Goldinger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dev.patrickgold.florisboard.ime.nlp
/**
* Abstract interface for a language model. Can house any n-grams with a minimum order of one.
*/
interface LanguageModel<T : Any, F : Comparable<F>> {
/**
* Tries to get the n-gram for the passed [tokens]. Throws a NPE if no match could be found.
*/
@Throws(NullPointerException::class)
fun getNgram(vararg tokens: T): Ngram<T, F>
/**
* Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while
* searching. Throws a NPE if no match could be found.
*/
@Throws(NullPointerException::class)
fun getNgram(ngram: Ngram<T, F>): Ngram<T, F>
/**
* Tries to get the n-gram for the passed [tokens]. Returns null if no match could be found.
*/
fun getNgramOrNull(vararg tokens: T): Ngram<T, F>?
/**
* Tries to get the n-gram for the passed [ngram], whereas the frequency is ignored while
* searching. Returns null if no match could be found.
*/
fun getNgramOrNull(ngram: Ngram<T, F>): Ngram<T, F>?
/**
* Checks if a given [ngram] exists within this model. If [doMatchFreq] is set to true, the
* frequency is also matched.
*/
fun hasNgram(ngram: Ngram<T, F>, doMatchFreq: Boolean = false): Boolean
/**
* Matches all n-grams which match the given [ngram], whereas the last item in the n-gram is
* is used to search for predictions.
*/
fun matchAllNgrams(
ngram: Ngram<T, F>,
maxEditDistance: Int,
maxTokenCount: Int,
allowPossiblyOffensive: Boolean
): List<WeightedToken<T, F>>
}
/**
* Mutable version of [LanguageModel].
*/
interface MutableLanguageModel<T : Any, F : Comparable<F>> : LanguageModel<T, F> {
fun deleteNgram(ngram: Ngram<T, F>)
fun insertNgram(ngram: Ngram<T, F>)
fun updateNgram(ngram: Ngram<T, F>)
}

5
app/src/main/java/dev/patrickgold/florisboard/ime/nlp/SuggestionList.kt
View File

@ -36,6 +36,8 @@ value class SuggestionList private constructor(
external fun nativeClear(nativePtr: NativePtr)
external fun nativeContains(nativePtr: NativePtr, element: Word): Boolean
external fun nativeGetOrNull(nativePtr: NativePtr, index: Int): Word?
external fun nativeGetIsPrimaryTokenAutoInsert(nativePtr: NativePtr): Boolean
external fun nativeSetIsPrimaryTokenAutoInsert(nativePtr: NativePtr, v: Boolean)
external fun nativeSize(nativePtr: NativePtr): Int
}
@ -75,6 +77,9 @@ value class SuggestionList private constructor(
override fun isEmpty(): Boolean = size <= 0
val isPrimaryTokenAutoInsert: Boolean
get() = nativeGetIsPrimaryTokenAutoInsert(_nativePtr)
override fun iterator(): Iterator<Word> {
return SuggestionListIterator(this)
}

24
app/src/main/java/dev/patrickgold/florisboard/ime/nlp/TextProcessor.kt
View File

@ -1,24 +0,0 @@
/*
* Copyright (C) 2021 Patrick Goldinger
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package dev.patrickgold.florisboard.ime.nlp
class TextProcessor {
data class Word(
val word: String,
val isPossiblyOffensive: Boolean = false
)
}

77
app/src/main/java/dev/patrickgold/florisboard/ime/text/TextInputManager.kt
View File

@ -27,13 +27,10 @@ import dev.patrickgold.florisboard.R
import dev.patrickgold.florisboard.debug.*
import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
import dev.patrickgold.florisboard.ime.core.*
import dev.patrickgold.florisboard.ime.dictionary.Dictionary
import dev.patrickgold.florisboard.ime.dictionary.DictionaryManager
import dev.patrickgold.florisboard.ime.extension.AssetManager
import dev.patrickgold.florisboard.ime.extension.AssetRef
import dev.patrickgold.florisboard.ime.extension.AssetSource
import dev.patrickgold.florisboard.ime.nlp.Token
import dev.patrickgold.florisboard.ime.nlp.toStringList
import dev.patrickgold.florisboard.ime.text.gestures.GlideTypingManager
import dev.patrickgold.florisboard.ime.text.gestures.SwipeAction
import dev.patrickgold.florisboard.ime.text.key.*
@ -42,7 +39,6 @@ import dev.patrickgold.florisboard.ime.text.layout.LayoutManager
import dev.patrickgold.florisboard.ime.text.smartbar.SmartbarView
import kotlinx.coroutines.*
import org.json.JSONArray
import java.util.*
import kotlin.math.roundToLong
/**
@ -74,8 +70,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
lateinit var textKeyboardIconSet: TextKeyboardIconSet
private set
private var textViewGroup: LinearLayout? = null
private val dictionaryManager: DictionaryManager = DictionaryManager.default()
private var activeDictionary: Dictionary<String, Int>? = null
private val dictionaryManager: DictionaryManager get() = DictionaryManager.default()
val inputEventDispatcher: InputEventDispatcher = InputEventDispatcher.new(
repeatableKeyCodes = intArrayOf(
KeyCode.ARROW_DOWN,
@ -420,11 +415,7 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
override fun onSubtypeChanged(newSubtype: Subtype) {
launch {
if (activeEditorInstance.isComposingEnabled) {
withContext(Dispatchers.IO) {
dictionaryManager.loadDictionary(AssetRef(AssetSource.Assets, "ime/dict/en.flict")).let {
activeDictionary = it.getOrDefault(null)
}
}
dictionaryManager.prepareDictionaries(newSubtype)
}
if (prefs.glide.enabled) {
GlideTypingManager.getInstance().setWordData(newSubtype)
@ -447,28 +438,24 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
if (activeEditorInstance.isComposingEnabled && !inputEventDispatcher.isPressed(KeyCode.DELETE) && !isGlidePostEffect) {
if (activeEditorInstance.shouldReevaluateComposingSuggestions) {
activeEditorInstance.shouldReevaluateComposingSuggestions = false
activeDictionary?.let {
launch(Dispatchers.Default) {
val startTime = System.nanoTime()
val suggestions = queryUserDictionary(
activeEditorInstance.cachedInput.currentWord.text,
florisboard.activeSubtype.locale
).toMutableList()
suggestions.addAll(it.getTokenPredictions(
precedingTokens = listOf(),
currentToken = Token(activeEditorInstance.cachedInput.currentWord.text),
maxSuggestionCount = 16,
allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive
).toStringList())
if (BuildConfig.DEBUG) {
val elapsed = (System.nanoTime() - startTime) / 1000.0
flogInfo { "sugg fetch time: $elapsed us" }
}
launch(Dispatchers.Default) {
val startTime = System.nanoTime()
dictionaryManager.suggest(
currentWord = activeEditorInstance.cachedInput.currentWord.text,
preceidingWords = listOf(),
subtype = florisboard.activeSubtype,
allowPossiblyOffensive = !prefs.suggestion.blockPossiblyOffensive,
maxSuggestionCount = 16
) { suggestions ->
withContext(Dispatchers.Main) {
smartbarView?.setCandidateSuggestionWords(startTime, suggestions)
smartbarView?.updateCandidateSuggestionCapsState()
}
}
if (BuildConfig.DEBUG) {
val elapsed = (System.nanoTime() - startTime) / 1000.0
flogInfo { "sugg fetch time: $elapsed us" }
}
}
} else {
smartbarView?.setCandidateSuggestionWords(System.nanoTime(), null)
@ -480,40 +467,6 @@ class TextInputManager private constructor() : CoroutineScope by MainScope(), In
smartbarView?.onPrimaryClipChanged()
}
private fun queryUserDictionary(word: String, locale: Locale): Set<String> {
val florisDao = dictionaryManager.florisUserDictionaryDao()
val systemDao = dictionaryManager.systemUserDictionaryDao()
if (florisDao == null && systemDao == null) {
return setOf()
}
val retList = mutableSetOf<String>()
if (prefs.dictionary.enableFlorisUserDictionary) {
florisDao?.query(word, locale)?.let {
for (entry in it) {
retList.add(entry.word)
}
}
florisDao?.queryShortcut(word, locale)?.let {
for (entry in it) {
retList.add(entry.word)
}
}
}
if (prefs.dictionary.enableSystemUserDictionary) {
systemDao?.query(word, locale)?.let {
for (entry in it) {
retList.add(entry.word)
}
}
systemDao?.queryShortcut(word, locale)?.let {
for (entry in it) {
retList.add(entry.word)
}
}
}
return retList
}
/**
* Updates the current caps state according to the [EditorInstance.cursorCapsMode], while
* respecting [capsLock] property and the correction.autoCapitalization preference.

6
app/src/main/java/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt
View File

@ -105,10 +105,12 @@ class GlideTypingManager : GlideTypingGesture.Listener, CoroutineScope by MainSc
textInputManager.isGlidePostEffect = true
textInputManager.smartbarView?.setCandidateSuggestionWords(
time,
suggestions.subList(
// FIXME
/*suggestions.subList(
1.coerceAtMost(min(commit.compareTo(false), suggestions.size)),
maxSuggestionsToShow.coerceAtMost(suggestions.size)
).map { textInputManager.fixCase(it) }
).map { textInputManager.fixCase(it) }*/
null
)
textInputManager.smartbarView?.updateCandidateSuggestionCapsState()
if (commit && suggestions.isNotEmpty()) {

5
app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/CandidateView.kt
View File

@ -36,6 +36,7 @@ import androidx.core.graphics.ColorUtils
import dev.patrickgold.florisboard.R
import dev.patrickgold.florisboard.ime.clip.FlorisClipboardManager
import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
import dev.patrickgold.florisboard.ime.nlp.SuggestionList
import dev.patrickgold.florisboard.ime.theme.Theme
import dev.patrickgold.florisboard.ime.theme.ThemeManager
import dev.patrickgold.florisboard.ime.theme.ThemeValue
@ -100,7 +101,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener {
themeManager = ThemeManager.defaultOrNull()
themeManager?.registerOnThemeUpdatedListener(this)
florisClipboardManager = FlorisClipboardManager.getInstanceOrNull()
updateCandidates(candidates)
recomputeCandidates()
}
override fun onDetachedFromWindow() {
@ -113,7 +114,7 @@ class CandidateView : View, ThemeManager.OnThemeUpdatedListener {
velocityTracker = null
}
fun updateCandidates(newCandidates: List<String>?) {
fun updateCandidates(newCandidates: SuggestionList?) {
candidates.clear()
if (newCandidates != null) {
candidates.addAll(newCandidates)

3
app/src/main/java/dev/patrickgold/florisboard/ime/text/smartbar/SmartbarView.kt
View File

@ -28,6 +28,7 @@ import dev.patrickgold.florisboard.ime.clip.provider.ClipboardItem
import dev.patrickgold.florisboard.ime.core.FlorisBoard
import dev.patrickgold.florisboard.ime.core.Preferences
import dev.patrickgold.florisboard.ime.core.Subtype
import dev.patrickgold.florisboard.ime.nlp.SuggestionList
import dev.patrickgold.florisboard.ime.text.key.KeyVariation
import dev.patrickgold.florisboard.ime.text.keyboard.KeyboardMode
import dev.patrickgold.florisboard.ime.theme.Theme
@ -275,7 +276,7 @@ class SmartbarView : ConstraintLayout, ThemeManager.OnThemeUpdatedListener {
}
}
fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: List<String>?) {
fun setCandidateSuggestionWords(suggestionInitDate: Long, suggestions: SuggestionList?) {
if (suggestionInitDate > lastSuggestionInitDate) {
lastSuggestionInitDate = suggestionInitDate
binding.candidates.updateCandidates(suggestions)