Implement word prediction support using the new NLP core

The IPC communication causes significant lag atm.
2024-09-19 19:42:20 +02:00 · 2023-05-31 13:58:55 +02:00 · 2023-05-31 13:58:55 +02:00 · 031d9fb75b
commit 031d9fb75b
parent eb23dc8ba1
13 changed files with 456 additions and 320 deletions
--- a/app/src/main/cpp/LatinNlpSession.cpp
+++ b/app/src/main/cpp/LatinNlpSession.cpp
@ -80,3 +80,25 @@ Java_dev_patrickgold_florisboard_ime_nlp_latin_LatinNlpSession_00024CXX_nativeSp
    json["suggestions"] = spelling_result.suggestions;
    return fl::jni::std2j_string(env, json.dump());
 }
+
+extern "C" JNIEXPORT fl::jni::NativeList JNICALL
+Java_dev_patrickgold_florisboard_ime_nlp_latin_LatinNlpSession_00024CXX_nativeSuggest( //
+    JNIEnv* env,
+    jobject,
+    jlong native_ptr,
+    fl::jni::NativeStr j_word,
+    fl::jni::NativeList j_prev_words,
+    jint flags
+) {
+    auto* session = reinterpret_cast<fl::nlp::LatinNlpSession*>(native_ptr);
+    auto word = fl::jni::j2std_string(env, j_word);
+    auto prev_words = fl::jni::j2std_list<std::string>(env, j_prev_words);
+    fl::nlp::SuggestionResults suggestion_results;
+    session->suggest(word, prev_words, flags, suggestion_results);
+    std::vector<fl::nlp::SuggestionCandidate> candidates;
+    candidates.reserve(suggestion_results.size());
+    for (auto& candidate_ptr : suggestion_results) {
+        candidates.push_back(std::move(*candidate_ptr));
+    }
+    return fl::jni::std2j_list(env, candidates);
+}
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/clipboard/ClipboardSuggestionCandidate.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/clipboard/ClipboardSuggestionCandidate.kt
@ -29,28 +29,24 @@ import dev.patrickgold.florisboard.lib.util.NetworkUtils
 *
 * @see SuggestionCandidate
 */
-data class ClipboardSuggestionCandidate(
+class ClipboardSuggestionCandidate(
    val clipboardItem: ClipboardItem,
-    override val sourceProvider: SuggestionProvider?,
-) : SuggestionCandidate {
-    override val text: CharSequence = clipboardItem.stringRepresentation()
-
-    override val secondaryText: CharSequence? = null
-
-    override val confidence: Double = 1.0
-
-    override val isEligibleForAutoCommit: Boolean = false
-
-    override val isEligibleForUserRemoval: Boolean = true
-
-    override val iconId: Int = when (clipboardItem.type) {
+    sourceProvider: SuggestionProvider?,
+) : SuggestionCandidate(
+    text = clipboardItem.stringRepresentation(),
+    secondaryText = null,
+    confidence = 1.0,
+    isEligibleForAutoCommit = false,
+    isEligibleForUserRemoval = true,
+    iconId = when (clipboardItem.type) {
        ItemType.TEXT -> when {
-            NetworkUtils.isEmailAddress(text) -> R.drawable.ic_email
-            NetworkUtils.isUrl(text) -> R.drawable.ic_link
-            NetworkUtils.isPhoneNumber(text) -> R.drawable.ic_phone
+            NetworkUtils.isEmailAddress(clipboardItem.stringRepresentation()) -> R.drawable.ic_email
+            NetworkUtils.isUrl(clipboardItem.stringRepresentation()) -> R.drawable.ic_link
+            NetworkUtils.isPhoneNumber(clipboardItem.stringRepresentation()) -> R.drawable.ic_phone
            else -> R.drawable.ic_assignment
        }
        ItemType.IMAGE -> R.drawable.ic_image
        ItemType.VIDEO -> R.drawable.ic_videocam
-    }
-}
+    },
+    sourceProvider = sourceProvider,
+)
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/dictionary/DictionaryManager.kt
@ -20,7 +20,6 @@ import android.content.Context
 import androidx.room.Room
 import dev.patrickgold.florisboard.app.florisPreferenceModel
 import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
-import dev.patrickgold.florisboard.ime.nlp.WordSuggestionCandidate
 import dev.patrickgold.florisboard.lib.FlorisLocale
 import java.lang.ref.WeakReference

@ -65,24 +64,24 @@ class DictionaryManager private constructor(context: Context) {
            if (prefs.dictionary.enableFlorisUserDictionary.get()) {
                florisDao?.query(word, locale)?.let {
                    for (entry in it) {
-                        add(WordSuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
+                        add(SuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
                    }
                }
                florisDao?.queryShortcut(word, locale)?.let {
                    for (entry in it) {
-                        add(WordSuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
+                        add(SuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
                    }
                }
            }
            if (prefs.dictionary.enableSystemUserDictionary.get()) {
                systemDao?.query(word, locale)?.let {
                    for (entry in it) {
-                        add(WordSuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
+                        add(SuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
                    }
                }
                systemDao?.queryShortcut(word, locale)?.let {
                    for (entry in it) {
-                        add(WordSuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
+                        add(SuggestionCandidate(entry.word, confidence = entry.freq / 255.0))
                    }
                }
            }
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/NlpManager.kt
@ -211,13 +211,12 @@ class NlpManager(context: Context) {
    fun suggest(subtype: Subtype, content: EditorContent) {
        val reqTime = SystemClock.uptimeMillis()
        scope.launch {
-            val suggestions = plugins.getOrNull(subtype.nlpProviders.spelling)?.spell(
+            val candidates = plugins.getOrNull(subtype.nlpProviders.spelling)?.suggest(
                subtypeId = subtype.id,
                word = content.composingText,
                prevWords = content.textBeforeSelection.split(" "), // TODO this split is incorrect
                flags = activeSuggestionRequestFlags(),
-            ) ?: SpellingResult.unspecified()
-            val candidates = suggestions.suggestions().map { WordSuggestionCandidate(it) }
+            ) ?: emptyList()
            flogDebug { "candidates: $candidates" }
            internalSuggestionsGuard.withLock {
                if (internalSuggestions.first < reqTime) {
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/han/HanShapeBasedLanguageProvider.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/han/HanShapeBasedLanguageProvider.kt
@ -31,7 +31,6 @@ import dev.patrickgold.florisboard.ime.nlp.SpellingProvider
 import dev.patrickgold.florisboard.ime.nlp.SpellingResult
 import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
 import dev.patrickgold.florisboard.ime.nlp.SuggestionProvider
-import dev.patrickgold.florisboard.ime.nlp.WordSuggestionCandidate
 import dev.patrickgold.florisboard.lib.devtools.flogDebug
 import dev.patrickgold.florisboard.lib.devtools.flogError
 import dev.patrickgold.florisboard.subtypeManager
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/latin/LatinLanguageProviderService.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/latin/LatinLanguageProviderService.kt
@ -21,6 +21,8 @@ import dev.patrickgold.florisboard.ime.core.ComputedSubtype
 import dev.patrickgold.florisboard.ime.keyboard.KeyProximityChecker
 import dev.patrickgold.florisboard.ime.nlp.SpellingProvider
 import dev.patrickgold.florisboard.ime.nlp.SpellingResult
+import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
+import dev.patrickgold.florisboard.ime.nlp.SuggestionProvider
 import dev.patrickgold.florisboard.ime.nlp.SuggestionRequestFlags
 import dev.patrickgold.florisboard.lib.FlorisLocale
 import dev.patrickgold.florisboard.lib.io.subFile
@ -61,7 +63,7 @@ private data class LatinNlpSessionWrapper(
    var session: LatinNlpSession,
 )

-class LatinLanguageProviderService : FlorisPluginService(), SpellingProvider {
+class LatinLanguageProviderService : FlorisPluginService(), SpellingProvider, SuggestionProvider {
    companion object {
        const val NlpSessionConfigFileName = "nlp_session_config.json"
        const val UserDictionaryFileName = "user_dict.fldic"
@ -122,10 +124,34 @@ class LatinLanguageProviderService : FlorisPluginService(), SpellingProvider {
    ): SpellingResult {
        return cachedSessionWrappers.withLock { sessionWrappers ->
            val sessionWrapper = sessionWrappers.find { it.subtype.id == subtypeId }
-            return@withLock sessionWrapper?.session?.spell(word, prevWords, flags) ?: SpellingResult.unspecified()
+            sessionWrapper?.session?.spell(word, prevWords, flags) ?: SpellingResult.unspecified()
        }
    }

+    override suspend fun suggest(
+        subtypeId: Long,
+        word: String,
+        prevWords: List<String>,
+        flags: SuggestionRequestFlags,
+    ): List<SuggestionCandidate> {
+        return cachedSessionWrappers.withLock { sessionWrappers ->
+            val sessionWrapper = sessionWrappers.find { it.subtype.id == subtypeId }
+            sessionWrapper?.session?.suggest(word, prevWords, flags) ?: emptyList()
+        }
+    }
+
+    override suspend fun notifySuggestionAccepted(subtypeId: Long, candidate: SuggestionCandidate) {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun notifySuggestionReverted(subtypeId: Long, candidate: SuggestionCandidate) {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun removeSuggestion(subtypeId: Long, candidate: SuggestionCandidate): Boolean {
+        TODO("Not yet implemented")
+    }
+
    override suspend fun destroy() {
        //
    }
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/latin/LatinNlpSession.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/latin/LatinNlpSession.kt
@ -19,13 +19,14 @@ package dev.patrickgold.florisboard.ime.nlp.latin
 import android.view.textservice.SuggestionsInfo
 import dev.patrickgold.florisboard.ime.keyboard.KeyProximityChecker
 import dev.patrickgold.florisboard.ime.nlp.SpellingResult
+import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
 import dev.patrickgold.florisboard.ime.nlp.SuggestionRequestFlags
 import dev.patrickgold.florisboard.lib.io.FsFile
 import dev.patrickgold.florisboard.lib.kotlin.tryOrNull
 import dev.patrickgold.florisboard.native.NativeInstanceWrapper
+import dev.patrickgold.florisboard.native.NativeList
 import dev.patrickgold.florisboard.native.NativePtr
 import dev.patrickgold.florisboard.native.NativeStr
-import dev.patrickgold.florisboard.native.NativeList
 import dev.patrickgold.florisboard.native.toJavaString
 import dev.patrickgold.florisboard.native.toNativeList
 import dev.patrickgold.florisboard.native.toNativeStr
@ -61,23 +62,41 @@ value class LatinNlpSession(private val _nativePtr: NativePtr = nativeInit()) :
        prevWords: List<String>,
        flags: SuggestionRequestFlags,
    ): SpellingResult {
-        return withContext(Dispatchers.IO) {
-            val nativeSpellingResultStr = nativeSpell(
-                nativePtr = _nativePtr,
-                word = word.toNativeStr(),
-                prevWords = prevWords.toNativeList(),
-                flags = flags.toInt(),
-            ).toJavaString()
-            val nativeSpellingResult = Json.decodeFromString<NativeSpellingResult>(nativeSpellingResultStr)
-            return@withContext tryOrNull {
+        return tryOrNull {
+            withContext(Dispatchers.IO) {
+                val nativeSpellingResultStr = nativeSpell(
+                    nativePtr = _nativePtr,
+                    word = word.toNativeStr(),
+                    prevWords = prevWords.toNativeList(),
+                    flags = flags.toInt(),
+                ).toJavaString()
+                val nativeSpellingResult = Json.decodeFromString<NativeSpellingResult>(nativeSpellingResultStr)
                SpellingResult(
                    SuggestionsInfo(
                        nativeSpellingResult.suggestionAttributes,
                        nativeSpellingResult.suggestions.toTypedArray(),
                    )
                )
-            } ?: SpellingResult.unspecified()
-        }
+            }
+        } ?: SpellingResult.unspecified()
+    }
+
+    suspend fun suggest(
+        word: String,
+        prevWords: List<String>,
+        flags: SuggestionRequestFlags,
+    ): List<SuggestionCandidate> {
+        //return tryOrNull {
+            return withContext(Dispatchers.IO) {
+                val nativeCandidatesList = nativeSuggest(
+                    nativePtr = _nativePtr,
+                    word = word.toNativeStr(),
+                    prevWords = prevWords.toNativeList(),
+                    flags = flags.toInt(),
+                ).toJavaString()
+                Json.decodeFromString(nativeCandidatesList)
+            }
+        //} ?: emptyList()
    }

    override fun nativePtr(): NativePtr {
@ -96,16 +115,25 @@ value class LatinNlpSession(private val _nativePtr: NativePtr = nativeInit()) :

    companion object CXX {
        external fun nativeInit(): NativePtr
+
        external fun nativeDispose(nativePtr: NativePtr)

        external fun nativeLoadFromConfigFile(nativePtr: NativePtr, configPath: NativeStr)
+
        external fun nativeSpell(
            nativePtr: NativePtr,
            word: NativeStr,
            prevWords: NativeList,
            flags: Int,
        ): NativeStr
-        //external fun nativeSuggest(word: NativeStr, prevWords: List<NativeStr>, flags: Int)
+
+        external fun nativeSuggest(
+            nativePtr: NativePtr,
+            word: NativeStr,
+            prevWords: NativeList,
+            flags: Int,
+        ): NativeList
+
        //external fun nativeTrain(sentence: List<NativeStr>, maxPrevWords: Int)
    }
 }
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/smartbar/CandidatesRow.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/smartbar/CandidatesRow.kt
@ -251,7 +251,7 @@ private fun CandidateItem(
                maxLines = 1,
                overflow = TextOverflow.Ellipsis,
            )
-            if (candidate.secondaryText != null) {
+            if (!candidate.secondaryText.isNullOrEmpty()) {
                Text(
                    text = candidate.secondaryText!!.toString(),
                    color = style.foreground.solidColor(context),
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/ime/text/gestures/GlideTypingManager.kt
@ -2,7 +2,7 @@ package dev.patrickgold.florisboard.ime.text.gestures

 import android.content.Context
 import dev.patrickgold.florisboard.app.florisPreferenceModel
-import dev.patrickgold.florisboard.ime.nlp.WordSuggestionCandidate
+import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
 import dev.patrickgold.florisboard.ime.text.keyboard.TextKey
 import dev.patrickgold.florisboard.keyboardManager
 import dev.patrickgold.florisboard.nlpManager
@ -86,7 +86,7 @@ class GlideTypingManager(context: Context) : GlideTypingGesture.Listener {
                        1.coerceAtMost(min(commit.compareTo(false), suggestions.size)),
                        maxSuggestionsToShow.coerceAtMost(suggestions.size)
                    ).map { keyboardManager.fixCase(it) }.forEach {
-                        add(WordSuggestionCandidate(it, confidence = 1.0))
+                        add(SuggestionCandidate(it, confidence = 1.0))
                    }
                }

--- a/app/src/main/kotlin/dev/patrickgold/florisboard/plugin/FlorisPluginIndexer.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/plugin/FlorisPluginIndexer.kt
@ -16,43 +16,17 @@

 package dev.patrickgold.florisboard.plugin

-import android.annotation.SuppressLint
 import android.content.BroadcastReceiver
 import android.content.ComponentName
 import android.content.Context
 import android.content.Intent
 import android.content.IntentFilter
-import android.content.ServiceConnection
 import android.content.pm.PackageManager
-import android.os.Handler
-import android.os.IBinder
-import android.os.Message
-import android.os.Messenger
-import android.view.textservice.SuggestionsInfo
 import dev.patrickgold.florisboard.BuildConfig
-import dev.patrickgold.florisboard.ime.core.ComputedSubtype
-import dev.patrickgold.florisboard.ime.nlp.SpellingProvider
-import dev.patrickgold.florisboard.ime.nlp.SpellingResult
-import dev.patrickgold.florisboard.ime.nlp.SuggestionRequest
-import dev.patrickgold.florisboard.ime.nlp.SuggestionRequestFlags
-import dev.patrickgold.florisboard.lib.devtools.flogDebug
-import dev.patrickgold.florisboard.lib.io.FlorisRef
 import dev.patrickgold.florisboard.lib.kotlin.guardedByLock
-import kotlinx.coroutines.CoroutineScope
-import kotlinx.coroutines.Dispatchers
-import kotlinx.coroutines.SupervisorJob
-import kotlinx.coroutines.channels.BufferOverflow
-import kotlinx.coroutines.flow.MutableSharedFlow
 import kotlinx.coroutines.flow.MutableStateFlow
-import kotlinx.coroutines.flow.asSharedFlow
 import kotlinx.coroutines.flow.asStateFlow
-import kotlinx.coroutines.flow.first
-import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
-import kotlinx.coroutines.withTimeoutOrNull
-import kotlinx.serialization.json.Json
-import java.util.concurrent.atomic.AtomicBoolean
-import java.util.concurrent.atomic.AtomicInteger

 class FlorisPluginIndexer(private val context: Context) {
    private val _pluginIndexFlow = MutableStateFlow(listOf<IndexedPlugin>())
@ -135,225 +109,3 @@ class FlorisPluginIndexer(private val context: Context) {
        }
    }
 }
-
-class IndexedPlugin(
-    val context: Context,
-    val serviceName: ComponentName,
-    var state: IndexedPluginState,
-    var metadata: FlorisPluginMetadata = FlorisPluginMetadata(""),
-) : SpellingProvider {
-    private var messageIdGenerator = AtomicInteger(1)
-    private var connection = IndexedPluginConnection(context)
-
-    override suspend fun create() {
-        if (isValidAndBound()) return
-        connection.bindService(serviceName)
-    }
-
-    override suspend fun preload(subtype: ComputedSubtype) {
-        val message = FlorisPluginMessage.requestToService(
-            action = FlorisPluginMessage.ACTION_PRELOAD,
-            id = messageIdGenerator.getAndIncrement(),
-            data = Json.encodeToString(ComputedSubtype.serializer(), subtype),
-        )
-        connection.sendMessage(message)
-    }
-
-    override suspend fun spell(
-        subtypeId: Long,
-        word: String,
-        prevWords: List<String>,
-        flags: SuggestionRequestFlags,
-    ): SpellingResult {
-        val request = SuggestionRequest(subtypeId, word, prevWords, flags)
-        val message = FlorisPluginMessage.requestToService(
-            action = FlorisPluginMessage.ACTION_SPELL,
-            id = messageIdGenerator.getAndIncrement(),
-            data = Json.encodeToString(SuggestionRequest.serializer(), request),
-        )
-        connection.sendMessage(message)
-        return withTimeoutOrNull(5000L) {
-            val replyMessage = connection.replyMessages.first { it.id == message.id }
-            val resultObj = replyMessage.obj as? SuggestionsInfo ?: return@withTimeoutOrNull null
-            SpellingResult(resultObj)
-        } ?: SpellingResult.unspecified()
-    }
-
-    override suspend fun destroy() {
-        if (!isValidAndBound()) return
-        connection.unbindService()
-    }
-
-    fun packageContext(): Context {
-        return context.createPackageContext(serviceName.packageName, 0)
-    }
-
-    fun configurationRoute(): String? {
-        if (!isValid()) return null
-        val configurationRoute = metadata.settingsActivity ?: return null
-        val ref = FlorisRef.from(configurationRoute)
-        return if (ref.isAppUi) ref.relativePath else null
-    }
-
-    fun settingsActivityIntent(): Intent? {
-        if (!isValid()) return null
-        val settingsActivityName = metadata.settingsActivity ?: return null
-        val intent = Intent().also {
-            it.component = ComponentName(serviceName.packageName, settingsActivityName)
-            it.flags = Intent.FLAG_ACTIVITY_NEW_TASK
-        }
-        return if (intent.resolveActivityInfo(context.packageManager, 0) != null) {
-            intent
-        } else {
-            null
-        }
-    }
-
-    fun isValid(): Boolean {
-        return state == IndexedPluginState.Ok
-    }
-
-    fun isValidAndBound(): Boolean {
-        return isValid() && connection.isBound()
-    }
-
-    fun isInternalPlugin(): Boolean {
-        return serviceName.packageName == BuildConfig.APPLICATION_ID
-    }
-
-    fun isExternalPlugin(): Boolean {
-        return !isInternalPlugin()
-    }
-
-    override fun toString(): String {
-        val packageContext = packageContext()
-        return """
-            IndexedPlugin {
-                serviceName=$serviceName
-                state=$state
-                isBound=${connection.isBound()}
-                metadata=${metadata.toString(packageContext).prependIndent("                ").substring(16)}
-            }
-        """.trimIndent()
-    }
-}
-
-class IndexedPluginConnection(private val context: Context) {
-    private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
-    private var serviceMessenger = MutableStateFlow<Messenger?>(null)
-    private val consumerMessenger = Messenger(IncomingHandler())
-    private var isBound = AtomicBoolean(false)
-    private val stagedOutgoingMessages = MutableSharedFlow<FlorisPluginMessage>(
-        replay = 8,
-        extraBufferCapacity = 8,
-        onBufferOverflow = BufferOverflow.DROP_OLDEST,
-    )
-    private val _replyMessages = MutableSharedFlow<FlorisPluginMessage>(
-        replay = 8,
-        extraBufferCapacity = 8,
-        onBufferOverflow = BufferOverflow.DROP_OLDEST,
-    )
-    val replyMessages = _replyMessages.asSharedFlow()
-
-    private val serviceConnection = object : ServiceConnection {
-        override fun onServiceConnected(name: ComponentName?, binder: IBinder?) {
-            flogDebug { "$name, $binder" }
-            if (name == null || binder == null) return
-
-            serviceMessenger.value = Messenger(binder)
-        }
-
-        override fun onServiceDisconnected(name: ComponentName?) {
-            flogDebug { "$name" }
-            if (name == null) return
-
-            serviceMessenger.value = null
-        }
-
-        override fun onBindingDied(name: ComponentName?) {
-            flogDebug { "$name" }
-            if (name == null) return
-
-            serviceMessenger.value = null
-            unbindService()
-            bindService(name)
-        }
-
-        override fun onNullBinding(name: ComponentName?) {
-            flogDebug { "$name" }
-            if (name == null) return
-
-            serviceMessenger.value = null
-            unbindService()
-        }
-    }
-
-    init {
-        scope.launch {
-            stagedOutgoingMessages.collect { message ->
-                val messenger = serviceMessenger.first { it != null }!!
-                messenger.send(message.also { it.replyTo = consumerMessenger }.toAndroidMessage())
-            }
-        }
-    }
-
-    fun isBound(): Boolean {
-        return isBound.get() && serviceMessenger.value != null
-    }
-
-    fun bindService(serviceName: ComponentName) {
-        if (isBound.getAndSet(true)) return
-        val intent = Intent().also {
-            it.component = serviceName
-            it.putExtra(FlorisPluginService.CONSUMER_PACKAGE_NAME, BuildConfig.APPLICATION_ID)
-            it.putExtra(FlorisPluginService.CONSUMER_VERSION_CODE, BuildConfig.VERSION_CODE)
-            it.putExtra(FlorisPluginService.CONSUMER_VERSION_NAME, BuildConfig.VERSION_NAME)
-        }
-        context.bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE)
-    }
-
-    fun unbindService() {
-        if (!isBound.getAndSet(false)) return
-        context.unbindService(serviceConnection)
-    }
-
-    fun sendMessage(message: FlorisPluginMessage) = runBlocking {
-        stagedOutgoingMessages.emit(message)
-    }
-
-    @SuppressLint("HandlerLeak")
-    inner class IncomingHandler : Handler(context.mainLooper) {
-        override fun handleMessage(msg: Message) {
-            val message = FlorisPluginMessage.fromAndroidMessage(msg)
-            val (source, type, _) = message.metadata()
-            if (source != FlorisPluginMessage.SOURCE_SERVICE || type != FlorisPluginMessage.TYPE_RESPONSE) {
-                return
-            }
-            runBlocking {
-                _replyMessages.emit(message)
-            }
-        }
-    }
-}
-
-enum class IndexedPluginError {
-    NoMetadata,
-    InvalidMetadata,
-}
-
-sealed class IndexedPluginState {
-    object Ok : IndexedPluginState()
-    data class Error(val type: IndexedPluginError, val exception: Exception?) : IndexedPluginState()
-
-    override fun toString(): String {
-        return when (this) {
-            is Ok -> "Ok"
-            is Error -> "Error($type, $exception)"
-        }
-    }
-}
-
-private fun stateOk() = IndexedPluginState.Ok
-
-private fun stateError(type: IndexedPluginError, exception: Exception? = null) =
-    IndexedPluginState.Error(type, exception)
--- a/app/src/main/kotlin/dev/patrickgold/florisboard/plugin/IndexedPlugin.kt
+++ b/app/src/main/kotlin/dev/patrickgold/florisboard/plugin/IndexedPlugin.kt
@ -0,0 +1,308 @@
+/*
+ * Copyright (C) 2023 Patrick Goldinger
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dev.patrickgold.florisboard.plugin
+
+import android.annotation.SuppressLint
+import android.content.ComponentName
+import android.content.Context
+import android.content.Intent
+import android.content.ServiceConnection
+import android.os.Handler
+import android.os.IBinder
+import android.os.Message
+import android.os.Messenger
+import android.view.textservice.SuggestionsInfo
+import dev.patrickgold.florisboard.BuildConfig
+import dev.patrickgold.florisboard.ime.core.ComputedSubtype
+import dev.patrickgold.florisboard.ime.nlp.SpellingProvider
+import dev.patrickgold.florisboard.ime.nlp.SpellingResult
+import dev.patrickgold.florisboard.ime.nlp.SuggestionCandidate
+import dev.patrickgold.florisboard.ime.nlp.SuggestionProvider
+import dev.patrickgold.florisboard.ime.nlp.SuggestionRequest
+import dev.patrickgold.florisboard.ime.nlp.SuggestionRequestFlags
+import dev.patrickgold.florisboard.lib.devtools.flogDebug
+import dev.patrickgold.florisboard.lib.io.FlorisRef
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.SupervisorJob
+import kotlinx.coroutines.channels.BufferOverflow
+import kotlinx.coroutines.flow.MutableSharedFlow
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.asSharedFlow
+import kotlinx.coroutines.flow.first
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.withTimeoutOrNull
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
+import java.util.concurrent.atomic.AtomicBoolean
+import java.util.concurrent.atomic.AtomicInteger
+
+class IndexedPlugin(
+    val context: Context,
+    val serviceName: ComponentName,
+    var state: IndexedPluginState,
+    var metadata: FlorisPluginMetadata = FlorisPluginMetadata(""),
+) : SpellingProvider, SuggestionProvider {
+    private var messageIdGenerator = AtomicInteger(1)
+    private var connection = IndexedPluginConnection(context)
+
+    override suspend fun create() {
+        if (isValidAndBound()) return
+        connection.bindService(serviceName)
+    }
+
+    override suspend fun preload(subtype: ComputedSubtype) {
+        val message = FlorisPluginMessage.requestToService(
+            action = FlorisPluginMessage.ACTION_PRELOAD,
+            id = messageIdGenerator.getAndIncrement(),
+            data = Json.encodeToString(ComputedSubtype.serializer(), subtype),
+        )
+        connection.sendMessage(message)
+    }
+
+    override suspend fun spell(
+        subtypeId: Long,
+        word: String,
+        prevWords: List<String>,
+        flags: SuggestionRequestFlags,
+    ): SpellingResult {
+        val request = SuggestionRequest(subtypeId, word, prevWords, flags)
+        val message = FlorisPluginMessage.requestToService(
+            action = FlorisPluginMessage.ACTION_SPELL,
+            id = messageIdGenerator.getAndIncrement(),
+            data = Json.encodeToString(request),
+        )
+        connection.sendMessage(message)
+        return withTimeoutOrNull(5000L) {
+            val replyMessage = connection.replyMessages.first { it.id == message.id }
+            val resultObj = replyMessage.obj as? SuggestionsInfo ?: return@withTimeoutOrNull null
+            return@withTimeoutOrNull SpellingResult(resultObj)
+        } ?: SpellingResult.unspecified()
+    }
+
+    override suspend fun suggest(
+        subtypeId: Long,
+        word: String,
+        prevWords: List<String>,
+        flags: SuggestionRequestFlags,
+    ): List<SuggestionCandidate> {
+        val request = SuggestionRequest(subtypeId, word, prevWords, flags)
+        val message = FlorisPluginMessage.requestToService(
+            action = FlorisPluginMessage.ACTION_SUGGEST,
+            id = messageIdGenerator.getAndIncrement(),
+            data = Json.encodeToString(request),
+        )
+        connection.sendMessage(message)
+        return withTimeoutOrNull(5000L) {
+            val replyMessage = connection.replyMessages.first { it.id == message.id }
+            val resultData = replyMessage.data ?: return@withTimeoutOrNull null
+            return@withTimeoutOrNull Json.decodeFromString(resultData)
+        } ?: emptyList()
+    }
+
+    override suspend fun notifySuggestionAccepted(subtypeId: Long, candidate: SuggestionCandidate) {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun notifySuggestionReverted(subtypeId: Long, candidate: SuggestionCandidate) {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun removeSuggestion(subtypeId: Long, candidate: SuggestionCandidate): Boolean {
+        TODO("Not yet implemented")
+    }
+
+    override suspend fun destroy() {
+        if (!isValidAndBound()) return
+        connection.unbindService()
+    }
+
+    fun packageContext(): Context {
+        return context.createPackageContext(serviceName.packageName, 0)
+    }
+
+    fun configurationRoute(): String? {
+        if (!isValid()) return null
+        val configurationRoute = metadata.settingsActivity ?: return null
+        val ref = FlorisRef.from(configurationRoute)
+        return if (ref.isAppUi) ref.relativePath else null
+    }
+
+    fun settingsActivityIntent(): Intent? {
+        if (!isValid()) return null
+        val settingsActivityName = metadata.settingsActivity ?: return null
+        val intent = Intent().also {
+            it.component = ComponentName(serviceName.packageName, settingsActivityName)
+            it.flags = Intent.FLAG_ACTIVITY_NEW_TASK
+        }
+        return if (intent.resolveActivityInfo(context.packageManager, 0) != null) {
+            intent
+        } else {
+            null
+        }
+    }
+
+    fun isValid(): Boolean {
+        return state == IndexedPluginState.Ok
+    }
+
+    fun isValidAndBound(): Boolean {
+        return isValid() && connection.isBound()
+    }
+
+    fun isInternalPlugin(): Boolean {
+        return serviceName.packageName == BuildConfig.APPLICATION_ID
+    }
+
+    fun isExternalPlugin(): Boolean {
+        return !isInternalPlugin()
+    }
+
+    override fun toString(): String {
+        val packageContext = packageContext()
+        return """
+            IndexedPlugin {
+                serviceName=$serviceName
+                state=$state
+                isBound=${connection.isBound()}
+                metadata=${metadata.toString(packageContext).prependIndent("                ").substring(16)}
+            }
+        """.trimIndent()
+    }
+}
+
+class IndexedPluginConnection(private val context: Context) {
+    private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
+    private var serviceMessenger = MutableStateFlow<Messenger?>(null)
+    private val consumerMessenger = Messenger(IncomingHandler())
+    private var isBound = AtomicBoolean(false)
+    private val stagedOutgoingMessages = MutableSharedFlow<FlorisPluginMessage>(
+        replay = 8,
+        extraBufferCapacity = 8,
+        onBufferOverflow = BufferOverflow.DROP_OLDEST,
+    )
+    private val _replyMessages = MutableSharedFlow<FlorisPluginMessage>(
+        replay = 8,
+        extraBufferCapacity = 8,
+        onBufferOverflow = BufferOverflow.DROP_OLDEST,
+    )
+    val replyMessages = _replyMessages.asSharedFlow()
+
+    private val serviceConnection = object : ServiceConnection {
+        override fun onServiceConnected(name: ComponentName?, binder: IBinder?) {
+            flogDebug { "$name, $binder" }
+            if (name == null || binder == null) return
+
+            serviceMessenger.value = Messenger(binder)
+        }
+
+        override fun onServiceDisconnected(name: ComponentName?) {
+            flogDebug { "$name" }
+            if (name == null) return
+
+            serviceMessenger.value = null
+        }
+
+        override fun onBindingDied(name: ComponentName?) {
+            flogDebug { "$name" }
+            if (name == null) return
+
+            serviceMessenger.value = null
+            unbindService()
+            bindService(name)
+        }
+
+        override fun onNullBinding(name: ComponentName?) {
+            flogDebug { "$name" }
+            if (name == null) return
+
+            serviceMessenger.value = null
+            unbindService()
+        }
+    }
+
+    init {
+        scope.launch {
+            stagedOutgoingMessages.collect { message ->
+                val messenger = serviceMessenger.first { it != null }!!
+                messenger.send(message.also { it.replyTo = consumerMessenger }.toAndroidMessage())
+            }
+        }
+    }
+
+    fun isBound(): Boolean {
+        return isBound.get() && serviceMessenger.value != null
+    }
+
+    fun bindService(serviceName: ComponentName) {
+        if (isBound.getAndSet(true)) return
+        val intent = Intent().also {
+            it.component = serviceName
+            it.putExtra(FlorisPluginService.CONSUMER_PACKAGE_NAME, BuildConfig.APPLICATION_ID)
+            it.putExtra(FlorisPluginService.CONSUMER_VERSION_CODE, BuildConfig.VERSION_CODE)
+            it.putExtra(FlorisPluginService.CONSUMER_VERSION_NAME, BuildConfig.VERSION_NAME)
+        }
+        context.bindService(intent, serviceConnection, Context.BIND_AUTO_CREATE)
+    }
+
+    fun unbindService() {
+        if (!isBound.getAndSet(false)) return
+        context.unbindService(serviceConnection)
+    }
+
+    fun sendMessage(message: FlorisPluginMessage) = runBlocking {
+        stagedOutgoingMessages.emit(message)
+    }
+
+    @SuppressLint("HandlerLeak")
+    inner class IncomingHandler : Handler(context.mainLooper) {
+        override fun handleMessage(msg: Message) {
+            val message = FlorisPluginMessage.fromAndroidMessage(msg)
+            val (source, type, _) = message.metadata()
+            if (source != FlorisPluginMessage.SOURCE_SERVICE || type != FlorisPluginMessage.TYPE_RESPONSE) {
+                return
+            }
+            runBlocking {
+                _replyMessages.emit(message)
+            }
+        }
+    }
+}
+
+enum class IndexedPluginError {
+    NoMetadata,
+    InvalidMetadata,
+}
+
+sealed class IndexedPluginState {
+    object Ok : IndexedPluginState()
+    data class Error(val type: IndexedPluginError, val exception: Exception?) : IndexedPluginState()
+
+    override fun toString(): String {
+        return when (this) {
+            is Ok -> "Ok"
+            is Error -> "Error($type, $exception)"
+        }
+    }
+}
+
+internal fun stateOk() = IndexedPluginState.Ok
+
+internal fun stateError(type: IndexedPluginError, exception: Exception? = null) =
+    IndexedPluginState.Error(type, exception)
--- a/ime-lib/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/SuggestionCandidate.kt
+++ b/ime-lib/src/main/kotlin/dev/patrickgold/florisboard/ime/nlp/SuggestionCandidate.kt
@ -17,12 +17,14 @@
 package dev.patrickgold.florisboard.ime.nlp

 import kotlinx.serialization.Serializable
+import kotlinx.serialization.Transient

 /**
 * Interface for a candidate item, which is returned by a suggestion provider and used by the UI logic to render
 * the candidate row.
 */
-interface SuggestionCandidate {
+@Serializable
+open class SuggestionCandidate(
    /**
     * Required primary text of a candidate item, must be non-null and non-blank. The value of this property will
     * be committed to the target editor when the user clicks on this candidate item (either replacing the current
@ -31,7 +33,7 @@ interface SuggestionCandidate {
     * In the UI it will be shown as the main label of a candidate item. Long texts that don't fit the maximum
     * candidate item width may be shortened and ellipsized.
     */
-    val text: CharSequence
+    val text: String,

    /**
     * Optional secondary text of a candidate item, can be used to provide additional context, e.g. for translation
@ -41,14 +43,14 @@ interface SuggestionCandidate {
     * is a non-null and non-blank character sequence. Long texts that don't fit the maximum candidate item width
     * may be shortened and ellipsized.
     */
-    val secondaryText: CharSequence?
+    val secondaryText: String? = null,

    /**
     * The confidence of this suggestion to be what the user wanted to type. Must be a value between 0.0 and 1.0 (both
     * inclusive), where 0.0 means no confidence and 1.0 means highest confidence. The confidence rating may be used to
     * sort and filter candidates if multiple providers provide suggestions for a single input.
     */
-    val confidence: Double
+    val confidence: Double = 0.0,

    /**
     * If true, it indicates that this candidate item should be automatically committed to the target editor once the
@ -60,14 +62,14 @@ interface SuggestionCandidate {
     * Only set this property to true if the algorithm has a high confidence that this suggestion is what the user
     * wanted to type.
     */
-    val isEligibleForAutoCommit: Boolean
+    val isEligibleForAutoCommit: Boolean = false,

    /**
     * If true, it indicates that this candidate item should be user removable (by long-pressing). This flag should
     * only be set if it actually makes sense for this type of candidate to be removable and if the linked source
     * provider supports this action.
     */
-    val isEligibleForUserRemoval: Boolean
+    val isEligibleForUserRemoval: Boolean = true,

    /**
     * Optional icon ID for showing an icon on the start of the candidate item. Mainly used for special suggestions
@ -77,29 +79,14 @@ interface SuggestionCandidate {
     * In the UI, if the ID is non-null, it will be shown to the start of the main label and scaled accordingly.
     * The color of the icon is entirely decided by the theme of the user. Icons that are monochrome work best.
     */
-    val iconId: Int?
+    @Transient
+    val iconId: Int? = null,

    /**
     * The source provider of this candidate. Is used for several callbacks for training, blacklisting of candidates on
     * user-request, and so on. If null, it means that the source provider is unknown or does not want to receive
     * callbacks.
     */
-    val sourceProvider: SuggestionProvider?
-}
-
-/**
- * Default implementation for a word candidate (autocorrect and next/current word suggestion).
- *
- * @see SuggestionCandidate
- */
-@Serializable
-data class WordSuggestionCandidate(
-    override val text: CharSequence,
-    override val secondaryText: CharSequence? = null,
-    override val confidence: Double = 0.0,
-    override val isEligibleForAutoCommit: Boolean = false,
-    override val isEligibleForUserRemoval: Boolean = true,
-) : SuggestionCandidate {
-    override val iconId: Int? = null
-    override var sourceProvider: SuggestionProvider? = null
-}
+    @Transient
+    val sourceProvider: SuggestionProvider? = null,
+)
--- a/plugin/src/main/kotlin/dev/patrickgold/florisboard/plugin/FlorisPluginService.kt
+++ b/plugin/src/main/kotlin/dev/patrickgold/florisboard/plugin/FlorisPluginService.kt
@ -36,6 +36,7 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.SerializationException
 import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.encodeToString
 import kotlinx.serialization.json.Json
 import java.lang.ref.WeakReference

@ -143,6 +144,25 @@ abstract class FlorisPluginService : Service(), NlpProvider {
                        if (service !is SuggestionProvider) {
                            error("This action can only be executed by a SuggestionProvider")
                        }
+                        val data = message.data ?: error("Request message contains no data")
+                        val id = message.id
+                        val replyToMessenger = message.replyTo ?: error("Request message contains no replyTo field")
+                        val suggestionRequest = Json.decodeFromString<SuggestionRequest>(data)
+                        service.scope.launch {
+                            flogDebug { "ACTION_SUGGEST: $suggestionRequest" }
+                            val candidatesList = service.suggest(
+                                suggestionRequest.subtypeId,
+                                suggestionRequest.word,
+                                suggestionRequest.prevWords,
+                                suggestionRequest.flags,
+                            )
+                            val responseMessage = FlorisPluginMessage.replyToConsumer(
+                                action = FlorisPluginMessage.ACTION_SUGGEST,
+                                id = id,
+                                data = Json.encodeToString(candidatesList),
+                            )
+                            replyToMessenger.send(responseMessage.toAndroidMessage())
+                        }
                    }
                }
            }