Delete swipe: exact visual lines via requestTextBoundsInfo (Android 14+), opt-in

app/src/main/java/helium314/keyboard/latin/LatinIME.kt

@@ -1045,6 +1045,12 @@ class LatinIME : InputMethodService(),
         keyboardSwitcher.deallocateMemory()
     }
 
+    override fun onUpdateCursorAnchorInfo(cursorAnchorInfo: android.view.inputmethod.CursorAnchorInfo) {
+        super.onUpdateCursorAnchorInfo(cursorAnchorInfo)
+        // only used by the delete swipe to know where the cursor line is, see KeyboardActionListenerImpl
+        inputLogic.connection.lastCursorAnchorInfo = cursorAnchorInfo
+    }
+
     override fun onUpdateSelection(
         oldSelStart: Int, oldSelEnd: Int,
         newSelStart: Int, newSelEnd: Int,

app/src/main/java/helium314/keyboard/latin/RichInputConnection.kt

@@ -4,9 +4,12 @@ import android.content.ClipData
 import android.content.ClipboardManager
 import android.content.Context
 import android.graphics.Color
+import android.graphics.RectF
 import android.inputmethodservice.InputMethodService
 import android.os.Build
 import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.os.SystemClock
 import android.text.SpannableString
 import android.text.SpannableStringBuilder
@@ -15,12 +18,16 @@ import android.text.TextUtils
 import android.text.style.BackgroundColorSpan
 import android.text.style.CharacterStyle
 import android.view.KeyEvent
+import android.view.inputmethod.CursorAnchorInfo
 import android.view.inputmethod.CompletionInfo
 import android.view.inputmethod.CorrectionInfo
 import android.view.inputmethod.ExtractedText
 import android.view.inputmethod.ExtractedTextRequest
 import android.view.inputmethod.InputConnection
 import android.view.inputmethod.InputMethodManager
+import android.view.inputmethod.TextBoundsInfo
+import android.view.inputmethod.TextBoundsInfoResult
+import androidx.annotation.RequiresApi
 import helium314.keyboard.compat.AppQuirksManager
 import helium314.keyboard.keyboard.KeyboardSwitcher
 import helium314.keyboard.latin.common.Constants
@@ -42,7 +49,9 @@ import helium314.keyboard.latin.utils.Log
 import helium314.keyboard.latin.utils.NgramContextUtils
 import helium314.keyboard.latin.utils.StatsUtils
 import helium314.keyboard.latin.utils.TextRange
+import java.util.concurrent.Executor
 import java.util.concurrent.TimeUnit
+import java.util.function.Consumer
 import kotlin.math.max
 
 /**
@@ -107,6 +116,7 @@ class RichInputConnection(private val mParent: InputMethodService) : PrivateComm
         }
         mIsActive = false
         mIC = null
+        lastCursorAnchorInfo = null
         mNestLevel = 0
         mPendingInFlightDeletions = 0
     }
@@ -1438,6 +1448,29 @@ class RichInputConnection(private val mParent: InputMethodService) : PrivateComm
 
     fun isCursorPositionKnown(): Boolean = INVALID_CURSOR_POSITION != mExpectedSelStart
 
+    /** Latest cursor anchor info the editor sent (needs [requestCursorUpdates] with monitoring), or null. */
+    @Volatile var lastCursorAnchorInfo: CursorAnchorInfo? = null
+
+    /**
+     * Asks the editor for the geometry of the characters inside [bounds] (editor coordinates), without touching
+     * the selection or sending key events. Android 14+ only; [callback] runs on the main thread with null if the
+     * editor doesn't support the request or failed. Returns false if the request could not be sent at all.
+     */
+    @RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE)
+    fun requestTextBoundsInfo(bounds: RectF, callback: (TextBoundsInfo?) -> Unit): Boolean {
+        val ic = mIC ?: return false
+        val mainHandler = Handler(Looper.getMainLooper())
+        return try {
+            ic.requestTextBoundsInfo(bounds, Executor { mainHandler.post(it) }, Consumer<TextBoundsInfoResult> { result ->
+                callback(if (result.resultCode == TextBoundsInfoResult.CODE_SUCCESS) result.textBoundsInfo else null)
+            })
+            true
+        } catch (e: Exception) {
+            Log.w(TAG, "requestTextBoundsInfo failed", e)
+            false
+        }
+    }
+
     fun requestCursorUpdates(enableMonitor: Boolean, requestImmediateCallback: Boolean): Boolean {
         mIC = mParent.currentInputConnection
 

app/src/main/java/helium314/keyboard/latin/settings/Defaults.kt

@@ -140,6 +140,7 @@ object Defaults {
     const val PREF_VERTICAL_SWIPE_THRESHOLD = 24
     const val PREF_DELETE_SWIPE = true
     const val PREF_DELETE_SWIPE_WORD_BY_WORD = false
+    const val PREF_DELETE_SWIPE_EXACT_LINES = false
     const val PREF_AUTOSPACE_AFTER_PUNCTUATION = false
     const val PREF_AUTOSPACE_AFTER_EMOJI = false
     const val PREF_AUTOSPACE_AFTER_SUGGESTION = true

app/src/main/java/helium314/keyboard/latin/settings/Settings.kt

@@ -396,6 +396,7 @@ class Settings private constructor() : SharedPreferences.OnSharedPreferenceChang
         const val PREF_VERTICAL_SWIPE_THRESHOLD = "vertical_swipe_threshold"
         const val PREF_DELETE_SWIPE = "delete_swipe"
         const val PREF_DELETE_SWIPE_WORD_BY_WORD = "delete_swipe_word_by_word"
+        const val PREF_DELETE_SWIPE_EXACT_LINES = "delete_swipe_exact_lines"
         const val PREF_FULL_KEYBOARD_SWIPE = "full_keyboard_swipe"
         const val PREF_FULL_SWIPE_SWITCH_LANGUAGE = "full_swipe_switch_language"
         const val PREF_AUTOSPACE_AFTER_PUNCTUATION = "autospace_after_punctuation"

app/src/main/java/helium314/keyboard/latin/settings/SettingsValues.kt

@@ -108,6 +108,7 @@ open class SettingsValues(
     val mClipboardSkipSensitive: Boolean
     val mDeleteSwipeEnabled: Boolean
     val mDeleteSwipeWordByWord: Boolean
+    val mDeleteSwipeExactLines: Boolean
     val mAutospaceAfterPunctuation: Boolean
     val mAutospaceAfterEmoji: Boolean
     val mAutospaceAfterSuggestion: Boolean
@@ -454,6 +455,7 @@ open class SettingsValues(
         mClipboardSkipSensitive = prefs.getBoolean(Settings.PREF_CLIPBOARD_SKIP_SENSITIVE, Defaults.PREF_CLIPBOARD_SKIP_SENSITIVE)
         mDeleteSwipeEnabled = prefs.getBoolean(Settings.PREF_DELETE_SWIPE, Defaults.PREF_DELETE_SWIPE)
         mDeleteSwipeWordByWord = prefs.getBoolean(Settings.PREF_DELETE_SWIPE_WORD_BY_WORD, Defaults.PREF_DELETE_SWIPE_WORD_BY_WORD)
+        mDeleteSwipeExactLines = prefs.getBoolean(Settings.PREF_DELETE_SWIPE_EXACT_LINES, Defaults.PREF_DELETE_SWIPE_EXACT_LINES)
         mAutospaceAfterPunctuation = prefs.getBoolean(Settings.PREF_AUTOSPACE_AFTER_PUNCTUATION, Defaults.PREF_AUTOSPACE_AFTER_PUNCTUATION)
         mAutospaceAfterEmoji = prefs.getBoolean(Settings.PREF_AUTOSPACE_AFTER_EMOJI, Defaults.PREF_AUTOSPACE_AFTER_EMOJI)
         mAutospaceAfterSuggestion = prefs.getBoolean(Settings.PREF_AUTOSPACE_AFTER_SUGGESTION, Defaults.PREF_AUTOSPACE_AFTER_SUGGESTION)

app/src/main/java/helium314/keyboard/latin/utils/VisualLines.kt

@@ -0,0 +1,64 @@
+// SPDX-License-Identifier: GPL-3.0-only
+package helium314.keyboard.latin.utils
+
+/**
+ * Helpers to turn the character geometry an editor reports (`InputConnection.requestTextBoundsInfo`, Android 14+)
+ * into the visual line boundaries the delete swipe uses. Kept free of Android classes so it can be unit tested.
+ */
+object VisualLines {
+    /**
+     * Offsets (absolute, ascending) of the first character of every visual line in a range of characters.
+     *
+     * [tops] and [bottoms] hold the vertical extent of each character, index 0 being the character at [start].
+     * A new line begins after a line feed, or when the vertical centre of a character lies outside the
+     * vertical extent of the line collected so far. The first entry is always [start], which is only a real
+     * line start if the range happens to begin at one (see [buildBoundaries]).
+     */
+    fun lineStarts(start: Int, tops: FloatArray, bottoms: FloatArray, lineFeed: BooleanArray): List<Int> {
+        val n = minOf(tops.size, bottoms.size, lineFeed.size)
+        if (n == 0) return emptyList()
+        val starts = mutableListOf(start)
+        var lineTop = tops[0]
+        var lineBottom = bottoms[0]
+        for (i in 1 until n) {
+            val centerY = (tops[i] + bottoms[i]) / 2
+            val newLine = lineFeed[i - 1] || centerY > lineBottom || centerY < lineTop
+            if (newLine) {
+                starts.add(start + i)
+                lineTop = tops[i]
+                lineBottom = bottoms[i]
+            } else {
+                lineTop = minOf(lineTop, tops[i])
+                lineBottom = maxOf(lineBottom, bottoms[i])
+            }
+        }
+        return starts
+    }
+
+    /**
+     * Boundaries for the delete swipe in the same layout as `getLineBoundariesBackwards`:
+     * index 0 = [baseStart], then the start of the line the selection begins in, then the lines above it.
+     *
+     * [exactStarts] are the line starts from [lineStarts]. If [firstIsRealLineStart] is false the range began in the
+     * middle of a line, so its first entry is not a real line start and is dropped. Lines older than the exact
+     * ones are continued from [approxBoundaries] (the estimate), so swiping further up than the editor was asked
+     * about still works. Entries below [minOffset] are not used (that is as far back as the swipe knows the text).
+     */
+    fun buildBoundaries(
+        baseStart: Int,
+        exactStarts: List<Int>,
+        firstIsRealLineStart: Boolean,
+        approxBoundaries: List<Int>,
+        minOffset: Int,
+    ): List<Int> {
+        val result = mutableListOf(baseStart)
+        val usable = if (firstIsRealLineStart) exactStarts else exactStarts.drop(1)
+        for (s in usable.asReversed()) {
+            if (s < result.last() && s >= minOffset) result.add(s)
+        }
+        for (b in approxBoundaries) {
+            if (b < result.last() && b >= minOffset) result.add(b)
+        }
+        return result
+    }
+}

app/src/main/java/helium314/keyboard/settings/screens/GestureTypingScreen.kt

@@ -105,6 +105,9 @@ fun GestureTypingScreen(
         add(Settings.PREF_TOOLBAR_SWIPE_DOWN_GESTURE)
         add(Settings.PREF_VERTICAL_SWIPE_THRESHOLD)
         add(Settings.PREF_DELETE_SWIPE)
+        // only useful where the editor can be asked for its lines (requestTextBoundsInfo, Android 14+)
+        if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.UPSIDE_DOWN_CAKE)
+            add(Settings.PREF_DELETE_SWIPE_EXACT_LINES)
 
         add(R.string.settings_category_touchpad)
         add(Settings.PREF_TOUCHPAD_SENSITIVITY)
@@ -356,6 +359,14 @@ fun createGestureTypingSettings(context: Context) = listOf(
             enabled = prefs.getBoolean(Settings.PREF_DELETE_SWIPE, Defaults.PREF_DELETE_SWIPE)
         )
     },
+    Setting(context, Settings.PREF_DELETE_SWIPE_EXACT_LINES, R.string.delete_swipe_exact_lines, R.string.delete_swipe_exact_lines_summary) {
+        val prefs = LocalContext.current.prefs()
+        SwitchPreference(
+            it,
+            Defaults.PREF_DELETE_SWIPE_EXACT_LINES,
+            enabled = prefs.getBoolean(Settings.PREF_DELETE_SWIPE, Defaults.PREF_DELETE_SWIPE)
+        )
+    },
 )
 
 @Preview

app/src/main/res/values/strings.xml

@@ -350,6 +350,10 @@
     <string name="delete_swipe_word_by_word">Delete word by word</string>
     <!-- Description for "delete_swipe_word_by_word" option. -->
     <string name="delete_swipe_word_by_word_summary">Select and delete whole words when swiping left from the delete key</string>
+    <!-- Preferences item for using the real visual lines when swiping up from the delete key (experimental) -->
+    <string name="delete_swipe_exact_lines">Exact lines when swiping up (experimental)</string>
+    <!-- Description for "delete_swipe_exact_lines" option. -->
+    <string name="delete_swipe_exact_lines_summary">Ask the text field where its lines break (Android 14 or newer, multi-line fields). If the field does not answer, lines are estimated as before</string>
     <!-- Preferences item and dialog title for backup and restore -->
     <string name="backup_restore_title">Backup and restore</string>
     <!-- Message for backup and restore dialog -->

app/src/test/java/helium314/keyboard/keyboard/WordDeleteSwipeTest.kt

@@ -3,7 +3,6 @@ package helium314.keyboard.keyboard
 
 import org.junit.Test
 import kotlin.test.assertEquals
-import kotlin.test.assertTrue
 
 class WordDeleteSwipeTest {
 
@@ -97,28 +96,6 @@ class WordDeleteSwipeTest {
         assertEquals(listOf(50, 44, 37), boundaries)
     }
 
-    @Test
-    fun `long line with newline exceeding 50 chars remains intact`() {
-        val text = "This is a line of text that has more than fifty characters in it without soft wrap\nsecond line"
-        val boundaries = KeyboardActionListenerImpl.getLineBoundariesBackwards(text, text.length)
-        val line2Start = text.indexOf("second line")
-        assertEquals(listOf(text.length, line2Start, 0), boundaries)
-        assertEquals("second line", text.substring(boundaries[1], boundaries[0]))
-        assertEquals("This is a line of text that has more than fifty characters in it without soft wrap", text.substring(boundaries[2], boundaries[1]).trim())
-    }
-
-    @Test
-    fun `multi sentence paragraph without newline splits at sentence boundaries`() {
-        val text = "First sentence of paragraph. Second sentence of paragraph is longer. Third sentence here."
-        val boundaries = KeyboardActionListenerImpl.getLineBoundariesBackwards(text, text.length)
-        for (i in 1 until boundaries.size) {
-            val start = boundaries[i]
-            assertTrue(start == 0 || Character.isWhitespace(text[start - 1]))
-        }
-        assertEquals(text.length, boundaries.first())
-        assertEquals(0, boundaries.last())
-    }
-
     @Test
     fun `diagonal swipe word boundaries before line start`() {
         val initialText = "first line\nsecond line\nthird line"

app/src/test/java/helium314/keyboard/latin/utils/VisualLinesTest.kt

@@ -0,0 +1,62 @@
+// SPDX-License-Identifier: GPL-3.0-only
+package helium314.keyboard.latin.utils
+
+import org.junit.Test
+import kotlin.test.assertEquals
+
+class VisualLinesTest {
+    // one entry per char, lines of the given lengths, 10px high, no linefeed flags
+    private fun geometry(vararg lineLengths: Int): Triple<FloatArray, FloatArray, BooleanArray> {
+        val tops = mutableListOf<Float>()
+        lineLengths.forEachIndexed { line, len -> repeat(len) { tops.add(line * 10f) } }
+        val t = tops.toFloatArray()
+        return Triple(t, FloatArray(t.size) { t[it] + 10f }, BooleanArray(t.size))
+    }
+
+    @Test
+    fun `soft wrapped lines are found by vertical position`() {
+        val (t, b, f) = geometry(5, 3, 4)
+        assertEquals(listOf(100, 105, 108), VisualLines.lineStarts(100, t, b, f))
+    }
+
+    @Test
+    fun `line feed starts a new line`() {
+        val t = floatArrayOf(0f, 0f, 0f, 10f, 10f)
+        val b = floatArrayOf(10f, 10f, 10f, 20f, 20f)
+        val f = booleanArrayOf(false, false, true, false, false)
+        assertEquals(listOf(0, 3), VisualLines.lineStarts(0, t, b, f))
+    }
+
+    @Test
+    fun `line feed flag alone is enough when geometry is identical`() {
+        val t = FloatArray(4)
+        val b = FloatArray(4) { 10f }
+        val f = booleanArrayOf(false, true, false, false)
+        assertEquals(listOf(0, 2), VisualLines.lineStarts(0, t, b, f))
+    }
+
+    @Test
+    fun `empty input gives no lines`() {
+        assertEquals(emptyList(), VisualLines.lineStarts(0, floatArrayOf(), floatArrayOf(), booleanArrayOf()))
+    }
+
+    @Test
+    fun `boundaries start at base and go upwards`() {
+        assertEquals(listOf(60, 45, 20, 0), VisualLines.buildBoundaries(60, listOf(0, 20, 45), true, emptyList(), 0))
+    }
+
+    @Test
+    fun `no duplicate when base is already a line start`() {
+        assertEquals(listOf(45, 20, 0), VisualLines.buildBoundaries(45, listOf(0, 20, 45), true, emptyList(), 0))
+    }
+
+    @Test
+    fun `partial first line is dropped and estimate continues`() {
+        assertEquals(listOf(60, 45, 25, 0), VisualLines.buildBoundaries(60, listOf(30, 45), false, listOf(60, 45, 25, 0), 0))
+    }
+
+    @Test
+    fun `entries before the known text are not used`() {
+        assertEquals(listOf(60, 45, 30), VisualLines.buildBoundaries(60, listOf(0, 30, 45), true, emptyList(), 30))
+    }
+}