Restore Android text selection and clipboard actions in Diary and Lumi

This commit is contained in:
waozi committed 2026-10-16 03:34:03 -03:00
1 parent 85e3b71d08
commit b342be2bcd
16 files changed
+748 -15

No files matched your search

+5
View File
@@ -1642,6 +1642,11 @@ test: raylib-text-input-zi-test
android-text-input-zi-test: $(ZI2C_BIN)
sh tests/android_text_input_zi_test.sh $(abspath $(dir $(ZI2C_BIN)))
.PHONY: android-text-selection-test
android-text-selection-test:
env -u DISPLAY -u WAYLAND_DISPLAY -u XAUTHORITY -u DBUS_SESSION_BUS_ADDRESS \
$(GRADLE) --no-daemon --project-dir tests/android-text-selection test
test: android-text-input-zi-test
.PHONY: main-platform-zi-test
+1
View File
@@ -793,6 +793,7 @@ LumiView :: (message: LumiMessage, commands: LumiCommands, revision: s64) -> Lum
.bounds = Bounds(left + Scale(14), top + text_top,
bubble_width - Scale(28), text_height),
.text = row.text, .font = frame.font,
.selectable = !selecting,
.wrap = cast(TextWrap)TextWrapAuto}
if !row.user {
formatted := ReplyText(index, row.text)
@@ -60,6 +60,7 @@ public class MainActivity extends NativeActivity {
private boolean firstFrameReady;
private long startupStarted;
private TextInputView textInputView;
private volatile TextSelectionMenu textSelectionMenu;
private boolean textInputVisible;
private volatile int pictureWindowState;
private volatile boolean pictureWindowEnabled;
@@ -448,6 +449,27 @@ public class MainActivity extends NativeActivity {
});
}
public void setTextSelectionMenu(long token, float x, float y, int flags) {
runOnUiThread(() -> {
if (textSelectionMenu != null && !isFinishing() && !isDestroyed()) {
textSelectionMenu.show(token, x, y, flags);
}
});
}
public int takeTextSelectionMenuAction(long token) {
TextSelectionMenu menu = textSelectionMenu;
return menu == null ? -1 : menu.take(token);
}
public byte[] readClipboardText() {
return TextClipboard.read(this);
}
public boolean writeClipboardText(byte[] text) {
return TextClipboard.write(this, text);
}
@Override
public boolean dispatchKeyEvent(KeyEvent event) {
if (textInputVisible && textInputView != null && textInputView.hasFocus()) {
@@ -1078,6 +1100,7 @@ public class MainActivity extends NativeActivity {
}
});
addContentView(textInputView, new ViewGroup.LayoutParams(1, 1));
textSelectionMenu = new TextSelectionMenu(getWindow().getDecorView());
showStartup();
synchronized (cachedInsets) {
@@ -0,0 +1,32 @@
package xyz.waozi.inbe;
import android.content.ClipData;
import android.content.ClipboardManager;
import android.content.Context;
import java.nio.charset.StandardCharsets;
/** Android clipboard transport; text commands and selection remain in Kryon. */
final class TextClipboard {
static byte[] read(Context context) {
ClipboardManager manager = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE);
if (manager == null) {
return null;
}
ClipData clip = manager.getPrimaryClip();
if (clip == null || clip.getItemCount() == 0) {
return new byte[0];
}
CharSequence text = clip.getItemAt(0).getText();
return text == null ? new byte[0] : text.toString().getBytes(StandardCharsets.UTF_8);
}
static boolean write(Context context, byte[] bytes) {
ClipboardManager manager = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE);
if (manager == null || bytes == null) {
return false;
}
manager.setPrimaryClip(ClipData.newPlainText(context.getApplicationInfo().loadLabel(
context.getPackageManager()), new String(bytes, StandardCharsets.UTF_8)));
return true;
}
}
@@ -0,0 +1,133 @@
package xyz.waozi.inbe;
import android.graphics.Rect;
import android.os.Build;
import android.view.ActionMode;
import android.view.Menu;
import android.view.MenuItem;
import android.view.View;
/** Presents Kryon's text commands using Android's localized selection toolbar. */
final class TextSelectionMenu {
private final View view;
private ActionMode mode;
private long shownToken;
private int flags;
private final Rect content = new Rect();
private volatile long resultToken;
private volatile int result;
TextSelectionMenu(View view) {
this.view = view;
}
synchronized int take(long token) {
return resultToken == token ? result : 0;
}
private synchronized void finish(long token, int command) {
resultToken = token;
result = command;
}
void show(long token, float x, float y, int flags) {
if (flags == 0) {
if (shownToken == token && mode != null) {
mode.finish();
}
return;
}
if (shownToken == token && resultToken == token && result != 0) {
return;
}
if (shownToken != token && mode != null) {
mode.finish();
}
shownToken = token;
boolean changed = this.flags != flags;
this.flags = flags;
content.set(Math.round(x), Math.round(y), Math.round(x) + 1, Math.round(y) + 1);
if (mode != null) {
if (changed) {
mode.invalidate();
}
if (Build.VERSION.SDK_INT >= 23) {
mode.invalidateContentRect();
}
return;
}
ActionMode.Callback callback;
if (Build.VERSION.SDK_INT >= 23) {
callback = new ActionMode.Callback2() {
@Override public boolean onCreateActionMode(ActionMode mode, Menu menu) {
return populate(menu);
}
@Override public boolean onPrepareActionMode(ActionMode mode, Menu menu) {
return populate(menu);
}
@Override public boolean onActionItemClicked(ActionMode mode, MenuItem item) {
return choose(mode, token, item.getItemId());
}
@Override public void onDestroyActionMode(ActionMode ended) {
dismissed(ended, token);
}
@Override public void onGetContentRect(ActionMode mode, View view, Rect out) {
out.set(content);
}
};
mode = view.startActionMode(callback, ActionMode.TYPE_FLOATING);
} else {
callback = new ActionMode.Callback() {
@Override public boolean onCreateActionMode(ActionMode mode, Menu menu) {
return populate(menu);
}
@Override public boolean onPrepareActionMode(ActionMode mode, Menu menu) {
return populate(menu);
}
@Override public boolean onActionItemClicked(ActionMode mode, MenuItem item) {
return choose(mode, token, item.getItemId());
}
@Override public void onDestroyActionMode(ActionMode ended) {
dismissed(ended, token);
}
};
mode = view.startActionMode(callback);
}
if (mode == null) {
finish(token, -1);
}
}
private boolean populate(Menu menu) {
menu.clear();
add(menu, 1, 1, android.R.string.cut);
add(menu, 2, 2, android.R.string.copy);
add(menu, 4, 3, android.R.string.paste);
add(menu, 8, 4, android.R.string.selectAll);
return menu.size() > 0;
}
private void add(Menu menu, int flag, int command, int title) {
if ((flags & flag) != 0) {
menu.add(Menu.NONE, command, command, title).setShowAsAction(MenuItem.SHOW_AS_ACTION_IF_ROOM);
}
}
private boolean choose(ActionMode mode, long token, int command) {
if (shownToken != token || command < 1 || command > 4 || (flags & (1 << (command - 1))) == 0) {
return false;
}
finish(token, command);
mode.finish();
return true;
}
private void dismissed(ActionMode ended, long token) {
if (mode == ended) {
mode = null;
}
if (shownToken == token && take(token) == 0) {
finish(token, -1);
}
}
}
+4
View File
@@ -27,6 +27,7 @@ PackageNow :: () -> s64 #foreign c "now_seconds";
#import, file "../platform/environment.zi";
#import "file_download"
#import "android_device"
#import "text_context_menu"
#import, file "../platform/android/android_activity.zi";
#import, file "../platform/android/graphics_context.zi";
#if #defined(PLATFORM_WEB) {
@@ -1593,6 +1594,9 @@ app_update_draw :: (vapp: *void, viewport: Rectangle) {
app.camera.zoom = render_scale
app.camera.offset.x = ifx (PreviewMode() != 0) then (0.0f) else (viewport.x + cast(float32)content_x)
app.camera.offset.y = ifx (PreviewMode() != 0) then (0.0f) else (viewport.y + cast(float32)top_height)
#if #defined(ANDROID_BUILD) {
TextMenuSetTransform(render_scale, app.camera.offset.x, app.camera.offset.y)
}
if PreviewMode() == 0 {
DirectDrawTransform(app.camera)
}
+11 -2
View File
@@ -1,3 +1,6 @@
#if #defined(ANDROID_BUILD) {
#import "android_clipboard"
}
#import "text_input"
#import "text_input_props"
#import, file "../app/text_buffers.zi";
@@ -72,6 +75,9 @@ TextInputCharacter :: () -> s32 {
#program_export
TextInputClipboardText :: () -> string {
#if #defined(ANDROID_BUILD) {
return AppTextFromCString(AndroidClipboardText(), 65536)
}
return AppTextFromCString(TextInputGetClipboardText(), 4096)
}
@@ -150,7 +156,7 @@ SampleTextFieldInput :: (focused: bool, typed: []u8) -> TextFieldInput {
input.cut = control && TextInputKeyPressed(TextInputKeyX)
input.paste = control && TextInputKeyPressed(TextInputKeyV)
if input.paste {
input.paste_text = AppTextFromCString(TextInputGetClipboardText(), 4096)
input.paste_text = TextInputClipboardText()
}
input.commit = TextInputKeyPressed(TextInputKeyEnter)
input.escape = TextInputKeyPressed(TextInputKeyEscape)
@@ -194,7 +200,7 @@ SampleTextAreaInput :: (focused: bool, typed: []u8) -> TextAreaInput {
input.cut = control && TextInputKeyPressed(TextInputKeyX)
input.paste = control && TextInputKeyPressed(TextInputKeyV)
if input.paste {
input.paste_text = AppTextFromCString(TextInputGetClipboardText(), 4096)
input.paste_text = TextInputClipboardText()
}
input.enter = TextInputKeyPressed(TextInputKeyEnter) && !control
input.commit = TextInputKeyPressed(TextInputKeyEnter) && control
@@ -204,6 +210,9 @@ SampleTextAreaInput :: (focused: bool, typed: []u8) -> TextAreaInput {
#program_export
TextInputWriteClipboard :: (value: string) -> bool {
#if #defined(ANDROID_BUILD) {
return AndroidClipboardWrite(value)
}
bytes: [4096]u8
if !AppCopyText(value, bytes[:]) { return false }
TextInputSetClipboardText(*bytes[0])
+5 -1
View File
@@ -781,6 +781,10 @@ LumiUiCall :: (context: *void, module: *u8, function: *u8,
}
if name == "WriteLumiUiHost" && count == 1 && WidgetValidLumiUiFrame(arguments[0]) {
frame := WidgetDecodeLumiUiFrame(arguments[0], *lumi_ui_codec)
// A cut's clipboard range can borrow lumi_input. Preserve it before
// replacing that buffer with the edited composer value.
clipboard := ifx frame.clipboard_write then PackageText(frame.clipboard_text, "") else ""
defer { PackageFree(clipboard) }
if frame.value.count >= LUMI_INPUT_MAX || !AppCopyText(frame.value, lumi_input[:]) {
return 0
}
@@ -795,7 +799,7 @@ LumiUiCall :: (context: *void, module: *u8, function: *u8,
lumi_ui.scroll = frame.scroll
lumi_ui.follow_end = frame.follow_end
if frame.clipboard_write {
unused TextInputWriteClipboard(frame.clipboard_text)
unused TextInputWriteClipboard(clipboard)
}
return 1
}
@@ -0,0 +1,158 @@
package xyz.waozi.inbe;
import static org.junit.Assert.*;
import android.content.Context;
import android.content.ClipData;
import android.content.ClipboardManager;
import android.graphics.Rect;
import android.net.Uri;
import android.view.ActionMode;
import android.view.Menu;
import android.view.MenuInflater;
import android.view.View;
import android.view.inputmethod.EditorInfo;
import android.view.inputmethod.InputConnection;
import android.widget.PopupMenu;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.RobolectricTestRunner;
import org.robolectric.RuntimeEnvironment;
import org.robolectric.annotation.Config;
import java.nio.charset.StandardCharsets;
@RunWith(RobolectricTestRunner.class)
@Config(sdk = {34, 36}, manifest = Config.NONE)
public class TextSelectionMenuTest {
private static final class Mode extends ActionMode {
final Callback callback;
final Menu menu;
final View view;
boolean finished;
Mode(View view, Callback callback) {
this.view = view;
this.callback = callback;
menu = new PopupMenu(view.getContext(), view).getMenu();
assertTrue(callback.onCreateActionMode(this, menu));
}
@Override public void setTitle(CharSequence title) {}
@Override public void setTitle(int title) {}
@Override public void setSubtitle(CharSequence subtitle) {}
@Override public void setSubtitle(int subtitle) {}
@Override public void setCustomView(View view) {}
@Override public void invalidate() { callback.onPrepareActionMode(this, menu); }
@Override public void finish() {
if (!finished) {
finished = true;
callback.onDestroyActionMode(this);
}
}
@Override public Menu getMenu() { return menu; }
@Override public MenuInflater getMenuInflater() { return new MenuInflater(view.getContext()); }
@Override public CharSequence getTitle() { return ""; }
@Override public CharSequence getSubtitle() { return ""; }
@Override public View getCustomView() { return null; }
}
private static final class ToolbarView extends View {
Mode last;
int count;
int type;
ToolbarView(Context context) { super(context); }
@Override public ActionMode startActionMode(ActionMode.Callback callback, int type) {
this.type = type;
count++;
last = new Mode(this, callback);
return last;
}
}
@Test public void floatingToolbarUsesPlatformLabelsAndPreservesChosenCommand() {
ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication());
TextSelectionMenu menu = new TextSelectionMenu(view);
menu.show(1, 120, 240, 15);
assertEquals(ActionMode.TYPE_FLOATING, view.type);
assertEquals(4, view.last.menu.size());
assertEquals(view.getContext().getString(android.R.string.copy), view.last.menu.findItem(2).getTitle());
Rect content = new Rect();
((ActionMode.Callback2)view.last.callback).onGetContentRect(view.last, view, content);
assertEquals(new Rect(120, 240, 121, 241), content);
assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(2)));
assertEquals(2, menu.take(1));
menu.show(1, 120, 240, 15);
assertEquals(1, view.count);
}
@Test public void chatToolbarHasOnlyCopyAndSelectAllAndRejectsStaleActions() {
ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication());
TextSelectionMenu menu = new TextSelectionMenu(view);
menu.show(1, 10, 20, 15);
Mode old = view.last;
menu.show(2, 20, 30, 10);
assertTrue(old.finished);
assertEquals(2, view.last.menu.size());
assertNull(view.last.menu.findItem(1));
assertNull(view.last.menu.findItem(3));
assertFalse(old.callback.onActionItemClicked(old, old.menu.findItem(2)));
assertEquals(0, menu.take(2));
assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(4)));
assertEquals(4, menu.take(2));
}
@Test public void dismissalDoesNotReopenUntilAnotherSelectionRequestsIt() {
ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication());
TextSelectionMenu menu = new TextSelectionMenu(view);
menu.show(1, 10, 20, 10);
view.last.finish();
assertEquals(-1, menu.take(1));
menu.show(1, 10, 20, 10);
assertEquals(1, view.count);
menu.show(2, 20, 30, 4);
assertEquals(2, view.count);
assertEquals(1, view.last.menu.size());
assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(3)));
assertEquals(3, menu.take(2));
}
@Test public void editingStillSupportsCompositionUnicodeAndStaleAcknowledgements() {
final String[] latest = {""};
final long[] revision = {0};
TextInputView editor = new TextInputView(RuntimeEnvironment.getApplication(), new TextInputView.Listener() {
@Override public void changed(long target, long next, byte[] text, int cursor, int anchor) {
latest[0] = new String(text, StandardCharsets.UTF_8);
revision[0] = next;
}
@Override public void enter() {}
});
editor.update(1, 0, "hello ".getBytes(StandardCharsets.UTF_8), 6, 6, false, false, true);
InputConnection connection = editor.onCreateInputConnection(new EditorInfo());
assertNotNull(connection);
assertTrue(connection.setComposingText("calm", 1));
assertTrue(connection.commitText("水😀", 1));
assertEquals("hello 水😀", latest[0]);
editor.update(1, 0, "old".getBytes(StandardCharsets.UTF_8), 3, 3, false, false, true);
assertEquals("hello 水😀", editor.getText().toString());
assertTrue(connection.setSelection(0, 5));
assertTrue(connection.commitText("fresh", 1));
assertEquals("fresh 水😀", latest[0]);
editor.update(2, 0, "saved".getBytes(StandardCharsets.UTF_8), 5, 5, false, false, true);
assertFalse(connection.commitText("late", 1));
assertEquals("saved", editor.getText().toString());
assertTrue(revision[0] > 0);
}
@Test public void systemClipboardPreservesUnicodeAndIgnoresNonTextClips() {
Context context = RuntimeEnvironment.getApplication();
byte[] text = "Diary\n水😀 é".getBytes(StandardCharsets.UTF_8);
assertTrue(TextClipboard.write(context, text));
assertArrayEquals(text, TextClipboard.read(context));
ClipboardManager clipboard = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE);
assertEquals("Diary\n水😀 é", clipboard.getPrimaryClip().getItemAt(0).getText().toString());
clipboard.setPrimaryClip(ClipData.newRawUri("image", Uri.parse("content://example/image")));
assertArrayEquals(new byte[0], TextClipboard.read(context));
assertTrue(TextClipboard.write(context, new byte[0]));
assertArrayEquals(new byte[0], TextClipboard.read(context));
assertFalse(TextClipboard.write(context, null));
}
}
+60
View File
@@ -0,0 +1,60 @@
plugins {
id 'java'
}
repositories {
google()
mavenCentral()
}
layout.buildDirectory = file('../../build/android-text-selection-test')
def sdkProperties = new Properties()
def sdkFile = file('../../droid/local.properties')
if (sdkFile.exists()) {
sdkFile.withInputStream { sdkProperties.load(it) }
}
def sdk = System.getenv('ANDROID_SDK_ROOT') ?: System.getenv('ANDROID_HOME') ?: sdkProperties.getProperty('sdk.dir')
if (!sdk) {
throw new GradleException('Set ANDROID_SDK_ROOT or droid/local.properties sdk.dir for Android text tests')
}
sourceSets {
main.java {
srcDir '../../droid/app/src/main/java'
include '**/TextSelectionMenu.java', '**/TextInputView.java', '**/TextClipboard.java'
}
test.java.srcDir '.'
}
dependencies {
compileOnly files("${sdk}/platforms/android-36/android.jar")
testImplementation files("${sdk}/platforms/android-36/android.jar")
testImplementation 'junit:junit:4.13.2'
testImplementation 'org.robolectric:robolectric:4.16'
}
// Robolectric's AndroidX dependencies contain classes.jar inside their AARs.
def aarDirectory = layout.buildDirectory.dir('aar-classes')
tasks.register('extractAndroidTestAars') {
inputs.files(configurations.testRuntimeClasspath)
outputs.dir(aarDirectory)
doLast {
aarDirectory.get().asFile.listFiles()?.each { it.setWritable(true) }
configurations.testRuntimeClasspath.files.findAll { it.name.endsWith('.aar') }.each { aar ->
copy {
from zipTree(aar)
include 'classes.jar'
into aarDirectory
rename 'classes.jar', "${aar.name}.jar"
fileMode 0644
}
}
}
}
test {
dependsOn 'extractAndroidTestAars'
classpath += files(aarDirectory.map { fileTree(it) { include '*.jar' } })
systemProperty 'robolectric.dependency.repo.url', 'https://repo.maven.apache.org/maven2'
testLogging {
events 'passed', 'failed', 'skipped'
exceptionFormat 'full'
}
}
@@ -0,0 +1 @@
rootProject.name = 'inbe-text-selection-test'
+1
View File
@@ -1,3 +1,4 @@
#import "android_text_input_host"
#import "android_device"
#import "android_text_editor"
#import, file "../src/platform/android/android_activity.zi";
+67
View File
@@ -0,0 +1,67 @@
// No graphics or desktop are used by the Android input transport test.
#import "drawing_props"
#import "geometry"
labels: s32;
#program_export
MeasureGlyphWidth :: (value: string, font: s32, typeface: string) -> s32 {
unused typeface
return (cast(s32)value.count) * font / 2
}
#program_export
MeasureGlyphLineHeight :: (font: s32, typeface: string) -> s32 {
unused typeface
return font
}
#program_export
RasterRoundedRectangle :: (bounds: Rectangle, radius: float32,
segments: s32, color: Color) {}
#program_export
RasterRoundedRectangleOutline :: (bounds: Rectangle, radius: float32,
segments: s32, width: float32, color: Color) {}
#program_export
RasterLine :: (bounds: Rectangle, color: Color) {}
#program_export
RasterText :: (value: string, x: s32, y: s32, font: s32, color: Color) {
labels += 1
}
#program_export
RasterTextClipped :: (value: string, x: s32, y: s32, font: s32,
color: Color, clip: Rectangle) {
labels += 1
}
#program_export
RasterImage :: (path: string, texture_id: u32, source: Rectangle,
destination: Rectangle, clip: Rectangle, origin: Vector2,
rotation: float32, radius: float32, tint: Color) {}
#program_export
RasterImageMesh :: (path: string, texture_id: u32, source: Rectangle,
points: []Vector2, columns: s32, rows: s32, clip: Rectangle,
tint: Color) {}
#program_export
ContextMenuLabelCount :: () -> s32 { return labels }
#program_export
SystemClipboardOpenHost :: () -> s32 {
return -1
}
#program_export
SystemClipboardByteHost :: (index: s32) -> s32 {
unused index
return -1
}
#program_export
SystemClipboardWriteHost :: (text: string) -> bool {
unused text
return false
}
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
"""Check selection on the dedicated Android 14 AVD, never a physical phone.
Uses a fresh debug-app profile. Run the AVD on private Xvfb with inherited
desktop variables scrubbed, then pass the current x86_64 debug APK.
"""
import argparse
import csv
import io
import json
from pathlib import Path
import re
import sqlite3
import subprocess
import time
from PIL import ImageOps
from android_diary_photo_device import APP_FILES, PACKAGE, ROOT, Device
TEXT = "Calm breathing keeps my diary clear."
class SelectionDevice(Device):
def prepare(self, apk, dark=0):
if apk:
self.adb("install", "-r", "-t", apk)
self.shell("pm", "clear", PACKAGE)
self.shell("pm", "grant", PACKAGE, "android.permission.POST_NOTIFICATIONS")
self.shell("input", "keyevent", "KEYCODE_WAKEUP")
self.shell("wm", "dismiss-keyguard")
self.start()
owner = self.settings(enabled_apps=31, main_tab=3, language="en", language_setup_done=1,
apps_setup_done=1, launcher_guide_seen=1, tutorial_seen=1,
lumi_guides_seen=127, lumi_online=0, lumi_archive_migrated=1,
package_updates_allowed=0, theme=7, theme_source=0,
theme_mode=2 if dark else 1, dark_mode=dark, ui_scale=10,
**{"cell.lumi.message.selection-fixture": json.dumps(dict(
time=1, user=0, kind=0, payload="", text="Gentle breathing brings calm."))})
day = self.shell("date", "+%F")
assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", day)
directory = APP_FILES + "/diary/" + owner
self.shell("run-as", PACKAGE, "mkdir", "-p", directory)
self.entry_path = directory + "/" + day + ".json"
fixture = self.output / "entry.json"
fixture.write_text(json.dumps(dict(version=1, date=day, text=TEXT, photos=[])))
self.put_private(fixture, self.entry_path)
self.output.joinpath("context.json").write_text(json.dumps(dict(entry=self.entry_path)))
self.start()
self.snapshot("prepared")
def word(self, name, target, minimum_y=0):
path = self.output / (name + ".png")
assert self.snapshot(name).size == (480, 960)
for mode in (11, 6):
output = subprocess.run(["tesseract", str(path), "stdout", "--psm", str(mode),
"-c", "tessedit_create_tsv=1"],
capture_output=True, text=True, check=True, timeout=20).stdout
words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t")
if row["text"].strip(".,:").lstrip("I|").lower() == target.split()[0].lower()
and int(row["top"]) >= minimum_y
and int(row["top"]) < (960 if target == "Message" else 600)]
if len(words) == 1:
row = words[0]
return int(row["left"]) + int(row["width"]) // 2, int(row["top"]) + int(row["height"]) // 2
raise AssertionError("Expected one rendered word: " + target)
def hold_word(self, name, word):
x, y = self.word(name, word)
self.shell("input", "swipe", x, y, x, y, 900)
time.sleep(0.5)
self.snapshot(name + "-selected")
def menu(self, command):
matches = [node for node in self.nodes() if node.attrib.get("text") == command
or node.attrib.get("content-desc") == command]
if matches:
assert len(matches) == 1
self.tap_node(matches[0])
else:
# NativeActivity's floating popup is rendered but not exposed by
# UIAutomator on this image. OCR its largest connected background.
screen = self.snapshot("menu-" + command.replace(" ", "-"))
pixels = screen.load()
remaining = {(x, y) for y in range(30, 600) for x in range(480)
if pixels[x, y] == (66, 66, 66)}
largest = set()
while remaining:
todo = [remaining.pop()]
component = set(todo)
while todo:
x, y = todo.pop()
for point in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)):
if point in remaining:
remaining.remove(point)
component.add(point)
todo.append(point)
if len(component) > len(largest):
largest = component
assert len(largest) > 1000, "Native toolbar did not render"
left = min(x for x, y in largest)
top = min(y for x, y in largest)
right = max(x for x, y in largest) + 1
bottom = max(y for x, y in largest) + 1
crop = ImageOps.invert(screen.crop((left, top, right, bottom)))
path = self.output / "toolbar-ocr.png"
crop.resize((crop.width * 3, crop.height * 3)).save(path)
output = subprocess.run(["tesseract", str(path), "stdout", "--psm", "6",
"-c", "tessedit_create_tsv=1"], capture_output=True,
text=True, check=True, timeout=60).stdout
words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t")
if row["text"].strip("|.,:").lower() == command.split()[0].lower()]
assert len(words) == 1, (command, output)
row = words[0]
x = left + (int(row["left"]) + int(row["width"]) // 2) // 3
y = top + (int(row["top"]) + int(row["height"]) // 2) // 3
self.shell("input", "tap", x, y)
time.sleep(0.5)
def text(self):
return json.loads(self.private_bytes(self.entry_path))["text"]
def check(self, apk):
for dark in (0, 1):
name = "sage-dark" if dark else "sage-light"
# Give each theme its own profile. After startup, navigate in the
# actual UI rather than rewriting a live profile's SQLite records.
self.prepare(apk, dark)
self.hold_word(name + "-diary-copy", "breathing")
self.menu("Copy")
self.hold_word(name + "-diary-replace", "Calm")
self.menu("Paste")
expected = "breathing breathing keeps my diary clear."
self.wait(lambda: self.text() == expected, "Diary paste saved")
# A real IME key, rather than adb's synthetic hardware typing.
self.shell("input", "tap", 240, 807)
self.wait(lambda: self.text() != expected, "continued IME typing")
typed = self.text()
assert typed in ("breathingg breathing keeps my diary clear.",
"breathingG breathing keeps my diary clear."), typed
self.snapshot(name + "-diary-paste-typed")
self.stop()
self.start()
assert self.text() == typed
self.snapshot(name + "-diary-restart")
self.shell("input", "tap", 66, 929)
time.sleep(2)
self.hold_word(name + "-chat-copy", "Gentle")
self.menu("Copy")
# Empty composer must still offer Paste on long-press.
time.sleep(6) # Let Android's clipboard preview stop covering it.
x, y = self.word(name + "-composer-empty", "Message")
self.shell("input", "tap", x, y)
time.sleep(2)
assert self.snapshot(name + "-composer-focused").size == (480, 960)
# The open IME moves this fixture's composer to y=575 in both
# themes. The dark placeholder has intentionally low contrast.
self.shell("input", "swipe", 70, 575, 70, 575, 900)
time.sleep(1)
self.menu("Paste")
time.sleep(2)
self.snapshot(name + "-chat-pasted")
self.shell("input", "tap", 240, 807)
time.sleep(2)
self.snapshot(name + "-chat-typed")
self.shell("input", "swipe", 45, 570, 45, 570, 900)
time.sleep(1)
self.menu("Cut")
time.sleep(2)
self.snapshot(name + "-composer-cut")
# Verify the exact composer text through the OS clipboard and
# persisted Diary data; caret blinking can confuse screenshot OCR.
self.shell("input", "tap", 414, 645) # Apps, above the open IME.
time.sleep(2)
x, y = self.word(name + "-apps", "Diary")
self.shell("input", "tap", x, y)
time.sleep(2)
self.hold_word(name + "-composer-roundtrip", "keeps")
self.menu("Paste")
expected = typed.replace("keeps", "Gentleg")
self.wait(lambda: self.text() == expected, "composer clipboard round trip")
self.stop()
database_path = self.output / "checked.db"
database_path.write_bytes(self.private_bytes(APP_FILES + "/inbe.db"))
for suffix in ("-wal", "-shm"):
Path(str(database_path) + suffix).unlink(missing_ok=True)
wal = self.probe("run-as", PACKAGE, "ls", APP_FILES + "/inbe.db-wal")
if wal:
Path(str(database_path) + "-wal").write_bytes(self.private_bytes(APP_FILES + "/inbe.db-wal"))
with sqlite3.connect(database_path) as database:
keys = database.execute("SELECT key FROM settings WHERE key LIKE 'cell.lumi.message.%'").fetchall()
assert keys == [("cell.lumi.message.selection-fixture",)], "The check must never send a message"
print(name + ": Diary copy/paste/IME typing/save/restart and offline chat word copy/composer paste passed",
flush=True)
self.output.joinpath("result.json").write_text(json.dumps(dict(
android_sdk=34, avd="inbe-diary-photo", scale=10,
themes=["sage_light", "sage_dark"],
passed=["long press", "native toolbar", "Diary copy/paste", "IME typing",
"Diary save/restart", "chat word copy", "unsent composer paste",
"composer IME typing/cut and exact clipboard round trip"],
status="Local debug APK on a private emulator; no physical phone, send, push or release"), indent=2) + "\n")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("prepare", "check", "test", "snapshot"))
parser.add_argument("--adb", default="/home/wao/Android/Sdk/platform-tools/adb")
parser.add_argument("--serial", default="emulator-5578")
parser.add_argument("--apk", type=Path)
parser.add_argument("--output", type=Path, default=ROOT / "build/android-text-selection-device")
args = parser.parse_args()
device = SelectionDevice(args.adb, args.serial, args.output)
try:
if args.action in ("prepare", "test"):
if not args.apk:
parser.error("prepare requires --apk")
if args.action == "prepare":
device.prepare(args.apk)
if args.action in ("check", "test"):
device.check(args.apk)
if args.action == "snapshot":
device.snapshot("current")
except Exception:
device.snapshot("failure")
raise
finally:
if args.action == "test":
device.stop()
if __name__ == "__main__":
main()
+1
View File
@@ -17,6 +17,7 @@ mkdir -p "$work/c"
--module-path "$root/build/packages/kss/src" --module-path "oqs=$root/build/packages/oqs/src" --module-path "kryon=$root/build/packages/kryon/src/ui" \
--module-path "$root/build/packages/kryon/src/backend" \
--module-path "$root/build/packages/game2d/src" \
--module-path "qrcode=$root/build/packages/qrcode/src" \
--module-path "$root/build/packages/daochi-client" \
-o "$work/c" "$root/tests/sync_recovery_behavior.zi"
"${CC:-cc}" -std=gnu11 -w -DZIRAN_BOUNDS_CHECK -ffunction-sections \
+12 -12
View File
@@ -53,17 +53,9 @@
"source": true,
"url": "https://github.com/raysan5/raylib.git"
},
{
"commit": "6a8ca21ab165a64af425da68944ead083e2ed97d",
"dependencies": {"raylib": "p5a77c9c35110e925"},
"id": "p701ca29a75e15af6",
"name": "kryon",
"ref": "master",
"url": "https://github.com/kryonlabs/kryon.git"
},
{
"commit": "41538659afbbda300915ef009b55c18f6de159b8",
"dependencies": {"kryon": "p701ca29a75e15af6"},
"dependencies": {"kryon": "pe4492807d82c43f5"},
"id": "p9504fb8e5fabefc6",
"name": "Kss",
"ref": "master",
@@ -79,7 +71,7 @@
},
{
"commit": "2ee29338c9b93fb4f4a56a49c7be09e9d2c2418f",
"dependencies": {"kryon": "p701ca29a75e15af6"},
"dependencies": {"kryon": "pe4492807d82c43f5"},
"id": "pa5dd7b8b83307b18",
"name": "plot",
"ref": "master",
@@ -87,7 +79,7 @@
},
{
"commit": "7aa01465ac3f988e0404f3c6dc244bf4b009f253",
"dependencies": {"kryon": "p701ca29a75e15af6"},
"dependencies": {"kryon": "pe4492807d82c43f5"},
"id": "paa08e1730a25bda0",
"name": "Game2D",
"ref": "master",
@@ -102,6 +94,14 @@
"source": true,
"url": "https://github.com/curl/curl.git"
},
{
"commit": "556ee25f11f2e6b979150acb94e3093a137ed47e",
"dependencies": {"raylib": "p5a77c9c35110e925"},
"id": "pe4492807d82c43f5",
"name": "kryon",
"ref": "master",
"url": "https://github.com/kryonlabs/kryon.git"
},
{
"commit": "51fa84aedaa6ae4f6734e0fcca9bfc35a184d297",
"dependencies": {},
@@ -121,7 +121,7 @@
}
],
"root": {
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "p701ca29a75e15af6", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pe4492807d82c43f5", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
"name": "Inbe"
},
"toolchain": {