diff --git a/Makefile b/Makefile index 926655166..8ef15368d 100644 --- a/Makefile +++ b/Makefile @@ -1642,6 +1642,11 @@ test: raylib-text-input-zi-test android-text-input-zi-test: $(ZI2C_BIN) sh tests/android_text_input_zi_test.sh $(abspath $(dir $(ZI2C_BIN))) +.PHONY: android-text-selection-test +android-text-selection-test: + env -u DISPLAY -u WAYLAND_DISPLAY -u XAUTHORITY -u DBUS_SESSION_BUS_ADDRESS \ + $(GRADLE) --no-daemon --project-dir tests/android-text-selection test + test: android-text-input-zi-test .PHONY: main-platform-zi-test diff --git a/apps/lumi/lumi_view.zi b/apps/lumi/lumi_view.zi index eefa088ee..a18303591 100644 --- a/apps/lumi/lumi_view.zi +++ b/apps/lumi/lumi_view.zi @@ -793,6 +793,7 @@ LumiView :: (message: LumiMessage, commands: LumiCommands, revision: s64) -> Lum .bounds = Bounds(left + Scale(14), top + text_top, bubble_width - Scale(28), text_height), .text = row.text, .font = frame.font, + .selectable = !selecting, .wrap = cast(TextWrap)TextWrapAuto} if !row.user { formatted := ReplyText(index, row.text) diff --git a/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java b/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java index c8f57477e..95e7d998c 100644 --- a/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java +++ b/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java @@ -60,6 +60,7 @@ public class MainActivity extends NativeActivity { private boolean firstFrameReady; private long startupStarted; private TextInputView textInputView; + private volatile TextSelectionMenu textSelectionMenu; private boolean textInputVisible; private volatile int pictureWindowState; private volatile boolean pictureWindowEnabled; @@ -448,6 +449,27 @@ public class MainActivity extends NativeActivity { }); } + public void setTextSelectionMenu(long token, float x, float y, int flags) { + runOnUiThread(() -> { + if (textSelectionMenu != null && !isFinishing() && !isDestroyed()) { + textSelectionMenu.show(token, x, y, flags); + } + }); + } + + public int takeTextSelectionMenuAction(long token) { + TextSelectionMenu menu = textSelectionMenu; + return menu == null ? -1 : menu.take(token); + } + + public byte[] readClipboardText() { + return TextClipboard.read(this); + } + + public boolean writeClipboardText(byte[] text) { + return TextClipboard.write(this, text); + } + @Override public boolean dispatchKeyEvent(KeyEvent event) { if (textInputVisible && textInputView != null && textInputView.hasFocus()) { @@ -1078,6 +1100,7 @@ public class MainActivity extends NativeActivity { } }); addContentView(textInputView, new ViewGroup.LayoutParams(1, 1)); + textSelectionMenu = new TextSelectionMenu(getWindow().getDecorView()); showStartup(); synchronized (cachedInsets) { diff --git a/droid/app/src/main/java/xyz/waozi/inbe/TextClipboard.java b/droid/app/src/main/java/xyz/waozi/inbe/TextClipboard.java new file mode 100644 index 000000000..4644cef0b --- /dev/null +++ b/droid/app/src/main/java/xyz/waozi/inbe/TextClipboard.java @@ -0,0 +1,32 @@ +package xyz.waozi.inbe; + +import android.content.ClipData; +import android.content.ClipboardManager; +import android.content.Context; +import java.nio.charset.StandardCharsets; + +/** Android clipboard transport; text commands and selection remain in Kryon. */ +final class TextClipboard { + static byte[] read(Context context) { + ClipboardManager manager = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE); + if (manager == null) { + return null; + } + ClipData clip = manager.getPrimaryClip(); + if (clip == null || clip.getItemCount() == 0) { + return new byte[0]; + } + CharSequence text = clip.getItemAt(0).getText(); + return text == null ? new byte[0] : text.toString().getBytes(StandardCharsets.UTF_8); + } + + static boolean write(Context context, byte[] bytes) { + ClipboardManager manager = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE); + if (manager == null || bytes == null) { + return false; + } + manager.setPrimaryClip(ClipData.newPlainText(context.getApplicationInfo().loadLabel( + context.getPackageManager()), new String(bytes, StandardCharsets.UTF_8))); + return true; + } +} diff --git a/droid/app/src/main/java/xyz/waozi/inbe/TextSelectionMenu.java b/droid/app/src/main/java/xyz/waozi/inbe/TextSelectionMenu.java new file mode 100644 index 000000000..d0a588a5b --- /dev/null +++ b/droid/app/src/main/java/xyz/waozi/inbe/TextSelectionMenu.java @@ -0,0 +1,133 @@ +package xyz.waozi.inbe; + +import android.graphics.Rect; +import android.os.Build; +import android.view.ActionMode; +import android.view.Menu; +import android.view.MenuItem; +import android.view.View; + +/** Presents Kryon's text commands using Android's localized selection toolbar. */ +final class TextSelectionMenu { + private final View view; + private ActionMode mode; + private long shownToken; + private int flags; + private final Rect content = new Rect(); + private volatile long resultToken; + private volatile int result; + + TextSelectionMenu(View view) { + this.view = view; + } + + synchronized int take(long token) { + return resultToken == token ? result : 0; + } + + private synchronized void finish(long token, int command) { + resultToken = token; + result = command; + } + + void show(long token, float x, float y, int flags) { + if (flags == 0) { + if (shownToken == token && mode != null) { + mode.finish(); + } + return; + } + if (shownToken == token && resultToken == token && result != 0) { + return; + } + if (shownToken != token && mode != null) { + mode.finish(); + } + shownToken = token; + boolean changed = this.flags != flags; + this.flags = flags; + content.set(Math.round(x), Math.round(y), Math.round(x) + 1, Math.round(y) + 1); + if (mode != null) { + if (changed) { + mode.invalidate(); + } + if (Build.VERSION.SDK_INT >= 23) { + mode.invalidateContentRect(); + } + return; + } + ActionMode.Callback callback; + if (Build.VERSION.SDK_INT >= 23) { + callback = new ActionMode.Callback2() { + @Override public boolean onCreateActionMode(ActionMode mode, Menu menu) { + return populate(menu); + } + @Override public boolean onPrepareActionMode(ActionMode mode, Menu menu) { + return populate(menu); + } + @Override public boolean onActionItemClicked(ActionMode mode, MenuItem item) { + return choose(mode, token, item.getItemId()); + } + @Override public void onDestroyActionMode(ActionMode ended) { + dismissed(ended, token); + } + @Override public void onGetContentRect(ActionMode mode, View view, Rect out) { + out.set(content); + } + }; + mode = view.startActionMode(callback, ActionMode.TYPE_FLOATING); + } else { + callback = new ActionMode.Callback() { + @Override public boolean onCreateActionMode(ActionMode mode, Menu menu) { + return populate(menu); + } + @Override public boolean onPrepareActionMode(ActionMode mode, Menu menu) { + return populate(menu); + } + @Override public boolean onActionItemClicked(ActionMode mode, MenuItem item) { + return choose(mode, token, item.getItemId()); + } + @Override public void onDestroyActionMode(ActionMode ended) { + dismissed(ended, token); + } + }; + mode = view.startActionMode(callback); + } + if (mode == null) { + finish(token, -1); + } + } + + private boolean populate(Menu menu) { + menu.clear(); + add(menu, 1, 1, android.R.string.cut); + add(menu, 2, 2, android.R.string.copy); + add(menu, 4, 3, android.R.string.paste); + add(menu, 8, 4, android.R.string.selectAll); + return menu.size() > 0; + } + + private void add(Menu menu, int flag, int command, int title) { + if ((flags & flag) != 0) { + menu.add(Menu.NONE, command, command, title).setShowAsAction(MenuItem.SHOW_AS_ACTION_IF_ROOM); + } + } + + private boolean choose(ActionMode mode, long token, int command) { + if (shownToken != token || command < 1 || command > 4 || (flags & (1 << (command - 1))) == 0) { + return false; + } + finish(token, command); + mode.finish(); + return true; + } + + private void dismissed(ActionMode ended, long token) { + if (mode == ended) { + mode = null; + } + if (shownToken == token && take(token) == 0) { + finish(token, -1); + } + } +} diff --git a/src/app/application.zi b/src/app/application.zi index 030985dbf..8dfeaa775 100644 --- a/src/app/application.zi +++ b/src/app/application.zi @@ -27,6 +27,7 @@ PackageNow :: () -> s64 #foreign c "now_seconds"; #import, file "../platform/environment.zi"; #import "file_download" #import "android_device" +#import "text_context_menu" #import, file "../platform/android/android_activity.zi"; #import, file "../platform/android/graphics_context.zi"; #if #defined(PLATFORM_WEB) { @@ -1593,6 +1594,9 @@ app_update_draw :: (vapp: *void, viewport: Rectangle) { app.camera.zoom = render_scale app.camera.offset.x = ifx (PreviewMode() != 0) then (0.0f) else (viewport.x + cast(float32)content_x) app.camera.offset.y = ifx (PreviewMode() != 0) then (0.0f) else (viewport.y + cast(float32)top_height) + #if #defined(ANDROID_BUILD) { + TextMenuSetTransform(render_scale, app.camera.offset.x, app.camera.offset.y) + } if PreviewMode() == 0 { DirectDrawTransform(app.camera) } diff --git a/src/platform/raylib_text_input.zi b/src/platform/raylib_text_input.zi index 664f00347..f43da476c 100644 --- a/src/platform/raylib_text_input.zi +++ b/src/platform/raylib_text_input.zi @@ -1,3 +1,6 @@ +#if #defined(ANDROID_BUILD) { + #import "android_clipboard" +} #import "text_input" #import "text_input_props" #import, file "../app/text_buffers.zi"; @@ -72,6 +75,9 @@ TextInputCharacter :: () -> s32 { #program_export TextInputClipboardText :: () -> string { + #if #defined(ANDROID_BUILD) { + return AppTextFromCString(AndroidClipboardText(), 65536) + } return AppTextFromCString(TextInputGetClipboardText(), 4096) } @@ -150,7 +156,7 @@ SampleTextFieldInput :: (focused: bool, typed: []u8) -> TextFieldInput { input.cut = control && TextInputKeyPressed(TextInputKeyX) input.paste = control && TextInputKeyPressed(TextInputKeyV) if input.paste { - input.paste_text = AppTextFromCString(TextInputGetClipboardText(), 4096) + input.paste_text = TextInputClipboardText() } input.commit = TextInputKeyPressed(TextInputKeyEnter) input.escape = TextInputKeyPressed(TextInputKeyEscape) @@ -194,7 +200,7 @@ SampleTextAreaInput :: (focused: bool, typed: []u8) -> TextAreaInput { input.cut = control && TextInputKeyPressed(TextInputKeyX) input.paste = control && TextInputKeyPressed(TextInputKeyV) if input.paste { - input.paste_text = AppTextFromCString(TextInputGetClipboardText(), 4096) + input.paste_text = TextInputClipboardText() } input.enter = TextInputKeyPressed(TextInputKeyEnter) && !control input.commit = TextInputKeyPressed(TextInputKeyEnter) && control @@ -204,6 +210,9 @@ SampleTextAreaInput :: (focused: bool, typed: []u8) -> TextAreaInput { #program_export TextInputWriteClipboard :: (value: string) -> bool { + #if #defined(ANDROID_BUILD) { + return AndroidClipboardWrite(value) + } bytes: [4096]u8 if !AppCopyText(value, bytes[:]) { return false } TextInputSetClipboardText(*bytes[0]) diff --git a/src/screens/lumi_screen.zi b/src/screens/lumi_screen.zi index 1a002cc57..84aaa4abd 100644 --- a/src/screens/lumi_screen.zi +++ b/src/screens/lumi_screen.zi @@ -781,6 +781,10 @@ LumiUiCall :: (context: *void, module: *u8, function: *u8, } if name == "WriteLumiUiHost" && count == 1 && WidgetValidLumiUiFrame(arguments[0]) { frame := WidgetDecodeLumiUiFrame(arguments[0], *lumi_ui_codec) + // A cut's clipboard range can borrow lumi_input. Preserve it before + // replacing that buffer with the edited composer value. + clipboard := ifx frame.clipboard_write then PackageText(frame.clipboard_text, "") else "" + defer { PackageFree(clipboard) } if frame.value.count >= LUMI_INPUT_MAX || !AppCopyText(frame.value, lumi_input[:]) { return 0 } @@ -795,7 +799,7 @@ LumiUiCall :: (context: *void, module: *u8, function: *u8, lumi_ui.scroll = frame.scroll lumi_ui.follow_end = frame.follow_end if frame.clipboard_write { - unused TextInputWriteClipboard(frame.clipboard_text) + unused TextInputWriteClipboard(clipboard) } return 1 } diff --git a/tests/android-text-selection/TextSelectionMenuTest.java b/tests/android-text-selection/TextSelectionMenuTest.java new file mode 100644 index 000000000..65ed89ded --- /dev/null +++ b/tests/android-text-selection/TextSelectionMenuTest.java @@ -0,0 +1,158 @@ +package xyz.waozi.inbe; + +import static org.junit.Assert.*; + +import android.content.Context; +import android.content.ClipData; +import android.content.ClipboardManager; +import android.graphics.Rect; +import android.net.Uri; +import android.view.ActionMode; +import android.view.Menu; +import android.view.MenuInflater; +import android.view.View; +import android.view.inputmethod.EditorInfo; +import android.view.inputmethod.InputConnection; +import android.widget.PopupMenu; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.RuntimeEnvironment; +import org.robolectric.annotation.Config; +import java.nio.charset.StandardCharsets; + +@RunWith(RobolectricTestRunner.class) +@Config(sdk = {34, 36}, manifest = Config.NONE) +public class TextSelectionMenuTest { + private static final class Mode extends ActionMode { + final Callback callback; + final Menu menu; + final View view; + boolean finished; + + Mode(View view, Callback callback) { + this.view = view; + this.callback = callback; + menu = new PopupMenu(view.getContext(), view).getMenu(); + assertTrue(callback.onCreateActionMode(this, menu)); + } + @Override public void setTitle(CharSequence title) {} + @Override public void setTitle(int title) {} + @Override public void setSubtitle(CharSequence subtitle) {} + @Override public void setSubtitle(int subtitle) {} + @Override public void setCustomView(View view) {} + @Override public void invalidate() { callback.onPrepareActionMode(this, menu); } + @Override public void finish() { + if (!finished) { + finished = true; + callback.onDestroyActionMode(this); + } + } + @Override public Menu getMenu() { return menu; } + @Override public MenuInflater getMenuInflater() { return new MenuInflater(view.getContext()); } + @Override public CharSequence getTitle() { return ""; } + @Override public CharSequence getSubtitle() { return ""; } + @Override public View getCustomView() { return null; } + } + + private static final class ToolbarView extends View { + Mode last; + int count; + int type; + ToolbarView(Context context) { super(context); } + @Override public ActionMode startActionMode(ActionMode.Callback callback, int type) { + this.type = type; + count++; + last = new Mode(this, callback); + return last; + } + } + + @Test public void floatingToolbarUsesPlatformLabelsAndPreservesChosenCommand() { + ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication()); + TextSelectionMenu menu = new TextSelectionMenu(view); + menu.show(1, 120, 240, 15); + assertEquals(ActionMode.TYPE_FLOATING, view.type); + assertEquals(4, view.last.menu.size()); + assertEquals(view.getContext().getString(android.R.string.copy), view.last.menu.findItem(2).getTitle()); + Rect content = new Rect(); + ((ActionMode.Callback2)view.last.callback).onGetContentRect(view.last, view, content); + assertEquals(new Rect(120, 240, 121, 241), content); + assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(2))); + assertEquals(2, menu.take(1)); + menu.show(1, 120, 240, 15); + assertEquals(1, view.count); + } + + @Test public void chatToolbarHasOnlyCopyAndSelectAllAndRejectsStaleActions() { + ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication()); + TextSelectionMenu menu = new TextSelectionMenu(view); + menu.show(1, 10, 20, 15); + Mode old = view.last; + menu.show(2, 20, 30, 10); + assertTrue(old.finished); + assertEquals(2, view.last.menu.size()); + assertNull(view.last.menu.findItem(1)); + assertNull(view.last.menu.findItem(3)); + assertFalse(old.callback.onActionItemClicked(old, old.menu.findItem(2))); + assertEquals(0, menu.take(2)); + assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(4))); + assertEquals(4, menu.take(2)); + } + + @Test public void dismissalDoesNotReopenUntilAnotherSelectionRequestsIt() { + ToolbarView view = new ToolbarView(RuntimeEnvironment.getApplication()); + TextSelectionMenu menu = new TextSelectionMenu(view); + menu.show(1, 10, 20, 10); + view.last.finish(); + assertEquals(-1, menu.take(1)); + menu.show(1, 10, 20, 10); + assertEquals(1, view.count); + menu.show(2, 20, 30, 4); + assertEquals(2, view.count); + assertEquals(1, view.last.menu.size()); + assertTrue(view.last.callback.onActionItemClicked(view.last, view.last.menu.findItem(3))); + assertEquals(3, menu.take(2)); + } + + @Test public void editingStillSupportsCompositionUnicodeAndStaleAcknowledgements() { + final String[] latest = {""}; + final long[] revision = {0}; + TextInputView editor = new TextInputView(RuntimeEnvironment.getApplication(), new TextInputView.Listener() { + @Override public void changed(long target, long next, byte[] text, int cursor, int anchor) { + latest[0] = new String(text, StandardCharsets.UTF_8); + revision[0] = next; + } + @Override public void enter() {} + }); + editor.update(1, 0, "hello ".getBytes(StandardCharsets.UTF_8), 6, 6, false, false, true); + InputConnection connection = editor.onCreateInputConnection(new EditorInfo()); + assertNotNull(connection); + assertTrue(connection.setComposingText("calm", 1)); + assertTrue(connection.commitText("水😀", 1)); + assertEquals("hello 水😀", latest[0]); + editor.update(1, 0, "old".getBytes(StandardCharsets.UTF_8), 3, 3, false, false, true); + assertEquals("hello 水😀", editor.getText().toString()); + assertTrue(connection.setSelection(0, 5)); + assertTrue(connection.commitText("fresh", 1)); + assertEquals("fresh 水😀", latest[0]); + editor.update(2, 0, "saved".getBytes(StandardCharsets.UTF_8), 5, 5, false, false, true); + assertFalse(connection.commitText("late", 1)); + assertEquals("saved", editor.getText().toString()); + assertTrue(revision[0] > 0); + } + + @Test public void systemClipboardPreservesUnicodeAndIgnoresNonTextClips() { + Context context = RuntimeEnvironment.getApplication(); + byte[] text = "Diary\n水😀 é".getBytes(StandardCharsets.UTF_8); + assertTrue(TextClipboard.write(context, text)); + assertArrayEquals(text, TextClipboard.read(context)); + ClipboardManager clipboard = (ClipboardManager)context.getSystemService(Context.CLIPBOARD_SERVICE); + assertEquals("Diary\n水😀 é", clipboard.getPrimaryClip().getItemAt(0).getText().toString()); + clipboard.setPrimaryClip(ClipData.newRawUri("image", Uri.parse("content://example/image"))); + assertArrayEquals(new byte[0], TextClipboard.read(context)); + assertTrue(TextClipboard.write(context, new byte[0])); + assertArrayEquals(new byte[0], TextClipboard.read(context)); + assertFalse(TextClipboard.write(context, null)); + } +} diff --git a/tests/android-text-selection/build.gradle b/tests/android-text-selection/build.gradle new file mode 100644 index 000000000..b32629726 --- /dev/null +++ b/tests/android-text-selection/build.gradle @@ -0,0 +1,60 @@ +plugins { + id 'java' +} + +repositories { + google() + mavenCentral() +} + +layout.buildDirectory = file('../../build/android-text-selection-test') +def sdkProperties = new Properties() +def sdkFile = file('../../droid/local.properties') +if (sdkFile.exists()) { + sdkFile.withInputStream { sdkProperties.load(it) } +} +def sdk = System.getenv('ANDROID_SDK_ROOT') ?: System.getenv('ANDROID_HOME') ?: sdkProperties.getProperty('sdk.dir') +if (!sdk) { + throw new GradleException('Set ANDROID_SDK_ROOT or droid/local.properties sdk.dir for Android text tests') +} +sourceSets { + main.java { + srcDir '../../droid/app/src/main/java' + include '**/TextSelectionMenu.java', '**/TextInputView.java', '**/TextClipboard.java' + } + test.java.srcDir '.' +} +dependencies { + compileOnly files("${sdk}/platforms/android-36/android.jar") + testImplementation files("${sdk}/platforms/android-36/android.jar") + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.robolectric:robolectric:4.16' +} + +// Robolectric's AndroidX dependencies contain classes.jar inside their AARs. +def aarDirectory = layout.buildDirectory.dir('aar-classes') +tasks.register('extractAndroidTestAars') { + inputs.files(configurations.testRuntimeClasspath) + outputs.dir(aarDirectory) + doLast { + aarDirectory.get().asFile.listFiles()?.each { it.setWritable(true) } + configurations.testRuntimeClasspath.files.findAll { it.name.endsWith('.aar') }.each { aar -> + copy { + from zipTree(aar) + include 'classes.jar' + into aarDirectory + rename 'classes.jar', "${aar.name}.jar" + fileMode 0644 + } + } + } +} +test { + dependsOn 'extractAndroidTestAars' + classpath += files(aarDirectory.map { fileTree(it) { include '*.jar' } }) + systemProperty 'robolectric.dependency.repo.url', 'https://repo.maven.apache.org/maven2' + testLogging { + events 'passed', 'failed', 'skipped' + exceptionFormat 'full' + } +} diff --git a/tests/android-text-selection/settings.gradle b/tests/android-text-selection/settings.gradle new file mode 100644 index 000000000..67018c768 --- /dev/null +++ b/tests/android-text-selection/settings.gradle @@ -0,0 +1 @@ +rootProject.name = 'inbe-text-selection-test' diff --git a/tests/android_text_input_behavior.zi b/tests/android_text_input_behavior.zi index 8bf4a113f..631d4f107 100644 --- a/tests/android_text_input_behavior.zi +++ b/tests/android_text_input_behavior.zi @@ -1,3 +1,4 @@ +#import "android_text_input_host" #import "android_device" #import "android_text_editor" #import, file "../src/platform/android/android_activity.zi"; diff --git a/tests/android_text_input_host.zi b/tests/android_text_input_host.zi new file mode 100644 index 000000000..cd646960e --- /dev/null +++ b/tests/android_text_input_host.zi @@ -0,0 +1,67 @@ +// No graphics or desktop are used by the Android input transport test. +#import "drawing_props" +#import "geometry" + +labels: s32; + +#program_export +MeasureGlyphWidth :: (value: string, font: s32, typeface: string) -> s32 { + unused typeface + return (cast(s32)value.count) * font / 2 +} + +#program_export +MeasureGlyphLineHeight :: (font: s32, typeface: string) -> s32 { + unused typeface + return font +} + +#program_export +RasterRoundedRectangle :: (bounds: Rectangle, radius: float32, + segments: s32, color: Color) {} + +#program_export +RasterRoundedRectangleOutline :: (bounds: Rectangle, radius: float32, + segments: s32, width: float32, color: Color) {} + +#program_export +RasterLine :: (bounds: Rectangle, color: Color) {} + +#program_export +RasterText :: (value: string, x: s32, y: s32, font: s32, color: Color) { + labels += 1 +} + +#program_export +RasterTextClipped :: (value: string, x: s32, y: s32, font: s32, + color: Color, clip: Rectangle) { + labels += 1 +} + +#program_export +RasterImage :: (path: string, texture_id: u32, source: Rectangle, + destination: Rectangle, clip: Rectangle, origin: Vector2, + rotation: float32, radius: float32, tint: Color) {} + +#program_export +RasterImageMesh :: (path: string, texture_id: u32, source: Rectangle, + points: []Vector2, columns: s32, rows: s32, clip: Rectangle, + tint: Color) {} + +#program_export +ContextMenuLabelCount :: () -> s32 { return labels } + +#program_export +SystemClipboardOpenHost :: () -> s32 { + return -1 +} +#program_export +SystemClipboardByteHost :: (index: s32) -> s32 { + unused index + return -1 +} +#program_export +SystemClipboardWriteHost :: (text: string) -> bool { + unused text + return false +} diff --git a/tests/android_text_selection_device.py b/tests/android_text_selection_device.py new file mode 100644 index 000000000..22e2f9bce --- /dev/null +++ b/tests/android_text_selection_device.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Check selection on the dedicated Android 14 AVD, never a physical phone. + +Uses a fresh debug-app profile. Run the AVD on private Xvfb with inherited +desktop variables scrubbed, then pass the current x86_64 debug APK. +""" +import argparse +import csv +import io +import json +from pathlib import Path +import re +import sqlite3 +import subprocess +import time + +from PIL import ImageOps + +from android_diary_photo_device import APP_FILES, PACKAGE, ROOT, Device + + +TEXT = "Calm breathing keeps my diary clear." + + +class SelectionDevice(Device): + def prepare(self, apk, dark=0): + if apk: + self.adb("install", "-r", "-t", apk) + self.shell("pm", "clear", PACKAGE) + self.shell("pm", "grant", PACKAGE, "android.permission.POST_NOTIFICATIONS") + self.shell("input", "keyevent", "KEYCODE_WAKEUP") + self.shell("wm", "dismiss-keyguard") + self.start() + owner = self.settings(enabled_apps=31, main_tab=3, language="en", language_setup_done=1, + apps_setup_done=1, launcher_guide_seen=1, tutorial_seen=1, + lumi_guides_seen=127, lumi_online=0, lumi_archive_migrated=1, + package_updates_allowed=0, theme=7, theme_source=0, + theme_mode=2 if dark else 1, dark_mode=dark, ui_scale=10, + **{"cell.lumi.message.selection-fixture": json.dumps(dict( + time=1, user=0, kind=0, payload="", text="Gentle breathing brings calm."))}) + day = self.shell("date", "+%F") + assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", day) + directory = APP_FILES + "/diary/" + owner + self.shell("run-as", PACKAGE, "mkdir", "-p", directory) + self.entry_path = directory + "/" + day + ".json" + fixture = self.output / "entry.json" + fixture.write_text(json.dumps(dict(version=1, date=day, text=TEXT, photos=[]))) + self.put_private(fixture, self.entry_path) + self.output.joinpath("context.json").write_text(json.dumps(dict(entry=self.entry_path))) + self.start() + self.snapshot("prepared") + + def word(self, name, target, minimum_y=0): + path = self.output / (name + ".png") + assert self.snapshot(name).size == (480, 960) + for mode in (11, 6): + output = subprocess.run(["tesseract", str(path), "stdout", "--psm", str(mode), + "-c", "tessedit_create_tsv=1"], + capture_output=True, text=True, check=True, timeout=20).stdout + words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t") + if row["text"].strip(".,:").lstrip("I|").lower() == target.split()[0].lower() + and int(row["top"]) >= minimum_y + and int(row["top"]) < (960 if target == "Message" else 600)] + if len(words) == 1: + row = words[0] + return int(row["left"]) + int(row["width"]) // 2, int(row["top"]) + int(row["height"]) // 2 + raise AssertionError("Expected one rendered word: " + target) + + def hold_word(self, name, word): + x, y = self.word(name, word) + self.shell("input", "swipe", x, y, x, y, 900) + time.sleep(0.5) + self.snapshot(name + "-selected") + + def menu(self, command): + matches = [node for node in self.nodes() if node.attrib.get("text") == command + or node.attrib.get("content-desc") == command] + if matches: + assert len(matches) == 1 + self.tap_node(matches[0]) + else: + # NativeActivity's floating popup is rendered but not exposed by + # UIAutomator on this image. OCR its largest connected background. + screen = self.snapshot("menu-" + command.replace(" ", "-")) + pixels = screen.load() + remaining = {(x, y) for y in range(30, 600) for x in range(480) + if pixels[x, y] == (66, 66, 66)} + largest = set() + while remaining: + todo = [remaining.pop()] + component = set(todo) + while todo: + x, y = todo.pop() + for point in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)): + if point in remaining: + remaining.remove(point) + component.add(point) + todo.append(point) + if len(component) > len(largest): + largest = component + assert len(largest) > 1000, "Native toolbar did not render" + left = min(x for x, y in largest) + top = min(y for x, y in largest) + right = max(x for x, y in largest) + 1 + bottom = max(y for x, y in largest) + 1 + crop = ImageOps.invert(screen.crop((left, top, right, bottom))) + path = self.output / "toolbar-ocr.png" + crop.resize((crop.width * 3, crop.height * 3)).save(path) + output = subprocess.run(["tesseract", str(path), "stdout", "--psm", "6", + "-c", "tessedit_create_tsv=1"], capture_output=True, + text=True, check=True, timeout=60).stdout + words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t") + if row["text"].strip("|.,:").lower() == command.split()[0].lower()] + assert len(words) == 1, (command, output) + row = words[0] + x = left + (int(row["left"]) + int(row["width"]) // 2) // 3 + y = top + (int(row["top"]) + int(row["height"]) // 2) // 3 + self.shell("input", "tap", x, y) + time.sleep(0.5) + + def text(self): + return json.loads(self.private_bytes(self.entry_path))["text"] + + def check(self, apk): + for dark in (0, 1): + name = "sage-dark" if dark else "sage-light" + # Give each theme its own profile. After startup, navigate in the + # actual UI rather than rewriting a live profile's SQLite records. + self.prepare(apk, dark) + self.hold_word(name + "-diary-copy", "breathing") + self.menu("Copy") + self.hold_word(name + "-diary-replace", "Calm") + self.menu("Paste") + expected = "breathing breathing keeps my diary clear." + self.wait(lambda: self.text() == expected, "Diary paste saved") + # A real IME key, rather than adb's synthetic hardware typing. + self.shell("input", "tap", 240, 807) + self.wait(lambda: self.text() != expected, "continued IME typing") + typed = self.text() + assert typed in ("breathingg breathing keeps my diary clear.", + "breathingG breathing keeps my diary clear."), typed + self.snapshot(name + "-diary-paste-typed") + self.stop() + self.start() + assert self.text() == typed + self.snapshot(name + "-diary-restart") + + self.shell("input", "tap", 66, 929) + time.sleep(2) + self.hold_word(name + "-chat-copy", "Gentle") + self.menu("Copy") + # Empty composer must still offer Paste on long-press. + time.sleep(6) # Let Android's clipboard preview stop covering it. + x, y = self.word(name + "-composer-empty", "Message") + self.shell("input", "tap", x, y) + time.sleep(2) + assert self.snapshot(name + "-composer-focused").size == (480, 960) + # The open IME moves this fixture's composer to y=575 in both + # themes. The dark placeholder has intentionally low contrast. + self.shell("input", "swipe", 70, 575, 70, 575, 900) + time.sleep(1) + self.menu("Paste") + time.sleep(2) + self.snapshot(name + "-chat-pasted") + self.shell("input", "tap", 240, 807) + time.sleep(2) + self.snapshot(name + "-chat-typed") + self.shell("input", "swipe", 45, 570, 45, 570, 900) + time.sleep(1) + self.menu("Cut") + time.sleep(2) + self.snapshot(name + "-composer-cut") + # Verify the exact composer text through the OS clipboard and + # persisted Diary data; caret blinking can confuse screenshot OCR. + self.shell("input", "tap", 414, 645) # Apps, above the open IME. + time.sleep(2) + x, y = self.word(name + "-apps", "Diary") + self.shell("input", "tap", x, y) + time.sleep(2) + self.hold_word(name + "-composer-roundtrip", "keeps") + self.menu("Paste") + expected = typed.replace("keeps", "Gentleg") + self.wait(lambda: self.text() == expected, "composer clipboard round trip") + self.stop() + database_path = self.output / "checked.db" + database_path.write_bytes(self.private_bytes(APP_FILES + "/inbe.db")) + for suffix in ("-wal", "-shm"): + Path(str(database_path) + suffix).unlink(missing_ok=True) + wal = self.probe("run-as", PACKAGE, "ls", APP_FILES + "/inbe.db-wal") + if wal: + Path(str(database_path) + "-wal").write_bytes(self.private_bytes(APP_FILES + "/inbe.db-wal")) + with sqlite3.connect(database_path) as database: + keys = database.execute("SELECT key FROM settings WHERE key LIKE 'cell.lumi.message.%'").fetchall() + assert keys == [("cell.lumi.message.selection-fixture",)], "The check must never send a message" + print(name + ": Diary copy/paste/IME typing/save/restart and offline chat word copy/composer paste passed", + flush=True) + self.output.joinpath("result.json").write_text(json.dumps(dict( + android_sdk=34, avd="inbe-diary-photo", scale=10, + themes=["sage_light", "sage_dark"], + passed=["long press", "native toolbar", "Diary copy/paste", "IME typing", + "Diary save/restart", "chat word copy", "unsent composer paste", + "composer IME typing/cut and exact clipboard round trip"], + status="Local debug APK on a private emulator; no physical phone, send, push or release"), indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("prepare", "check", "test", "snapshot")) + parser.add_argument("--adb", default="/home/wao/Android/Sdk/platform-tools/adb") + parser.add_argument("--serial", default="emulator-5578") + parser.add_argument("--apk", type=Path) + parser.add_argument("--output", type=Path, default=ROOT / "build/android-text-selection-device") + args = parser.parse_args() + device = SelectionDevice(args.adb, args.serial, args.output) + try: + if args.action in ("prepare", "test"): + if not args.apk: + parser.error("prepare requires --apk") + if args.action == "prepare": + device.prepare(args.apk) + if args.action in ("check", "test"): + device.check(args.apk) + if args.action == "snapshot": + device.snapshot("current") + except Exception: + device.snapshot("failure") + raise + finally: + if args.action == "test": + device.stop() + + +if __name__ == "__main__": + main() diff --git a/tests/sync_recovery_zi_test.sh b/tests/sync_recovery_zi_test.sh index 97db8eaab..72dc35985 100644 --- a/tests/sync_recovery_zi_test.sh +++ b/tests/sync_recovery_zi_test.sh @@ -17,6 +17,7 @@ mkdir -p "$work/c" --module-path "$root/build/packages/kss/src" --module-path "oqs=$root/build/packages/oqs/src" --module-path "kryon=$root/build/packages/kryon/src/ui" \ --module-path "$root/build/packages/kryon/src/backend" \ --module-path "$root/build/packages/game2d/src" \ + --module-path "qrcode=$root/build/packages/qrcode/src" \ --module-path "$root/build/packages/daochi-client" \ -o "$work/c" "$root/tests/sync_recovery_behavior.zi" "${CC:-cc}" -std=gnu11 -w -DZIRAN_BOUNDS_CHECK -ffunction-sections \ diff --git a/ziran.lock b/ziran.lock index 007b194bf..f7f88efa6 100644 --- a/ziran.lock +++ b/ziran.lock @@ -53,17 +53,9 @@ "source": true, "url": "https://github.com/raysan5/raylib.git" }, - { - "commit": "6a8ca21ab165a64af425da68944ead083e2ed97d", - "dependencies": {"raylib": "p5a77c9c35110e925"}, - "id": "p701ca29a75e15af6", - "name": "kryon", - "ref": "master", - "url": "https://github.com/kryonlabs/kryon.git" - }, { "commit": "41538659afbbda300915ef009b55c18f6de159b8", - "dependencies": {"kryon": "p701ca29a75e15af6"}, + "dependencies": {"kryon": "pe4492807d82c43f5"}, "id": "p9504fb8e5fabefc6", "name": "Kss", "ref": "master", @@ -79,7 +71,7 @@ }, { "commit": "2ee29338c9b93fb4f4a56a49c7be09e9d2c2418f", - "dependencies": {"kryon": "p701ca29a75e15af6"}, + "dependencies": {"kryon": "pe4492807d82c43f5"}, "id": "pa5dd7b8b83307b18", "name": "plot", "ref": "master", @@ -87,7 +79,7 @@ }, { "commit": "7aa01465ac3f988e0404f3c6dc244bf4b009f253", - "dependencies": {"kryon": "p701ca29a75e15af6"}, + "dependencies": {"kryon": "pe4492807d82c43f5"}, "id": "paa08e1730a25bda0", "name": "Game2D", "ref": "master", @@ -102,6 +94,14 @@ "source": true, "url": "https://github.com/curl/curl.git" }, + { + "commit": "556ee25f11f2e6b979150acb94e3093a137ed47e", + "dependencies": {"raylib": "p5a77c9c35110e925"}, + "id": "pe4492807d82c43f5", + "name": "kryon", + "ref": "master", + "url": "https://github.com/kryonlabs/kryon.git" + }, { "commit": "51fa84aedaa6ae4f6734e0fcca9bfc35a184d297", "dependencies": {}, @@ -121,7 +121,7 @@ } ], "root": { - "dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "p701ca29a75e15af6", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"}, + "dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pe4492807d82c43f5", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"}, "name": "Inbe" }, "toolchain": {