Files
breeze/tests/android_text_selection_device.py

235 lines
11 KiB
Python

#!/usr/bin/env python3
"""Check selection on the dedicated Android 14 AVD, never a physical phone.
Uses a fresh debug-app profile. Run the AVD on private Xvfb with inherited
desktop variables scrubbed, then pass the current x86_64 debug APK.
"""
import argparse
import csv
import io
import json
from pathlib import Path
import re
import sqlite3
import subprocess
import time
from PIL import ImageOps
from android_diary_photo_device import APP_FILES, PACKAGE, ROOT, Device
TEXT = "Calm breathing keeps my diary clear."
class SelectionDevice(Device):
def prepare(self, apk, dark=0):
if apk:
self.adb("install", "-r", "-t", apk)
self.shell("pm", "clear", PACKAGE)
self.shell("pm", "grant", PACKAGE, "android.permission.POST_NOTIFICATIONS")
self.shell("input", "keyevent", "KEYCODE_WAKEUP")
self.shell("wm", "dismiss-keyguard")
self.start()
owner = self.settings(enabled_apps=31, main_tab=3, language="en", language_setup_done=1,
apps_setup_done=1, launcher_guide_seen=1, tutorial_seen=1,
lumi_guides_seen=127, lumi_online=0, lumi_archive_migrated=1,
package_updates_allowed=0, theme=7, theme_source=0,
theme_mode=2 if dark else 1, dark_mode=dark, ui_scale=10,
**{"cell.lumi.message.selection-fixture": json.dumps(dict(
time=1, user=0, kind=0, payload="", text="Gentle breathing brings calm."))})
day = self.shell("date", "+%F")
assert re.fullmatch(r"\d{4}-\d{2}-\d{2}", day)
directory = APP_FILES + "/diary/" + owner
self.shell("run-as", PACKAGE, "mkdir", "-p", directory)
self.entry_path = directory + "/" + day + ".json"
fixture = self.output / "entry.json"
fixture.write_text(json.dumps(dict(version=1, date=day, text=TEXT, photos=[])))
self.put_private(fixture, self.entry_path)
self.output.joinpath("context.json").write_text(json.dumps(dict(entry=self.entry_path)))
self.start()
self.snapshot("prepared")
def word(self, name, target, minimum_y=0):
path = self.output / (name + ".png")
assert self.snapshot(name).size == (480, 960)
for mode in (11, 6):
output = subprocess.run(["tesseract", str(path), "stdout", "--psm", str(mode),
"-c", "tessedit_create_tsv=1"],
capture_output=True, text=True, check=True, timeout=20).stdout
words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t")
if row["text"].strip(".,:").lstrip("I|").lower() == target.split()[0].lower()
and int(row["top"]) >= minimum_y
and int(row["top"]) < (960 if target == "Message" else 600)]
if len(words) == 1:
row = words[0]
return int(row["left"]) + int(row["width"]) // 2, int(row["top"]) + int(row["height"]) // 2
raise AssertionError("Expected one rendered word: " + target)
def hold_word(self, name, word):
x, y = self.word(name, word)
self.shell("input", "swipe", x, y, x, y, 900)
time.sleep(0.5)
self.snapshot(name + "-selected")
def menu(self, command):
matches = [node for node in self.nodes() if node.attrib.get("text") == command
or node.attrib.get("content-desc") == command]
if matches:
assert len(matches) == 1
self.tap_node(matches[0])
else:
# NativeActivity's floating popup is rendered but not exposed by
# UIAutomator on this image. OCR its largest connected background.
screen = self.snapshot("menu-" + command.replace(" ", "-"))
pixels = screen.load()
remaining = {(x, y) for y in range(30, 600) for x in range(480)
if pixels[x, y] == (66, 66, 66)}
largest = set()
while remaining:
todo = [remaining.pop()]
component = set(todo)
while todo:
x, y = todo.pop()
for point in ((x - 1, y), (x + 1, y), (x, y - 1), (x, y + 1)):
if point in remaining:
remaining.remove(point)
component.add(point)
todo.append(point)
if len(component) > len(largest):
largest = component
assert len(largest) > 1000, "Native toolbar did not render"
left = min(x for x, y in largest)
top = min(y for x, y in largest)
right = max(x for x, y in largest) + 1
bottom = max(y for x, y in largest) + 1
crop = ImageOps.invert(screen.crop((left, top, right, bottom)))
path = self.output / "toolbar-ocr.png"
crop.resize((crop.width * 3, crop.height * 3)).save(path)
output = subprocess.run(["tesseract", str(path), "stdout", "--psm", "6",
"-c", "tessedit_create_tsv=1"], capture_output=True,
text=True, check=True, timeout=60).stdout
words = [row for row in csv.DictReader(io.StringIO(output), delimiter="\t")
if row["text"].strip("|.,:").lower() == command.split()[0].lower()]
assert len(words) == 1, (command, output)
row = words[0]
x = left + (int(row["left"]) + int(row["width"]) // 2) // 3
y = top + (int(row["top"]) + int(row["height"]) // 2) // 3
self.shell("input", "tap", x, y)
time.sleep(0.5)
def text(self):
return json.loads(self.private_bytes(self.entry_path))["text"]
def check(self, apk):
for dark in (0, 1):
name = "sage-dark" if dark else "sage-light"
# Give each theme its own profile. After startup, navigate in the
# actual UI rather than rewriting a live profile's SQLite records.
self.prepare(apk, dark)
self.hold_word(name + "-diary-copy", "breathing")
self.menu("Copy")
self.hold_word(name + "-diary-replace", "Calm")
self.menu("Paste")
expected = "breathing breathing keeps my diary clear."
self.wait(lambda: self.text() == expected, "Diary paste saved")
# A real IME key, rather than adb's synthetic hardware typing.
self.shell("input", "tap", 240, 807)
self.wait(lambda: self.text() != expected, "continued IME typing")
typed = self.text()
assert typed in ("breathingg breathing keeps my diary clear.",
"breathingG breathing keeps my diary clear."), typed
self.snapshot(name + "-diary-paste-typed")
self.stop()
self.start()
assert self.text() == typed
self.snapshot(name + "-diary-restart")
self.shell("input", "tap", 66, 929)
time.sleep(2)
self.hold_word(name + "-chat-copy", "Gentle")
self.menu("Copy")
# Empty composer must still offer Paste on long-press.
time.sleep(6) # Let Android's clipboard preview stop covering it.
x, y = self.word(name + "-composer-empty", "Message")
self.shell("input", "tap", x, y)
time.sleep(2)
assert self.snapshot(name + "-composer-focused").size == (480, 960)
# The open IME moves this fixture's composer to y=575 in both
# themes. The dark placeholder has intentionally low contrast.
self.shell("input", "swipe", 70, 575, 70, 575, 900)
time.sleep(1)
self.menu("Paste")
time.sleep(2)
self.snapshot(name + "-chat-pasted")
self.shell("input", "tap", 240, 807)
time.sleep(2)
self.snapshot(name + "-chat-typed")
self.shell("input", "swipe", 45, 570, 45, 570, 900)
time.sleep(1)
self.menu("Cut")
time.sleep(2)
self.snapshot(name + "-composer-cut")
# Verify the exact composer text through the OS clipboard and
# persisted Diary data; caret blinking can confuse screenshot OCR.
self.shell("input", "tap", 414, 645) # Apps, above the open IME.
time.sleep(2)
x, y = self.word(name + "-apps", "Diary")
self.shell("input", "tap", x, y)
time.sleep(2)
self.hold_word(name + "-composer-roundtrip", "keeps")
self.menu("Paste")
expected = typed.replace("keeps", "Gentleg")
self.wait(lambda: self.text() == expected, "composer clipboard round trip")
self.stop()
database_path = self.output / "checked.db"
database_path.write_bytes(self.private_bytes(APP_FILES + "/inbe.db"))
for suffix in ("-wal", "-shm"):
Path(str(database_path) + suffix).unlink(missing_ok=True)
wal = self.probe("run-as", PACKAGE, "ls", APP_FILES + "/inbe.db-wal")
if wal:
Path(str(database_path) + "-wal").write_bytes(self.private_bytes(APP_FILES + "/inbe.db-wal"))
with sqlite3.connect(database_path) as database:
keys = database.execute("SELECT key FROM settings WHERE key LIKE 'cell.lumi.message.%'").fetchall()
assert keys == [("cell.lumi.message.selection-fixture",)], "The check must never send a message"
print(name + ": Diary copy/paste/IME typing/save/restart and offline chat word copy/composer paste passed",
flush=True)
self.output.joinpath("result.json").write_text(json.dumps(dict(
android_sdk=34, avd="inbe-diary-photo", scale=10,
themes=["sage_light", "sage_dark"],
passed=["long press", "native toolbar", "Diary copy/paste", "IME typing",
"Diary save/restart", "chat word copy", "unsent composer paste",
"composer IME typing/cut and exact clipboard round trip"],
status="Local debug APK on a private emulator; no physical phone, send, push or release"), indent=2) + "\n")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=("prepare", "check", "test", "snapshot"))
parser.add_argument("--adb", default="/home/wao/Android/Sdk/platform-tools/adb")
parser.add_argument("--serial", default="emulator-5578")
parser.add_argument("--apk", type=Path)
parser.add_argument("--output", type=Path, default=ROOT / "build/android-text-selection-device")
args = parser.parse_args()
device = SelectionDevice(args.adb, args.serial, args.output)
try:
if args.action in ("prepare", "test"):
if not args.apk:
parser.error("prepare requires --apk")
if args.action == "prepare":
device.prepare(args.apk)
if args.action in ("check", "test"):
device.check(args.apk)
if args.action == "snapshot":
device.snapshot("current")
except Exception:
device.snapshot("failure")
raise
finally:
if args.action == "test":
device.stop()
if __name__ == "__main__":
main()