From 78083bbeb399476c58e2bd7fa5b60cf238c5bdc3 Mon Sep 17 00:00:00 2001 From: Waozi Date: Thu, 22 Oct 2026 02:39:59 -0300 Subject: [PATCH] Use Daochi Client for resumable account sync --- CHANGELOG.md | 34 +--------- .../java/xyz/waozi/inbe/HttpTransport.java | 24 ++++++- .../java/xyz/waozi/inbe/MainActivity.java | 4 ++ .../metadata/android/en-US/changelogs/175.txt | 34 +--------- src/platform/network_transport.zi | 12 +++- src/storage/storage_settings.zi | 2 +- src/storage/sync_async.zi | 67 ++++++++++++++----- src/storage/sync_exchange.zi | 31 ++++++--- src/storage/sync_page_store.zi | 43 ++++++++++++ src/storage/sync_runtime.zi | 61 +++-------------- tests/AndroidHttpTransportTest.java | 1 + tests/flow_fake/http_transfer.zi | 9 ++- tests/storage_sync_behavior.zi | 41 ++++++++++++ ziran.lock | 44 ++++++------ 14 files changed, 239 insertions(+), 168 deletions(-) create mode 100644 src/storage/sync_page_store.zi diff --git a/CHANGELOG.md b/CHANGELOG.md index 1865b0c41..06a03a225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,37 +2,9 @@ ## [2.1.38] - 2026-10-21 -- Connecting an account starts sync automatically, and existing connections recover a missing node address. -- Sync startup failures show their cause and retry correctly instead of remaining queued. -- Move apps by their handles in Edit apps, including apps not downloaded yet; dragging a row scrolls the list. -- Open main settings from Apps’ top-right corner instead of a separate Settings app. -- Tapping Apps again keeps it open, and every app row keeps its normal appearance and options menu. -- Profile has one header, and Back returns through Account and Activity before Apps even when Profile is unpinned. -- Lists and Leaderboard come preinstalled, the app list uses a simple divider, and every row stays clear of navigation. -- Account IDs use the first and last four characters, and Account has one heading. -- Downloads keep transferring while package installation waits for scrolling and dragging to finish. -- Installed collections discover their games over successive frames instead of opening every game at once. -- Apps asks about automatic updates again, and discovering apps retries after a connection failure. -- Open a friend to view and copy their full public ID. -- Account shows one current sync status, and local database size is distinguished from node storage. -- Switch Apps between a scrollable list and a grid; your chosen view is remembered. -- Pin Profile alongside your other apps. -- Apps opens faster when first showing your apps. -- Arcade switches to newly installed game updates without keeping an older campaign open. -- Android launches use current game packages even when an older developer copy remains on the device. -- Shovey's own Back button returns to its game chooser. -- Escape closes the open dialog, including while an input is focused. -- Scrolling stays responsive while a text field is focused. -- Profile Activity shows activity without repeating profile destinations. -- Every achievement has its own meaningful icon, without numbered badge frames. -- Android account-key backups use the system Save dialog and confirm that the chosen file was written. -- Share lists with name suggestions and an automatically updated list of everyone with access. -- Give collaborators full owner rights, editing access, or read-only access, and remove access from the same dialog. -- Shared-list edits preserve every task. -- Undo restores a deleted task with its comment and completion state. -- The Habits shop scrolls through all recent activity and offers a soda reward symbol. -- Games hide the surrounding navigation and keep a clear return path to Arcade. -- Library chapters turn with a swipe and use a warm paper background without the sidebar. +- Simplify Apps editing, settings, navigation and app loading. +- Fix duplicate Profile headers, scrolling, Escape handling and Android key backups. +- Fix large-account sync, sharing controls and activity and achievement views. ## [2.1.37] - 2026-10-20 diff --git a/droid/app/src/main/java/xyz/waozi/inbe/HttpTransport.java b/droid/app/src/main/java/xyz/waozi/inbe/HttpTransport.java index 3acfe344d..fb6d71eaf 100644 --- a/droid/app/src/main/java/xyz/waozi/inbe/HttpTransport.java +++ b/droid/app/src/main/java/xyz/waozi/inbe/HttpTransport.java @@ -40,6 +40,7 @@ final class HttpTransport { volatile boolean cancelled; boolean sent; int state = -1; + int failure; byte[] response; Transfer(String method, URL url, int capacity, int timeout) { @@ -181,7 +182,9 @@ final class HttpTransport { HttpURLConnection connection = null; int status = 0; int receivedStatus = 0; + int receivedBytes = 0; String stage = "connect"; + String failure = "io"; byte[] response = null; try { if (transfer.cancelled) { @@ -212,6 +215,7 @@ final class HttpTransport { stage = "response"; receivedStatus = connection.getResponseCode(); if (connection.getContentLength() >= transfer.capacity) { + failure = "capacity"; throw new java.io.IOException("response capacity exceeded"); } InputStream stream = receivedStatus >= 400 @@ -224,10 +228,16 @@ final class HttpTransport { byte[] buffer = new byte[Math.min(transfer.capacity, 8192)]; int count; while ((count = input.read(buffer)) != -1) { - if (transfer.cancelled || count > transfer.capacity - bytes.size() - 1) { - throw new java.io.IOException("response cancelled or capacity exceeded"); + if (transfer.cancelled) { + failure = "cancelled"; + throw new java.io.IOException("response cancelled"); + } + if (count > transfer.capacity - bytes.size() - 1) { + failure = "capacity"; + throw new java.io.IOException("response capacity exceeded"); } bytes.write(buffer, 0, count); + receivedBytes = bytes.size(); } } } @@ -236,7 +246,8 @@ final class HttpTransport { } catch (Exception error) { // Class and stage only: exception messages may contain private URLs. System.err.println("HTTP: request failed stage=" + stage + " cause=" + - error.getClass().getSimpleName() + " status=" + receivedStatus + + error.getClass().getSimpleName() + " reason=" + failure + + " status=" + receivedStatus + " bytes=" + receivedBytes + " capacity=" + transfer.capacity); } finally { if (connection != null) { @@ -246,6 +257,7 @@ final class HttpTransport { synchronized (transfer) { if (!transfer.cancelled && transfers.get(id) == transfer) { transfer.response = response; + transfer.failure = status > 0 ? 0 : ("capacity".equals(failure) ? 2 : 1); transfer.state = status; } } @@ -272,6 +284,12 @@ final class HttpTransport { } } + int failure(int id) { + Transfer transfer = transfers.get(id); + if (transfer == null) { return 1; } + synchronized (transfer) { return transfer.failure; } + } + void cancel(int id) { Transfer transfer = transfers.remove(id); if (transfer == null) { diff --git a/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java b/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java index fb291dc60..5d018470c 100644 --- a/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java +++ b/droid/app/src/main/java/xyz/waozi/inbe/MainActivity.java @@ -281,6 +281,10 @@ public class MainActivity extends NativeActivity { return httpTransport.poll(handle); } + public int httpFailure(int handle) { + return httpTransport.failure(handle); + } + public byte[] httpResponse(int handle) { return httpTransport.response(handle); } diff --git a/fastlane/metadata/android/en-US/changelogs/175.txt b/fastlane/metadata/android/en-US/changelogs/175.txt index 9449c03e4..f1041f565 100644 --- a/fastlane/metadata/android/en-US/changelogs/175.txt +++ b/fastlane/metadata/android/en-US/changelogs/175.txt @@ -1,31 +1,3 @@ -- Connecting an account starts sync automatically, and existing connections recover a missing node address. -- Sync startup failures show their cause and retry correctly instead of remaining queued. -- Move apps by their handles in Edit apps, including apps not downloaded yet; dragging a row scrolls the list. -- Open main settings from Apps’ top-right corner instead of a separate Settings app. -- Tapping Apps again keeps it open, and every app row keeps its normal appearance and options menu. -- Profile has one header, and Back returns through Account and Activity before Apps even when Profile is unpinned. -- Lists and Leaderboard come preinstalled, the app list uses a simple divider, and every row stays clear of navigation. -- Account IDs use the first and last four characters, and Account has one heading. -- Downloads keep transferring while package installation waits for scrolling and dragging to finish. -- Installed collections discover their games over successive frames instead of opening every game at once. -- Apps asks about automatic updates again, and discovering apps retries after a connection failure. -- Open a friend to view and copy their full public ID. -- Account shows one current sync status, and local database size is distinguished from node storage. -- Switch Apps between a scrollable list and a grid; your chosen view is remembered. -- Pin Profile alongside your other apps. -- Apps opens faster when first showing your apps. -- Arcade switches to newly installed game updates without keeping an older campaign open. -- Android launches use current game packages even when an older developer copy remains on the device. -- Shovey's own Back button returns to its game chooser. -- Escape closes the open dialog, including while an input is focused. -- Scrolling stays responsive while a text field is focused. -- Profile Activity shows activity without repeating profile destinations. -- Every achievement has its own meaningful icon, without numbered badge frames. -- Android account-key backups use the system Save dialog and confirm that the chosen file was written. -- Share lists with name suggestions and an automatically updated list of everyone with access. -- Give collaborators full owner rights, editing access, or read-only access, and remove access from the same dialog. -- Shared-list edits preserve every task. -- Undo restores a deleted task with its comment and completion state. -- The Habits shop scrolls through all recent activity and offers a soda reward symbol. -- Games hide the surrounding navigation and keep a clear return path to Arcade. -- Library chapters turn with a swipe and use a warm paper background without the sidebar. +- Simplify Apps editing, settings, navigation and app loading. +- Fix duplicate Profile headers, scrolling, Escape handling and Android key backups. +- Fix large-account sync, sharing controls and activity and achievement views. diff --git a/src/platform/network_transport.zi b/src/platform/network_transport.zi index bce2d4724..8f4e18883 100644 --- a/src/platform/network_transport.zi +++ b/src/platform/network_transport.zi @@ -27,7 +27,7 @@ NetworkTransfer :: struct { StartNetworkTransfer :: (context: *void, request: HttpRequest, output: []u8) -> bool { transfer := cast(*NetworkTransfer)context return transfer != null && - HttpTransferStart(*transfer.transfer, request, output, NetworkDeadline(request.url)) + HttpTransferStart(*transfer.transfer, request, output, NetworkDeadline(request.url)) } PollNetworkTransfer :: (context: *void, status: *s32) -> s32 { @@ -45,9 +45,17 @@ CancelNetworkTransfer :: (context: *void) { } } +NetworkTransferFailure :: (context: *void) -> s32 { + transfer := cast(*NetworkTransfer)context + if transfer != null && transfer.transfer.failure == HTTP_TRANSFER_FAILURE_CAPACITY { + return TRANSFER_FAILURE_CAPACITY + } + return TRANSFER_FAILURE_TRANSPORT +} + NetworkTransport :: (transfer: *NetworkTransfer) -> AsyncTransport { return AsyncTransport.{.context = cast(*void)transfer, .start = StartNetworkTransfer, - .poll = PollNetworkTransfer, .cancel = CancelNetworkTransfer} + .poll = PollNetworkTransfer, .cancel = CancelNetworkTransfer, .failure = NetworkTransferFailure} } #if #defined(PLATFORM_WEB) == false && #defined(ANDROID_BUILD) == false { diff --git a/src/storage/storage_settings.zi b/src/storage/storage_settings.zi index e50c7add8..b8fcfb246 100644 --- a/src/storage/storage_settings.zi +++ b/src/storage/storage_settings.zi @@ -270,7 +270,7 @@ storage_setting_present :: (key: string) -> bool { storage_setting_text_equals :: (key: string, value: string) -> s32 { stmt: *void = null ok: s32 = 0 - if storage_db_handle() == null || key.count == 0 || value.count == 0 { + if storage_db_handle() == null || key.count == 0 { return 0 } sql_cx4: [71] u8 diff --git a/src/storage/sync_async.zi b/src/storage/sync_async.zi index a1dfdfe2a..63260f014 100644 --- a/src/storage/sync_async.zi +++ b/src/storage/sync_async.zi @@ -5,6 +5,8 @@ Auth :: #import "auth"; #import "client" #import "async_client" #import "async_sync" +#import "sync_flow" +#import, file "sync_page_store.zi"; #import "async_account" #import "delegated" #import "async_delegated" @@ -191,6 +193,49 @@ PrepareStoredOperation :: (operation: *StoredOperation, server: string, return true } +LoadOperationSyncPage :: (context: *void, state: *SyncPageState) -> bool { + operation := cast(*StoredOperation)context + return StoredIdentityCurrent(operation) && + LoadStoredSyncPage(TextUntilNul(operation.server[:]), state) +} + +SaveOperationSyncPage :: (context: *void, state: *SyncPageState) -> bool { + operation := cast(*StoredOperation)context + return StoredIdentityCurrent(operation) && + SaveStoredSyncPage(TextUntilNul(operation.server[:]), state) +} + +PrepareOperationSyncBody :: (context: *void, maximum_records: s32) -> string { + operation := cast(*StoredOperation)context + if !StoredIdentityCurrent(operation) { + return "" + } + if operation.payload != null { + storage_free_sync_payload_json(operation.payload) + } + operation.payload = BuildStoredSyncPayload( + operation.client.identity.account_id, operation.client.identity.public_key, maximum_records) + return TextFromCString(operation.payload) +} + +AcceptOperationSyncPage :: (context: *void, response: string, next: *SyncPageState) -> bool { + operation := cast(*StoredOperation)context + if !StoredIdentityCurrent(operation) || ApplyStoredSyncResponse(response, next) == 0 { + return false + } + storage_purge_synced_deleted_data() + if storage_last_sync_changed() != 0 { + operation.changed = 1 + } + SyncNodeStorageRecordResponse(TextUntilNul(operation.server[:]), response) + return true +} + +OperationSyncClock :: (context: *void) -> s64 { + unused context + return Clock.UnixNow() +} + BeginStoredSync :: (operation: *StoredOperation, server: string, transport: AsyncTransport) -> bool { if !PrepareStoredOperation(operation, server, true) { @@ -217,12 +262,6 @@ BeginStoredSync :: (operation: *StoredOperation, server: string, return false } operation.result = cast(s32)AuthResult.AUTH_PAYLOAD_FAILED - operation.payload = storage_build_sync_payload_json( - operation.client.identity.account_id, operation.client.identity.public_key) - if operation.payload == null { - ReleaseStoredOperation(operation) - return false - } operation.response = cast(*u8)AsyncAllocate(AsyncResponseCapacity) if operation.response == null { ReleaseStoredOperation(operation) @@ -231,8 +270,10 @@ BeginStoredSync :: (operation: *StoredOperation, server: string, operation.result = cast(s32)AuthResult.AUTH_REQUEST_FAILED if !BeginSync(*operation.sync, operation.client, *operation.session, DeviceFromKey(*operation.material), Clock.UnixNow(), null, NativeNonce, - transport, TextFromCString(operation.payload), - BytesFromPointer(operation.response, AsyncResponseCapacity)) { + transport, "", BytesFromPointer(operation.response, AsyncResponseCapacity), + SyncFlow.{.context = cast(*void)operation, .load = LoadOperationSyncPage, + .prepare = PrepareOperationSyncBody, .save = SaveOperationSyncPage, + .accept = AcceptOperationSyncPage, .clock = OperationSyncClock}) { ReleaseStoredOperation(operation) return false } @@ -373,15 +414,7 @@ PollStoredOperation :: (operation: *StoredOperation) -> bool { } response := TextUntilNul(BytesFromPointer(operation.response, AsyncResponseCapacity)) server := TextUntilNul(operation.server[:]) - if operation.result == cast(s32)AuthResult.AUTH_OK { - if storage_apply_sync_response_json(response) == 0 { - operation.result = cast(s32)AuthResult.AUTH_PAYLOAD_FAILED - } else { - storage_purge_synced_deleted_data() - operation.changed = storage_last_sync_changed() - SyncNodeStorageRecordResponse(server, response) - } - } else if operation.sync.status == 413 && SyncNodeStorageRecordFull(server, response) { + if operation.sync.status == 413 && SyncNodeStorageRecordFull(server, response) { operation.result = cast(s32)SyncResult.SYNC_STORAGE_FULL } } else if operation.is_deletion { diff --git a/src/storage/sync_exchange.zi b/src/storage/sync_exchange.zi index 05c5d772c..123b4953b 100644 --- a/src/storage/sync_exchange.zi +++ b/src/storage/sync_exchange.zi @@ -21,6 +21,8 @@ Habits :: #import, file "storage_habits.zi"; #import, file "sync_record_crypto.zi"; #import, file "storage_json_sql.zi"; #import, file "sync_apply.zi"; +#import, file "sync_page_store.zi"; +#import "sync_flow" c :: #system_library "c"; malloc :: (size: usize) -> *void #foreign c; free :: (ptr: *void) #foreign c; @@ -30,6 +32,10 @@ LOG_INFO :: 3 #program_export storage_build_sync_payload_json :: (user_id_hash: string, public_key_hex: string) -> *u8 { + return BuildStoredSyncPayload(user_id_hash, public_key_hex, STORAGE_SYNC_OP_BATCH_LIMIT) +} + +BuildStoredSyncPayload :: (user_id_hash: string, public_key_hex: string, maximum_records: s32) -> *u8 { json: StorageJsonBuilder since_server_version: s64 through_seq: s64 @@ -76,9 +82,9 @@ storage_build_sync_payload_json :: (user_id_hash: string, public_key_hex: string if storage_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, 0) < STORAGE_SYNC_RECORD_FIELDS { since_server_version = 0 } - // Chunked diary photos must stay within the public node's 1 MiB request. - batch_limit: s32 = cast(s32)STORAGE_SYNC_OP_BATCH_LIMIT - if db_select_int64_text("SELECT EXISTS(SELECT 1 FROM sync_outbox WHERE entity_type='cell_record')", 0) != 0 { + batch_limit := maximum_records + // Chunked diary photos must stay within the public node's request limit. + if batch_limit > 64 && db_select_int64_text("SELECT EXISTS(SELECT 1 FROM sync_outbox WHERE entity_type='cell_record')", 0) != 0 { batch_limit = 64 } through_seq = storage_sync_outbox_batch_seq(batch_limit) @@ -99,11 +105,11 @@ storage_build_sync_payload_json :: (user_id_hash: string, public_key_hex: string storage_json_builder_append_text(*json, ",") storage_json_builder_key_value_text(*json, "client_id", storage_sync_client_id()) server_version_text: [24] u8 - AppFormatS32(cast(s32)since_server_version, server_version_text[:]) + unused AppFormatS64(since_server_version, server_version_text[:]) storage_json_builder_append_text(*json, ",\"since_server_version\":") storage_json_builder_append_text(*json, AppTextFromCString(*server_version_text[0], 24)) client_clock_text: [24] u8 - AppFormatS32(cast(s32)storage_meta_int64(STORAGE_SYNC_SERVER_CLOCK_KEY, 0), client_clock_text[:]) + unused AppFormatS64(storage_meta_int64(STORAGE_SYNC_SERVER_CLOCK_KEY, 0), client_clock_text[:]) storage_json_builder_append_text(*json, ",\"client_clock\":") storage_json_builder_append_text(*json, AppTextFromCString(*client_clock_text[0], 24)) if storage_meta_int64(STORAGE_SYNC_FULL_REPLACE_KEY, 0) != 0 { @@ -157,6 +163,11 @@ storage_sync_response_has_changes :: (json: string) -> s32 { #program_export storage_apply_sync_response_json :: (response_json: string) -> s32 { + return ApplyStoredSyncResponse(response_json, null) +} + +ApplyStoredSyncResponse :: (response_json: string, page: *SyncPageState) -> s32 { + incomplete := page != null && page.pending server_version: s64 server_clock: s64 old_server_version: s64 @@ -230,7 +241,7 @@ storage_apply_sync_response_json :: (response_json: string) -> s32 { return 0 } storage_clear_uploaded_outbox(storage_state_handle().pending_sync_outbox_seq) - if storage_apply_sync_records(response_json) == 0 { + if storage_apply_sync_records(response_json) == 0 || (page != null && !SaveStoredSyncPage(storage_get_setting_text("sync_server_url"), page)) { exec_sql("ROLLBACK TO sync_apply") exec_sql("RELEASE sync_apply") return 0 @@ -255,10 +266,10 @@ storage_apply_sync_response_json :: (response_json: string) -> s32 { if server_version > old_server_version && storage_sync_response_has_changes(response_json) != 0 { storage_state_handle().last_sync_changed = 1 } - if server_version > 0 { + if server_version > 0 && !incomplete { storage_set_meta_int64("sync_last_server_version", server_version) } - if server_clock > 0 { + if server_clock > 0 && !incomplete { storage_set_meta_int64(STORAGE_SYNC_SERVER_CLOCK_KEY, server_clock) } if storage_json_extract_text(response_json, "$.server_state_hash", *server_hash[0], @@ -276,6 +287,10 @@ storage_apply_sync_response_json :: (response_json: string) -> s32 { storage_set_meta_int64(STORAGE_SYNC_APPLY_REVIEW_KEY, 0) set_meta_int64_buf(STORAGE_SYNC_FULL_REPLACE_KEY, 0) storage_state_handle().pending_sync_outbox_seq = 0 + if incomplete { + storage_schedule_persist() + return 1 + } storage_set_meta_int64("sync_full_upload_done", 1) storage_set_meta_int64(STORAGE_SYNC_BACKFILL_KEY, 1) storage_set_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 1) diff --git a/src/storage/sync_page_store.zi b/src/storage/sync_page_store.zi new file mode 100644 index 000000000..9b8986137 --- /dev/null +++ b/src/storage/sync_page_store.zi @@ -0,0 +1,43 @@ +// Inbe persists Daochi Client's cursor with this account and node. The +// library owns page requests, validation, retries and completion decisions. +#import "sync_flow" +#import "text_buffer" +#import "c_string" +#import, file "sync_nodes.zi"; +#import, file "storage_settings.zi"; +#import, file "../cells/package_files.zi"; + +LoadStoredSyncPage :: (server: string, state: *SyncPageState) -> bool { + address := PackageText(server, "") + defer { PackageFree(address) } + enabled := SyncNodeCursorKey(address, "record_pages") + pending := SyncNodeCursorKey(address, "record_pending") + cursor := SyncNodeCursorKey(address, "record_cursor") + defer { + PackageFree(enabled) + PackageFree(pending) + PackageFree(cursor) + } + state.enabled = storage_get_setting_int(enabled, 0) != 0 + state.pending = storage_get_setting_int(pending, 0) != 0 + return CopyCString(storage_get_setting_text(cursor), state.cursor[:]) +} + +SaveStoredSyncPage :: (server: string, state: *SyncPageState) -> bool { + address := PackageText(server, "") + defer { PackageFree(address) } + enabled := SyncNodeCursorKey(address, "record_pages") + pending := SyncNodeCursorKey(address, "record_pending") + cursor := SyncNodeCursorKey(address, "record_cursor") + defer { + PackageFree(enabled) + PackageFree(pending) + PackageFree(cursor) + } + storage_set_setting_int(enabled, ifx state.enabled then 1 else 0) + storage_set_setting_int(pending, ifx state.pending then 1 else 0) + storage_set_setting_text(cursor, TextUntilNul(state.cursor[:])) + return storage_setting_text_equals(enabled, ifx state.enabled then "1" else "0") != 0 && + storage_setting_text_equals(pending, ifx state.pending then "1" else "0") != 0 && + storage_setting_text_equals(cursor, TextUntilNul(state.cursor[:])) != 0 +} diff --git a/src/storage/sync_runtime.zi b/src/storage/sync_runtime.zi index 29f39e9e6..d7ba58b3b 100644 --- a/src/storage/sync_runtime.zi +++ b/src/storage/sync_runtime.zi @@ -6,6 +6,8 @@ #import "account" #import "social" #import "sync" + #import, file "sync_async.zi"; + #import, file "../platform/network_transport.zi"; #import "events" #import "delegated" #import, file "delegate_store.zi"; @@ -37,6 +39,7 @@ malloc :: (size: usize) -> *void #foreign c; free :: (pointer: *void) #foreign c; time :: (out: *s64) -> s64 #foreign c; + SyncWait :: (microseconds: u32) -> s32 #foreign c "usleep"; SyncResponseCapacity :: 16777216; SocialSend :: 1; @@ -94,62 +97,20 @@ if StoredAccountMode() == AccountModeDelegate { return SyncStoredDelegate(base_url) } - account: AccountKeys - if sync_account_load(*account) == 0 { - return cast(s32)AuthResult.AUTH_NO_ACCOUNT - } - material: DeviceKeyMaterial - if !LoadOrCreateDeviceKey(*material) { - return cast(s32)AuthResult.AUTH_SIGN_FAILED - } - client_id := storage_sync_client_id() - account_id := AccountFieldText(*account.public_id[0], - ACCOUNT_PUBLIC_ID_HEX_SIZE) - public_key := AccountFieldText(*account.public_key_hex[0], - ACCOUNT_PUBLIC_KEY_HEX_SIZE) - // An updated device downloads first and then sends its changes in a - // second exchange right away. downloading_first := storage_sync_records_download_first() - payload := storage_build_sync_payload_json(account_id, public_key) - if payload == null { - WipeDeviceKey(*material) - return cast(s32)AuthResult.AUTH_PAYLOAD_FAILED + operation: StoredOperation + network: NetworkTransfer + if !BeginStoredSync(*operation, base_url, NetworkTransport(*network)) { + return operation.result } - response := cast(*u8)malloc(SyncResponseCapacity) - if response == null { - storage_free_sync_payload_json(payload) - WipeDeviceKey(*material) - return cast(s32)AuthResult.AUTH_PAYLOAD_FAILED + while !PollStoredOperation(*operation) { + unused SyncWait(10000) } - now := time(null) - session: Session - LoadSavedSession(*session, now) - status: s32 = 0 - result := SyncAccountPayload(base_url, *account, client_id, - *session, DeviceFromKey(*material), now, - TextFromCString(payload), - BytesFromPointer(response, SyncResponseCapacity), *status) - SaveSession(*session) - storage_free_sync_payload_json(payload) - WipeDeviceKey(*material) - reply := TextUntilNul(BytesFromPointer(response, SyncResponseCapacity)) - outcome := cast(s32)result - if result == cast(AuthResult)AuthResult.AUTH_OK { - if storage_apply_sync_response_json(reply) == 0 { - outcome = cast(s32)AuthResult.AUTH_PAYLOAD_FAILED - } else { - storage_purge_synced_deleted_data() - SyncNodeStorageRecordResponse(base_url, reply) - } - } else if status == 413 && SyncNodeStorageRecordFull(base_url, reply) { - outcome = cast(s32)SyncResult.SYNC_STORAGE_FULL - } - free(cast(*void)response) - if outcome == cast(s32)AuthResult.AUTH_OK && downloading_first && + if operation.result == cast(s32)AuthResult.AUTH_OK && downloading_first && !storage_sync_records_download_first() { return SyncStoredAccount(base_url) } - return outcome + return operation.result } // Signs in to a node for photo copies and registers this device there. diff --git a/tests/AndroidHttpTransportTest.java b/tests/AndroidHttpTransportTest.java index f3fc9a2db..3c3146989 100644 --- a/tests/AndroidHttpTransportTest.java +++ b/tests/AndroidHttpTransportTest.java @@ -130,6 +130,7 @@ public final class AndroidHttpTransportTest { id = request(transport, "GET", base + path, 8, 1000, new byte[0]); require(waitFor(transport, id) == 0 && transport.response(id) == null, "bounded response " + path); + require(transport.failure(id) == 2, "capacity failure classification " + path); transport.cancel(id); } id = request(transport, "GET", base + "/redirect", 64, 1000, new byte[0]); diff --git a/tests/flow_fake/http_transfer.zi b/tests/flow_fake/http_transfer.zi index 24a132355..258739cea 100644 --- a/tests/flow_fake/http_transfer.zi +++ b/tests/flow_fake/http_transfer.zi @@ -9,8 +9,11 @@ flow_fake_c :: #system_library "c"; // main.c of tests/telegram_account_flow_zi_test.sh moves the fixture clock. FlowClockAdvance :: (seconds: s64) #foreign flow_fake_c; +HTTP_TRANSFER_FAILURE_CAPACITY :: 2 + HttpTransfer :: struct { handle: s32 + failure: s32 job: *void output: []u8 } @@ -51,9 +54,9 @@ HttpTransferStart :: (transfer: *HttpTransfer, request: HttpRequest, output: []u FlowObserve(request.url) authorization: [8193]u8 configured := (request.accept.count == 0 || FlowHeader(operation, "Accept", request.accept)) && - (request.content_type.count == 0 || FlowHeader(operation, "Content-Type", request.content_type)) && - (request.token.count == 0 || (HttpTransferBearer(request.token, authorization[:]) && - FlowHeader(operation, "Authorization", TextUntilNul(authorization[:])))) + (request.content_type.count == 0 || FlowHeader(operation, "Content-Type", request.content_type)) && + (request.token.count == 0 || (HttpTransferBearer(request.token, authorization[:]) && + FlowHeader(operation, "Authorization", TextUntilNul(authorization[:])))) for index: cast(s64)0..request.headers.count - 1 { configured = configured && FlowHeader(operation, request.headers[index].name, request.headers[index].value) } diff --git a/tests/storage_sync_behavior.zi b/tests/storage_sync_behavior.zi index 0af733608..d199b1b6d 100644 --- a/tests/storage_sync_behavior.zi +++ b/tests/storage_sync_behavior.zi @@ -44,6 +44,8 @@ #import, file "../src/storage/media_node_session.zi"; #import, file "../src/storage/sync_session.zi"; #import "auth" +#import "sync_flow" +#import, file "../src/storage/sync_page_store.zi"; #import "text_buffer" #import, file "../src/storage/sqlite.zi"; #import "json_scan" @@ -91,6 +93,44 @@ test_node_cursors :: () { storage_close() } +test_record_page_storage :: () { + root := make_root("sync-record-pages") + check_true("init paged device", storage_init(root) != 0) + set_account() + storage_set_setting_text("sync_server_url", "https://one.example") + page: SyncPageState + page.enabled = true + page.pending = true + unused AppCopyText("{\"version\":7,\"collection\":\"private.inbe.v1.habits\",\"id\":\"habit-1\"}", page.cursor[:]) + check_true("persist page recovery", SaveStoredSyncPage("https://one.example", *page)) + loaded: SyncPageState + check_true("load saved progress", LoadStoredSyncPage("https://one.example", *loaded)) + check_true("resume the last committed cursor", loaded.enabled && loaded.pending && + TextUntilNul(loaded.cursor[:]) == TextUntilNul(page.cursor[:])) + check_true("another node has its own state", LoadStoredSyncPage("https://two.example", *loaded) && !loaded.enabled) + storage_set_setting_text("sync_public_id", "other-account") + check_true("another account has its own state", LoadStoredSyncPage("https://one.example", *loaded) && !loaded.enabled) + set_account() + check_true("restore the original account progress", LoadStoredSyncPage("https://one.example", *loaded) && loaded.pending) + response := "{\"protocol_version\":6,\"server_version\":50,\"server_clock\":50,\"changes\":{\"encrypted_records\":[]}}" + check_true("apply a partial page", ApplyStoredSyncResponse(response, *page) != 0) + check_true("keep watermark until the terminal page", storage_meta_int64("sync_last_server_version", 0) == 0) + check_true("keep repairs pending during download", storage_meta_int64("sync_full_upload_done", 0) == 0) + page.pending = false + page.cursor[0] = cast(u8)0 + // A failing cursor write must reject the page and leave previous progress. + check_true("install cursor failure fixture", SqliteExecText(storage_db_handle(), + "CREATE TEMP TRIGGER reject_cursor BEFORE UPDATE ON settings WHEN NEW.key GLOB 'sync_node_*.record_cursor' BEGIN SELECT RAISE(ABORT,'test cursor failure'); END") == SQLITE_OK) + check_true("failed cursor save rejects the page", ApplyStoredSyncResponse(response, *page) == 0) + check_true("remove cursor failure fixture", SqliteExecText(storage_db_handle(), "DROP TRIGGER reject_cursor") == SQLITE_OK) + check_true("failed page does not advance watermark", storage_meta_int64("sync_last_server_version", 0) == 0) + check_true("failed page keeps the committed cursor", LoadStoredSyncPage("https://one.example", *loaded) && loaded.pending && loaded.cursor[0] != cast(u8)0) + check_true("accept terminal page", ApplyStoredSyncResponse(response, *page) != 0) + check_true("terminal page advances the watermark", storage_meta_int64("sync_last_server_version", 0) == 50) + check_true("terminal page clears recovery cursor", LoadStoredSyncPage("https://one.example", *loaded) && !loaded.pending && loaded.cursor[0] == cast(u8)0) + storage_close() +} + // Bytes a node stores for one Diary file in the payload, counted like // Daochi's account storage: every stored field of each encrypted record. payload_file_bytes :: (name: string, hash: string, chunks_only: bool) -> s64 { @@ -2160,6 +2200,7 @@ Check :: () -> s32 { guard := support_begin("/tmp/inbe-storage-sync-zi-test/") if guard != 0 { return guard } test_node_cursors() + test_record_page_storage() test_node_storage_and_media() test_payload_after_upload_marker() test_backfill_includes_existing_habits() diff --git a/ziran.lock b/ziran.lock index 11bf6fbcd..2bea998fc 100644 --- a/ziran.lock +++ b/ziran.lock @@ -1,14 +1,6 @@ { "format": 1, "packages": [ - { - "commit": "08969407c431b7c512f76a58033e5d4101523353", - "dependencies": {"raylib": "p5a77c9c35110e925"}, - "id": "p01e33d1c2eade6ee", - "name": "kryon", - "ref": "master", - "url": "https://github.com/kryonlabs/kryon.git" - }, { "commit": "b31299434c785d0f2bd82f36b5bfd2fd36ba1216", "dependencies": {}, @@ -20,7 +12,7 @@ }, { "commit": "db5349c476bdc243c22b9c73fda4ec06d0a25ffc", - "dependencies": {"crypto": "pf039d7a776333200", "kryon": "p01e33d1c2eade6ee"}, + "dependencies": {"crypto": "pf039d7a776333200", "kryon": "p428f68b688585898"}, "id": "p0c45ce9ed9a0f8cf", "name": "Apps", "ref": "master", @@ -36,7 +28,7 @@ }, { "commit": "1cb9df183089a77c8a16448a556186f30b261a98", - "dependencies": {"kryon": "p01e33d1c2eade6ee"}, + "dependencies": {"kryon": "p428f68b688585898"}, "id": "p1b13dda539ed2a39", "name": "Kss", "ref": "master", @@ -44,12 +36,28 @@ }, { "commit": "bad2caaa84ef3eae61e40c99bf572d2b060d0ca2", - "dependencies": {"kryon": "p01e33d1c2eade6ee"}, + "dependencies": {"kryon": "p428f68b688585898"}, "id": "p20eb23d3e6db3711", "name": "Game2D", "ref": "master", "url": "https://github.com/kryonlabs/game2d.git" }, + { + "commit": "84c85dcee0dae247ddf5b1ffe8b3a0fa47d4e1dc", + "dependencies": {"oqs": "p7098ac9cd92f7e5f"}, + "id": "p2727791b55a0fa95", + "name": "DaochiClient", + "ref": "master", + "url": "https://github.com/ziranlang/daochi-client.git" + }, + { + "commit": "3f9dc9505f3cb7cea145812a0842cb6beae1e73c", + "dependencies": {"raylib": "p5a77c9c35110e925"}, + "id": "p428f68b688585898", + "name": "kryon", + "ref": "master", + "url": "https://github.com/kryonlabs/kryon.git" + }, { "commit": "fafa2cea97d0cb379d832086ba45a989fa6f9c8e", "dependencies": {}, @@ -61,20 +69,12 @@ }, { "commit": "6cca06d8bfe92965c23fc3565bf85589ea5f67d3", - "dependencies": {"daochi_client": "p6cee0aacfacc847c", "kryon": "p01e33d1c2eade6ee", "monocypher": "p809b4d31a48625a3", "qrcode": "p0dfa70137405b096"}, + "dependencies": {"daochi_client": "p2727791b55a0fa95", "kryon": "p428f68b688585898", "monocypher": "p809b4d31a48625a3", "qrcode": "p0dfa70137405b096"}, "id": "p6c7fcf5c8bd3d9db", "name": "Self", "ref": "master", "url": "https://github.com/waozixyz/self.git" }, - { - "commit": "a2d71cf96ac5d19c113b1fc55c7f6005d4d6d26e", - "dependencies": {"oqs": "p7098ac9cd92f7e5f"}, - "id": "p6cee0aacfacc847c", - "name": "DaochiClient", - "ref": "master", - "url": "https://github.com/ziranlang/daochi-client.git" - }, { "commit": "22d2ddbdc236796953c905a05a2cebf5ab1b4ff3", "dependencies": {"liboqs": "p7766f6d294641ca8"}, @@ -120,7 +120,7 @@ }, { "commit": "4b0e25bd2df86b54a8f5af3db48282bd41daab1a", - "dependencies": {"kryon": "p01e33d1c2eade6ee"}, + "dependencies": {"kryon": "p428f68b688585898"}, "id": "pe8e21a1f2b5ff52c", "name": "plot", "ref": "master", @@ -145,7 +145,7 @@ } ], "root": { - "dependencies": {"apps": "p0c45ce9ed9a0f8cf", "crypto": "pf039d7a776333200", "curl": "pd4bcd11b7d623073", "daochi_client": "p6cee0aacfacc847c", "game2d": "p20eb23d3e6db3711", "kryon": "p01e33d1c2eade6ee", "kss": "p1b13dda539ed2a39", "liboqs": "p7766f6d294641ca8", "mingcute": "pe1027b9d16704793", "monocypher": "p809b4d31a48625a3", "oqs": "p7098ac9cd92f7e5f", "plot": "pe8e21a1f2b5ff52c", "qrcode": "p0dfa70137405b096", "rini": "p09fa77e94b0dfc6c", "self": "p6c7fcf5c8bd3d9db", "sqlite": "peaa43c4b7ef72edf"}, + "dependencies": {"apps": "p0c45ce9ed9a0f8cf", "crypto": "pf039d7a776333200", "curl": "pd4bcd11b7d623073", "daochi_client": "p2727791b55a0fa95", "game2d": "p20eb23d3e6db3711", "kryon": "p428f68b688585898", "kss": "p1b13dda539ed2a39", "liboqs": "p7766f6d294641ca8", "mingcute": "pe1027b9d16704793", "monocypher": "p809b4d31a48625a3", "oqs": "p7098ac9cd92f7e5f", "plot": "pe8e21a1f2b5ff52c", "qrcode": "p0dfa70137405b096", "rini": "p09fa77e94b0dfc6c", "self": "p6c7fcf5c8bd3d9db", "sqlite": "peaa43c4b7ef72edf"}, "name": "Inbe" }, "toolchain": {