diff --git a/Makefile b/Makefile index 6450fee2d..a9f76fb1d 100644 --- a/Makefile +++ b/Makefile @@ -2968,6 +2968,12 @@ sync-recovery-test: build-laws $(ZI2C_BIN) test: sync-recovery-test +.PHONY: sync-record-fields-test +sync-record-fields-test: + python3 tests/sync_record_fields_test.py + +test: sync-record-fields-test + # Validate before producing release packages, including direct artifact targets. $(APPIMAGE_TARGET) $(DEB_TARGET) $(RPM_TARGET) $(SNAP_TARGET) $(FLATPAK_TARGET) $(CLICK_TARGET) $(CHROME_WEB_STORE_ZIP) $(FIREFOX_ADDONS_ZIP): | version-check android-release android-bundle android-copy-release-apks android-copy-fdroid-apk android-copy-bundle windows-setup site: version-check diff --git a/src/storage/schema.zi b/src/storage/schema.zi index 76c8a7a73..75c2086f3 100644 --- a/src/storage/schema.zi +++ b/src/storage/schema.zi @@ -232,6 +232,9 @@ migrate_schema :: () -> s32 { if !SchemaExec(SchemaHabitChanges) { return SchemaUpgradeFailed() } + if !SchemaExec(SchemaSyncRecordCopies) { + return SchemaUpgradeFailed() + } if SchemaHasColumn("habit_days", "answers") && !SchemaExec(SchemaDayAnswersToEntries) { return SchemaUpgradeFailed() } diff --git a/src/storage/schema_sql.zi b/src/storage/schema_sql.zi index 9fc03a2ba..2e8870883 100644 --- a/src/storage/schema_sql.zi +++ b/src/storage/schema_sql.zi @@ -262,6 +262,17 @@ CREATE TABLE IF NOT EXISTS habit_changes( id TEXT PRIMARY KEY, CREATE INDEX IF NOT EXISTS habit_changes_habit ON habit_changes(habit_id,changed_at); SQL +// Every synced record as it last arrived or left, with all its fields. A +// version that doesn't know some of a record's fields keeps them here and +// sends them back under its own, so it never removes what a newer version +// added, on this device or for the account's other devices. +SchemaSyncRecordCopies :: #string SQL +CREATE TABLE IF NOT EXISTS sync_record_copies( collection TEXT NOT NULL, + id TEXT NOT NULL, + json TEXT NOT NULL, + PRIMARY KEY(collection,id)) +SQL + // A habit's entries: each a moment of a day, or a span from its start to // its end (0 while it runs), with its answers to the habit's questions as // a JSON array. An entry belongs to the day it started. diff --git a/src/storage/storage_core.zi b/src/storage/storage_core.zi index 0bc6c77e1..83353776b 100644 --- a/src/storage/storage_core.zi +++ b/src/storage/storage_core.zi @@ -437,6 +437,7 @@ storage_reset_sync_state :: () { storage_set_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 0) storage_set_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, 0) exec_sql("DELETE FROM sync_outbox") + exec_sql("DELETE FROM sync_record_copies") if storage_has_sync_account() != 0 { storage_enqueue_all_sync_state() } @@ -1491,6 +1492,60 @@ storage_encrypted_record_cleanup :: (plain: *StorageJsonBuilder, aad: *StorageJs return ok } +// Keeps a synced record with every field it has, as it arrived or left. +storage_sync_record_copy_store :: (collection: string, record_id: string, json: string) { + stmt: *void = null + if storage_db_handle() == null || collection.count == 0 || record_id.count == 0 { + return + } + sql: [256]u8 + AppCopyText("INSERT INTO sync_record_copies(collection,id,json) SELECT ?1,?2,?3 WHERE json_valid(?3) ON CONFLICT(collection,id) DO UPDATE SET json=excluded.json", sql[:]) + if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK { + return + } + bind_text(stmt, 1, collection) + bind_text(stmt, 2, record_id) + bind_text(stmt, 3, json) + unused SqliteStep(stmt) + SqliteFinalize(stmt) +} + +// A record goes out with every field it last had, under this version's own +// fields, so an older version never removes fields a newer one added. The +// merged record becomes the kept copy. +storage_sync_record_merge :: (plain: *StorageJsonBuilder, collection: string, record_id: string) { + stmt: *void = null + if storage_db_handle() == null || plain.ok == 0 || plain.data == null { + return + } + own := AppTextFromCString(plain.data, cast(s32)(plain.len + 1)) + sql: [256]u8 + AppCopyText("SELECT json_patch(json,?3) FROM sync_record_copies WHERE collection=?1 AND id=?2 AND json_valid(json) AND json_valid(?3) AND json_type(json)='object' AND json_type(?3)='object'", sql[:]) + if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK { + return + } + bind_text(stmt, 1, collection) + bind_text(stmt, 2, record_id) + bind_text(stmt, 3, own) + merged: StorageJsonBuilder + merged.ok = 1 + if SqliteStep(stmt) == SQLITE_ROW { + value := SqliteColumnText(stmt, 0) + if value != null { + storage_json_builder_append(*merged, value) + } + } + SqliteFinalize(stmt) + if merged.ok != 0 && merged.data != null && merged.len > 0 { + storage_json_builder_free(plain) + plain.* = merged + } else { + storage_json_builder_free(*merged) + } + storage_sync_record_copy_store(collection, record_id, + AppTextFromCString(plain.data, cast(s32)(plain.len + 1))) +} + storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, collection: string, key_id: string, record_id: string, entity_type: string, entity_id: string, local_date: s32, queued_at: s64, deleted: s32) -> s32 { plain: StorageJsonBuilder aad: StorageJsonBuilder @@ -1509,6 +1564,10 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co if plain.ok == 0 || plain.data == null { return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok) } + storage_sync_record_merge(*plain, collection, record_id) + if plain.ok == 0 || plain.data == null { + return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok) + } storage_sha256_hex(plain.data, plain.len, *content_hash[0]) sealed_len = plain.len + 16 sealed = cast(*u8)malloc(cast(usize)sealed_len) @@ -2372,6 +2431,10 @@ storage_apply_encrypted_records :: (response_json: string) -> s32 { } ok = ifx (record_id.count > 0 && key_id.count > 0 && nonce.count > 0 && ciphertext.count > 0 && storage_decrypt_record(collection, record_id, key_id, nonce, ciphertext, content_hash, *plaintext) != 0) then (1) else (0) + if ok != 0 { + storage_sync_record_copy_store(collection, record_id, + AppTextFromCString(plaintext, 2147483647)) + } if ok == 0 { LogStorageText(LOG_WARNING, "SYNC: encrypted record decrypt failed: ", collection) @@ -2611,7 +2674,8 @@ storage_apply_remote_full_snapshot :: (response_json: string) -> s32 { return ok } -storage_clear_local_sync_data_sqls: [11] string = .[ +storage_clear_local_sync_data_sqls: [12] string = .[ +"DELETE FROM sync_record_copies", "DELETE FROM habit_entries", "DELETE FROM habit_changes", "DELETE FROM elist_items", diff --git a/tests/storage_sync_behavior.zi b/tests/storage_sync_behavior.zi index c54e27ebd..9b88800c4 100644 --- a/tests/storage_sync_behavior.zi +++ b/tests/storage_sync_behavior.zi @@ -552,6 +552,42 @@ test_older_records_download_again_once :: () { storage_close() } +// A record's fields this version doesn't read, such as ones a newer version +// added, go back out with it under this version's own changes, so an older +// version never removes them for the account's other devices. +test_unknown_record_fields_go_back_out :: () { + first := make_root("sync-fields-a") + second := make_root("sync-fields-b") + check_true("init device a", storage_init(first) != 0) + set_account() + habits_a = Habits.{} + check_true("add habit", habits_add_custom(*habits_a, "Cold Shower", Green(), 0, 0) == 0) + habits_save(*habits_a) + habits_free(*habits_a) + check_true("device a payload", build_payload()) + records := records_array() + storage_close() + + check_true("init device b", storage_init(second) != 0) + set_account() + check_true("device b applies", storage_apply_sync_response_json(response_with(5, records)) != 0) + storage_close() + check_int("device b keeps the record as it arrived", + raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits'"), 1) + // A field from a newer version that this one doesn't read. + raw_exec(second, "UPDATE sync_record_copies SET json=json_set(json,'$.future_field','kept') WHERE collection='private.inbe.v1.habits'") + check_true("reopen device b", storage_init(second) != 0) + habits_b = Habits.{} + check_true("device b loads", storage_habits_load(cast(*void)*habits_b) != 0) + habits_b.items[0].icon = 23 + habits_save(*habits_b) + habits_free(*habits_b) + check_true("device b payload", build_payload()) + storage_close() + check_int("the change goes out with the field it doesn't know", + raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits' AND json_extract(json,'$.future_field')='kept' AND json_extract(json,'$.icon')=23"), 1) +} + // Device A's records applied on device B come out as the same habits. test_records_round_trip_between_devices :: () { first := make_root("sync-device-a") @@ -1698,6 +1734,7 @@ Check :: () -> s32 { test_orphan_habit_days_reset_cursor() test_habit_name_repair_runs_once() test_older_records_download_again_once() + test_unknown_record_fields_go_back_out() test_records_round_trip_between_devices() test_cell_records_round_trip() test_cell_record_identity() diff --git a/tests/sync_record_fields.txt b/tests/sync_record_fields.txt new file mode 100644 index 000000000..ed582ef57 --- /dev/null +++ b/tests/sync_record_fields.txt @@ -0,0 +1,69 @@ +version 2 +activity +after +answers +before +changed_at +ciphertext +collection +color_b +color_g +color_r +comment +completed +completed_at +content_hash +count +counter_enabled +counter_target +deactivated +deleted_at +done +duration +duration_seconds +ended_at +energy +goal +habit_id +hash +high +icon +id +json +key +key_id +kind +list_id +local_date +low +mood_after +mood_before +name +nonce +note +question +question_text +questions +reminder_hour +rounds_hash +schedule +session_id +sort_order +source +span +span_goal +span_high +span_low +started_at +stress +sync_activity +sync_mode +tags +text +timestamp +title +topic +type +updated_at +value +weekdays diff --git a/tests/sync_record_fields_test.py b/tests/sync_record_fields_test.py new file mode 100644 index 000000000..69ffa89e7 --- /dev/null +++ b/tests/sync_record_fields_test.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Synced records may only gain fields together with STORAGE_SYNC_RECORD_FIELDS. + +A device keeps only the fields its app version reads from a record. When a +newer version reads more, raising STORAGE_SYNC_RECORD_FIELDS makes every +device download its records again once, so it gets the new fields. This +check records the fields read for the current value and fails when they +change without it. +""" +from pathlib import Path +import re +import sys + +ROOT = Path(__file__).resolve().parents[1] +RECORDED = ROOT / "tests/sync_record_fields.txt" + + +def read_fields(): + fields = set() + for path in sorted((ROOT / "src/storage").glob("*.zi")): + fields.update(re.findall(r"json_extract\(value,\s*'\$\.([A-Za-z_][A-Za-z0-9_.]*)'\)", path.read_text())) + return fields + + +def current_version(): + core = (ROOT / "src/storage/storage_core.zi").read_text() + return int(re.search(r"^STORAGE_SYNC_RECORD_FIELDS :: (\d+)$", core, re.M)[1]) + + +def main(): + fields = read_fields() + version = current_version() + if "--record" in sys.argv: + RECORDED.write_text(f"version {version}\n" + "".join(f"{name}\n" for name in sorted(fields))) + return 0 + lines = RECORDED.read_text().split() + recorded_version = int(lines[1]) + recorded = set(lines[2:]) + if recorded_version == version and recorded == fields: + print(f"Synced record fields: {len(fields)} fields for record version {version}") + return 0 + added = sorted(fields - recorded) + removed = sorted(recorded - fields) + if recorded_version == version: + print("Synced record fields changed without raising STORAGE_SYNC_RECORD_FIELDS " + f"(src/storage/storage_core.zi): added {added}, removed {removed}. Raise it so " + "devices download their records again, then run " + "python3 tests/sync_record_fields_test.py --record", file=sys.stderr) + else: + print("STORAGE_SYNC_RECORD_FIELDS changed; record its fields with " + "python3 tests/sync_record_fields_test.py --record", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main())