Never drop synced fields an app version doesn't know
Each device now keeps every synced record with all its fields, as it arrived or left. When it sends a record again, its own fields go on top of that copy, so an older version never removes fields a newer version added, on its own device or for the account's other devices. Clearing local sync data or resetting sync clears these copies too. A new check lists every field the sync reads from records. When that list changes, STORAGE_SYNC_RECORD_FIELDS has to rise with it, so every device downloads its records again once and gets the new fields.
This commit is contained in:
1 parent
01cce2c41c
commit
880f5b8c63
7 files changed
+247
-1
No files matched your search
@@ -2968,6 +2968,12 @@ sync-recovery-test: build-laws $(ZI2C_BIN)
|
||||
|
||||
test: sync-recovery-test
|
||||
|
||||
.PHONY: sync-record-fields-test
|
||||
sync-record-fields-test:
|
||||
python3 tests/sync_record_fields_test.py
|
||||
|
||||
test: sync-record-fields-test
|
||||
|
||||
# Validate before producing release packages, including direct artifact targets.
|
||||
$(APPIMAGE_TARGET) $(DEB_TARGET) $(RPM_TARGET) $(SNAP_TARGET) $(FLATPAK_TARGET) $(CLICK_TARGET) $(CHROME_WEB_STORE_ZIP) $(FIREFOX_ADDONS_ZIP): | version-check
|
||||
android-release android-bundle android-copy-release-apks android-copy-fdroid-apk android-copy-bundle windows-setup site: version-check
|
||||
|
||||
@@ -232,6 +232,9 @@ migrate_schema :: () -> s32 {
|
||||
if !SchemaExec(SchemaHabitChanges) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
if !SchemaExec(SchemaSyncRecordCopies) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
if SchemaHasColumn("habit_days", "answers") && !SchemaExec(SchemaDayAnswersToEntries) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
|
||||
@@ -262,6 +262,17 @@ CREATE TABLE IF NOT EXISTS habit_changes( id TEXT PRIMARY KEY,
|
||||
CREATE INDEX IF NOT EXISTS habit_changes_habit ON habit_changes(habit_id,changed_at);
|
||||
SQL
|
||||
|
||||
// Every synced record as it last arrived or left, with all its fields. A
|
||||
// version that doesn't know some of a record's fields keeps them here and
|
||||
// sends them back under its own, so it never removes what a newer version
|
||||
// added, on this device or for the account's other devices.
|
||||
SchemaSyncRecordCopies :: #string SQL
|
||||
CREATE TABLE IF NOT EXISTS sync_record_copies( collection TEXT NOT NULL,
|
||||
id TEXT NOT NULL,
|
||||
json TEXT NOT NULL,
|
||||
PRIMARY KEY(collection,id))
|
||||
SQL
|
||||
|
||||
// A habit's entries: each a moment of a day, or a span from its start to
|
||||
// its end (0 while it runs), with its answers to the habit's questions as
|
||||
// a JSON array. An entry belongs to the day it started.
|
||||
|
||||
@@ -437,6 +437,7 @@ storage_reset_sync_state :: () {
|
||||
storage_set_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 0)
|
||||
storage_set_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, 0)
|
||||
exec_sql("DELETE FROM sync_outbox")
|
||||
exec_sql("DELETE FROM sync_record_copies")
|
||||
if storage_has_sync_account() != 0 {
|
||||
storage_enqueue_all_sync_state()
|
||||
}
|
||||
@@ -1491,6 +1492,60 @@ storage_encrypted_record_cleanup :: (plain: *StorageJsonBuilder, aad: *StorageJs
|
||||
return ok
|
||||
}
|
||||
|
||||
// Keeps a synced record with every field it has, as it arrived or left.
|
||||
storage_sync_record_copy_store :: (collection: string, record_id: string, json: string) {
|
||||
stmt: *void = null
|
||||
if storage_db_handle() == null || collection.count == 0 || record_id.count == 0 {
|
||||
return
|
||||
}
|
||||
sql: [256]u8
|
||||
AppCopyText("INSERT INTO sync_record_copies(collection,id,json) SELECT ?1,?2,?3 WHERE json_valid(?3) ON CONFLICT(collection,id) DO UPDATE SET json=excluded.json", sql[:])
|
||||
if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK {
|
||||
return
|
||||
}
|
||||
bind_text(stmt, 1, collection)
|
||||
bind_text(stmt, 2, record_id)
|
||||
bind_text(stmt, 3, json)
|
||||
unused SqliteStep(stmt)
|
||||
SqliteFinalize(stmt)
|
||||
}
|
||||
|
||||
// A record goes out with every field it last had, under this version's own
|
||||
// fields, so an older version never removes fields a newer one added. The
|
||||
// merged record becomes the kept copy.
|
||||
storage_sync_record_merge :: (plain: *StorageJsonBuilder, collection: string, record_id: string) {
|
||||
stmt: *void = null
|
||||
if storage_db_handle() == null || plain.ok == 0 || plain.data == null {
|
||||
return
|
||||
}
|
||||
own := AppTextFromCString(plain.data, cast(s32)(plain.len + 1))
|
||||
sql: [256]u8
|
||||
AppCopyText("SELECT json_patch(json,?3) FROM sync_record_copies WHERE collection=?1 AND id=?2 AND json_valid(json) AND json_valid(?3) AND json_type(json)='object' AND json_type(?3)='object'", sql[:])
|
||||
if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK {
|
||||
return
|
||||
}
|
||||
bind_text(stmt, 1, collection)
|
||||
bind_text(stmt, 2, record_id)
|
||||
bind_text(stmt, 3, own)
|
||||
merged: StorageJsonBuilder
|
||||
merged.ok = 1
|
||||
if SqliteStep(stmt) == SQLITE_ROW {
|
||||
value := SqliteColumnText(stmt, 0)
|
||||
if value != null {
|
||||
storage_json_builder_append(*merged, value)
|
||||
}
|
||||
}
|
||||
SqliteFinalize(stmt)
|
||||
if merged.ok != 0 && merged.data != null && merged.len > 0 {
|
||||
storage_json_builder_free(plain)
|
||||
plain.* = merged
|
||||
} else {
|
||||
storage_json_builder_free(*merged)
|
||||
}
|
||||
storage_sync_record_copy_store(collection, record_id,
|
||||
AppTextFromCString(plain.data, cast(s32)(plain.len + 1)))
|
||||
}
|
||||
|
||||
storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, collection: string, key_id: string, record_id: string, entity_type: string, entity_id: string, local_date: s32, queued_at: s64, deleted: s32) -> s32 {
|
||||
plain: StorageJsonBuilder
|
||||
aad: StorageJsonBuilder
|
||||
@@ -1509,6 +1564,10 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
|
||||
if plain.ok == 0 || plain.data == null {
|
||||
return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok)
|
||||
}
|
||||
storage_sync_record_merge(*plain, collection, record_id)
|
||||
if plain.ok == 0 || plain.data == null {
|
||||
return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok)
|
||||
}
|
||||
storage_sha256_hex(plain.data, plain.len, *content_hash[0])
|
||||
sealed_len = plain.len + 16
|
||||
sealed = cast(*u8)malloc(cast(usize)sealed_len)
|
||||
@@ -2372,6 +2431,10 @@ storage_apply_encrypted_records :: (response_json: string) -> s32 {
|
||||
}
|
||||
ok = ifx (record_id.count > 0 && key_id.count > 0 && nonce.count > 0 && ciphertext.count > 0 && storage_decrypt_record(collection, record_id, key_id, nonce,
|
||||
ciphertext, content_hash, *plaintext) != 0) then (1) else (0)
|
||||
if ok != 0 {
|
||||
storage_sync_record_copy_store(collection, record_id,
|
||||
AppTextFromCString(plaintext, 2147483647))
|
||||
}
|
||||
if ok == 0 {
|
||||
LogStorageText(LOG_WARNING, "SYNC: encrypted record decrypt failed: ",
|
||||
collection)
|
||||
@@ -2611,7 +2674,8 @@ storage_apply_remote_full_snapshot :: (response_json: string) -> s32 {
|
||||
return ok
|
||||
}
|
||||
|
||||
storage_clear_local_sync_data_sqls: [11] string = .[
|
||||
storage_clear_local_sync_data_sqls: [12] string = .[
|
||||
"DELETE FROM sync_record_copies",
|
||||
"DELETE FROM habit_entries",
|
||||
"DELETE FROM habit_changes",
|
||||
"DELETE FROM elist_items",
|
||||
|
||||
@@ -552,6 +552,42 @@ test_older_records_download_again_once :: () {
|
||||
storage_close()
|
||||
}
|
||||
|
||||
// A record's fields this version doesn't read, such as ones a newer version
|
||||
// added, go back out with it under this version's own changes, so an older
|
||||
// version never removes them for the account's other devices.
|
||||
test_unknown_record_fields_go_back_out :: () {
|
||||
first := make_root("sync-fields-a")
|
||||
second := make_root("sync-fields-b")
|
||||
check_true("init device a", storage_init(first) != 0)
|
||||
set_account()
|
||||
habits_a = Habits.{}
|
||||
check_true("add habit", habits_add_custom(*habits_a, "Cold Shower", Green(), 0, 0) == 0)
|
||||
habits_save(*habits_a)
|
||||
habits_free(*habits_a)
|
||||
check_true("device a payload", build_payload())
|
||||
records := records_array()
|
||||
storage_close()
|
||||
|
||||
check_true("init device b", storage_init(second) != 0)
|
||||
set_account()
|
||||
check_true("device b applies", storage_apply_sync_response_json(response_with(5, records)) != 0)
|
||||
storage_close()
|
||||
check_int("device b keeps the record as it arrived",
|
||||
raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits'"), 1)
|
||||
// A field from a newer version that this one doesn't read.
|
||||
raw_exec(second, "UPDATE sync_record_copies SET json=json_set(json,'$.future_field','kept') WHERE collection='private.inbe.v1.habits'")
|
||||
check_true("reopen device b", storage_init(second) != 0)
|
||||
habits_b = Habits.{}
|
||||
check_true("device b loads", storage_habits_load(cast(*void)*habits_b) != 0)
|
||||
habits_b.items[0].icon = 23
|
||||
habits_save(*habits_b)
|
||||
habits_free(*habits_b)
|
||||
check_true("device b payload", build_payload())
|
||||
storage_close()
|
||||
check_int("the change goes out with the field it doesn't know",
|
||||
raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits' AND json_extract(json,'$.future_field')='kept' AND json_extract(json,'$.icon')=23"), 1)
|
||||
}
|
||||
|
||||
// Device A's records applied on device B come out as the same habits.
|
||||
test_records_round_trip_between_devices :: () {
|
||||
first := make_root("sync-device-a")
|
||||
@@ -1698,6 +1734,7 @@ Check :: () -> s32 {
|
||||
test_orphan_habit_days_reset_cursor()
|
||||
test_habit_name_repair_runs_once()
|
||||
test_older_records_download_again_once()
|
||||
test_unknown_record_fields_go_back_out()
|
||||
test_records_round_trip_between_devices()
|
||||
test_cell_records_round_trip()
|
||||
test_cell_record_identity()
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
version 2
|
||||
activity
|
||||
after
|
||||
answers
|
||||
before
|
||||
changed_at
|
||||
ciphertext
|
||||
collection
|
||||
color_b
|
||||
color_g
|
||||
color_r
|
||||
comment
|
||||
completed
|
||||
completed_at
|
||||
content_hash
|
||||
count
|
||||
counter_enabled
|
||||
counter_target
|
||||
deactivated
|
||||
deleted_at
|
||||
done
|
||||
duration
|
||||
duration_seconds
|
||||
ended_at
|
||||
energy
|
||||
goal
|
||||
habit_id
|
||||
hash
|
||||
high
|
||||
icon
|
||||
id
|
||||
json
|
||||
key
|
||||
key_id
|
||||
kind
|
||||
list_id
|
||||
local_date
|
||||
low
|
||||
mood_after
|
||||
mood_before
|
||||
name
|
||||
nonce
|
||||
note
|
||||
question
|
||||
question_text
|
||||
questions
|
||||
reminder_hour
|
||||
rounds_hash
|
||||
schedule
|
||||
session_id
|
||||
sort_order
|
||||
source
|
||||
span
|
||||
span_goal
|
||||
span_high
|
||||
span_low
|
||||
started_at
|
||||
stress
|
||||
sync_activity
|
||||
sync_mode
|
||||
tags
|
||||
text
|
||||
timestamp
|
||||
title
|
||||
topic
|
||||
type
|
||||
updated_at
|
||||
value
|
||||
weekdays
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Synced records may only gain fields together with STORAGE_SYNC_RECORD_FIELDS.
|
||||
|
||||
A device keeps only the fields its app version reads from a record. When a
|
||||
newer version reads more, raising STORAGE_SYNC_RECORD_FIELDS makes every
|
||||
device download its records again once, so it gets the new fields. This
|
||||
check records the fields read for the current value and fails when they
|
||||
change without it.
|
||||
"""
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
RECORDED = ROOT / "tests/sync_record_fields.txt"
|
||||
|
||||
|
||||
def read_fields():
|
||||
fields = set()
|
||||
for path in sorted((ROOT / "src/storage").glob("*.zi")):
|
||||
fields.update(re.findall(r"json_extract\(value,\s*'\$\.([A-Za-z_][A-Za-z0-9_.]*)'\)", path.read_text()))
|
||||
return fields
|
||||
|
||||
|
||||
def current_version():
|
||||
core = (ROOT / "src/storage/storage_core.zi").read_text()
|
||||
return int(re.search(r"^STORAGE_SYNC_RECORD_FIELDS :: (\d+)$", core, re.M)[1])
|
||||
|
||||
|
||||
def main():
|
||||
fields = read_fields()
|
||||
version = current_version()
|
||||
if "--record" in sys.argv:
|
||||
RECORDED.write_text(f"version {version}\n" + "".join(f"{name}\n" for name in sorted(fields)))
|
||||
return 0
|
||||
lines = RECORDED.read_text().split()
|
||||
recorded_version = int(lines[1])
|
||||
recorded = set(lines[2:])
|
||||
if recorded_version == version and recorded == fields:
|
||||
print(f"Synced record fields: {len(fields)} fields for record version {version}")
|
||||
return 0
|
||||
added = sorted(fields - recorded)
|
||||
removed = sorted(recorded - fields)
|
||||
if recorded_version == version:
|
||||
print("Synced record fields changed without raising STORAGE_SYNC_RECORD_FIELDS "
|
||||
f"(src/storage/storage_core.zi): added {added}, removed {removed}. Raise it so "
|
||||
"devices download their records again, then run "
|
||||
"python3 tests/sync_record_fields_test.py --record", file=sys.stderr)
|
||||
else:
|
||||
print("STORAGE_SYNC_RECORD_FIELDS changed; record its fields with "
|
||||
"python3 tests/sync_record_fields_test.py --record", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user