Never drop synced fields an app version doesn't know

Each device now keeps every synced record with all its fields, as it
arrived or left. When it sends a record again, its own fields go on top of
that copy, so an older version never removes fields a newer version added,
on its own device or for the account's other devices. Clearing local sync
data or resetting sync clears these copies too.

A new check lists every field the sync reads from records. When that list
changes, STORAGE_SYNC_RECORD_FIELDS has to rise with it, so every device
downloads its records again once and gets the new fields.
This commit is contained in:
waozi committed 2026-10-13 22:21:23 -03:00
1 parent 01cce2c41c
commit 880f5b8c63
7 files changed
+247 -1

No files matched your search

+6
View File
@@ -2968,6 +2968,12 @@ sync-recovery-test: build-laws $(ZI2C_BIN)
test: sync-recovery-test
.PHONY: sync-record-fields-test
sync-record-fields-test:
python3 tests/sync_record_fields_test.py
test: sync-record-fields-test
# Validate before producing release packages, including direct artifact targets.
$(APPIMAGE_TARGET) $(DEB_TARGET) $(RPM_TARGET) $(SNAP_TARGET) $(FLATPAK_TARGET) $(CLICK_TARGET) $(CHROME_WEB_STORE_ZIP) $(FIREFOX_ADDONS_ZIP): | version-check
android-release android-bundle android-copy-release-apks android-copy-fdroid-apk android-copy-bundle windows-setup site: version-check
+3
View File
@@ -232,6 +232,9 @@ migrate_schema :: () -> s32 {
if !SchemaExec(SchemaHabitChanges) {
return SchemaUpgradeFailed()
}
if !SchemaExec(SchemaSyncRecordCopies) {
return SchemaUpgradeFailed()
}
if SchemaHasColumn("habit_days", "answers") && !SchemaExec(SchemaDayAnswersToEntries) {
return SchemaUpgradeFailed()
}
+11
View File
@@ -262,6 +262,17 @@ CREATE TABLE IF NOT EXISTS habit_changes( id TEXT PRIMARY KEY,
CREATE INDEX IF NOT EXISTS habit_changes_habit ON habit_changes(habit_id,changed_at);
SQL
// Every synced record as it last arrived or left, with all its fields. A
// version that doesn't know some of a record's fields keeps them here and
// sends them back under its own, so it never removes what a newer version
// added, on this device or for the account's other devices.
SchemaSyncRecordCopies :: #string SQL
CREATE TABLE IF NOT EXISTS sync_record_copies( collection TEXT NOT NULL,
id TEXT NOT NULL,
json TEXT NOT NULL,
PRIMARY KEY(collection,id))
SQL
// A habit's entries: each a moment of a day, or a span from its start to
// its end (0 while it runs), with its answers to the habit's questions as
// a JSON array. An entry belongs to the day it started.
+65 -1
View File
@@ -437,6 +437,7 @@ storage_reset_sync_state :: () {
storage_set_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 0)
storage_set_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, 0)
exec_sql("DELETE FROM sync_outbox")
exec_sql("DELETE FROM sync_record_copies")
if storage_has_sync_account() != 0 {
storage_enqueue_all_sync_state()
}
@@ -1491,6 +1492,60 @@ storage_encrypted_record_cleanup :: (plain: *StorageJsonBuilder, aad: *StorageJs
return ok
}
// Keeps a synced record with every field it has, as it arrived or left.
storage_sync_record_copy_store :: (collection: string, record_id: string, json: string) {
stmt: *void = null
if storage_db_handle() == null || collection.count == 0 || record_id.count == 0 {
return
}
sql: [256]u8
AppCopyText("INSERT INTO sync_record_copies(collection,id,json) SELECT ?1,?2,?3 WHERE json_valid(?3) ON CONFLICT(collection,id) DO UPDATE SET json=excluded.json", sql[:])
if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK {
return
}
bind_text(stmt, 1, collection)
bind_text(stmt, 2, record_id)
bind_text(stmt, 3, json)
unused SqliteStep(stmt)
SqliteFinalize(stmt)
}
// A record goes out with every field it last had, under this version's own
// fields, so an older version never removes fields a newer one added. The
// merged record becomes the kept copy.
storage_sync_record_merge :: (plain: *StorageJsonBuilder, collection: string, record_id: string) {
stmt: *void = null
if storage_db_handle() == null || plain.ok == 0 || plain.data == null {
return
}
own := AppTextFromCString(plain.data, cast(s32)(plain.len + 1))
sql: [256]u8
AppCopyText("SELECT json_patch(json,?3) FROM sync_record_copies WHERE collection=?1 AND id=?2 AND json_valid(json) AND json_valid(?3) AND json_type(json)='object' AND json_type(?3)='object'", sql[:])
if SqlitePrepare(storage_db_handle(), *sql[0], -1, *stmt, null) != SQLITE_OK {
return
}
bind_text(stmt, 1, collection)
bind_text(stmt, 2, record_id)
bind_text(stmt, 3, own)
merged: StorageJsonBuilder
merged.ok = 1
if SqliteStep(stmt) == SQLITE_ROW {
value := SqliteColumnText(stmt, 0)
if value != null {
storage_json_builder_append(*merged, value)
}
}
SqliteFinalize(stmt)
if merged.ok != 0 && merged.data != null && merged.len > 0 {
storage_json_builder_free(plain)
plain.* = merged
} else {
storage_json_builder_free(*merged)
}
storage_sync_record_copy_store(collection, record_id,
AppTextFromCString(plain.data, cast(s32)(plain.len + 1)))
}
storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, collection: string, key_id: string, record_id: string, entity_type: string, entity_id: string, local_date: s32, queued_at: s64, deleted: s32) -> s32 {
plain: StorageJsonBuilder
aad: StorageJsonBuilder
@@ -1509,6 +1564,10 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
if plain.ok == 0 || plain.data == null {
return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok)
}
storage_sync_record_merge(*plain, collection, record_id)
if plain.ok == 0 || plain.data == null {
return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok)
}
storage_sha256_hex(plain.data, plain.len, *content_hash[0])
sealed_len = plain.len + 16
sealed = cast(*u8)malloc(cast(usize)sealed_len)
@@ -2372,6 +2431,10 @@ storage_apply_encrypted_records :: (response_json: string) -> s32 {
}
ok = ifx (record_id.count > 0 && key_id.count > 0 && nonce.count > 0 && ciphertext.count > 0 && storage_decrypt_record(collection, record_id, key_id, nonce,
ciphertext, content_hash, *plaintext) != 0) then (1) else (0)
if ok != 0 {
storage_sync_record_copy_store(collection, record_id,
AppTextFromCString(plaintext, 2147483647))
}
if ok == 0 {
LogStorageText(LOG_WARNING, "SYNC: encrypted record decrypt failed: ",
collection)
@@ -2611,7 +2674,8 @@ storage_apply_remote_full_snapshot :: (response_json: string) -> s32 {
return ok
}
storage_clear_local_sync_data_sqls: [11] string = .[
storage_clear_local_sync_data_sqls: [12] string = .[
"DELETE FROM sync_record_copies",
"DELETE FROM habit_entries",
"DELETE FROM habit_changes",
"DELETE FROM elist_items",
+37
View File
@@ -552,6 +552,42 @@ test_older_records_download_again_once :: () {
storage_close()
}
// A record's fields this version doesn't read, such as ones a newer version
// added, go back out with it under this version's own changes, so an older
// version never removes them for the account's other devices.
test_unknown_record_fields_go_back_out :: () {
first := make_root("sync-fields-a")
second := make_root("sync-fields-b")
check_true("init device a", storage_init(first) != 0)
set_account()
habits_a = Habits.{}
check_true("add habit", habits_add_custom(*habits_a, "Cold Shower", Green(), 0, 0) == 0)
habits_save(*habits_a)
habits_free(*habits_a)
check_true("device a payload", build_payload())
records := records_array()
storage_close()
check_true("init device b", storage_init(second) != 0)
set_account()
check_true("device b applies", storage_apply_sync_response_json(response_with(5, records)) != 0)
storage_close()
check_int("device b keeps the record as it arrived",
raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits'"), 1)
// A field from a newer version that this one doesn't read.
raw_exec(second, "UPDATE sync_record_copies SET json=json_set(json,'$.future_field','kept') WHERE collection='private.inbe.v1.habits'")
check_true("reopen device b", storage_init(second) != 0)
habits_b = Habits.{}
check_true("device b loads", storage_habits_load(cast(*void)*habits_b) != 0)
habits_b.items[0].icon = 23
habits_save(*habits_b)
habits_free(*habits_b)
check_true("device b payload", build_payload())
storage_close()
check_int("the change goes out with the field it doesn't know",
raw_count(second, "SELECT COUNT(*) FROM sync_record_copies WHERE collection='private.inbe.v1.habits' AND json_extract(json,'$.future_field')='kept' AND json_extract(json,'$.icon')=23"), 1)
}
// Device A's records applied on device B come out as the same habits.
test_records_round_trip_between_devices :: () {
first := make_root("sync-device-a")
@@ -1698,6 +1734,7 @@ Check :: () -> s32 {
test_orphan_habit_days_reset_cursor()
test_habit_name_repair_runs_once()
test_older_records_download_again_once()
test_unknown_record_fields_go_back_out()
test_records_round_trip_between_devices()
test_cell_records_round_trip()
test_cell_record_identity()
+69
View File
@@ -0,0 +1,69 @@
version 2
activity
after
answers
before
changed_at
ciphertext
collection
color_b
color_g
color_r
comment
completed
completed_at
content_hash
count
counter_enabled
counter_target
deactivated
deleted_at
done
duration
duration_seconds
ended_at
energy
goal
habit_id
hash
high
icon
id
json
key
key_id
kind
list_id
local_date
low
mood_after
mood_before
name
nonce
note
question
question_text
questions
reminder_hour
rounds_hash
schedule
session_id
sort_order
source
span
span_goal
span_high
span_low
started_at
stress
sync_activity
sync_mode
tags
text
timestamp
title
topic
type
updated_at
value
weekdays
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
"""Synced records may only gain fields together with STORAGE_SYNC_RECORD_FIELDS.
A device keeps only the fields its app version reads from a record. When a
newer version reads more, raising STORAGE_SYNC_RECORD_FIELDS makes every
device download its records again once, so it gets the new fields. This
check records the fields read for the current value and fails when they
change without it.
"""
from pathlib import Path
import re
import sys
ROOT = Path(__file__).resolve().parents[1]
RECORDED = ROOT / "tests/sync_record_fields.txt"
def read_fields():
fields = set()
for path in sorted((ROOT / "src/storage").glob("*.zi")):
fields.update(re.findall(r"json_extract\(value,\s*'\$\.([A-Za-z_][A-Za-z0-9_.]*)'\)", path.read_text()))
return fields
def current_version():
core = (ROOT / "src/storage/storage_core.zi").read_text()
return int(re.search(r"^STORAGE_SYNC_RECORD_FIELDS :: (\d+)$", core, re.M)[1])
def main():
fields = read_fields()
version = current_version()
if "--record" in sys.argv:
RECORDED.write_text(f"version {version}\n" + "".join(f"{name}\n" for name in sorted(fields)))
return 0
lines = RECORDED.read_text().split()
recorded_version = int(lines[1])
recorded = set(lines[2:])
if recorded_version == version and recorded == fields:
print(f"Synced record fields: {len(fields)} fields for record version {version}")
return 0
added = sorted(fields - recorded)
removed = sorted(recorded - fields)
if recorded_version == version:
print("Synced record fields changed without raising STORAGE_SYNC_RECORD_FIELDS "
f"(src/storage/storage_core.zi): added {added}, removed {removed}. Raise it so "
"devices download their records again, then run "
"python3 tests/sync_record_fields_test.py --record", file=sys.stderr)
else:
print("STORAGE_SYNC_RECORD_FIELDS changed; record its fields with "
"python3 tests/sync_record_fields_test.py --record", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())