Isolate Lumi private data and preserve legacy storage safely

This commit is contained in:
waozi committed 2026-10-08 01:19:19 -03:00
1 parent 39c5018790
commit 1b5d03ab34
10 files changed
+552 -63

No files matched your search

+22
View File
@@ -45,3 +45,25 @@ meditation audio stay inside their respective profile directories. `PREFIX`
can be set explicitly for a system installation; it does not change the
production data location. Debug and production windows have separate desktop
identities and single-instance locks.
## Private Lumi sync collection
New Lumi records use `private.inbe.v2.lumi` and key generation `inbe-lumi-1`.
Only validated `cell.lumi.*` records belong to that collection. Its key is
HMAC-SHA-256 of the owner's full ML-DSA private key with the distinct context
`inbe-private-lumi-record-key-v2`, using the existing record encryption format.
Diary, feedback, sessions and other private records retain their existing
owner-only collections and legacy key context. A Lumi delegate must receive
only the approved Lumi key; the legacy global key and combined
`private.inbe.v1.cells` collection must never be delegated.
Opening an owner profile queues existing Lumi settings for upload under the
new collection and saves `sync_lumi_collection_v2_done` in the same SQLite
savepoint. Existing settings, history and pending outbox entries are retained.
The marker makes restarts idempotent; a failed marker write rolls back the
queued copies so a restart can retry. A legacy Lumi record received later is
still decrypted with the old key, preserved locally, and queued under the new
key. The old combined collection remains readable by the owner so historical
Diary and feedback records continue to restore. Grant revocation cannot erase
plaintext or keys already received by a delegate; new grants and key generations
must be approved separately.
+7 -2
View File
@@ -1,5 +1,6 @@
// Account-owned cell records share the v6 encrypted outbox. Device settings,
// signing keys and credentials never enter this collection.
// Account-owned cell records share the v6 encrypted outbox. Lumi has its own
// collection and key; Diary and feedback retain the owner-only legacy scope.
// Device settings, signing keys and credentials never enter either collection.
#import, file "storage_core.zi";
#import, file "sqlite.zi";
#import, file "state.zi";
@@ -25,6 +26,10 @@ CellRecordValid :: (key: string) -> bool {
return true
}
LumiCellRecordValid :: (key: string) -> bool {
return CellRecordValid(key) && key[:10] == "cell.lumi."
}
CellRecordRead :: (key: string) -> string {
if !CellRecordValid(key) {
return ""
+6
View File
@@ -54,6 +54,7 @@ storage_migrate_default_habit_ids :: () -> s32 #foreign c;
storage_migrate_habit_ids_to_uuid :: () -> s32 #foreign c;
storage_migrate_default_meditation_activity_mask :: () -> s32 #foreign c;
storage_migrate_sync_enabled_flag :: () -> s32 #foreign c;
storage_migrate_lumi_collection :: () -> s32 #foreign c;
storage_materialize_session_habit_days :: () -> s32 #foreign c;
migrate_legacy_file_sessions_once :: (root: *u8) -> s32 #foreign c;
#import, file "user.zi";
@@ -215,6 +216,11 @@ storage_init :: (root: string) -> s32 {
storage_migrate_habit_ids_to_uuid()
storage_migrate_default_meditation_activity_mask()
storage_migrate_sync_enabled_flag()
if storage_migrate_lumi_collection() == 0 {
sqlite3_close(storage_state_handle().db)
storage_state_handle().db = null
return 0
}
storage_materialize_session_habit_days()
if migrate_legacy_file_sessions_once(*storage_state_handle().root[0]) == 0 {
LogStorage(LOG_WARNING, "STORAGE: historical session import deferred; source files retained")
+12 -22
View File
@@ -180,11 +180,10 @@ JsonEscapeLength :: (text: string) -> s64 {
return length
}
#program_export
storage_json_builder_append_string :: (json: *StorageJsonBuilder,
value: *u8) {
// Escape the complete slice, including embedded NUL bytes. The builder's
// checked capacity is the limit; fields are never silently omitted.
JsonAppendStringText :: (json: *StorageJsonBuilder, text: string) {
if json == null || json.ok == 0 { return }
text: string = json_cstring_view(value)
length := JsonEscapeLength(text)
if length < 0 || storage_json_builder_reserve(json, length + 1) == 0 {
json.ok = 0
@@ -247,26 +246,17 @@ storage_json_builder_append_string :: (json: *StorageJsonBuilder,
json.len = write
}
#program_export
storage_json_builder_append_string :: (json: *StorageJsonBuilder,
value: *u8) {
JsonAppendStringText(json, json_cstring_view(value))
}
#program_export
storage_json_builder_key_value_text :: (json: *StorageJsonBuilder, key: string, value: string) {
key_buf: [256] u8
value_buf: [4096] u8
if key.count >= 256 || value.count >= 4096 {
return
}
index: s32 = 0
while index < cast(s32)key.count {
key_buf[index] = key[index]
index += 1
}
key_buf[index] = cast(u8)0
index = 0
while index < cast(s32)value.count {
value_buf[index] = value[index]
index += 1
}
value_buf[index] = cast(u8)0
storage_json_builder_append_key_string(json, *key_buf[0], *value_buf[0])
JsonAppendStringText(json, key)
JsonAppendSlice(json, ":")
JsonAppendStringText(json, value)
}
#program_export
+164 -25
View File
@@ -16,6 +16,7 @@ Scan :: #import "json_scan"
#import, file "storage_habit_materialize.zi";
#import, file "sync_sql.zi";
#import, file "cell_records.zi";
#import, file "storage_layout.zi";
#import, file "export_sql.zi";
#import, file "sync_crypto.zi";
#import, file "sync_crypto_hmac.zi";
@@ -353,6 +354,39 @@ storage_enqueue_all_sync_state :: () {
}
}
#program_export
// The outbox and marker commit together. The old settings and server history
// remain intact; new writes and queued copies use the separate Lumi scope.
storage_migrate_lumi_collection :: () -> s32 {
if storage_db_handle() == null {
return 0
}
if storage_meta_int64(LumiCollectionMigrationKey, 0) != 0 ||
storage_get_setting_text(STORAGE_SYNC_PRIVATE_KEY_KEY).count == 0 {
return 1
}
if exec_sql("SAVEPOINT inbe_lumi_collection_migration") == 0 {
return 0
}
queued := exec_sql("INSERT INTO sync_outbox(entity_type,entity_id,local_date,queued_at) SELECT 'cell_record',key,0,updated_at FROM settings WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.lumi.*' AND length(key) BETWEEN 12 AND 180 AND key NOT GLOB '*[^a-z0-9.-]*' ON CONFLICT(entity_type,entity_id,local_date) DO NOTHING")
marked: s32 = 0
if queued != 0 {
marked = exec_sql("INSERT INTO meta(key,value) VALUES('sync_lumi_collection_v2_done','1') ON CONFLICT(key) DO UPDATE SET value=excluded.value")
}
if queued == 0 || marked == 0 {
unused exec_sql("ROLLBACK TO inbe_lumi_collection_migration")
unused exec_sql("RELEASE inbe_lumi_collection_migration")
return 0
}
if exec_sql("RELEASE inbe_lumi_collection_migration") == 0 {
unused exec_sql("ROLLBACK TO inbe_lumi_collection_migration")
unused exec_sql("RELEASE inbe_lumi_collection_migration")
return 0
}
storage_schedule_persist()
return 1
}
#program_export
storage_reset_sync_state :: () {
if storage_db_handle() == null {
@@ -1189,9 +1223,15 @@ storage_append_sync_op_payload :: (json: *StorageJsonBuilder, entity_type: strin
}
}
storage_encrypted_collection_for_entity :: (entity_type: string) -> string {
storage_encrypted_collection_for_entity :: (entity_type: string, entity_id: string) -> string {
if entity_type == "cell_record" {
return "private.inbe.v1.cells"
if LumiCellRecordValid(entity_id) {
return LumiPrivateCollection
}
if !CellRecordValid(entity_id) {
return ""
}
return LegacyPrivateCellsCollection
}
if entity_type == "app_preference" {
return "private.inbe.v1.app-preferences"
@@ -1266,6 +1306,34 @@ storage_private_record_key :: (out: *u8) -> s32 {
out)
}
// The collection and generation must agree. In particular, no legacy key can
// decrypt Lumi v2, and the Lumi key cannot decrypt Diary, feedback or sessions.
#program_export
storage_private_collection_key :: (collection: string, key_id: string, out: *u8) -> s32 {
if out == null {
return 0
}
SyncCryptoWipe(out, 32)
if collection == LumiPrivateCollection {
if key_id != LumiPrivateKeyID {
return 0
}
return storage_private_record_key_for_context(LumiPrivateKeyContext, out)
}
if key_id != STORAGE_SYNC_RECORD_KEY_ID ||
storage_decrypted_array_name(collection).count == 0 {
return 0
}
return storage_private_record_key(out)
}
storage_encrypted_collection_key_id :: (collection: string) -> string {
if collection == LumiPrivateCollection {
return LumiPrivateKeyID
}
return STORAGE_SYNC_RECORD_KEY_ID
}
storage_encrypted_record_cleanup :: (plain: *StorageJsonBuilder, aad: *StorageJsonBuilder,
sealed: *u8, sealed_hex: *u8, ok: s32) -> s32 {
storage_json_builder_free(plain)
@@ -1275,7 +1343,7 @@ storage_encrypted_record_cleanup :: (plain: *StorageJsonBuilder, aad: *StorageJs
return ok
}
storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, collection: string, record_id: string, entity_type: string, entity_id: string, local_date: s32, queued_at: s64, deleted: s32) -> s32 {
storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, collection: string, key_id: string, record_id: string, entity_type: string, entity_id: string, local_date: s32, queued_at: s64, deleted: s32) -> s32 {
plain: StorageJsonBuilder
aad: StorageJsonBuilder
nonce: [12] u8
@@ -1285,7 +1353,7 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
content_hash: [SYNC_PUBLIC_ID_HEX_SIZE] u8
sealed_len: s64
ok: s32 = 0
if collection.count == 0 || record_id.count == 0 {
if collection.count == 0 || key_id.count == 0 || record_id.count == 0 {
return 0
}
plain.ok = 1
@@ -1305,7 +1373,7 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
storage_json_builder_append_text(*aad, "\n")
storage_json_builder_append_text(*aad, record_id)
storage_json_builder_append_text(*aad, "\n")
storage_json_builder_append_text(*aad, STORAGE_SYNC_RECORD_KEY_ID)
storage_json_builder_append_text(*aad, key_id)
if aad.ok == 0 || aad.data == null {
return storage_encrypted_record_cleanup(*plain, *aad, sealed, sealed_hex, ok)
}
@@ -1325,7 +1393,7 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
storage_json_builder_append_text(json, ",")
storage_json_builder_key_value_text(json, "id", record_id)
storage_json_builder_append_text(json, ",")
storage_json_builder_key_value_text(json, "key_id", STORAGE_SYNC_RECORD_KEY_ID)
storage_json_builder_key_value_text(json, "key_id", key_id)
storage_json_builder_append_text(json, ",")
storage_json_builder_key_value_text(json, "nonce", AppTextFromCString(*nonce_hex[0], 25))
storage_json_builder_append_text(json, ",")
@@ -1347,12 +1415,10 @@ storage_append_encrypted_record_json :: (json: *StorageJsonBuilder, key: *u8, co
storage_append_encrypted_records_json :: (json: *StorageJsonBuilder, through_seq: s64) {
stmt: *void = null
key: [32] u8
has_key: s32
first: s32 = 1
step_result: s32 = SQLITE_DONE
storage_json_builder_append_text(json, "\"encrypted_records\":[")
has_key = storage_private_record_key(*key[0])
if has_key == 0 || storage_db_handle() == null {
if storage_db_handle() == null {
json.ok = 0
} else {
sql_cx13: [98] u8
@@ -1368,7 +1434,8 @@ storage_append_encrypted_records_json :: (json: *StorageJsonBuilder, through_seq
entity_id: string = AppTextFromCString(SqliteColumnText(stmt, 2), 8192)
local_date: s32 = sqlite3_column_int(stmt, 3)
queued_at: s64 = SqliteColumnInt64(stmt, 4)
collection: string = storage_encrypted_collection_for_entity(entity_type)
collection: string = storage_encrypted_collection_for_entity(entity_type, entity_id)
key_id: string = storage_encrypted_collection_key_id(collection)
is_delete: s32 = storage_sync_op_is_delete(entity_type, entity_id, local_date)
record_id: [180] u8
if collection.count == 0 || storage_encrypted_record_id(entity_type, entity_id, local_date,
@@ -1380,12 +1447,18 @@ storage_append_encrypted_records_json :: (json: *StorageJsonBuilder, through_seq
storage_json_builder_append_text(json, ",")
}
first = 0
if storage_append_encrypted_record_json(json, *key[0], collection, AppTextFromCString(*record_id[0], 180),
entity_type, entity_id, local_date,
queued_at, is_delete) == 0 {
if storage_private_collection_key(collection, key_id, *key[0]) == 0 {
json.ok = 0
break
}
if storage_append_encrypted_record_json(json, *key[0], collection, key_id, AppTextFromCString(*record_id[0], 180),
entity_type, entity_id, local_date,
queued_at, is_delete) == 0 {
SyncCryptoWipe(*key[0], 32)
json.ok = 0
break
}
SyncCryptoWipe(*key[0], 32)
}
if step_result != SQLITE_DONE {
json.ok = 0
@@ -1413,7 +1486,7 @@ storage_build_sync_payload_json :: (user_id_hash: string, public_key_hex: string
if storage_db_handle() == null || user_id_hash.count == 0 {
return null
}
if migrate_schema() == 0 {
if migrate_schema() == 0 || storage_migrate_lumi_collection() == 0 {
return null
}
if storage_materialize_session_habit_days() == 0 {
@@ -1809,7 +1882,7 @@ storage_apply_sync_social_json :: (response_json: string) -> s32 {
}
storage_decrypted_array_name :: (collection: string) -> string {
if collection == "private.inbe.v1.cells" {
if collection == LegacyPrivateCellsCollection || collection == LumiPrivateCollection {
return "cell_records"
}
if collection == "private.inbe.v1.app-preferences" {
@@ -1833,7 +1906,66 @@ storage_decrypted_array_name :: (collection: string) -> string {
return ""
}
storage_apply_decrypted_record :: (collection: string, plaintext: string) -> s32 {
// Cell plaintext has one object with one occurrence of each known string
// field. Decode member names when checking uniqueness, including escapes.
storage_cell_plaintext_fields_valid :: (plaintext: string) -> bool {
cursor := Scan.Space(plaintext, 0)
if cursor < 0 || cursor >= cast(s32)plaintext.count ||
plaintext[cursor] != cast(u8)123 {
return false
}
cursor = Scan.Space(plaintext, cursor + 1)
seen: s32 = 0
while cursor < cast(s32)plaintext.count {
field_bytes: [32]u8
field_count := Scan.ReadStringText(plaintext, cursor, field_bytes[:])
field_span := Scan.ReadString(plaintext, cursor)
if field_count <= 0 || !field_span.valid {
return false
}
field := TextFromBytes(field_bytes[:field_count])
flag: s32 = 0
if field == "key" {
flag = 1
} else if field == "value" {
flag = 2
} else if field == "updated_at" {
flag = 4
} else {
return false
}
if (seen & flag) != 0 {
return false
}
seen = seen | flag
cursor = Scan.Space(plaintext, field_span.after)
if cursor >= cast(s32)plaintext.count || plaintext[cursor] != cast(u8)58 {
return false
}
cursor = Scan.Space(plaintext, cursor + 1)
if cursor >= cast(s32)plaintext.count || plaintext[cursor] != cast(u8)34 {
return false
}
value_span := Scan.ReadString(plaintext, cursor)
if !value_span.valid {
return false
}
cursor = Scan.Space(plaintext, value_span.after)
if cursor >= cast(s32)plaintext.count {
return false
}
if plaintext[cursor] == cast(u8)125 {
return seen == 7 && Scan.Space(plaintext, cursor + 1) == cast(s32)plaintext.count
}
if plaintext[cursor] != cast(u8)44 {
return false
}
cursor = Scan.Space(plaintext, cursor + 1)
}
return false
}
storage_apply_decrypted_record :: (collection: string, record_id: string, plaintext: string) -> s32 {
array_name: string = storage_decrypted_array_name(collection)
response: StorageJsonBuilder = StorageJsonBuilder.{}
record_response: string
@@ -1858,6 +1990,10 @@ storage_apply_decrypted_record :: (collection: string, plaintext: string) -> s32
return 0
}
if array_name == "cell_records" {
if record_id.count == 0 || !storage_cell_plaintext_fields_valid(plaintext) {
storage_json_builder_free(*response)
return 0
}
records := Scan.FieldAt(record_response, Scan.FieldAt(record_response, 0, "changes"), "cell_records")
if Scan.ArrayCount(record_response, records) != 1 {
storage_json_builder_free(*response)
@@ -1869,11 +2005,19 @@ storage_apply_decrypted_record :: (collection: string, plaintext: string) -> s32
key_count := Scan.ReadStringText(record_response, Scan.FieldAt(record_response, at, "key"), key_bytes[:])
value_count := Scan.ReadStringText(record_response, Scan.FieldAt(record_response, at, "value"), value_bytes[:])
if key_count <= 0 || key_count >= 181 || value_count <= 0 || value_count >= 65536 ||
!CellRecordValid(TextFromBytes(key_bytes[:key_count])) {
TextFromBytes(key_bytes[:key_count]) != record_id ||
!CellRecordValid(TextFromBytes(key_bytes[:key_count])) ||
(collection == LumiPrivateCollection && !LumiCellRecordValid(TextFromBytes(key_bytes[:key_count]))) {
storage_json_builder_free(*response)
return 0
}
ok = storage_exec_json_user_sql(SyncCellRecordsSQL, record_response)
if ok != 0 && collection == LegacyPrivateCellsCollection &&
LumiCellRecordValid(TextFromBytes(key_bytes[:key_count])) {
// An older device may upload legacy Lumi after the local migration.
// Preserve its record and queue a copy under the isolated key.
ok = storage_enqueue_sync_entity("cell_record", TextFromBytes(key_bytes[:key_count]), 0)
}
} else if array_name == "app_preferences" {
records := Scan.FieldAt(record_response, Scan.FieldAt(record_response, 0, "changes"), "app_preferences")
count := Scan.ArrayCount(record_response, records)
@@ -1931,7 +2075,6 @@ storage_decrypted_record_cleanup :: (aad: *StorageJsonBuilder, sealed: *u8,
}
storage_decrypt_record :: (collection: string, record_id: string, key_id: string, nonce_hex: string, ciphertext_hex: string, content_hash: string, plaintext_out: **u8) -> s32 {
context: string
aad: StorageJsonBuilder = StorageJsonBuilder.{}
key: [32] u8
nonce: [12] u8
@@ -1940,11 +2083,7 @@ storage_decrypt_record :: (collection: string, record_id: string, key_id: string
actual_hash: [SYNC_PUBLIC_ID_HEX_SIZE] u8
sealed_len: s64
plaintext_out.* = null
if storage_compare_text(key_id, STORAGE_SYNC_RECORD_KEY_ID) != 0 {
return 0
}
context = STORAGE_SYNC_RECORD_KEY_CONTEXT
if ciphertext_hex.count < 32 || ciphertext_hex.count % 2 != 0 || SyncCryptoHexToBytes(nonce_hex, *nonce[0], size_of(type_of(nonce))) == 0 || storage_private_record_key_for_context(context, *key[0]) == 0 {
if ciphertext_hex.count < 32 || ciphertext_hex.count % 2 != 0 || SyncCryptoHexToBytes(nonce_hex, *nonce[0], size_of(type_of(nonce))) == 0 || storage_private_collection_key(collection, key_id, *key[0]) == 0 {
return 0
}
sealed_len = cast(s64)ciphertext_hex.count / 2
@@ -2076,7 +2215,7 @@ storage_apply_encrypted_records :: (response_json: string) -> s32 {
habit_batch_count += 1
habit_batch_collection = collection
} else {
ok = storage_apply_decrypted_record(collection,
ok = storage_apply_decrypted_record(collection, record_id,
AppTextFromCString(plaintext, 2147483647))
if ok == 0 {
LogStorageText(LOG_WARNING, "SYNC: encrypted record apply failed: ",
@@ -2087,7 +2226,7 @@ storage_apply_encrypted_records :: (response_json: string) -> s32 {
index += 1
}
if ok != 0 && habit_batch_count > 0 && habit_batch.ok != 0 && habit_batch.data != null {
ok = storage_apply_decrypted_record(habit_batch_collection,
ok = storage_apply_decrypted_record(habit_batch_collection, "",
AppTextFromCString(habit_batch.data, cast(s32)(habit_batch.len + 1)))
if ok == 0 {
LogStorage(LOG_WARNING, "SYNC: encrypted habit records apply failed")
+8
View File
@@ -23,3 +23,11 @@ ExportSessionsCsv :: "inbe-sessions.csv"
CurrentWebHome :: "/home/inbe"
PreviousWebHome :: "/home/breathing"
WebExportTemporaryPath :: "/tmp/inbe-web-export.zip"
// The owner-approved Mini App scope contains Lumi records only. The legacy
// collection remains readable by an owner for migration of older accounts.
LumiPrivateCollection :: "private.inbe.v2.lumi"
LegacyPrivateCellsCollection :: "private.inbe.v1.cells"
LumiPrivateKeyContext :: "inbe-private-lumi-record-key-v2"
LumiPrivateKeyID :: "inbe-lumi-1"
LumiCollectionMigrationKey :: "sync_lumi_collection_v2_done"
+6
View File
@@ -22,6 +22,12 @@
#law ExportFilenames custom ExportFilenameFormat == "inbe-%lld.zip" && ExportFilenameFallback == "inbe.zip" && ExportFilenamePrefix == "inbe-" && ExportFilenameSuffix == ".zip" && ExportSessionsCsv == "inbe-sessions.csv";
#law TemporaryNames custom ImportDatabaseTemporaryName == "import-inbe.db" && InspectDatabaseTemporaryName == "import-inspect-inbe.db" && ExportDatabaseTemporaryName == "export-inbe.db" && WebExportTemporaryPath == "/tmp/inbe-web-export.zip" && ArchiveDirectorySuffix == ".before-breathing-migration-";
// Product intent: delegate access is limited to Lumi, with a separate key.
// These obligations add the migration boundary without changing old layouts.
#law LumiCollectionIsIsolated custom LumiPrivateCollection == "private.inbe.v2.lumi" && LegacyPrivateCellsCollection == "private.inbe.v1.cells" && LumiPrivateCollection != LegacyPrivateCellsCollection;
#law LumiKeyIsSeparate custom LumiPrivateKeyContext == "inbe-private-lumi-record-key-v2" && LumiPrivateKeyID == "inbe-lumi-1" && LumiPrivateKeyContext != "breathing-private-record-key-v1" && LumiPrivateKeyID != "breathing-main-1";
#law LumiMigrationMarker custom LumiCollectionMigrationKey == "sync_lumi_collection_v2_done";
#program_export
storage_layout_laws_entry :: () -> s32 {
return 0
+10
View File
@@ -171,4 +171,14 @@ mutate storage/storage_layout.zi storage/storage_layout_laws.zi \
's/^PreviousDatabaseEntry :: .*/PreviousDatabaseEntry :: "inbe-data\/inbe.db"/' \
HistoricalEntryStaysImportable
mutate storage/storage_layout.zi storage/storage_layout_laws.zi \
's/^LumiPrivateCollection :: .*/LumiPrivateCollection :: "private.inbe.v1.cells"/' \
LumiCollectionIsIsolated
mutate storage/storage_layout.zi storage/storage_layout_laws.zi \
's/^LumiPrivateKeyContext :: .*/LumiPrivateKeyContext :: "breathing-private-record-key-v1"/' \
LumiKeyIsSeparate
mutate storage/storage_layout.zi storage/storage_layout_laws.zi \
's/^LumiPrivateKeyID :: .*/LumiPrivateKeyID :: "breathing-main-1"/' \
LumiKeyIsSeparate
echo "law mutation test passed"
+23
View File
@@ -1,4 +1,5 @@
#import, file "../src/storage/json.zi";
#import "byte_text_linux"
CStringEquals :: (actual: *u8, expected: string) -> bool {
if actual == null {
@@ -59,6 +60,28 @@ Answer :: () -> s32 {
return 6
}
storage_json_builder_free(*json)
// Encrypted photo chunks exceed the old 4095-byte temporary buffer.
large: [5000]u8
for index: cast(s32)0..cast(s32)4999 {
large[index] = cast(u8)120
}
json.ok = 1
storage_json_builder_key_value_text(*json, "ciphertext", TextFromBytes(large[:]))
if json.ok == 0 || json.len != 5015 || json.data[13] != cast(u8)34 ||
json.data[14] != cast(u8)120 || json.data[5013] != cast(u8)120 ||
json.data[5014] != cast(u8)34 || json.data[5015] != cast(u8)0 {
return 7
}
storage_json_builder_free(*json)
binary: [3]u8 = .[97, 0, 98]
json.ok = 1
storage_json_builder_key_value_text(*json, "binary", TextFromBytes(binary[:]))
if json.ok == 0 || !CStringEquals(json.data, "\"binary\":\"a\\u0000b\"") {
return 8
}
storage_json_builder_free(*json)
return 0
}
+294 -14
View File
@@ -9,6 +9,10 @@
#import, file "../src/storage/db.zi";
#import, file "../src/storage/session_mood.zi";
#import, file "../src/storage/storage_core.zi";
#import, file "../src/storage/json.zi";
#import, file "../src/storage/storage_layout.zi";
#import, file "../src/storage/sync_crypto.zi";
#import, file "../src/storage/sync_crypto_random.zi";
#import, file "../src/storage/storage_sessions.zi";
#import, file "../src/habits/habit_types.zi";
#import, file "../src/habits/habit_memory.zi";
@@ -42,20 +46,20 @@ set_account :: () {
storage_set_setting_text("sync_private_key", TextFromCString(*key[0]))
}
payload_text: TextBuffer;
payload_text: string;
// Builds the sync payload into `payload_text`; false when none is produced.
build_payload :: () -> bool {
payload := storage_build_sync_payload_json("test-public-id", "test-public-key")
if payload == null { return false }
payload_text = TextBuffer.{}
append(*payload_text, TextFromCString(payload))
PackageFree(payload_text)
payload_text = PackageText(TextFromCString(payload), "")
storage_free_sync_payload_json(payload)
return true
}
payload :: () -> string {
return TextFromCString(*payload_text.bytes[0])
return payload_text
}
record_count :: () -> s32 {
@@ -98,20 +102,22 @@ records_array :: () -> string {
return text[begin:end]
}
response_buffer: TextBuffer;
response_buffer: StorageJsonBuilder;
// A v6 server response carrying `records` for a device to apply.
response_with :: (version: s32, records: string) -> string {
part: [24]u8
response_buffer = TextBuffer.{}
append(*response_buffer, "{\"protocol_version\":6,\"status\":\"ok\",\"server_version\":")
append(*response_buffer, FormatInteger(cast(s64)version, part[:]))
append(*response_buffer, ",\"server_clock\":")
append(*response_buffer, FormatInteger(cast(s64)version, part[:]))
append(*response_buffer, ",\"changes\":{\"encrypted_records\":")
append(*response_buffer, records)
append(*response_buffer, "}}")
return TextFromCString(*response_buffer.bytes[0])
storage_json_builder_free(*response_buffer)
response_buffer = StorageJsonBuilder.{}
response_buffer.ok = 1
storage_json_builder_append_text(*response_buffer, "{\"protocol_version\":6,\"status\":\"ok\",\"server_version\":")
storage_json_builder_append_text(*response_buffer, FormatInteger(cast(s64)version, part[:]))
storage_json_builder_append_text(*response_buffer, ",\"server_clock\":")
storage_json_builder_append_text(*response_buffer, FormatInteger(cast(s64)version, part[:]))
storage_json_builder_append_text(*response_buffer, ",\"changes\":{\"encrypted_records\":")
storage_json_builder_append_text(*response_buffer, records)
storage_json_builder_append_text(*response_buffer, "}}")
return AppTextFromCString(response_buffer.data, cast(s32)(response_buffer.len + 1))
}
empty_response :: (version: s32) -> string {
@@ -1076,6 +1082,274 @@ test_review_completion_changes :: () {
storage_close()
}
// Real encrypted fixtures for the historical owner collection and isolated
// Lumi collection. They use the same account seed on two synthetic devices.
cell_record_id_fixture :: (collection: string, key_id: string, record_id: string, cell_key: string) -> string {
key: [32]u8
fixture: StorageJsonBuilder
fixture.ok = 1
if storage_private_collection_key(collection, key_id, *key[0]) == 0 {
return ""
}
storage_json_builder_append_text(*fixture, "[")
ok := storage_append_encrypted_record_json(*fixture, *key[0], collection,
key_id, record_id, "cell_record", cell_key, 0, now_seconds(), 0)
SyncCryptoWipe(*key[0], 32)
storage_json_builder_append_text(*fixture, "]")
result: string = ""
if ok != 0 && fixture.ok != 0 {
result = PackageText(AppTextFromCString(fixture.data, cast(s32)(fixture.len + 1)), "")
}
storage_json_builder_free(*fixture)
return result
}
cell_record_fixture :: (collection: string, key_id: string, cell_key: string) -> string {
return cell_record_id_fixture(collection, key_id, cell_key, cell_key)
}
test_cell_record_identity :: () {
source := make_root("sync-cell-identity-source")
target := make_root("sync-cell-identity-target")
outer := "cell.lumi.message.outer"
inner := "cell.lumi.message.inner"
check_true("init record identity source", storage_init(source) != 0)
set_account()
check_true("seed encrypted cell identity fixture", CellRecordWrite(inner, "Bound history"))
wrong_lumi := cell_record_id_fixture(LumiPrivateCollection, LumiPrivateKeyID, outer, inner)
wrong_legacy := cell_record_id_fixture(LegacyPrivateCellsCollection, STORAGE_SYNC_RECORD_KEY_ID, outer, inner)
correct_lumi := cell_record_fixture(LumiPrivateCollection, LumiPrivateKeyID, inner)
defer {
PackageFree(wrong_lumi)
PackageFree(wrong_legacy)
PackageFree(correct_lumi)
}
storage_close()
check_true("init record identity target", storage_init(target) != 0)
set_account()
check_true("authenticated Lumi outer ID cannot write different embedded key",
storage_apply_sync_response_json(response_with(1, wrong_lumi)) == 0)
check_true("authenticated legacy outer ID cannot write different embedded key",
storage_apply_sync_response_json(response_with(1, wrong_legacy)) == 0)
check_int("identity mismatch changes no cell", read_db_count("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.*'"), 0)
check_int("identity mismatch queues no record", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 0)
check_int("identity mismatch preserves cursor", cast(s32)storage_meta_int64("sync_last_server_version", 0), 0)
check_true("matching authenticated ID still restores",
storage_apply_sync_response_json(response_with(1, correct_lumi)) != 0)
check_true("matching record data survives", storage_get_setting_text(inner) == "Bound history")
storage_close()
}
test_cell_plaintext_single_record :: () {
root := make_root("sync-cell-single-record")
check_true("init single cell record fixture", storage_init(root) != 0)
set_account()
record_id := "cell.lumi.message.bound"
cases: [11]string = .[
"{\"key\":\"cell.lumi.message.other\",\"value\":\"wrong key\",\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"one\",\"updated_at\":\"2026-10-07T00:00:00Z\"},{\"key\":\"cell.lumi.message.other\",\"value\":\"two\",\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"key\":\"cell.lumi.message.other\",\"value\":\"duplicate key\",\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"\\u006bey\":\"cell.lumi.message.other\",\"value\":\"escaped duplicate key\",\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"one\",\"value\":\"two\",\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"timestamp ambiguity\",\"updated_at\":\"2026-10-07T00:00:00Z\",\"updated_at\":\"2026-10-08T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":{\"nested\":\"not a string\"},\"updated_at\":\"2026-10-07T00:00:00Z\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"missing timestamp\"}",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"unknown field\",\"updated_at\":\"2026-10-07T00:00:00Z\",\"extra\":1}",
"[{\"key\":\"cell.lumi.message.bound\",\"value\":\"nested record\",\"updated_at\":\"2026-10-07T00:00:00Z\"}]",
"{\"key\":\"cell.lumi.message.bound\",\"value\":\"trailing object\",\"updated_at\":\"2026-10-07T00:00:00Z\"}{}"
]
for plaintext: cases {
check_true("malformed single-cell plaintext rejected before storage",
storage_apply_decrypted_record(LumiPrivateCollection, record_id, plaintext) == 0)
}
check_true("cell record requires authenticated outer ID",
storage_apply_decrypted_record(LumiPrivateCollection, "", "{\"key\":\"cell.lumi.message.bound\",\"value\":\"unbound\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") == 0)
check_true("matching outer ID cannot admit Diary to Lumi",
storage_apply_decrypted_record(LumiPrivateCollection, "cell.diary.file.bound", "{\"key\":\"cell.diary.file.bound\",\"value\":\"wrong scope\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") == 0)
check_int("malformed plaintext changes no cell", read_db_count("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.*'"), 0)
check_int("malformed plaintext queues no record", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 0)
check_true("one matching cell record remains valid",
storage_apply_decrypted_record(LumiPrivateCollection, record_id, "{\"key\":\"cell.lumi.message.bound\",\"value\":\"one valid record\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") != 0)
check_true("valid single record data applied", storage_get_setting_text(record_id) == "one valid record")
storage_close()
}
test_lumi_collection_keys :: () {
root := make_root("sync-lumi-keys")
check_true("init isolated key device", storage_init(root) != 0)
legacy: [32]u8
lumi: [32]u8
legacy_hex: [65]u8
lumi_hex: [65]u8
check_true("Lumi key needs owner material",
storage_private_collection_key(LumiPrivateCollection, LumiPrivateKeyID, *lumi[0]) == 0)
set_account()
check_true("legacy cell key remains available to owner",
storage_private_collection_key(LegacyPrivateCellsCollection, STORAGE_SYNC_RECORD_KEY_ID, *legacy[0]) != 0)
check_true("separate Lumi key is available to owner",
storage_private_collection_key(LumiPrivateCollection, LumiPrivateKeyID, *lumi[0]) != 0)
unused SyncCryptoBytesToHex(*legacy[0], 32, *legacy_hex[0], 65)
unused SyncCryptoBytesToHex(*lumi[0], 32, *lumi_hex[0], 65)
check_true("Lumi key differs from global legacy key",
TextFromCString(*legacy_hex[0]) != TextFromCString(*lumi_hex[0]))
check_true("legacy generation is rejected for Lumi",
storage_private_collection_key(LumiPrivateCollection, STORAGE_SYNC_RECORD_KEY_ID, *lumi[0]) == 0)
check_true("Lumi generation is rejected for old cells",
storage_private_collection_key(LegacyPrivateCellsCollection, LumiPrivateKeyID, *legacy[0]) == 0)
check_true("Lumi generation is rejected for session history",
storage_private_collection_key("private.inbe.v1.sessions", LumiPrivateKeyID, *legacy[0]) == 0)
check_true("unapproved generation is rejected",
storage_private_collection_key(LumiPrivateCollection, "inbe-lumi-2", *lumi[0]) == 0)
check_true("legacy session key remains available",
storage_private_collection_key("private.inbe.v1.sessions", STORAGE_SYNC_RECORD_KEY_ID, *legacy[0]) != 0)
SyncCryptoWipe(*legacy[0], 32)
SyncCryptoWipe(*lumi[0], 32)
SyncCryptoWipe(*legacy_hex[0], 65)
SyncCryptoWipe(*lumi_hex[0], 65)
storage_close()
}
test_lumi_collection_migration :: () {
root := make_root("sync-lumi-older-layout")
check_true("init old collection fixture", storage_init(root) != 0)
set_account()
check_true("seed old Lumi history", CellRecordWrite("cell.lumi.message.old", "{\"text\":\"Earlier chat\",\"time\":1,\"user\":1}"))
check_true("seed old Diary history", CellRecordWrite("cell.diary.file.old", "Earlier diary"))
check_true("seed old feedback history", CellRecordWrite("cell.feedback.message.old", "Earlier feedback"))
storage_set_setting_text("lumi_pending_action", "preserved pending command")
storage_close()
// This fixture has the pre-upgrade sync layout: the v1 upload/backfill
// markers are complete and Lumi has no v2 marker or queued copy.
raw_exec(root, "DELETE FROM sync_outbox;DELETE FROM meta WHERE key='sync_lumi_collection_v2_done';INSERT OR REPLACE INTO meta(key,value) VALUES('sync_full_upload_done','1'),('sync_backfill_v2_done','1');INSERT INTO sync_outbox(entity_type,entity_id,local_date,queued_at) VALUES('cell_record','cell.diary.file.old',0,123);INSERT INTO settings(user_id,key,value,updated_at) SELECT id,'cell.lumi.INVALID','ignored malformed old key',1 FROM users LIMIT 1;")
check_true("upgrade old collection fixture", storage_init(root) != 0)
check_int("Lumi migration marker saved", cast(s32)storage_meta_int64(LumiCollectionMigrationKey, 0), 1)
check_int("only valid old Lumi history queued", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_id GLOB 'cell.lumi.*'"), 1)
check_int("existing Diary pending upload preserved", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_id='cell.diary.file.old' AND queued_at=123"), 1)
check_true("old Lumi plaintext remains intact", storage_get_setting_text("cell.lumi.message.old") == "{\"text\":\"Earlier chat\",\"time\":1,\"user\":1}")
LumiStoreReset()
check_int("old conversation remains loadable after migration", LumiStoreCount(), 1)
check_true("old conversation text remains loadable", LumiStoreRow(0).text == "Earlier chat")
check_true("old Diary remains intact", storage_get_setting_text("cell.diary.file.old") == "Earlier diary")
check_true("old feedback remains intact", storage_get_setting_text("cell.feedback.message.old") == "Earlier feedback")
check_true("pending command remains intact", storage_get_setting_text("lumi_pending_action") == "preserved pending command")
queued_seq := read_db_count("SELECT MAX(seq) FROM sync_outbox")
check_true("new-key copies can be uploaded", build_payload())
check_true("migration payload uses Lumi v2", contains(payload(), "private.inbe.v2.lumi") && contains(payload(), "inbe-lumi-1"))
check_true("migration keeps Diary in owner collection", contains(payload(), "private.inbe.v1.cells") && contains(payload(), "breathing-main-1"))
storage_close()
check_true("restart migrated device", storage_init(root) != 0)
LumiStoreReset()
check_int("old conversation remains loadable after restart", LumiStoreCount(), 1)
check_int("restart does not change queued copies", read_db_count("SELECT MAX(seq) FROM sync_outbox"), queued_seq)
check_int("restart does not duplicate migration", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 2)
exec_db_sql("DELETE FROM sync_outbox;")
storage_close()
check_true("restart after migration acknowledgement", storage_init(root) != 0)
check_int("acknowledged migration stays complete", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 0)
storage_close()
}
test_lumi_collection_migration_rollback :: () {
root := make_root("sync-lumi-migration-rollback")
check_true("init interrupted migration fixture", storage_init(root) != 0)
set_account()
check_true("seed interrupted old Lumi", CellRecordWrite("cell.lumi.message.interrupted", "Older history"))
exec_db_sql("DELETE FROM sync_outbox;CREATE TRIGGER block_lumi_marker BEFORE INSERT ON meta WHEN NEW.key='sync_lumi_collection_v2_done' BEGIN SELECT RAISE(ABORT,'synthetic marker failure');END;")
check_true("migration reports marker failure", storage_migrate_lumi_collection() == 0)
check_int("failed migration rolls back queued copies", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 0)
check_int("failed migration does not save marker", cast(s32)storage_meta_int64(LumiCollectionMigrationKey, 0), 0)
check_true("failed migration retains old history", storage_get_setting_text("cell.lumi.message.interrupted") == "Older history")
exec_db_sql("DROP TRIGGER block_lumi_marker;")
storage_close()
check_true("restart retries interrupted migration", storage_init(root) != 0)
check_int("retried migration queues one copy", read_db_count("SELECT COUNT(*) FROM sync_outbox"), 1)
check_int("retried migration saves marker", cast(s32)storage_meta_int64(LumiCollectionMigrationKey, 0), 1)
storage_close()
}
test_legacy_cells_and_lumi_scope :: () {
first := make_root("sync-lumi-legacy-source")
second := make_root("sync-lumi-legacy-restore")
check_true("init old encrypted record source", storage_init(first) != 0)
set_account()
check_true("seed historical combined Lumi", CellRecordWrite("cell.lumi.message.historical", "Historical chat"))
check_true("seed historical combined Diary", CellRecordWrite("cell.diary.file.historical", "Historical diary"))
check_true("seed historical combined feedback", CellRecordWrite("cell.feedback.message.historical", "Historical feedback"))
legacy_lumi := cell_record_fixture(LegacyPrivateCellsCollection, STORAGE_SYNC_RECORD_KEY_ID, "cell.lumi.message.historical")
legacy_diary := cell_record_fixture(LegacyPrivateCellsCollection, STORAGE_SYNC_RECORD_KEY_ID, "cell.diary.file.historical")
legacy_feedback := cell_record_fixture(LegacyPrivateCellsCollection, STORAGE_SYNC_RECORD_KEY_ID, "cell.feedback.message.historical")
forbidden_diary := cell_record_fixture(LumiPrivateCollection, LumiPrivateKeyID, "cell.diary.file.historical")
forbidden_feedback := cell_record_fixture(LumiPrivateCollection, LumiPrivateKeyID, "cell.feedback.message.historical")
defer {
PackageFree(legacy_lumi)
PackageFree(legacy_diary)
PackageFree(legacy_feedback)
PackageFree(forbidden_diary)
PackageFree(forbidden_feedback)
}
storage_close()
check_true("init historical restore device", storage_init(second) != 0)
set_account()
check_true("legacy global-key Lumi remains decryptable", storage_apply_sync_response_json(response_with(1, legacy_lumi)) != 0)
check_true("legacy global-key Diary remains decryptable", storage_apply_sync_response_json(response_with(2, legacy_diary)) != 0)
check_true("legacy global-key feedback remains decryptable", storage_apply_sync_response_json(response_with(3, legacy_feedback)) != 0)
check_true("legacy Lumi restored intact", storage_get_setting_text("cell.lumi.message.historical") == "Historical chat")
check_true("legacy Diary restored intact", storage_get_setting_text("cell.diary.file.historical") == "Historical diary")
check_true("legacy feedback restored intact", storage_get_setting_text("cell.feedback.message.historical") == "Historical feedback")
check_int("late legacy Lumi queues a new-key copy", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_id='cell.lumi.message.historical'"), 1)
check_true("migration payload after legacy restore builds", build_payload())
check_true("legacy Lumi reuploads only to its isolated collection", contains(payload(), "private.inbe.v2.lumi") && !contains(payload(), "private.inbe.v1.cells"))
check_true("cryptographically valid Diary is rejected in Lumi collection", storage_apply_sync_response_json(response_with(4, forbidden_diary)) == 0)
check_true("cryptographically valid feedback is rejected in Lumi collection", storage_apply_sync_response_json(response_with(4, forbidden_feedback)) == 0)
check_int("rejected foreign records do not advance sync cursor", cast(s32)storage_meta_int64("sync_last_server_version", 0), 3)
storage_close()
}
test_cell_records_round_trip :: () {
first := make_root("sync-cells-a")
second := make_root("sync-cells-b")
check_true("init cell device a", storage_init(first) != 0)
set_account()
LumiStoreReset()
LumiStoreAppend("Private conversation 水", true)
LumiStoreAppend("Private reply", false)
check_true("save private diary", DiaryRecordSave("2026-10-07.json", "{\"text\":\"Private diary 水\"}"))
photo: [4097]u8
for index: cast(s32)0..cast(s32)4096 {
photo[index] = cast(u8)(index % 256)
}
check_true("save binary diary photo", DiaryRecordSave("photo-test.png", TextFromBytes(photo[:])))
check_true("cell payload builds", build_payload())
check_true("Lumi uses separate encrypted collection", contains(payload(), "private.inbe.v2.lumi"))
check_true("Diary retains legacy owner collection", contains(payload(), "private.inbe.v1.cells"))
check_true("private chat is absent on wire", !contains(payload(), "Private conversation"))
check_true("private diary is absent on wire", !contains(payload(), "Private diary"))
records := records_array()
check_true("photo fixture exceeds the old bounded payload buffer", payload().count > 16384)
check_true("full cell payload is valid JSON", storage_json_valid(payload()) != 0)
check_true("cell record array is valid JSON", storage_json_valid(records) != 0)
storage_close()
check_true("init cell device b", storage_init(second) != 0)
set_account()
response := response_with(5, records)
check_true("full cell response is valid JSON", storage_json_valid(response) != 0)
check_true("encrypted cell restore", storage_apply_sync_response_json(response) != 0)
LumiStoreReset()
check_int("conversation restored", LumiStoreCount(), 2)
check_true("chat text survives", LumiStoreRow(0).text == "Private conversation 水")
diary := DiaryRecordRead("2026-10-07.json")
check_true("diary text survives", diary == "{\"text\":\"Private diary 水\"}")
PackageFree(diary)
restored := DiaryRecordRead("photo-test.png")
check_true("binary photo survives", restored == TextFromBytes(photo[:]))
PackageFree(restored)
check_int("restore does not requeue cells", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_type='cell_record'"), 0)
check_true("cell collection cannot overwrite signing key",
storage_apply_decrypted_record("private.inbe.v1.cells", "sync_private_key", "{\"key\":\"sync_private_key\",\"value\":\"forged\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") == 0)
storage_close()
}
#program_export
Check :: () -> s32 {
guard := support_begin("/tmp/inbe-storage-sync-zi-test/")
@@ -1093,6 +1367,12 @@ Check :: () -> s32 {
test_habit_name_repair_runs_once()
test_records_round_trip_between_devices()
test_cell_records_round_trip()
test_cell_record_identity()
test_cell_plaintext_single_record()
test_lumi_collection_keys()
test_lumi_collection_migration()
test_lumi_collection_migration_rollback()
test_legacy_cells_and_lumi_scope()
test_deletion_round_trip()
test_counter_counts_apply()
test_queued_edit_beats_equal_time_remote()