Files
breeze/src/storage/sync_exchange.zi
T

355 lines
15 KiB
Ziran

// One sync exchange with the node: the request payload, and applying its
// response, including a full snapshot that may need review first.
#import, file "../app/text_buffers.zi";
#import, file "state.zi";
#import, file "json.zi";
#import, file "schema.zi";
#import, file "storage_habit_materialize.zi";
#import, file "sync_node_storage.zi";
#import "keys"
#import, file "db.zi";
#import, file "storage_sessions.zi";
#import, file "sync_review.zi";
// Its own set_meta_buf would be ambiguous with the core's.
Habits :: #import, file "storage_habits.zi";
#import, file "sql_query.zi";
#import, file "storage_core.zi";
#import, file "sync_outbox.zi";
#import, file "sync_migrations.zi";
#import, file "sync_state.zi";
#import, file "storage_settings.zi";
#import, file "sync_record_crypto.zi";
#import, file "storage_json_sql.zi";
#import, file "sync_apply.zi";
#import, file "sync_page_store.zi";
#import "sync_flow"
c :: #system_library "c";
malloc :: (size: usize) -> *void #foreign c;
free :: (ptr: *void) #foreign c;
storage_sync_review_delete_json :: () #foreign c "storage_sync_review_delete_json";
set_meta :: (key: *u8, value: *u8) #foreign c "set_meta";
LOG_INFO :: 3
#program_export
storage_build_sync_payload_json :: (user_id_hash: string, public_key_hex: string) -> *u8 {
return BuildStoredSyncPayload(user_id_hash, public_key_hex, STORAGE_SYNC_OP_BATCH_LIMIT)
}
BuildStoredSyncPayload :: (user_id_hash: string, public_key_hex: string, maximum_records: s32) -> *u8 {
json: StorageJsonBuilder
since_server_version: s64
through_seq: s64
full_upload_done: s32
force_zero_day_repair: s32
shadow_key: [32] u8
can_shadow_encrypt: s32
if storage_db_handle() == null || user_id_hash.count == 0 {
return null
}
if migrate_schema() == 0 || storage_migrate_lumi_collection() == 0 {
return null
}
if storage_materialize_session_habit_days() == 0 {
return null
}
if storage_meta_int64(STORAGE_SYNC_BACKFILL_KEY, 0) == 0 {
storage_enqueue_all_sync_state()
}
// A node set not to store photos must not receive them from any queue.
if storage_withhold_node_media() == 0 {
return null
}
can_shadow_encrypt = storage_private_record_key(*shadow_key[0])
SecureWipe(*shadow_key[0], 32)
if can_shadow_encrypt == 0 {
return null
}
since_server_version = storage_meta_int64("sync_last_server_version", 0)
full_upload_done = 0
if storage_meta_int64("sync_full_upload_done", 0) != 0 {
full_upload_done = 1
}
if storage_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 0) == 0 || storage_has_orphan_habit_days() != 0 {
since_server_version = 0
}
force_zero_day_repair = 0
if storage_meta_int64(STORAGE_SYNC_ZERO_HABIT_DAY_REPAIR_KEY, 0) == 0 {
force_zero_day_repair = 1
}
if force_zero_day_repair != 0 {
since_server_version = 0
}
if storage_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, 0) < STORAGE_SYNC_RECORD_FIELDS {
since_server_version = 0
}
batch_limit := maximum_records
// Chunked diary photos must stay within the public node's request limit.
if batch_limit > 64 && db_select_int64_text("SELECT EXISTS(SELECT 1 FROM sync_outbox WHERE entity_type='cell_record')", 0) != 0 {
batch_limit = 64
}
through_seq = storage_sync_outbox_batch_seq(batch_limit)
if storage_sync_records_download_first() {
through_seq = 0
}
storage_state_handle().pending_sync_outbox_seq = through_seq
json.ok = 1
protocol_text: [32] u8
storage_json_builder_append_text(*json, "{")
AppFormatS32(SYNC_PROTOCOL_VERSION, protocol_text[:])
storage_json_builder_append_text(*json, "\"protocol_version\":")
storage_json_builder_append_text(*json, AppTextFromCString(*protocol_text[0], 32))
storage_json_builder_append_text(*json, ",")
storage_json_builder_append_text(*json, "\"app_id\":\"inbe\",")
storage_json_builder_append_text(*json, "\"client_capabilities\":[\"v6-device-transactions\",\"encrypted-primary\"],")
storage_json_builder_key_value_text(*json, "user_id_hash", user_id_hash)
storage_json_builder_append_text(*json, ",")
storage_json_builder_key_value_text(*json, "client_id", storage_sync_client_id())
server_version_text: [24] u8
unused AppFormatS64(since_server_version, server_version_text[:])
storage_json_builder_append_text(*json, ",\"since_server_version\":")
storage_json_builder_append_text(*json, AppTextFromCString(*server_version_text[0], 24))
client_clock_text: [24] u8
unused AppFormatS64(storage_meta_int64(STORAGE_SYNC_SERVER_CLOCK_KEY, 0), client_clock_text[:])
storage_json_builder_append_text(*json, ",\"client_clock\":")
storage_json_builder_append_text(*json, AppTextFromCString(*client_clock_text[0], 24))
if storage_meta_int64(STORAGE_SYNC_FULL_REPLACE_KEY, 0) != 0 {
storage_json_builder_append_text(*json, ",\"full_sync_requested\":true")
}
{
last_server_hash: string = storage_meta_text(STORAGE_SYNC_LAST_SERVER_HASH_KEY)
if force_zero_day_repair == 0 && last_server_hash.count > 0 {
storage_json_builder_append_text(*json, ",")
storage_json_builder_key_value_text(*json, "last_server_state_hash", last_server_hash)
}
}
if since_server_version <= 0 || full_upload_done == 0 {
storage_json_builder_append_text(*json, ",\"bootstrap\":true")
}
if public_key_hex.count > 0 {
storage_json_builder_append_text(*json, ",")
storage_json_builder_key_value_text(*json, "public_key", public_key_hex)
}
storage_json_builder_append_text(*json, ",")
storage_append_encrypted_records_json(*json, through_seq)
storage_json_builder_append_text(*json, "}")
if json.ok == 0 || json.data == null {
free(cast(*void)json.data)
return null
}
LogStorage(LOG_INFO, "SYNC: payload built")
return json.data
}
#program_export
storage_free_sync_payload_json :: (payload: *u8) {
free(cast(*void)payload)
}
storage_sync_response_has_changes :: (json: string) -> s32 {
if storage_json_array_has_items(json, "$.changes.encrypted_records") != 0 {
return 1
}
if storage_json_array_has_items(json, "$.data.encrypted_records") != 0 {
return 1
}
if storage_json_array_has_items(json, "$.data.social") != 0 {
return 1
}
if storage_json_array_has_items(json, "$.changes.social_cache") != 0 {
return 1
}
return 0
}
#program_export
storage_apply_sync_response_json :: (response_json: string) -> s32 {
return ApplyStoredSyncResponse(response_json, null)
}
ApplyStoredSyncResponse :: (response_json: string, page: *SyncPageState) -> s32 {
incomplete := page != null && page.pending
server_version: s64
server_clock: s64
old_server_version: s64
protocol_version: s64
server_hash: [80] u8
account_alias: [40] u8
if storage_db_handle() == null || response_json.count == 0 || response_json[0] == #char "\0" {
return 0
}
if migrate_schema() == 0 {
LogStorage(LOG_WARNING, "SYNC: response migration failed")
return 0
}
storage_state_handle().last_sync_changed = 0
if storage_json_valid(response_json) == 0 {
LogStorage(LOG_WARNING, "SYNC: response JSON invalid")
return 0
}
protocol_version = storage_json_extract_int64(response_json, "$.protocol_version", 0)
if protocol_version != SYNC_PROTOCOL_VERSION {
protocol_text: [24]u8
if AppFormatS64(protocol_version, protocol_text[:]) {
LogStorageText(LOG_WARNING, "SYNC: unsupported response protocol: ",
AppTextFromCString(*protocol_text[0], 24))
}
return 0
}
if storage_json_extract_text(response_json, "$.account_alias", *account_alias[0],
size_of(type_of(account_alias))) != 0 {
storage_set_setting_text(STORAGE_SYNC_ACCOUNT_ALIAS_KEY, AppTextFromCString(*account_alias[0], 40))
}
// The newest server version this device has heard of, even when a full
// snapshot waits for review and the applied version stays behind. Live
// updates compare against it, so a waiting review is not news each time.
seen_server_version := storage_json_extract_int64(response_json, "$.server_version", 0)
if seen_server_version > storage_meta_int64(STORAGE_SYNC_SEEN_SERVER_VERSION_KEY, 0) {
storage_set_meta_int64(STORAGE_SYNC_SEEN_SERVER_VERSION_KEY, seen_server_version)
}
if storage_json_extract_int64(response_json, "$.full_snapshot_required", 0) != 0 && storage_meta_int64(STORAGE_SYNC_APPLY_REVIEW_KEY, 0) == 0 {
if storage_has_any() == 0 {
return storage_apply_remote_full_snapshot(response_json)
}
if storage_has_pending_sync_outbox() != 0 {
LogStorage(LOG_INFO, "SYNC: merging full snapshot with pending local edits")
} else if storage_sync_review_json_should_auto_apply_remote(response_json) != 0 {
return storage_apply_remote_full_snapshot(response_json)
} else if storage_sync_review_json_has_visible_diff(response_json) == 0 {
// The same merge preview shown to the user has no changes. Apply
// its bookkeeping without replacing or removing local records.
LogStorage(LOG_INFO, "SYNC: full snapshot has no visible merge changes")
} else {
if storage_sync_review_write_json(response_json) == 0 {
LogStorage(LOG_WARNING, "SYNC: response review save failed")
return 0
}
{
meta_key_buffer: [64] u8
meta_val_buffer: [4] u8
AppCopyText(STORAGE_SYNC_PENDING_REVIEW_KEY, meta_key_buffer[:])
AppCopyText("1", meta_val_buffer[:])
set_meta(*meta_key_buffer[0], *meta_val_buffer[0])
}
storage_state_handle().pending_sync_outbox_seq = 0
storage_schedule_persist()
LogStorage(LOG_WARNING, "SYNC: full snapshot review required")
return 1
}
}
old_server_version = storage_meta_int64("sync_last_server_version", 0)
if exec_sql("SAVEPOINT sync_apply") == 0 {
return 0
}
storage_clear_uploaded_outbox(storage_state_handle().pending_sync_outbox_seq)
if storage_apply_sync_records(response_json) == 0 || (page != null && !SaveStoredSyncPage(storage_get_setting_text("sync_server_url"), page)) {
exec_sql("ROLLBACK TO sync_apply")
exec_sql("RELEASE sync_apply")
return 0
}
if exec_sql("RELEASE sync_apply") == 0 {
// An unreleased savepoint keeps the transaction, and the connection,
// held by this worker thread after it returns.
exec_sql("ROLLBACK TO sync_apply")
exec_sql("RELEASE sync_apply")
return 0
}
server_version = storage_json_extract_int64(response_json, "$.server_version", 0)
server_clock = storage_json_extract_int64(response_json, "$.server_clock", 0)
storage_log_number("SYNC: response server_version=", server_version)
storage_log_number("SYNC: previous server_version=", old_server_version)
storage_log_number("SYNC: encrypted records=", cast(s64)(
storage_json_array_count(response_json, "$.changes.encrypted_records") +
storage_json_array_count(response_json, "$.data.encrypted_records")))
storage_log_number("SYNC: social records=", cast(s64)(
storage_json_array_count(response_json, "$.changes.social_cache") +
storage_json_array_count(response_json, "$.data.social")))
if server_version > old_server_version && storage_sync_response_has_changes(response_json) != 0 {
storage_state_handle().last_sync_changed = 1
}
if server_version > 0 && !incomplete {
storage_set_meta_int64("sync_last_server_version", server_version)
}
if server_clock > 0 && !incomplete {
storage_set_meta_int64(STORAGE_SYNC_SERVER_CLOCK_KEY, server_clock)
}
if storage_json_extract_text(response_json, "$.server_state_hash", *server_hash[0],
size_of(type_of(server_hash))) != 0 {
{
meta_key_buffer: [64] u8
AppCopyText(STORAGE_SYNC_LAST_SERVER_HASH_KEY, meta_key_buffer[:])
set_meta(*meta_key_buffer[0], *server_hash[0])
}
}
set_meta_buf(STORAGE_SYNC_PENDING_REVIEW_KEY, "")
if storage_meta_int64(STORAGE_SYNC_APPLY_REVIEW_KEY, 0) == 0 {
storage_sync_review_delete_json()
}
storage_set_meta_int64(STORAGE_SYNC_APPLY_REVIEW_KEY, 0)
set_meta_int64_buf(STORAGE_SYNC_FULL_REPLACE_KEY, 0)
storage_state_handle().pending_sync_outbox_seq = 0
if incomplete {
storage_schedule_persist()
return 1
}
storage_set_meta_int64("sync_full_upload_done", 1)
storage_set_meta_int64(STORAGE_SYNC_BACKFILL_KEY, 1)
storage_set_meta_int64(STORAGE_SYNC_HABIT_NAME_REPAIR_KEY, 1)
storage_set_meta_int64(STORAGE_SYNC_ZERO_HABIT_DAY_REPAIR_KEY, 1)
storage_set_meta_int64(STORAGE_SYNC_RECORD_FIELDS_KEY, STORAGE_SYNC_RECORD_FIELDS)
Habits.storage_mark_habits_initialized()
storage_schedule_persist()
return 1
}
storage_apply_remote_full_snapshot :: (response_json: string) -> s32 {
copy: *u8
ok: s32
previous_pending_seq: s64 = storage_state_handle().pending_sync_outbox_seq
previous_last_sync_changed: s32 = storage_state_handle().last_sync_changed
if response_json.count == 0 || response_json.count >= 2147483647 {
return 0
}
copy = cast(*u8)malloc(cast(usize)response_json.count + 1)
if copy == null {
return 0
}
index: s32 = 0
while index < cast(s32)response_json.count {
copy[index] = response_json[index]
index += 1
}
copy[index] = 0
if exec_sql("SAVEPOINT sync_replace") == 0 {
free(cast(*void)copy)
return 0
}
if storage_clear_local_sync_data() == 0 {
LogStorage(LOG_WARNING, "SYNC: full snapshot local clear failed")
exec_sql("ROLLBACK TO sync_replace")
exec_sql("RELEASE sync_replace")
free(cast(*void)copy)
return 0
}
storage_set_meta_int64(STORAGE_SYNC_APPLY_REVIEW_KEY, 1)
ok = storage_apply_sync_response_json(
AppTextFromCString(copy, cast(s32)response_json.count + 1))
if ok == 0 {
exec_sql("ROLLBACK TO sync_replace")
exec_sql("RELEASE sync_replace")
storage_state_handle().pending_sync_outbox_seq = previous_pending_seq
storage_state_handle().last_sync_changed = previous_last_sync_changed
} else if exec_sql("RELEASE sync_replace") == 0 {
exec_sql("ROLLBACK TO sync_replace")
exec_sql("RELEASE sync_replace")
storage_state_handle().pending_sync_outbox_seq = previous_pending_seq
storage_state_handle().last_sync_changed = previous_last_sync_changed
ok = 0
}
if ok != 0 {
storage_sync_review_delete_json()
}
free(cast(*void)copy)
return ok
}