Sync Diary photos as one sealed copy each

A Diary photo was synced as 1 KB hex chunks inside hex ciphertext, so a
node stored about 4.5 times the photo and the device kept a second hex
copy. A photo's entry now syncs only its hash and size; after each good
sync the photo is sealed with a key made for it alone and stored on the
node as one encrypted copy (its size plus 16 bytes), and the account's
other devices download and verify it. Photos from before upload the same
way and their chunks shrink to one byte each. A node without photo upload
or a full node only keeps the copy off that node; photos always save.

The desktop moves copies on the sync worker, Android and the web one
request per frame. Signed GET requests now sign their empty body.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
waoziandClaude Opus 5.5 committed 2026-10-17 01:55:27 -03:00
1 parent 1083e90d35
commit c4babc97dc
17 files changed
+1237 -34

No files matched your search

+1
View File
@@ -19,6 +19,7 @@
- Top up Dew centers the Monero address under its QR code.
- Profile shows how much of your storage is used, with a bar, next to your plan and in Storage plans.
- Free storage is 20 MB. Profile shows the free storage your sync node gives instead of a fixed size.
- Diary photos sync as one encrypted copy each, about the photo's own size on your sync node instead of about four and a half times, and are no longer stored twice on your device. Photos added before upload the same way and free their old space on the node.
- Every practice is optional: Practice shows only installed ones. Wim Hof comes installed and can be removed; Meditation and Breathing Patterns come with the app and are added without a download.
- Manage practices lists every practice with its size and one menu that says where it is: On Practice, On device or Not downloaded, each with what choosing it does. On device takes a practice off Practice but keeps it on your device, so adding it back needs no download; once a practice is downloaded, Uninstall asks once, then frees the space. A paid practice offers its unlock instead. Drag a practice by its handle onto another to change the order Practice shows them in. Your progress stays either way.
- Lumi asks to download or unlock a practice before starting one that isn't on your device.
+96
View File
@@ -1,4 +1,6 @@
#import "async_client"
#import "auth"
#import "text_buffer"
#import, file "sync_safety_policy.zi";
#import "byte_text_linux"
#import "url"
@@ -32,8 +34,10 @@ GetTime :: () -> float64 #foreign c;
#import, file "app_types.zi";
#import, file "app.zi";
#import, file "sync_safety.zi";
#import, file "../cells/diary_blobs.zi";
#if #defined(PLATFORM_WEB) == false && #defined(ANDROID_BUILD) == false {
#import, file "../storage/sync_runtime.zi";
#import, file "../cells/diary_blobs_native.zi";
}
#if #defined(PLATFORM_WEB) || #defined(ANDROID_BUILD) {
#import, file "../storage/sync_async.zi";
@@ -137,6 +141,16 @@ g_sync_events_retry_at: float64;
#if ASYNC {
g_sync_operation: StoredOperation;
g_sync_transfer: NetworkTransfer;
// Diary photo copies move one request at a time after a good sync.
g_diary_blob: StoredBlob;
g_diary_pass: DiaryBlobPass;
g_diary_request: DiaryBlobRequest;
g_diary_output: *u8;
g_diary_output_bytes: s64;
g_diary_pass_active: bool;
g_diary_pass_due: bool;
g_diary_pass_url: [512]u8;
g_diary_pass_account: [72]u8;
g_social_async: SocialRefresh;
g_social_transfer: NetworkTransfer;
g_social_streak_key: [96]u8;
@@ -429,6 +443,9 @@ app_social_apply_result :: (app: *InnerBreeze, requests_json: string,
if result == cast(SyncResult)SYNC_OK && storage_last_sync_changed() != 0 {
changed = 1
}
if result == cast(SyncResult)SYNC_OK {
DiaryBlobsSendNative(AppTextFromCString(*args.url[0], 256))
}
SyncWorkerFree(cast(*void)args)
}
@@ -534,6 +551,80 @@ app_social_apply_result :: (app: *InnerBreeze, requests_json: string,
}
}
#if ASYNC {
app_diary_blobs_end :: () {
if g_diary_blob.active {
CancelStoredBlob(*g_diary_blob)
}
DiaryBlobAbandon(*g_diary_request)
if g_diary_output != null {
SyncWorkerFree(cast(*void)g_diary_output)
g_diary_output = null
}
g_diary_output_bytes = 0
g_diary_pass_active = false
}
// Moves Diary photo copies one request at a time, each frame, after a
// good sync (cells/diary_blobs.zi). A switched account ends the pass.
app_diary_blobs_tick :: () {
if g_diary_pass_active &&
storage_get_setting_text("sync_public_id") != AppTextFromCString(*g_diary_pass_account[0], 72) {
app_diary_blobs_end()
return
}
if g_diary_blob.active {
if !PollStoredBlob(*g_diary_blob) {
return
}
keep := false
length := g_diary_blob.length
if g_diary_blob.result == cast(s32)AuthResult.AUTH_OK && length >= 0 && length < g_diary_output_bytes {
keep = DiaryBlobFinish(*g_diary_pass, *g_diary_request, g_diary_blob.status,
BytesFromPointer(g_diary_output, cast(u64)length))
}
DiaryBlobAbandon(*g_diary_request)
SyncWorkerFree(cast(*void)g_diary_output)
g_diary_output = null
g_diary_output_bytes = 0
if !keep {
app_diary_blobs_end()
return
}
}
if !g_diary_pass_active {
if !g_diary_pass_due || g_sync.sync_running != 0 {
return
}
g_diary_pass_due = false
if !DiaryBlobPlan(*g_diary_pass, AppTextFromCString(*g_diary_pass_url[0], 512)) {
return
}
unused AppCopyText(storage_get_setting_text("sync_public_id"), g_diary_pass_account[:])
g_diary_pass_active = true
}
if !DiaryBlobNext(*g_diary_pass, *g_diary_request) {
app_diary_blobs_end()
return
}
capacity: s64 = 4096
if g_diary_request.answer_bytes > capacity {
capacity = g_diary_request.answer_bytes
}
g_diary_output = cast(*u8)SyncWorkerAlloc(cast(usize)capacity)
if g_diary_output == null {
app_diary_blobs_end()
return
}
g_diary_output_bytes = capacity
if !BeginStoredBlob(*g_diary_blob, AppTextFromCString(*g_diary_pass_url[0], 512),
g_diary_request.method, TextUntilNul(g_diary_request.path[:]), g_diary_request.body,
BytesFromPointer(g_diary_output, cast(u64)capacity)) {
app_diary_blobs_end()
}
}
}
app_collect_finished_sync :: () {
result: s32 = 0
changed: s32 = 0
@@ -549,12 +640,17 @@ app_collect_finished_sync :: () {
}
}
#if ASYNC {
app_diary_blobs_tick()
if g_sync.sync_running == 0 || !PollStoredOperation(*g_sync_operation) {
return
}
result = g_sync_operation.result
changed = g_sync_operation.changed
g_sync.sync_running = 0
if result == SYNC_OK {
unused AppCopyText(AppTextFromCString(*g_sync_operation.server[0], 512), g_diary_pass_url[:])
g_diary_pass_due = true
}
}
app_sync_record_result(result)
if result == SYNC_OK {
+385
View File
@@ -0,0 +1,385 @@
// Diary photos and videos travel between a device and its node as sealed
// copies (diary_records.zi), one request per copy, after each good sync:
//
// - A photo this device has whose entry names no copy yet is sealed and
// uploaded; its entry then names the copy, so the account's other devices
// can download it.
// - A photo whose copy is named but whose file is not on this device is
// downloaded, opened and saved as its file. A copy not uploaded yet (404)
// is tried again after a later sync.
// - A copy this device has that the node does not, for example after
// switching nodes, is uploaded again.
// - A photo from before copies, kept as hex chunks, is uploaded the same
// way; its chunks then shrink to one byte each, freeing their room on the
// node.
//
// Nothing here stops a photo from saving: a node without photo upload, or
// one that is full, only keeps its copy off that node. The host sends each
// request (signed for the account and device) and passes the answer back.
#import "byte_text_linux"
#import "vec"
#import "text"
#import "json_scan"
#import "text_buffer"
#import "keys"
#import "crypto"
#import, file "diary_records.zi";
#import, file "diary_store.zi";
#import, file "package_files.zi";
#import, file "../storage/cell_records.zi";
#import, file "../storage/storage_core.zi";
#import, file "../storage/sqlite.zi";
#import, file "../storage/db.zi";
#import, file "../storage/state.zi";
#import, file "../storage/paths.zi";
#import, file "../storage/sync_nodes.zi";
#import, file "../storage/sync_node_storage.zi";
#import, file "../app/text_buffers.zi";
// One pass after a sync moves at most this many files and bytes, so a sync
// is never held up for long; the next sync continues.
DIARY_BLOB_PASS_FILES :: 8
DIARY_BLOB_PASS_BYTES :: 33554432
DIARY_BLOB_UPLOAD :: 1
DIARY_BLOB_DOWNLOAD :: 2
DiaryBlobJob :: struct {
kind: s32
name: [201]u8
}
DiaryBlobPass :: struct {
// The node's normalized address.
node: [256]u8
jobs: [DIARY_BLOB_PASS_FILES]DiaryBlobJob
count: s32
next: s32
// The node refused an upload for room; downloads go on.
uploads_stopped: bool
}
// One request: PUT stores body as the copy at path; GET answers with the
// copy, which needs answer_bytes of room.
DiaryBlobRequest :: struct {
kind: s32
name: [201]u8
method: string
path: [128]u8
body: string
answer_bytes: s64
blob: [65]u8
key: [65]u8
// An entry from before copies: its hash, size and chunk count.
chunked: bool
hash: [65]u8
size: s64
chunks: s32
}
DiaryBlobEntriesSQL :: #string SQL_END
SELECT substr(key,17),value FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
AND key NOT GLOB '*.json' AND json_valid(value)
ORDER BY updated_at DESC
SQL_END;
DiaryBlobOnNode :: (node: string, blob: string) -> bool {
statement: *void
if SqlitePrepareText(storage_db_handle(),
"SELECT 1 FROM diary_blobs WHERE node=?1 AND blob=?2", -1,
*statement, null) != SQLITE_OK {
return false
}
defer { SqliteFinalize(statement) }
unused bind_text(statement, 1, node)
unused bind_text(statement, 2, blob)
return SqliteStep(statement) == SQLITE_ROW
}
DiaryBlobRemember :: (node: string, blob: string) {
statement: *void
if SqlitePrepareText(storage_db_handle(),
"INSERT OR IGNORE INTO diary_blobs(node,blob) VALUES(?1,?2)", -1,
*statement, null) != SQLITE_OK {
return
}
unused bind_text(statement, 1, node)
unused bind_text(statement, 2, blob)
unused SqliteStep(statement)
SqliteFinalize(statement)
}
// An entry from before copies: a photo kept as chunks.
DiaryBlobChunked :: (entry: string, request: *DiaryBlobRequest) -> bool {
count := ReadNumber(entry, FieldAt(entry, 0, "count"))
size := ReadNumber(entry, FieldAt(entry, 0, "size"))
if !count.valid || !size.valid || size.value < 1 || size.value > cast(float64)DIARY_FILE_LIMIT ||
count.value != cast(float64)((cast(s64)size.value + DIARY_CHUNK_BYTES - 1) / DIARY_CHUNK_BYTES) ||
ReadStringText(entry, FieldAt(entry, 0, "hash"), request.hash[:]) != 64 ||
!DiaryHex64(TextFromBytes(request.hash[:64])) {
return false
}
request.chunked = true
request.size = cast(s64)size.value
request.chunks = cast(s32)count.value
return true
}
DiaryBlobAdd :: (pass: *DiaryBlobPass, kind: s32, name: string) {
if pass.count >= DIARY_BLOB_PASS_FILES {
return
}
pass.jobs[pass.count].kind = kind
unused AppCopyText(name, pass.jobs[pass.count].name[:])
pass.count += 1
}
// Chooses this pass's files: downloads first, so a photo shows on this
// device soon, then uploads while the node takes photos and the account can
// seal them.
DiaryBlobPlan :: (pass: *DiaryBlobPass, node: string) -> bool {
pass.* = DiaryBlobPass.{}
if !SyncNodeAddress(node, pass.node[:]) || storage_db_handle() == null {
return false
}
address := TextUntilNul(pass.node[:])
key: [32]u8
uploads := SyncNodeMediaUpload(address) &&
storage_private_record_key_for_context(DIARY_BLOB_CONTEXT, *key[0]) != 0
SecureWipe(*key[0], 32)
bytes: s64 = 0
for round: 0..1 {
statement: *void
if SqlitePrepareText(storage_db_handle(), DiaryBlobEntriesSQL, -1,
*statement, null) != SQLITE_OK {
return false
}
while pass.count < DIARY_BLOB_PASS_FILES && bytes < DIARY_BLOB_PASS_BYTES &&
SqliteStep(statement) == SQLITE_ROW {
name := AppTextFromCString(SqliteColumnText(statement, 0), 201)
entry := AppTextFromCString(SqliteColumnText(statement, 1), 65536)
if !DiaryStoreNameValid(name) {
continue
}
path := DiaryStorePath(name)
local := PackageFileSize(path)
PackageFree(path)
media := DiaryMediaParse(entry)
if round == 0 {
if DiaryMediaUploaded(media) && local != media.size {
DiaryBlobAdd(pass, DIARY_BLOB_DOWNLOAD, name)
bytes += media.size
}
continue
}
if !uploads {
continue
}
if media.valid {
if local == media.size && (!DiaryMediaUploaded(media) ||
!DiaryBlobOnNode(address, TextFromBytes(media.blob[:64]))) {
DiaryBlobAdd(pass, DIARY_BLOB_UPLOAD, name)
bytes += media.size
}
continue
}
old: DiaryBlobRequest
if DiaryBlobChunked(entry, *old) {
DiaryBlobAdd(pass, DIARY_BLOB_UPLOAD, name)
bytes += old.size
}
}
SqliteFinalize(statement)
}
return pass.count > 0
}
// The bytes of a photo this device has, or "": its file, or for a photo
// from before copies, its chunks when the file is gone.
DiaryBlobLocalBytes :: (name: string, hash: string, size: s64, chunked: bool) -> string {
path := DiaryStorePath(name)
data := PackageRead(path, DIARY_FILE_LIMIT)
PackageFree(path)
if data.count == size {
descriptor := cast(*TextDescriptor)(*data)
actual: [65]u8
storage_sha256_hex(descriptor.data, data.count, *actual[0])
if TextFromCString(*actual[0]) == hash {
return data
}
}
PackageFree(data)
if chunked {
return DiaryRecordRead(name)
}
return ""
}
DiaryBlobPath :: (request: *DiaryBlobRequest) {
out: Vec(u8)
BuilderPrint(*out, "/api/v1/blobs/inbe/%", TextFromBytes(request.blob[:64]))
path := BuilderFinish(out)
unused AppCopyText(path, request.path[:])
PackageFree(path)
}
// Prepares the pass's next request, or false when the pass is done.
DiaryBlobNext :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest) -> bool {
while pass.next < pass.count {
job := pass.jobs[pass.next]
pass.next += 1
request.* = DiaryBlobRequest.{}
request.kind = job.kind
request.name = job.name
name := TextUntilNul(job.name[:])
key := DiaryRecordKey(name)
entry := CellRecordRead(key)
PackageFree(key)
media := DiaryMediaParse(entry)
if job.kind == DIARY_BLOB_DOWNLOAD {
PackageFree(entry)
if !DiaryMediaUploaded(media) {
continue
}
request.method = "GET"
request.blob = media.blob
request.key = media.key
request.hash = media.hash
request.size = media.size
request.answer_bytes = media.size + DIARY_BLOB_TAG + 1
DiaryBlobPath(request)
return true
}
if pass.uploads_stopped {
PackageFree(entry)
continue
}
if media.valid {
request.hash = media.hash
request.size = media.size
} else if !DiaryBlobChunked(entry, request) {
PackageFree(entry)
continue
}
PackageFree(entry)
hash := TextFromBytes(request.hash[:64])
data := DiaryBlobLocalBytes(name, hash, request.size, request.chunked)
if data.count == 0 {
continue
}
file_key: [32]u8
sealed: string = ""
if DiaryMediaUploaded(media) {
if HexToBytes(TextFromBytes(media.key[:64]), *file_key[0], 32) != 0 {
sealed = DiaryBlobSeal(data, *file_key[0])
}
} else if DiaryBlobKey(hash, *file_key[0]) {
sealed = DiaryBlobSeal(data, *file_key[0])
}
PackageFree(data)
if sealed.count == 0 {
SecureWipe(*file_key[0], 32)
continue
}
descriptor := cast(*TextDescriptor)(*sealed)
storage_sha256_hex(descriptor.data, sealed.count, *request.blob[0])
unused BytesToHex(*file_key[0], 32, *request.key[0], 65)
SecureWipe(*file_key[0], 32)
// A named copy is re-made only to reach another node; it must be
// the same copy.
if DiaryMediaUploaded(media) && TextFromBytes(request.blob[:64]) != TextFromBytes(media.blob[:64]) {
PackageFree(sealed)
continue
}
request.method = "PUT"
request.body = sealed
DiaryBlobPath(request)
return true
}
return false
}
// A photo from before copies frees its chunks once its copy is on a node:
// each becomes one byte, so its room on the node shrinks with it.
DiaryBlobShrinkChunks :: (request: *DiaryBlobRequest) {
storage_settings_begin_write()
defer { storage_settings_end_write() }
for index: cast(s32)0..request.chunks - 1 {
chunk := DiaryChunkKey(TextFromBytes(request.hash[:64]), index)
unused CellRecordWrite(chunk, "-")
PackageFree(chunk)
}
}
// Applies one answer. False stops the pass: the node or the account cannot
// take more now.
DiaryBlobFinish :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest, status: s32,
answer: []u8) -> bool {
name := TextUntilNul(request.name[:])
node := TextUntilNul(pass.node[:])
blob := TextFromBytes(request.blob[:64])
if request.kind == DIARY_BLOB_UPLOAD {
PackageFree(request.body)
request.body = ""
if status == 413 {
pass.uploads_stopped = true
unused SyncNodeStorageRecordFull(node, TextFromBytes(answer))
return true
}
if status != 204 && status != 200 {
return false
}
DiaryBlobRemember(node, blob)
media := DiaryMediaRead(name)
if DiaryMediaUploaded(media) {
return true
}
// The entry still describes this file; it now names the copy.
if (media.valid && TextFromBytes(media.hash[:64]) == TextFromBytes(request.hash[:64])) ||
request.chunked {
described: DiaryMedia
described.valid = true
described.hash = request.hash
described.size = request.size
entry := DiaryMediaUploadedEntry(described, blob, TextFromBytes(request.key[:64]))
key := DiaryRecordKey(name)
written := CellRecordWrite(key, entry)
PackageFree(key)
PackageFree(entry)
if written && request.chunked {
DiaryBlobShrinkChunks(request)
}
}
return true
}
if status == 404 {
return true
}
if status != 200 {
return false
}
media := DiaryMediaRead(name)
if !DiaryMediaUploaded(media) || TextFromBytes(media.blob[:64]) != blob {
return true
}
plain := DiaryBlobOpen(media, answer)
if plain.count == 0 {
return true
}
directory := DiaryStoreDirectory()
path := DiaryStorePath(name)
if storage_ensure_dir(directory) != 0 && PackageWrite(path, plain) {
DiaryBlobRemember(node, blob)
diary_blob_generation += 1
}
PackageFree(directory)
PackageFree(path)
PackageFree(plain)
return true
}
DiaryBlobAbandon :: (request: *DiaryBlobRequest) {
PackageFree(request.body)
request.body = ""
}
+46
View File
@@ -0,0 +1,46 @@
// The desktop moves Diary photo copies (diary_blobs.zi) on its sync worker,
// right after a good sync, one blocking request at a time.
#if #defined(PLATFORM_WEB) == false && #defined(ANDROID_BUILD) == false {
#import "auth"
#import "byte_text_linux"
#import "text_buffer"
#import, file "diary_blobs.zi";
#import, file "../storage/sync_runtime.zi";
diary_blobs_native_c :: #system_library "c";
DiaryBlobsAllocate :: (size: usize) -> *void #foreign diary_blobs_native_c "malloc";
DiaryBlobsRelease :: (pointer: *void) #foreign diary_blobs_native_c "free";
// A failed request ends the pass; the next sync tries again.
DiaryBlobsSendNative :: (url: string) {
pass: DiaryBlobPass
if !DiaryBlobPlan(*pass, url) {
return
}
request: DiaryBlobRequest
while DiaryBlobNext(*pass, *request) {
capacity: s64 = 4096
if request.answer_bytes > capacity {
capacity = request.answer_bytes
}
output := cast(*u8)DiaryBlobsAllocate(cast(usize)capacity)
if output == null {
DiaryBlobAbandon(*request)
return
}
status: s32 = 0
length: s64 = 0
sent := SendStoredBlob(url, request.method, TextUntilNul(request.path[:]), request.body,
BytesFromPointer(output, cast(u64)capacity), *status, *length)
keep := false
if sent == cast(s32)AuthResult.AUTH_OK && length >= 0 && length < capacity {
keep = DiaryBlobFinish(*pass, *request, status, BytesFromPointer(output, cast(u64)length))
}
DiaryBlobAbandon(*request)
DiaryBlobsRelease(cast(*void)output)
if !keep {
return
}
}
}
}
+189 -3
View File
@@ -1,9 +1,19 @@
// Immutable photo/text chunks and a per-file manifest share encrypted sync.
// A manifest becomes readable only when all chunks and its hash validate.
// Each Diary file has an entry, cell.diary.file.NAME, that syncs.
//
// A day document (YYYY-MM-DD.json) syncs its bytes as hex chunks beside the
// entry, which becomes readable only when all chunks and its hash validate.
//
// A photo or video stays a file on the device. Its entry syncs
// {"hash","size"}; once its sealed copy is on a node it also names that copy,
// "blob" (the copy's SHA-256, its address on the node) and "key". The copy is
// the file sealed with a key made for that file alone, so it takes the
// file's size plus 16 bytes. Entries with "count" are photos from before,
// still kept as chunks until their copy is uploaded (diary_blobs.zi).
#import "json_scan"
#import "byte_text_linux"
#import "vec"
#import "text"
#import "keys"
#import, file "../storage/cell_records.zi";
#import "crypto"
#import, file "lumi_store.zi";
@@ -11,6 +21,16 @@
#import, file "../storage/storage_core.zi";
DIARY_CHUNK_BYTES :: 1024
DIARY_FILE_LIMIT :: 16777216
// Sealing adds a 16-byte tag; the nonce is fixed (see DiaryBlobSeal).
DIARY_BLOB_TAG :: 16
// The account's Diary copy key comes from the account key with this context.
DIARY_BLOB_CONTEXT :: "inbe-diary-blob-key-v1"
DIARY_BLOB_AAD :: "inbe.diary.blob.v1"
diary_records_c :: #system_library "c";
DiaryAllocate :: (size: usize) -> *void #foreign diary_records_c "malloc";
DiaryRelease :: (pointer: *void) #foreign diary_records_c "free";
DiaryRecordKey :: (name: string) -> string {
return PackageText("cell.diary.file.", name)
@@ -22,8 +42,13 @@ DiaryChunkKey :: (hash: string, index: s32) -> string {
return BuilderFinish(out)
}
// Day documents are YYYY-MM-DD.json; every other Diary file is media.
DiaryRecordMedia :: (name: string) -> bool {
return name.count < 5 || name[name.count - 5:] != ".json"
}
DiaryRecordSave :: (name: string, data: string) -> bool {
if data.count <= 0 || data.count > 16777216 {
if data.count <= 0 || data.count > DIARY_FILE_LIMIT {
return false
}
descriptor := cast(*TextDescriptor)(*data)
@@ -40,6 +65,15 @@ DiaryRecordSave :: (name: string, data: string) -> bool {
if length == 64 && TextFromBytes(old_hash[:length]) == hash {
return true
}
if DiaryRecordMedia(name) {
// The file holds the bytes; only the entry syncs until the sealed
// copy is uploaded.
entry: Vec(u8)
BuilderPrint(*entry, "{\"hash\":\"%\",\"size\":%}", hash, data.count)
compact := BuilderFinish(entry)
defer { PackageFree(compact) }
return CellRecordWrite(key, compact)
}
storage_settings_begin_write()
defer { storage_settings_end_write() }
for index: cast(s32)0..count - 1 {
@@ -101,3 +135,155 @@ DiaryRecordRead :: (name: string) -> string {
}
return value
}
// A photo's entry without chunks, or not valid when the entry has chunks or
// does not parse.
DiaryMedia :: struct {
valid: bool
hash: [65]u8
size: s64
// Both empty until the sealed copy is on a node.
blob: [65]u8
key: [65]u8
}
DiaryHex64 :: (text: string) -> bool {
if text.count != 64 {
return false
}
for byte: text {
if !((byte >= 48 && byte <= 57) || (byte >= 97 && byte <= 102)) {
return false
}
}
return true
}
DiaryMediaParse :: (entry: string) -> DiaryMedia {
result: DiaryMedia
if entry.count == 0 || FieldAt(entry, 0, "count") >= 0 {
return result
}
size := ReadNumber(entry, FieldAt(entry, 0, "size"))
if ReadStringText(entry, FieldAt(entry, 0, "hash"), result.hash[:]) != 64 ||
!DiaryHex64(TextFromBytes(result.hash[:64])) || !size.valid || size.value < 1 ||
size.value > cast(float64)DIARY_FILE_LIMIT {
return DiaryMedia.{}
}
result.size = cast(s64)size.value
if FieldAt(entry, 0, "blob") >= 0 {
if ReadStringText(entry, FieldAt(entry, 0, "blob"), result.blob[:]) != 64 ||
ReadStringText(entry, FieldAt(entry, 0, "key"), result.key[:]) != 64 ||
!DiaryHex64(TextFromBytes(result.blob[:64])) || !DiaryHex64(TextFromBytes(result.key[:64])) {
return DiaryMedia.{}
}
}
result.valid = true
return result
}
DiaryMediaRead :: (name: string) -> DiaryMedia {
key := DiaryRecordKey(name)
entry := CellRecordRead(key)
defer { PackageFree(key); PackageFree(entry) }
return DiaryMediaParse(entry)
}
DiaryMediaUploaded :: (media: DiaryMedia) -> bool {
return media.valid && media.blob[0] != cast(u8)0
}
// Whether data is exactly the file an entry describes.
DiaryMediaMatches :: (media: DiaryMedia, data: string) -> bool {
if !media.valid || data.count != media.size {
return false
}
descriptor := cast(*TextDescriptor)(*data)
actual: [65]u8
storage_sha256_hex(descriptor.data, data.count, *actual[0])
return TextFromCString(*actual[0]) == TextFromBytes(media.hash[:64])
}
// The key that seals one file comes from the account's Diary key and the
// file's hash, so every device of the account makes the same copy.
DiaryBlobKey :: (hash: string, output: *u8) -> bool {
master: [32]u8
if storage_private_record_key_for_context(DIARY_BLOB_CONTEXT, *master[0]) == 0 {
return false
}
ok := HmacSha256Text(*master[0], 32, hash, output) != 0
SecureWipe(*master[0], 32)
return ok
}
DiaryBlobAad :: (output: []u8) {
text := DIARY_BLOB_AAD
for index: cast(s64)0..text.count - 1 {
output[index] = text[index]
}
}
// The file sealed for a node, or "" when it cannot be sealed. Its key seals
// only this file, so the fixed zero nonce is never used twice with a key.
DiaryBlobSeal :: (data: string, key: *u8) -> string {
if data.count <= 0 || data.count > DIARY_FILE_LIMIT {
return ""
}
count := data.count + DIARY_BLOB_TAG
output := cast(*u8)DiaryAllocate(cast(usize)count)
if output == null {
return ""
}
nonce: [12]u8
aad: [32]u8
DiaryBlobAad(aad[:])
descriptor := cast(*TextDescriptor)(*data)
if ChaCha20Poly1305Seal(key, *nonce[0], descriptor.data, data.count, *aad[0],
cast(s64)DIARY_BLOB_AAD.count, output, count) == 0 {
DiaryRelease(cast(*void)output)
return ""
}
return TextFromBytes(BytesFromPointer(output, cast(u64)count))
}
// The file inside a sealed copy, or "" when the copy is not the one its
// entry names.
DiaryBlobOpen :: (media: DiaryMedia, sealed: []u8) -> string {
if !DiaryMediaUploaded(media) || cast(s64)sealed.count != media.size + DIARY_BLOB_TAG {
return ""
}
actual: [65]u8
storage_sha256_hex(*sealed[0], cast(s64)sealed.count, *actual[0])
if TextFromCString(*actual[0]) != TextFromBytes(media.blob[:64]) {
return ""
}
key: [32]u8
if HexToBytes(TextFromBytes(media.key[:64]), *key[0], 32) == 0 {
return ""
}
output := cast(*u8)DiaryAllocate(cast(usize)media.size)
if output == null {
SecureWipe(*key[0], 32)
return ""
}
nonce: [12]u8
aad: [32]u8
DiaryBlobAad(aad[:])
opened := ChaCha20Poly1305Open(*key[0], *nonce[0], *sealed[0], cast(s64)sealed.count,
*aad[0], cast(s64)DIARY_BLOB_AAD.count, output, media.size) != 0
SecureWipe(*key[0], 32)
value := TextFromBytes(BytesFromPointer(output, cast(u64)media.size))
if !opened || !DiaryMediaMatches(media, value) {
DiaryRelease(cast(*void)output)
return ""
}
return value
}
// The entry once the sealed copy is on a node.
DiaryMediaUploadedEntry :: (media: DiaryMedia, blob: string, key: string) -> string {
out: Vec(u8)
BuilderPrint(*out, "{\"hash\":\"%\",\"size\":%,\"blob\":\"%\",\"key\":\"%\"}",
TextFromBytes(media.hash[:64]), media.size, blob, key)
return BuilderFinish(out)
}
+33 -1
View File
@@ -38,6 +38,10 @@ diary_prepared: bool;
diary_account: string;
diary_preview_name: string;
diary_preview_bytes: string;
// Bumped when a photo's file arrives from a node, so a placeholder shown for
// it is read again.
diary_blob_generation: s64;
diary_preview_generation: s64;
#if !#defined(_WIN32) && !#defined(PLATFORM_WEB) {
DiaryLock :: (descriptor: s32, operation: s32) -> s32 #foreign c "flock";
}
@@ -189,6 +193,24 @@ DiaryStoreRead :: (name: string) -> DiaryFileResult {
}
result.missing = !PathExists(path)
PackageFree(diary_file_bytes)
diary_file_bytes = ""
media := DiaryMediaRead(name)
if media.valid {
// A photo is its file. Until the file arrives from a node, it shows
// as not on this device.
diary_file_bytes = PackageRead(path, DIARY_FILE_LIMIT)
if !DiaryMediaMatches(media, diary_file_bytes) {
PackageFree(diary_file_bytes)
diary_file_bytes = ""
result.missing = false
result.succeeded = false
return result
}
result.missing = false
result.data = diary_file_bytes
result.succeeded = true
return result
}
diary_file_bytes = DiaryRecordRead(name)
if diary_file_bytes.count > 0 {
result.missing = false
@@ -382,14 +404,24 @@ DiaryStorePreview :: (name: string) -> string {
if !DiaryStoreNameValid(name) || !DiaryStorePrepare() {
return ""
}
if name != diary_preview_name {
if name != diary_preview_name || diary_preview_generation != diary_blob_generation {
PackageFree(diary_preview_name)
PackageFree(diary_preview_bytes)
diary_preview_name = PackageText(name, "")
diary_preview_generation = diary_blob_generation
path := DiaryStorePath(name)
defer {
PackageFree(path)
}
media := DiaryMediaRead(name)
if media.valid {
diary_preview_bytes = PackageRead(path, DIARY_FILE_LIMIT)
if !DiaryMediaMatches(media, diary_preview_bytes) {
PackageFree(diary_preview_bytes)
diary_preview_bytes = ""
}
return diary_preview_bytes
}
diary_preview_bytes = DiaryRecordRead(name)
record_key := DiaryRecordKey(name)
manifest := CellRecordRead(record_key)
+2 -2
View File
@@ -29,9 +29,9 @@ AccountFieldText :: (data: *u8, capacity: s64) -> string {
return TextFromBytes(BytesFromPointer(data, cast(u64)length))
}
// A signed GET has an empty body, whose SHA-256 is still signed.
DigestBody :: (body: string, output: []u8) -> bool {
if body.count == 0 || body.count > 9223372036854775807 ||
output.count < 65 {
if body.count < 0 || output.count < 65 {
return false
}
descriptor: *TextDescriptor = cast(*TextDescriptor)(*body)
+3
View File
@@ -235,6 +235,9 @@ migrate_schema :: () -> s32 {
if !SchemaExec(SchemaSyncRecordCopies) {
return SchemaUpgradeFailed()
}
if !SchemaExec(SchemaDiaryBlobs) {
return SchemaUpgradeFailed()
}
if SchemaHasColumn("habit_days", "answers") && !SchemaExec(SchemaDayAnswersToEntries) {
return SchemaUpgradeFailed()
}
+9
View File
@@ -278,6 +278,15 @@ CREATE TABLE IF NOT EXISTS sync_record_copies( collection TEXT NOT NULL,
PRIMARY KEY(collection,id))
SQL
// The sealed Diary photo copies (diary_blobs.zi) each node is known to
// hold, by the node's normalized address, so a copy uploads once per node.
// Only this device keeps it; a lost row means one more upload.
SchemaDiaryBlobs :: #string SQL
CREATE TABLE IF NOT EXISTS diary_blobs( node TEXT NOT NULL,
blob TEXT NOT NULL,
PRIMARY KEY(node,blob))
SQL
// A habit's entries: each a moment of a day, or a span from its start to
// its end (0 while it runs), with its answers to the habit's questions as
// a JSON array. An entry belongs to the day it started.
+98
View File
@@ -12,6 +12,7 @@ Auth :: #import "auth";
#import, file "delegate_storage_types.zi";
#import, file "delegate_storage.zi";
#import "net_http"
#import "http_transfer"
#import "url"
#import "byte_text_linux"
Clock :: #import "date_time_linux";
@@ -400,3 +401,100 @@ PollStoredOperation :: (operation: *StoredOperation) -> bool {
ReleaseStoredOperation(operation)
return true
}
// One signed request for a sealed Diary copy (cells/diary_blobs.zi),
// advanced each frame like a sync: PUT stores the body at path, GET answers
// with the copy in output. It uses the sync's saved session; without one it
// does not start, and the next sync signs in again.
StoredBlob :: struct {
active: bool
transfer: HttpTransfer
transaction: [8192]u8
token: [4096]u8
url: [1024]u8
method: [8]u8
headers: [1]HttpHeader
status: s32
length: s64
result: s32
}
// The body and output must stay valid until PollStoredBlob finishes or
// CancelStoredBlob returns.
BeginStoredBlob :: (blob: *StoredBlob, server: string, method: string, path: string,
body: string, output: []u8) -> bool {
if blob == null || blob.active || StoredAccountMode() != AccountModeOwner {
return false
}
blob.* = StoredBlob.{}
blob.result = cast(s32)AuthResult.AUTH_NO_ACCOUNT
account: AccountKeys
if sync_account_load(*account) == 0 {
return false
}
defer { SecureWipe(*account.private_key_hex[0], ACCOUNT_PRIVATE_KEY_HEX_SIZE) }
material: DeviceKeyMaterial
if !LoadOrCreateDeviceKey(*material) {
return false
}
defer { WipeDeviceKey(*material) }
client: Client
client_id: [128]u8
normalized: [512]u8
if !AppCopyText(storage_sync_client_id(), client_id[:]) ||
!NormalizeUrl(server, normalized[:]) ||
!OpenAccountClient(TextUntilNul(normalized[:]), *account, TextUntilNul(client_id[:]), *client) {
return false
}
now := Clock.UnixNow()
session: Session
LoadSavedSession(*session, now)
if session.token[0] == cast(u8)0 {
return false
}
blob.result = cast(s32)AuthResult.AUTH_SIGN_FAILED
if PrepareRequestHeader(client, *session, DeviceFromKey(*material), now, null, NativeNonce,
method, path, body, blob.transaction[:]) != cast(AuthResult)AuthResult.AUTH_OK ||
!JoinUrl(TextUntilNul(normalized[:]), path, blob.url[:]) ||
!AppCopyText(TextUntilNul(session.token[:]), blob.token[:]) ||
!AppCopyText(method, blob.method[:]) {
return false
}
blob.headers[0] = HttpHeader.{.name = "X-Daochi-Tx", .value = TextUntilNul(blob.transaction[:])}
request := HttpRequest.{.method = TextUntilNul(blob.method[:]), .url = TextUntilNul(blob.url[:]),
.token = TextUntilNul(blob.token[:]), .body = body, .headers = blob.headers[:]}
if body.count > 0 {
request.content_type = "application/octet-stream"
}
blob.result = cast(s32)AuthResult.AUTH_REQUEST_FAILED
// A copy of up to 16 MB needs longer than a sync's request.
if !HttpTransferStart(*blob.transfer, request, output, 180000) {
return false
}
blob.active = true
return true
}
// True once the request has finished; status and length then hold its
// answer, and result is AUTH_OK when one arrived.
PollStoredBlob :: (blob: *StoredBlob) -> bool {
if blob == null || !blob.active {
return true
}
state := HttpTransferPollLength(*blob.transfer, *blob.status, *blob.length)
if state < 0 {
return false
}
blob.active = false
blob.result = ifx (state == 1) then (cast(s32)AuthResult.AUTH_OK) else (cast(s32)AuthResult.AUTH_REQUEST_FAILED)
return true
}
CancelStoredBlob :: (blob: *StoredBlob) {
if blob == null || !blob.active {
return
}
HttpTransferCancel(*blob.transfer)
blob.active = false
blob.result = cast(s32)AuthResult.AUTH_REQUEST_FAILED
}
+13 -7
View File
@@ -2,7 +2,8 @@
// any. This device keeps the last report per account and node, and whether
// Diary photos and videos upload to that node. A Diary file entry always
// syncs, so other devices see that a photo exists even where its bytes are
// withheld; only the photo's chunks stay off the node.
// withheld; only the photo's sealed copy (cells/diary_blobs.zi), or for a
// photo from before copies its chunks, stays off the node.
#import "json_scan"
#import "byte_text_linux"
#import "text"
@@ -48,13 +49,18 @@ WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
ON CONFLICT(entity_type,entity_id,local_date) DO UPDATE SET queued_at=excluded.queued_at
SQL_END;
// A sealed copy is its file plus a 16-byte tag; a photo from before copies
// still costs its chunks.
SyncMediaLocalBytesSQL :: #string SQL_END
SELECT COALESCE(SUM(125+LENGTH(key)+2*(LENGTH(key)+LENGTH(value)+73)),0) FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
AND substr(key,18,64) IN (
SELECT json_extract(value,'$.hash') FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
AND key NOT GLOB '*.json' AND json_valid(value))
SELECT COALESCE((SELECT SUM(json_extract(value,'$.size')+16) FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
AND key NOT GLOB '*.json' AND json_valid(value) AND json_type(value,'$.count') IS NULL),0)
+COALESCE((SELECT SUM(125+LENGTH(key)+2*(LENGTH(key)+LENGTH(value)+73)) FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
AND substr(key,18,64) IN (
SELECT json_extract(value,'$.hash') FROM settings
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
AND key NOT GLOB '*.json' AND json_valid(value) AND json_type(value,'$.count') IS NOT NULL)),0)
SQL_END;
// Keys follow the node's normalized address, so a report and a setting made
+67
View File
@@ -12,6 +12,9 @@
#import, file "delegate_storage_types.zi";
#import, file "delegate_storage.zi";
#import "net_ws_curl_linux"
#import "net_http"
#import "net_http_curl_linux"
#import "url"
#import "byte_text_linux"
#import "text_buffer"
#import, file "account_types.zi";
@@ -144,6 +147,70 @@ SyncStoredAccount :: (base_url: string) -> s32 {
return outcome
}
// One signed request for a sealed Diary copy (cells/diary_blobs.zi): PUT
// stores body at path, GET answers with the copy in output. The bearer
// session is the sync's, renewed here when it has run out.
#program_export
SendStoredBlob :: (base_url: string, method: string, path: string, body: string,
output: []u8, status: *s32, length: *s64) -> s32 {
status.* = 0
length.* = 0
if StoredAccountMode() != AccountModeOwner {
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
}
account: AccountKeys
if sync_account_load(*account) == 0 {
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
}
defer { SecureWipe(*account.private_key_hex[0], ACCOUNT_PRIVATE_KEY_HEX_SIZE) }
material: DeviceKeyMaterial
if !LoadOrCreateDeviceKey(*material) {
return cast(s32)AuthResult.AUTH_SIGN_FAILED
}
defer { WipeDeviceKey(*material) }
client: Client
if !OpenAccountClient(base_url, *account, storage_sync_client_id(), *client) {
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
}
client.send = SendNative
now := time(null)
session: Session
LoadSavedSession(*session, now)
if !TokenValid(*session, now) {
signed_in := Login(client, *session)
if signed_in != cast(AuthResult)AuthResult.AUTH_OK {
return cast(s32)signed_in
}
SaveSession(*session)
}
transaction: [8192]u8
prepared := PrepareRequestHeader(client, *session, DeviceFromKey(*material), now,
null, NativeNonce, method, path, body, transaction[:])
if prepared != cast(AuthResult)AuthResult.AUTH_OK {
return cast(s32)prepared
}
server: [512]u8
url: [1024]u8
if !NormalizeUrl(base_url, server[:]) || !JoinUrl(TextUntilNul(server[:]), path, url[:]) {
return cast(s32)AuthResult.AUTH_INVALID_URL
}
headers: [1]HttpHeader
headers[0] = HttpHeader.{.name = "X-Daochi-Tx", .value = TextUntilNul(transaction[:])}
request := HttpRequest.{.method = method, .url = TextUntilNul(url[:]),
.token = TextUntilNul(session.token[:]), .body = body, .headers = headers[:]}
if body.count > 0 {
request.content_type = "application/octet-stream"
}
// A copy of up to 16 MB needs longer than a sync's request.
result := SendCurlWithin(request, output, 180000)
status.* = result.status
length.* = result.length
if result.code != 0 || result.truncated {
return cast(s32)AuthResult.AUTH_REQUEST_FAILED
}
return cast(s32)AuthResult.AUTH_OK
}
// Expired context can perform only this key-bound challenge/session exchange.
// Data payload construction remains unavailable until the answer is accepted.
RenewStoredDelegateNative :: (base_url: string) -> AuthResult {
+168
View File
@@ -20,6 +20,9 @@ ModuleTestFailRun :: () #foreign c_cache;
#import, file "../src/cells/diary_types.zi";
Dates :: #import, file "../apps/diary/diary_date.zi";
#import, file "../src/cells/diary_store.zi";
#import, file "../src/cells/diary_records.zi";
#import, file "../src/cells/diary_blobs.zi";
#import, file "../src/storage/cell_records.zi";
#import, file "../src/cells/package_files.zi";
#import, file "../src/app/app_types.zi";
#import, file "../src/core/types.zi";
@@ -43,6 +46,8 @@ Dates :: #import, file "../apps/diary/diary_date.zi";
#import, file "../src/app/text_buffers.zi";
#import, file "../src/app/app_size.zi";
#import "byte_text_linux"
#import "text_buffer"
#import "crypto"
#import, file "../src/app/assets.zi";
#import, file "../src/app/locale.zi";
#import, file "../src/storage/sync_node_storage.zi";
@@ -274,6 +279,7 @@ CheckDiary :: () {
photo = DiaryRun(DiaryMessage.{.action = DIARY_LOAD, .year = 2024, .month = 3, .day = 1})
Expect(photo.photo_count == 1 && photo.photo_name == "A second photo 🌿", 105)
CheckDiaryPhotoOnFullNode()
CheckDiaryPhotoCopies()
Expect(DiaryStoreSave("2024-03-02.json", "damaged"), 92)
bad := DiaryRun(DiaryMessage.{.action = DIARY_LOAD, .year = 2024, .month = 3, .day = 2})
bad.action = DIARY_SAVE
@@ -353,6 +359,168 @@ CheckDiaryPhotoOnFullNode :: () {
storage_set_sync_enabled(sync_was_enabled)
}
DiaryTestImage :: (bytes: []u8, seed: s32) -> string {
header: [8]u8 = .[cast(u8)137, cast(u8)80, cast(u8)78, cast(u8)71, cast(u8)13, cast(u8)10, cast(u8)26, cast(u8)10]
for index: cast(s32)0..cast(s32)bytes.count - 1 {
bytes[index] = ifx index < 8 then header[index] else cast(u8)((index * 7 + seed) % 251)
}
return TextFromBytes(bytes)
}
DiaryTestBytes :: (text: string) -> []u8 {
descriptor := cast(*TextDescriptor)(*text)
return BytesFromPointer(descriptor.data, cast(u64)text.count)
}
DiaryTestHash :: (data: []u8, output: []u8) -> string {
storage_sha256_hex(*data[0], cast(s64)data.count, *output[0])
return TextFromCString(*output[0])
}
// The pass's job for one photo, or 0; other Diary tests leave photos too.
DiaryTestJob :: (pass: *DiaryBlobPass, name: string) -> s32 {
for index: cast(s32)0..pass.count - 1 {
if TextUntilNul(pass.jobs[index].name[:]) == name {
return pass.jobs[index].kind
}
}
return 0
}
// The pass's request for one photo, skipping the others.
DiaryTestNext :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest, name: string) -> bool {
while DiaryBlobNext(pass, request) {
if TextUntilNul(request.name[:]) == name {
return true
}
DiaryBlobAbandon(request)
}
return false
}
// A photo is its file: only its entry syncs, and its sealed copy, the file's
// size plus 16 bytes, travels to a node and back to the account's devices.
CheckDiaryPhotoCopies :: () {
node := "https://copies.test"
sync_was_enabled := storage_sync_enabled()
storage_set_sync_enabled(1)
storage_set_setting_text("sync_server_url", node)
// The account key lets this device seal copies.
private: [5121]u8
for index: cast(s32)0..cast(s32)5119 {
private[index] = ifx index % 2 == 0 then #char "a" else #char "7"
}
storage_set_setting_text(STORAGE_SYNC_PRIVATE_KEY_KEY, TextFromBytes(private[:5120]))
png: [3000]u8
image := DiaryTestImage(png[:], 3)
source := "copied-photo.png"
unused RemoveFile(source)
file := OpenExclusivePrivate(source)
written := file.valid && WriteTextAt(file, image, 0) == image.count
Expect(CloseFile(file) && written, 430)
name := DiaryStoreAttach(source)
unused RemoveFile(source)
Expect(name.count > 0, 431)
media := DiaryMediaRead(name)
hash_text: [65]u8
hash := DiaryTestHash(png[:], hash_text[:])
Expect(media.valid && !DiaryMediaUploaded(media) && media.size == 3000 &&
TextFromBytes(media.hash[:64]) == hash, 432)
no_chunks := PackageText(PackageText("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.diary.chunk.", hash), ".*'")
Expect(ModuleTestCount(no_chunks) == 0, 433)
PackageFree(no_chunks)
// A pass seals and uploads it, then its entry names the copy.
pass: DiaryBlobPass
request: DiaryBlobRequest
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, name) == DIARY_BLOB_UPLOAD, 434)
Expect(DiaryTestNext(*pass, *request, name) && request.method == "PUT" &&
request.body.count == image.count + DIARY_BLOB_TAG && request.body != image, 435)
sealed := PackageText(request.body, "")
blob_text: [65]u8
blob := DiaryTestHash(DiaryTestBytes(sealed), blob_text[:])
blob_path := PackageText("/api/v1/blobs/inbe/", blob)
Expect(TextUntilNul(request.path[:]) == blob_path, 436)
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]), 437)
media = DiaryMediaRead(name)
Expect(DiaryMediaUploaded(media) && TextFromBytes(media.blob[:64]) == blob, 438)
unused DiaryBlobPlan(*pass, node)
Expect(DiaryTestJob(*pass, name) == 0, 439)
// Another device of the account has only the entry: the photo shows as
// not on the device until its copy arrives, opened and checked.
path := DiaryStorePath(name)
unused RemoveFile(path)
Expect(!DiaryStoreRead(name).succeeded && DiaryStorePreview(name).count == 0, 440)
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, name) == DIARY_BLOB_DOWNLOAD, 441)
Expect(DiaryTestNext(*pass, *request, name) && request.method == "GET" &&
request.answer_bytes == sealed.count + 1, 442)
tampered := PackageText(sealed, "")
changed := DiaryTestBytes(tampered)
changed[20] = cast(u8)((cast(s32)changed[20] + 1) % 256)
generation := diary_blob_generation
Expect(DiaryBlobFinish(*pass, *request, 200, changed) &&
!PathExists(path) && diary_blob_generation == generation, 443)
Expect(DiaryBlobFinish(*pass, *request, 404, png[0:0]) && !PathExists(path), 444)
Expect(DiaryBlobFinish(*pass, *request, 200, DiaryTestBytes(sealed)) &&
diary_blob_generation > generation, 445)
Expect(DiaryStoreRead(name).data == image && DiaryStorePreview(name) == image, 446)
// A new node gets the same copy once.
other := "https://other-copies.test"
Expect(DiaryBlobPlan(*pass, other) && DiaryTestJob(*pass, name) == DIARY_BLOB_UPLOAD, 447)
Expect(DiaryTestNext(*pass, *request, name) && request.body == sealed, 448)
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]), 449)
unused DiaryBlobPlan(*pass, other)
Expect(DiaryTestJob(*pass, name) == 0, 459)
// A node without photo upload receives none; a full one stops uploads.
third := "https://third-copies.test"
Expect(SyncNodeSetMediaUpload(third, false) && !DiaryBlobPlan(*pass, third), 450)
Expect(SyncNodeSetMediaUpload(third, true) && DiaryBlobPlan(*pass, third) &&
DiaryTestNext(*pass, *request, name), 451)
full := "{\"code\":\"storage_full\",\"account_storage\":{\"used_bytes\":20971000,\"limit_bytes\":20971520}}"
full_text := PackageText(full, "")
Expect(DiaryBlobFinish(*pass, *request, 413, DiaryTestBytes(full_text)) &&
pass.uploads_stopped && SyncNodeStorageRead(third).full, 452)
PackageFree(full_text)
// A photo from before copies, kept as hex chunks, uploads the same copy
// and its chunks shrink to one byte each.
older: [2500]u8
old_image := DiaryTestImage(older[:], 9)
old_name := "photo-20240301000000-1.bin"
old_hash_text: [65]u8
old_hash := DiaryTestHash(older[:], old_hash_text[:])
for index: cast(s32)0..2 {
start := index * 1024
size: s32 = 1024
if start + size > 2500 {
size = 2500 - start
}
hex: [2049]u8
unused BytesToHex(*older[start], cast(s64)size, *hex[0], 2049)
chunk := DiaryChunkKey(old_hash, index)
Expect(CellRecordWrite(chunk, TextFromCString(*hex[0])), 453)
PackageFree(chunk)
}
old_entry := PackageText(PackageText("{\"hash\":\"", old_hash), "\",\"count\":3,\"size\":2500}")
old_key := DiaryRecordKey(old_name)
Expect(CellRecordWrite(old_key, old_entry) && DiaryStoreRead(old_name).data == old_image, 454)
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, old_name) == DIARY_BLOB_UPLOAD &&
DiaryTestNext(*pass, *request, old_name) && request.chunked && request.body.count == 2500 + DIARY_BLOB_TAG, 455)
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]) && DiaryMediaUploaded(DiaryMediaRead(old_name)), 456)
shrunk := PackageText(PackageText("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.diary.chunk.", old_hash), ".*' AND value='-'")
Expect(ModuleTestCount(shrunk) == 3, 457)
PackageFree(shrunk)
Expect(DiaryStoreRead(old_name).data == old_image, 458)
PackageFree(old_key)
PackageFree(old_entry)
PackageFree(tampered)
PackageFree(blob_path)
PackageFree(sealed)
PackageFree(path)
PackageFree(name)
storage_set_setting_text(STORAGE_SYNC_PRIVATE_KEY_KEY, "")
storage_set_setting_text("sync_server_url", "")
storage_set_sync_enabled(sync_was_enabled)
}
CheckPackagedLocales :: () {
Expect(LocaleLanguageCount() == 12, 200)
for index: cast(s32)0..LocaleLanguageCount() - 1 {
+64 -14
View File
@@ -117,6 +117,41 @@ queued_diary_chunks :: () -> s32 {
return read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_type='cell_record' AND entity_id GLOB 'cell.diary.chunk.*'")
}
// A photo saved before sealed copies: hex chunks and an entry that counts
// them, as older versions still send.
save_legacy_photo :: (name: string, data: string) -> bool {
descriptor := cast(*TextDescriptor)(*data)
hash_bytes: [65]u8
storage_sha256_hex(descriptor.data, data.count, *hash_bytes[0])
hash := TextFromCString(*hash_bytes[0])
count := cast(s32)((data.count + DIARY_CHUNK_BYTES - 1) / DIARY_CHUNK_BYTES)
for index: cast(s32)0..count - 1 {
start := cast(s64)index * DIARY_CHUNK_BYTES
size := data.count - start
if size > DIARY_CHUNK_BYTES {
size = DIARY_CHUNK_BYTES
}
encoded: [DIARY_CHUNK_BYTES * 2 + 1]u8
if BytesToHex(*descriptor.data[start], size, *encoded[0], encoded.count) == 0 {
return false
}
chunk := DiaryChunkKey(hash, index)
written := CellRecordWrite(chunk, TextFromCString(*encoded[0]))
PackageFree(chunk)
if !written {
return false
}
}
out: Vec(u8)
BuilderPrint(*out, "{\"hash\":\"%\",\"count\":%,\"size\":%}", hash, count, data.count)
entry := BuilderFinish(out)
key := DiaryRecordKey(name)
written := CellRecordWrite(key, entry)
PackageFree(key)
PackageFree(entry)
return written
}
format_bytes :: (bytes: s64) -> string {
output: [32]u8
if !SyncStorageFormatBytes(bytes, output[:]) {
@@ -125,8 +160,9 @@ format_bytes :: (bytes: s64) -> string {
return PackageText(TextFromCString(*output[0]), "")
}
// A node set not to store photos receives each Diary file entry and the day
// documents, never photo chunks. Reports and limits.
// A photo syncs only its entry; its sealed copy travels separately
// (cells/diary_blobs.zi). A node set not to store photos still never
// receives the chunks of a photo from before copies. Reports and limits.
test_node_storage_and_media :: () {
root := make_root("sync-node-media")
check_true("init media device", storage_init(root) != 0)
@@ -140,15 +176,23 @@ test_node_storage_and_media :: () {
photo[index] = cast(u8)(index % 251)
}
check_true("save diary photo", DiaryRecordSave("photo-1-1.bin", TextFromBytes(photo[:])))
check_int("day and photo chunks queue", queued_diary_chunks(), 4)
check_int("a photo queues no chunks", queued_diary_chunks(), 1)
check_true("photos upload by default", SyncNodeMediaUpload(node))
check_true("payload with photos builds", build_payload())
check_int("file entries and every chunk upload", record_count(), 6)
photo_hash := PackageText(read_db_text("SELECT json_extract(value,'$.hash') FROM settings WHERE key='cell.diary.file.photo-1-1.bin'"), "")
check_int("photo entry names its content", cast(s32)photo_hash.count, 64)
check_true("local photo bytes equal the node's chunk bytes",
SyncMediaLocalBytes() == payload_file_bytes("photo-1-1.bin", photo_hash, true))
check_true("payload with a photo builds", build_payload())
check_int("file entries and the day chunk upload", record_count(), 3)
check_true("a photo's copy is its size plus 16 bytes", SyncMediaLocalBytes() == 3016)
older: [2500]u8
for index: cast(s32)0..cast(s32)2499 {
older[index] = cast(u8)((index * 3) % 251)
}
check_true("keep a photo from before copies", save_legacy_photo("photo-0-1.bin", TextFromBytes(older[:])))
check_int("its chunks queue", queued_diary_chunks(), 4)
check_true("payload with the old photo builds", build_payload())
check_int("entries and every chunk upload", record_count(), 7)
photo_hash := PackageText(read_db_text("SELECT json_extract(value,'$.hash') FROM settings WHERE key='cell.diary.file.photo-0-1.bin'"), "")
check_int("old photo entry names its content", cast(s32)photo_hash.count, 64)
check_true("local photo bytes count the copy and the old chunks",
SyncMediaLocalBytes() == 3016 + payload_file_bytes("photo-0-1.bin", photo_hash, true))
PackageFree(photo_hash)
check_true("turn photos off for the node", SyncNodeSetMediaUpload(node, false))
@@ -156,11 +200,12 @@ test_node_storage_and_media :: () {
check_true("other nodes keep photos on", SyncNodeMediaUpload("https://other.example"))
check_int("photo chunks leave the queue", queued_diary_chunks(), 1)
check_true("payload without photos builds", build_payload())
check_int("only entries and the day chunk upload", record_count(), 3)
check_true("photo entry still syncs", contains(payload(), "cell.diary.file.photo-1-1.bin"))
check_int("only entries and the day chunk upload", record_count(), 4)
check_true("photo entries still sync", contains(payload(), "cell.diary.file.photo-1-1.bin") &&
contains(payload(), "cell.diary.file.photo-0-1.bin"))
storage_enqueue_all_sync_state()
check_true("backfill payload builds", build_payload())
check_int("a backfill does not send photos", record_count(), 3)
check_int("a backfill does not send photos", record_count(), 4)
check_true("turn photos back on", SyncNodeSetMediaUpload(node, true))
check_int("photos queue again", queued_diary_chunks(), 4)
@@ -1754,7 +1799,9 @@ test_cell_records_round_trip :: () {
for index: cast(s32)0..cast(s32)4096 {
photo[index] = cast(u8)(index % 256)
}
check_true("save binary diary photo", DiaryRecordSave("photo-test.png", TextFromBytes(photo[:])))
// Photos from older versions still arrive as binary chunks.
check_true("save binary diary photo", save_legacy_photo("photo-test.png", TextFromBytes(photo[:])))
check_true("save a photo entry", DiaryRecordSave("photo-new.png", TextFromBytes(photo[:4000])))
check_true("cell payload builds", build_payload())
check_true("Lumi uses separate encrypted collection", contains(payload(), "private.inbe.v2.lumi"))
check_true("Diary retains legacy owner collection", contains(payload(), "private.inbe.v1.cells"))
@@ -1780,6 +1827,9 @@ test_cell_records_round_trip :: () {
restored := DiaryRecordRead("photo-test.png")
check_true("binary photo survives", restored == TextFromBytes(photo[:]))
PackageFree(restored)
entry := DiaryMediaRead("photo-new.png")
check_true("a photo entry arrives without its bytes", entry.valid && entry.size == 4000 &&
!DiaryMediaUploaded(entry) && DiaryRecordRead("photo-new.png").count == 0)
check_int("restore does not requeue cells", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_type='cell_record'"), 0)
check_true("cell collection cannot overwrite signing key",
storage_apply_decrypted_record("private.inbe.v1.cells", "sync_private_key", "{\"key\":\"sync_private_key\",\"value\":\"forged\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") == 0)
+54
View File
@@ -28,6 +28,10 @@
#import "authorization"
#import, file "../src/core/types.zi";
#import, file "../src/cells/package_files.zi";
#import, file "../src/cells/diary_store.zi";
#import, file "../src/cells/diary_records.zi";
#import, file "../src/cells/diary_blobs_native.zi";
#import "file_linux"
// Two clients, a friend, and fresh installs syncing through a real Daochi
// server (with a proxy that drops one response), from the first upload to
@@ -155,6 +159,54 @@ synchronize :: (url: string) {
check_true("sync drained the queued changes", false)
}
// A Diary photo travels as one sealed copy, which the node keeps at the
// photo's size plus 16 bytes, and arrives whole on the account's other
// device.
expect_photo_copy :: (url: string, a: string, b: string) {
open_client(a)
png: [6000]u8
for index: cast(s32)0..cast(s32)5999 {
png[index] = cast(u8)((index * 13 + 7) % 251)
}
header: [8]u8 = .[cast(u8)137, cast(u8)80, cast(u8)78, cast(u8)71, cast(u8)13, cast(u8)10, cast(u8)26, cast(u8)10]
for index: cast(s32)0..7 {
png[index] = header[index]
}
image := TextFromBytes(png[:])
source: TextBuffer
append(*source, data_root())
append(*source, "/photo-source.png")
source_path := TextFromCString(*source.bytes[0])
unused RemoveFile(source_path)
file := OpenExclusivePrivate(source_path)
written := file.valid && WriteTextAt(file, image, 0) == image.count
check_true("write the photo to attach", CloseFile(file) && written)
name := DiaryStoreAttach(source_path)
check_true("attach the photo", name.count > 0)
DiaryStoreClose()
synchronize(url)
DiaryBlobsSendNative(url)
media := DiaryMediaRead(name)
check_true("the photo's copy is on the node", DiaryMediaUploaded(media))
synchronize(url)
blob_path: TextBuffer
append(*blob_path, data_root())
append(*blob_path, "/server.db.blobs/")
append(*blob_path, storage_get_setting_text("sync_public_id"))
append(*blob_path, "/inbe/")
append(*blob_path, TextFromBytes(media.blob[:64]))
check_int("the node keeps the photo's size plus 16 bytes",
cast(s32)PackageFileSize(TextFromCString(*blob_path.bytes[0])), 6016)
open_client(b)
synchronize(url)
check_true("the photo waits on the second device", !DiaryStoreRead(name).succeeded)
DiaryStoreClose()
DiaryBlobsSendNative(url)
check_true("the photo arrives whole on the second device", DiaryStoreRead(name).data == image)
DiaryStoreClose()
PackageFree(name)
}
mood_is :: (id: string, want: s32) -> bool {
loaded: SessionMoodCheckin
return storage_load_session_checkin(id, *loaded) != 0 && loaded.mood_after == want
@@ -309,6 +361,8 @@ Check :: () -> s32 {
check_true("completion reaches other device", ActivityRead(live_document, *observed) && observed.active == 0)
PackageFree(live_document)
check_int("live activity does not duplicate results", storage_session_count(), 1)
expect_photo_copy(url, a, b)
open_client(b)
check_true("complete the item", storage_elist_update_item(item_text, "Pack bag", "Passport", 1, 0) != 0)
synchronize(url)
open_client(a)
+3 -1
View File
@@ -28,11 +28,13 @@ mkdir -p "$work/generated"
cat > "$work/generated/main.c" <<'C'
#include <stdio.h>
#include "sync_server_behavior.h"
/* Diary files are flushed by the browser's storage only on the web. */
void app_web_storage_flush(void) {}
int main(void)
{
int result = Check();
if (result != 0) fprintf(stderr, "sync server check failed with code %d\n", result);
if (result == 0) puts("PASS two-client sync, habit/session/list/alias/friend recovery, offline retry, conflict, backup restore, deletion");
if (result == 0) puts("PASS two-client sync, habit/session/list/alias/friend recovery, Diary photo copies, offline retry, conflict, backup restore, deletion");
return result == 0 ? 0 : 1;
}
C
+6 -6
View File
@@ -55,7 +55,7 @@
},
{
"commit": "41538659afbbda300915ef009b55c18f6de159b8",
"dependencies": {"kryon": "pe4492807d82c43f5"},
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
"id": "p9504fb8e5fabefc6",
"name": "Kss",
"ref": "master",
@@ -71,7 +71,7 @@
},
{
"commit": "2ee29338c9b93fb4f4a56a49c7be09e9d2c2418f",
"dependencies": {"kryon": "pe4492807d82c43f5"},
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
"id": "pa5dd7b8b83307b18",
"name": "plot",
"ref": "master",
@@ -79,7 +79,7 @@
},
{
"commit": "7aa01465ac3f988e0404f3c6dc244bf4b009f253",
"dependencies": {"kryon": "pe4492807d82c43f5"},
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
"id": "paa08e1730a25bda0",
"name": "Game2D",
"ref": "master",
@@ -95,9 +95,9 @@
"url": "https://github.com/curl/curl.git"
},
{
"commit": "556ee25f11f2e6b979150acb94e3093a137ed47e",
"commit": "98d63a194ee790deace033f77f66163a78122069",
"dependencies": {"raylib": "p5a77c9c35110e925"},
"id": "pe4492807d82c43f5",
"id": "pd71ea76b67bd47e2",
"name": "kryon",
"ref": "master",
"url": "https://github.com/kryonlabs/kryon.git"
@@ -121,7 +121,7 @@
}
],
"root": {
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pe4492807d82c43f5", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pd71ea76b67bd47e2", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
"name": "Inbe"
},
"toolchain": {