Sync Diary photos as one sealed copy each
A Diary photo was synced as 1 KB hex chunks inside hex ciphertext, so a node stored about 4.5 times the photo and the device kept a second hex copy. A photo's entry now syncs only its hash and size; after each good sync the photo is sealed with a key made for it alone and stored on the node as one encrypted copy (its size plus 16 bytes), and the account's other devices download and verify it. Photos from before upload the same way and their chunks shrink to one byte each. A node without photo upload or a full node only keeps the copy off that node; photos always save. The desktop moves copies on the sync worker, Android and the web one request per frame. Signed GET requests now sign their empty body. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
1083e90d35
commit
c4babc97dc
17 files changed
+1237
-34
No files matched your search
@@ -19,6 +19,7 @@
|
||||
- Top up Dew centers the Monero address under its QR code.
|
||||
- Profile shows how much of your storage is used, with a bar, next to your plan and in Storage plans.
|
||||
- Free storage is 20 MB. Profile shows the free storage your sync node gives instead of a fixed size.
|
||||
- Diary photos sync as one encrypted copy each, about the photo's own size on your sync node instead of about four and a half times, and are no longer stored twice on your device. Photos added before upload the same way and free their old space on the node.
|
||||
- Every practice is optional: Practice shows only installed ones. Wim Hof comes installed and can be removed; Meditation and Breathing Patterns come with the app and are added without a download.
|
||||
- Manage practices lists every practice with its size and one menu that says where it is: On Practice, On device or Not downloaded, each with what choosing it does. On device takes a practice off Practice but keeps it on your device, so adding it back needs no download; once a practice is downloaded, Uninstall asks once, then frees the space. A paid practice offers its unlock instead. Drag a practice by its handle onto another to change the order Practice shows them in. Your progress stays either way.
|
||||
- Lumi asks to download or unlock a practice before starting one that isn't on your device.
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
#import "async_client"
|
||||
#import "auth"
|
||||
#import "text_buffer"
|
||||
#import, file "sync_safety_policy.zi";
|
||||
#import "byte_text_linux"
|
||||
#import "url"
|
||||
@@ -32,8 +34,10 @@ GetTime :: () -> float64 #foreign c;
|
||||
#import, file "app_types.zi";
|
||||
#import, file "app.zi";
|
||||
#import, file "sync_safety.zi";
|
||||
#import, file "../cells/diary_blobs.zi";
|
||||
#if #defined(PLATFORM_WEB) == false && #defined(ANDROID_BUILD) == false {
|
||||
#import, file "../storage/sync_runtime.zi";
|
||||
#import, file "../cells/diary_blobs_native.zi";
|
||||
}
|
||||
#if #defined(PLATFORM_WEB) || #defined(ANDROID_BUILD) {
|
||||
#import, file "../storage/sync_async.zi";
|
||||
@@ -137,6 +141,16 @@ g_sync_events_retry_at: float64;
|
||||
#if ASYNC {
|
||||
g_sync_operation: StoredOperation;
|
||||
g_sync_transfer: NetworkTransfer;
|
||||
// Diary photo copies move one request at a time after a good sync.
|
||||
g_diary_blob: StoredBlob;
|
||||
g_diary_pass: DiaryBlobPass;
|
||||
g_diary_request: DiaryBlobRequest;
|
||||
g_diary_output: *u8;
|
||||
g_diary_output_bytes: s64;
|
||||
g_diary_pass_active: bool;
|
||||
g_diary_pass_due: bool;
|
||||
g_diary_pass_url: [512]u8;
|
||||
g_diary_pass_account: [72]u8;
|
||||
g_social_async: SocialRefresh;
|
||||
g_social_transfer: NetworkTransfer;
|
||||
g_social_streak_key: [96]u8;
|
||||
@@ -429,6 +443,9 @@ app_social_apply_result :: (app: *InnerBreeze, requests_json: string,
|
||||
if result == cast(SyncResult)SYNC_OK && storage_last_sync_changed() != 0 {
|
||||
changed = 1
|
||||
}
|
||||
if result == cast(SyncResult)SYNC_OK {
|
||||
DiaryBlobsSendNative(AppTextFromCString(*args.url[0], 256))
|
||||
}
|
||||
SyncWorkerFree(cast(*void)args)
|
||||
}
|
||||
|
||||
@@ -534,6 +551,80 @@ app_social_apply_result :: (app: *InnerBreeze, requests_json: string,
|
||||
}
|
||||
}
|
||||
|
||||
#if ASYNC {
|
||||
app_diary_blobs_end :: () {
|
||||
if g_diary_blob.active {
|
||||
CancelStoredBlob(*g_diary_blob)
|
||||
}
|
||||
DiaryBlobAbandon(*g_diary_request)
|
||||
if g_diary_output != null {
|
||||
SyncWorkerFree(cast(*void)g_diary_output)
|
||||
g_diary_output = null
|
||||
}
|
||||
g_diary_output_bytes = 0
|
||||
g_diary_pass_active = false
|
||||
}
|
||||
|
||||
// Moves Diary photo copies one request at a time, each frame, after a
|
||||
// good sync (cells/diary_blobs.zi). A switched account ends the pass.
|
||||
app_diary_blobs_tick :: () {
|
||||
if g_diary_pass_active &&
|
||||
storage_get_setting_text("sync_public_id") != AppTextFromCString(*g_diary_pass_account[0], 72) {
|
||||
app_diary_blobs_end()
|
||||
return
|
||||
}
|
||||
if g_diary_blob.active {
|
||||
if !PollStoredBlob(*g_diary_blob) {
|
||||
return
|
||||
}
|
||||
keep := false
|
||||
length := g_diary_blob.length
|
||||
if g_diary_blob.result == cast(s32)AuthResult.AUTH_OK && length >= 0 && length < g_diary_output_bytes {
|
||||
keep = DiaryBlobFinish(*g_diary_pass, *g_diary_request, g_diary_blob.status,
|
||||
BytesFromPointer(g_diary_output, cast(u64)length))
|
||||
}
|
||||
DiaryBlobAbandon(*g_diary_request)
|
||||
SyncWorkerFree(cast(*void)g_diary_output)
|
||||
g_diary_output = null
|
||||
g_diary_output_bytes = 0
|
||||
if !keep {
|
||||
app_diary_blobs_end()
|
||||
return
|
||||
}
|
||||
}
|
||||
if !g_diary_pass_active {
|
||||
if !g_diary_pass_due || g_sync.sync_running != 0 {
|
||||
return
|
||||
}
|
||||
g_diary_pass_due = false
|
||||
if !DiaryBlobPlan(*g_diary_pass, AppTextFromCString(*g_diary_pass_url[0], 512)) {
|
||||
return
|
||||
}
|
||||
unused AppCopyText(storage_get_setting_text("sync_public_id"), g_diary_pass_account[:])
|
||||
g_diary_pass_active = true
|
||||
}
|
||||
if !DiaryBlobNext(*g_diary_pass, *g_diary_request) {
|
||||
app_diary_blobs_end()
|
||||
return
|
||||
}
|
||||
capacity: s64 = 4096
|
||||
if g_diary_request.answer_bytes > capacity {
|
||||
capacity = g_diary_request.answer_bytes
|
||||
}
|
||||
g_diary_output = cast(*u8)SyncWorkerAlloc(cast(usize)capacity)
|
||||
if g_diary_output == null {
|
||||
app_diary_blobs_end()
|
||||
return
|
||||
}
|
||||
g_diary_output_bytes = capacity
|
||||
if !BeginStoredBlob(*g_diary_blob, AppTextFromCString(*g_diary_pass_url[0], 512),
|
||||
g_diary_request.method, TextUntilNul(g_diary_request.path[:]), g_diary_request.body,
|
||||
BytesFromPointer(g_diary_output, cast(u64)capacity)) {
|
||||
app_diary_blobs_end()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
app_collect_finished_sync :: () {
|
||||
result: s32 = 0
|
||||
changed: s32 = 0
|
||||
@@ -549,12 +640,17 @@ app_collect_finished_sync :: () {
|
||||
}
|
||||
}
|
||||
#if ASYNC {
|
||||
app_diary_blobs_tick()
|
||||
if g_sync.sync_running == 0 || !PollStoredOperation(*g_sync_operation) {
|
||||
return
|
||||
}
|
||||
result = g_sync_operation.result
|
||||
changed = g_sync_operation.changed
|
||||
g_sync.sync_running = 0
|
||||
if result == SYNC_OK {
|
||||
unused AppCopyText(AppTextFromCString(*g_sync_operation.server[0], 512), g_diary_pass_url[:])
|
||||
g_diary_pass_due = true
|
||||
}
|
||||
}
|
||||
app_sync_record_result(result)
|
||||
if result == SYNC_OK {
|
||||
|
||||
@@ -0,0 +1,385 @@
|
||||
// Diary photos and videos travel between a device and its node as sealed
|
||||
// copies (diary_records.zi), one request per copy, after each good sync:
|
||||
//
|
||||
// - A photo this device has whose entry names no copy yet is sealed and
|
||||
// uploaded; its entry then names the copy, so the account's other devices
|
||||
// can download it.
|
||||
// - A photo whose copy is named but whose file is not on this device is
|
||||
// downloaded, opened and saved as its file. A copy not uploaded yet (404)
|
||||
// is tried again after a later sync.
|
||||
// - A copy this device has that the node does not, for example after
|
||||
// switching nodes, is uploaded again.
|
||||
// - A photo from before copies, kept as hex chunks, is uploaded the same
|
||||
// way; its chunks then shrink to one byte each, freeing their room on the
|
||||
// node.
|
||||
//
|
||||
// Nothing here stops a photo from saving: a node without photo upload, or
|
||||
// one that is full, only keeps its copy off that node. The host sends each
|
||||
// request (signed for the account and device) and passes the answer back.
|
||||
#import "byte_text_linux"
|
||||
#import "vec"
|
||||
#import "text"
|
||||
#import "json_scan"
|
||||
#import "text_buffer"
|
||||
#import "keys"
|
||||
#import "crypto"
|
||||
#import, file "diary_records.zi";
|
||||
#import, file "diary_store.zi";
|
||||
#import, file "package_files.zi";
|
||||
#import, file "../storage/cell_records.zi";
|
||||
#import, file "../storage/storage_core.zi";
|
||||
#import, file "../storage/sqlite.zi";
|
||||
#import, file "../storage/db.zi";
|
||||
#import, file "../storage/state.zi";
|
||||
#import, file "../storage/paths.zi";
|
||||
#import, file "../storage/sync_nodes.zi";
|
||||
#import, file "../storage/sync_node_storage.zi";
|
||||
#import, file "../app/text_buffers.zi";
|
||||
|
||||
// One pass after a sync moves at most this many files and bytes, so a sync
|
||||
// is never held up for long; the next sync continues.
|
||||
DIARY_BLOB_PASS_FILES :: 8
|
||||
DIARY_BLOB_PASS_BYTES :: 33554432
|
||||
|
||||
DIARY_BLOB_UPLOAD :: 1
|
||||
DIARY_BLOB_DOWNLOAD :: 2
|
||||
|
||||
DiaryBlobJob :: struct {
|
||||
kind: s32
|
||||
name: [201]u8
|
||||
}
|
||||
|
||||
DiaryBlobPass :: struct {
|
||||
// The node's normalized address.
|
||||
node: [256]u8
|
||||
jobs: [DIARY_BLOB_PASS_FILES]DiaryBlobJob
|
||||
count: s32
|
||||
next: s32
|
||||
// The node refused an upload for room; downloads go on.
|
||||
uploads_stopped: bool
|
||||
}
|
||||
|
||||
// One request: PUT stores body as the copy at path; GET answers with the
|
||||
// copy, which needs answer_bytes of room.
|
||||
DiaryBlobRequest :: struct {
|
||||
kind: s32
|
||||
name: [201]u8
|
||||
method: string
|
||||
path: [128]u8
|
||||
body: string
|
||||
answer_bytes: s64
|
||||
blob: [65]u8
|
||||
key: [65]u8
|
||||
// An entry from before copies: its hash, size and chunk count.
|
||||
chunked: bool
|
||||
hash: [65]u8
|
||||
size: s64
|
||||
chunks: s32
|
||||
}
|
||||
|
||||
DiaryBlobEntriesSQL :: #string SQL_END
|
||||
SELECT substr(key,17),value FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
|
||||
AND key NOT GLOB '*.json' AND json_valid(value)
|
||||
ORDER BY updated_at DESC
|
||||
SQL_END;
|
||||
|
||||
DiaryBlobOnNode :: (node: string, blob: string) -> bool {
|
||||
statement: *void
|
||||
if SqlitePrepareText(storage_db_handle(),
|
||||
"SELECT 1 FROM diary_blobs WHERE node=?1 AND blob=?2", -1,
|
||||
*statement, null) != SQLITE_OK {
|
||||
return false
|
||||
}
|
||||
defer { SqliteFinalize(statement) }
|
||||
unused bind_text(statement, 1, node)
|
||||
unused bind_text(statement, 2, blob)
|
||||
return SqliteStep(statement) == SQLITE_ROW
|
||||
}
|
||||
|
||||
DiaryBlobRemember :: (node: string, blob: string) {
|
||||
statement: *void
|
||||
if SqlitePrepareText(storage_db_handle(),
|
||||
"INSERT OR IGNORE INTO diary_blobs(node,blob) VALUES(?1,?2)", -1,
|
||||
*statement, null) != SQLITE_OK {
|
||||
return
|
||||
}
|
||||
unused bind_text(statement, 1, node)
|
||||
unused bind_text(statement, 2, blob)
|
||||
unused SqliteStep(statement)
|
||||
SqliteFinalize(statement)
|
||||
}
|
||||
|
||||
// An entry from before copies: a photo kept as chunks.
|
||||
DiaryBlobChunked :: (entry: string, request: *DiaryBlobRequest) -> bool {
|
||||
count := ReadNumber(entry, FieldAt(entry, 0, "count"))
|
||||
size := ReadNumber(entry, FieldAt(entry, 0, "size"))
|
||||
if !count.valid || !size.valid || size.value < 1 || size.value > cast(float64)DIARY_FILE_LIMIT ||
|
||||
count.value != cast(float64)((cast(s64)size.value + DIARY_CHUNK_BYTES - 1) / DIARY_CHUNK_BYTES) ||
|
||||
ReadStringText(entry, FieldAt(entry, 0, "hash"), request.hash[:]) != 64 ||
|
||||
!DiaryHex64(TextFromBytes(request.hash[:64])) {
|
||||
return false
|
||||
}
|
||||
request.chunked = true
|
||||
request.size = cast(s64)size.value
|
||||
request.chunks = cast(s32)count.value
|
||||
return true
|
||||
}
|
||||
|
||||
DiaryBlobAdd :: (pass: *DiaryBlobPass, kind: s32, name: string) {
|
||||
if pass.count >= DIARY_BLOB_PASS_FILES {
|
||||
return
|
||||
}
|
||||
pass.jobs[pass.count].kind = kind
|
||||
unused AppCopyText(name, pass.jobs[pass.count].name[:])
|
||||
pass.count += 1
|
||||
}
|
||||
|
||||
// Chooses this pass's files: downloads first, so a photo shows on this
|
||||
// device soon, then uploads while the node takes photos and the account can
|
||||
// seal them.
|
||||
DiaryBlobPlan :: (pass: *DiaryBlobPass, node: string) -> bool {
|
||||
pass.* = DiaryBlobPass.{}
|
||||
if !SyncNodeAddress(node, pass.node[:]) || storage_db_handle() == null {
|
||||
return false
|
||||
}
|
||||
address := TextUntilNul(pass.node[:])
|
||||
key: [32]u8
|
||||
uploads := SyncNodeMediaUpload(address) &&
|
||||
storage_private_record_key_for_context(DIARY_BLOB_CONTEXT, *key[0]) != 0
|
||||
SecureWipe(*key[0], 32)
|
||||
bytes: s64 = 0
|
||||
for round: 0..1 {
|
||||
statement: *void
|
||||
if SqlitePrepareText(storage_db_handle(), DiaryBlobEntriesSQL, -1,
|
||||
*statement, null) != SQLITE_OK {
|
||||
return false
|
||||
}
|
||||
while pass.count < DIARY_BLOB_PASS_FILES && bytes < DIARY_BLOB_PASS_BYTES &&
|
||||
SqliteStep(statement) == SQLITE_ROW {
|
||||
name := AppTextFromCString(SqliteColumnText(statement, 0), 201)
|
||||
entry := AppTextFromCString(SqliteColumnText(statement, 1), 65536)
|
||||
if !DiaryStoreNameValid(name) {
|
||||
continue
|
||||
}
|
||||
path := DiaryStorePath(name)
|
||||
local := PackageFileSize(path)
|
||||
PackageFree(path)
|
||||
media := DiaryMediaParse(entry)
|
||||
if round == 0 {
|
||||
if DiaryMediaUploaded(media) && local != media.size {
|
||||
DiaryBlobAdd(pass, DIARY_BLOB_DOWNLOAD, name)
|
||||
bytes += media.size
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !uploads {
|
||||
continue
|
||||
}
|
||||
if media.valid {
|
||||
if local == media.size && (!DiaryMediaUploaded(media) ||
|
||||
!DiaryBlobOnNode(address, TextFromBytes(media.blob[:64]))) {
|
||||
DiaryBlobAdd(pass, DIARY_BLOB_UPLOAD, name)
|
||||
bytes += media.size
|
||||
}
|
||||
continue
|
||||
}
|
||||
old: DiaryBlobRequest
|
||||
if DiaryBlobChunked(entry, *old) {
|
||||
DiaryBlobAdd(pass, DIARY_BLOB_UPLOAD, name)
|
||||
bytes += old.size
|
||||
}
|
||||
}
|
||||
SqliteFinalize(statement)
|
||||
}
|
||||
return pass.count > 0
|
||||
}
|
||||
|
||||
// The bytes of a photo this device has, or "": its file, or for a photo
|
||||
// from before copies, its chunks when the file is gone.
|
||||
DiaryBlobLocalBytes :: (name: string, hash: string, size: s64, chunked: bool) -> string {
|
||||
path := DiaryStorePath(name)
|
||||
data := PackageRead(path, DIARY_FILE_LIMIT)
|
||||
PackageFree(path)
|
||||
if data.count == size {
|
||||
descriptor := cast(*TextDescriptor)(*data)
|
||||
actual: [65]u8
|
||||
storage_sha256_hex(descriptor.data, data.count, *actual[0])
|
||||
if TextFromCString(*actual[0]) == hash {
|
||||
return data
|
||||
}
|
||||
}
|
||||
PackageFree(data)
|
||||
if chunked {
|
||||
return DiaryRecordRead(name)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
DiaryBlobPath :: (request: *DiaryBlobRequest) {
|
||||
out: Vec(u8)
|
||||
BuilderPrint(*out, "/api/v1/blobs/inbe/%", TextFromBytes(request.blob[:64]))
|
||||
path := BuilderFinish(out)
|
||||
unused AppCopyText(path, request.path[:])
|
||||
PackageFree(path)
|
||||
}
|
||||
|
||||
// Prepares the pass's next request, or false when the pass is done.
|
||||
DiaryBlobNext :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest) -> bool {
|
||||
while pass.next < pass.count {
|
||||
job := pass.jobs[pass.next]
|
||||
pass.next += 1
|
||||
request.* = DiaryBlobRequest.{}
|
||||
request.kind = job.kind
|
||||
request.name = job.name
|
||||
name := TextUntilNul(job.name[:])
|
||||
key := DiaryRecordKey(name)
|
||||
entry := CellRecordRead(key)
|
||||
PackageFree(key)
|
||||
media := DiaryMediaParse(entry)
|
||||
if job.kind == DIARY_BLOB_DOWNLOAD {
|
||||
PackageFree(entry)
|
||||
if !DiaryMediaUploaded(media) {
|
||||
continue
|
||||
}
|
||||
request.method = "GET"
|
||||
request.blob = media.blob
|
||||
request.key = media.key
|
||||
request.hash = media.hash
|
||||
request.size = media.size
|
||||
request.answer_bytes = media.size + DIARY_BLOB_TAG + 1
|
||||
DiaryBlobPath(request)
|
||||
return true
|
||||
}
|
||||
if pass.uploads_stopped {
|
||||
PackageFree(entry)
|
||||
continue
|
||||
}
|
||||
if media.valid {
|
||||
request.hash = media.hash
|
||||
request.size = media.size
|
||||
} else if !DiaryBlobChunked(entry, request) {
|
||||
PackageFree(entry)
|
||||
continue
|
||||
}
|
||||
PackageFree(entry)
|
||||
hash := TextFromBytes(request.hash[:64])
|
||||
data := DiaryBlobLocalBytes(name, hash, request.size, request.chunked)
|
||||
if data.count == 0 {
|
||||
continue
|
||||
}
|
||||
file_key: [32]u8
|
||||
sealed: string = ""
|
||||
if DiaryMediaUploaded(media) {
|
||||
if HexToBytes(TextFromBytes(media.key[:64]), *file_key[0], 32) != 0 {
|
||||
sealed = DiaryBlobSeal(data, *file_key[0])
|
||||
}
|
||||
} else if DiaryBlobKey(hash, *file_key[0]) {
|
||||
sealed = DiaryBlobSeal(data, *file_key[0])
|
||||
}
|
||||
PackageFree(data)
|
||||
if sealed.count == 0 {
|
||||
SecureWipe(*file_key[0], 32)
|
||||
continue
|
||||
}
|
||||
descriptor := cast(*TextDescriptor)(*sealed)
|
||||
storage_sha256_hex(descriptor.data, sealed.count, *request.blob[0])
|
||||
unused BytesToHex(*file_key[0], 32, *request.key[0], 65)
|
||||
SecureWipe(*file_key[0], 32)
|
||||
// A named copy is re-made only to reach another node; it must be
|
||||
// the same copy.
|
||||
if DiaryMediaUploaded(media) && TextFromBytes(request.blob[:64]) != TextFromBytes(media.blob[:64]) {
|
||||
PackageFree(sealed)
|
||||
continue
|
||||
}
|
||||
request.method = "PUT"
|
||||
request.body = sealed
|
||||
DiaryBlobPath(request)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A photo from before copies frees its chunks once its copy is on a node:
|
||||
// each becomes one byte, so its room on the node shrinks with it.
|
||||
DiaryBlobShrinkChunks :: (request: *DiaryBlobRequest) {
|
||||
storage_settings_begin_write()
|
||||
defer { storage_settings_end_write() }
|
||||
for index: cast(s32)0..request.chunks - 1 {
|
||||
chunk := DiaryChunkKey(TextFromBytes(request.hash[:64]), index)
|
||||
unused CellRecordWrite(chunk, "-")
|
||||
PackageFree(chunk)
|
||||
}
|
||||
}
|
||||
|
||||
// Applies one answer. False stops the pass: the node or the account cannot
|
||||
// take more now.
|
||||
DiaryBlobFinish :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest, status: s32,
|
||||
answer: []u8) -> bool {
|
||||
name := TextUntilNul(request.name[:])
|
||||
node := TextUntilNul(pass.node[:])
|
||||
blob := TextFromBytes(request.blob[:64])
|
||||
if request.kind == DIARY_BLOB_UPLOAD {
|
||||
PackageFree(request.body)
|
||||
request.body = ""
|
||||
if status == 413 {
|
||||
pass.uploads_stopped = true
|
||||
unused SyncNodeStorageRecordFull(node, TextFromBytes(answer))
|
||||
return true
|
||||
}
|
||||
if status != 204 && status != 200 {
|
||||
return false
|
||||
}
|
||||
DiaryBlobRemember(node, blob)
|
||||
media := DiaryMediaRead(name)
|
||||
if DiaryMediaUploaded(media) {
|
||||
return true
|
||||
}
|
||||
// The entry still describes this file; it now names the copy.
|
||||
if (media.valid && TextFromBytes(media.hash[:64]) == TextFromBytes(request.hash[:64])) ||
|
||||
request.chunked {
|
||||
described: DiaryMedia
|
||||
described.valid = true
|
||||
described.hash = request.hash
|
||||
described.size = request.size
|
||||
entry := DiaryMediaUploadedEntry(described, blob, TextFromBytes(request.key[:64]))
|
||||
key := DiaryRecordKey(name)
|
||||
written := CellRecordWrite(key, entry)
|
||||
PackageFree(key)
|
||||
PackageFree(entry)
|
||||
if written && request.chunked {
|
||||
DiaryBlobShrinkChunks(request)
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
if status == 404 {
|
||||
return true
|
||||
}
|
||||
if status != 200 {
|
||||
return false
|
||||
}
|
||||
media := DiaryMediaRead(name)
|
||||
if !DiaryMediaUploaded(media) || TextFromBytes(media.blob[:64]) != blob {
|
||||
return true
|
||||
}
|
||||
plain := DiaryBlobOpen(media, answer)
|
||||
if plain.count == 0 {
|
||||
return true
|
||||
}
|
||||
directory := DiaryStoreDirectory()
|
||||
path := DiaryStorePath(name)
|
||||
if storage_ensure_dir(directory) != 0 && PackageWrite(path, plain) {
|
||||
DiaryBlobRemember(node, blob)
|
||||
diary_blob_generation += 1
|
||||
}
|
||||
PackageFree(directory)
|
||||
PackageFree(path)
|
||||
PackageFree(plain)
|
||||
return true
|
||||
}
|
||||
|
||||
DiaryBlobAbandon :: (request: *DiaryBlobRequest) {
|
||||
PackageFree(request.body)
|
||||
request.body = ""
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// The desktop moves Diary photo copies (diary_blobs.zi) on its sync worker,
|
||||
// right after a good sync, one blocking request at a time.
|
||||
#if #defined(PLATFORM_WEB) == false && #defined(ANDROID_BUILD) == false {
|
||||
#import "auth"
|
||||
#import "byte_text_linux"
|
||||
#import "text_buffer"
|
||||
#import, file "diary_blobs.zi";
|
||||
#import, file "../storage/sync_runtime.zi";
|
||||
|
||||
diary_blobs_native_c :: #system_library "c";
|
||||
DiaryBlobsAllocate :: (size: usize) -> *void #foreign diary_blobs_native_c "malloc";
|
||||
DiaryBlobsRelease :: (pointer: *void) #foreign diary_blobs_native_c "free";
|
||||
|
||||
// A failed request ends the pass; the next sync tries again.
|
||||
DiaryBlobsSendNative :: (url: string) {
|
||||
pass: DiaryBlobPass
|
||||
if !DiaryBlobPlan(*pass, url) {
|
||||
return
|
||||
}
|
||||
request: DiaryBlobRequest
|
||||
while DiaryBlobNext(*pass, *request) {
|
||||
capacity: s64 = 4096
|
||||
if request.answer_bytes > capacity {
|
||||
capacity = request.answer_bytes
|
||||
}
|
||||
output := cast(*u8)DiaryBlobsAllocate(cast(usize)capacity)
|
||||
if output == null {
|
||||
DiaryBlobAbandon(*request)
|
||||
return
|
||||
}
|
||||
status: s32 = 0
|
||||
length: s64 = 0
|
||||
sent := SendStoredBlob(url, request.method, TextUntilNul(request.path[:]), request.body,
|
||||
BytesFromPointer(output, cast(u64)capacity), *status, *length)
|
||||
keep := false
|
||||
if sent == cast(s32)AuthResult.AUTH_OK && length >= 0 && length < capacity {
|
||||
keep = DiaryBlobFinish(*pass, *request, status, BytesFromPointer(output, cast(u64)length))
|
||||
}
|
||||
DiaryBlobAbandon(*request)
|
||||
DiaryBlobsRelease(cast(*void)output)
|
||||
if !keep {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+189
-3
@@ -1,9 +1,19 @@
|
||||
// Immutable photo/text chunks and a per-file manifest share encrypted sync.
|
||||
// A manifest becomes readable only when all chunks and its hash validate.
|
||||
// Each Diary file has an entry, cell.diary.file.NAME, that syncs.
|
||||
//
|
||||
// A day document (YYYY-MM-DD.json) syncs its bytes as hex chunks beside the
|
||||
// entry, which becomes readable only when all chunks and its hash validate.
|
||||
//
|
||||
// A photo or video stays a file on the device. Its entry syncs
|
||||
// {"hash","size"}; once its sealed copy is on a node it also names that copy,
|
||||
// "blob" (the copy's SHA-256, its address on the node) and "key". The copy is
|
||||
// the file sealed with a key made for that file alone, so it takes the
|
||||
// file's size plus 16 bytes. Entries with "count" are photos from before,
|
||||
// still kept as chunks until their copy is uploaded (diary_blobs.zi).
|
||||
#import "json_scan"
|
||||
#import "byte_text_linux"
|
||||
#import "vec"
|
||||
#import "text"
|
||||
#import "keys"
|
||||
#import, file "../storage/cell_records.zi";
|
||||
#import "crypto"
|
||||
#import, file "lumi_store.zi";
|
||||
@@ -11,6 +21,16 @@
|
||||
#import, file "../storage/storage_core.zi";
|
||||
|
||||
DIARY_CHUNK_BYTES :: 1024
|
||||
DIARY_FILE_LIMIT :: 16777216
|
||||
// Sealing adds a 16-byte tag; the nonce is fixed (see DiaryBlobSeal).
|
||||
DIARY_BLOB_TAG :: 16
|
||||
// The account's Diary copy key comes from the account key with this context.
|
||||
DIARY_BLOB_CONTEXT :: "inbe-diary-blob-key-v1"
|
||||
DIARY_BLOB_AAD :: "inbe.diary.blob.v1"
|
||||
|
||||
diary_records_c :: #system_library "c";
|
||||
DiaryAllocate :: (size: usize) -> *void #foreign diary_records_c "malloc";
|
||||
DiaryRelease :: (pointer: *void) #foreign diary_records_c "free";
|
||||
|
||||
DiaryRecordKey :: (name: string) -> string {
|
||||
return PackageText("cell.diary.file.", name)
|
||||
@@ -22,8 +42,13 @@ DiaryChunkKey :: (hash: string, index: s32) -> string {
|
||||
return BuilderFinish(out)
|
||||
}
|
||||
|
||||
// Day documents are YYYY-MM-DD.json; every other Diary file is media.
|
||||
DiaryRecordMedia :: (name: string) -> bool {
|
||||
return name.count < 5 || name[name.count - 5:] != ".json"
|
||||
}
|
||||
|
||||
DiaryRecordSave :: (name: string, data: string) -> bool {
|
||||
if data.count <= 0 || data.count > 16777216 {
|
||||
if data.count <= 0 || data.count > DIARY_FILE_LIMIT {
|
||||
return false
|
||||
}
|
||||
descriptor := cast(*TextDescriptor)(*data)
|
||||
@@ -40,6 +65,15 @@ DiaryRecordSave :: (name: string, data: string) -> bool {
|
||||
if length == 64 && TextFromBytes(old_hash[:length]) == hash {
|
||||
return true
|
||||
}
|
||||
if DiaryRecordMedia(name) {
|
||||
// The file holds the bytes; only the entry syncs until the sealed
|
||||
// copy is uploaded.
|
||||
entry: Vec(u8)
|
||||
BuilderPrint(*entry, "{\"hash\":\"%\",\"size\":%}", hash, data.count)
|
||||
compact := BuilderFinish(entry)
|
||||
defer { PackageFree(compact) }
|
||||
return CellRecordWrite(key, compact)
|
||||
}
|
||||
storage_settings_begin_write()
|
||||
defer { storage_settings_end_write() }
|
||||
for index: cast(s32)0..count - 1 {
|
||||
@@ -101,3 +135,155 @@ DiaryRecordRead :: (name: string) -> string {
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// A photo's entry without chunks, or not valid when the entry has chunks or
|
||||
// does not parse.
|
||||
DiaryMedia :: struct {
|
||||
valid: bool
|
||||
hash: [65]u8
|
||||
size: s64
|
||||
// Both empty until the sealed copy is on a node.
|
||||
blob: [65]u8
|
||||
key: [65]u8
|
||||
}
|
||||
|
||||
DiaryHex64 :: (text: string) -> bool {
|
||||
if text.count != 64 {
|
||||
return false
|
||||
}
|
||||
for byte: text {
|
||||
if !((byte >= 48 && byte <= 57) || (byte >= 97 && byte <= 102)) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
DiaryMediaParse :: (entry: string) -> DiaryMedia {
|
||||
result: DiaryMedia
|
||||
if entry.count == 0 || FieldAt(entry, 0, "count") >= 0 {
|
||||
return result
|
||||
}
|
||||
size := ReadNumber(entry, FieldAt(entry, 0, "size"))
|
||||
if ReadStringText(entry, FieldAt(entry, 0, "hash"), result.hash[:]) != 64 ||
|
||||
!DiaryHex64(TextFromBytes(result.hash[:64])) || !size.valid || size.value < 1 ||
|
||||
size.value > cast(float64)DIARY_FILE_LIMIT {
|
||||
return DiaryMedia.{}
|
||||
}
|
||||
result.size = cast(s64)size.value
|
||||
if FieldAt(entry, 0, "blob") >= 0 {
|
||||
if ReadStringText(entry, FieldAt(entry, 0, "blob"), result.blob[:]) != 64 ||
|
||||
ReadStringText(entry, FieldAt(entry, 0, "key"), result.key[:]) != 64 ||
|
||||
!DiaryHex64(TextFromBytes(result.blob[:64])) || !DiaryHex64(TextFromBytes(result.key[:64])) {
|
||||
return DiaryMedia.{}
|
||||
}
|
||||
}
|
||||
result.valid = true
|
||||
return result
|
||||
}
|
||||
|
||||
DiaryMediaRead :: (name: string) -> DiaryMedia {
|
||||
key := DiaryRecordKey(name)
|
||||
entry := CellRecordRead(key)
|
||||
defer { PackageFree(key); PackageFree(entry) }
|
||||
return DiaryMediaParse(entry)
|
||||
}
|
||||
|
||||
DiaryMediaUploaded :: (media: DiaryMedia) -> bool {
|
||||
return media.valid && media.blob[0] != cast(u8)0
|
||||
}
|
||||
|
||||
// Whether data is exactly the file an entry describes.
|
||||
DiaryMediaMatches :: (media: DiaryMedia, data: string) -> bool {
|
||||
if !media.valid || data.count != media.size {
|
||||
return false
|
||||
}
|
||||
descriptor := cast(*TextDescriptor)(*data)
|
||||
actual: [65]u8
|
||||
storage_sha256_hex(descriptor.data, data.count, *actual[0])
|
||||
return TextFromCString(*actual[0]) == TextFromBytes(media.hash[:64])
|
||||
}
|
||||
|
||||
// The key that seals one file comes from the account's Diary key and the
|
||||
// file's hash, so every device of the account makes the same copy.
|
||||
DiaryBlobKey :: (hash: string, output: *u8) -> bool {
|
||||
master: [32]u8
|
||||
if storage_private_record_key_for_context(DIARY_BLOB_CONTEXT, *master[0]) == 0 {
|
||||
return false
|
||||
}
|
||||
ok := HmacSha256Text(*master[0], 32, hash, output) != 0
|
||||
SecureWipe(*master[0], 32)
|
||||
return ok
|
||||
}
|
||||
|
||||
DiaryBlobAad :: (output: []u8) {
|
||||
text := DIARY_BLOB_AAD
|
||||
for index: cast(s64)0..text.count - 1 {
|
||||
output[index] = text[index]
|
||||
}
|
||||
}
|
||||
|
||||
// The file sealed for a node, or "" when it cannot be sealed. Its key seals
|
||||
// only this file, so the fixed zero nonce is never used twice with a key.
|
||||
DiaryBlobSeal :: (data: string, key: *u8) -> string {
|
||||
if data.count <= 0 || data.count > DIARY_FILE_LIMIT {
|
||||
return ""
|
||||
}
|
||||
count := data.count + DIARY_BLOB_TAG
|
||||
output := cast(*u8)DiaryAllocate(cast(usize)count)
|
||||
if output == null {
|
||||
return ""
|
||||
}
|
||||
nonce: [12]u8
|
||||
aad: [32]u8
|
||||
DiaryBlobAad(aad[:])
|
||||
descriptor := cast(*TextDescriptor)(*data)
|
||||
if ChaCha20Poly1305Seal(key, *nonce[0], descriptor.data, data.count, *aad[0],
|
||||
cast(s64)DIARY_BLOB_AAD.count, output, count) == 0 {
|
||||
DiaryRelease(cast(*void)output)
|
||||
return ""
|
||||
}
|
||||
return TextFromBytes(BytesFromPointer(output, cast(u64)count))
|
||||
}
|
||||
|
||||
// The file inside a sealed copy, or "" when the copy is not the one its
|
||||
// entry names.
|
||||
DiaryBlobOpen :: (media: DiaryMedia, sealed: []u8) -> string {
|
||||
if !DiaryMediaUploaded(media) || cast(s64)sealed.count != media.size + DIARY_BLOB_TAG {
|
||||
return ""
|
||||
}
|
||||
actual: [65]u8
|
||||
storage_sha256_hex(*sealed[0], cast(s64)sealed.count, *actual[0])
|
||||
if TextFromCString(*actual[0]) != TextFromBytes(media.blob[:64]) {
|
||||
return ""
|
||||
}
|
||||
key: [32]u8
|
||||
if HexToBytes(TextFromBytes(media.key[:64]), *key[0], 32) == 0 {
|
||||
return ""
|
||||
}
|
||||
output := cast(*u8)DiaryAllocate(cast(usize)media.size)
|
||||
if output == null {
|
||||
SecureWipe(*key[0], 32)
|
||||
return ""
|
||||
}
|
||||
nonce: [12]u8
|
||||
aad: [32]u8
|
||||
DiaryBlobAad(aad[:])
|
||||
opened := ChaCha20Poly1305Open(*key[0], *nonce[0], *sealed[0], cast(s64)sealed.count,
|
||||
*aad[0], cast(s64)DIARY_BLOB_AAD.count, output, media.size) != 0
|
||||
SecureWipe(*key[0], 32)
|
||||
value := TextFromBytes(BytesFromPointer(output, cast(u64)media.size))
|
||||
if !opened || !DiaryMediaMatches(media, value) {
|
||||
DiaryRelease(cast(*void)output)
|
||||
return ""
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
// The entry once the sealed copy is on a node.
|
||||
DiaryMediaUploadedEntry :: (media: DiaryMedia, blob: string, key: string) -> string {
|
||||
out: Vec(u8)
|
||||
BuilderPrint(*out, "{\"hash\":\"%\",\"size\":%,\"blob\":\"%\",\"key\":\"%\"}",
|
||||
TextFromBytes(media.hash[:64]), media.size, blob, key)
|
||||
return BuilderFinish(out)
|
||||
}
|
||||
@@ -38,6 +38,10 @@ diary_prepared: bool;
|
||||
diary_account: string;
|
||||
diary_preview_name: string;
|
||||
diary_preview_bytes: string;
|
||||
// Bumped when a photo's file arrives from a node, so a placeholder shown for
|
||||
// it is read again.
|
||||
diary_blob_generation: s64;
|
||||
diary_preview_generation: s64;
|
||||
#if !#defined(_WIN32) && !#defined(PLATFORM_WEB) {
|
||||
DiaryLock :: (descriptor: s32, operation: s32) -> s32 #foreign c "flock";
|
||||
}
|
||||
@@ -189,6 +193,24 @@ DiaryStoreRead :: (name: string) -> DiaryFileResult {
|
||||
}
|
||||
result.missing = !PathExists(path)
|
||||
PackageFree(diary_file_bytes)
|
||||
diary_file_bytes = ""
|
||||
media := DiaryMediaRead(name)
|
||||
if media.valid {
|
||||
// A photo is its file. Until the file arrives from a node, it shows
|
||||
// as not on this device.
|
||||
diary_file_bytes = PackageRead(path, DIARY_FILE_LIMIT)
|
||||
if !DiaryMediaMatches(media, diary_file_bytes) {
|
||||
PackageFree(diary_file_bytes)
|
||||
diary_file_bytes = ""
|
||||
result.missing = false
|
||||
result.succeeded = false
|
||||
return result
|
||||
}
|
||||
result.missing = false
|
||||
result.data = diary_file_bytes
|
||||
result.succeeded = true
|
||||
return result
|
||||
}
|
||||
diary_file_bytes = DiaryRecordRead(name)
|
||||
if diary_file_bytes.count > 0 {
|
||||
result.missing = false
|
||||
@@ -382,14 +404,24 @@ DiaryStorePreview :: (name: string) -> string {
|
||||
if !DiaryStoreNameValid(name) || !DiaryStorePrepare() {
|
||||
return ""
|
||||
}
|
||||
if name != diary_preview_name {
|
||||
if name != diary_preview_name || diary_preview_generation != diary_blob_generation {
|
||||
PackageFree(diary_preview_name)
|
||||
PackageFree(diary_preview_bytes)
|
||||
diary_preview_name = PackageText(name, "")
|
||||
diary_preview_generation = diary_blob_generation
|
||||
path := DiaryStorePath(name)
|
||||
defer {
|
||||
PackageFree(path)
|
||||
}
|
||||
media := DiaryMediaRead(name)
|
||||
if media.valid {
|
||||
diary_preview_bytes = PackageRead(path, DIARY_FILE_LIMIT)
|
||||
if !DiaryMediaMatches(media, diary_preview_bytes) {
|
||||
PackageFree(diary_preview_bytes)
|
||||
diary_preview_bytes = ""
|
||||
}
|
||||
return diary_preview_bytes
|
||||
}
|
||||
diary_preview_bytes = DiaryRecordRead(name)
|
||||
record_key := DiaryRecordKey(name)
|
||||
manifest := CellRecordRead(record_key)
|
||||
|
||||
@@ -29,9 +29,9 @@ AccountFieldText :: (data: *u8, capacity: s64) -> string {
|
||||
return TextFromBytes(BytesFromPointer(data, cast(u64)length))
|
||||
}
|
||||
|
||||
// A signed GET has an empty body, whose SHA-256 is still signed.
|
||||
DigestBody :: (body: string, output: []u8) -> bool {
|
||||
if body.count == 0 || body.count > 9223372036854775807 ||
|
||||
output.count < 65 {
|
||||
if body.count < 0 || output.count < 65 {
|
||||
return false
|
||||
}
|
||||
descriptor: *TextDescriptor = cast(*TextDescriptor)(*body)
|
||||
|
||||
@@ -235,6 +235,9 @@ migrate_schema :: () -> s32 {
|
||||
if !SchemaExec(SchemaSyncRecordCopies) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
if !SchemaExec(SchemaDiaryBlobs) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
if SchemaHasColumn("habit_days", "answers") && !SchemaExec(SchemaDayAnswersToEntries) {
|
||||
return SchemaUpgradeFailed()
|
||||
}
|
||||
|
||||
@@ -278,6 +278,15 @@ CREATE TABLE IF NOT EXISTS sync_record_copies( collection TEXT NOT NULL,
|
||||
PRIMARY KEY(collection,id))
|
||||
SQL
|
||||
|
||||
// The sealed Diary photo copies (diary_blobs.zi) each node is known to
|
||||
// hold, by the node's normalized address, so a copy uploads once per node.
|
||||
// Only this device keeps it; a lost row means one more upload.
|
||||
SchemaDiaryBlobs :: #string SQL
|
||||
CREATE TABLE IF NOT EXISTS diary_blobs( node TEXT NOT NULL,
|
||||
blob TEXT NOT NULL,
|
||||
PRIMARY KEY(node,blob))
|
||||
SQL
|
||||
|
||||
// A habit's entries: each a moment of a day, or a span from its start to
|
||||
// its end (0 while it runs), with its answers to the habit's questions as
|
||||
// a JSON array. An entry belongs to the day it started.
|
||||
|
||||
@@ -12,6 +12,7 @@ Auth :: #import "auth";
|
||||
#import, file "delegate_storage_types.zi";
|
||||
#import, file "delegate_storage.zi";
|
||||
#import "net_http"
|
||||
#import "http_transfer"
|
||||
#import "url"
|
||||
#import "byte_text_linux"
|
||||
Clock :: #import "date_time_linux";
|
||||
@@ -400,3 +401,100 @@ PollStoredOperation :: (operation: *StoredOperation) -> bool {
|
||||
ReleaseStoredOperation(operation)
|
||||
return true
|
||||
}
|
||||
|
||||
// One signed request for a sealed Diary copy (cells/diary_blobs.zi),
|
||||
// advanced each frame like a sync: PUT stores the body at path, GET answers
|
||||
// with the copy in output. It uses the sync's saved session; without one it
|
||||
// does not start, and the next sync signs in again.
|
||||
StoredBlob :: struct {
|
||||
active: bool
|
||||
transfer: HttpTransfer
|
||||
transaction: [8192]u8
|
||||
token: [4096]u8
|
||||
url: [1024]u8
|
||||
method: [8]u8
|
||||
headers: [1]HttpHeader
|
||||
status: s32
|
||||
length: s64
|
||||
result: s32
|
||||
}
|
||||
|
||||
// The body and output must stay valid until PollStoredBlob finishes or
|
||||
// CancelStoredBlob returns.
|
||||
BeginStoredBlob :: (blob: *StoredBlob, server: string, method: string, path: string,
|
||||
body: string, output: []u8) -> bool {
|
||||
if blob == null || blob.active || StoredAccountMode() != AccountModeOwner {
|
||||
return false
|
||||
}
|
||||
blob.* = StoredBlob.{}
|
||||
blob.result = cast(s32)AuthResult.AUTH_NO_ACCOUNT
|
||||
account: AccountKeys
|
||||
if sync_account_load(*account) == 0 {
|
||||
return false
|
||||
}
|
||||
defer { SecureWipe(*account.private_key_hex[0], ACCOUNT_PRIVATE_KEY_HEX_SIZE) }
|
||||
material: DeviceKeyMaterial
|
||||
if !LoadOrCreateDeviceKey(*material) {
|
||||
return false
|
||||
}
|
||||
defer { WipeDeviceKey(*material) }
|
||||
client: Client
|
||||
client_id: [128]u8
|
||||
normalized: [512]u8
|
||||
if !AppCopyText(storage_sync_client_id(), client_id[:]) ||
|
||||
!NormalizeUrl(server, normalized[:]) ||
|
||||
!OpenAccountClient(TextUntilNul(normalized[:]), *account, TextUntilNul(client_id[:]), *client) {
|
||||
return false
|
||||
}
|
||||
now := Clock.UnixNow()
|
||||
session: Session
|
||||
LoadSavedSession(*session, now)
|
||||
if session.token[0] == cast(u8)0 {
|
||||
return false
|
||||
}
|
||||
blob.result = cast(s32)AuthResult.AUTH_SIGN_FAILED
|
||||
if PrepareRequestHeader(client, *session, DeviceFromKey(*material), now, null, NativeNonce,
|
||||
method, path, body, blob.transaction[:]) != cast(AuthResult)AuthResult.AUTH_OK ||
|
||||
!JoinUrl(TextUntilNul(normalized[:]), path, blob.url[:]) ||
|
||||
!AppCopyText(TextUntilNul(session.token[:]), blob.token[:]) ||
|
||||
!AppCopyText(method, blob.method[:]) {
|
||||
return false
|
||||
}
|
||||
blob.headers[0] = HttpHeader.{.name = "X-Daochi-Tx", .value = TextUntilNul(blob.transaction[:])}
|
||||
request := HttpRequest.{.method = TextUntilNul(blob.method[:]), .url = TextUntilNul(blob.url[:]),
|
||||
.token = TextUntilNul(blob.token[:]), .body = body, .headers = blob.headers[:]}
|
||||
if body.count > 0 {
|
||||
request.content_type = "application/octet-stream"
|
||||
}
|
||||
blob.result = cast(s32)AuthResult.AUTH_REQUEST_FAILED
|
||||
// A copy of up to 16 MB needs longer than a sync's request.
|
||||
if !HttpTransferStart(*blob.transfer, request, output, 180000) {
|
||||
return false
|
||||
}
|
||||
blob.active = true
|
||||
return true
|
||||
}
|
||||
|
||||
// True once the request has finished; status and length then hold its
|
||||
// answer, and result is AUTH_OK when one arrived.
|
||||
PollStoredBlob :: (blob: *StoredBlob) -> bool {
|
||||
if blob == null || !blob.active {
|
||||
return true
|
||||
}
|
||||
state := HttpTransferPollLength(*blob.transfer, *blob.status, *blob.length)
|
||||
if state < 0 {
|
||||
return false
|
||||
}
|
||||
blob.active = false
|
||||
blob.result = ifx (state == 1) then (cast(s32)AuthResult.AUTH_OK) else (cast(s32)AuthResult.AUTH_REQUEST_FAILED)
|
||||
return true
|
||||
}
|
||||
|
||||
CancelStoredBlob :: (blob: *StoredBlob) {
|
||||
if blob == null || !blob.active {
|
||||
return
|
||||
}
|
||||
HttpTransferCancel(*blob.transfer)
|
||||
blob.active = false
|
||||
blob.result = cast(s32)AuthResult.AUTH_REQUEST_FAILED
|
||||
}
|
||||
@@ -2,7 +2,8 @@
|
||||
// any. This device keeps the last report per account and node, and whether
|
||||
// Diary photos and videos upload to that node. A Diary file entry always
|
||||
// syncs, so other devices see that a photo exists even where its bytes are
|
||||
// withheld; only the photo's chunks stay off the node.
|
||||
// withheld; only the photo's sealed copy (cells/diary_blobs.zi), or for a
|
||||
// photo from before copies its chunks, stays off the node.
|
||||
#import "json_scan"
|
||||
#import "byte_text_linux"
|
||||
#import "text"
|
||||
@@ -48,13 +49,18 @@ WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
|
||||
ON CONFLICT(entity_type,entity_id,local_date) DO UPDATE SET queued_at=excluded.queued_at
|
||||
SQL_END;
|
||||
|
||||
// A sealed copy is its file plus a 16-byte tag; a photo from before copies
|
||||
// still costs its chunks.
|
||||
SyncMediaLocalBytesSQL :: #string SQL_END
|
||||
SELECT COALESCE(SUM(125+LENGTH(key)+2*(LENGTH(key)+LENGTH(value)+73)),0) FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
|
||||
AND substr(key,18,64) IN (
|
||||
SELECT json_extract(value,'$.hash') FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
|
||||
AND key NOT GLOB '*.json' AND json_valid(value))
|
||||
SELECT COALESCE((SELECT SUM(json_extract(value,'$.size')+16) FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
|
||||
AND key NOT GLOB '*.json' AND json_valid(value) AND json_type(value,'$.count') IS NULL),0)
|
||||
+COALESCE((SELECT SUM(125+LENGTH(key)+2*(LENGTH(key)+LENGTH(value)+73)) FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.chunk.*'
|
||||
AND substr(key,18,64) IN (
|
||||
SELECT json_extract(value,'$.hash') FROM settings
|
||||
WHERE user_id=(SELECT id FROM users LIMIT 1) AND key GLOB 'cell.diary.file.*'
|
||||
AND key NOT GLOB '*.json' AND json_valid(value) AND json_type(value,'$.count') IS NOT NULL)),0)
|
||||
SQL_END;
|
||||
|
||||
// Keys follow the node's normalized address, so a report and a setting made
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
#import, file "delegate_storage_types.zi";
|
||||
#import, file "delegate_storage.zi";
|
||||
#import "net_ws_curl_linux"
|
||||
#import "net_http"
|
||||
#import "net_http_curl_linux"
|
||||
#import "url"
|
||||
#import "byte_text_linux"
|
||||
#import "text_buffer"
|
||||
#import, file "account_types.zi";
|
||||
@@ -144,6 +147,70 @@ SyncStoredAccount :: (base_url: string) -> s32 {
|
||||
return outcome
|
||||
}
|
||||
|
||||
// One signed request for a sealed Diary copy (cells/diary_blobs.zi): PUT
|
||||
// stores body at path, GET answers with the copy in output. The bearer
|
||||
// session is the sync's, renewed here when it has run out.
|
||||
#program_export
|
||||
SendStoredBlob :: (base_url: string, method: string, path: string, body: string,
|
||||
output: []u8, status: *s32, length: *s64) -> s32 {
|
||||
status.* = 0
|
||||
length.* = 0
|
||||
if StoredAccountMode() != AccountModeOwner {
|
||||
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
|
||||
}
|
||||
account: AccountKeys
|
||||
if sync_account_load(*account) == 0 {
|
||||
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
|
||||
}
|
||||
defer { SecureWipe(*account.private_key_hex[0], ACCOUNT_PRIVATE_KEY_HEX_SIZE) }
|
||||
material: DeviceKeyMaterial
|
||||
if !LoadOrCreateDeviceKey(*material) {
|
||||
return cast(s32)AuthResult.AUTH_SIGN_FAILED
|
||||
}
|
||||
defer { WipeDeviceKey(*material) }
|
||||
client: Client
|
||||
if !OpenAccountClient(base_url, *account, storage_sync_client_id(), *client) {
|
||||
return cast(s32)AuthResult.AUTH_NO_ACCOUNT
|
||||
}
|
||||
client.send = SendNative
|
||||
now := time(null)
|
||||
session: Session
|
||||
LoadSavedSession(*session, now)
|
||||
if !TokenValid(*session, now) {
|
||||
signed_in := Login(client, *session)
|
||||
if signed_in != cast(AuthResult)AuthResult.AUTH_OK {
|
||||
return cast(s32)signed_in
|
||||
}
|
||||
SaveSession(*session)
|
||||
}
|
||||
transaction: [8192]u8
|
||||
prepared := PrepareRequestHeader(client, *session, DeviceFromKey(*material), now,
|
||||
null, NativeNonce, method, path, body, transaction[:])
|
||||
if prepared != cast(AuthResult)AuthResult.AUTH_OK {
|
||||
return cast(s32)prepared
|
||||
}
|
||||
server: [512]u8
|
||||
url: [1024]u8
|
||||
if !NormalizeUrl(base_url, server[:]) || !JoinUrl(TextUntilNul(server[:]), path, url[:]) {
|
||||
return cast(s32)AuthResult.AUTH_INVALID_URL
|
||||
}
|
||||
headers: [1]HttpHeader
|
||||
headers[0] = HttpHeader.{.name = "X-Daochi-Tx", .value = TextUntilNul(transaction[:])}
|
||||
request := HttpRequest.{.method = method, .url = TextUntilNul(url[:]),
|
||||
.token = TextUntilNul(session.token[:]), .body = body, .headers = headers[:]}
|
||||
if body.count > 0 {
|
||||
request.content_type = "application/octet-stream"
|
||||
}
|
||||
// A copy of up to 16 MB needs longer than a sync's request.
|
||||
result := SendCurlWithin(request, output, 180000)
|
||||
status.* = result.status
|
||||
length.* = result.length
|
||||
if result.code != 0 || result.truncated {
|
||||
return cast(s32)AuthResult.AUTH_REQUEST_FAILED
|
||||
}
|
||||
return cast(s32)AuthResult.AUTH_OK
|
||||
}
|
||||
|
||||
// Expired context can perform only this key-bound challenge/session exchange.
|
||||
// Data payload construction remains unavailable until the answer is accepted.
|
||||
RenewStoredDelegateNative :: (base_url: string) -> AuthResult {
|
||||
|
||||
@@ -20,6 +20,9 @@ ModuleTestFailRun :: () #foreign c_cache;
|
||||
#import, file "../src/cells/diary_types.zi";
|
||||
Dates :: #import, file "../apps/diary/diary_date.zi";
|
||||
#import, file "../src/cells/diary_store.zi";
|
||||
#import, file "../src/cells/diary_records.zi";
|
||||
#import, file "../src/cells/diary_blobs.zi";
|
||||
#import, file "../src/storage/cell_records.zi";
|
||||
#import, file "../src/cells/package_files.zi";
|
||||
#import, file "../src/app/app_types.zi";
|
||||
#import, file "../src/core/types.zi";
|
||||
@@ -43,6 +46,8 @@ Dates :: #import, file "../apps/diary/diary_date.zi";
|
||||
#import, file "../src/app/text_buffers.zi";
|
||||
#import, file "../src/app/app_size.zi";
|
||||
#import "byte_text_linux"
|
||||
#import "text_buffer"
|
||||
#import "crypto"
|
||||
#import, file "../src/app/assets.zi";
|
||||
#import, file "../src/app/locale.zi";
|
||||
#import, file "../src/storage/sync_node_storage.zi";
|
||||
@@ -274,6 +279,7 @@ CheckDiary :: () {
|
||||
photo = DiaryRun(DiaryMessage.{.action = DIARY_LOAD, .year = 2024, .month = 3, .day = 1})
|
||||
Expect(photo.photo_count == 1 && photo.photo_name == "A second photo 🌿", 105)
|
||||
CheckDiaryPhotoOnFullNode()
|
||||
CheckDiaryPhotoCopies()
|
||||
Expect(DiaryStoreSave("2024-03-02.json", "damaged"), 92)
|
||||
bad := DiaryRun(DiaryMessage.{.action = DIARY_LOAD, .year = 2024, .month = 3, .day = 2})
|
||||
bad.action = DIARY_SAVE
|
||||
@@ -353,6 +359,168 @@ CheckDiaryPhotoOnFullNode :: () {
|
||||
storage_set_sync_enabled(sync_was_enabled)
|
||||
}
|
||||
|
||||
DiaryTestImage :: (bytes: []u8, seed: s32) -> string {
|
||||
header: [8]u8 = .[cast(u8)137, cast(u8)80, cast(u8)78, cast(u8)71, cast(u8)13, cast(u8)10, cast(u8)26, cast(u8)10]
|
||||
for index: cast(s32)0..cast(s32)bytes.count - 1 {
|
||||
bytes[index] = ifx index < 8 then header[index] else cast(u8)((index * 7 + seed) % 251)
|
||||
}
|
||||
return TextFromBytes(bytes)
|
||||
}
|
||||
|
||||
DiaryTestBytes :: (text: string) -> []u8 {
|
||||
descriptor := cast(*TextDescriptor)(*text)
|
||||
return BytesFromPointer(descriptor.data, cast(u64)text.count)
|
||||
}
|
||||
|
||||
DiaryTestHash :: (data: []u8, output: []u8) -> string {
|
||||
storage_sha256_hex(*data[0], cast(s64)data.count, *output[0])
|
||||
return TextFromCString(*output[0])
|
||||
}
|
||||
|
||||
// The pass's job for one photo, or 0; other Diary tests leave photos too.
|
||||
DiaryTestJob :: (pass: *DiaryBlobPass, name: string) -> s32 {
|
||||
for index: cast(s32)0..pass.count - 1 {
|
||||
if TextUntilNul(pass.jobs[index].name[:]) == name {
|
||||
return pass.jobs[index].kind
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// The pass's request for one photo, skipping the others.
|
||||
DiaryTestNext :: (pass: *DiaryBlobPass, request: *DiaryBlobRequest, name: string) -> bool {
|
||||
while DiaryBlobNext(pass, request) {
|
||||
if TextUntilNul(request.name[:]) == name {
|
||||
return true
|
||||
}
|
||||
DiaryBlobAbandon(request)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// A photo is its file: only its entry syncs, and its sealed copy, the file's
|
||||
// size plus 16 bytes, travels to a node and back to the account's devices.
|
||||
CheckDiaryPhotoCopies :: () {
|
||||
node := "https://copies.test"
|
||||
sync_was_enabled := storage_sync_enabled()
|
||||
storage_set_sync_enabled(1)
|
||||
storage_set_setting_text("sync_server_url", node)
|
||||
// The account key lets this device seal copies.
|
||||
private: [5121]u8
|
||||
for index: cast(s32)0..cast(s32)5119 {
|
||||
private[index] = ifx index % 2 == 0 then #char "a" else #char "7"
|
||||
}
|
||||
storage_set_setting_text(STORAGE_SYNC_PRIVATE_KEY_KEY, TextFromBytes(private[:5120]))
|
||||
png: [3000]u8
|
||||
image := DiaryTestImage(png[:], 3)
|
||||
source := "copied-photo.png"
|
||||
unused RemoveFile(source)
|
||||
file := OpenExclusivePrivate(source)
|
||||
written := file.valid && WriteTextAt(file, image, 0) == image.count
|
||||
Expect(CloseFile(file) && written, 430)
|
||||
name := DiaryStoreAttach(source)
|
||||
unused RemoveFile(source)
|
||||
Expect(name.count > 0, 431)
|
||||
media := DiaryMediaRead(name)
|
||||
hash_text: [65]u8
|
||||
hash := DiaryTestHash(png[:], hash_text[:])
|
||||
Expect(media.valid && !DiaryMediaUploaded(media) && media.size == 3000 &&
|
||||
TextFromBytes(media.hash[:64]) == hash, 432)
|
||||
no_chunks := PackageText(PackageText("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.diary.chunk.", hash), ".*'")
|
||||
Expect(ModuleTestCount(no_chunks) == 0, 433)
|
||||
PackageFree(no_chunks)
|
||||
// A pass seals and uploads it, then its entry names the copy.
|
||||
pass: DiaryBlobPass
|
||||
request: DiaryBlobRequest
|
||||
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, name) == DIARY_BLOB_UPLOAD, 434)
|
||||
Expect(DiaryTestNext(*pass, *request, name) && request.method == "PUT" &&
|
||||
request.body.count == image.count + DIARY_BLOB_TAG && request.body != image, 435)
|
||||
sealed := PackageText(request.body, "")
|
||||
blob_text: [65]u8
|
||||
blob := DiaryTestHash(DiaryTestBytes(sealed), blob_text[:])
|
||||
blob_path := PackageText("/api/v1/blobs/inbe/", blob)
|
||||
Expect(TextUntilNul(request.path[:]) == blob_path, 436)
|
||||
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]), 437)
|
||||
media = DiaryMediaRead(name)
|
||||
Expect(DiaryMediaUploaded(media) && TextFromBytes(media.blob[:64]) == blob, 438)
|
||||
unused DiaryBlobPlan(*pass, node)
|
||||
Expect(DiaryTestJob(*pass, name) == 0, 439)
|
||||
// Another device of the account has only the entry: the photo shows as
|
||||
// not on the device until its copy arrives, opened and checked.
|
||||
path := DiaryStorePath(name)
|
||||
unused RemoveFile(path)
|
||||
Expect(!DiaryStoreRead(name).succeeded && DiaryStorePreview(name).count == 0, 440)
|
||||
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, name) == DIARY_BLOB_DOWNLOAD, 441)
|
||||
Expect(DiaryTestNext(*pass, *request, name) && request.method == "GET" &&
|
||||
request.answer_bytes == sealed.count + 1, 442)
|
||||
tampered := PackageText(sealed, "")
|
||||
changed := DiaryTestBytes(tampered)
|
||||
changed[20] = cast(u8)((cast(s32)changed[20] + 1) % 256)
|
||||
generation := diary_blob_generation
|
||||
Expect(DiaryBlobFinish(*pass, *request, 200, changed) &&
|
||||
!PathExists(path) && diary_blob_generation == generation, 443)
|
||||
Expect(DiaryBlobFinish(*pass, *request, 404, png[0:0]) && !PathExists(path), 444)
|
||||
Expect(DiaryBlobFinish(*pass, *request, 200, DiaryTestBytes(sealed)) &&
|
||||
diary_blob_generation > generation, 445)
|
||||
Expect(DiaryStoreRead(name).data == image && DiaryStorePreview(name) == image, 446)
|
||||
// A new node gets the same copy once.
|
||||
other := "https://other-copies.test"
|
||||
Expect(DiaryBlobPlan(*pass, other) && DiaryTestJob(*pass, name) == DIARY_BLOB_UPLOAD, 447)
|
||||
Expect(DiaryTestNext(*pass, *request, name) && request.body == sealed, 448)
|
||||
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]), 449)
|
||||
unused DiaryBlobPlan(*pass, other)
|
||||
Expect(DiaryTestJob(*pass, name) == 0, 459)
|
||||
// A node without photo upload receives none; a full one stops uploads.
|
||||
third := "https://third-copies.test"
|
||||
Expect(SyncNodeSetMediaUpload(third, false) && !DiaryBlobPlan(*pass, third), 450)
|
||||
Expect(SyncNodeSetMediaUpload(third, true) && DiaryBlobPlan(*pass, third) &&
|
||||
DiaryTestNext(*pass, *request, name), 451)
|
||||
full := "{\"code\":\"storage_full\",\"account_storage\":{\"used_bytes\":20971000,\"limit_bytes\":20971520}}"
|
||||
full_text := PackageText(full, "")
|
||||
Expect(DiaryBlobFinish(*pass, *request, 413, DiaryTestBytes(full_text)) &&
|
||||
pass.uploads_stopped && SyncNodeStorageRead(third).full, 452)
|
||||
PackageFree(full_text)
|
||||
// A photo from before copies, kept as hex chunks, uploads the same copy
|
||||
// and its chunks shrink to one byte each.
|
||||
older: [2500]u8
|
||||
old_image := DiaryTestImage(older[:], 9)
|
||||
old_name := "photo-20240301000000-1.bin"
|
||||
old_hash_text: [65]u8
|
||||
old_hash := DiaryTestHash(older[:], old_hash_text[:])
|
||||
for index: cast(s32)0..2 {
|
||||
start := index * 1024
|
||||
size: s32 = 1024
|
||||
if start + size > 2500 {
|
||||
size = 2500 - start
|
||||
}
|
||||
hex: [2049]u8
|
||||
unused BytesToHex(*older[start], cast(s64)size, *hex[0], 2049)
|
||||
chunk := DiaryChunkKey(old_hash, index)
|
||||
Expect(CellRecordWrite(chunk, TextFromCString(*hex[0])), 453)
|
||||
PackageFree(chunk)
|
||||
}
|
||||
old_entry := PackageText(PackageText("{\"hash\":\"", old_hash), "\",\"count\":3,\"size\":2500}")
|
||||
old_key := DiaryRecordKey(old_name)
|
||||
Expect(CellRecordWrite(old_key, old_entry) && DiaryStoreRead(old_name).data == old_image, 454)
|
||||
Expect(DiaryBlobPlan(*pass, node) && DiaryTestJob(*pass, old_name) == DIARY_BLOB_UPLOAD &&
|
||||
DiaryTestNext(*pass, *request, old_name) && request.chunked && request.body.count == 2500 + DIARY_BLOB_TAG, 455)
|
||||
Expect(DiaryBlobFinish(*pass, *request, 204, png[0:0]) && DiaryMediaUploaded(DiaryMediaRead(old_name)), 456)
|
||||
shrunk := PackageText(PackageText("SELECT COUNT(*) FROM settings WHERE key GLOB 'cell.diary.chunk.", old_hash), ".*' AND value='-'")
|
||||
Expect(ModuleTestCount(shrunk) == 3, 457)
|
||||
PackageFree(shrunk)
|
||||
Expect(DiaryStoreRead(old_name).data == old_image, 458)
|
||||
PackageFree(old_key)
|
||||
PackageFree(old_entry)
|
||||
PackageFree(tampered)
|
||||
PackageFree(blob_path)
|
||||
PackageFree(sealed)
|
||||
PackageFree(path)
|
||||
PackageFree(name)
|
||||
storage_set_setting_text(STORAGE_SYNC_PRIVATE_KEY_KEY, "")
|
||||
storage_set_setting_text("sync_server_url", "")
|
||||
storage_set_sync_enabled(sync_was_enabled)
|
||||
}
|
||||
|
||||
CheckPackagedLocales :: () {
|
||||
Expect(LocaleLanguageCount() == 12, 200)
|
||||
for index: cast(s32)0..LocaleLanguageCount() - 1 {
|
||||
|
||||
@@ -117,6 +117,41 @@ queued_diary_chunks :: () -> s32 {
|
||||
return read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_type='cell_record' AND entity_id GLOB 'cell.diary.chunk.*'")
|
||||
}
|
||||
|
||||
// A photo saved before sealed copies: hex chunks and an entry that counts
|
||||
// them, as older versions still send.
|
||||
save_legacy_photo :: (name: string, data: string) -> bool {
|
||||
descriptor := cast(*TextDescriptor)(*data)
|
||||
hash_bytes: [65]u8
|
||||
storage_sha256_hex(descriptor.data, data.count, *hash_bytes[0])
|
||||
hash := TextFromCString(*hash_bytes[0])
|
||||
count := cast(s32)((data.count + DIARY_CHUNK_BYTES - 1) / DIARY_CHUNK_BYTES)
|
||||
for index: cast(s32)0..count - 1 {
|
||||
start := cast(s64)index * DIARY_CHUNK_BYTES
|
||||
size := data.count - start
|
||||
if size > DIARY_CHUNK_BYTES {
|
||||
size = DIARY_CHUNK_BYTES
|
||||
}
|
||||
encoded: [DIARY_CHUNK_BYTES * 2 + 1]u8
|
||||
if BytesToHex(*descriptor.data[start], size, *encoded[0], encoded.count) == 0 {
|
||||
return false
|
||||
}
|
||||
chunk := DiaryChunkKey(hash, index)
|
||||
written := CellRecordWrite(chunk, TextFromCString(*encoded[0]))
|
||||
PackageFree(chunk)
|
||||
if !written {
|
||||
return false
|
||||
}
|
||||
}
|
||||
out: Vec(u8)
|
||||
BuilderPrint(*out, "{\"hash\":\"%\",\"count\":%,\"size\":%}", hash, count, data.count)
|
||||
entry := BuilderFinish(out)
|
||||
key := DiaryRecordKey(name)
|
||||
written := CellRecordWrite(key, entry)
|
||||
PackageFree(key)
|
||||
PackageFree(entry)
|
||||
return written
|
||||
}
|
||||
|
||||
format_bytes :: (bytes: s64) -> string {
|
||||
output: [32]u8
|
||||
if !SyncStorageFormatBytes(bytes, output[:]) {
|
||||
@@ -125,8 +160,9 @@ format_bytes :: (bytes: s64) -> string {
|
||||
return PackageText(TextFromCString(*output[0]), "")
|
||||
}
|
||||
|
||||
// A node set not to store photos receives each Diary file entry and the day
|
||||
// documents, never photo chunks. Reports and limits.
|
||||
// A photo syncs only its entry; its sealed copy travels separately
|
||||
// (cells/diary_blobs.zi). A node set not to store photos still never
|
||||
// receives the chunks of a photo from before copies. Reports and limits.
|
||||
test_node_storage_and_media :: () {
|
||||
root := make_root("sync-node-media")
|
||||
check_true("init media device", storage_init(root) != 0)
|
||||
@@ -140,15 +176,23 @@ test_node_storage_and_media :: () {
|
||||
photo[index] = cast(u8)(index % 251)
|
||||
}
|
||||
check_true("save diary photo", DiaryRecordSave("photo-1-1.bin", TextFromBytes(photo[:])))
|
||||
check_int("day and photo chunks queue", queued_diary_chunks(), 4)
|
||||
check_int("a photo queues no chunks", queued_diary_chunks(), 1)
|
||||
check_true("photos upload by default", SyncNodeMediaUpload(node))
|
||||
check_true("payload with photos builds", build_payload())
|
||||
check_int("file entries and every chunk upload", record_count(), 6)
|
||||
|
||||
photo_hash := PackageText(read_db_text("SELECT json_extract(value,'$.hash') FROM settings WHERE key='cell.diary.file.photo-1-1.bin'"), "")
|
||||
check_int("photo entry names its content", cast(s32)photo_hash.count, 64)
|
||||
check_true("local photo bytes equal the node's chunk bytes",
|
||||
SyncMediaLocalBytes() == payload_file_bytes("photo-1-1.bin", photo_hash, true))
|
||||
check_true("payload with a photo builds", build_payload())
|
||||
check_int("file entries and the day chunk upload", record_count(), 3)
|
||||
check_true("a photo's copy is its size plus 16 bytes", SyncMediaLocalBytes() == 3016)
|
||||
older: [2500]u8
|
||||
for index: cast(s32)0..cast(s32)2499 {
|
||||
older[index] = cast(u8)((index * 3) % 251)
|
||||
}
|
||||
check_true("keep a photo from before copies", save_legacy_photo("photo-0-1.bin", TextFromBytes(older[:])))
|
||||
check_int("its chunks queue", queued_diary_chunks(), 4)
|
||||
check_true("payload with the old photo builds", build_payload())
|
||||
check_int("entries and every chunk upload", record_count(), 7)
|
||||
photo_hash := PackageText(read_db_text("SELECT json_extract(value,'$.hash') FROM settings WHERE key='cell.diary.file.photo-0-1.bin'"), "")
|
||||
check_int("old photo entry names its content", cast(s32)photo_hash.count, 64)
|
||||
check_true("local photo bytes count the copy and the old chunks",
|
||||
SyncMediaLocalBytes() == 3016 + payload_file_bytes("photo-0-1.bin", photo_hash, true))
|
||||
PackageFree(photo_hash)
|
||||
|
||||
check_true("turn photos off for the node", SyncNodeSetMediaUpload(node, false))
|
||||
@@ -156,11 +200,12 @@ test_node_storage_and_media :: () {
|
||||
check_true("other nodes keep photos on", SyncNodeMediaUpload("https://other.example"))
|
||||
check_int("photo chunks leave the queue", queued_diary_chunks(), 1)
|
||||
check_true("payload without photos builds", build_payload())
|
||||
check_int("only entries and the day chunk upload", record_count(), 3)
|
||||
check_true("photo entry still syncs", contains(payload(), "cell.diary.file.photo-1-1.bin"))
|
||||
check_int("only entries and the day chunk upload", record_count(), 4)
|
||||
check_true("photo entries still sync", contains(payload(), "cell.diary.file.photo-1-1.bin") &&
|
||||
contains(payload(), "cell.diary.file.photo-0-1.bin"))
|
||||
storage_enqueue_all_sync_state()
|
||||
check_true("backfill payload builds", build_payload())
|
||||
check_int("a backfill does not send photos", record_count(), 3)
|
||||
check_int("a backfill does not send photos", record_count(), 4)
|
||||
check_true("turn photos back on", SyncNodeSetMediaUpload(node, true))
|
||||
check_int("photos queue again", queued_diary_chunks(), 4)
|
||||
|
||||
@@ -1754,7 +1799,9 @@ test_cell_records_round_trip :: () {
|
||||
for index: cast(s32)0..cast(s32)4096 {
|
||||
photo[index] = cast(u8)(index % 256)
|
||||
}
|
||||
check_true("save binary diary photo", DiaryRecordSave("photo-test.png", TextFromBytes(photo[:])))
|
||||
// Photos from older versions still arrive as binary chunks.
|
||||
check_true("save binary diary photo", save_legacy_photo("photo-test.png", TextFromBytes(photo[:])))
|
||||
check_true("save a photo entry", DiaryRecordSave("photo-new.png", TextFromBytes(photo[:4000])))
|
||||
check_true("cell payload builds", build_payload())
|
||||
check_true("Lumi uses separate encrypted collection", contains(payload(), "private.inbe.v2.lumi"))
|
||||
check_true("Diary retains legacy owner collection", contains(payload(), "private.inbe.v1.cells"))
|
||||
@@ -1780,6 +1827,9 @@ test_cell_records_round_trip :: () {
|
||||
restored := DiaryRecordRead("photo-test.png")
|
||||
check_true("binary photo survives", restored == TextFromBytes(photo[:]))
|
||||
PackageFree(restored)
|
||||
entry := DiaryMediaRead("photo-new.png")
|
||||
check_true("a photo entry arrives without its bytes", entry.valid && entry.size == 4000 &&
|
||||
!DiaryMediaUploaded(entry) && DiaryRecordRead("photo-new.png").count == 0)
|
||||
check_int("restore does not requeue cells", read_db_count("SELECT COUNT(*) FROM sync_outbox WHERE entity_type='cell_record'"), 0)
|
||||
check_true("cell collection cannot overwrite signing key",
|
||||
storage_apply_decrypted_record("private.inbe.v1.cells", "sync_private_key", "{\"key\":\"sync_private_key\",\"value\":\"forged\",\"updated_at\":\"2026-10-07T00:00:00Z\"}") == 0)
|
||||
|
||||
@@ -28,6 +28,10 @@
|
||||
#import "authorization"
|
||||
#import, file "../src/core/types.zi";
|
||||
#import, file "../src/cells/package_files.zi";
|
||||
#import, file "../src/cells/diary_store.zi";
|
||||
#import, file "../src/cells/diary_records.zi";
|
||||
#import, file "../src/cells/diary_blobs_native.zi";
|
||||
#import "file_linux"
|
||||
|
||||
// Two clients, a friend, and fresh installs syncing through a real Daochi
|
||||
// server (with a proxy that drops one response), from the first upload to
|
||||
@@ -155,6 +159,54 @@ synchronize :: (url: string) {
|
||||
check_true("sync drained the queued changes", false)
|
||||
}
|
||||
|
||||
// A Diary photo travels as one sealed copy, which the node keeps at the
|
||||
// photo's size plus 16 bytes, and arrives whole on the account's other
|
||||
// device.
|
||||
expect_photo_copy :: (url: string, a: string, b: string) {
|
||||
open_client(a)
|
||||
png: [6000]u8
|
||||
for index: cast(s32)0..cast(s32)5999 {
|
||||
png[index] = cast(u8)((index * 13 + 7) % 251)
|
||||
}
|
||||
header: [8]u8 = .[cast(u8)137, cast(u8)80, cast(u8)78, cast(u8)71, cast(u8)13, cast(u8)10, cast(u8)26, cast(u8)10]
|
||||
for index: cast(s32)0..7 {
|
||||
png[index] = header[index]
|
||||
}
|
||||
image := TextFromBytes(png[:])
|
||||
source: TextBuffer
|
||||
append(*source, data_root())
|
||||
append(*source, "/photo-source.png")
|
||||
source_path := TextFromCString(*source.bytes[0])
|
||||
unused RemoveFile(source_path)
|
||||
file := OpenExclusivePrivate(source_path)
|
||||
written := file.valid && WriteTextAt(file, image, 0) == image.count
|
||||
check_true("write the photo to attach", CloseFile(file) && written)
|
||||
name := DiaryStoreAttach(source_path)
|
||||
check_true("attach the photo", name.count > 0)
|
||||
DiaryStoreClose()
|
||||
synchronize(url)
|
||||
DiaryBlobsSendNative(url)
|
||||
media := DiaryMediaRead(name)
|
||||
check_true("the photo's copy is on the node", DiaryMediaUploaded(media))
|
||||
synchronize(url)
|
||||
blob_path: TextBuffer
|
||||
append(*blob_path, data_root())
|
||||
append(*blob_path, "/server.db.blobs/")
|
||||
append(*blob_path, storage_get_setting_text("sync_public_id"))
|
||||
append(*blob_path, "/inbe/")
|
||||
append(*blob_path, TextFromBytes(media.blob[:64]))
|
||||
check_int("the node keeps the photo's size plus 16 bytes",
|
||||
cast(s32)PackageFileSize(TextFromCString(*blob_path.bytes[0])), 6016)
|
||||
open_client(b)
|
||||
synchronize(url)
|
||||
check_true("the photo waits on the second device", !DiaryStoreRead(name).succeeded)
|
||||
DiaryStoreClose()
|
||||
DiaryBlobsSendNative(url)
|
||||
check_true("the photo arrives whole on the second device", DiaryStoreRead(name).data == image)
|
||||
DiaryStoreClose()
|
||||
PackageFree(name)
|
||||
}
|
||||
|
||||
mood_is :: (id: string, want: s32) -> bool {
|
||||
loaded: SessionMoodCheckin
|
||||
return storage_load_session_checkin(id, *loaded) != 0 && loaded.mood_after == want
|
||||
@@ -309,6 +361,8 @@ Check :: () -> s32 {
|
||||
check_true("completion reaches other device", ActivityRead(live_document, *observed) && observed.active == 0)
|
||||
PackageFree(live_document)
|
||||
check_int("live activity does not duplicate results", storage_session_count(), 1)
|
||||
expect_photo_copy(url, a, b)
|
||||
open_client(b)
|
||||
check_true("complete the item", storage_elist_update_item(item_text, "Pack bag", "Passport", 1, 0) != 0)
|
||||
synchronize(url)
|
||||
open_client(a)
|
||||
|
||||
@@ -28,11 +28,13 @@ mkdir -p "$work/generated"
|
||||
cat > "$work/generated/main.c" <<'C'
|
||||
#include <stdio.h>
|
||||
#include "sync_server_behavior.h"
|
||||
/* Diary files are flushed by the browser's storage only on the web. */
|
||||
void app_web_storage_flush(void) {}
|
||||
int main(void)
|
||||
{
|
||||
int result = Check();
|
||||
if (result != 0) fprintf(stderr, "sync server check failed with code %d\n", result);
|
||||
if (result == 0) puts("PASS two-client sync, habit/session/list/alias/friend recovery, offline retry, conflict, backup restore, deletion");
|
||||
if (result == 0) puts("PASS two-client sync, habit/session/list/alias/friend recovery, Diary photo copies, offline retry, conflict, backup restore, deletion");
|
||||
return result == 0 ? 0 : 1;
|
||||
}
|
||||
C
|
||||
|
||||
+6
-6
@@ -55,7 +55,7 @@
|
||||
},
|
||||
{
|
||||
"commit": "41538659afbbda300915ef009b55c18f6de159b8",
|
||||
"dependencies": {"kryon": "pe4492807d82c43f5"},
|
||||
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
|
||||
"id": "p9504fb8e5fabefc6",
|
||||
"name": "Kss",
|
||||
"ref": "master",
|
||||
@@ -71,7 +71,7 @@
|
||||
},
|
||||
{
|
||||
"commit": "2ee29338c9b93fb4f4a56a49c7be09e9d2c2418f",
|
||||
"dependencies": {"kryon": "pe4492807d82c43f5"},
|
||||
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
|
||||
"id": "pa5dd7b8b83307b18",
|
||||
"name": "plot",
|
||||
"ref": "master",
|
||||
@@ -79,7 +79,7 @@
|
||||
},
|
||||
{
|
||||
"commit": "7aa01465ac3f988e0404f3c6dc244bf4b009f253",
|
||||
"dependencies": {"kryon": "pe4492807d82c43f5"},
|
||||
"dependencies": {"kryon": "pd71ea76b67bd47e2"},
|
||||
"id": "paa08e1730a25bda0",
|
||||
"name": "Game2D",
|
||||
"ref": "master",
|
||||
@@ -95,9 +95,9 @@
|
||||
"url": "https://github.com/curl/curl.git"
|
||||
},
|
||||
{
|
||||
"commit": "556ee25f11f2e6b979150acb94e3093a137ed47e",
|
||||
"commit": "98d63a194ee790deace033f77f66163a78122069",
|
||||
"dependencies": {"raylib": "p5a77c9c35110e925"},
|
||||
"id": "pe4492807d82c43f5",
|
||||
"id": "pd71ea76b67bd47e2",
|
||||
"name": "kryon",
|
||||
"ref": "master",
|
||||
"url": "https://github.com/kryonlabs/kryon.git"
|
||||
@@ -121,7 +121,7 @@
|
||||
}
|
||||
],
|
||||
"root": {
|
||||
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pe4492807d82c43f5", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
|
||||
"dependencies": {"curl": "pc4b263336680098b", "daochi_client": "p96e1f539ccb91af5", "game2d": "paa08e1730a25bda0", "kryon": "pd71ea76b67bd47e2", "kss": "p9504fb8e5fabefc6", "liboqs": "pfb3df7cb222b1bdc", "mingcute": "p56788fa3dab3e73a", "monocypher": "p2ff634c7b007140d", "oqs": "p55635fc8976895a9", "plot": "pa5dd7b8b83307b18", "qrcode": "pf0ce9cea355c9228", "rini": "p09fa77e94b0dfc6c", "sqlite": "p151b83a361c86f2d"},
|
||||
"name": "Inbe"
|
||||
},
|
||||
"toolchain": {
|
||||
|
||||
Reference in new issue
Block a user