diff --git a/src/platform/plan9/sqlite3_stub.zi b/src/platform/plan9/sqlite3_stub.zi index 9bfb239e1..f8e8a8fa6 100644 --- a/src/platform/plan9/sqlite3_stub.zi +++ b/src/platform/plan9/sqlite3_stub.zi @@ -200,6 +200,12 @@ return null } + #program_export + sqlite3_mutex_alloc :: (kind: s32) -> *void { + unused kind + return null + } + #program_export sqlite3_mutex_enter :: (mutex: *void) { unused mutex diff --git a/src/storage/db.zi b/src/storage/db.zi index 83eb83126..b2227779d 100644 --- a/src/storage/db.zi +++ b/src/storage/db.zi @@ -6,7 +6,6 @@ LOG_WARNING :: 4 LOG_ERROR :: 5 NativeTraceLog :: (level: s32, message: *u8, args: ..any) #foreign c "TraceLog"; sqlite3_open_v2 :: (path: *u8, database: **void, flags: s32, vfs: *u8) -> s32 #foreign sqlite3_lib "sqlite3_open_v2"; -sqlite3_close :: (database: *void) -> s32 #foreign sqlite3_lib "sqlite3_close"; sqlite3_busy_timeout :: (database: *void, milliseconds: s32) -> s32 #foreign sqlite3_lib "sqlite3_busy_timeout"; sqlite3_total_changes64 :: (database: *void) -> s64 #foreign sqlite3_lib "sqlite3_total_changes64"; @@ -53,7 +52,7 @@ storage_data_revision :: () -> s64 { changes := storage_connection_changes() data_version := storage_connection_data_version() if changes != storage_revision_changes || data_version != storage_revision_data_version || - storage_connection_generation != storage_revision_generation { + storage_connection_generation != storage_revision_generation { storage_revision_value += 1 storage_revision_changes = changes storage_revision_data_version = data_version @@ -229,14 +228,14 @@ storage_init :: (root: string) -> s32 { SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, null) != SQLITE_OK { LogStorage(LOG_ERROR, "STORAGE: failed to open database") - sqlite3_close(storage_state_handle().db) + unused SqliteClose(storage_state_handle().db) storage_state_handle().db = null return 0 } storage_connection_generation += 1 sqlite3_busy_timeout(storage_state_handle().db, 1000) if schema_create() == 0 || migrate_schema() == 0 || load_or_create_user() == 0 { - sqlite3_close(storage_state_handle().db) + unused SqliteClose(storage_state_handle().db) storage_state_handle().db = null return 0 } @@ -245,7 +244,7 @@ storage_init :: (root: string) -> s32 { storage_migrate_default_meditation_activity_mask() storage_migrate_sync_enabled_flag() if storage_migrate_lumi_collection() == 0 || storage_migrate_habit_settings() == 0 { - sqlite3_close(storage_state_handle().db) + unused SqliteClose(storage_state_handle().db) storage_state_handle().db = null return 0 } @@ -260,7 +259,7 @@ storage_init :: (root: string) -> s32 { storage_close :: () { storage_connection_generation += 1 if storage_state_handle().db != null { - sqlite3_close(storage_state_handle().db) + unused SqliteClose(storage_state_handle().db) storage_state_handle().db = null } storage_state_handle().pending_sync_outbox_seq = 0 diff --git a/src/storage/sqlite.zi b/src/storage/sqlite.zi index ac8e5d563..e30c2e7ca 100644 --- a/src/storage/sqlite.zi +++ b/src/storage/sqlite.zi @@ -41,6 +41,122 @@ BackupStepRaw :: (backup: *void, pages: s32) -> s32 #foreign sqlite "sqlite3_bac BackupFinishRaw :: (backup: *void) -> s32 #foreign sqlite "sqlite3_backup_finish"; ExecRaw :: (database: *void, sql: *u8, callback: *void, argument: *void, error: **u8) -> s32 #foreign sqlite "sqlite3_exec"; +MutexAllocRaw :: (kind: s32) -> *void #foreign sqlite "sqlite3_mutex_alloc"; +MutexEnterRaw :: (mutex: *void) #foreign sqlite "sqlite3_mutex_enter"; +MutexLeaveRaw :: (mutex: *void) #foreign sqlite "sqlite3_mutex_leave"; +CompareBytes :: (first: *void, second: *void, size: usize) -> s32 #foreign c "memcmp"; +TextLength :: (text: *u8) -> usize #foreign c "strlen"; + +// Prepared statements kept for the next caller of the same SQL on the same +// connection. Preparing parses and plans a query every time, and the app's +// queries repeat with new bindings, so a finished statement is reset and +// kept instead of finalized. The cache's own lock is never held while +// SQLite works, because callers may already hold a connection's lock. +SQLITE_MUTEX_STATIC_APP1 :: 8 +SQLITE_KEPT_STATEMENTS :: 48 +SQLITE_KEPT_SQL_LIMIT :: 2048 + +SqliteKept :: struct { + db: *void + statement: *void + sql: *u8 + length: usize + hash: u64 + busy: bool + used: u64 +} + +sqlite_kept: [SQLITE_KEPT_STATEMENTS]SqliteKept; +sqlite_kept_clock: u64; + +SqliteKeptLock :: () -> *void { + mutex := MutexAllocRaw(SQLITE_MUTEX_STATIC_APP1) + MutexEnterRaw(mutex) + return mutex +} + +SqliteKeptHash :: (sql: *u8, length: usize) -> u64 { + hash: u64 = 14695981039346656037 + index: usize = 0 + while index < length { + hash = (hash ^ cast(u64)sql[index]) * 1099511628211 + index += 1 + } + return hash +} + +// A kept statement for this SQL that no one is using, now taken; null when +// there is none. +SqliteKeptTake :: (db: *void, sql: *u8, length: usize, hash: u64) -> *void { + mutex := SqliteKeptLock() + defer { MutexLeaveRaw(mutex) } + for index: cast(s32)0..SQLITE_KEPT_STATEMENTS - 1 { + kept := *sqlite_kept[index] + if kept.statement != null && !kept.busy && kept.db == db && kept.length == length && + kept.hash == hash && CompareBytes(cast(*void)kept.sql, cast(*void)sql, length) == 0 { + kept.busy = true + sqlite_kept_clock += 1 + kept.used = sqlite_kept_clock + return kept.statement + } + } + return null +} + +// Keeps a statement just prepared, in use by its caller. The statement the +// cache gives up for it, if any, is returned for finalizing. +SqliteKeptAdd :: (db: *void, statement: *void, sql: *u8, length: usize, hash: u64) -> *void { + copy := cast(*u8)malloc(length + 1) + if copy == null { + return null + } + for index: cast(usize)0..length - 1 { + copy[index] = sql[index] + } + copy[length] = cast(u8)0 + mutex := SqliteKeptLock() + defer { MutexLeaveRaw(mutex) } + slot: s32 = -1 + for index: cast(s32)0..SQLITE_KEPT_STATEMENTS - 1 { + kept := sqlite_kept[index] + if kept.statement == null { + slot = index + break + } + if !kept.busy && (slot < 0 || kept.used < sqlite_kept[slot].used) { + slot = index + } + } + if slot < 0 { + free(cast(*void)copy) + return null + } + evicted := sqlite_kept[slot].statement + free(cast(*void)sqlite_kept[slot].sql) + sqlite_kept_clock += 1 + sqlite_kept[slot] = SqliteKept.{.db = db, .statement = statement, .sql = copy, .length = length, + .hash = hash, .busy = true, .used = sqlite_kept_clock} + return evicted +} + +SqliteKeptFind :: (statement: *void) -> s32 { + mutex := SqliteKeptLock() + defer { MutexLeaveRaw(mutex) } + for index: cast(s32)0..SQLITE_KEPT_STATEMENTS - 1 { + if sqlite_kept[index].statement == statement && sqlite_kept[index].busy { + return index + } + } + return -1 +} + +SqliteKeptRelease :: (index: s32, statement: *void) { + mutex := SqliteKeptLock() + defer { MutexLeaveRaw(mutex) } + if sqlite_kept[index].statement == statement { + sqlite_kept[index].busy = false + } +} CopyStringBytes :: (source: string, destination: *u8) { index: s32 = 0 @@ -52,10 +168,31 @@ CopyStringBytes :: (source: string, destination: *u8) { } #scope_export +// Statements a caller walks through with `tail` are prepared afresh. #program_export SqlitePrepare :: (db: *void, sql: *u8, length: s32, statement: **void, tail: **u8) -> s32 { - return PrepareRaw(db, sql, length, statement, tail) + if tail != null || sql == null || statement == null { + return PrepareRaw(db, sql, length, statement, tail) + } + size := ifx length < 0 then TextLength(sql) else cast(usize)length + if size == cast(usize)0 || size > cast(usize)SQLITE_KEPT_SQL_LIMIT { + return PrepareRaw(db, sql, length, statement, tail) + } + hash := SqliteKeptHash(sql, size) + kept := SqliteKeptTake(db, sql, size, hash) + if kept != null { + statement.* = kept + return SQLITE_OK + } + result := PrepareRaw(db, sql, length, statement, null) + if result == SQLITE_OK && statement.* != null { + evicted := SqliteKeptAdd(db, statement.*, sql, size, hash) + if evicted != null { + unused FinalizeRaw(evicted) + } + } + return result } #scope_export @@ -69,14 +206,14 @@ SqlitePrepareText :: (db: *void, sql: string, length: s32, if sql.count < 512 { short_sql: [512]u8 CopyStringBytes(sql, *short_sql[0]) - return PrepareRaw(db, *short_sql[0], length, statement, null) + return SqlitePrepare(db, *short_sql[0], length, statement, null) } buffer: *u8 = cast(*u8)malloc(cast(usize)sql.count + 1) if buffer == null { return SQLITE_NOMEM } CopyStringBytes(sql, buffer) - result := PrepareRaw(db, buffer, length, statement, null) + result := SqlitePrepare(db, buffer, length, statement, null) free(cast(*void)buffer) return result } @@ -110,9 +247,18 @@ SqliteStep :: (statement: *void) -> s32 { return StepRaw(statement) } +// A kept statement is reset, with its bindings cleared, for the next caller +// of its SQL. Like finalizing, that reports the last step's error. #program_export SqliteFinalize :: (statement: *void) -> s32 { - return FinalizeRaw(statement) + index := SqliteKeptFind(statement) + if index < 0 { + return FinalizeRaw(statement) + } + result := ResetRaw(statement) + unused ClearBindingsRaw(statement) + SqliteKeptRelease(index, statement) + return result } #program_export @@ -171,9 +317,38 @@ SqliteOpen :: (path: *u8, database: **void, flags: s32) -> s32 { #program_export SqliteClose :: (database: *void) -> s32 { + SqliteReleaseKept(database) return CloseRaw(database) } +// Finalizes the connection's kept statements that are not in use; one still +// in use is its caller's to finalize. SQLite will not close a connection +// with statements left, and the next prepare of their SQL starts afresh. +#program_export +SqliteReleaseKept :: (database: *void) { + idle: [SQLITE_KEPT_STATEMENTS]*void + idle_count: s32 = 0 + { + mutex := SqliteKeptLock() + defer { MutexLeaveRaw(mutex) } + for index: cast(s32)0..SQLITE_KEPT_STATEMENTS - 1 { + kept := *sqlite_kept[index] + if kept.statement == null || kept.db != database { + continue + } + if !kept.busy { + idle[idle_count] = kept.statement + idle_count += 1 + } + free(cast(*void)kept.sql) + kept.* = SqliteKept.{} + } + } + for index: cast(s32)0..idle_count - 1 { + unused FinalizeRaw(idle[index]) + } +} + #program_export SqliteBackupInit :: (destination: *void, source: *void) -> *void { name: [5]u8 diff --git a/tests/break_stats_link_test.c b/tests/break_stats_link_test.c index d0259b266..17862512e 100644 --- a/tests/break_stats_link_test.c +++ b/tests/break_stats_link_test.c @@ -21,6 +21,8 @@ #include #include +int32_t SqliteClose(void *database); + static LocalDateTime forced_local; static bool use_forced_local; @@ -200,7 +202,8 @@ main(void) assert(sqlite3_column_int(stmt, 0) == BREAK_TYPE_COUNT); assert(sqlite3_finalize(stmt) == SQLITE_OK); } - assert(sqlite3_close(database) == SQLITE_OK); + // Statements the storage wrapper keeps for reuse are released by its close. + assert(SqliteClose(database) == SQLITE_OK); puts("break stats Ziran and SQLite passed"); return 0; } diff --git a/tests/schema_zi_test.c b/tests/schema_zi_test.c index 27730dd97..a7ed13bca 100644 --- a/tests/schema_zi_test.c +++ b/tests/schema_zi_test.c @@ -100,9 +100,12 @@ static void open_memory_database(void) memset(current_user_id, 0, sizeof current_user_id); } +int32_t SqliteClose(void *database); + static void close_database(void) { - assert(sqlite3_close(database) == SQLITE_OK); + // Statements the wrapper keeps for reuse are released by its close. + assert(SqliteClose(database) == SQLITE_OK); database = NULL; } diff --git a/tests/sqlite_text_zi_test.c b/tests/sqlite_text_zi_test.c index bfd992265..c8995251e 100644 --- a/tests/sqlite_text_zi_test.c +++ b/tests/sqlite_text_zi_test.c @@ -11,6 +11,7 @@ int32_t SqliteBindTextString(void *statement, int32_t index, String value, int32_t length); int32_t SqliteStep(void *statement); int32_t SqliteFinalize(void *statement); +int32_t SqliteClose(void *database); int32_t SqliteExecText(void *database, String sql); int @@ -58,7 +59,8 @@ main(void) assert(SqliteExecText(db, StringView(long_sql, 6000)) == SQLITE_OK); assert(sqlite3_exec(db, "INSERT INTO test VALUES(7)", NULL, NULL, NULL) == SQLITE_OK); - assert(sqlite3_close(db) == SQLITE_OK); + // Statements the wrapper keeps for reuse are released by its close. + assert(SqliteClose(db) == SQLITE_OK); puts("Ziran SQLite text preserves long and binary values after binding"); return 0; } diff --git a/tests/storage_sync_behavior.zi b/tests/storage_sync_behavior.zi index 8b1f5edb4..2ea939415 100644 --- a/tests/storage_sync_behavior.zi +++ b/tests/storage_sync_behavior.zi @@ -1397,6 +1397,9 @@ test_review_preview_and_choices :: () { "{\"protocol_version\":6,\"changes\":{\"encrypted_records\":[{\"collection\":\"private.inbe.v1.sessions\",\"id\":\"bad\",\"key_id\":\"bad\",\"nonce\":\"bad\",\"ciphertext\":\"bad\"}]}}", 2, *plan) == 0) check_true("failed preview returned no partial text", plan == null) check_int("failed preview kept local habit", read_db_count("SELECT COUNT(*) FROM habits WHERE id='local-habit'"), 1) + // The failure is injected where a statement is prepared, so none kept + // from the previews above may stand in for it. + SqliteReleaseKept(storage_db_handle()) ReviewFieldFailure(1) check_true("failed field preview is rejected", storage_sync_review_plan_for_json(snapshot, 2, *plan) == 0) check_true("failed field preview returned no partial text", plan == null)