959 lines
36 KiB
Ziran
959 lines
36 KiB
Ziran
#import, file "../apps/practice/recovery.zi";
|
|
// Crash behaviour of storage. A writer process saves through the real
|
|
// storage code until tests/storage_crash_zi_test.sh kills it with SIGKILL
|
|
// at a random moment. A checker process then opens the same data the way
|
|
// the next start of the app would, and checks that nothing acknowledged was
|
|
// lost and nothing is half written. The restore role checks recovery from a
|
|
// damaged database with the last good copy.
|
|
//
|
|
// CRASH_ROLE is writer, checker or restore. CRASH_SCENARIO names the data
|
|
// a writer and its checker share under APP_DATA_ROOT:
|
|
// sessions storage_save_session in a loop
|
|
// payloads PackageWrite of payload files, as a photo or media upload
|
|
// copies the last good copy written at every start
|
|
// drafts storage_practice_draft_save in a loop
|
|
// The writer appends each number to <scenario>.acks only once the save
|
|
// returned, so every acknowledged number must survive the kill.
|
|
#import "byte_text_linux"
|
|
#import "file_linux"
|
|
#import "number_text"
|
|
#import, file "storage_test_support.zi";
|
|
#import, file "../src/app/text_buffers.zi";
|
|
#import, file "../src/storage/data.zi";
|
|
#import, file "../src/storage/state.zi";
|
|
#import, file "../src/storage/db.zi";
|
|
#import, file "../src/storage/paths.zi";
|
|
#import, file "../src/storage/database_backup.zi";
|
|
#import, file "../src/storage/practice_draft.zi";
|
|
#import, file "../src/storage/storage_core.zi";
|
|
#import, file "../src/storage/storage_sessions.zi";
|
|
#import, file "../src/apps/package_files.zi";
|
|
|
|
crash_libc :: #system_library "c";
|
|
CrashGetEnv :: (name: *u8) -> *u8 #foreign crash_libc "getenv";
|
|
CrashTime :: (output: *s64) -> s64 #foreign crash_libc "time";
|
|
CrashAllocate :: (size: usize) -> *u8 #foreign crash_libc "malloc";
|
|
CrashSetEnvironment :: (name: *u8, value: *u8, overwrite: s32) -> s32 #foreign crash_libc "setenv";
|
|
CrashUnsetEnvironment :: (name: *u8) -> s32 #foreign crash_libc "unsetenv";
|
|
CrashTimezoneChanged :: () #foreign crash_libc "tzset";
|
|
|
|
// A separate connection to any SQLite file, as an outside reader would open it.
|
|
crash_sqlite :: #system_library "sqlite3";
|
|
CrashSqliteOpen :: (path: *u8, database: **void) -> s32 #foreign crash_sqlite "sqlite3_open";
|
|
CrashSqliteClose :: (database: *void) -> s32 #foreign crash_sqlite "sqlite3_close";
|
|
CrashSqlitePrepare :: (database: *void, sql: *u8, length: s32, statement: **void,
|
|
tail: **u8) -> s32 #foreign crash_sqlite "sqlite3_prepare_v2";
|
|
CrashSqliteStep :: (statement: *void) -> s32 #foreign crash_sqlite "sqlite3_step";
|
|
CrashSqliteColumnInt :: (statement: *void, column: s32) -> s32 #foreign crash_sqlite "sqlite3_column_int";
|
|
CrashSqliteColumnText :: (statement: *void, column: s32) -> *u8 #foreign crash_sqlite "sqlite3_column_text";
|
|
CrashSqliteFinalize :: (statement: *void) -> s32 #foreign crash_sqlite "sqlite3_finalize";
|
|
|
|
// PackageWrite flushes the browser's filesystem; a native test has none.
|
|
#program_export "app_web_storage_flush"
|
|
CrashWebStorageFlush :: (log_success: s32) {
|
|
unused log_success
|
|
}
|
|
|
|
// Storage must only ever see this test's own data folder.
|
|
CrashDataFolder :: "/build/storage-crash-zi-test/data"
|
|
PayloadFileCount :: 4
|
|
// A writer that is never killed stops by itself, and the runner fails it.
|
|
WriterSeconds :: 30
|
|
CopyFillerRows :: 3000
|
|
DayInSeconds :: 86400
|
|
|
|
CrashText :: struct {
|
|
bytes: [4096]u8
|
|
length: s64
|
|
}
|
|
|
|
CrashTextAdd :: (text: *CrashText, part: string) {
|
|
index: s64 = 0
|
|
while index < part.count && text.length < 4095 {
|
|
text.bytes[text.length] = part[index]
|
|
text.length += 1
|
|
index += 1
|
|
}
|
|
text.bytes[text.length] = cast(u8)0
|
|
}
|
|
|
|
CrashTextNumber :: (text: *CrashText, value: s64) {
|
|
digits: [24]u8
|
|
CrashTextAdd(text, FormatInteger(value, digits[:]))
|
|
}
|
|
|
|
// The text in `bytes`, a CrashText's bytes with its length.
|
|
CrashTextView :: (bytes: []u8, length: s64) -> string {
|
|
return TextFromBytes(bytes[0:length])
|
|
}
|
|
|
|
environment_name: [64]u8;
|
|
|
|
EnvironmentText :: (name: string) -> string {
|
|
if !AppCopyText(name, environment_name[:]) {
|
|
return ""
|
|
}
|
|
value := CrashGetEnv(*environment_name[0])
|
|
if value == null {
|
|
return ""
|
|
}
|
|
return TextFromCString(value)
|
|
}
|
|
|
|
// `<data root>/<scenario>`, or a file inside it when `name` is not empty.
|
|
CrashPath :: (output: *CrashText, scenario: string, name: string) {
|
|
output.length = 0
|
|
CrashTextAdd(output, data_root())
|
|
CrashTextAdd(output, "/")
|
|
CrashTextAdd(output, scenario)
|
|
if name.count > 0 {
|
|
CrashTextAdd(output, "/")
|
|
CrashTextAdd(output, name)
|
|
}
|
|
}
|
|
|
|
CrashAcksPath :: (output: *CrashText, scenario: string) {
|
|
CrashPath(output, scenario, "")
|
|
CrashTextAdd(output, ".acks")
|
|
}
|
|
|
|
CrashOpenStorage :: (scenario: string) -> bool {
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
if storage_ensure_dir(CrashTextView(root.bytes[:], root.length)) == 0 {
|
|
return false
|
|
}
|
|
return storage_init(CrashTextView(root.bytes[:], root.length)) != 0
|
|
}
|
|
|
|
// The acknowledged numbers. They count up by one from 1, because a writer
|
|
// continues after the last number acknowledged before it. Only lines that
|
|
// end in a newline count; a writer writes after the last of them.
|
|
CrashAcks :: struct {
|
|
last: s64
|
|
valid: bool
|
|
end: s64
|
|
}
|
|
|
|
CrashReadAcks :: (scenario: string) -> CrashAcks {
|
|
result: CrashAcks
|
|
result.valid = true
|
|
path: CrashText
|
|
CrashAcksPath(*path, scenario)
|
|
contents := PackageRead(CrashTextView(path.bytes[:], path.length), 1 << 26)
|
|
defer {
|
|
PackageFree(contents)
|
|
}
|
|
value: s64 = 0
|
|
digits: s64 = 0
|
|
index: s64 = 0
|
|
while index < contents.count {
|
|
byte := contents[index]
|
|
if byte == cast(u8)10 {
|
|
if digits == 0 || value != result.last + 1 {
|
|
result.valid = false
|
|
}
|
|
result.last = value
|
|
result.end = index + 1
|
|
value = 0
|
|
digits = 0
|
|
} else if byte >= cast(u8)48 && byte <= cast(u8)57 {
|
|
value = value * 10 + cast(s64)(byte - cast(u8)48)
|
|
digits += 1
|
|
} else {
|
|
result.valid = false
|
|
}
|
|
index += 1
|
|
}
|
|
return result
|
|
}
|
|
|
|
CrashAckFile :: struct {
|
|
file: FileHandle
|
|
offset: s64
|
|
}
|
|
|
|
CrashOpenAcks :: (scenario: string, acks: CrashAcks, output: *CrashAckFile) -> bool {
|
|
path: CrashText
|
|
CrashAcksPath(*path, scenario)
|
|
output.file = OpenUpdate(CrashTextView(path.bytes[:], path.length))
|
|
output.offset = acks.end
|
|
return output.file.valid
|
|
}
|
|
|
|
// One write per line, so a kill never leaves part of a number behind.
|
|
CrashAcknowledge :: (acks: *CrashAckFile, value: s64) -> bool {
|
|
line: CrashText
|
|
CrashTextNumber(*line, value)
|
|
CrashTextAdd(*line, "\n")
|
|
written := WriteTextAt(acks.file, CrashTextView(line.bytes[:], line.length), acks.offset)
|
|
if written != line.length {
|
|
return false
|
|
}
|
|
acks.offset += written
|
|
return true
|
|
}
|
|
|
|
writer_started_at: s64;
|
|
|
|
WriterHasTime :: () -> bool {
|
|
return CrashTime(null) - writer_started_at < WriterSeconds
|
|
}
|
|
|
|
// A whole SQLite file passes PRAGMA integrity_check.
|
|
CrashIntegrityOk :: (path: *u8) -> bool {
|
|
database: *void = null
|
|
statement: *void = null
|
|
if CrashSqliteOpen(path, *database) != 0 || database == null {
|
|
if database != null {
|
|
unused CrashSqliteClose(database)
|
|
}
|
|
return false
|
|
}
|
|
query: CrashText
|
|
CrashTextAdd(*query, "PRAGMA integrity_check")
|
|
healthy := false
|
|
if CrashSqlitePrepare(database, *query.bytes[0], -1, *statement, null) == 0 &&
|
|
CrashSqliteStep(statement) == 100 {
|
|
answer := CrashSqliteColumnText(statement, 0)
|
|
healthy = answer != null && TextFromCString(answer) == "ok" &&
|
|
CrashSqliteStep(statement) == 101
|
|
}
|
|
if statement != null {
|
|
unused CrashSqliteFinalize(statement)
|
|
}
|
|
unused CrashSqliteClose(database)
|
|
return healthy
|
|
}
|
|
|
|
// The first column of the first row of `sql` in any SQLite file, or -1.
|
|
CrashCount :: (path: *u8, sql: *CrashText) -> s64 {
|
|
database: *void = null
|
|
statement: *void = null
|
|
count: s64 = -1
|
|
if CrashSqliteOpen(path, *database) != 0 || database == null {
|
|
if database != null {
|
|
unused CrashSqliteClose(database)
|
|
}
|
|
return count
|
|
}
|
|
if CrashSqlitePrepare(database, *sql.bytes[0], -1, *statement, null) == 0 &&
|
|
CrashSqliteStep(statement) == 100 {
|
|
count = cast(s64)CrashSqliteColumnInt(statement, 0)
|
|
}
|
|
if statement != null {
|
|
unused CrashSqliteFinalize(statement)
|
|
}
|
|
unused CrashSqliteClose(database)
|
|
return count
|
|
}
|
|
|
|
// Live sessions whose single round lasts `first` to `last` seconds, counted
|
|
// once per length.
|
|
CrashSessionsQuery :: (output: *CrashText, first: s64, last: s64) {
|
|
output.length = 0
|
|
CrashTextAdd(output,
|
|
"SELECT COUNT(DISTINCT r.seconds) FROM session_rounds r JOIN sessions s ON s.id=r.session_id WHERE s.deleted_at=0 AND r.seconds BETWEEN ")
|
|
CrashTextNumber(output, first)
|
|
CrashTextAdd(output, " AND ")
|
|
CrashTextNumber(output, last)
|
|
}
|
|
|
|
CrashSaveSession :: (seconds: s64) -> bool {
|
|
rounds: [1]s32
|
|
rounds[0] = cast(s32)seconds
|
|
return storage_save_session(*rounds[0], 1, null, 0) != 0
|
|
}
|
|
|
|
// Payload `number` is written to file number % PayloadFileCount. Its size
|
|
// and bytes depend on the number, so a mix of two versions never matches.
|
|
PayloadPath :: (output: *CrashText, scenario: string, file_index: s64) {
|
|
CrashPath(output, scenario, "packages/payload-")
|
|
letter: [1]u8
|
|
letter[0] = cast(u8)(97 + file_index)
|
|
CrashTextAdd(output, TextFromBytes(letter[:]))
|
|
CrashTextAdd(output, ".bin")
|
|
}
|
|
|
|
PayloadMake :: (number: s64) -> string {
|
|
size := 98304 + (number % 5) * 49152
|
|
data := CrashAllocate(cast(usize)size)
|
|
if data == null {
|
|
return ""
|
|
}
|
|
header: CrashText
|
|
CrashTextAdd(*header, "payload ")
|
|
CrashTextNumber(*header, number)
|
|
CrashTextAdd(*header, "\n")
|
|
index: s64 = 0
|
|
while index < size {
|
|
if index < header.length {
|
|
data[index] = header.bytes[index]
|
|
} else {
|
|
data[index] = cast(u8)(33 + (number * 131 + index * 7) % 90)
|
|
}
|
|
index += 1
|
|
}
|
|
return TextFromBytes(BytesFromPointer(data, cast(u64)size))
|
|
}
|
|
|
|
PayloadMatches :: (contents: string, number: s64) -> bool {
|
|
if number <= 0 {
|
|
return false
|
|
}
|
|
expected := PayloadMake(number)
|
|
defer {
|
|
PackageFree(expected)
|
|
}
|
|
return expected.count > 0 && contents == expected
|
|
}
|
|
|
|
DraftMake :: (output: *CrashText, number: s64) {
|
|
output.length = 0
|
|
CrashTextAdd(output, "{\"practice\":\"breathing\",\"round\":")
|
|
CrashTextNumber(output, number)
|
|
CrashTextAdd(output, ",\"elapsed_seconds\":")
|
|
CrashTextNumber(output, number * 7)
|
|
CrashTextAdd(output, ",\"note\":\"")
|
|
index: s64 = 0
|
|
while index < 1500 {
|
|
letter: [1]u8
|
|
letter[0] = cast(u8)(97 + (number + index) % 26)
|
|
CrashTextAdd(output, TextFromBytes(letter[:]))
|
|
index += 1
|
|
}
|
|
CrashTextAdd(output, "\"}")
|
|
}
|
|
|
|
WriteSessions :: (acks: *CrashAckFile, first: s64) -> s32 {
|
|
number := first
|
|
while WriterHasTime() {
|
|
if !CrashSaveSession(number) {
|
|
return 21
|
|
}
|
|
if !CrashAcknowledge(acks, number) {
|
|
return 22
|
|
}
|
|
number += 1
|
|
}
|
|
return 23
|
|
}
|
|
|
|
WritePayloads :: (scenario: string, acks: *CrashAckFile, first: s64) -> s32 {
|
|
folder: CrashText
|
|
CrashPath(*folder, scenario, "packages")
|
|
if storage_ensure_dir(CrashTextView(folder.bytes[:], folder.length)) == 0 {
|
|
return 31
|
|
}
|
|
number := first
|
|
while WriterHasTime() {
|
|
path: CrashText
|
|
PayloadPath(*path, scenario, number % PayloadFileCount)
|
|
contents := PayloadMake(number)
|
|
if contents.count == 0 {
|
|
return 32
|
|
}
|
|
written := PackageWrite(CrashTextView(path.bytes[:], path.length), contents)
|
|
PackageFree(contents)
|
|
if !written {
|
|
return 33
|
|
}
|
|
if !CrashAcknowledge(acks, number) {
|
|
return 34
|
|
}
|
|
number += 1
|
|
}
|
|
return 35
|
|
}
|
|
|
|
// Every number saves a session, then starts storage again with the copy
|
|
// due, so the kill can land anywhere in writing the copy.
|
|
WriteCopies :: (scenario: string, acks: *CrashAckFile, first: s64) -> s32 {
|
|
// Some volume, so that writing a copy takes long enough to interrupt.
|
|
filler := exec_sql("WITH RECURSIVE filler(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM filler WHERE x<3000) INSERT OR IGNORE INTO meta(key,value) SELECT 'crash-test-filler-'||x,hex(randomblob(256)) FROM filler")
|
|
if filler == 0 {
|
|
return 41
|
|
}
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
number := first
|
|
while WriterHasTime() {
|
|
if !CrashSaveSession(number) {
|
|
return 42
|
|
}
|
|
storage_set_meta_int64(LastGoodCopyKey, 0)
|
|
storage_close()
|
|
if storage_init(CrashTextView(root.bytes[:], root.length)) == 0 {
|
|
return 43
|
|
}
|
|
if !PathExists(CrashTextView(copy.bytes[:], copy.length)) {
|
|
return 44
|
|
}
|
|
if !CrashAcknowledge(acks, number) {
|
|
return 45
|
|
}
|
|
number += 1
|
|
}
|
|
return 46
|
|
}
|
|
|
|
WriteDrafts :: (acks: *CrashAckFile, first: s64) -> s32 {
|
|
number := first
|
|
while WriterHasTime() {
|
|
draft: CrashText
|
|
DraftMake(*draft, number)
|
|
if storage_practice_draft_save(CrashTextView(draft.bytes[:], draft.length)) == 0 {
|
|
return 51
|
|
}
|
|
if !CrashAcknowledge(acks, number) {
|
|
return 52
|
|
}
|
|
number += 1
|
|
}
|
|
return 53
|
|
}
|
|
|
|
CrashWriter :: (scenario: string) -> s32 {
|
|
writer_started_at = CrashTime(null)
|
|
acks := CrashReadAcks(scenario)
|
|
if !acks.valid {
|
|
return 11
|
|
}
|
|
if !CrashOpenStorage(scenario) {
|
|
return 12
|
|
}
|
|
acknowledgements: CrashAckFile
|
|
if !CrashOpenAcks(scenario, acks, *acknowledgements) {
|
|
return 13
|
|
}
|
|
first := acks.last + 1
|
|
if scenario == "sessions" {
|
|
return WriteSessions(*acknowledgements, first)
|
|
}
|
|
if scenario == "payloads" {
|
|
return WritePayloads(scenario, *acknowledgements, first)
|
|
}
|
|
if scenario == "copies" {
|
|
return WriteCopies(scenario, *acknowledgements, first)
|
|
}
|
|
if scenario == "drafts" {
|
|
return WriteDrafts(*acknowledgements, first)
|
|
}
|
|
return 14
|
|
}
|
|
|
|
CheckSessions :: (last: s64) {
|
|
query: CrashText
|
|
CrashSessionsQuery(*query, 1, last)
|
|
check_int("every acknowledged session is present",
|
|
read_db_count(CrashTextView(query.bytes[:], query.length)), cast(s32)last)
|
|
CrashSessionsQuery(*query, last + 1, last + 1000000)
|
|
check_true("only the session in flight may be saved without its acknowledgement",
|
|
read_db_count(CrashTextView(query.bytes[:], query.length)) <= 1)
|
|
check_int("no session is left without its rounds",
|
|
read_db_count("SELECT COUNT(*) FROM sessions s WHERE s.deleted_at=0 AND NOT EXISTS (SELECT 1 FROM session_rounds r WHERE r.session_id=s.id)"),
|
|
0)
|
|
// Nothing syncs in this test, so every saved session keeps its entry.
|
|
check_int("no session is left without its sync entry",
|
|
read_db_count("SELECT COUNT(*) FROM sessions s WHERE s.deleted_at=0 AND NOT EXISTS (SELECT 1 FROM sync_outbox o WHERE o.entity_type='session' AND o.entity_id=s.id)"),
|
|
0)
|
|
}
|
|
|
|
// Before storage starts, which may write a new copy: the copy on disk is
|
|
// whole and holds everything acknowledged, since each acknowledgement
|
|
// follows a copy.
|
|
CheckCopyBeforeStart :: (scenario: string, last: s64) {
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
if !PathExists(CrashTextView(copy.bytes[:], copy.length)) {
|
|
check_true("a last good copy exists once one was acknowledged", last == 0)
|
|
return
|
|
}
|
|
check_true("the last good copy passes integrity_check", CrashIntegrityOk(*copy.bytes[0]))
|
|
if last > 0 {
|
|
query: CrashText
|
|
CrashSessionsQuery(*query, 1, last)
|
|
check_true("the last good copy holds every acknowledged session",
|
|
CrashCount(*copy.bytes[0], *query) == last)
|
|
}
|
|
}
|
|
|
|
CheckPayloads :: (scenario: string, last: s64) {
|
|
file_index: s64 = 0
|
|
while file_index < PayloadFileCount {
|
|
newest := last
|
|
while newest >= 1 && newest % PayloadFileCount != file_index {
|
|
newest -= 1
|
|
}
|
|
in_flight := last + 1
|
|
path: CrashText
|
|
PayloadPath(*path, scenario, file_index)
|
|
if !PathExists(CrashTextView(path.bytes[:], path.length)) {
|
|
check_true("a payload file is missing only before its first write", newest < 1)
|
|
} else {
|
|
contents := PackageRead(CrashTextView(path.bytes[:], path.length), 1 << 24)
|
|
whole := PayloadMatches(contents, newest)
|
|
if !whole && in_flight % PayloadFileCount == file_index {
|
|
whole = PayloadMatches(contents, in_flight)
|
|
}
|
|
check_true("a payload file holds a whole acknowledged or in-flight version", whole)
|
|
PackageFree(contents)
|
|
}
|
|
file_index += 1
|
|
}
|
|
}
|
|
|
|
draft_bytes: [8192]u8;
|
|
|
|
// The draft is the last acknowledged one, or the next one when its save
|
|
// committed just before the kill.
|
|
CheckDraft :: (last: s64) {
|
|
length := storage_practice_draft_load(draft_bytes[:])
|
|
if length == 0 {
|
|
check_true("no draft only before the first save", last == 0)
|
|
return
|
|
}
|
|
check_true("the draft loads", length > 0)
|
|
if length <= 0 {
|
|
return
|
|
}
|
|
loaded := TextFromBytes(draft_bytes[0:length])
|
|
acknowledged: CrashText
|
|
DraftMake(*acknowledged, last)
|
|
next: CrashText
|
|
DraftMake(*next, last + 1)
|
|
whole := (last > 0 && loaded == CrashTextView(acknowledged.bytes[:], acknowledged.length)) || loaded == CrashTextView(next.bytes[:], next.length)
|
|
check_true("the draft is the acknowledged one or the one in flight", whole)
|
|
}
|
|
|
|
CrashChecker :: (scenario: string) -> s32 {
|
|
acks := CrashReadAcks(scenario)
|
|
check_true("acknowledgements count up by one", acks.valid)
|
|
database: CrashText
|
|
CrashPath(*database, scenario, "inbe.db")
|
|
if PathExists(CrashTextView(database.bytes[:], database.length)) {
|
|
check_true("the database passes integrity_check after the kill",
|
|
CrashIntegrityOk(*database.bytes[0]))
|
|
} else {
|
|
check_true("a database exists once a write was acknowledged", acks.last == 0)
|
|
}
|
|
if scenario == "copies" {
|
|
CheckCopyBeforeStart(scenario, acks.last)
|
|
}
|
|
if !CrashOpenStorage(scenario) {
|
|
check_true("storage opens after the kill", false)
|
|
return failures + 1
|
|
}
|
|
check_int("the database is not reported damaged", storage_database_damaged(), 0)
|
|
if scenario == "sessions" || scenario == "copies" {
|
|
CheckSessions(acks.last)
|
|
}
|
|
if scenario == "payloads" {
|
|
CheckPayloads(scenario, acks.last)
|
|
}
|
|
if scenario == "drafts" {
|
|
CheckDraft(acks.last)
|
|
}
|
|
storage_close()
|
|
return failures
|
|
}
|
|
|
|
// The whole file, released with PackageFree; empty when it is missing.
|
|
FileContents :: (path: *CrashText) -> string {
|
|
return PackageRead(CrashTextView(path.bytes[:], path.length), 1 << 26)
|
|
}
|
|
|
|
WriteFileText :: (path: *CrashText, text: string) -> bool {
|
|
file := OpenReplace(CrashTextView(path.bytes[:], path.length))
|
|
if !file.valid {
|
|
return false
|
|
}
|
|
written := WriteTextAt(file, text, 0) == text.count
|
|
closed := CloseFile(file)
|
|
return written && closed
|
|
}
|
|
|
|
// Overwrites `count` bytes at `offset` with `byte`.
|
|
DamageFile :: (path: *CrashText, offset: s64, count: s64, byte: u8) -> bool {
|
|
file := OpenUpdate(CrashTextView(path.bytes[:], path.length))
|
|
if !file.valid {
|
|
return false
|
|
}
|
|
bytes: [4096]u8
|
|
index: s64 = 0
|
|
while index < count && index < 4096 {
|
|
bytes[index] = byte
|
|
index += 1
|
|
}
|
|
written := WriteAt(file, bytes[0:index], offset) == index
|
|
closed := CloseFile(file)
|
|
return written && closed
|
|
}
|
|
|
|
CopySessionCount :: (scenario: string, first: s64, last: s64) -> s64 {
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
query: CrashText
|
|
CrashSessionsQuery(*query, first, last)
|
|
return CrashCount(*copy.bytes[0], *query)
|
|
}
|
|
|
|
LiveSessionCount :: (first: s64, last: s64) -> s32 {
|
|
query: CrashText
|
|
CrashSessionsQuery(*query, first, last)
|
|
return read_db_count(CrashTextView(query.bytes[:], query.length))
|
|
}
|
|
|
|
// The `<database>.damaged-<unixtime>` name a restore between `earliest` and
|
|
// `latest` chose, or an empty path.
|
|
FindDamagedPath :: (output: *CrashText, scenario: string, earliest: s64, latest: s64) {
|
|
stamp := earliest
|
|
while stamp <= latest + 1 {
|
|
CrashPath(output, scenario, "inbe.db.damaged-")
|
|
CrashTextNumber(output, stamp)
|
|
if PathExists(CrashTextView(output.bytes[:], output.length)) {
|
|
return
|
|
}
|
|
stamp += 1
|
|
}
|
|
output.length = 0
|
|
}
|
|
|
|
FileHolds :: (path: *CrashText, expected: string) -> bool {
|
|
contents := FileContents(path)
|
|
defer {
|
|
PackageFree(contents)
|
|
}
|
|
return contents.count > 0 && contents == expected
|
|
}
|
|
|
|
// A day passes, so the next start writes a new copy.
|
|
AgeLastGoodCopy :: () {
|
|
storage_set_meta_int64(LastGoodCopyKey, CrashTime(null) - DayInSeconds - 10)
|
|
}
|
|
|
|
CheckCopyGate :: (scenario: string) {
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
check_true("the first start writes a last good copy", PathExists(CrashTextView(copy.bytes[:], copy.length)))
|
|
check_true("the copy time is kept", storage_meta_int64(LastGoodCopyKey, 0) > 0)
|
|
check_true("save sessions before the copy", CrashSaveSession(1001) && CrashSaveSession(1002) &&
|
|
CrashSaveSession(1003))
|
|
storage_close()
|
|
check_int("start again within a day", storage_init(CrashTextView(root.bytes[:], root.length)), 1)
|
|
check_true("a copy less than a day old is kept as it was",
|
|
CopySessionCount(scenario, 1001, 1003) == 0)
|
|
AgeLastGoodCopy()
|
|
storage_close()
|
|
check_int("start again a day later", storage_init(CrashTextView(root.bytes[:], root.length)), 1)
|
|
check_true("a day-old copy is renewed", CopySessionCount(scenario, 1001, 1003) == 3)
|
|
storage_close()
|
|
// A missing copy is written again whatever its time says.
|
|
unused RemoveFile(CrashTextView(copy.bytes[:], copy.length))
|
|
check_int("start again without the copy", storage_init(CrashTextView(root.bytes[:], root.length)), 1)
|
|
check_true("a missing copy is written again", CopySessionCount(scenario, 1001, 1003) == 3)
|
|
}
|
|
|
|
// A damaged header beside a -wal of committed sessions, as a crash leaves
|
|
// one: storage does not open and changes no byte of either file. The last
|
|
// good copy then replaces the database, the damaged files are kept, and the
|
|
// old -wal is not replayed into the restored database.
|
|
CheckRestoreDamagedHeader :: (scenario: string) {
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
database: CrashText
|
|
CrashPath(*database, scenario, "inbe.db")
|
|
wal: CrashText
|
|
CrashPath(*wal, scenario, "inbe.db-wal")
|
|
check_true("save sessions after the copy", CrashSaveSession(2004) && CrashSaveSession(2005))
|
|
committed_wal := FileContents(*wal)
|
|
check_true("the open database has a -wal", committed_wal.count > 0)
|
|
storage_close()
|
|
check_true("damage the database header", DamageFile(*database, 0, 16, cast(u8)88))
|
|
check_true("put the crash's -wal back", WriteFileText(*wal, committed_wal))
|
|
damaged := FileContents(*database)
|
|
check_true("read the damaged database", damaged.count > 0)
|
|
|
|
check_int("a damaged database does not open",
|
|
storage_init(CrashTextView(root.bytes[:], root.length)), 0)
|
|
check_int("the database is reported damaged", storage_database_damaged(), 1)
|
|
check_true("a failed open leaves the database byte for byte", FileHolds(*database, damaged))
|
|
check_true("a failed open leaves the -wal byte for byte", FileHolds(*wal, committed_wal))
|
|
check_int("opening it again fails the same way",
|
|
storage_init(CrashTextView(root.bytes[:], root.length)), 0)
|
|
check_true("the second open leaves it too", FileHolds(*database, damaged))
|
|
|
|
before := CrashTime(null)
|
|
check_int("restore the last good copy", storage_restore_last_good(), 1)
|
|
after := CrashTime(null)
|
|
check_int("a restored database is not reported damaged", storage_database_damaged(), 0)
|
|
check_int("the restored data matches the copy", LiveSessionCount(1001, 1003), 3)
|
|
check_int("the old -wal is not replayed into the restored database",
|
|
LiveSessionCount(2004, 2005), 0)
|
|
|
|
kept: CrashText
|
|
FindDamagedPath(*kept, scenario, before, after)
|
|
check_true("the damaged database is kept", kept.length > 0)
|
|
if kept.length > 0 {
|
|
check_true("the kept database is byte for byte the damaged one", FileHolds(*kept, damaged))
|
|
kept_wal: CrashText = kept
|
|
CrashTextAdd(*kept_wal, "-wal")
|
|
check_true("the damaged -wal is kept byte for byte", FileHolds(*kept_wal, committed_wal))
|
|
}
|
|
check_true("the damaged -wal is not beside the restored database",
|
|
!FileHolds(*wal, committed_wal))
|
|
check_true("the restored database passes integrity_check", CrashIntegrityOk(*database.bytes[0]))
|
|
PackageFree(damaged)
|
|
PackageFree(committed_wal)
|
|
storage_close()
|
|
}
|
|
|
|
// Damage inside the file that leaves the header readable is found too.
|
|
CheckRestoreDamagedPage :: () {
|
|
scenario := "restore-page"
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
remove_tree(CrashTextView(root.bytes[:], root.length))
|
|
database: CrashText
|
|
CrashPath(*database, scenario, "inbe.db")
|
|
check_true("open a database to damage inside", CrashOpenStorage(scenario))
|
|
number: s64 = 3001
|
|
while number <= 3200 {
|
|
unused CrashSaveSession(number)
|
|
number += 1
|
|
}
|
|
AgeLastGoodCopy()
|
|
storage_close()
|
|
check_true("copy the sessions", CrashOpenStorage(scenario))
|
|
storage_close()
|
|
check_true("damage a page inside the database", DamageFile(*database, 4096 * 3, 4096, cast(u8)255))
|
|
damaged := FileContents(*database)
|
|
check_int("a database with a damaged page does not open", storage_init(CrashTextView(root.bytes[:], root.length)), 0)
|
|
check_int("a damaged page is reported", storage_database_damaged(), 1)
|
|
check_true("a damaged page leaves the database byte for byte", FileHolds(*database, damaged))
|
|
check_int("restore after a damaged page", storage_restore_last_good(), 1)
|
|
check_int("all copied sessions are back", LiveSessionCount(3001, 3200), 200)
|
|
PackageFree(damaged)
|
|
storage_close()
|
|
}
|
|
|
|
// Without a valid copy, a restore changes nothing.
|
|
CheckRestoreWithoutCopy :: () {
|
|
scenario := "restore-none"
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
remove_tree(CrashTextView(root.bytes[:], root.length))
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
check_true("open a database without a copy", CrashOpenStorage(scenario))
|
|
check_true("save a session to keep", CrashSaveSession(4001))
|
|
unused RemoveFile(CrashTextView(copy.bytes[:], copy.length))
|
|
check_int("no copy: nothing to restore", storage_restore_last_good(), 0)
|
|
check_true("storage stays open without a copy", storage_db_handle() != null)
|
|
check_int("no copy: the data stays", LiveSessionCount(4001, 4001), 1)
|
|
check_true("write an invalid copy", WriteFileText(*copy, "not a database at all"))
|
|
check_int("an invalid copy is not restored", storage_restore_last_good(), 0)
|
|
check_int("an invalid copy: the data stays", LiveSessionCount(4001, 4001), 1)
|
|
check_true("an invalid copy is left as it was", FileHolds(*copy, "not a database at all"))
|
|
kept: CrashText
|
|
now := CrashTime(null)
|
|
FindDamagedPath(*kept, scenario, now - 60, now)
|
|
check_true("nothing is moved aside without a copy", kept.length == 0)
|
|
storage_close()
|
|
}
|
|
|
|
// A database that went missing: the new, empty one does not replace the
|
|
// last good copy, which is kept beside it with everything it held.
|
|
CheckMissingDatabaseKeepsCopy :: () {
|
|
scenario := "restore-missing"
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
remove_tree(CrashTextView(root.bytes[:], root.length))
|
|
database: CrashText
|
|
CrashPath(*database, scenario, "inbe.db")
|
|
copy: CrashText
|
|
CrashPath(*copy, scenario, "inbe.db.last-good")
|
|
check_true("open a database that will go missing", CrashOpenStorage(scenario))
|
|
check_true("save a session before it goes missing", CrashSaveSession(5001))
|
|
AgeLastGoodCopy()
|
|
storage_close()
|
|
check_true("copy the session", CrashOpenStorage(scenario))
|
|
storage_close()
|
|
check_true("the database goes missing", RemoveFile(CrashTextView(database.bytes[:], database.length)))
|
|
before := CrashTime(null)
|
|
check_true("a new database opens", CrashOpenStorage(scenario))
|
|
after := CrashTime(null)
|
|
check_int("the new database is empty", LiveSessionCount(5001, 5001), 0)
|
|
check_true("the new database has its own copy", PathExists(CrashTextView(copy.bytes[:], copy.length)))
|
|
earlier: CrashText
|
|
stamp := before
|
|
while stamp <= after + 1 {
|
|
CrashPath(*earlier, scenario, "inbe.db.last-good.before-")
|
|
CrashTextNumber(*earlier, stamp)
|
|
if PathExists(CrashTextView(earlier.bytes[:], earlier.length)) {
|
|
break
|
|
}
|
|
earlier.length = 0
|
|
stamp += 1
|
|
}
|
|
check_true("the earlier copy is kept", earlier.length > 0)
|
|
if earlier.length > 0 {
|
|
query: CrashText
|
|
CrashSessionsQuery(*query, 5001, 5001)
|
|
check_true("the earlier copy still holds its session", CrashCount(*earlier.bytes[0], *query) == 1)
|
|
}
|
|
storage_close()
|
|
}
|
|
|
|
CrashRestore :: () -> s32 {
|
|
scenario := "restore"
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
remove_tree(CrashTextView(root.bytes[:], root.length))
|
|
if !CrashOpenStorage(scenario) {
|
|
check_true("open storage for the restore checks", false)
|
|
return failures + 1
|
|
}
|
|
CheckCopyGate(scenario)
|
|
CheckRestoreDamagedHeader(scenario)
|
|
CheckRestoreDamagedPage()
|
|
CheckRestoreWithoutCopy()
|
|
CheckMissingDatabaseKeepsCopy()
|
|
CheckPracticeRecoveryDecision()
|
|
return failures
|
|
}
|
|
|
|
#program_export
|
|
Check :: () -> s32 {
|
|
if !contains(data_root(), CrashDataFolder) {
|
|
return 100
|
|
}
|
|
install_english_catalog()
|
|
role := EnvironmentText("CRASH_ROLE")
|
|
scenario := EnvironmentText("CRASH_SCENARIO")
|
|
known := scenario == "sessions" || scenario == "payloads" || scenario == "copies" ||
|
|
scenario == "drafts"
|
|
if role == "restore" {
|
|
return CrashRestore()
|
|
}
|
|
if !known {
|
|
return 101
|
|
}
|
|
if role == "writer" {
|
|
return CrashWriter(scenario)
|
|
}
|
|
if role == "checker" {
|
|
return CrashChecker(scenario)
|
|
}
|
|
return 102
|
|
}
|
|
|
|
CheckPracticeRecoveryDecision :: () {
|
|
storage_close()
|
|
scenario := "recovery-decision"
|
|
root: CrashText
|
|
CrashPath(*root, scenario, "")
|
|
remove_tree(CrashTextView(root.bytes[:], root.length))
|
|
check_true("open recovery decision storage", CrashOpenStorage(scenario))
|
|
draft := PracticeRecoveryDraft.{.started_at = 1794800000, .activity = 0,
|
|
.topic = 2, .count = 2}
|
|
draft.rounds[0] = 24
|
|
draft.rounds[1] = 35
|
|
buffer: [1024]u8
|
|
check_true("encode unfinished rounds", PracticeDraftEncode(draft, buffer[:]))
|
|
json := AppTextFromCString(*buffer[0], 1024)
|
|
parsed: PracticeRecoveryDraft
|
|
check_true("parse a checkpoint", PracticeDraftParse(json, *parsed))
|
|
check_true("checkpoint keeps original start", parsed.started_at == draft.started_at)
|
|
check_int("checkpoint keeps completed rounds", parsed.rounds[1], 35)
|
|
before := storage_session_count()
|
|
check_int("write draft for Discard", storage_practice_draft_save(json), 1)
|
|
check_true("Discard succeeds", PracticeDraftDecide(false))
|
|
check_int("Discard stores no session", storage_session_count(), before)
|
|
check_true("Discard removes draft", storage_practice_draft_load(buffer[:]) == 0)
|
|
check_int("write draft for Save", storage_practice_draft_save(json), 1)
|
|
unused exec_sql("CREATE TEMP TRIGGER fail_practice_recovery BEFORE INSERT ON sessions BEGIN SELECT RAISE(ABORT, 'test failure'); END")
|
|
check_true("failed Save reports failure", !PracticeDraftDecide(true))
|
|
check_true("failed Save retains draft", storage_practice_draft_load(buffer[:]) > 0)
|
|
check_int("failed Save stores no session", storage_session_count(), before)
|
|
unused exec_sql("DROP TRIGGER fail_practice_recovery")
|
|
check_true("Save succeeds", PracticeDraftDecide(true))
|
|
check_int("Save adds one session", storage_session_count(), before + 1)
|
|
check_int("Save keeps start, activity and topic", read_db_count("SELECT COUNT(*) FROM sessions WHERE started_at=1794800000 AND activity=0 AND topic=2"), 1)
|
|
check_int("Save keeps both completed rounds", read_db_count("SELECT COUNT(*) FROM session_rounds WHERE seconds IN (24,35)"), 2)
|
|
check_true("Save removes draft", storage_practice_draft_load(buffer[:]) == 0)
|
|
check_true("second Save cannot duplicate", !PracticeDraftDecide(true))
|
|
check_int("no duplicate session", storage_session_count(), before + 1)
|
|
check_true("malformed checkpoint rejected", !PracticeDraftParse("{\"schema\":1}", *parsed))
|
|
check_int("refresh copy after deliberate deletion", storage_refresh_last_good(), 1)
|
|
unused storage_delete_all_sessions()
|
|
check_int("restore refreshed copy", storage_restore_last_good(), 1)
|
|
check_int("restore cannot resurrect deleted sessions", storage_session_count(), 0)
|
|
CheckPracticeRecoveryRepeatedHour()
|
|
CheckPracticeCheckpointTriggers()
|
|
}
|
|
|
|
CheckPracticeRecoveryRepeatedHour :: () {
|
|
previous_timezone: [256]u8
|
|
name: [3]u8
|
|
timezone: [64]u8
|
|
previous := EnvironmentText("TZ")
|
|
had_timezone := CrashGetEnv(*environment_name[0]) != null
|
|
check_true("remember the test process timezone", AppCopyText(previous, previous_timezone[:]))
|
|
unused AppCopyText("TZ", name[:])
|
|
unused AppCopyText("EST5EDT,M3.2.0,M11.1.0", timezone[:])
|
|
check_int("select a repeated local hour", CrashSetEnvironment(*name[0], *timezone[0], 1), 0)
|
|
CrashTimezoneChanged()
|
|
defer {
|
|
if had_timezone {
|
|
unused CrashSetEnvironment(*name[0], *previous_timezone[0], 1)
|
|
} else {
|
|
unused CrashUnsetEnvironment(*name[0])
|
|
}
|
|
CrashTimezoneChanged()
|
|
}
|
|
before := storage_session_count()
|
|
draft := PracticeRecoveryDraft.{.started_at = 1793511000, .activity = 1, .count = 1}
|
|
draft.rounds[0] = 40
|
|
buffer: [1024]u8
|
|
for occurrence: 0..1 {
|
|
// Both starts read 01:30 locally on 2026-11-01, one hour apart.
|
|
draft.started_at = 1793511000 + occurrence * 3600
|
|
check_true("encode the repeated hour", PracticeDraftEncode(draft, buffer[:]))
|
|
check_int("checkpoint the repeated hour", storage_practice_draft_save(AppTextFromCString(*buffer[0], 1024)), 1)
|
|
check_true("save the repeated hour", PracticeDraftDecide(true))
|
|
}
|
|
check_int("both occurrences remain separate sessions", storage_session_count(), before + 2)
|
|
check_int("recovery preserves exact starts across the repeated hour",
|
|
read_db_count("SELECT COUNT(*) FROM sessions WHERE started_at IN (1793511000,1793514600) AND local_date=20261101"), 2)
|
|
}
|
|
|
|
CheckPracticeCheckpointTriggers :: () {
|
|
buffer: [1024]u8
|
|
parsed: PracticeRecoveryDraft
|
|
PracticeDraftBegin(1)
|
|
PracticeDraftElapsed(1, 10, 0, false)
|
|
length := storage_practice_draft_load(buffer[:])
|
|
check_true("start records an empty checkpoint", length > 0 &&
|
|
PracticeDraftParse(TextFromBytes(buffer[:length]), *parsed) && parsed.count == 0)
|
|
PracticeDraftPause()
|
|
length = storage_practice_draft_load(buffer[:])
|
|
check_true("pause keeps the latest elapsed time", length > 0 &&
|
|
PracticeDraftParse(TextFromBytes(buffer[:length]), *parsed) && parsed.rounds[0] == 10)
|
|
practice_draft_saved_at = now_seconds() - 30
|
|
PracticeDraftElapsed(1, 40, 0, false)
|
|
length = storage_practice_draft_load(buffer[:])
|
|
check_true("thirty seconds checkpoints elapsed time", length > 0 &&
|
|
PracticeDraftParse(TextFromBytes(buffer[:length]), *parsed) && parsed.rounds[0] == 40)
|
|
PracticeDraftElapsed(1, 41, 1, false)
|
|
length = storage_practice_draft_load(buffer[:])
|
|
check_true("round boundary checkpoints immediately", length > 0 &&
|
|
PracticeDraftParse(TextFromBytes(buffer[:length]), *parsed) && parsed.rounds[0] == 41)
|
|
PracticeDraftEnd()
|
|
check_true("normal completion clears the draft", storage_practice_draft_load(buffer[:]) == 0)
|
|
}
|