Size the text index to each language

The index had a fixed number of entries: past it a language lost the
index and every text read the whole language file. It is now allocated
for the keys each language actually has, with its slots at most half
full, so a lookup is one or two probes and no language can outgrow it.
Replaces the 8192-entry limit and its capacity test.
This commit is contained in:
waozi committed 2026-10-19 01:33:12 -03:00
1 parent 374d01957c
commit 719ab49118
3 files changed
+88 -58

No files matched your search

-1
View File
@@ -4,7 +4,6 @@
asset-text-tests:
@python3 tests/font_glyph_coverage_test.py
@python3 tests/locale_keys_test.py
@python3 tests/locale_capacity_test.py
@python3 tests/settings_keys_test.py
.PHONY: version-check
+88 -35
View File
@@ -14,19 +14,41 @@ locale_extension: LocaleExtension;
// Index entries borrow mounted catalog bytes. Rebuild the catalogs whenever
// the asset provider changes, before looking up any of their borrowed slices.
// A language with more keys than this loses its index, and every text then
// reads the whole file: at 2058 keys over a 2048 limit that was three
// quarters of each frame on a phone. tests/locale_capacity_test.py keeps the
// languages well under it.
LocaleCatalogCapacity :: 8192
LocaleCatalogSlots :: 16384
// The index is sized to its language: a fixed 2048-entry index overflowed at
// 2058 English keys, and every text then read the whole file, three quarters
// of each frame on a phone.
LocaleCatalog :: struct {
entries: [LocaleCatalogCapacity]LocaleEntry
slots: [LocaleCatalogSlots]s32
count: s32
entries: *LocaleEntry
entry_room: s64
count: s64
// Open addressing, a power of two at least twice the keys; 0 is empty and
// n is entries[n - 1].
slots: *s32
slot_count: s64
complete: bool
}
LocaleMemory :: (count: u64, size: u64) -> *void #foreign c_locale "calloc";
LocaleMemoryFree :: (memory: *void) #foreign c_locale "free";
LocaleSliceDescriptor :: struct {
data: *void
length: s64
}
LocaleEntries :: (catalog: *LocaleCatalog) -> []LocaleEntry {
result: []LocaleEntry
descriptor := cast(*LocaleSliceDescriptor)(*result)
descriptor.data = cast(*void)catalog.entries
descriptor.length = ifx catalog.entries == null then 0 else catalog.count
return result
}
LocaleSlots :: (catalog: *LocaleCatalog) -> []s32 {
result: []s32
descriptor := cast(*LocaleSliceDescriptor)(*result)
descriptor.data = cast(*void)catalog.slots
descriptor.length = ifx catalog.slots == null then 0 else catalog.slot_count
return result
}
english_catalog: LocaleCatalog;
active_catalog: LocaleCatalog;
@@ -40,48 +62,82 @@ active_catalog_ready: bool;
active_catalog_language_index: s32;
catalog_asset_generation: u64;
LocaleCatalogSlot :: (key: string) -> s32 {
LocaleCatalogSlot :: (key: string, slots: s64) -> s32 {
hash: u32 = 2166136261
index: s32 = 0
while index < cast(s32)key.count {
hash = (hash ^ cast(u32)key[index]) * cast(u32)16777619
index += 1
}
return cast(s32)(hash % cast(u32)LocaleCatalogSlots)
return cast(s32)(hash & cast(u32)(slots - 1))
}
LocaleCatalogPut :: (catalog: *LocaleCatalog, key: string, value: string) {
slot := LocaleCatalogSlot(key)
probes: s32 = 0
while probes < LocaleCatalogSlots {
slot := LocaleCatalogSlot(key, catalog.slot_count)
probes: s64 = 0
while probes < catalog.slot_count {
stored := catalog.slots[slot]
if stored == 0 {
if catalog.count >= LocaleCatalogCapacity {
if catalog.count >= catalog.entry_room {
catalog.complete = false
return
}
catalog.entries[catalog.count] = LocaleEntry.{key, value }
catalog.entries[catalog.count] = LocaleEntry.{key, value}
catalog.count += 1
catalog.slots[slot] = catalog.count
catalog.slots[slot] = cast(s32)catalog.count
return
}
if catalog.entries[stored - 1].key == key {
catalog.entries[stored - 1].value = value
return
}
slot = (slot + 1) % LocaleCatalogSlots
slot = cast(s32)((cast(s64)slot + 1) & (catalog.slot_count - 1))
probes += 1
}
catalog.complete = false
}
LocaleCatalogBuild :: (catalog: *LocaleCatalog, source: string) {
// Room for every key of `source`, with the slots at most half full.
LocaleCatalogSize :: (catalog: *LocaleCatalog, source: string) -> bool {
keys: s64 = 0
cursor: s32 = 0
while cursor < cast(s32)source.count {
line := LocaleNextLine(source, cursor)
cursor = line.next
if line.end - line.start > 2 && source[line.start] == cast(u8)91 && source[line.end - 1] == cast(u8)93 {
keys += 1
}
}
wanted: s64 = 16
while wanted < keys * 2 {
wanted *= 2
}
catalog.count = 0
catalog.complete = true
slot: s32 = 0
while slot < LocaleCatalogSlots {
if keys > catalog.entry_room || wanted != catalog.slot_count {
LocaleMemoryFree(cast(*void)catalog.entries)
LocaleMemoryFree(cast(*void)catalog.slots)
catalog.entries = cast(*LocaleEntry)LocaleMemory(cast(u64)(keys + 1), cast(u64)size_of(LocaleEntry))
catalog.slots = cast(*s32)LocaleMemory(cast(u64)wanted, cast(u64)size_of(s32))
if catalog.entries == null || catalog.slots == null {
LocaleMemoryFree(cast(*void)catalog.entries)
LocaleMemoryFree(cast(*void)catalog.slots)
catalog.* = LocaleCatalog.{}
return false
}
catalog.entry_room = keys + 1
catalog.slot_count = wanted
return true
}
for slot: cast(s64)0..wanted - 1 {
catalog.slots[slot] = 0
slot += 1
}
return true
}
LocaleCatalogBuild :: (catalog: *LocaleCatalog, source: string) {
catalog.complete = LocaleCatalogSize(catalog, source)
if !catalog.complete {
return
}
cursor: s32 = 0
@@ -129,12 +185,12 @@ LocaleCatalogBuild :: (catalog: *LocaleCatalog, source: string) {
LocaleCatalogFind :: (entries: []LocaleEntry, slots: []s32,
key: string) -> LocaleTextResult {
result: LocaleTextResult
if key.count == 0 {
if key.count == 0 || slots.count == 0 {
return result
}
slot := LocaleCatalogSlot(key)
probes: s32 = 0
while probes < LocaleCatalogSlots {
slot := LocaleCatalogSlot(key, slots.count)
probes: s64 = 0
while probes < slots.count {
stored := slots[slot]
if stored == 0 {
return result
@@ -144,7 +200,7 @@ LocaleCatalogFind :: (entries: []LocaleEntry, slots: []s32,
result.found = true
return result
}
slot = (slot + 1) % LocaleCatalogSlots
slot = cast(s32)((cast(s64)slot + 1) & (slots.count - 1))
probes += 1
}
return result
@@ -331,13 +387,11 @@ LocaleText :: (key: string) -> string {
return LocaleResolveCatalog(active_catalog_asset.data,
english_catalog_asset.data, key)
}
lookup := LocaleCatalogFind(active_catalog.entries[:],
active_catalog.slots[:], key)
lookup := LocaleCatalogFind(LocaleEntries(*active_catalog), LocaleSlots(*active_catalog), key)
if lookup.found {
return lookup.text
}
lookup = LocaleCatalogFind(english_catalog.entries[:],
english_catalog.slots[:], key)
lookup = LocaleCatalogFind(LocaleEntries(*english_catalog), LocaleSlots(*english_catalog), key)
if lookup.found {
return lookup.text
}
@@ -353,8 +407,7 @@ LocaleEnglishText :: (key: string) -> string {
return LocaleResolveCatalog(english_catalog_asset.data,
english_catalog_asset.data, key)
}
lookup := LocaleCatalogFind(english_catalog.entries[:],
english_catalog.slots[:], key)
lookup := LocaleCatalogFind(LocaleEntries(*english_catalog), LocaleSlots(*english_catalog), key)
if lookup.found {
return lookup.text
}
-22
View File
@@ -1,22 +0,0 @@
"""Every language fits the app's text index with room to grow.
Past LocaleCatalogCapacity a language loses its index and each text reads the
whole file, which made every screen lag on phones."""
from pathlib import Path
import re
import sys
ROOT = Path(__file__).resolve().parents[1]
source = (ROOT / "src/app/locale.zi").read_text()
capacity = int(re.search(r"^LocaleCatalogCapacity :: (\d+)$", source, re.M)[1])
slots = int(re.search(r"^LocaleCatalogSlots :: (\d+)$", source, re.M)[1])
failed = slots < capacity * 2
if failed:
print(f"LocaleCatalogSlots {slots} must be at least twice LocaleCatalogCapacity {capacity}")
for path in sorted((ROOT / "locales").glob("*.txt")):
keys = len(re.findall(r"^\[[^\]]+\]$", path.read_text(), re.M))
# Room for a quarter more keys before the index could fill.
if keys * 5 > capacity * 4:
print(f"{path.name}: {keys} keys leave too little room in the {capacity}-entry text index")
failed = True
sys.exit(1 if failed else 0)