Load pages in Ziran instead of a Python worker
Navigation no longer forks python3 scripts/fetch.py and round-trips JSON through temporary files. src/url.zi resolves links per RFC 3986 and keeps the old rules (http, https and local file URLs only, no credentials). src/transport.zi streams curl, as Ziran's std HTTPS GET does, through std/process_channel_linux and drains it each frame without blocking; redirects may only reach HTTP(S), documents stay within 512 KiB, only HTML and text types load and invalid UTF-8 becomes U+FFFD. Local files and the start page (now embedded in src/start_page.zi) are read directly, so the app ships no data files. Typed addresses no longer resolve against the current page, so "example.com" typed on the start page loads. tests/transport_test.zi checks resolution, limits, local files and HTTP redirects; Python remains only as its fixture server. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
f37a51b31c
commit
487ccd22cd
13 files changed
+836
-259
No files matched your search
@@ -1,8 +1,11 @@
|
||||
# Browser
|
||||
|
||||
The HTML parser, CSS cascade, document model, layout and product UI belong in
|
||||
this repository, written in Ziran with Kryon. The Python worker only fetches
|
||||
documents and resolves URLs; it never renders a page or invokes another browser.
|
||||
this repository, written in Ziran with Kryon. Use real Ziran packages and std
|
||||
modules instead of side programs: `src/url.zi` resolves addresses and
|
||||
`src/transport.zi` loads documents through `std/process_channel_linux` and curl,
|
||||
as Ziran's std HTTPS GET does. Never reintroduce a Python worker or invoke
|
||||
another browser; Python is allowed only as a test fixture server.
|
||||
Keep credentials out of logs. Do not run page scripts or submitted forms as
|
||||
shell commands. Tests and screenshots run on private Xvfb with inherited
|
||||
display and session variables removed. Never edit dependency checkouts.
|
||||
@@ -4,20 +4,18 @@ PREFIX ?= $(HOME)/.local
|
||||
.PHONY: build check test window-test install
|
||||
build:
|
||||
$(ZIRAN) build
|
||||
cp scripts/fetch.py scripts/start.html build/
|
||||
check:
|
||||
$(ZIRAN) check
|
||||
# The transport test needs curl; its Python harness only serves fixtures.
|
||||
test: check
|
||||
python3 -m unittest discover -s tests -v
|
||||
$(ZIRAN) build --project --root tests --target=c --entry document_test:Main --exe -o build/document-test tests/document_test.zi
|
||||
build/document-test/document_test
|
||||
$(ZIRAN) build --project --root tests --target=c --entry transport_test:Main --exe -o build/transport-test tests/transport_test.zi
|
||||
python3 tests/transport_test.py build/transport-test/transport_test
|
||||
window-test: build
|
||||
env -u DISPLAY -u WAYLAND_DISPLAY -u XAUTHORITY -u DBUS_SESSION_BUS_ADDRESS -u SESSION_MANAGER YUE_DESKTOP_RECOVERY=0 xvfb-run -a -s '-screen 0 1280x1024x24' env BROWSER_PRIVATE_XVFB=1 python3 tests/window_test.py
|
||||
install: build
|
||||
mkdir -p "$(PREFIX)/bin" "$(PREFIX)/lib/browser" "$(PREFIX)/share/applications" "$(PREFIX)/share/icons/hicolor/scalable/apps"
|
||||
install -m 755 build/browser-desktop "$(PREFIX)/lib/browser/browser-desktop"
|
||||
install -m 644 scripts/fetch.py scripts/start.html "$(PREFIX)/lib/browser/"
|
||||
mkdir -p "$(PREFIX)/bin" "$(PREFIX)/share/applications" "$(PREFIX)/share/icons/hicolor/scalable/apps"
|
||||
install -m 755 build/browser-desktop "$(PREFIX)/bin/kryon-browser"
|
||||
install -m 644 assets/browser.svg "$(PREFIX)/share/icons/hicolor/scalable/apps/kryon-browser.svg"
|
||||
printf '%s\n' '#!/bin/sh' 'exec "$(PREFIX)/lib/browser/browser-desktop" "$$@"' > "$(PREFIX)/bin/kryon-browser"
|
||||
chmod 755 "$(PREFIX)/bin/kryon-browser"
|
||||
printf '%s\n' '[Desktop Entry]' 'Type=Application' 'Name=Browser' 'Exec=$(PREFIX)/bin/kryon-browser' 'Icon=kryon-browser' 'Terminal=false' 'Categories=Network;WebBrowser;' > "$(PREFIX)/share/applications/kryon-browser.desktop"
|
||||
@@ -7,8 +7,9 @@ independently from the desktop. No Chromium or WebKit engine renders pages.
|
||||
|
||||
Enter an HTTP(S) address or local `file:///` URL. Links navigate within the app.
|
||||
Back and Next restore history and scroll positions. Pages wrap to the window
|
||||
and scroll independently. Ctrl+L selects the address. Fetching runs outside the
|
||||
UI frame in a Python IO worker; parsing and rendering stay in Ziran.
|
||||
and scroll independently. Ctrl+L selects the address. Pages load outside the
|
||||
UI frame: `src/transport.zi` streams curl through Ziran's process channel
|
||||
and `src/url.zi` resolves links, so the whole browser is Ziran plus curl.
|
||||
|
||||
The initial engine supports headings, paragraphs, lists, links, preformatted
|
||||
text, image descriptions and malformed HTML recovery. Embedded/inline CSS
|
||||
@@ -23,7 +24,8 @@ cookies and authentication remain future work. Limits are 512 KiB of input and
|
||||
|
||||
## Build and verify
|
||||
|
||||
Install Ziran, SDL2, Cairo, FreeType, X11 and Python 3. Dependencies and compiler
|
||||
Install Ziran, SDL2, Cairo, FreeType, X11 and curl; Python 3 only serves test
|
||||
fixtures. Dependencies and compiler
|
||||
are pinned in `ziran.lock`; local development uses ignored package overrides.
|
||||
|
||||
```sh
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Document IO only. Ziran owns HTML, CSS, layout and rendering."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from urllib.parse import urljoin, urlsplit, unquote
|
||||
from urllib.request import Request, urlopen, build_opener, HTTPRedirectHandler
|
||||
|
||||
LIMIT = 524288
|
||||
|
||||
|
||||
def destination(value, base=""):
|
||||
if value == "about:start":
|
||||
return value
|
||||
if ":" not in value and not base:
|
||||
value = "https://" + value
|
||||
url = urljoin(base, value)
|
||||
parts = urlsplit(url)
|
||||
if parts.scheme not in ("http", "https", "file") or parts.username or parts.password:
|
||||
raise ValueError("Enter an HTTP(S) address or a local file URL.")
|
||||
if parts.scheme == "file" and parts.netloc not in ("", "localhost"):
|
||||
raise ValueError("Only local file URLs are supported.")
|
||||
if parts.scheme in ("http", "https") and not parts.netloc:
|
||||
raise ValueError("Enter a complete website address.")
|
||||
return url
|
||||
|
||||
|
||||
class Redirects(HTTPRedirectHandler):
|
||||
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
||||
url = destination(newurl, req.full_url)
|
||||
if urlsplit(url).scheme not in ("http", "https"):
|
||||
raise ValueError("A website cannot redirect to a local file.")
|
||||
return super().redirect_request(req, fp, code, msg, headers, url)
|
||||
|
||||
|
||||
def fetch(request):
|
||||
url = destination(request.get("url", "about:start"), request.get("base", ""))
|
||||
if url == "about:start":
|
||||
content = (Path(__file__).resolve().parent / "start.html").read_bytes()
|
||||
elif urlsplit(url).scheme == "file":
|
||||
with Path(unquote(urlsplit(url).path)).open("rb") as source:
|
||||
content = source.read(LIMIT + 1)
|
||||
else:
|
||||
req = Request(url, headers={"User-Agent": "KryonBrowser/0.1", "Accept": "text/html,text/plain;q=0.9"})
|
||||
with build_opener(Redirects()).open(req, timeout=15) as response:
|
||||
url = response.url
|
||||
if response.headers.get_content_type() not in ("text/html", "text/plain", "application/xhtml+xml"):
|
||||
raise ValueError("This resource is not an HTML or text document.")
|
||||
content = response.read(LIMIT + 1)
|
||||
if len(content) > LIMIT:
|
||||
raise ValueError("This document exceeds the 512 KiB page limit.")
|
||||
html = content.decode("utf-8", errors="replace")
|
||||
if urlsplit(url).path.endswith(('.txt', '.zi', '.css')):
|
||||
import html as escaping
|
||||
html = "<pre>" + escaping.escape(html) + "</pre>"
|
||||
return {"url": url, "html": html, "error": ""}
|
||||
|
||||
|
||||
def main():
|
||||
os.umask(0o077)
|
||||
output = Path(sys.argv[2])
|
||||
try:
|
||||
result = fetch(json.loads(Path(sys.argv[1]).read_text()))
|
||||
except ValueError as exc:
|
||||
result = {"error": str(exc)}
|
||||
except Exception:
|
||||
result = {"error": "Could not load this page. Check the address and your connection."}
|
||||
fd, filename = tempfile.mkstemp(dir=output.parent)
|
||||
with os.fdopen(fd, "w") as stream:
|
||||
json.dump(result, stream, ensure_ascii=False)
|
||||
os.replace(filename, output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,20 +0,0 @@
|
||||
<!doctype html><html><head><title>Browser</title>
|
||||
<style>
|
||||
body { background: #f4f7fb; color: #263246; padding: 20px; }
|
||||
main { max-width: 760px; margin: 12px auto; padding: 26px; background: white; }
|
||||
h1 { font-size: 38px; color: #254c7b; }
|
||||
h2 { color: #254c7b; font-size: 26px; }
|
||||
.quiet { color: #67768b; font-size: 15px; }
|
||||
a { color: #2468bc; }
|
||||
</style></head><body><main>
|
||||
<p class="quiet">ZIRAN + KRYON</p>
|
||||
<h1>A small browser, built from scratch.</h1>
|
||||
<p>Enter a web address or a file URL above. HTML becomes a document tree, CSS styles that tree, and Kryon draws the page.</p>
|
||||
<h2>Simple pages, clearly rendered</h2>
|
||||
<p>Headings, paragraphs, lists, links, preformatted text and image descriptions work here. Text wraps to the window and the page scrolls independently.</p>
|
||||
<p>Styles can set colors, backgrounds, font sizes, spacing, widths and visibility using element, class, ID and inline selectors.</p>
|
||||
<h2>A first engine</h2>
|
||||
<p>This early browser supports embedded and inline CSS. JavaScript, forms, image decoding, external stylesheets, flexbox and grid are not implemented yet.</p>
|
||||
<a href="https://example.com">Visit example.com</a>
|
||||
<p class="quiet">No Chromium or WebKit rendering engine. Document fetching runs outside the UI thread.</p>
|
||||
</main></body></html>
|
||||
+1
-1
@@ -59,7 +59,7 @@ Frame :: (session: Session, viewport: Rectangle) -> s32 {
|
||||
result:=AddressField(session,field_bounds,key,modifiers,typed); State.cursor=result.cursor; State.anchor=result.anchor; State.focused=result.focused
|
||||
if result.edit.changed { State.Edit(result.edit.start,result.edit.end,result.edit.replacement) }
|
||||
Window.Record(20,"Address",field_bounds.x,field_bounds.y,field_bounds.width,field_bounds.height,true)
|
||||
if Action(session,13,ifx State.loading then "Wait" else "Go",.{x+width-62.0,y,62.0,38.0},!State.loading) || (result.committed && !State.loading) { State.Navigate(TextUntilNul(State.address[:])); State.focused=false }
|
||||
if Action(session,13,ifx State.loading then "Wait" else "Go",.{x+width-62.0,y,62.0,38.0},!State.loading) || (result.committed && !State.loading) { State.Navigate(TextUntilNul(State.address[:]),true,true); State.focused=false }
|
||||
page: Rectangle=.{viewport.x,viewport.y+60.0,viewport.width,viewport.height-92.0}
|
||||
background: BoxProps; background.key=2; background.bounds=page; background.fill=ColorFromPacked(Document.page_color); Box(session,background)
|
||||
if Document.count>0 { Paint(session,page) }
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// The built-in start page, kept in Ziran so the app needs no data files.
|
||||
Url :: #import "url";
|
||||
|
||||
lines: [21]string = .{
|
||||
"<!doctype html><html><head><title>Browser</title>",
|
||||
"<style>",
|
||||
"body { background: #f4f7fb; color: #263246; padding: 20px; }",
|
||||
"main { max-width: 760px; margin: 12px auto; padding: 26px; background: white; }",
|
||||
"h1 { font-size: 38px; color: #254c7b; }",
|
||||
"h2 { color: #254c7b; font-size: 26px; }",
|
||||
".quiet { color: #67768b; font-size: 15px; }",
|
||||
"a { color: #2468bc; }",
|
||||
"</style></head><body><main>",
|
||||
"<p class='quiet'>ZIRAN + KRYON</p>",
|
||||
"<h1>A small browser, built from scratch.</h1>",
|
||||
"<p>Enter a web address or a file URL above. HTML becomes a document tree, CSS styles that tree, and Kryon draws the page.</p>",
|
||||
"<h2>Simple pages, clearly rendered</h2>",
|
||||
"<p>Headings, paragraphs, lists, links, preformatted text and image descriptions work here. Text wraps to the window and the page scrolls independently.</p>",
|
||||
"<p>Styles can set colors, backgrounds, font sizes, spacing, widths and visibility using element, class, ID and inline selectors.</p>",
|
||||
"<h2>A first engine</h2>",
|
||||
"<p>This early browser supports embedded and inline CSS. JavaScript, forms, image decoding, external stylesheets, flexbox and grid are not implemented yet.</p>",
|
||||
"<a href='https://example.com'>Visit example.com</a>",
|
||||
"<p class='quiet'>No Chromium or WebKit rendering engine. Pages load outside the UI frame.</p>",
|
||||
"</main></body></html>",
|
||||
""
|
||||
};
|
||||
|
||||
// Writes the page into OUT and returns its length.
|
||||
#program_export
|
||||
Write :: (out: []u8) -> s32 {
|
||||
length: s32 = 0
|
||||
for line: lines {
|
||||
length = Url.Append(out, length, line)
|
||||
length = Url.Append(out, length, "\n")
|
||||
}
|
||||
if length < 0 { return 0 }
|
||||
return length
|
||||
}
|
||||
+6
-10
@@ -1,6 +1,5 @@
|
||||
Transport :: #import "transport";
|
||||
Document :: #import "document";
|
||||
#import "std/json_scan"
|
||||
#import "std/text"
|
||||
#import "std/text_buffer"
|
||||
#import "std/byte_text_linux"
|
||||
@@ -22,6 +21,7 @@ anchor: s32;
|
||||
focused: bool;
|
||||
scroll: s32;
|
||||
|
||||
// JSON helpers for the desktop host's state reply (desktop.zi).
|
||||
Release :: (value: string) { descriptor:=cast(*TextDescriptor)(*value); Free(cast(*void)descriptor.data) }
|
||||
Quoted :: (out: *Vec(u8), value: string) {
|
||||
BuilderPrint(out,"\"")
|
||||
@@ -33,31 +33,27 @@ Quoted :: (out: *Vec(u8), value: string) {
|
||||
}
|
||||
Property :: (out: *Vec(u8), key: string, value: string) { BuilderPrint(out,",\"%\":",key); Quoted(out,value) }
|
||||
Set :: (out: []u8, value: string) { Document.Copy(out,value) }
|
||||
Navigate :: (target: string, remember: bool=true) {
|
||||
// A TYPED address stands alone; a link resolves against the current page.
|
||||
Navigate :: (target: string, remember: bool=true, typed: bool=false) {
|
||||
if loading || target=="" { return }
|
||||
if remember && position>=0 { history_scroll[position]=scroll }
|
||||
if remember {
|
||||
if position>=63 { for i: 0..cast(s32)62 { history[i]=history[i+1]; history_scroll[i]=history_scroll[i+1] }; position=62 }
|
||||
position+=1; history_count=position+1; Set(history[position][:],target); history_scroll[position]=0
|
||||
}
|
||||
out: Vec(u8); BuilderPrint(*out,"{\"url\":"); Quoted(*out,target); Property(*out,"base",TextUntilNul(url[:])); BuilderPrint(*out,"}")
|
||||
request:=BuilderFinish(out); loading=Transport.Start(request); Release(request)
|
||||
base: [4097]u8; if !typed { Set(base[:],TextUntilNul(url[:])) }
|
||||
loading=Transport.Start(target,TextUntilNul(base[:]))
|
||||
Set(message[:],ifx loading then "Loading…" else Transport.error)
|
||||
}
|
||||
Back :: (delta: s32) {
|
||||
if loading || position+delta<0 || position+delta>=history_count { return }
|
||||
history_scroll[position]=scroll; position+=delta; target: [4097]u8; Set(target[:],TextUntilNul(history[position][:])); Navigate(TextUntilNul(target[:]),false)
|
||||
}
|
||||
Decode :: (key: string, out: []u8) {
|
||||
value:=Transport.Value(); size:=ReadStringText(value,FieldAt(value,0,key),out)
|
||||
if size<0 || size>=cast(s32)out.count { out[0]=0 } else { out[size]=0 }
|
||||
}
|
||||
Tick :: () {
|
||||
if !started { started=true; Navigate("about:start") }
|
||||
if !Transport.Poll() { return }; loading=false
|
||||
if Transport.error!="" { Set(message[:],Transport.error); return }
|
||||
Decode("error",message[:]); if message[0]!=0 { return }
|
||||
Decode("url",url[:]); Decode("html",html[:]); Set(address[:],TextUntilNul(url[:])); Set(history[position][:],TextUntilNul(url[:]))
|
||||
Set(url[:],Transport.CurrentUrl()); Set(html[:],Transport.Html()); Set(address[:],TextUntilNul(url[:])); Set(history[position][:],TextUntilNul(url[:]))
|
||||
Document.Load(TextUntilNul(html[:])); changed=true; scroll=history_scroll[position]; cursor=cast(s32)TextUntilNul(address[:]).count; anchor=cursor
|
||||
Set(message[:],ifx Document.truncated then "Page preview limit reached" else TextUntilNul(Document.title[:]))
|
||||
}
|
||||
|
||||
+296
-77
@@ -1,93 +1,312 @@
|
||||
// A single asynchronous worker keeps network IO outside the UI frame.
|
||||
#import "std/byte_text_linux"
|
||||
#import "std/text_buffer"
|
||||
libc :: #system_library "libc";
|
||||
Open :: (path: *u8, flags: s32, mode: ..any) -> s32 #foreign libc "open";
|
||||
Read :: (descriptor: s32, output: *u8, length: u64) -> s64 #foreign libc "read";
|
||||
Write :: (descriptor: s32, input: *u8, length: u64) -> s64 #foreign libc "write";
|
||||
Close :: (descriptor: s32) -> s32 #foreign libc "close";
|
||||
Fork :: () -> s32 #foreign libc "fork";
|
||||
Exec :: (file: *u8, arg0: *u8, args: ..any) -> s32 #foreign libc "execlp";
|
||||
Exit :: (code: s32) #foreign libc "_exit";
|
||||
Wait :: (process: s32, status: *s32, options: s32) -> s32 #foreign libc "waitpid";
|
||||
Temporary :: (pattern: *u8) -> *u8 #foreign libc "mkdtemp";
|
||||
Unlink :: (path: *u8) -> s32 #foreign libc "unlink";
|
||||
Environment :: (name: *u8) -> *u8 #foreign libc "getenv";
|
||||
ReadExecutableLink :: (path: *u8, out: *u8, size: u64) -> s64 #foreign libc "readlink";
|
||||
Access :: (path: *u8, mode: s32) -> s32 #foreign libc "access";
|
||||
// Document IO for navigation, entirely in Ziran. Web pages come from curl
|
||||
// (which Ziran's std HTTPS GET also relies on) in an owned process channel
|
||||
// that Poll drains without blocking, so a slow site never stalls a frame.
|
||||
// Local files and the built-in start page are read directly. Redirects may
|
||||
// only reach HTTP(S), documents are capped at 512 KiB, only HTML and text
|
||||
// content types load, and invalid UTF-8 becomes U+FFFD.
|
||||
Channel :: #import "std/process_channel_linux";
|
||||
Files :: #import "std/file";
|
||||
Url :: #import "url";
|
||||
StartPage :: #import "start_page";
|
||||
|
||||
LIMIT :: 524288;
|
||||
META :: "\n@@kryon-browser-meta@@\n";
|
||||
TRAILER_CAPACITY :: 8192;
|
||||
PAGE_CAPACITY :: 1048576;
|
||||
READS_PER_POLL :: 64;
|
||||
URL_BYTES :: 4096;
|
||||
|
||||
ERROR_LOAD :: "Could not load this page. Check the address and your connection.";
|
||||
ERROR_LIMIT :: "This document exceeds the 512 KiB page limit.";
|
||||
ERROR_TYPE :: "This resource is not an HTML or text document.";
|
||||
ERROR_REDIRECT :: "A website can only redirect to another web address.";
|
||||
ERROR_START :: "Cannot start the page loader (curl).";
|
||||
|
||||
pid: s32;
|
||||
request_path: [4096]u8;
|
||||
response_path: [4096]u8;
|
||||
helper: [4096]u8;
|
||||
data: [1048576]u8;
|
||||
size: s32;
|
||||
error: string;
|
||||
initialized: bool;
|
||||
loading: bool;
|
||||
ready: bool;
|
||||
channel: Channel.ProcessChannel;
|
||||
received: [LIMIT + TRAILER_CAPACITY]u8;
|
||||
received_size: s32;
|
||||
request_url: [URL_BYTES]u8;
|
||||
request_length: s32;
|
||||
final_url: [URL_BYTES]u8;
|
||||
final_length: s32;
|
||||
path_bytes: [URL_BYTES]u8;
|
||||
page: [PAGE_CAPACITY]u8;
|
||||
page_length: s32;
|
||||
|
||||
Copy :: (out: []u8, value: string) {
|
||||
count := cast(s32)value.count; if count >= cast(s32)out.count { count = cast(s32)out.count - 1 }
|
||||
for i: 0..count - 1 { out[i] = value[i] }; out[count] = 0
|
||||
EndsWith :: (value: string, suffix: string) -> bool {
|
||||
if suffix.count > value.count { return false }
|
||||
return value[value.count - suffix.count:value.count] == suffix
|
||||
}
|
||||
|
||||
Init :: () -> bool {
|
||||
if initialized { return true }
|
||||
folder: [64]u8; Copy(folder[:], "/tmp/kryon-browser-XXXXXX")
|
||||
if Temporary(*folder[0]) == null { error = "Cannot create private request storage"; return false }
|
||||
root := TextUntilNul(folder[:]); prefix := cast(s32)root.count
|
||||
Copy(request_path[:],root); Copy(request_path[prefix:],"/request.json")
|
||||
Copy(response_path[:],root); Copy(response_path[prefix:],"/response.json")
|
||||
name: [64]u8; Copy(name[:], "KRYON_BROWSER_FETCH")
|
||||
override := Environment(*name[0])
|
||||
if override != null { Copy(helper[:], TextFromCString(override)) }
|
||||
else {
|
||||
Copy(name[:], "/proc/self/exe")
|
||||
length := cast(s32)ReadExecutableLink(*name[0], *helper[0], 4000)
|
||||
if length <= 0 { error="Cannot locate the Browser transport"; return false }
|
||||
while length > 0 && helper[length-1] != 47 { length -= 1 }
|
||||
Copy(helper[length:], "fetch.py")
|
||||
}
|
||||
if Access(*helper[0],4) != 0 { error="Browser transport is missing; rebuild or reinstall the app"; return false }
|
||||
initialized = true; return true
|
||||
// Appends one escaped byte for a text document shown as <pre>.
|
||||
AppendByte :: (out: []u8, at: s32, byte: u8, escape: bool) -> s32 {
|
||||
if at < 0 { return -1 }
|
||||
if escape && byte == 60 { return Url.Append(out, at, "<") }
|
||||
if escape && byte == 62 { return Url.Append(out, at, ">") }
|
||||
if escape && byte == 38 { return Url.Append(out, at, "&") }
|
||||
if at + 1 >= cast(s32)out.count { return -1 }
|
||||
out[at] = byte
|
||||
return at + 1
|
||||
}
|
||||
|
||||
Start :: (request: string) -> bool {
|
||||
if pid > 0 || !Init() { return false }
|
||||
error=""; size=0; unused Unlink(*response_path[0])
|
||||
descriptor := Open(*request_path[0], 577, cast(u32)384)
|
||||
if descriptor < 0 { error="Cannot write Browser request"; return false }
|
||||
bytes := cast(*TextDescriptor)(*request); written: s64
|
||||
while written < request.count {
|
||||
n := Write(descriptor, *bytes.data[written], cast(u64)(request.count-written))
|
||||
if n <= 0 { break }; written += n
|
||||
// Copies BYTES as valid UTF-8, replacing each invalid byte with U+FFFD, and
|
||||
// stops at RESERVE bytes before the end of OUT.
|
||||
#program_export
|
||||
AppendUtf8 :: (out: []u8, at: s32, bytes: []u8, escape: bool, reserve: s32) -> s32 {
|
||||
limited := out[0:cast(s32)out.count - reserve]
|
||||
length := at
|
||||
total := cast(s32)bytes.count
|
||||
index: s32 = 0
|
||||
while index < total && length >= 0 {
|
||||
lead := bytes[index]
|
||||
need: s32 = 0
|
||||
low: u8 = 128
|
||||
high: u8 = 191
|
||||
if lead >= 194 && lead <= 223 { need = 1 }
|
||||
else if lead == 224 { need = 2; low = 160 }
|
||||
else if (lead >= 225 && lead <= 236) || lead == 238 || lead == 239 { need = 2 }
|
||||
else if lead == 237 { need = 2; high = 159 }
|
||||
else if lead == 240 { need = 3; low = 144 }
|
||||
else if lead >= 241 && lead <= 243 { need = 3 }
|
||||
else if lead == 244 { need = 3; high = 143 }
|
||||
valid := lead < 128 || need > 0
|
||||
if need > 0 {
|
||||
if index + need >= total { valid = false }
|
||||
else {
|
||||
for step: 1..need {
|
||||
next := bytes[index + step]
|
||||
minimum: u8 = 128
|
||||
maximum: u8 = 191
|
||||
if step == 1 { minimum = low; maximum = high }
|
||||
if next < minimum || next > maximum { valid = false }
|
||||
}
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
// U+FFFD REPLACEMENT CHARACTER
|
||||
if length + 3 >= cast(s32)limited.count { return length }
|
||||
out[length] = 239
|
||||
out[length + 1] = 191
|
||||
out[length + 2] = 189
|
||||
length += 3
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
if need == 0 {
|
||||
next_length := AppendByte(limited, length, lead, escape)
|
||||
if next_length < 0 { return length }
|
||||
length = next_length
|
||||
index += 1
|
||||
continue
|
||||
}
|
||||
if length + need + 1 >= cast(s32)limited.count { return length }
|
||||
for step: 0..need {
|
||||
out[length] = bytes[index + step]
|
||||
length += 1
|
||||
}
|
||||
index += need + 1
|
||||
}
|
||||
unused Close(descriptor)
|
||||
if written != request.count { error="Cannot write complete Browser request"; return false }
|
||||
pid=Fork()
|
||||
if pid < 0 { pid=0; error="Cannot start Browser transport"; return false }
|
||||
if pid == 0 {
|
||||
command: [16]u8; Copy(command[:],"python3")
|
||||
unused Exec(*command[0], *command[0], *helper[0], *request_path[0], *response_path[0], cast(*u8)null)
|
||||
Exit(127)
|
||||
return length
|
||||
}
|
||||
|
||||
// Publishes BODY loaded from URL as the page; text files become <pre>.
|
||||
Finish :: (body: []u8, url: string) {
|
||||
final_length = Url.Append(final_url[:], 0, url)
|
||||
if final_length < 0 { final_length = 0 }
|
||||
parts := Url.SplitUrl(url)
|
||||
plain := EndsWith(parts.path, ".txt") || EndsWith(parts.path, ".zi") || EndsWith(parts.path, ".css")
|
||||
length: s32 = 0
|
||||
if plain { length = Url.Append(page[:], length, "<pre>") }
|
||||
length = AppendUtf8(page[:], length, body, plain, 16)
|
||||
if plain { length = Url.Append(page[:], length, "</pre>") }
|
||||
if length < 0 { length = 0 }
|
||||
page_length = length
|
||||
}
|
||||
|
||||
ReadLocal :: (path: string, url: string) {
|
||||
decoded, ok := Url.DecodePath(path, path_bytes[:])
|
||||
if !ok {
|
||||
error = ERROR_LOAD
|
||||
return
|
||||
}
|
||||
file := Files.OpenRead(decoded)
|
||||
if !file.valid {
|
||||
error = ERROR_LOAD
|
||||
return
|
||||
}
|
||||
size: s32 = 0
|
||||
while size <= LIMIT {
|
||||
count := Files.ReadAt(file, received[size:LIMIT + 1], cast(s64)size)
|
||||
if count <= 0 { break }
|
||||
size += cast(s32)count
|
||||
}
|
||||
unused Files.CloseFile(file)
|
||||
if size > LIMIT {
|
||||
error = ERROR_LIMIT
|
||||
return
|
||||
}
|
||||
Finish(received[0:size], url)
|
||||
}
|
||||
|
||||
// The media type before any parameters, lower-cased, or false when it is
|
||||
// neither HTML nor text. A missing type counts as text, as HTTP clients do.
|
||||
#program_export
|
||||
AcceptedType :: (value: string) -> bool {
|
||||
end: s32 = 0
|
||||
while end < cast(s32)value.count && value[end] != 59 { end += 1 }
|
||||
start: s32 = 0
|
||||
while start < end && value[start] == 32 { start += 1 }
|
||||
while end > start && value[end - 1] == 32 { end -= 1 }
|
||||
media: [64]u8
|
||||
length := Url.AppendLower(media[:], 0, value[start:end])
|
||||
if length < 0 { return false }
|
||||
kind := TextView(media[0:length])
|
||||
return kind == "" || kind == "text/html" || kind == "text/plain" || kind == "application/xhtml+xml"
|
||||
}
|
||||
|
||||
LastMarker :: (bytes: []u8) -> s32 {
|
||||
marker := META
|
||||
count := cast(s32)marker.count
|
||||
index := cast(s32)bytes.count - count
|
||||
while index >= 0 {
|
||||
matched := true
|
||||
for offset: 0..count - 1 {
|
||||
if bytes[index + offset] != marker[offset] {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if matched { return index }
|
||||
index -= 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
Interpret :: (code: s32) {
|
||||
marker := LastMarker(received[0:received_size])
|
||||
if code == 63 {
|
||||
error = ERROR_LIMIT
|
||||
return
|
||||
}
|
||||
if code == 1 {
|
||||
error = ERROR_REDIRECT
|
||||
return
|
||||
}
|
||||
if code != 0 || marker < 0 {
|
||||
error = ERROR_LOAD
|
||||
return
|
||||
}
|
||||
if marker > LIMIT {
|
||||
error = ERROR_LIMIT
|
||||
return
|
||||
}
|
||||
meta := TextView(received[marker + cast(s32)META.count:received_size])
|
||||
line_end: s32 = 0
|
||||
while line_end < cast(s32)meta.count && meta[line_end] != 10 { line_end += 1 }
|
||||
effective := meta[0:line_end]
|
||||
type_start := line_end + 1
|
||||
type_end := type_start
|
||||
while type_end < cast(s32)meta.count && meta[type_end] != 10 { type_end += 1 }
|
||||
content_type := ""
|
||||
if type_start <= cast(s32)meta.count { content_type = meta[type_start:type_end] }
|
||||
if !AcceptedType(content_type) {
|
||||
error = ERROR_TYPE
|
||||
return
|
||||
}
|
||||
if effective == "" { effective = TextView(request_url[0:request_length]) }
|
||||
Finish(received[0:marker], effective)
|
||||
}
|
||||
|
||||
// Starts loading TARGET resolved against BASE (empty for a typed address).
|
||||
// False with error set when the address is refused or nothing can start.
|
||||
#program_export
|
||||
Start :: (target: string, base: string) -> bool {
|
||||
if loading { return false }
|
||||
error = ""
|
||||
ready = false
|
||||
page_length = 0
|
||||
final_length = 0
|
||||
received_size = 0
|
||||
url, problem := Url.Destination(target, base, request_url[:])
|
||||
if problem != "" {
|
||||
// Named constants, so error never views the request buffer.
|
||||
error = Url.ERROR_SCHEME
|
||||
if problem == Url.ERROR_FOREIGN_FILE { error = Url.ERROR_FOREIGN_FILE }
|
||||
if problem == Url.ERROR_HOST { error = Url.ERROR_HOST }
|
||||
if problem == Url.ERROR_LENGTH { error = Url.ERROR_LENGTH }
|
||||
return false
|
||||
}
|
||||
if url == "about:start" {
|
||||
page_length = StartPage.Write(page[:])
|
||||
final_length = Url.Append(final_url[:], 0, "about:start")
|
||||
loading = true
|
||||
ready = true
|
||||
return true
|
||||
}
|
||||
request_length = cast(s32)url.count
|
||||
parts := Url.SplitUrl(url)
|
||||
if parts.scheme == "file" {
|
||||
ReadLocal(parts.path, url)
|
||||
loading = true
|
||||
ready = true
|
||||
return true
|
||||
}
|
||||
arguments: [22]string = .["curl", "--silent", "--location", "--max-redirs", "10",
|
||||
"--proto", "=http,https", "--proto-redir", "=http,https", "--max-time", "15",
|
||||
"--max-filesize", "524288", "--compressed", "--fail", "--user-agent",
|
||||
"KryonBrowser/0.1", "--header", "Accept: text/html,text/plain;q=0.9",
|
||||
"--write-out", "\n@@kryon-browser-meta@@\n%{url_effective}\n%{content_type}\n", ""]
|
||||
arguments[21] = url
|
||||
channel = Channel.StartChannel(arguments[:], false, true)
|
||||
if channel.pid <= 0 {
|
||||
error = ERROR_START
|
||||
return false
|
||||
}
|
||||
loading = true
|
||||
return true
|
||||
}
|
||||
|
||||
// True once the current load finished, successfully or with error set.
|
||||
#program_export
|
||||
Poll :: () -> bool {
|
||||
if pid <= 0 { return false }
|
||||
status: s32; done := Wait(pid,*status,1)
|
||||
if done == 0 { return false }; pid=0
|
||||
if done < 0 || status != 0 { error="Browser transport stopped unexpectedly. Refresh to retry."; return true }
|
||||
descriptor := Open(*response_path[0],0)
|
||||
if descriptor < 0 { error="Browser returned no response. Refresh to retry."; return true }
|
||||
size=0
|
||||
while size < 1048575 {
|
||||
n := Read(descriptor,*data[size],cast(u64)(1048575-size))
|
||||
if n <= 0 { break }; size += cast(s32)n
|
||||
if !loading { return false }
|
||||
if ready {
|
||||
loading = false
|
||||
ready = false
|
||||
return true
|
||||
}
|
||||
extra: u8; remaining := Read(descriptor,*extra,1); unused Close(descriptor)
|
||||
if remaining > 0 { error="Browser response exceeded the preview limit"; size=0 }
|
||||
data[size]=0; unused Unlink(*request_path[0]); unused Unlink(*response_path[0])
|
||||
finished := false
|
||||
reads: s32 = 0
|
||||
while reads < READS_PER_POLL {
|
||||
if received_size >= cast(s32)received.count {
|
||||
unused Channel.CancelChannel(*channel)
|
||||
loading = false
|
||||
error = ERROR_LIMIT
|
||||
return true
|
||||
}
|
||||
count := Channel.ChannelReceive(channel, received[received_size:received.count], 0)
|
||||
if count < 0 { break }
|
||||
if count == 0 {
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
received_size += cast(s32)count
|
||||
reads += 1
|
||||
}
|
||||
if !finished { return false }
|
||||
unused Channel.WaitChannel(*channel, 2000)
|
||||
code := channel.code
|
||||
unused Channel.CloseChannel(*channel, 0)
|
||||
loading = false
|
||||
Interpret(code)
|
||||
return true
|
||||
}
|
||||
Value :: () -> string { return TextFromBytes(data[:size]) }
|
||||
|
||||
#program_export
|
||||
CurrentUrl :: () -> string { return TextView(final_url[0:final_length]) }
|
||||
|
||||
#program_export
|
||||
Html :: () -> string { return TextView(page[0:page_length]) }
|
||||
+268
@@ -0,0 +1,268 @@
|
||||
// Navigation addresses, resolved the way a browser resolves a link: an RFC
|
||||
// 3986 reference against the current page. Only http, https and local file
|
||||
// URLs without credentials load; "about:start" names the built-in page.
|
||||
|
||||
URL_CAPACITY :: 4096;
|
||||
MAX_SEGMENTS :: 512;
|
||||
|
||||
ERROR_SCHEME :: "Enter an HTTP(S) address or a local file URL.";
|
||||
ERROR_FOREIGN_FILE :: "Only local file URLs are supported.";
|
||||
ERROR_HOST :: "Enter a complete website address.";
|
||||
ERROR_LENGTH :: "This address is too long.";
|
||||
|
||||
UrlParts :: struct {
|
||||
scheme: string
|
||||
has_authority: bool
|
||||
authority: string
|
||||
path: string
|
||||
has_query: bool
|
||||
query: string
|
||||
has_fragment: bool
|
||||
fragment: string
|
||||
}
|
||||
|
||||
IsAlpha :: (byte: u8) -> bool {
|
||||
return (byte >= 65 && byte <= 90) || (byte >= 97 && byte <= 122)
|
||||
}
|
||||
|
||||
IsSchemeByte :: (byte: u8) -> bool {
|
||||
return IsAlpha(byte) || (byte >= 48 && byte <= 57) || byte == 43 || byte == 45 || byte == 46
|
||||
}
|
||||
|
||||
// Splits scheme, authority, path, query and fragment without decoding them.
|
||||
#program_export
|
||||
SplitUrl :: (value: string) -> UrlParts {
|
||||
parts: UrlParts
|
||||
total := cast(s32)value.count
|
||||
cursor: s32 = 0
|
||||
if total > 0 && IsAlpha(value[0]) {
|
||||
index: s32 = 1
|
||||
while index < total && IsSchemeByte(value[index]) { index += 1 }
|
||||
if index < total && value[index] == 58 {
|
||||
parts.scheme = value[0:index]
|
||||
cursor = index + 1
|
||||
}
|
||||
}
|
||||
if cursor + 1 < total && value[cursor] == 47 && value[cursor + 1] == 47 {
|
||||
start := cursor + 2
|
||||
end := start
|
||||
while end < total && value[end] != 47 && value[end] != 63 && value[end] != 35 { end += 1 }
|
||||
parts.has_authority = true
|
||||
parts.authority = value[start:end]
|
||||
cursor = end
|
||||
}
|
||||
path_end := cursor
|
||||
while path_end < total && value[path_end] != 63 && value[path_end] != 35 { path_end += 1 }
|
||||
parts.path = value[cursor:path_end]
|
||||
cursor = path_end
|
||||
if cursor < total && value[cursor] == 63 {
|
||||
end := cursor + 1
|
||||
while end < total && value[end] != 35 { end += 1 }
|
||||
parts.has_query = true
|
||||
parts.query = value[cursor + 1:end]
|
||||
cursor = end
|
||||
}
|
||||
if cursor < total && value[cursor] == 35 {
|
||||
parts.has_fragment = true
|
||||
parts.fragment = value[cursor + 1:total]
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// Appends PIECE and returns the new length, or -1 once OUT is full; a
|
||||
// negative length stays negative so callers check once at the end.
|
||||
Append :: (out: []u8, at: s32, piece: string) -> s32 {
|
||||
if at < 0 { return -1 }
|
||||
if at + cast(s32)piece.count >= cast(s32)out.count { return -1 }
|
||||
index := at
|
||||
for byte: piece {
|
||||
out[index] = byte
|
||||
index += 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
AppendLower :: (out: []u8, at: s32, piece: string) -> s32 {
|
||||
if at < 0 { return -1 }
|
||||
if at + cast(s32)piece.count >= cast(s32)out.count { return -1 }
|
||||
index := at
|
||||
for byte: piece {
|
||||
value := byte
|
||||
if value >= 65 && value <= 90 { value += 32 }
|
||||
out[index] = value
|
||||
index += 1
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
// RFC 3986 remove_dot_segments: "." and ".." never reach the request.
|
||||
AppendWithoutDots :: (out: []u8, at: s32, path: string) -> s32 {
|
||||
total := cast(s32)path.count
|
||||
starts: [MAX_SEGMENTS]s32
|
||||
ends: [MAX_SEGMENTS]s32
|
||||
count: s32 = 0
|
||||
absolute := total > 0 && path[0] == 47
|
||||
index: s32 = 0
|
||||
if absolute { index = 1 }
|
||||
trailing := false
|
||||
while index <= total {
|
||||
end := index
|
||||
while end < total && path[end] != 47 { end += 1 }
|
||||
segment := path[index:end]
|
||||
if segment == "." {
|
||||
trailing = true
|
||||
} else if segment == ".." {
|
||||
if count > 0 { count -= 1 }
|
||||
trailing = true
|
||||
} else {
|
||||
if count >= MAX_SEGMENTS { return -1 }
|
||||
starts[count] = index
|
||||
ends[count] = end
|
||||
count += 1
|
||||
trailing = false
|
||||
}
|
||||
index = end + 1
|
||||
}
|
||||
length := at
|
||||
if absolute { length = Append(out, length, "/") }
|
||||
if count > 0 {
|
||||
for segment: 0..count - 1 {
|
||||
if segment > 0 { length = Append(out, length, "/") }
|
||||
length = Append(out, length, path[starts[segment]:ends[segment]])
|
||||
}
|
||||
if trailing { length = Append(out, length, "/") }
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
AppendTail :: (out: []u8, at: s32, has_query: bool, query: string, has_fragment: bool,
|
||||
fragment: string) -> s32 {
|
||||
length := at
|
||||
if has_query {
|
||||
length = Append(out, length, "?")
|
||||
length = Append(out, length, query)
|
||||
}
|
||||
if has_fragment {
|
||||
length = Append(out, length, "#")
|
||||
length = Append(out, length, fragment)
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
// Resolves REFERENCE against BASE (RFC 3986 section 5.2.2) into OUT and
|
||||
// returns its length, or -1 when it does not fit.
|
||||
#program_export
|
||||
ResolveReference :: (reference: string, base: string, out: []u8) -> s32 {
|
||||
r := SplitUrl(reference)
|
||||
b := SplitUrl(base)
|
||||
length: s32 = 0
|
||||
if r.scheme != "" || base == "" {
|
||||
if r.scheme != "" {
|
||||
length = AppendLower(out, length, r.scheme)
|
||||
length = Append(out, length, ":")
|
||||
}
|
||||
if r.has_authority {
|
||||
length = Append(out, length, "//")
|
||||
length = Append(out, length, r.authority)
|
||||
}
|
||||
length = AppendWithoutDots(out, length, r.path)
|
||||
return AppendTail(out, length, r.has_query, r.query, r.has_fragment, r.fragment)
|
||||
}
|
||||
if b.scheme != "" {
|
||||
length = AppendLower(out, length, b.scheme)
|
||||
length = Append(out, length, ":")
|
||||
}
|
||||
if r.has_authority {
|
||||
length = Append(out, length, "//")
|
||||
length = Append(out, length, r.authority)
|
||||
length = AppendWithoutDots(out, length, r.path)
|
||||
return AppendTail(out, length, r.has_query, r.query, r.has_fragment, r.fragment)
|
||||
}
|
||||
if b.has_authority {
|
||||
length = Append(out, length, "//")
|
||||
length = Append(out, length, b.authority)
|
||||
}
|
||||
if r.path == "" {
|
||||
length = Append(out, length, b.path)
|
||||
if r.has_query {
|
||||
return AppendTail(out, length, true, r.query, r.has_fragment, r.fragment)
|
||||
}
|
||||
return AppendTail(out, length, b.has_query, b.query, r.has_fragment, r.fragment)
|
||||
}
|
||||
if r.path[0] == 47 {
|
||||
length = AppendWithoutDots(out, length, r.path)
|
||||
} else {
|
||||
merged: [URL_CAPACITY]u8
|
||||
merged_length: s32 = 0
|
||||
if b.has_authority && b.path == "" {
|
||||
merged_length = Append(merged[:], 0, "/")
|
||||
} else {
|
||||
slash := cast(s32)b.path.count - 1
|
||||
while slash >= 0 && b.path[slash] != 47 { slash -= 1 }
|
||||
if slash >= 0 { merged_length = Append(merged[:], 0, b.path[0:slash + 1]) }
|
||||
}
|
||||
merged_length = Append(merged[:], merged_length, r.path)
|
||||
if merged_length < 0 { return -1 }
|
||||
length = AppendWithoutDots(out, length, TextView(merged[0:merged_length]))
|
||||
}
|
||||
return AppendTail(out, length, r.has_query, r.query, r.has_fragment, r.fragment)
|
||||
}
|
||||
|
||||
// The address to load for VALUE on the page at BASE, written into OUT. A
|
||||
// typed address passes an empty BASE; without a scheme it means https.
|
||||
#program_export
|
||||
Destination :: (value: string, base: string, out: []u8) -> (url: string, error: string) {
|
||||
if value == "about:start" { return "about:start", "" }
|
||||
has_colon := false
|
||||
for byte: value { if byte == 58 { has_colon = true } }
|
||||
source := value
|
||||
prefixed: [URL_CAPACITY]u8
|
||||
if !has_colon && base == "" {
|
||||
prefixed_length := Append(prefixed[:], 0, "https://")
|
||||
prefixed_length = Append(prefixed[:], prefixed_length, value)
|
||||
if prefixed_length < 0 { return "", ERROR_LENGTH }
|
||||
source = TextView(prefixed[0:prefixed_length])
|
||||
}
|
||||
length := ResolveReference(source, base, out)
|
||||
if length < 0 { return "", ERROR_LENGTH }
|
||||
url := TextView(out[0:length])
|
||||
parts := SplitUrl(url)
|
||||
web := parts.scheme == "http" || parts.scheme == "https"
|
||||
if !web && parts.scheme != "file" { return "", ERROR_SCHEME }
|
||||
for byte: parts.authority { if byte == 64 { return "", ERROR_SCHEME } }
|
||||
if parts.scheme == "file" && parts.authority != "" && parts.authority != "localhost" {
|
||||
return "", ERROR_FOREIGN_FILE
|
||||
}
|
||||
if web && parts.authority == "" { return "", ERROR_HOST }
|
||||
return url, ""
|
||||
}
|
||||
|
||||
HexValue :: (byte: u8) -> s32 {
|
||||
if byte >= 48 && byte <= 57 { return cast(s32)(byte - 48) }
|
||||
if byte >= 97 && byte <= 102 { return cast(s32)(byte - 87) }
|
||||
if byte >= 65 && byte <= 70 { return cast(s32)(byte - 55) }
|
||||
return -1
|
||||
}
|
||||
|
||||
// Percent-decodes a file URL path into OUT; false for an encoded NUL or a
|
||||
// path that does not fit.
|
||||
#program_export
|
||||
DecodePath :: (path: string, out: []u8) -> (value: string, ok: bool) {
|
||||
total := cast(s32)path.count
|
||||
length: s32 = 0
|
||||
index: s32 = 0
|
||||
while index < total {
|
||||
if length >= cast(s32)out.count - 1 { return "", false }
|
||||
byte := path[index]
|
||||
if byte == 37 && index + 2 < total && HexValue(path[index + 1]) >= 0 &&
|
||||
HexValue(path[index + 2]) >= 0 {
|
||||
byte = cast(u8)(HexValue(path[index + 1]) * 16 + HexValue(path[index + 2]))
|
||||
index += 2
|
||||
}
|
||||
if byte == 0 { return "", false }
|
||||
out[length] = byte
|
||||
length += 1
|
||||
index += 1
|
||||
}
|
||||
return TextView(out[0:length]), true
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
spec = importlib.util.spec_from_file_location("fetch", Path(__file__).resolve().parents[1] / "scripts/fetch.py")
|
||||
fetch = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(fetch)
|
||||
|
||||
|
||||
class FetchTests(unittest.TestCase):
|
||||
def test_relative_urls_and_default_scheme(self):
|
||||
self.assertEqual(fetch.destination("../next.html", "https://example.com/a/index.html"), "https://example.com/next.html")
|
||||
self.assertEqual(fetch.destination("example.com"), "https://example.com")
|
||||
for url in ("javascript:alert(1)", "ftp://example.com", "https://name:[email protected]", "file://foreign/tmp/page"):
|
||||
with self.assertRaises(ValueError):
|
||||
fetch.destination(url)
|
||||
|
||||
def test_start_page_and_file_limit(self):
|
||||
self.assertIn("<h1>", fetch.fetch({"url": "about:start"})["html"])
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder) / "page.html"
|
||||
path.write_text("<h1>Local HTML</h1>")
|
||||
self.assertEqual(fetch.fetch({"url": path.as_uri()})["html"], path.read_text())
|
||||
path.write_bytes(b"x" * (fetch.LIMIT + 1))
|
||||
with self.assertRaises(ValueError):
|
||||
fetch.fetch({"url": path.as_uri()})
|
||||
|
||||
def test_redirect_cannot_open_local_files(self):
|
||||
from urllib.request import Request
|
||||
with self.assertRaises(ValueError):
|
||||
fetch.Redirects().redirect_request(Request("https://example.com"), None, 302, "", {}, "file:///private")
|
||||
|
||||
def test_http_fetch_follows_redirect_and_preserves_final_base(self):
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import threading
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == "/redirect":
|
||||
self.send_response(302)
|
||||
self.send_header("Location", "/pages/index.html")
|
||||
self.end_headers()
|
||||
else:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"<h1>HTTP document</h1><a href='next.html'>Next</a>")
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
origin = f"http://127.0.0.1:{server.server_port}"
|
||||
result = fetch.fetch({"url": origin + "/redirect"})
|
||||
self.assertEqual(result["url"], origin + "/pages/index.html")
|
||||
self.assertIn("HTTP document", result["html"])
|
||||
self.assertEqual(fetch.destination("next.html", result["url"]), origin + "/pages/next.html")
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join(2)
|
||||
server.server_close()
|
||||
@@ -0,0 +1,68 @@
|
||||
"""Fixture server for tests/transport_test.zi; every check runs in Ziran."""
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
|
||||
LIMIT = 524288
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def send(self, status, content_type, body, headers=()):
|
||||
self.send_response(status)
|
||||
if content_type:
|
||||
self.send_header("Content-Type", content_type)
|
||||
for name, value in headers:
|
||||
self.send_header(name, value)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
try:
|
||||
self.wfile.write(body)
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
pass # curl stops oversized documents early, as intended
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == "/redirect":
|
||||
self.send(302, "", b"", [("Location", "/pages/index.html")])
|
||||
elif self.path == "/pages/index.html":
|
||||
self.send(200, "text/html; charset=utf-8", b"<h1>HTTP document</h1><a href='next.html'>Next</a>")
|
||||
elif self.path == "/image":
|
||||
self.send(200, "image/png", b"\x89PNG\r\n")
|
||||
elif self.path == "/to-file":
|
||||
self.send(302, "", b"", [("Location", "file:///etc/hostname")])
|
||||
elif self.path == "/big":
|
||||
self.send(200, "text/html", b"x" * (LIMIT + 1))
|
||||
else:
|
||||
self.send(404, "text/html", b"<h1>Missing</h1>")
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
binary = Path(sys.argv[1]).resolve()
|
||||
with tempfile.TemporaryDirectory(prefix="browser-transport-") as folder:
|
||||
root = Path(folder)
|
||||
(root / "local page.html").write_text("<h1>Local HTML</h1>")
|
||||
(root / "big.html").write_bytes(b"x" * (LIMIT + 1))
|
||||
(root / "notes.txt").write_text("a <b> & c")
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
env = dict(os.environ, BROWSER_TEST_DIR=str(root),
|
||||
BROWSER_TEST_ORIGIN=f"http://127.0.0.1:{server.server_port}")
|
||||
env.pop("http_proxy", None)
|
||||
env.pop("HTTP_PROXY", None)
|
||||
return subprocess.run([str(binary)], env=env, timeout=120).returncode
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join(2)
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,144 @@
|
||||
// Address resolution and document loading without a display. Run through
|
||||
// tests/transport_test.py, which serves BROWSER_TEST_ORIGIN and writes local
|
||||
// fixtures under BROWSER_TEST_DIR.
|
||||
Url :: #import "url";
|
||||
Transport :: #import "transport";
|
||||
#import "std/byte_text_linux"
|
||||
test_libc :: #system_library "libc";
|
||||
TestSleep :: (micros: u32) -> s32 #foreign test_libc "usleep";
|
||||
TestEnvironment :: (name: *u8) -> *u8 #foreign test_libc "getenv";
|
||||
|
||||
EnvironmentText :: (name: string) -> string {
|
||||
bytes: [64]u8
|
||||
if name.count >= 64 { return "" }
|
||||
for index: 0..cast(s32)name.count - 1 { bytes[index] = name[index] }
|
||||
bytes[name.count] = 0
|
||||
value := TestEnvironment(*bytes[0])
|
||||
if value == null { return "" }
|
||||
return TextFromCString(value)
|
||||
}
|
||||
|
||||
failures: s32;
|
||||
resolved: [4096]u8;
|
||||
joined: [4096]u8;
|
||||
|
||||
Check :: (ok: bool, name: string) { if !ok { print("FAIL: %\n", name); failures += 1 } }
|
||||
|
||||
Resolves :: (value: string, base: string, expected: string) {
|
||||
url, error := Url.Destination(value, base, resolved[:])
|
||||
if error != "" || url != expected { print(" got '%' error '%'\n", url, error) }
|
||||
Check(error == "" && url == expected, expected)
|
||||
}
|
||||
|
||||
Refused :: (value: string, base: string, expected_error: string) {
|
||||
url, error := Url.Destination(value, base, resolved[:])
|
||||
Check(error != "" && (expected_error == "" || error == expected_error), value)
|
||||
}
|
||||
|
||||
Join :: (left: string, right: string) -> string {
|
||||
length := Url.Append(joined[:], 0, left)
|
||||
length = Url.Append(joined[:], length, right)
|
||||
if length < 0 { return "" }
|
||||
return TextView(joined[0:length])
|
||||
}
|
||||
|
||||
// Loads TARGET and waits for the result; false when it never finishes.
|
||||
Load :: (target: string, base: string) -> bool {
|
||||
if !Transport.Start(target, base) { return true }
|
||||
for attempt: 0..400 {
|
||||
if Transport.Poll() { return true }
|
||||
unused TestSleep(50000)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Contains :: (value: string, wanted: string) -> bool {
|
||||
if wanted.count > value.count { return false }
|
||||
last := cast(s32)(value.count - wanted.count)
|
||||
for index: 0..last {
|
||||
if value[index:index + cast(s32)wanted.count] == wanted { return true }
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
Main :: () -> s32 {
|
||||
Resolves("../next.html", "https://example.com/a/index.html", "https://example.com/next.html")
|
||||
Resolves("example.com", "", "https://example.com")
|
||||
Resolves("next.html?x=1#y", "http://host/p/q", "http://host/p/next.html?x=1#y")
|
||||
Resolves("//cdn.example/x", "https://a/b", "https://cdn.example/x")
|
||||
Resolves("?q", "https://a/b/c?d#e", "https://a/b/c?q")
|
||||
Resolves("#f", "https://a/b?c", "https://a/b?c#f")
|
||||
Resolves("/x/./y/../z", "https://a/b", "https://a/x/z")
|
||||
Resolves("child/", "https://a", "https://a/child/")
|
||||
Resolves("HTTPS://Example.com/Path", "", "https://Example.com/Path")
|
||||
Resolves("about:start", "", "about:start")
|
||||
Refused("javascript:alert(1)", "", Url.ERROR_SCHEME)
|
||||
Refused("ftp://example.com", "", Url.ERROR_SCHEME)
|
||||
Refused("https://name:[email protected]", "", Url.ERROR_SCHEME)
|
||||
Refused("file://foreign/tmp/page", "", Url.ERROR_FOREIGN_FILE)
|
||||
Refused("https:///path", "", Url.ERROR_HOST)
|
||||
Refused("next.html", "about:start", "")
|
||||
|
||||
Check(Transport.AcceptedType("text/html; charset=utf-8"), "HTML content type")
|
||||
Check(Transport.AcceptedType("Application/XHTML+XML"), "XHTML content type")
|
||||
Check(Transport.AcceptedType(""), "missing content type counts as text")
|
||||
Check(!Transport.AcceptedType("image/png"), "binary content type refused")
|
||||
invalid: [4]u8 = .[65, 255, 60, 66]
|
||||
cleaned: [32]u8
|
||||
length := Transport.AppendUtf8(cleaned[:], 0, invalid[:], true, 0)
|
||||
Check(length == 9 && cleaned[0] == 65 && cleaned[1] == 239 && cleaned[2] == 191 &&
|
||||
cleaned[3] == 189 && TextView(cleaned[4:9]) == "<B", "invalid UTF-8 and escaping")
|
||||
|
||||
Check(Load("about:start", "") && Transport.error == "" && Contains(Transport.Html(), "<h1>"),
|
||||
"built-in start page")
|
||||
|
||||
directory := EnvironmentText("BROWSER_TEST_DIR")
|
||||
origin := EnvironmentText("BROWSER_TEST_ORIGIN")
|
||||
if directory == "" || origin == "" {
|
||||
print("FAIL: run through tests/transport_test.py\n")
|
||||
return failures + 1
|
||||
}
|
||||
page_url: [4096]u8
|
||||
page_length := Url.Append(page_url[:], 0, Join(Join("file://", directory), "/local%20page.html"))
|
||||
page := TextView(page_url[0:page_length])
|
||||
Check(Load(page, "") && Transport.error == "" && Transport.Html() == "<h1>Local HTML</h1>",
|
||||
"local file with an encoded space")
|
||||
big_url: [4096]u8
|
||||
big_length := Url.Append(big_url[:], 0, Join(Join("file://", directory), "/big.html"))
|
||||
Check(Load(TextView(big_url[0:big_length]), "") && Transport.error == Transport.ERROR_LIMIT,
|
||||
"local file over the page limit")
|
||||
notes_url: [4096]u8
|
||||
notes_length := Url.Append(notes_url[:], 0, Join(Join("file://", directory), "/notes.txt"))
|
||||
Check(Load(TextView(notes_url[0:notes_length]), "") && Transport.Html() == "<pre>a <b> & c</pre>",
|
||||
"text file shown as escaped preformatted text")
|
||||
|
||||
redirect: [4096]u8
|
||||
redirect_length := Url.Append(redirect[:], 0, Join(origin, "/redirect"))
|
||||
Check(Load(TextView(redirect[0:redirect_length]), "") && Transport.error == "" &&
|
||||
Contains(Transport.Html(), "HTTP document"), "HTTP redirect loads the target document")
|
||||
Check(Transport.CurrentUrl() == Join(origin, "/pages/index.html"), "final URL after redirect")
|
||||
current: [4096]u8
|
||||
current_length := Url.Append(current[:], 0, Transport.CurrentUrl())
|
||||
Resolves("next.html", TextView(current[0:current_length]), Join(origin, "/pages/next.html"))
|
||||
image: [4096]u8
|
||||
image_length := Url.Append(image[:], 0, Join(origin, "/image"))
|
||||
Check(Load(TextView(image[0:image_length]), "") && Transport.error == Transport.ERROR_TYPE,
|
||||
"binary documents refused")
|
||||
to_file: [4096]u8
|
||||
to_file_length := Url.Append(to_file[:], 0, Join(origin, "/to-file"))
|
||||
Check(Load(TextView(to_file[0:to_file_length]), "") && Transport.error == Transport.ERROR_REDIRECT,
|
||||
"a website cannot redirect to a local file")
|
||||
missing: [4096]u8
|
||||
missing_length := Url.Append(missing[:], 0, Join(origin, "/missing"))
|
||||
Check(Load(TextView(missing[0:missing_length]), "") && Transport.error == Transport.ERROR_LOAD,
|
||||
"HTTP errors report a load failure")
|
||||
big_web: [4096]u8
|
||||
big_web_length := Url.Append(big_web[:], 0, Join(origin, "/big"))
|
||||
Check(Load(TextView(big_web[0:big_web_length]), "") && Transport.error == Transport.ERROR_LIMIT,
|
||||
"web document over the page limit")
|
||||
|
||||
if failures == 0 {
|
||||
print("URL resolution, start page, local files, HTTP redirects, type and size limits verified\n")
|
||||
}
|
||||
return failures
|
||||
}
|
||||
Reference in new issue
Block a user