123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488 | package render_test
import (
"io/fs"
"maps"
"net/http"
"os"
"path"
"path/filepath"
"slices"
"strings"
"testing"
"code.bruce-hill.com/repo-renderer/internal/config"
"code.bruce-hill.com/repo-renderer/internal/httpd"
)
// TestHTTPDConf holds examples/httpd.conf
// to the rendered site. It reads the config and answers every URL of a
// rendered site, the private one too, the way httpd would by
// httpd.conf(5), with a login and without: each file the renderer wrote
// is served, by the URL the pages link to, with the right headers.
func TestHTTPDConf(t *testing.T) {
e := newEnv(t, "proj", "")
e.configure(e.privateConfig())
e.place("proj", "public")
e.seed()
e.git(e.work, "tag", "v1.0")
e.git(e.work, "push", "-q", "origin", "v1.0")
e.git("", "clone", "-q", "--bare", e.site.Lay.GitDir("proj"), e.site.Lay.GitDir("secret"))
if err := e.site.RenderAll(false); err != nil {
t.Fatal(err)
}
conf := readHTTPDConf(t, filepath.Join("..", "..", "examples", "httpd.conf"))
htdocs := e.site.Lay.Htdocs
// The example's private root is the test's private_htdocs.
roots := map[string]string{"": htdocs, "/private/repo-render": e.site.Cfg.PrivateHtdocs}
files := map[string]string{} // URL: the file it serves
for _, root := range []string{htdocs, e.site.Cfg.PrivateHtdocs} {
if err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
// A .gz twin is what gzip-static serves for its file, not a URL.
if twin, ok := strings.CutSuffix(p, ".gz"); ok {
if _, err := os.Stat(twin); err == nil {
return nil
}
}
rel, err := filepath.Rel(root, p)
if err != nil {
return err
}
files[publicURL(filepath.ToSlash(rel))] = p
return nil
}); err != nil {
t.Fatal(err)
}
}
want := map[string]int{} // URLs whose answer the site depends on, logged in
for _, u := range []string{"/code/proj", "/code/proj/src/main.c", "/code/proj/foo.html",
"/search/proj/", "/types/proj/code/c", "/raw/proj/foo.html", "/static/style.css",
"/git/proj.git/HEAD", "/git/proj.git/info/refs", "/git/proj.git/objects/info/packs",
"/private/", "/private/code/secret", "/private/code/secret/src/main.c", "/private/search/secret/",
"/private/types/secret/code/c", "/private/raw/secret/foo.html", "/private/git/secret.git/info/refs"} {
want[u] = http.StatusOK
}
for u, code := range map[string]int{
"/private/code/proj": http.StatusNotFound, "/code/secret": http.StatusNotFound,
"/code/proj/nope": http.StatusNotFound, "/raw/proj/nope": http.StatusNotFound,
// A directory, as httpd has it: redirected to its path with a
// slash, which is forbidden without an index.html. The search
// form submits to the slash, as the redirect would drop its query.
"/search/proj": http.StatusMovedPermanently,
"/raw/proj/src": http.StatusMovedPermanently, "/raw/proj/src/": http.StatusForbidden,
// What git probes for and expects to be missing: a 403 here makes
// it print an error on every clone.
"/git/proj.git/objects/00/00000000000000000000000000000000000000": http.StatusNotFound,
"/git/proj.git/objects/info/alternates": http.StatusNotFound,
"/git/proj.git/objects/info/http-alternates": http.StatusNotFound,
} {
want[u] = code
}
mustContain(t, "search form", e.read(e.page("")), `action="/search/proj/"`)
urls := slices.Sorted(maps.Keys(files))
for u := range want {
if _, ok := files[u]; !ok {
urls = append(urls, u)
}
}
for _, u := range urls {
for _, login := range []bool{true, false} {
got := conf.serve(t, roots, u, login)
private := strings.HasPrefix(u, config.PrivatePrefix+"/")
if private && !login {
if got.status != http.StatusUnauthorized {
t.Errorf("GET %s without a login: %d", u, got.status)
}
continue
}
if w, ok := want[u]; ok {
if got.status != w {
t.Errorf("GET %s: %d, want %d", u, got.status, w)
}
} else if got.status != http.StatusOK {
t.Errorf("GET %s: %d, want the file %s", u, got.status, files[u])
}
if f, ok := files[u]; ok && got.status == http.StatusOK && got.file != f {
t.Errorf("GET %s: serves %s, not %s", u, got.file, f)
}
if private && got.status == http.StatusOK && strings.Contains(got.header["Cache-Control"], "public") {
t.Errorf("GET %s: private file cached as public: %q", u, got.header["Cache-Control"])
}
// The rules below are the same on both sites.
u := strings.TrimPrefix(u, config.PrivatePrefix)
if csp := got.header["Content-Security-Policy"]; strings.HasPrefix(u, "/raw/") || strings.HasPrefix(u, "/svg/") {
if csp != "sandbox" {
t.Errorf("GET %s: the repository's own file served with CSP %q", u, csp)
}
} else if !strings.Contains(csp, "default-src 'none'") || !strings.Contains(csp, "frame-ancestors 'self'") {
t.Errorf("GET %s: CSP %q", u, csp)
}
// URLs named by what they hold are cached for good, and all others
// are asked for again each time, so no page outlives a push.
named := strings.HasPrefix(u, "/snapshot/") || strings.HasPrefix(u, "/static/") ||
strings.HasSuffix(u, "/corpus.json") || strings.Contains(u, "/objects/pack/pack-")
if cc := got.header["Cache-Control"]; got.status == http.StatusOK && named != strings.Contains(cc, "immutable") {
t.Errorf("GET %s: Cache-Control %q", u, cc)
} else if got.status == http.StatusOK && !named && cc != "no-cache" {
t.Errorf("GET %s: Cache-Control %q, want no-cache", u, cc)
}
if got.status == http.StatusOK && got.gz && !got.gzipStatic && !strings.HasPrefix(u, "/raw/") {
t.Errorf("GET %s: %s.gz is never served: no gzip-static", u, got.file)
}
// A repository's foo.gz is a file of its own, not foo compressed.
if strings.HasPrefix(u, "/raw/") && got.gzipStatic {
t.Errorf("GET %s: gzip-static on the repositories' own files", u)
}
}
}
// httpd's errdocs serves these pages for the site's errors.
e.mustExist("err/404.html", "err/403.html")
mustContain(t, "404 page", e.read("err/404.html"), "<h1>Page not found</h1>", `href="/static/style.css?v=`)
mustContain(t, "403 page", e.read("err/403.html"), "<h1>Nothing to show here</h1>")
}
// publicURL is the URL httpd serves a file in htdocs, or in
// private_htdocs, at.
func publicURL(rel string) string {
if rest, ok := strings.CutPrefix(rel, config.PrivateDir+"/"); ok {
return config.PrivatePrefix + publicURL(rest)
}
for _, dir := range []string{"code/", "types/"} {
if strings.HasPrefix(rel, dir) && strings.HasSuffix(rel, ".html") {
return "/" + strings.TrimSuffix(rel, ".html")
}
}
if dir, ok := strings.CutSuffix(rel, "index.html"); ok && (dir == "" || strings.HasSuffix(dir, "/")) {
return "/" + dir
}
return "/" + rel
}
// ---- a model of httpd(8) serving an httpd.conf ----
type httpdConf struct {
header map[string]string // server-wide headers
gzipStatic bool
locations []httpdLocation
}
type httpdLocation struct {
loc httpd.Location
rewrite string
gzipStatic *bool // nil: the server's
block bool
header map[string]string // set, or "" for removed
// root is the location's own root in the chroot, "" for the
// server's.
root string
auth bool // asks for a login
// elsewhere strips the path to serve from its root (acme-client's
// challenges), so no URL of the site may land in it.
elsewhere bool
}
type httpdAnswer struct {
status int
file, body string
header map[string]string
gz bool // a .gz twin exists
gzipStatic bool
block bool
}
// serve answers a GET of u as httpd.conf(5), server_http.c and
// server_file.c have it: the first matching location wins; a rewrite
// happens once, and then the new path is looked up again; that location
// asks for a login, and serves from its root, the server's or its own,
// which roots maps to a directory; a directory is redirected to its path
// with a slash, which serves its index.html, looked up again, or is
// forbidden.
func (c *httpdConf) serve(t *testing.T, roots map[string]string, u string, login bool) httpdAnswer {
t.Helper()
p := u
if i := strings.IndexByte(p, '?'); i >= 0 {
p = p[:i]
}
loc, caps := c.lookup(p)
if loc != nil && loc.rewrite != "" {
p = path.Clean(httpd.Rewrite(loc.rewrite, caps))
loc, _ = c.lookup(p)
}
if loc != nil && loc.elsewhere {
t.Fatalf("%s lands in a location that serves files from outside the site", u)
}
a := c.answer(loc)
if a.block {
a.status = http.StatusForbidden
return a
}
if loc != nil && loc.auth && !login {
a.status = http.StatusUnauthorized
return a
}
dir := func(loc *httpdLocation) string {
root := ""
if loc != nil {
root = loc.root
}
d, ok := roots[root]
if !ok {
t.Fatalf("%s: no directory for root %q", u, root)
}
return d
}
a.file = filepath.Join(dir(loc), filepath.FromSlash(p))
if st, err := os.Stat(a.file); err == nil && st.IsDir() {
if !strings.HasSuffix(p, "/") {
a.status = http.StatusMovedPermanently
a.header["Location"] = p + "/"
return a
}
p += "index.html"
loc, _ = c.lookup(p)
a = c.answer(loc)
a.file = filepath.Join(dir(loc), filepath.FromSlash(p))
if _, err := os.Stat(a.file); err != nil {
a.status = http.StatusForbidden
return a
}
}
b, err := os.ReadFile(a.file)
if err != nil {
a.status = http.StatusNotFound
return a
}
_, err = os.Stat(a.file + ".gz")
a.status, a.body, a.gz = http.StatusOK, string(b), err == nil
return a
}
// answer is what loc, or the server when it is nil, sends with a response.
func (c *httpdConf) answer(loc *httpdLocation) httpdAnswer {
// static-cache-control, on by default, sends no-cache with every file.
a := httpdAnswer{header: map[string]string{"Cache-Control": "no-cache"}}
for k, v := range c.header {
a.header[k] = v
}
a.gzipStatic = c.gzipStatic
if loc != nil {
for k, v := range loc.header {
a.header[k] = v
}
if loc.gzipStatic != nil {
a.gzipStatic = *loc.gzipStatic
}
}
a.block = loc != nil && loc.block
return a
}
func (c *httpdConf) lookup(p string) (*httpdLocation, []string) {
for i := range c.locations {
if caps, ok := c.locations[i].loc.Match(p); ok {
return &c.locations[i], caps
}
}
return nil, nil
}
// readHTTPDConf reads the server block of an httpd.conf, expanding macros.
// Anything it does not model fails the test, so a new directive gets a
// model before the test is trusted to check it.
func readHTTPDConf(t *testing.T, file string) *httpdConf {
t.Helper()
src, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
stmts := parseHTTPDConf(t, tokenizeHTTPDConf(t, string(src)))
c := &httpdConf{header: map[string]string{}}
servers := 0
for _, s := range stmts {
switch s.words[0] {
case "types":
case "server":
servers++
for _, o := range s.block {
switch o.words[0] {
case "listen", "tls", "root", "default", "errdocs", "alias", "log":
case "gzip-static":
c.gzipStatic = true
case "header":
setHeader(t, c.header, o.words)
case "location":
c.locations = append(c.locations, readLocation(t, o))
default:
t.Fatalf("httpd.conf: server option %q has no model", o.words)
}
}
default:
t.Fatalf("httpd.conf: %q has no model", s.words)
}
}
if servers != 1 {
t.Fatalf("httpd.conf: %d server blocks, want 1", servers)
}
return c
}
func readLocation(t *testing.T, s httpdStmt) httpdLocation {
t.Helper()
l := httpdLocation{header: map[string]string{}}
switch {
case len(s.words) == 3 && s.words[1] == "match":
l.loc = httpd.PatternLocation(s.words[2])
case len(s.words) == 2:
l.loc = httpd.GlobLocation(s.words[1])
default:
t.Fatalf("httpd.conf: location %q has no model", s.words)
}
if err := l.loc.Check(); err != nil {
t.Fatalf("httpd.conf: %v", err)
}
for _, o := range s.block {
switch {
case len(o.words) == 1 && o.words[0] == "gzip-static":
l.gzipStatic = new(true)
case len(o.words) == 2 && o.words[0] == "no" && o.words[1] == "gzip-static":
l.gzipStatic = new(false)
case len(o.words) == 2 && o.words[0] == "root":
l.root = o.words[1]
case len(o.words) == 3 && o.words[0] == "request" && o.words[1] == "strip":
l.elsewhere = true
case len(o.words) == 4 && o.words[0] == "authenticate" && o.words[2] == "with":
l.auth = true
case len(o.words) == 1 && o.words[0] == "block":
l.block = true
case len(o.words) == 3 && o.words[0] == "request" && o.words[1] == "rewrite":
l.rewrite = o.words[2]
if err := httpd.CheckRewrite(l.loc, l.rewrite); err != nil {
t.Fatalf("httpd.conf: %v", err)
}
case o.words[0] == "header":
setHeader(t, l.header, o.words)
default:
t.Fatalf("httpd.conf: location option %q has no model", o.words)
}
}
return l
}
// setHeader models `header set NAME VALUE always` and `header remove NAME
// always`, and Cache-Control without always.
func setHeader(t *testing.T, h map[string]string, w []string) {
t.Helper()
switch {
case len(w) == 5 && w[1] == "set" && w[4] == "always":
h[w[2]] = w[3]
case len(w) == 4 && w[1] == "remove" && w[3] == "always":
h[w[2]] = ""
case len(w) == 4 && w[1] == "set" && w[2] == "Cache-Control":
// Successful responses only, which is all it is for.
h[w[2]] = w[3]
default:
t.Fatalf("httpd.conf: %q has no model", w)
}
}
type httpdStmt struct {
words []string
block []httpdStmt
}
// tokenizeHTTPDConf splits a config into words, "{", "}" and "\n", with
// comments dropped, quotes removed and $macros expanded.
func tokenizeHTTPDConf(t *testing.T, src string) []string {
t.Helper()
var toks []string
macros := map[string]string{}
for line := range strings.Lines(src) {
var words []string
quoted := map[int]bool{}
for i := 0; i < len(line); {
switch c := line[i]; {
case c == '#':
i = len(line)
case c == ' ' || c == '\t' || c == '\n':
i++
case c == '"':
end := strings.IndexByte(line[i+1:], '"')
if end < 0 {
t.Fatalf("httpd.conf: unterminated quote: %q", line)
}
quoted[len(words)] = true
words = append(words, line[i+1:i+1+end])
i += end + 2
case c == '{' || c == '}' || c == '=':
words = append(words, string(c))
i++
default:
end := strings.IndexAny(line[i:], " \t\n{}=#\"")
if end < 0 {
end = len(line) - i
}
words = append(words, line[i:i+end])
i += end
}
}
if len(words) == 3 && words[1] == "=" {
macros[words[0]] = words[2]
continue
}
for i, w := range words {
if name, ok := strings.CutPrefix(w, "$"); ok && !quoted[i] {
v, ok := macros[name]
if !ok {
t.Fatalf("httpd.conf: undefined macro %s", w)
}
w = v
}
toks = append(toks, w)
}
toks = append(toks, "\n")
}
return toks
}
func parseHTTPDConf(t *testing.T, toks []string) []httpdStmt {
t.Helper()
stmts, rest := parseHTTPDBlock(t, toks)
if len(rest) != 0 {
t.Fatalf("httpd.conf: stray %q", rest[0])
}
return stmts
}
func parseHTTPDBlock(t *testing.T, toks []string) ([]httpdStmt, []string) {
var stmts []httpdStmt
var cur httpdStmt
flush := func() {
if len(cur.words) > 0 {
stmts = append(stmts, cur)
}
cur = httpdStmt{}
}
for len(toks) > 0 {
tok := toks[0]
toks = toks[1:]
switch tok {
case "\n":
flush()
case "{":
cur.block, toks = parseHTTPDBlock(t, toks)
flush()
case "}":
flush()
return stmts, toks
default:
cur.words = append(cur.words, tok)
}
}
flush()
return stmts, nil
}
|