Added TokenBucket to limit the rate of validation failures (#151)

Added new TockenBucket named `acmeClientFailLimit` to avoid being banned because of the [Failed validation limit](https://letsencrypt.org/docs/failed-validation-limit/) of Let's Encrypt.

The behaviour is similar to the other limiters blocking the `obtainCert` func ensuring rate under limit.

Co-authored-by: fsologureng <sologuren@estudiohum.cl>
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/151
Reviewed-by: 6543 <6543@obermui.de>
Co-authored-by: Felipe Leopoldo Sologuren Gutiérrez <fsologureng@noreply.codeberg.org>
Co-committed-by: Felipe Leopoldo Sologuren Gutiérrez <fsologureng@noreply.codeberg.org>

.ecrc

@@ -0,0 +1,9 @@
+{
+  "Exclude": [
+    ".git",
+    "go.mod", "go.sum",
+    "vendor",
+    "LICENSE",
+    "_test.go"
+  ]
+}

.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+tab_width = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.go]
+indent_style = tab
+
+[*.md]
+trim_trailing_whitespace = false
+indent_size = 1

.golangci.yml

@@ -0,0 +1,20 @@
+linters-settings:
+  gocritic:
+    enabled-tags:
+      - diagnostic
+      - experimental
+      - opinionated
+      - performance
+      - style
+    disabled-checks:
+      - importShadow
+      - ifElseChain
+      - hugeParam
+
+linters:
+  enable:
+    - unconvert
+    - gocritic
+
+run:
+  timeout: 5m

.woodpecker.yml

@@ -1,5 +1,3 @@
-branches: main
-
 pipeline:
   # use vendor to cache dependencies
   vendor:
@@ -17,18 +15,33 @@ pipeline:
       - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
       - golangci-lint run --timeout 5m --build-tags integration
 
+  editor-config:
+    group: compliant
+    image: mstruebing/editorconfig-checker
+
   build:
     group: compliant
-    image: a6543/golang_just
+    image: codeberg.org/6543/docker-images/golang_just
     commands:
       - go version
       - just build
     when:
       event: [ "pull_request", "push" ]
 
+  docker-dryrun:
+    group: compliant
+    image: plugins/kaniko
+    settings:
+      dockerfile: Dockerfile
+      no_push: true
+      tags: latest
+    when:
+      event: [ "pull_request", "push" ]
+      path: Dockerfile
+
   build-tag:
     group: compliant
-    image: a6543/golang_just
+    image: codeberg.org/6543/docker-images/golang_just
     commands:
       - go version
       - just build-tag ${CI_COMMIT_TAG##v}
@@ -36,14 +49,14 @@ pipeline:
       event: [ "tag" ]
 
   test:
-    image: a6543/golang_just
     group: test
+    image: codeberg.org/6543/docker-images/golang_just
     commands:
       - just test
 
   integration-tests:
-    image: a6543/golang_just
     group: test
+    image: codeberg.org/6543/docker-images/golang_just
     commands:
       - just integration
     environment:
@@ -67,3 +80,32 @@ pipeline:
       - DRONE_COMMIT_REF=${CI_COMMIT_REF}
     when:
       event: [ "tag" ]
+
+  docker-next:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tags: next
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "push" ]
+      branch: ${CI_REPO_DEFAULT_BRANCH}
+
+  docker-tag:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tags: [ latest, "${CI_COMMIT_TAG}" ]
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "tag" ]

Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:alpine as build
+
+WORKDIR /workspace
+
+RUN apk add ca-certificates
+COPY . .
+RUN CGO_ENABLED=0 go build .
+
+FROM scratch
+COPY --from=build /workspace/pages /pages
+COPY --from=build \
+    /etc/ssl/certs/ca-certificates.crt \
+    /etc/ssl/certs/ca-certificates.crt
+
+ENTRYPOINT ["/pages"]

Justfile

@@ -6,7 +6,8 @@ dev:
     export PAGES_DOMAIN=localhost.mock.directory
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
-    go run . --verbose
+    export LOG_LEVEL=trace
+    go run .
 
 build:
     CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
@@ -17,6 +18,7 @@ build-tag VERSION:
 lint: tool-golangci tool-gofumpt
     [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
     golangci-lint run --timeout 5m --build-tags integration
+    # TODO: run editorconfig-checker
 
 fmt: tool-gofumpt
     gofumpt -w --extra .
@@ -36,10 +38,10 @@ tool-gofumpt:
     fi
 
 test:
-    go test -race codeberg.org/codeberg/pages/server/...
+    go test -race codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
 
 test-run TEST:
-    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
+    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
 
 integration:
     go test -race -tags integration codeberg.org/codeberg/pages/integration/...

README.md

@@ -8,7 +8,6 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag
 **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
 and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
 
-
 ## Quickstart
 
 This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
@@ -20,17 +19,16 @@ first line will be the canonical domain/URL; all other occurrences will be redir
 2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
 "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
 If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
-	`www.example.org. IN CNAME main.pages.example.codeberg.page.`
+  `www.example.org. IN CNAME main.pages.example.codeberg.page.`
 
 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
 for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
 record that points to your repo (just like the CNAME record):
-	`example.org IN ALIAS codeberg.page.`
+  `example.org IN ALIAS codeberg.page.`
-	`example.org IN TXT main.pages.example.codeberg.page.`
+  `example.org IN TXT main.pages.example.codeberg.page.`
 
 Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
 
-
 ## Deployment
 
 **Warning: Some Caveats Apply**  
@@ -70,7 +68,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg
 - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
 - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.  
   See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
-- `DEBUG` (default: false): Set this to true to enable debug logging.
+- `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
 
 
 ## Contributing to the development

cmd/flags.go

@@ -5,12 +5,6 @@ import (
 )
 
 var ServeFlags = []cli.Flag{
-	&cli.BoolFlag{
-		Name: "verbose",
-		// TODO: Usage
-		EnvVars: []string{"DEBUG"},
-	},
-
 	// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
 	// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
 	// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
@@ -69,6 +63,25 @@ var ServeFlags = []cli.Flag{
 		// TODO: desc
 		EnvVars: []string{"ENABLE_HTTP_SERVER"},
 	},
+	// Server Options
+	&cli.BoolFlag{
+		Name:    "enable-lfs-support",
+		Usage:   "enable lfs support, require gitea v1.17.0 as backend",
+		EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		Name:    "enable-symlink-support",
+		Usage:   "follow symlinks if enabled, require gitea v1.18.0 as backend",
+		EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+		Value:   true,
+	},
+	&cli.StringFlag{
+		Name:    "log-level",
+		Value:   "warn",
+		Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
+		EnvVars: []string{"LOG_LEVEL"},
+	},
 
 	// ACME
 	&cli.StringFlag{

cmd/main.go

@@ -1,12 +1,13 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"errors"
 	"fmt"
 	"net"
+	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -19,32 +20,35 @@ import (
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
 	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/handler"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
 // TODO: make it a flag
-var AllowedCorsDomains = [][]byte{
+var AllowedCorsDomains = []string{
-	[]byte("fonts.codeberg.org"),
+	"fonts.codeberg.org",
-	[]byte("design.codeberg.org"),
+	"design.codeberg.org",
 }
 
 // BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
 // TODO: Make it a flag too
-var BlacklistedPaths = [][]byte{
+var BlacklistedPaths = []string{
-	[]byte("/.well-known/acme-challenge/"),
+	"/.well-known/acme-challenge/",
 }
 
 // Serve sets up and starts the web server.
 func Serve(ctx *cli.Context) error {
-	verbose := ctx.Bool("verbose")
+	// Initalize the logger.
-	if !verbose {
+	logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	if err != nil {
+		return err
 	}
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
 
 	giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
 	giteaAPIToken := ctx.String("gitea-api-token")
 	rawDomain := ctx.String("raw-domain")
-	mainDomainSuffix := []byte(ctx.String("pages-domain"))
+	mainDomainSuffix := ctx.String("pages-domain")
 	rawInfoPage := ctx.String("raw-info-page")
 	listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
 	enableHTTPServer := ctx.Bool("enable-http-server")
@@ -61,13 +65,13 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	allowedCorsDomains := AllowedCorsDomains
-	if len(rawDomain) != 0 {
+	if rawDomain != "" {
-		allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain))
+		allowedCorsDomains = append(allowedCorsDomains, rawDomain)
 	}
 
 	// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
-	if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
+	if !strings.HasPrefix(mainDomainSuffix, ".") {
-		mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
+		mainDomainSuffix = "." + mainDomainSuffix
 	}
 
 	keyCache := cache.NewKeyValueCache()
@@ -76,32 +80,28 @@ func Serve(ctx *cli.Context) error {
 	canonicalDomainCache := cache.NewKeyValueCache()
 	// dnsLookupCache stores DNS lookups for custom domains
 	dnsLookupCache := cache.NewKeyValueCache()
-	// branchTimestampCache stores branch timestamps for faster cache checking
+	// clientResponseCache stores responses from the Gitea server
-	branchTimestampCache := cache.NewKeyValueCache()
+	clientResponseCache := cache.NewKeyValueCache()
-	// fileResponseCache stores responses from the Gitea server
-	// TODO: make this an MRU cache with a size limit
-	fileResponseCache := cache.NewKeyValueCache()
 
-	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken)
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
 	if err != nil {
 		return fmt.Errorf("could not create new gitea client: %v", err)
 	}
 
 	// Create handler based on settings
-	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
+	httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
 		giteaClient,
-		giteaRoot, rawInfoPage,
+		rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
-		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
+		dnsLookupCache, canonicalDomainCache)
 
-	fastServer := server.SetupServer(handler)
+	httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
-	httpServer := server.SetupHTTPACMEChallengeServer(challengeCache)
 
 	// Setup listener and TLS
 	log.Info().Msgf("Listening on https://%s", listeningAddress)
 	listener, err := net.Listen("tcp", listeningAddress)
 	if err != nil {
-		return fmt.Errorf("couldn't create listener: %s", err)
+		return fmt.Errorf("couldn't create listener: %v", err)
 	}
 
 	// TODO: make "key-database.pogreb" set via flag
@@ -134,8 +134,8 @@ func Serve(ctx *cli.Context) error {
 
 	if enableHTTPServer {
 		go func() {
-			log.Info().Timestamp().Msg("Start listening on :80")
+			log.Info().Msg("Start HTTP server listening on :80")
-			err := httpServer.ListenAndServe("[::]:80")
+			err := http.ListenAndServe("[::]:80", httpHandler)
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
 			}
@@ -143,9 +143,8 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	// Start the web fastServer
-	log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr())
+	log.Info().Msgf("Start listening on %s", listener.Addr())
-	err = fastServer.Serve(listener)
+	if err := http.Serve(listener, httpsHandler); err != nil {
-	if err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")
 	}
 

go.mod

@@ -1,17 +1,17 @@
 module codeberg.org/codeberg/pages
 
-go 1.18
+go 1.19
 
 require (
+	code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
 	github.com/akrylysov/pogreb v0.10.1
 	github.com/go-acme/lego/v4 v4.5.3
+	github.com/joho/godotenv v1.4.0
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
-	github.com/rs/zerolog v1.26.0
+	github.com/rs/zerolog v1.27.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
-	github.com/valyala/fasthttp v1.31.0
-	github.com/valyala/fastjson v1.6.3
 )
 
 require (
@@ -30,7 +30,6 @@ require (
 	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
 	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
-	github.com/andybalholm/brotli v1.0.2 // indirect
 	github.com/aws/aws-sdk-go v1.39.0 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
 	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
@@ -38,6 +37,7 @@ require (
 	github.com/cpu/goacmedns v0.1.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davidmz/go-pageant v1.0.2 // indirect
 	github.com/deepmap/oapi-codegen v1.6.1 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
@@ -45,6 +45,7 @@ require (
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
 	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-fed/httpsig v1.1.0 // indirect
 	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
 	github.com/gofrs/uuid v3.2.0+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
@@ -56,14 +57,13 @@ require (
 	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
 	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
 	github.com/jarcoal/httpmock v1.0.6 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/joho/godotenv v1.4.0 // indirect
 	github.com/json-iterator/go v1.1.7 // indirect
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
-	github.com/klauspost/compress v1.13.4 // indirect
 	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
@@ -72,7 +72,8 @@ require (
 	github.com/liquidweb/go-lwApi v0.0.5 // indirect
 	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
 	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/miekg/dns v1.1.43 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
@@ -103,15 +104,14 @@ require (
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/stretchr/objx v0.3.0 // indirect
 	github.com/transip/gotransip/v6 v6.6.1 // indirect
-	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
 	github.com/vultr/govultr/v2 v2.7.1 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
-	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
-	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
 	google.golang.org/api v0.20.0 // indirect

go.sum

@@ -22,6 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
+code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk=
+code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/Azure/azure-sdk-for-go v32.4.0+incompatible h1:1JP8SKfroEakYiQU2ZyPDosh8w2Tg9UopKt88VyQPt4=
 github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
@@ -66,8 +68,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 h1:dkj8/dxOQ4L1XpwCzRLqukvUBbxuNdz3FeyvHFnRjmo=
 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA=
-github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E=
-github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -95,7 +95,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
 github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
@@ -106,6 +106,8 @@ github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
+github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
 github.com/deepmap/oapi-codegen v1.6.1 h1:2BvsmRb6pogGNtr8Ann+esAbSKFXx2CZN18VpAMecnw=
 github.com/deepmap/oapi-codegen v1.6.1/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -136,6 +138,8 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ
 github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s=
 github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
+github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI=
+github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -182,7 +186,6 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -243,6 +246,9 @@ github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdv
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
+github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -282,8 +288,6 @@ github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcM
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
-github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b h1:DzHy0GlWeF0KAglaTMY7Q+khIFoG8toHP+wLFBVBQJc=
 github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -321,12 +325,15 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -428,8 +435,8 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
+github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
-github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
+github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -491,15 +498,9 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
-github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.31.0 h1:lrauRLII19afgCs2fnWRJ4M5IkV0lo2FqA61uGkNBfE=
-github.com/valyala/fasthttp v1.31.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus=
-github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc=
-github.com/valyala/fastjson v1.6.3/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
-github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc=
 github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 h1:TFXGGMHmml4rs29PdPisC/aaCzOxUu1Vsh9on/IpUfE=
 github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14/go.mod h1:RWc47jtnVuQv6+lY3c768WtXCas/Xi+U5UFc5xULmYg=
 github.com/vultr/govultr/v2 v2.7.1 h1:uF9ERet++Gb+7Cqs3p1P6b6yebeaZqVd7t5P2uZCaJU=
@@ -510,7 +511,6 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -537,8 +537,10 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
-golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
+golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -569,7 +571,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -603,8 +604,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
-golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -666,11 +667,13 @@ golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -723,7 +726,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

html/404.html

@@ -3,7 +3,7 @@
 <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width">
-    <title>%status</title>
+    <title>%status%</title>
 
     <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
     <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
@@ -23,11 +23,11 @@
 <body>
     <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
     <h1 class="mb-0 text-primary">
-		Page not found!
+        Page not found!
     </h1>
     <h5 class="text-center" style="max-width: 25em;">
-    	Sorry, but this page couldn't be found or is inaccessible (%status).<br/>
+        Sorry, but this page couldn't be found or is inaccessible (%status%).<br/>
-		We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
+        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
     </h5>
     <small class="text-muted">
         <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">

html/error.go

@@ -1,24 +1,50 @@
 package html
 
 import (
-	"bytes"
+	"html/template"
+	"net/http"
 	"strconv"
+	"strings"
 
-	"github.com/valyala/fasthttp"
+	"codeberg.org/codeberg/pages/server/context"
 )
 
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
-// with the provided status code.
+// with "%status%" and %message% replaced with the provided statusCode and msg
-func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
+func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
-	ctx.Response.SetStatusCode(code)
+	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
-	ctx.Response.Header.SetContentType("text/html; charset=utf-8")
+	ctx.RespWriter.WriteHeader(statusCode)
-	message := fasthttp.StatusMessage(code)
+
-	if code == fasthttp.StatusMisdirectedRequest {
+	msg = generateResponse(msg, statusCode)
-		message += " - domain not specified in <code>.domains</code> file"
+
+	_, _ = ctx.RespWriter.Write([]byte(msg))
+}
+
+// TODO: use template engine
+func generateResponse(msg string, statusCode int) string {
+	if msg == "" {
+		msg = strings.ReplaceAll(NotFoundPage,
+			"%status%",
+			strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
+	} else {
+		msg = strings.ReplaceAll(
+			strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)),
+			"%status%",
+			http.StatusText(statusCode))
 	}
-	if code == fasthttp.StatusFailedDependency {
+
+	return msg
+}
+
+func errorMessage(statusCode int) string {
+	message := http.StatusText(statusCode)
+
+	switch statusCode {
+	case http.StatusMisdirectedRequest:
+		message += " - domain not specified in <code>.domains</code> file"
+	case http.StatusFailedDependency:
 		message += " - target repo/branch doesn't exist or is private"
 	}
-	// TODO: use template engine?
+
-	ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
+	return message
 }

html/error.html

@@ -0,0 +1,38 @@
+<!doctype html>
+<html class="codeberg-design">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width">
+    <title>%status%</title>
+
+    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
+    <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
+    <link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
+
+    <style>
+        body {
+            margin: 0; padding: 1rem; box-sizing: border-box;
+            width: 100%; min-height: 100vh;
+            display: flex;
+            flex-direction: column;
+            align-items: center;
+            justify-content: center;
+        }
+    </style>
+</head>
+<body>
+    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
+    <h1 class="mb-0 text-primary">
+        %status%!
+    </h1>
+    <h5 class="text-center" style="max-width: 25em;">
+        Sorry, but this page couldn't be served.<br/>
+        We got an <b>"%message%"</b><br/>
+        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
+    </h5>
+    <small class="text-muted">
+        <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">
+        Static pages made easy - <a href="https://codeberg.page">Codeberg Pages</a>
+    </small>
+</body>
+</html>

html/error_test.go

@@ -0,0 +1,38 @@
+package html
+
+import (
+	"net/http"
+	"strings"
+	"testing"
+)
+
+func TestValidMessage(t *testing.T) {
+	testString := "requested blacklisted path"
+	statusCode := http.StatusForbidden
+
+	expected := strings.ReplaceAll(
+		strings.ReplaceAll(ErrorPage, "%message%", testString),
+		"%status%",
+		http.StatusText(statusCode))
+	actual := generateResponse(testString, statusCode)
+
+	if expected != actual {
+		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
+	}
+}
+
+func TestMessageWithHtml(t *testing.T) {
+	testString := `abc<img src=1 onerror=alert("xss");`
+	escapedString := "abc&lt;img src=1 onerror=alert(&#34;xss&#34;);"
+	statusCode := http.StatusNotFound
+
+	expected := strings.ReplaceAll(
+		strings.ReplaceAll(ErrorPage, "%message%", escapedString),
+		"%status%",
+		http.StatusText(statusCode))
+	actual := generateResponse(testString, statusCode)
+
+	if expected != actual {
+		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
+	}
+}

html/html.go

@@ -3,4 +3,7 @@ package html
 import _ "embed"
 
 //go:embed 404.html
-var NotFoundPage []byte
+var NotFoundPage string
+
+//go:embed error.html
+var ErrorPage string

integration/get_test.go

@@ -7,16 +7,17 @@ import (
 	"bytes"
 	"crypto/tls"
 	"io"
+	"log"
 	"net/http"
 	"net/http/cookiejar"
+	"strings"
 	"testing"
 
-	"github.com/rs/zerolog/log"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestGetRedirect(t *testing.T) {
-	log.Printf("=== TestGetRedirect ===\n")
+	log.Println("=== TestGetRedirect ===")
 	// test custom domain redirect
 	resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
 	assert.NoError(t, err)
@@ -24,13 +25,13 @@ func TestGetRedirect(t *testing.T) {
 		t.FailNow()
 	}
 	assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
-	assert.EqualValues(t, 0, getSize(resp.Body))
+	assert.EqualValues(t, `<a href="https://www.cabr2.de/">Temporary Redirect</a>.`, strings.TrimSpace(string(getBytes(resp.Body))))
 }
 
 func TestGetContent(t *testing.T) {
-	log.Printf("=== TestGetContent ===\n")
+	log.Println("=== TestGetContent ===")
 	// test get image
-	resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg")
 	assert.NoError(t, err)
 	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
 		t.FailNow()
@@ -41,53 +42,98 @@ func TestGetContent(t *testing.T) {
 	assert.Len(t, resp.Header.Get("ETag"), 42)
 
 	// specify branch
-	resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
+	resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/pag/@master/")
 	assert.NoError(t, err)
-	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+	if !assert.NotNil(t, resp) {
 		t.FailNow()
 	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
 	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
 	assert.True(t, getSize(resp.Body) > 1000)
-	assert.Len(t, resp.Header.Get("ETag"), 42)
+	assert.Len(t, resp.Header.Get("ETag"), 44)
 
 	// access branch name contains '/'
-	resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
+	resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/blumia/@docs~main/")
 	assert.NoError(t, err)
 	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
 		t.FailNow()
 	}
 	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
 	assert.True(t, getSize(resp.Body) > 100)
-	assert.Len(t, resp.Header.Get("ETag"), 42)
+	assert.Len(t, resp.Header.Get("ETag"), 44)
 
 	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
 }
 
 func TestCustomDomain(t *testing.T) {
-	log.Printf("=== TestCustomDomain ===\n")
+	log.Println("=== TestCustomDomain ===")
 	resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
 	assert.NoError(t, err)
-	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+	if !assert.NotNil(t, resp) {
 		t.FailNow()
 	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
 	assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
 	assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
 	assert.EqualValues(t, 106, getSize(resp.Body))
 }
 
 func TestGetNotFound(t *testing.T) {
-	log.Printf("=== TestGetNotFound ===\n")
+	log.Println("=== TestGetNotFound ===")
 	// test custom not found pages
-	resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/pages-404-demo/blah")
 	assert.NoError(t, err)
-	if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+	if !assert.NotNil(t, resp) {
 		t.FailNow()
 	}
+	assert.EqualValues(t, http.StatusNotFound, resp.StatusCode)
 	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
 	assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
 	assert.EqualValues(t, 37, getSize(resp.Body))
 }
 
+func TestFollowSymlink(t *testing.T) {
+	log.Printf("=== TestFollowSymlink ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+	body := getBytes(resp.Body)
+	assert.EqualValues(t, 4, len(body))
+	assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+	log.Printf("=== TestLFSSupport ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	body := strings.TrimSpace(string(getBytes(resp.Body)))
+	assert.EqualValues(t, 12, len(body))
+	assert.EqualValues(t, "actual value", body)
+}
+
+func TestGetOptions(t *testing.T) {
+	log.Println("=== TestGetOptions ===")
+	req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", http.NoBody)
+	resp, err := getTestHTTPSClient().Do(req)
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusNoContent, resp.StatusCode)
+	assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{
@@ -101,6 +147,12 @@ func getTestHTTPSClient() *http.Client {
 	}
 }
 
+func getBytes(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Bytes()
+}
+
 func getSize(stream io.Reader) int {
 	buf := new(bytes.Buffer)
 	_, _ = buf.ReadFrom(stream)

integration/main_test.go

@@ -5,30 +5,30 @@ package integration
 
 import (
 	"context"
+	"log"
 	"os"
 	"testing"
 	"time"
 
 	"codeberg.org/codeberg/pages/cmd"
 
-	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 )
 
 func TestMain(m *testing.M) {
-	log.Printf("=== TestMain: START Server ===\n")
+	log.Println("=== TestMain: START Server ===")
 	serverCtx, serverCancel := context.WithCancel(context.Background())
 	if err := startServer(serverCtx); err != nil {
-		log.Fatal().Msgf("could not start server: %v", err)
+		log.Fatalf("could not start server: %v", err)
 	}
 	defer func() {
 		serverCancel()
-		log.Printf("=== TestMain: Server STOPED ===\n")
+		log.Println("=== TestMain: Server STOPED ===")
 	}()
 
 	time.Sleep(10 * time.Second)
 
-	os.Exit(m.Run())
+	m.Run()
 }
 
 func startServer(ctx context.Context) error {
@@ -48,7 +48,7 @@ func startServer(ctx context.Context) error {
 
 	go func() {
 		if err := app.RunContext(ctx, args); err != nil {
-			log.Fatal().Msgf("run server error: %v", err)
+			log.Fatalf("run server error: %v", err)
 		}
 	}()
 

server/certificates/certificates.go

@@ -12,7 +12,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -37,7 +36,7 @@ import (
 )
 
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
-func TLSConfig(mainDomainSuffix []byte,
+func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
 	dnsProvider string,
 	acmeUseRateLimits bool,
@@ -48,45 +47,47 @@ func TLSConfig(mainDomainSuffix []byte,
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
 			sni := strings.ToLower(strings.TrimSpace(info.ServerName))
-			sniBytes := []byte(sni)
 			if len(sni) < 1 {
 				return nil, errors.New("missing sni")
 			}
 
 			if info.SupportedProtos != nil {
 				for _, proto := range info.SupportedProtos {
-					if proto == tlsalpn01.ACMETLS1Protocol {
+					if proto != tlsalpn01.ACMETLS1Protocol {
-						challenge, ok := challengeCache.Get(sni)
+						continue
-						if !ok {
-							return nil, errors.New("no challenge for this domain")
-						}
-						cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
-						if err != nil {
-							return nil, err
-						}
-						return cert, nil
 					}
+
+					challenge, ok := challengeCache.Get(sni)
+					if !ok {
+						return nil, errors.New("no challenge for this domain")
+					}
+					cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+					if err != nil {
+						return nil, err
+					}
+					return cert, nil
 				}
 			}
 
 			targetOwner := ""
-			if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
+			if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
 				// deliver default certificate for the main domain (*.codeberg.page)
-				sniBytes = mainDomainSuffix
+				sni = mainDomainSuffix
-				sni = string(sniBytes)
 			} else {
 				var targetRepo, targetBranch string
-				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
 				if targetOwner == "" {
 					// DNS not set up, return main certificate to redirect to the docs
-					sniBytes = mainDomainSuffix
+					sni = mainDomainSuffix
-					sni = string(sniBytes)
 				} else {
-					_, _ = targetRepo, targetBranch
+					targetOpt := &upstream.Options{
-					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
+						TargetOwner:  targetOwner,
+						TargetRepo:   targetRepo,
+						TargetBranch: targetBranch,
+					}
+					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
 					if !valid {
-						sniBytes = mainDomainSuffix
+						sni = mainDomainSuffix
-						sni = string(sniBytes)
 					}
 				}
 			}
@@ -99,9 +100,9 @@ func TLSConfig(mainDomainSuffix []byte,
 			var tlsCertificate tls.Certificate
 			var err error
 			var ok bool
-			if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
+			if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
 				// request a new certificate
-				if bytes.Equal(sniBytes, mainDomainSuffix) {
+				if strings.EqualFold(sni, mainDomainSuffix) {
 					return nil, errors.New("won't request certificate for main domain, something really bad has happened")
 				}
 
@@ -118,6 +119,7 @@ func TLSConfig(mainDomainSuffix []byte,
 		},
 		PreferServerCipherSuites: true,
 		NextProtos: []string{
+			"h2",
 			"http/1.1",
 			tlsalpn01.ACMETLS1Protocol,
 		},
@@ -161,6 +163,9 @@ var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
 // rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
 var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
 
+// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
+var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour)
+
 type AcmeTLSChallengeProvider struct {
 	challengeCache cache.SetGetKey
 }
@@ -193,9 +198,9 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 	return nil
 }
 
-func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
+func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
 	// parse certificate from database
-	res, err := certDB.Get(string(sni))
+	res, err := certDB.Get(sni)
 	if err != nil {
 		panic(err) // TODO: no panic
 	}
@@ -209,14 +214,14 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 	}
 
 	// TODO: document & put into own function
-	if !bytes.Equal(sni, mainDomainSuffix) {
+	if !strings.EqualFold(sni, mainDomainSuffix) {
 		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
 		if err != nil {
 			panic(err)
 		}
 
 		// renew certificates 7 days before they expire
-		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(7 * 24 * time.Hour)) {
+		if tlsCertificate.Leaf.NotAfter.Before(time.Now().Add(7 * 24 * time.Hour)) {
 			// TODO: add ValidUntil to custom res struct
 			if res.CSR != nil && len(res.CSR) > 0 {
 				// CSR stores the time when the renewal shall be tried again
@@ -227,9 +232,9 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 			}
 			go (func() {
 				res.CSR = nil // acme client doesn't like CSR to be set
-				tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+				tlsCertificate, err = obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 				if err != nil {
-					log.Printf("Couldn't renew certificate for %s: %s", sni, err)
+					log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err)
 				}
 			})()
 		}
@@ -240,7 +245,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 
 var obtainLocks = sync.Map{}
 
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
+func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
 	name := strings.TrimPrefix(domains[0], "*")
 	if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
 		domains = domains[1:]
@@ -253,7 +258,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			time.Sleep(100 * time.Millisecond)
 			_, working = obtainLocks.Load(name)
 		}
-		cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+		cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
 		if !ok {
 			return tls.Certificate{}, errors.New("certificate failed in synchronous request")
 		}
@@ -262,7 +267,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 	defer obtainLocks.Delete(name)
 
 	if acmeClient == nil {
-		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil
+		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase), nil
 	}
 
 	// request actual cert
@@ -272,10 +277,13 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 		if acmeUseRateLimits {
 			acmeClientRequestLimit.Take()
 		}
-		log.Printf("Renewing certificate for %v", domains)
+		log.Debug().Msgf("Renewing certificate for: %v", domains)
 		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
 		if err != nil {
-			log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
+			log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
+			if acmeUseRateLimits {
+				acmeClientFailLimit.Take()
+			}
 			res = nil
 		}
 	}
@@ -290,29 +298,36 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			acmeClientOrderLimit.Take()
 			acmeClientRequestLimit.Take()
 		}
-		log.Printf("Requesting new certificate for %v", domains)
+		log.Debug().Msgf("Re-requesting new certificate for %v", domains)
 		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
 			Domains:    domains,
 			Bundle:     true,
 			MustStaple: false,
 		})
+		if acmeUseRateLimits && err != nil {
+			acmeClientFailLimit.Take()
+		}
 	}
 	if err != nil {
-		log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
+		log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
 		if renew != nil && renew.CertURL != "" {
 			tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
-			if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
+			if err != nil {
+				return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
+			}
+			leaf, err := leaf(&tlsCertificate)
+			if err == nil && leaf.NotAfter.After(time.Now()) {
 				// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
 				renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
 				if err := keyDatabase.Put(name, renew); err != nil {
-					return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+					return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
 				}
 				return tlsCertificate, nil
 			}
 		}
-		return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+		return mockCert(domains[0], err.Error(), mainDomainSuffix, keyDatabase), err
 	}
-	log.Printf("Obtained certificate for %v", domains)
+	log.Debug().Msgf("Obtained certificate for %v", domains)
 
 	if err := keyDatabase.Put(name, res); err != nil {
 		return tls.Certificate{}, err
@@ -329,7 +344,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 	var myAcmeAccount AcmeAccount
 	var myAcmeConfig *lego.Config
 
-	if account, err := ioutil.ReadFile(configFile); err == nil {
+	if account, err := os.ReadFile(configFile); err == nil {
 		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
 			return nil, err
 		}
@@ -345,7 +360,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 		_, err := lego.NewClient(myAcmeConfig)
 		if err != nil {
 			// TODO: should we fail hard instead?
-			log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+			log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 		}
 		return myAcmeConfig, nil
 	} else if !os.IsNotExist(err) {
@@ -366,13 +381,13 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
 	tempClient, err := lego.NewClient(myAcmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		// accept terms & log in to EAB
 		if acmeEabKID == "" || acmeEabHmac == "" {
 			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
 			if err != nil {
-				log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
 			} else {
 				myAcmeAccount.Registration = reg
 			}
@@ -383,7 +398,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 				HmacEncoded:          acmeEabHmac,
 			})
 			if err != nil {
-				log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
 			} else {
 				myAcmeAccount.Registration = reg
 			}
@@ -392,12 +407,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 		if myAcmeAccount.Registration != nil {
 			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
 			if err != nil {
-				log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
+				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
 				select {}
 			}
-			err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
+			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
 			if err != nil {
-				log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
+				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
 				select {}
 			}
 		}
@@ -406,62 +421,62 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 	return myAcmeConfig, nil
 }
 
-func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
-	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
+	mainCertBytes, err := certDB.Get(mainDomainSuffix)
 	if err != nil {
 		return fmt.Errorf("cert database is not working")
 	}
 
 	acmeClient, err = lego.NewClient(acmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
 		if err != nil {
-			log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
 		}
 		if enableHTTPServer {
 			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
 			if err != nil {
-				log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
 			}
 		}
 	}
 
 	mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		if dnsProvider == "" {
 			// using mock server, don't use wildcard certs
 			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
 			if err != nil {
-				log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
 			}
 		} else {
 			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
 			if err != nil {
-				log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
+				log.Error().Err(err).Msg("Can't create DNS Challenge provider")
 			}
 			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
 			if err != nil {
-				log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create DNS-01 provider")
 			}
 		}
 	}
 
 	if mainCertBytes == nil {
-		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 		if err != nil {
-			log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
+			log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
 		}
 	}
 
 	return nil
 }
 
-func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
 	for {
 		// clean up expired certs
 		now := time.Now()
@@ -469,7 +484,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 		keyDatabaseIterator := certDB.Items()
 		key, resBytes, err := keyDatabaseIterator.Next()
 		for err == nil {
-			if !bytes.Equal(key, mainDomainSuffix) {
+			if !strings.EqualFold(string(key), mainDomainSuffix) {
 				resGob := bytes.NewBuffer(resBytes)
 				resDec := gob.NewDecoder(resGob)
 				res := &certificate.Resource{}
@@ -479,10 +494,10 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 				}
 
 				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
-				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
+				if err != nil || tlsCertificates[0].NotAfter.Before(now) {
 					err := certDB.Delete(string(key))
 					if err != nil {
-						log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
+						log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
 					} else {
 						expiredCertCount++
 					}
@@ -490,31 +505,31 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 			}
 			key, resBytes, err = keyDatabaseIterator.Next()
 		}
-		log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
+		log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
 
 		// compact the database
 		msg, err := certDB.Compact()
 		if err != nil {
-			log.Printf("[ERROR] Compacting key database failed: %s", err)
+			log.Error().Err(err).Msg("Compacting key database failed")
 		} else {
-			log.Printf("[INFO] Compacted key database (%s)", msg)
+			log.Debug().Msgf("Compacted key database: %s", msg)
 		}
 
 		// update main cert
-		res, err := certDB.Get(string(mainDomainSuffix))
+		res, err := certDB.Get(mainDomainSuffix)
 		if err != nil {
-			log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
+			log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
 		} else if res == nil {
-			log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
+			log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", mainDomainSuffix)
 		} else {
 			tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 
 			// renew main certificate 30 days before it expires
-			if !tlsCertificates[0].NotAfter.After(time.Now().Add(30 * 24 * time.Hour)) {
+			if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
 				go (func() {
-					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 					if err != nil {
-						log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
+						log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
 					}
 				})()
 			}
@@ -527,3 +542,12 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 		}
 	}
 }
+
+// leaf returns the parsed leaf certificate, either from c.leaf or by parsing
+// the corresponding c.Certificate[0].
+func leaf(c *tls.Certificate) (*x509.Certificate, error) {
+	if c.Leaf != nil {
+		return c.Leaf, nil
+	}
+	return x509.ParseCertificate(c.Certificate[0])
+}

server/context/context.go

@@ -0,0 +1,64 @@
+package context
+
+import (
+	stdContext "context"
+	"net/http"
+
+	"codeberg.org/codeberg/pages/server/utils"
+)
+
+type Context struct {
+	RespWriter http.ResponseWriter
+	Req        *http.Request
+	StatusCode int
+}
+
+func New(w http.ResponseWriter, r *http.Request) *Context {
+	return &Context{
+		RespWriter: w,
+		Req:        r,
+		StatusCode: http.StatusOK,
+	}
+}
+
+func (c *Context) Context() stdContext.Context {
+	if c.Req != nil {
+		return c.Req.Context()
+	}
+	return stdContext.Background()
+}
+
+func (c *Context) Response() *http.Response {
+	if c.Req != nil && c.Req.Response != nil {
+		return c.Req.Response
+	}
+	return nil
+}
+
+func (c *Context) String(raw string, status ...int) {
+	code := http.StatusOK
+	if len(status) != 0 {
+		code = status[0]
+	}
+	c.RespWriter.WriteHeader(code)
+	_, _ = c.RespWriter.Write([]byte(raw))
+}
+
+func (c *Context) Redirect(uri string, statusCode int) {
+	http.Redirect(c.RespWriter, c.Req, uri, statusCode)
+}
+
+// Path returns requested path.
+//
+// The returned bytes are valid until your request handler returns.
+func (c *Context) Path() string {
+	return c.Req.URL.Path
+}
+
+func (c *Context) Host() string {
+	return c.Req.URL.Host
+}
+
+func (c *Context) TrimHostPort() string {
+	return utils.TrimHostPort(c.Req.Host)
+}

server/database/mock.go

@@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error {
 func (p tmpDB) Get(name string) (*certificate.Resource, error) {
 	cert, has := p.intern.Get(name)
 	if !has {
-		return nil, fmt.Errorf("cert for '%s' not found", name)
+		return nil, fmt.Errorf("cert for %q not found", name)
 	}
 	return cert.(*certificate.Resource), nil
 }

server/database/setup.go

@@ -44,7 +44,7 @@ func (p aDB) Get(name string) (*certificate.Resource, error) {
 	if resBytes == nil {
 		return nil, nil
 	}
-	if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
+	if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
 		return nil, err
 	}
 	return cert, nil
@@ -72,7 +72,7 @@ func (p aDB) sync() {
 	for {
 		err := p.intern.Sync()
 		if err != nil {
-			log.Err(err).Msg("Syncing cert database failed")
+			log.Error().Err(err).Msg("Syncing cert database failed")
 		}
 		select {
 		case <-p.ctx.Done():

server/dns/const.go

@@ -1,6 +0,0 @@
-package dns
-
-import "time"
-
-// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var lookupCacheTimeout = 15 * time.Minute

server/dns/dns.go

@@ -3,10 +3,14 @@ package dns
 import (
 	"net"
 	"strings"
+	"time"
 
 	"codeberg.org/codeberg/pages/server/cache"
 )
 
+// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
+var lookupCacheTimeout = 15 * time.Minute
+
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
 func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {

server/gitea/cache.go

@@ -0,0 +1,115 @@
+package gitea
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/server/cache"
+)
+
+const (
+	// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
+	defaultBranchCacheTimeout = 15 * time.Minute
+
+	// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
+	// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
+	// picked up faster, while still allowing the content to be cached longer if nothing changes.
+	branchExistenceCacheTimeout = 5 * time.Minute
+
+	// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
+	// on your available memory.
+	// TODO: move as option into cache interface
+	fileCacheTimeout = 5 * time.Minute
+
+	// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
+	fileCacheSizeLimit = int64(1000 * 1000)
+)
+
+type FileResponse struct {
+	Exists    bool
+	IsSymlink bool
+	ETag      string
+	MimeType  string
+	Body      []byte
+}
+
+func (f FileResponse) IsEmpty() bool {
+	return len(f.Body) != 0
+}
+
+func (f FileResponse) createHttpResponse(cacheKey string) (header http.Header, statusCode int) {
+	header = make(http.Header)
+
+	if f.Exists {
+		statusCode = http.StatusOK
+	} else {
+		statusCode = http.StatusNotFound
+	}
+
+	if f.IsSymlink {
+		header.Set(giteaObjectTypeHeader, objTypeSymlink)
+	}
+	header.Set(ETagHeader, f.ETag)
+	header.Set(ContentTypeHeader, f.MimeType)
+	header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
+	header.Set(PagesCacheIndicatorHeader, "true")
+
+	log.Trace().Msgf("fileCache for %q used", cacheKey)
+	return header, statusCode
+}
+
+type BranchTimestamp struct {
+	Branch    string
+	Timestamp time.Time
+	notFound  bool
+}
+
+type writeCacheReader struct {
+	originalReader io.ReadCloser
+	buffer         *bytes.Buffer
+	rileResponse   *FileResponse
+	cacheKey       string
+	cache          cache.SetGetKey
+	hasError       bool
+}
+
+func (t *writeCacheReader) Read(p []byte) (n int, err error) {
+	n, err = t.originalReader.Read(p)
+	if err != nil {
+		log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
+		t.hasError = true
+	} else if n > 0 {
+		_, _ = t.buffer.Write(p[:n])
+	}
+	return
+}
+
+func (t *writeCacheReader) Close() error {
+	if !t.hasError {
+		fc := *t.rileResponse
+		fc.Body = t.buffer.Bytes()
+		_ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
+	}
+	log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
+	return t.originalReader.Close()
+}
+
+func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
+	if r == nil || cache == nil || cacheKey == "" {
+		log.Error().Msg("could not create CacheReader")
+		return nil
+	}
+
+	return &writeCacheReader{
+		originalReader: r,
+		buffer:         bytes.NewBuffer(make([]byte, 0)),
+		rileResponse:   &f,
+		cache:          cache,
+		cacheKey:       cacheKey,
+	}
+}

server/gitea/client.go

@@ -1,134 +1,284 @@
 package gitea
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"mime"
+	"net/http"
 	"net/url"
+	"path"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/valyala/fasthttp"
+	"code.gitea.io/sdk/gitea"
-	"github.com/valyala/fastjson"
+	"github.com/rs/zerolog/log"
-)
 
-const giteaAPIRepos = "/api/v1/repos/"
+	"codeberg.org/codeberg/pages/server/cache"
+)
 
 var ErrorNotFound = errors.New("not found")
 
+const (
+	// cache key prefixe
+	branchTimestampCacheKeyPrefix = "branchTime"
+	defaultBranchCacheKeyPrefix   = "defaultBranch"
+	rawContentCacheKeyPrefix      = "rawContent"
+
+	// pages server
+	PagesCacheIndicatorHeader = "X-Pages-Cache"
+	symlinkReadLimit          = 10000
+
+	// gitea
+	giteaObjectTypeHeader = "X-Gitea-Object-Type"
+	objTypeSymlink        = "symlink"
+
+	// std
+	ETagHeader          = "ETag"
+	ContentTypeHeader   = "Content-Type"
+	ContentLengthHeader = "Content-Length"
+)
+
 type Client struct {
-	giteaRoot      string
+	sdkClient     *gitea.Client
-	giteaAPIToken  string
+	responseCache cache.SetGetKey
-	fastClient     *fasthttp.Client
+
-	infoTimeout    time.Duration
+	giteaRoot string
-	contentTimeout time.Duration
+
+	followSymlinks bool
+	supportLFS     bool
+
+	forbiddenMimeTypes map[string]bool
+	defaultMimeType    string
 }
 
-type FileResponse struct {
+func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
-	Exists   bool
-	ETag     []byte
-	MimeType string
-	Body     []byte
-}
-
-// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
-func joinURL(baseURL string, paths ...string) string {
-	p := make([]string, 0, len(paths))
-	for i := range paths {
-		path := strings.TrimSpace(paths[i])
-		path = strings.Trim(path, "/")
-		if len(path) != 0 {
-			p = append(p, path)
-		}
-	}
-
-	return baseURL + "/" + strings.Join(p, "/")
-}
-
-func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
-
-func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
 	rootURL, err := url.Parse(giteaRoot)
+	if err != nil {
+		return nil, err
+	}
 	giteaRoot = strings.Trim(rootURL.String(), "/")
 
+	stdClient := http.Client{Timeout: 10 * time.Second}
+
+	// TODO: pass down
+	var (
+		forbiddenMimeTypes map[string]bool
+		defaultMimeType    string
+	)
+
+	if forbiddenMimeTypes == nil {
+		forbiddenMimeTypes = make(map[string]bool)
+	}
+	if defaultMimeType == "" {
+		defaultMimeType = "application/octet-stream"
+	}
+
+	sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
 	return &Client{
-		giteaRoot:      giteaRoot,
+		sdkClient:     sdk,
-		giteaAPIToken:  giteaAPIToken,
+		responseCache: respCache,
-		infoTimeout:    5 * time.Second,
+
-		contentTimeout: 10 * time.Second,
+		giteaRoot: giteaRoot,
-		fastClient:     getFastHTTPClient(),
+
+		followSymlinks: followSymlinks,
+		supportLFS:     supportLFS,
+
+		forbiddenMimeTypes: forbiddenMimeTypes,
+		defaultMimeType:    defaultMimeType,
 	}, err
 }
 
+func (client *Client) ContentWebLink(targetOwner, targetRepo, branch, resource string) string {
+	return path.Join(client.giteaRoot, targetOwner, targetRepo, "src/branch", branch, resource)
+}
+
 func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
-	res, err := client.do(client.contentTimeout, url)
 	if err != nil {
 		return nil, err
 	}
-
+	defer reader.Close()
-	switch res.StatusCode() {
+	return io.ReadAll(reader)
-	case fasthttp.StatusOK:
-		return res.Body(), nil
-	case fasthttp.StatusNotFound:
-		return nil, ErrorNotFound
-	default:
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
-	}
 }
 
-func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
+	cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource)
-	res, err := client.do(client.contentTimeout, url)
+	log := log.With().Str("cache_key", cacheKey).Logger()
-	if err != nil {
+
-		return nil, err
+	// handle if cache entry exist
+	if cache, ok := client.responseCache.Get(cacheKey); ok {
+		cache := cache.(FileResponse)
+		cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
+		// TODO: check against some timestamp missmatch?!?
+		if cache.Exists {
+			if cache.IsSymlink {
+				linkDest := string(cache.Body)
+				log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
+				return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+			} else {
+				log.Debug().Msg("[cache] return bytes")
+				return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
+			}
+		} else {
+			return nil, cachedHeader, cachedStatusCode, ErrorNotFound
+		}
 	}
 
-	if err != nil {
+	// not in cache, open reader via gitea api
-		return nil, err
+	reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
-	}
+	if resp != nil {
+		switch resp.StatusCode {
+		case http.StatusOK:
+			// first handle symlinks
+			{
+				objType := resp.Header.Get(giteaObjectTypeHeader)
+				log.Trace().Msgf("server raw content object %q", objType)
+				if client.followSymlinks && objType == objTypeSymlink {
+					defer reader.Close()
+					// read limited chars for symlink
+					linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit))
+					if err != nil {
+						return nil, nil, http.StatusInternalServerError, err
+					}
+					linkDest := strings.TrimSpace(string(linkDestBytes))
 
-	switch res.StatusCode() {
+					// we store symlink not content to reduce duplicates in cache
-	case fasthttp.StatusOK:
+					if err := client.responseCache.Set(cacheKey, FileResponse{
-		return res, nil
+						Exists:    true,
-	case fasthttp.StatusNotFound:
+						IsSymlink: true,
-		return nil, ErrorNotFound
+						Body:      []byte(linkDest),
-	default:
+						ETag:      resp.Header.Get(ETagHeader),
-		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+					}, fileCacheTimeout); err != nil {
+						log.Error().Err(err).Msg("[cache] error on cache write")
+					}
+
+					log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
+					return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+				}
+			}
+
+			// now we are sure it's content so set the MIME type
+			mimeType := client.getMimeTypeByExtension(resource)
+			resp.Response.Header.Set(ContentTypeHeader, mimeType)
+
+			if !shouldRespBeSavedToCache(resp.Response) {
+				return reader, resp.Response.Header, resp.StatusCode, err
+			}
+
+			// now we write to cache and respond at the sime time
+			fileResp := FileResponse{
+				Exists:   true,
+				ETag:     resp.Header.Get(ETagHeader),
+				MimeType: mimeType,
+			}
+			return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
+
+		case http.StatusNotFound:
+			if err := client.responseCache.Set(cacheKey, FileResponse{
+				Exists: false,
+				ETag:   resp.Header.Get(ETagHeader),
+			}, fileCacheTimeout); err != nil {
+				log.Error().Err(err).Msg("[cache] error on cache write")
+			}
+
+			return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound
+		default:
+			return nil, resp.Response.Header, resp.StatusCode, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
+		}
 	}
+	return nil, nil, http.StatusInternalServerError, err
 }
 
-func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
-	res, err := client.do(client.infoTimeout, url)
+
+	if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
+		branchTimeStamp := stamp.(*BranchTimestamp)
+		if branchTimeStamp.notFound {
+			log.Trace().Msgf("[cache] use branch %q not found", branchName)
+			return &BranchTimestamp{}, ErrorNotFound
+		}
+		log.Trace().Msgf("[cache] use branch %q exist", branchName)
+		return branchTimeStamp, nil
+	}
+
+	branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
 	if err != nil {
-		return time.Time{}, err
+		if resp != nil && resp.StatusCode == http.StatusNotFound {
+			log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
+			if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
+				log.Error().Err(err).Msg("[cache] error on cache write")
+			}
+			return &BranchTimestamp{}, ErrorNotFound
+		}
+		return &BranchTimestamp{}, err
 	}
-	if res.StatusCode() != fasthttp.StatusOK {
+	if resp.StatusCode != http.StatusOK {
-		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+		return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
 	}
-	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+
+	stamp := &BranchTimestamp{
+		Branch:    branch.Name,
+		Timestamp: branch.Commit.Timestamp,
+	}
+
+	log.Trace().Msgf("set cache branch [%s] exist", branchName)
+	if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
+		log.Error().Err(err).Msg("[cache] error on cache write")
+	}
+	return stamp, nil
 }
 
 func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
-	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
-	res, err := client.do(client.infoTimeout, url)
+
+	if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil {
+		return branch.(string), nil
+	}
+
+	repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName)
 	if err != nil {
 		return "", err
 	}
-	if res.StatusCode() != fasthttp.StatusOK {
+	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+		return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
 	}
-	return fastjson.GetString(res.Body(), "default_branch"), nil
+
+	branch := repo.DefaultBranch
+	if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
+		log.Error().Err(err).Msg("[cache] error on cache write")
+	}
+	return branch, nil
 }
 
-func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+func (client *Client) getMimeTypeByExtension(resource string) string {
-	req := fasthttp.AcquireRequest()
+	mimeType := mime.TypeByExtension(path.Ext(resource))
-
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	req.SetRequestURI(url)
+	if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
-	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+		mimeType = client.defaultMimeType
-	res := fasthttp.AcquireResponse()
+	}
-
+	log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
-	err := client.fastClient.DoTimeout(req, res, timeout)
+	return mimeType
-
+}
-	return res, err
+
+func shouldRespBeSavedToCache(resp *http.Response) bool {
+	if resp == nil {
+		return false
+	}
+
+	contentLengthRaw := resp.Header.Get(ContentLengthHeader)
+	if contentLengthRaw == "" {
+		return false
+	}
+
+	contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
+	if err != nil {
+		log.Error().Err(err).Msg("could not parse content length")
+	}
+
+	// if content to big or could not be determined we not cache it
+	return contentLeng > 0 && contentLeng < fileCacheSizeLimit
 }

server/gitea/client_test.go

@@ -1,23 +0,0 @@
-package gitea
-
-import (
-	"net/url"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestJoinURL(t *testing.T) {
-	baseURL := ""
-	assert.EqualValues(t, "/", joinURL(baseURL))
-	assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
-
-	baseURL = "http://wwow.url.com"
-	assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
-
-	baseURL = "http://wow.url.com/subpath/2"
-	assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
-	assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
-	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
-	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
-}

server/gitea/fasthttp.go

@@ -1,15 +0,0 @@
-package gitea
-
-import (
-	"time"
-
-	"github.com/valyala/fasthttp"
-)
-
-func getFastHTTPClient() *fasthttp.Client {
-	return &fasthttp.Client{
-		MaxConnDuration:    60 * time.Second,
-		MaxConnWaitTimeout: 1000 * time.Millisecond,
-		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-	}
-}

server/handler.go

@@ -1,305 +0,0 @@
-package server
-
-import (
-	"bytes"
-	"strings"
-
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
-	"github.com/valyala/fasthttp"
-
-	"codeberg.org/codeberg/pages/html"
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/dns"
-	"codeberg.org/codeberg/pages/server/gitea"
-	"codeberg.org/codeberg/pages/server/upstream"
-	"codeberg.org/codeberg/pages/server/utils"
-	"codeberg.org/codeberg/pages/server/version"
-)
-
-// Handler handles a single HTTP request to the web server.
-func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaClient *gitea.Client,
-	giteaRoot, rawInfoPage string,
-	blacklistedPaths, allowedCorsDomains [][]byte,
-	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
-) func(ctx *fasthttp.RequestCtx) {
-	return func(ctx *fasthttp.RequestCtx) {
-		log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
-
-		ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
-
-		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
-		ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
-
-		// Enable browser caching for up to 10 minutes
-		ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
-
-		trimmedHost := utils.TrimHostPort(ctx.Request.Host())
-
-		// Add HSTS for RawDomain and MainDomainSuffix
-		if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
-			ctx.Response.Header.Set("Strict-Transport-Security", hsts)
-		}
-
-		// Block all methods not required for static pages
-		if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
-			ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
-			ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
-			return
-		}
-
-		// Block blacklisted paths (like ACME challenges)
-		for _, blacklistedPath := range blacklistedPaths {
-			if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
-				html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
-				return
-			}
-		}
-
-		// Allow CORS for specified domains
-		allowCors := false
-		for _, allowedCorsDomain := range allowedCorsDomains {
-			if bytes.Equal(trimmedHost, allowedCorsDomain) {
-				allowCors = true
-				break
-			}
-		}
-		if allowCors {
-			ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
-			ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
-		}
-		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
-		if ctx.IsOptions() {
-			ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
-			return
-		}
-
-		// Prepare request information to Gitea
-		var targetOwner, targetRepo, targetBranch, targetPath string
-		targetOptions := &upstream.Options{
-			TryIndexPages: true,
-		}
-
-		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
-		// also disallow search indexing and add a Link header to the canonical URL.
-		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
-			if repo == "" {
-				log.Debug().Msg("tryBranch: repo == ''")
-				return false
-			}
-
-			// Replace "~" to "/" so we can access branch that contains slash character
-			// Branch name cannot contain "~" so doing this is okay
-			branch = strings.ReplaceAll(branch, "~", "/")
-
-			// Check if the branch exists, otherwise treat it as a file path
-			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
-			if branchTimestampResult == nil {
-				log.Debug().Msg("tryBranch: branch doesn't exist")
-				return false
-			}
-
-			// Branch exists, use it
-			targetRepo = repo
-			targetPath = strings.Trim(strings.Join(path, "/"), "/")
-			targetBranch = branchTimestampResult.Branch
-
-			targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
-
-			if canonicalLink != "" {
-				// Hide from search machines & add canonical link
-				ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
-				ctx.Response.Header.Set("Link",
-					strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
-						"; rel=\"canonical\"",
-				)
-			}
-
-			log.Debug().Msg("tryBranch: true")
-			return true
-		}
-
-		log.Debug().Msg("preparations")
-		if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
-			// Serve raw content from RawDomain
-			log.Debug().Msg("raw domain")
-
-			targetOptions.TryIndexPages = false
-			if targetOptions.ForbiddenMimeTypes == nil {
-				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
-			}
-			targetOptions.ForbiddenMimeTypes["text/html"] = true
-			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
-
-			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-			if len(pathElements) < 2 {
-				// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
-				ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
-				return
-			}
-			targetOwner = pathElements[0]
-			targetRepo = pathElements[1]
-
-			// raw.codeberg.org/example/myrepo/@main/index.html
-			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
-				log.Debug().Msg("raw domain preparations, now trying with specified branch")
-				if tryBranch(log,
-					targetRepo, pathElements[2][1:], pathElements[3:],
-					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
-				) {
-					log.Debug().Msg("tryBranch, now trying upstream 1")
-					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						canonicalDomainCache, branchTimestampCache, fileResponseCache)
-					return
-				}
-				log.Debug().Msg("missing branch")
-				html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-				return
-			}
-
-			log.Debug().Msg("raw domain preparations, now trying with default branch")
-			tryBranch(log,
-				targetRepo, "", pathElements[2:],
-				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
-			)
-			log.Debug().Msg("tryBranch, now trying upstream 2")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-				canonicalDomainCache, branchTimestampCache, fileResponseCache)
-			return
-
-		} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-			// Serve pages from subdomains of MainDomainSuffix
-			log.Debug().Msg("main domain suffix")
-
-			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-			targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
-			targetRepo = pathElements[0]
-			targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
-
-			if targetOwner == "www" {
-				// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
-				ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
-				return
-			}
-
-			// Check if the first directory is a repo with the second directory as a branch
-			// example.codeberg.page/myrepo/@main/index.html
-			if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
-				if targetRepo == "pages" {
-					// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
-					ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
-					return
-				}
-
-				log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
-				if tryBranch(log,
-					pathElements[0], pathElements[1][1:], pathElements[2:],
-					"/"+pathElements[0]+"/%p",
-				) {
-					log.Debug().Msg("tryBranch, now trying upstream 3")
-					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						canonicalDomainCache, branchTimestampCache, fileResponseCache)
-				} else {
-					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-				}
-				return
-			}
-
-			// Check if the first directory is a branch for the "pages" repo
-			// example.codeberg.page/@main/index.html
-			if strings.HasPrefix(pathElements[0], "@") {
-				log.Debug().Msg("main domain preparations, now trying with specified branch")
-				if tryBranch(log,
-					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
-					log.Debug().Msg("tryBranch, now trying upstream 4")
-					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						canonicalDomainCache, branchTimestampCache, fileResponseCache)
-				} else {
-					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-				}
-				return
-			}
-
-			// Check if the first directory is a repo with a "pages" branch
-			// example.codeberg.page/myrepo/index.html
-			// example.codeberg.page/pages/... is not allowed here.
-			log.Debug().Msg("main domain preparations, now trying with specified repo")
-			if pathElements[0] != "pages" && tryBranch(log,
-				pathElements[0], "pages", pathElements[1:], "") {
-				log.Debug().Msg("tryBranch, now trying upstream 5")
-				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					canonicalDomainCache, branchTimestampCache, fileResponseCache)
-				return
-			}
-
-			// Try to use the "pages" repo on its default branch
-			// example.codeberg.page/index.html
-			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
-			if tryBranch(log,
-				"pages", "", pathElements, "") {
-				log.Debug().Msg("tryBranch, now trying upstream 6")
-				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					canonicalDomainCache, branchTimestampCache, fileResponseCache)
-				return
-			}
-
-			// Couldn't find a valid repo/branch
-			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return
-		} else {
-			trimmedHostStr := string(trimmedHost)
-
-			// Serve pages from external domains
-			targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
-			if targetOwner == "" {
-				html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-				return
-			}
-
-			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-			canonicalLink := ""
-			if strings.HasPrefix(pathElements[0], "@") {
-				targetBranch = pathElements[0][1:]
-				pathElements = pathElements[1:]
-				canonicalLink = "/%p"
-			}
-
-			// Try to use the given repo on the given branch or the default branch
-			log.Debug().Msg("custom domain preparations, now trying with details from DNS")
-			if tryBranch(log,
-				targetRepo, targetBranch, pathElements, canonicalLink) {
-				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
-				if !valid {
-					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
-					return
-				} else if canonicalDomain != trimmedHostStr {
-					// only redirect if the target is also a codeberg page!
-					targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
-					if targetOwner != "" {
-						ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
-						return
-					}
-
-					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-					return
-				}
-
-				log.Debug().Msg("tryBranch, now trying upstream 7")
-				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
-					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					canonicalDomainCache, branchTimestampCache, fileResponseCache)
-				return
-			}
-
-			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return
-		}
-	}
-}

server/handler/handler.go

@@ -0,0 +1,113 @@
+package handler
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/version"
+)
+
+const (
+	headerAccessControlAllowOrigin  = "Access-Control-Allow-Origin"
+	headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
+	defaultPagesRepo                = "pages"
+	defaultPagesBranch              = "pages"
+)
+
+// Handler handles a single HTTP request to the web server.
+func Handler(mainDomainSuffix, rawDomain string,
+	giteaClient *gitea.Client,
+	rawInfoPage string,
+	blacklistedPaths, allowedCorsDomains []string,
+	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
+		ctx := context.New(w, req)
+
+		ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
+
+		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
+		ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
+
+		// Enable browser caching for up to 10 minutes
+		ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600")
+
+		trimmedHost := ctx.TrimHostPort()
+
+		// Add HSTS for RawDomain and MainDomainSuffix
+		if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+			ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
+		}
+
+		// Handle all http methods
+		ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions)
+		switch ctx.Req.Method {
+		case http.MethodOptions:
+			// return Allow header
+			ctx.RespWriter.WriteHeader(http.StatusNoContent)
+			return
+		case http.MethodGet,
+			http.MethodHead:
+			// end switch case and handle allowed requests
+			break
+		default:
+			// Block all methods not required for static pages
+			ctx.String("Method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		// Block blacklisted paths (like ACME challenges)
+		for _, blacklistedPath := range blacklistedPaths {
+			if strings.HasPrefix(ctx.Path(), blacklistedPath) {
+				html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
+				return
+			}
+		}
+
+		// Allow CORS for specified domains
+		allowCors := false
+		for _, allowedCorsDomain := range allowedCorsDomains {
+			if strings.EqualFold(trimmedHost, allowedCorsDomain) {
+				allowCors = true
+				break
+			}
+		}
+		if allowCors {
+			ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*")
+			ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead)
+		}
+
+		// Prepare request information to Gitea
+		pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
+
+		if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
+			log.Debug().Msg("raw domain request detecded")
+			handleRaw(log, ctx, giteaClient,
+				mainDomainSuffix, rawInfoPage,
+				trimmedHost,
+				pathElements,
+				canonicalDomainCache)
+		} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+			log.Debug().Msg("subdomain request detecded")
+			handleSubDomain(log, ctx, giteaClient,
+				mainDomainSuffix,
+				trimmedHost,
+				pathElements,
+				canonicalDomainCache)
+		} else {
+			log.Debug().Msg("custom domain request detecded")
+			handleCustomDomain(log, ctx, giteaClient,
+				mainDomainSuffix,
+				trimmedHost,
+				pathElements,
+				dnsLookupCache, canonicalDomainCache)
+		}
+	}
+}

server/handler/handler_custom_domain.go

@@ -0,0 +1,71 @@
+package handler
+
+import (
+	"net/http"
+	"path"
+	"strings"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+	"github.com/rs/zerolog"
+)
+
+func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+	mainDomainSuffix string,
+	trimmedHost string,
+	pathElements []string,
+	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+) {
+	// Serve pages from custom domains
+	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache)
+	if targetOwner == "" {
+		html.ReturnErrorPage(ctx,
+			"could not obtain repo owner from custom domain",
+			http.StatusFailedDependency)
+		return
+	}
+
+	pathParts := pathElements
+	canonicalLink := false
+	if strings.HasPrefix(pathElements[0], "@") {
+		targetBranch = pathElements[0][1:]
+		pathParts = pathElements[1:]
+		canonicalLink = true
+	}
+
+	// Try to use the given repo on the given branch or the default branch
+	log.Debug().Msg("custom domain preparations, now trying with details from DNS")
+	if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+		TryIndexPages: true,
+		TargetOwner:   targetOwner,
+		TargetRepo:    targetRepo,
+		TargetBranch:  targetBranch,
+		TargetPath:    path.Join(pathParts...),
+	}, canonicalLink); works {
+		canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
+		if !valid {
+			html.ReturnErrorPage(ctx, "domain not specified in <code>.domains</code> file", http.StatusMisdirectedRequest)
+			return
+		} else if canonicalDomain != trimmedHost {
+			// only redirect if the target is also a codeberg page!
+			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache)
+			if targetOwner != "" {
+				ctx.Redirect("https://"+canonicalDomain+targetOpt.TargetPath, http.StatusTemporaryRedirect)
+				return
+			}
+
+			html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
+			return
+		}
+
+		log.Debug().Msg("tryBranch, now trying upstream 7")
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		return
+	}
+
+	html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
+}

server/handler/handler_raw_domain.go

@@ -0,0 +1,67 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/rs/zerolog"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+)
+
+func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+	mainDomainSuffix, rawInfoPage string,
+	trimmedHost string,
+	pathElements []string,
+	canonicalDomainCache cache.SetGetKey,
+) {
+	// Serve raw content from RawDomain
+	log.Debug().Msg("raw domain")
+
+	if len(pathElements) < 2 {
+		// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
+		ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
+		return
+	}
+
+	// raw.codeberg.org/example/myrepo/@main/index.html
+	if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
+		log.Debug().Msg("raw domain preparations, now trying with specified branch")
+		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+			ServeRaw:     true,
+			TargetOwner:  pathElements[0],
+			TargetRepo:   pathElements[1],
+			TargetBranch: pathElements[2][1:],
+			TargetPath:   path.Join(pathElements[3:]...),
+		}, true); works {
+			log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			return
+		}
+		log.Debug().Msg("missing branch info")
+		html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency)
+		return
+	}
+
+	log.Debug().Msg("raw domain preparations, now trying with default branch")
+	if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+		TryIndexPages: false,
+		ServeRaw:      true,
+		TargetOwner:   pathElements[0],
+		TargetRepo:    pathElements[1],
+		TargetPath:    path.Join(pathElements[2:]...),
+	}, true); works {
+		log.Trace().Msg("tryUpstream: serve raw domain with default branch")
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+	} else {
+		html.ReturnErrorPage(ctx,
+			fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
+			http.StatusNotFound)
+	}
+}

server/handler/handler_sub_domain.go

@@ -0,0 +1,120 @@
+package handler
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/rs/zerolog"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+)
+
+func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+	mainDomainSuffix string,
+	trimmedHost string,
+	pathElements []string,
+	canonicalDomainCache cache.SetGetKey,
+) {
+	// Serve pages from subdomains of MainDomainSuffix
+	log.Debug().Msg("main domain suffix")
+
+	targetOwner := strings.TrimSuffix(trimmedHost, mainDomainSuffix)
+	targetRepo := pathElements[0]
+
+	if targetOwner == "www" {
+		// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
+		ctx.Redirect("https://"+mainDomainSuffix[1:]+ctx.Path(), http.StatusPermanentRedirect)
+		return
+	}
+
+	// Check if the first directory is a repo with the second directory as a branch
+	// example.codeberg.page/myrepo/@main/index.html
+	if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
+		if targetRepo == defaultPagesRepo {
+			// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
+			ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
+			return
+		}
+
+		log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
+		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+			TryIndexPages: true,
+			TargetOwner:   targetOwner,
+			TargetRepo:    pathElements[0],
+			TargetBranch:  pathElements[1][1:],
+			TargetPath:    path.Join(pathElements[2:]...),
+		}, true); works {
+			log.Trace().Msg("tryUpstream: serve with specified repo and branch")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		} else {
+			html.ReturnErrorPage(ctx,
+				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
+				http.StatusFailedDependency)
+		}
+		return
+	}
+
+	// Check if the first directory is a branch for the defaultPagesRepo
+	// example.codeberg.page/@main/index.html
+	if strings.HasPrefix(pathElements[0], "@") {
+		log.Debug().Msg("main domain preparations, now trying with specified branch")
+		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+			TryIndexPages: true,
+			TargetOwner:   targetOwner,
+			TargetRepo:    defaultPagesRepo,
+			TargetBranch:  pathElements[0][1:],
+			TargetPath:    path.Join(pathElements[1:]...),
+		}, true); works {
+			log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		} else {
+			html.ReturnErrorPage(ctx,
+				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
+				http.StatusFailedDependency)
+		}
+		return
+	}
+
+	// Check if the first directory is a repo with a defaultPagesRepo branch
+	// example.codeberg.page/myrepo/index.html
+	// example.codeberg.page/pages/... is not allowed here.
+	log.Debug().Msg("main domain preparations, now trying with specified repo")
+	if pathElements[0] != defaultPagesRepo {
+		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+			TryIndexPages: true,
+			TargetOwner:   targetOwner,
+			TargetRepo:    pathElements[0],
+			TargetBranch:  defaultPagesBranch,
+			TargetPath:    path.Join(pathElements[1:]...),
+		}, false); works {
+			log.Debug().Msg("tryBranch, now trying upstream 5")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			return
+		}
+	}
+
+	// Try to use the defaultPagesRepo on its default branch
+	// example.codeberg.page/index.html
+	log.Debug().Msg("main domain preparations, now trying with default repo/branch")
+	if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+		TryIndexPages: true,
+		TargetOwner:   targetOwner,
+		TargetRepo:    defaultPagesRepo,
+		TargetPath:    path.Join(pathElements...),
+	}, false); works {
+		log.Debug().Msg("tryBranch, now trying upstream 6")
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		return
+	}
+
+	// Couldn't find a valid repo/branch
+	html.ReturnErrorPage(ctx,
+		fmt.Sprintf("could not find a valid repository[%s]", targetRepo),
+		http.StatusNotFound)
+}

server/handler/handler_test.go

@@ -0,0 +1,49 @@
+package handler
+
+import (
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"github.com/rs/zerolog/log"
+)
+
+func TestHandlerPerformance(t *testing.T) {
+	giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
+	testHandler := Handler(
+		"codeberg.page", "raw.codeberg.org",
+		giteaClient,
+		"https://docs.codeberg.org/pages/raw-content/",
+		[]string{"/.well-known/acme-challenge/"},
+		[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
+		cache.NewKeyValueCache(),
+		cache.NewKeyValueCache(),
+	)
+
+	testCase := func(uri string, status int) {
+		t.Run(uri, func(t *testing.T) {
+			req := httptest.NewRequest("GET", uri, nil)
+			w := httptest.NewRecorder()
+
+			log.Printf("Start: %v\n", time.Now())
+			start := time.Now()
+			testHandler(w, req)
+			end := time.Now()
+			log.Printf("Done: %v\n", time.Now())
+
+			resp := w.Result()
+
+			if resp.StatusCode != status {
+				t.Errorf("request failed with status code %d", resp.StatusCode)
+			} else {
+				t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
+			}
+		})
+	}
+
+	testCase("https://mondstern.codeberg.page/", 404) // TODO: expect 200
+	testCase("https://codeberg.page/", 404)           // TODO: expect 200
+	testCase("https://example.momar.xyz/", 424)
+}

server/handler/hsts.go

@@ -0,0 +1,15 @@
+package handler
+
+import (
+	"strings"
+)
+
+// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
+// string for custom domains.
+func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string {
+	if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) {
+		return "max-age=63072000; includeSubdomains; preload"
+	} else {
+		return ""
+	}
+}

server/handler/try.go

@@ -0,0 +1,76 @@
+package handler
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/rs/zerolog"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+)
+
+// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
+func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
+	mainDomainSuffix, trimmedHost string,
+	options *upstream.Options,
+	canonicalDomainCache cache.SetGetKey,
+) {
+	// check if a canonical domain exists on a request on MainDomain
+	if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+		canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
+		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
+			canonicalPath := ctx.Req.RequestURI
+			if options.TargetRepo != defaultPagesRepo {
+				path := strings.SplitN(canonicalPath, "/", 3)
+				if len(path) >= 3 {
+					canonicalPath = "/" + path[2]
+				}
+			}
+			ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	// Add host for debugging.
+	options.Host = trimmedHost
+
+	// Try to request the file from the Gitea API
+	if !options.Upstream(ctx, giteaClient) {
+		html.ReturnErrorPage(ctx, "", ctx.StatusCode)
+	}
+}
+
+// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty,
+// it will also disallow search indexing and add a Link header to the canonical URL.
+func tryBranch(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
+	targetOptions *upstream.Options, canonicalLink bool,
+) (*upstream.Options, bool) {
+	if targetOptions.TargetOwner == "" || targetOptions.TargetRepo == "" {
+		log.Debug().Msg("tryBranch: owner or repo is empty")
+		return nil, false
+	}
+
+	// Replace "~" to "/" so we can access branch that contains slash character
+	// Branch name cannot contain "~" so doing this is okay
+	targetOptions.TargetBranch = strings.ReplaceAll(targetOptions.TargetBranch, "~", "/")
+
+	// Check if the branch exists, otherwise treat it as a file path
+	branchExist, _ := targetOptions.GetBranchTimestamp(giteaClient)
+	if !branchExist {
+		log.Debug().Msg("tryBranch: branch doesn't exist")
+		return nil, false
+	}
+
+	if canonicalLink {
+		// Hide from search machines & add canonical link
+		ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex")
+		ctx.RespWriter.Header().Set("Link", targetOptions.ContentWebLink(giteaClient)+"; rel=\"canonical\"")
+	}
+
+	log.Debug().Msg("tryBranch: true")
+	return targetOptions, true
+}

server/handler_test.go

@@ -1,51 +0,0 @@
-package server
-
-import (
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/valyala/fasthttp"
-
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/gitea"
-)
-
-func TestHandlerPerformance(t *testing.T) {
-	giteaRoot := "https://codeberg.org"
-	giteaClient, _ := gitea.NewClient(giteaRoot, "")
-	testHandler := Handler(
-		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
-		giteaClient,
-		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
-		[][]byte{[]byte("/.well-known/acme-challenge/")},
-		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
-		cache.NewKeyValueCache(),
-		cache.NewKeyValueCache(),
-		cache.NewKeyValueCache(),
-		cache.NewKeyValueCache(),
-	)
-
-	testCase := func(uri string, status int) {
-		ctx := &fasthttp.RequestCtx{
-			Request:  *fasthttp.AcquireRequest(),
-			Response: *fasthttp.AcquireResponse(),
-		}
-		ctx.Request.SetRequestURI(uri)
-		fmt.Printf("Start: %v\n", time.Now())
-		start := time.Now()
-		testHandler(ctx)
-		end := time.Now()
-		fmt.Printf("Done: %v\n", time.Now())
-		if ctx.Response.StatusCode() != status {
-			t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
-		} else {
-			t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-		}
-	}
-
-	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
-	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
-	testCase("https://example.momar.xyz/", 424)       // TODO: expect 200
-	testCase("https://codeberg.page/", 424)           // TODO: expect 200
-}

server/helpers.go

@@ -1,15 +0,0 @@
-package server
-
-import (
-	"bytes"
-)
-
-// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
-// string for custom domains.
-func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
-	if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
-		return "max-age=63072000; includeSubdomains; preload"
-	} else {
-		return ""
-	}
-}

server/setup.go

@@ -1,53 +1,27 @@
 package server
 
 import (
-	"bytes"
 	"net/http"
-	"time"
+	"strings"
-
-	"github.com/rs/zerolog/log"
-
-	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/utils"
 )
 
-type fasthttpLogger struct{}
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
+	challengePath := "/.well-known/acme-challenge/"
 
-func (fasthttpLogger) Printf(format string, args ...interface{}) {
+	return func(w http.ResponseWriter, req *http.Request) {
-	log.Printf("[FASTHTTP] "+format, args...)
+		ctx := context.New(w, req)
-}
+		if strings.HasPrefix(ctx.Path(), challengePath) {
-
+			challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
-func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
+			if !ok || challenge == nil {
-	// Enable compression by wrapping the handler with the compression function provided by FastHTTP
+				ctx.String("no challenge for this token", http.StatusNotFound)
-	compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
-
-	return &fasthttp.Server{
-		Handler:                      compressedHandler,
-		DisablePreParseMultipartForm: true,
-		NoDefaultServerHeader:        true,
-		NoDefaultDate:                true,
-		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
-		Logger:                       fasthttpLogger{},
-	}
-}
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
-	challengePath := []byte("/.well-known/acme-challenge/")
-
-	return &fasthttp.Server{
-		Handler: func(ctx *fasthttp.RequestCtx) {
-			if bytes.HasPrefix(ctx.Path(), challengePath) {
-				challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
-				if !ok || challenge == nil {
-					ctx.SetStatusCode(http.StatusNotFound)
-					ctx.SetBodyString("no challenge for this token")
-				}
-				ctx.SetBodyString(challenge.(string))
-			} else {
-				ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
 			}
-		},
+			ctx.String(challenge.(string))
+		} else {
+			ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
+		}
 	}
 }

server/try.go

@@ -1,49 +0,0 @@
-package server
-
-import (
-	"bytes"
-	"strings"
-
-	"github.com/valyala/fasthttp"
-
-	"codeberg.org/codeberg/pages/html"
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/gitea"
-	"codeberg.org/codeberg/pages/server/upstream"
-)
-
-// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
-	mainDomainSuffix, trimmedHost []byte,
-
-	targetOptions *upstream.Options,
-	targetOwner, targetRepo, targetBranch, targetPath string,
-
-	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
-) {
-	// check if a canonical domain exists on a request on MainDomain
-	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
-		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
-			canonicalPath := string(ctx.RequestURI())
-			if targetRepo != "pages" {
-				path := strings.SplitN(canonicalPath, "/", 3)
-				if len(path) >= 3 {
-					canonicalPath = "/" + path[2]
-				}
-			}
-			ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
-			return
-		}
-	}
-
-	targetOptions.TargetOwner = targetOwner
-	targetOptions.TargetRepo = targetRepo
-	targetOptions.TargetBranch = targetBranch
-	targetOptions.TargetPath = targetPath
-
-	// Try to request the file from the Gitea API
-	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
-	}
-}

server/upstream/const.go

@@ -1,24 +0,0 @@
-package upstream
-
-import "time"
-
-// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
-var defaultBranchCacheTimeout = 15 * time.Minute
-
-// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
-// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
-// picked up faster, while still allowing the content to be cached longer if nothing changes.
-var branchExistenceCacheTimeout = 5 * time.Minute
-
-// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
-// on your available memory.
-// TODO: move as option into cache interface
-var fileCacheTimeout = 5 * time.Minute
-
-// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-var fileCacheSizeLimit = 1024 * 1024
-
-// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
-var canonicalDomainCacheTimeout = 15 * time.Minute
-
-const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -2,18 +2,26 @@ package upstream
 
 import (
 	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
 
+// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
+var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"
+
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
 	var (
 		domains []string
 		valid   bool
 	)
-	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
+	if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
 		domains = cachedValue.([]string)
 		for _, domain := range domains {
 			if domain == actualDomain {
@@ -22,7 +30,7 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
 			}
 		}
 	} else {
-		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
+		body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, canonicalDomainConfig)
 		if err == nil {
 			for _, domain := range strings.Split(string(body), "\n") {
 				domain = strings.ToLower(domain)
@@ -36,15 +44,17 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
 					valid = true
 				}
 			}
+		} else {
+			log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, o.TargetOwner, o.TargetRepo)
 		}
-		domains = append(domains, targetOwner+mainDomainSuffix)
+		domains = append(domains, o.TargetOwner+mainDomainSuffix)
 		if domains[len(domains)-1] == actualDomain {
 			valid = true
 		}
-		if targetRepo != "" && targetRepo != "pages" {
+		if o.TargetRepo != "" && o.TargetRepo != "pages" {
-			domains[len(domains)-1] += "/" + targetRepo
+			domains[len(domains)-1] += "/" + o.TargetRepo
 		}
-		_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
+		_ = canonicalDomainCache.Set(o.TargetOwner+"/"+o.TargetRepo+"/"+o.TargetBranch, domains, canonicalDomainCacheTimeout)
 	}
 	return domains[0], valid
 }

server/upstream/header.go

@@ -0,0 +1,28 @@
+package upstream
+
+import (
+	"net/http"
+	"time"
+
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+// setHeader set values to response header
+func (o *Options) setHeader(ctx *context.Context, header http.Header) {
+	if eTag := header.Get(gitea.ETagHeader); eTag != "" {
+		ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
+	}
+	if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
+		ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
+	}
+	if length := header.Get(gitea.ContentLengthHeader); length != "" {
+		ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
+	}
+	if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
+		ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
+	} else {
+		ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
+	}
+	ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
+}

server/upstream/helper.go

@@ -1,72 +1,47 @@
 package upstream
 
 import (
-	"mime"
+	"errors"
-	"path"
+	"fmt"
-	"strconv"
+
-	"strings"
+	"github.com/rs/zerolog/log"
-	"time"
 
-	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
 
-type branchTimestamp struct {
+// GetBranchTimestamp finds the default branch (if branch is "") and save branch and it's last modification time to Options
-	Branch    string
+func (o *Options) GetBranchTimestamp(giteaClient *gitea.Client) (bool, error) {
-	Timestamp time.Time
+	log := log.With().Strs("BranchInfo", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch}).Logger()
-}
 
-// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
+	if o.TargetBranch == "" {
-// (or nil if the branch doesn't exist)
-func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
-	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
-		if result == nil {
-			return nil
-		}
-		return result.(*branchTimestamp)
-	}
-	result := &branchTimestamp{
-		Branch: branch,
-	}
-	if len(branch) == 0 {
 		// Get default branch
-		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(o.TargetOwner, o.TargetRepo)
 		if err != nil {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
+			log.Err(err).Msg("Could't fetch default branch from repository")
-			return nil
+			return false, err
 		}
-		result.Branch = defaultBranch
+		log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
+		o.TargetBranch = defaultBranch
 	}
 
-	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch)
 	if err != nil {
-		return nil
+		if !errors.Is(err, gitea.ErrorNotFound) {
-	}
+			log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch")
-	result.Timestamp = timestamp
-	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
-	return result
-}
-
-func (o *Options) getMimeTypeByExtension() string {
-	if o.ForbiddenMimeTypes == nil {
-		o.ForbiddenMimeTypes = make(map[string]bool)
-	}
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
 		}
+		return false, err
 	}
-	return mimeType
+
+	if timestamp == nil || timestamp.Branch == "" {
+		return false, fmt.Errorf("empty response")
+	}
+
+	log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp)
+	o.BranchTimestamp = timestamp.Timestamp
+	o.TargetBranch = timestamp.Branch
+	return true, nil
 }
 
-func (o *Options) generateUri() string {
+func (o *Options) ContentWebLink(giteaClient *gitea.Client) string {
-	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+	return giteaClient.ContentWebLink(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath) + "; rel=\"canonical\""
-}
-
-func (o *Options) timestamp() string {
-	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -1,21 +1,27 @@
 package upstream
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
-	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/html"
-	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
 
+const (
+	headerLastModified    = "Last-Modified"
+	headerIfModifiedSince = "If-Modified-Since"
+
+	rawMime = "text/plain; charset=utf-8"
+)
+
 // upstreamIndexPages lists pages that may be considered as index pages for directories.
 var upstreamIndexPages = []string{
 	"index.html",
@@ -28,64 +34,75 @@ var upstreamNotFoundPages = []string{
 
 // Options provides various options for the upstream request.
 type Options struct {
-	TargetOwner,
+	TargetOwner  string
-	TargetRepo,
+	TargetRepo   string
-	TargetBranch,
+	TargetBranch string
-	TargetPath,
+	TargetPath   string
 
-	DefaultMimeType string
+	// Used for debugging purposes.
-	ForbiddenMimeTypes map[string]bool
+	Host string
-	TryIndexPages      bool
+
-	BranchTimestamp    time.Time
+	TryIndexPages   bool
+	BranchTimestamp time.Time
 	// internal
 	appendTrailingSlash bool
 	redirectIfExists    string
+
+	ServeRaw bool
 }
 
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
 
-	// Check if the branch exists and when it was modified
+	if o.TargetOwner == "" || o.TargetRepo == "" {
-	if o.BranchTimestamp.IsZero() {
+		html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
-		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
-
-		if branch == nil {
-			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return true
-		}
-		o.TargetBranch = branch.Branch
-		o.BranchTimestamp = branch.Timestamp
-	}
-
-	if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
-		html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
 		return true
 	}
 
-	// Check if the browser has a cached version
+	// Check if the branch exists and when it was modified
-	if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
+	if o.BranchTimestamp.IsZero() {
-		if !ifModifiedSince.Before(o.BranchTimestamp) {
+		branchExist, err := o.GetBranchTimestamp(giteaClient)
-			ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
+		// handle 404
+		if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist {
+			html.ReturnErrorPage(ctx,
+				fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
+				http.StatusNotFound)
+			return true
+		}
+
+		// handle unexpected errors
+		if err != nil {
+			html.ReturnErrorPage(ctx,
+				fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err),
+				http.StatusFailedDependency)
 			return true
 		}
 	}
-	log.Debug().Msg("preparations")
 
-	// Make a GET request to the upstream URL
+	// Check if the browser has a cached version
-	uri := o.generateUri()
+	if ctx.Response() != nil {
-	var res *fasthttp.Response
+		if ifModifiedSince, err := time.Parse(time.RFC1123, ctx.Response().Header.Get(headerIfModifiedSince)); err == nil {
-	var cachedResponse gitea.FileResponse
+			if ifModifiedSince.After(o.BranchTimestamp) {
-	var err error
+				ctx.RespWriter.WriteHeader(http.StatusNotModified)
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
+				log.Trace().Msg("check response against last modified: valid")
-		cachedResponse = cachedValue.(gitea.FileResponse)
+				return true
-	} else {
+			}
-		res, err = giteaClient.ServeRawContent(uri)
+		}
+		log.Trace().Msg("check response against last modified: outdated")
 	}
-	log.Debug().Msg("acquisition")
 
-	// Handle errors
+	log.Debug().Msg("Preparing")
-	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
+
+	reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath)
+	if reader != nil {
+		defer reader.Close()
+	}
+
+	log.Debug().Msg("Aquisting")
+
+	// Handle not found error
+	if err != nil && errors.Is(err, gitea.ErrorNotFound) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -93,25 +110,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+				if optionsForIndexPages.Upstream(ctx, giteaClient) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-						Exists: false,
-					}, fileCacheTimeout)
 					return true
 				}
 			}
 			// compatibility fix for GitHub Pages (/example → /example.html)
 			optionsForIndexPages.appendTrailingSlash = false
-			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
+			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+			if optionsForIndexPages.Upstream(ctx, giteaClient) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-					Exists: false,
-				}, fileCacheTimeout)
 				return true
 			}
 		}
-		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+
+		ctx.StatusCode = http.StatusNotFound
 		if o.TryIndexPages {
 			// copy the o struct & try if a not found page exists
 			optionsForNotFoundPages := *o
@@ -119,92 +131,70 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 			optionsForNotFoundPages.appendTrailingSlash = false
 			for _, notFoundPage := range upstreamNotFoundPages {
 				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
-				if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-						Exists: false,
-					}, fileCacheTimeout)
 					return true
 				}
 			}
 		}
-		if res != nil {
-			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-				Exists: false,
-			}, fileCacheTimeout)
-		}
 		return false
 	}
-	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
+
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
+	// handle unexpected client errors
-		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+	if err != nil || reader == nil || statusCode != http.StatusOK {
+		log.Debug().Msg("Handling error")
+		var msg string
+
+		if err != nil {
+			msg = "gitea client returned unexpected error"
+			log.Error().Err(err).Msg(msg)
+			msg = fmt.Sprintf("%s: %v", msg, err)
+		}
+		if reader == nil {
+			msg = "gitea client returned no reader"
+			log.Error().Msg(msg)
+		}
+		if statusCode != http.StatusOK {
+			msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+			log.Error().Msg(msg)
+		}
+
+		html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError)
 		return true
 	}
 
 	// Append trailing slash if missing (for index files), and redirect to fix filenames in general
 	// o.appendTrailingSlash is only true when looking for index pages
-	if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
+	if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
-		ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
+		ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
 		return true
 	}
-	if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
+	if strings.HasSuffix(ctx.Path(), "/index.html") {
-		ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
+		ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
 		return true
 	}
 	if o.redirectIfExists != "" {
-		ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
+		ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect)
 		return true
 	}
-	log.Debug().Msg("error handling")
 
-	// Set the MIME type
+	// Set ETag & MIME
-	mimeType := o.getMimeTypeByExtension()
+	o.setHeader(ctx, header)
-	ctx.Response.Header.SetContentType(mimeType)
 
-	// Set ETag
+	log.Debug().Msg("Prepare response")
-	if cachedResponse.Exists {
-		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
-	} else if res != nil {
-		cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
-		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
-	}
 
-	if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
+	ctx.RespWriter.WriteHeader(ctx.StatusCode)
-		// Everything's okay so far
-		ctx.Response.SetStatusCode(fasthttp.StatusOK)
-	}
-	ctx.Response.Header.SetLastModified(o.BranchTimestamp)
-
-	log.Debug().Msg("response preparations")
 
 	// Write the response body to the original request
-	var cacheBodyWriter bytes.Buffer
+	if reader != nil {
-	if res != nil {
+		_, err := io.Copy(ctx.RespWriter, reader)
-		if res.Header.ContentLength() > fileCacheSizeLimit {
+		if err != nil {
-			// fasthttp else will set "Content-Length: 0"
+			log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
-			ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+			html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
-
+			return true
-			err = res.BodyWriteTo(ctx.Response.BodyWriter())
-		} else {
-			// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
-			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
-	} else {
-		_, err = ctx.Write(cachedResponse.Body)
 	}
-	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
-		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
-		return true
-	}
-	log.Debug().Msg("response")
 
-	if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
+	log.Debug().Msg("Sending response")
-		cachedResponse.Exists = true
-		cachedResponse.MimeType = mimeType
-		cachedResponse.Body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
-	}
 
 	return true
 }

server/utils/utils.go

@@ -1,9 +1,11 @@
 package utils
 
-import "bytes"
+import (
+	"strings"
+)
 
-func TrimHostPort(host []byte) []byte {
+func TrimHostPort(host string) string {
-	i := bytes.IndexByte(host, ':')
+	i := strings.IndexByte(host, ':')
 	if i >= 0 {
 		return host[:i]
 	}

server/utils/utils_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestTrimHostPort(t *testing.T) {
-	assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
+	assert.EqualValues(t, "aa", TrimHostPort("aa"))
-	assert.EqualValues(t, "", TrimHostPort([]byte(":")))
+	assert.EqualValues(t, "", TrimHostPort(":"))
-	assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
+	assert.EqualValues(t, "example.com", TrimHostPort("example.com:80"))
 }