Fix panic in formatting not found message (#276)

Fix panic that was found in the logs, `targetOpt` is `nil`:

http2: panic serving 10.0.3.1:[...]: runtime error: invalid memory address or nil pointer dereference
net/http.(*http2serverConn).runHandler.func1()
        /usr/local/go/src/net/http/h2_bundle.go:6104 +0x145
panic({0x19c6820, 0x2d66db0})
        /usr/local/go/src/runtime/panic.go:884 +0x213
codeberg.org/codeberg/pages/server/handler.handleSubDomain({{0x2008c68, 0xc00047df90}, 0x2, {0x0, 0x0}, {0xc0fe3ef800, 0x55, 0x1f4}, {0xc00047dfa0, 0x1, ...}, ...}, ...)
        /woodpecker/src/codeberg.org/Codeberg/pages-server/server/handler/handler_sub_domain.go:59 +0x5e0

Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/276
Reviewed-by: crapStone <codeberg@crapstone.dev>
Co-authored-by: Gusted <postmaster@gusted.xyz>
Co-committed-by: Gusted <postmaster@gusted.xyz>

.gitignore

@@ -7,3 +7,4 @@ build/
 vendor/
 pages
 certs.sqlite
+.bash_history

.woodpecker.yml

@@ -1,7 +1,10 @@
-pipeline:
+when:
+  branch: main
+
+steps:
   # use vendor to cache dependencies
   vendor:
-    image: golang:1.20
+    image: golang:1.21
     commands:
       - go mod vendor
 
@@ -65,19 +68,6 @@ pipeline:
       - RAW_DOMAIN=raw.localhost.mock.directory
       - PORT=4430
 
-  # TODO: remove in next version
-  integration-tests-legacy:
-    group: test
-    image: codeberg.org/6543/docker-images/golang_just
-    commands:
-      - just integration
-    environment:
-      - ACME_API=https://acme.mock.directory
-      - PAGES_DOMAIN=localhost.mock.directory
-      - RAW_DOMAIN=raw.localhost.mock.directory
-      - PORT=4430
-      - DB_TYPE=
-
   release:
     image: plugins/gitea-release
     settings:

FEATURES.md

@@ -0,0 +1,51 @@
+# Features
+
+## Custom domains
+
+Custom domains can be used by creating a `.domains` file with the domain name, e.g.:
+
+```text
+codeberg.page
+```
+
+You also have to set some DNS records, see the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/using-custom-domain/).
+
+## Redirects
+
+Redirects can be created with a `_redirects` file with the following format:
+
+```text
+# Comment
+from  to  [status]
+```
+
+* Lines starting with `#` are ignored
+* `from` - the path to redirect from (Note: repository and branch names are removed from request URLs)
+* `to` - the path or URL to redirect to
+* `status` - status code to use when redirecting (default 301)
+
+### Status codes
+
+* `200` - returns content from specified path (no external URLs) without changing the URL (rewrite)
+* `301` - Moved Permanently (Permanent redirect)
+* `302` - Found (Temporary redirect)
+
+### Examples
+
+#### SPA (single-page application) rewrite
+
+Redirects all paths to `/index.html` for single-page apps.
+
+```text
+/*  /index.html 200
+```
+
+#### Splats
+
+Redirects every path under `/articles` to `/posts` while keeping the path.
+
+```text
+/articles/*  /posts/:splat  302
+```
+
+Example: `/articles/2022/10/12/post-1/` -> `/posts/2022/10/12/post-1/`

Justfile

@@ -9,6 +9,8 @@ dev:
     export PAGES_DOMAIN=localhost.mock.directory
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
+    export HTTP_PORT=8880
+    export ENABLE_HTTP_SERVER=true
     export LOG_LEVEL=trace
     go run -tags '{{TAGS}}' .
 
@@ -27,7 +29,7 @@ fmt: tool-gofumpt
 
 clean:
     go clean ./...
-    rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json
+    rm -rf build/ integration/certs.sqlite integration/acme-account.json
 
 tool-golangci:
     @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
@@ -50,3 +52,6 @@ integration:
 
 integration-run TEST:
     go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
+
+docker:
+    docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just

README.md

@@ -1,5 +1,11 @@
 # Codeberg Pages
 
+[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue)](https://opensource.org/license/eupl-1-2/)
+[![status-badge](https://ci.codeberg.org/api/badges/Codeberg/pages-server/status.svg)](https://ci.codeberg.org/Codeberg/pages-server)
+<a href="https://matrix.to/#/#gitea-pages-server:matrix.org" title="Join the Matrix room at https://matrix.to/#/#gitea-pages-server:matrix.org">
+  <img src="https://img.shields.io/matrix/gitea-pages-server:matrix.org?label=matrix">
+</a>
+
 Gitea lacks the ability to host static pages from Git.
 The Codeberg Pages Server addresses this lack by implementing a standalone service
 that connects to Gitea via API.
@@ -8,6 +14,8 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag
 **End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
 and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
 
+<a href="https://codeberg.org/Codeberg/pages-server"> <img src="https://codeberg.org/Codeberg/GetItOnCodeberg/raw/branch/main/get-it-on-blue-on-white.svg" alt="Get It On Codeberg" width="250"/> </a>
+
 ## Quickstart
 
 This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
@@ -29,6 +37,10 @@ record that points to your repo (just like the CNAME record):
 
 Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
 
+## Chat for admins & devs
+
+[matrix: #gitea-pages-server:matrix.org](https://matrix.to/#/#gitea-pages-server:matrix.org)
+
 ## Deployment
 
 **Warning: Some Caveats Apply**  
@@ -48,29 +60,28 @@ but if you want to run it on a shared IP address (and not a standalone),
 you'll need to configure your reverse proxy not to terminate the TLS connections,
 but forward the requests on the IP level to the Pages Server.
 
-You can check out a proof of concept in the `haproxy-sni` folder,
-and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+You can check out a proof of concept in the `examples/haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/examples/haproxy-sni/haproxy.cfg#L38).
 
-### Environment
+### Environment Variables
 
 - `HOST` & `PORT` (default: `[::]` & `443`): listen address.
 - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
-- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
+- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources (must be subdomain of `PAGES_DOMAIN`).
 - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
 - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
-- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
-- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).  
+- `RAW_INFO_PAGE` (default: <https://docs.codeberg.org/pages/raw-content/>): info page for raw resources, shown if no resource is provided.
+- `ACME_API` (default: <https://acme-v02.api.letsencrypt.org/directory>): set this to <https://acme.mock.director> to use invalid certificates without any verification (great for debugging).  
   ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
-- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
+- `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders.
 - `ACME_EAB_KID` &  `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL.
 - `ACME_ACCEPT_TERMS` (default: use self-signed certificate): Set this to "true" to accept the Terms of Service of your ACME provider.
 - `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL.
 - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
 - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.  
-  See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
+  See <https://go-acme.github.io/lego/dns/> for available values & additional environment variables.
 - `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
 
-
 ## Contributing to the development
 
 The Codeberg team is very open to your contribution.
@@ -78,31 +89,36 @@ Since we are working nicely in a team, it might be hard at times to get started
 (still check out the issues, we always aim to have some things to get you started).
 
 If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
-feel free to ping us in an issue or in a general Matrix chatgroup.
+feel free to ping us in an issue or in a general [Matrix chat room](#chat-for-admins--devs).
 
-You can also contact the maintainers of this project:
+You can also contact the maintainer(s) of this project:
+
+- [crapStone](https://codeberg.org/crapStone) [(Matrix)](https://matrix.to/#/@crapstone:obermui.de)
+
+Previous maintainers:
 
 - [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
 - [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
 
 ### First steps
 
-The code of this repository is split in several modules.
-While heavy refactoring work is currently undergo, you can easily understand the basic structure:
+The code of this repository is split in several modules.  
+The [Architecture is explained](https://codeberg.org/Codeberg/pages-server/wiki/Architecture) in the wiki.
+
 The `cmd` folder holds the data necessary for interacting with the service via the cli.
-If you are considering to deploy the service yourself, make sure to check it out.
 The heart of the software lives in the `server` folder and is split in several modules.
-After scanning the code, you should quickly be able to understand their function and start hacking on them.
 
 Again: Feel free to get in touch with us for any questions that might arise.
 Thank you very much.
 
-
 ### Test Server
 
-run `just dev`
+Make sure you have [golang](https://go.dev) v1.21 or newer and [just](https://just.systems/man/en/) installed.
+
+run `just dev`  
 now this pages should work:
- - https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg
- - https://momar.localhost.mock.directory:4430/ci-testing/
- - https://momar.localhost.mock.directory:4430/pag/@master/
- - https://mock-pages.codeberg-test.org:4430/README.md
+
+- <https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg>
+- <https://momar.localhost.mock.directory:4430/ci-testing/>
+- <https://momar.localhost.mock.directory:4430/pag/@master/>
+- <https://mock-pages.codeberg-test.org:4430/README.md>

cmd/certs.go

@@ -4,11 +4,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/rs/zerolog"
-	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
-
-	"codeberg.org/codeberg/pages/server/database"
 )
 
 var Certs = &cli.Command{
@@ -25,63 +21,8 @@ var Certs = &cli.Command{
 			Usage:  "remove a certificate from the database",
 			Action: removeCert,
 		},
-		{
-			Name:   "migrate",
-			Usage:  "migrate from \"pogreb\" driver to dbms driver",
-			Action: migrateCerts,
-		},
 	},
-	Flags: append(CertStorageFlags, []cli.Flag{
-		&cli.BoolFlag{
-			Name:    "verbose",
-			Usage:   "print trace info",
-			EnvVars: []string{"VERBOSE"},
-			Value:   false,
-		},
-	}...),
-}
-
-func migrateCerts(ctx *cli.Context) error {
-	dbType := ctx.String("db-type")
-	if dbType == "" {
-		dbType = "sqlite3"
-	}
-	dbConn := ctx.String("db-conn")
-	dbPogrebConn := ctx.String("db-pogreb")
-	verbose := ctx.Bool("verbose")
-
-	log.Level(zerolog.InfoLevel)
-	if verbose {
-		log.Level(zerolog.TraceLevel)
-	}
-
-	xormDB, err := database.NewXormDB(dbType, dbConn)
-	if err != nil {
-		return fmt.Errorf("could not connect to database: %w", err)
-	}
-	defer xormDB.Close()
-
-	pogrebDB, err := database.NewPogreb(dbPogrebConn)
-	if err != nil {
-		return fmt.Errorf("could not open database: %w", err)
-	}
-	defer pogrebDB.Close()
-
-	fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
-
-	certs, err := pogrebDB.Items(0, 0)
-	if err != nil {
-		return err
-	}
-
-	for _, cert := range certs {
-		if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
-			return err
-		}
-	}
-
-	fmt.Println("... done")
-	return nil
+	Flags: CertStorageFlags,
 }
 
 func listCerts(ctx *cli.Context) error {

cmd/flags.go

@@ -6,20 +6,15 @@ import (
 
 var (
 	CertStorageFlags = []cli.Flag{
-		&cli.StringFlag{
-			// TODO: remove in next version
-			// DEPRICATED
-			Name:    "db-pogreb",
-			Value:   "key-database.pogreb",
-			EnvVars: []string{"DB_POGREB"},
-		},
 		&cli.StringFlag{
 			Name:    "db-type",
-			Value:   "", // TODO: "sqlite3" in next version
+			Usage:   "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io",
+			Value:   "sqlite3",
 			EnvVars: []string{"DB_TYPE"},
 		},
 		&cli.StringFlag{
 			Name:    "db-conn",
+			Usage:   "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access",
 			Value:   "certs.sqlite",
 			EnvVars: []string{"DB_CONN"},
 		},
@@ -77,13 +72,6 @@ var (
 			EnvVars: []string{"RAW_DOMAIN"},
 			Value:   "raw.codeberg.page",
 		},
-		// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
-		&cli.StringFlag{
-			Name:    "raw-info-page",
-			Usage:   "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
-			EnvVars: []string{"RAW_INFO_PAGE"},
-			Value:   "https://docs.codeberg.org/codeberg-pages/raw-content/",
-		},
 
 		// #########################
 		// ### Page Server Setup ###
@@ -94,15 +82,21 @@ var (
 			EnvVars: []string{"HOST"},
 			Value:   "[::]",
 		},
-		&cli.StringFlag{
+		&cli.UintFlag{
 			Name:    "port",
-			Usage:   "specifies port of listening address",
-			EnvVars: []string{"PORT"},
-			Value:   "443",
+			Usage:   "specifies the https port to listen to ssl requests",
+			EnvVars: []string{"PORT", "HTTPS_PORT"},
+			Value:   443,
+		},
+		&cli.UintFlag{
+			Name:    "http-port",
+			Usage:   "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true",
+			EnvVars: []string{"HTTP_PORT"},
+			Value:   80,
 		},
 		&cli.BoolFlag{
-			Name: "enable-http-server",
-			// TODO: desc
+			Name:    "enable-http-server",
+			Usage:   "start a http server to redirect to https and respond to http acme challenges",
 			EnvVars: []string{"ENABLE_HTTP_SERVER"},
 		},
 		&cli.StringFlag{
@@ -111,6 +105,13 @@ var (
 			Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
 			EnvVars: []string{"LOG_LEVEL"},
 		},
+		// Default branches to fetch assets from
+		&cli.StringSliceFlag{
+			Name:    "pages-branch",
+			Usage:   "define a branch to fetch assets from",
+			EnvVars: []string{"PAGES_BRANCHES"},
+			Value:   cli.NewStringSlice("pages"),
+		},
 
 		// ############################
 		// ### ACME Client Settings ###
@@ -132,24 +133,30 @@ var (
 			Value:   true,
 		},
 		&cli.BoolFlag{
-			Name: "acme-accept-terms",
-			// TODO: Usage
+			Name:    "acme-accept-terms",
+			Usage:   "To accept the ACME ToS",
 			EnvVars: []string{"ACME_ACCEPT_TERMS"},
 		},
 		&cli.StringFlag{
-			Name: "acme-eab-kid",
-			// TODO: Usage
+			Name:    "acme-eab-kid",
+			Usage:   "Register the current account to the ACME server with external binding.",
 			EnvVars: []string{"ACME_EAB_KID"},
 		},
 		&cli.StringFlag{
-			Name: "acme-eab-hmac",
-			// TODO: Usage
+			Name:    "acme-eab-hmac",
+			Usage:   "Register the current account to the ACME server with external binding.",
 			EnvVars: []string{"ACME_EAB_HMAC"},
 		},
 		&cli.StringFlag{
-			Name: "dns-provider",
-			// TODO: Usage
+			Name:    "dns-provider",
+			Usage:   "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/",
 			EnvVars: []string{"DNS_PROVIDER"},
 		},
+		&cli.StringFlag{
+			Name:    "acme-account-config",
+			Usage:   "json file of acme account",
+			Value:   "acme-account.json",
+			EnvVars: []string{"ACME_ACCOUNT_CONFIG"},
+		},
 	}...)
 )

cmd/main.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -15,7 +14,6 @@ import (
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 
-	"codeberg.org/codeberg/pages/server"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/gitea"
@@ -47,22 +45,14 @@ func Serve(ctx *cli.Context) error {
 	giteaRoot := ctx.String("gitea-root")
 	giteaAPIToken := ctx.String("gitea-api-token")
 	rawDomain := ctx.String("raw-domain")
+	defaultBranches := ctx.StringSlice("pages-branch")
 	mainDomainSuffix := ctx.String("pages-domain")
-	rawInfoPage := ctx.String("raw-info-page")
-	listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
+	listeningHost := ctx.String("host")
+	listeningSSLPort := ctx.Uint("port")
+	listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
+	listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
 	enableHTTPServer := ctx.Bool("enable-http-server")
 
-	acmeAPI := ctx.String("acme-api-endpoint")
-	acmeMail := ctx.String("acme-email")
-	acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
-	acmeAcceptTerms := ctx.Bool("acme-accept-terms")
-	acmeEabKID := ctx.String("acme-eab-kid")
-	acmeEabHmac := ctx.String("acme-eab-hmac")
-	dnsProvider := ctx.String("dns-provider")
-	if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
-		return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
-	}
-
 	allowedCorsDomains := AllowedCorsDomains
 	if rawDomain != "" {
 		allowedCorsDomains = append(allowedCorsDomains, rawDomain)
@@ -73,6 +63,10 @@ func Serve(ctx *cli.Context) error {
 		mainDomainSuffix = "." + mainDomainSuffix
 	}
 
+	if len(defaultBranches) == 0 {
+		return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
+	}
+
 	// Init ssl cert database
 	certDB, closeFn, err := openCertDB(ctx)
 	if err != nil {
@@ -86,6 +80,8 @@ func Serve(ctx *cli.Context) error {
 	canonicalDomainCache := cache.NewKeyValueCache()
 	// dnsLookupCache stores DNS lookups for custom domains
 	dnsLookupCache := cache.NewKeyValueCache()
+	// redirectsCache stores redirects in _redirects files
+	redirectsCache := cache.NewKeyValueCache()
 	// clientResponseCache stores responses from the Gitea server
 	clientResponseCache := cache.NewKeyValueCache()
 
@@ -94,56 +90,59 @@ func Serve(ctx *cli.Context) error {
 		return fmt.Errorf("could not create new gitea client: %v", err)
 	}
 
-	// Create handler based on settings
-	httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
-		giteaClient,
-		rawInfoPage,
-		BlacklistedPaths, allowedCorsDomains,
-		dnsLookupCache, canonicalDomainCache)
+	acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache)
+	if err != nil {
+		return err
+	}
 
-	httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
+	if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil {
+		return err
+	}
 
-	// Setup listener and TLS
-	log.Info().Msgf("Listening on https://%s", listeningAddress)
-	listener, err := net.Listen("tcp", listeningAddress)
+	// Create listener for SSL connections
+	log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
+	listener, err := net.Listen("tcp", listeningSSLAddress)
 	if err != nil {
 		return fmt.Errorf("couldn't create listener: %v", err)
 	}
 
+	// Setup listener for SSL connections
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
 		giteaClient,
-		dnsProvider,
-		acmeUseRateLimits,
+		acmeClient,
+		defaultBranches[0],
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
 
-	acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
-	if err != nil {
-		return err
-	}
-
-	if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
-		return err
-	}
-
 	interval := 12 * time.Hour
 	certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
 	defer cancelCertMaintain()
-	go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
+	go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
 
 	if enableHTTPServer {
+		// Create handler for http->https redirect and http acme challenges
+		httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
+
+		// Create listener for http and start listening
 		go func() {
-			log.Info().Msg("Start HTTP server listening on :80")
-			err := http.ListenAndServe("[::]:80", httpHandler)
+			log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
+			err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
 			}
 		}()
 	}
 
-	// Start the web fastServer
-	log.Info().Msgf("Start listening on %s", listener.Addr())
-	if err := http.Serve(listener, httpsHandler); err != nil {
+	// Create ssl handler based on settings
+	sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
+		giteaClient,
+		BlacklistedPaths, allowedCorsDomains,
+		defaultBranches,
+		dnsLookupCache, canonicalDomainCache, redirectsCache)
+
+	// Start the ssl listener
+	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
+	if err := http.Serve(listener, sslHandler); err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")
 	}
 

cmd/setup.go

@@ -1,38 +1,23 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/rs/zerolog/log"
 	"github.com/urfave/cli/v2"
 
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
 )
 
+var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
+
 func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
-	if ctx.String("db-type") != "" {
-		log.Trace().Msg("use xorm mode")
-		certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not connect to database: %w", err)
-		}
-	} else {
-		// TODO: remove in next version
-		fmt.Println(`
-######################
-## W A R N I N G !!! #
-######################
-
-You use "pogreb" witch is deprecated and will be removed in the next version.
-Please switch to sqlite, mysql or postgres !!!
-
-The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`)
-		log.Error().Msg("depricated \"pogreb\" used\n")
-
-		certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("could not create database: %w", err)
-		}
+	certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not connect to database: %w", err)
 	}
 
 	closeFn = func() {
@@ -43,3 +28,37 @@ The simplest way is, to use './pages certs migrate' and set environment var DB_T
 
 	return certDB, closeFn, nil
 }
+
+func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) {
+	acmeAPI := ctx.String("acme-api-endpoint")
+	acmeMail := ctx.String("acme-email")
+	acmeEabHmac := ctx.String("acme-eab-hmac")
+	acmeEabKID := ctx.String("acme-eab-kid")
+	acmeAcceptTerms := ctx.Bool("acme-accept-terms")
+	dnsProvider := ctx.String("dns-provider")
+	acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
+	acmeAccountConf := ctx.String("acme-account-config")
+
+	// check config
+	if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
+		return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
+	}
+	if acmeEabHmac != "" && acmeEabKID == "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
+	} else if acmeEabHmac == "" && acmeEabKID != "" {
+		return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
+	}
+
+	return certificates.NewAcmeClient(
+		acmeAccountConf,
+		acmeAPI,
+		acmeMail,
+		acmeEabHmac,
+		acmeEabKID,
+		dnsProvider,
+		acmeAcceptTerms,
+		enableHTTPServer,
+		acmeUseRateLimits,
+		challengeCache,
+	)
+}

go.mod

@@ -1,20 +1,23 @@
 module codeberg.org/codeberg/pages
 
-go 1.20
+go 1.21
+
+toolchain go1.21.4
 
 require (
-	code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
+	code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
-	github.com/akrylysov/pogreb v0.10.1
 	github.com/go-acme/lego/v4 v4.5.3
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/joho/godotenv v1.4.0
 	github.com/lib/pq v1.10.7
 	github.com/mattn/go-sqlite3 v1.14.16
+	github.com/microcosm-cc/bluemonday v1.0.26
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
 	github.com/rs/zerolog v1.27.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
+	golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
 	xorm.io/xorm v1.3.2
 )
 
@@ -35,6 +38,7 @@ require (
 	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
 	github.com/aws/aws-sdk-go v1.39.0 // indirect
+	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
 	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
 	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
@@ -61,6 +65,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
 	github.com/gophercloud/gophercloud v0.16.0 // indirect
 	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/gorilla/css v1.0.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
@@ -115,11 +120,11 @@ require (
 	github.com/vultr/govultr/v2 v2.7.1 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
-	golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
 	google.golang.org/api v0.20.0 // indirect
 	google.golang.org/appengine v1.6.5 // indirect

go.sum

@@ -22,8 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
 cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
-code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk=
-code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE=
+code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f h1:nMmwDgUIAWj9XQjzHz5unC3ZMfhhwHRk6rnuwLzdu1o=
+code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
@@ -71,8 +71,6 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs=
 github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
-github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
-github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -90,6 +88,8 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN
 github.com/aws/aws-sdk-go v1.39.0 h1:74BBwkEmiqBbi2CGflEh34l0YNtIibTjZsibGarkNjo=
 github.com/aws/aws-sdk-go v1.39.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
+github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -251,8 +251,9 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
+github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -280,6 +281,8 @@ github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae/go.mod h1:wx8HMD
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
+github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
 github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
@@ -479,6 +482,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S
 github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
 github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
+github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
 github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
@@ -758,8 +763,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
 golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c=
-golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -770,6 +775,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w=
+golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -789,8 +796,9 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -826,8 +834,9 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE=
 golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -899,11 +908,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
+golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -911,8 +921,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -962,14 +973,14 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=

html/404.html

@@ -1,37 +0,0 @@
-<!doctype html>
-<html class="codeberg-design">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width">
-    <title>%status%</title>
-
-    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
-    <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
-    <link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
-
-    <style>
-        body {
-            margin: 0; padding: 1rem; box-sizing: border-box;
-            width: 100%; min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-        }
-    </style>
-</head>
-<body>
-    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
-    <h1 class="mb-0 text-primary">
-        Page not found!
-    </h1>
-    <h5 class="text-center" style="max-width: 25em;">
-        Sorry, but this page couldn't be found or is inaccessible (%status%).<br/>
-        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
-    </h5>
-    <small class="text-muted">
-        <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">
-        Static pages made easy - <a href="https://codeberg.page">Codeberg Pages</a>
-    </small>
-</body>
-</html>

html/error.go

@@ -1,50 +0,0 @@
-package html
-
-import (
-	"html/template"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"codeberg.org/codeberg/pages/server/context"
-)
-
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
-// with "%status%" and %message% replaced with the provided statusCode and msg
-func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
-	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
-	ctx.RespWriter.WriteHeader(statusCode)
-
-	msg = generateResponse(msg, statusCode)
-
-	_, _ = ctx.RespWriter.Write([]byte(msg))
-}
-
-// TODO: use template engine
-func generateResponse(msg string, statusCode int) string {
-	if msg == "" {
-		msg = strings.ReplaceAll(NotFoundPage,
-			"%status%",
-			strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
-	} else {
-		msg = strings.ReplaceAll(
-			strings.ReplaceAll(ErrorPage, "%message%", template.HTMLEscapeString(msg)),
-			"%status%",
-			http.StatusText(statusCode))
-	}
-
-	return msg
-}
-
-func errorMessage(statusCode int) string {
-	message := http.StatusText(statusCode)
-
-	switch statusCode {
-	case http.StatusMisdirectedRequest:
-		message += " - domain not specified in <code>.domains</code> file"
-	case http.StatusFailedDependency:
-		message += " - target repo/branch doesn't exist or is private"
-	}
-
-	return message
-}

html/error.html

@@ -1,38 +0,0 @@
-<!doctype html>
-<html class="codeberg-design">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width">
-    <title>%status%</title>
-
-    <link rel="stylesheet" href="https://design.codeberg.org/design-kit/codeberg.css" />
-    <link href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css" rel="stylesheet" />
-    <link href="https://fonts.codeberg.org/dist/fontawesome5/css/all.min.css" rel="stylesheet" />
-
-    <style>
-        body {
-            margin: 0; padding: 1rem; box-sizing: border-box;
-            width: 100%; min-height: 100vh;
-            display: flex;
-            flex-direction: column;
-            align-items: center;
-            justify-content: center;
-        }
-    </style>
-</head>
-<body>
-    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
-    <h1 class="mb-0 text-primary">
-        %status%!
-    </h1>
-    <h5 class="text-center" style="max-width: 25em;">
-        Sorry, but this page couldn't be served.<br/>
-        We got an <b>"%message%"</b><br/>
-        We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
-    </h5>
-    <small class="text-muted">
-        <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">
-        Static pages made easy - <a href="https://codeberg.page">Codeberg Pages</a>
-    </small>
-</body>
-</html>

html/error_test.go

@@ -1,38 +0,0 @@
-package html
-
-import (
-	"net/http"
-	"strings"
-	"testing"
-)
-
-func TestValidMessage(t *testing.T) {
-	testString := "requested blacklisted path"
-	statusCode := http.StatusForbidden
-
-	expected := strings.ReplaceAll(
-		strings.ReplaceAll(ErrorPage, "%message%", testString),
-		"%status%",
-		http.StatusText(statusCode))
-	actual := generateResponse(testString, statusCode)
-
-	if expected != actual {
-		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
-	}
-}
-
-func TestMessageWithHtml(t *testing.T) {
-	testString := `abc<img src=1 onerror=alert("xss");`
-	escapedString := "abc&lt;img src=1 onerror=alert(&#34;xss&#34;);"
-	statusCode := http.StatusNotFound
-
-	expected := strings.ReplaceAll(
-		strings.ReplaceAll(ErrorPage, "%message%", escapedString),
-		"%status%",
-		http.StatusText(statusCode))
-	actual := generateResponse(testString, statusCode)
-
-	if expected != actual {
-		t.Errorf("generated response did not match: expected: '%s', got: '%s'", expected, actual)
-	}
-}

html/html.go

@@ -1,9 +1,53 @@
 package html
 
-import _ "embed"
+import (
+	_ "embed"
+	"net/http"
+	"text/template" // do not use html/template here, we sanitize the message before passing it to the template
 
-//go:embed 404.html
-var NotFoundPage string
+	"codeberg.org/codeberg/pages/server/context"
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/rs/zerolog/log"
+)
 
-//go:embed error.html
-var ErrorPage string
+//go:embed templates/error.html
+var errorPage string
+
+var (
+	errorTemplate = template.Must(template.New("error").Parse(errorPage))
+	sanitizer     = createBlueMondayPolicy()
+)
+
+type TemplateContext struct {
+	StatusCode int
+	StatusText string
+	Message    string
+}
+
+// ReturnErrorPage sets the response status code and writes the error page to the response body.
+// The error page contains a sanitized version of the message and the statusCode both in text and numeric form.
+//
+// Currently, only the following html tags are supported: <code>
+func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
+	ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
+	ctx.RespWriter.WriteHeader(statusCode)
+
+	templateContext := TemplateContext{
+		StatusCode: statusCode,
+		StatusText: http.StatusText(statusCode),
+		Message:    sanitizer.Sanitize(msg),
+	}
+
+	err := errorTemplate.Execute(ctx.RespWriter, templateContext)
+	if err != nil {
+		log.Err(err).Str("message", msg).Int("status", statusCode).Msg("could not write response")
+	}
+}
+
+func createBlueMondayPolicy() *bluemonday.Policy {
+	p := bluemonday.NewPolicy()
+
+	p.AllowElements("code")
+
+	return p
+}

html/html_test.go

@@ -0,0 +1,54 @@
+package html
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSanitizerSimpleString(t *testing.T) {
+	str := "simple text message without any html elements"
+
+	assert.Equal(t, str, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithCodeTag(t *testing.T) {
+	str := "simple text message with <code>html</code> tag"
+
+	assert.Equal(t, str, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithCodeTagWithAttribute(t *testing.T) {
+	str := "simple text message with <code id=\"code\">html</code> tag"
+	expected := "simple text message with <code>html</code> tag"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATag(t *testing.T) {
+	str := "simple text message with <a>a link to another page</a>"
+	expected := "simple text message with a link to another page"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithATagAndHref(t *testing.T) {
+	str := "simple text message with <a href=\"http://evil.site\">a link to another page</a>"
+	expected := "simple text message with a link to another page"
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithImgTag(t *testing.T) {
+	str := "simple text message with a <img alt=\"not found\" src=\"http://evil.site\">"
+	expected := "simple text message with a "
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}
+
+func TestSanitizerStringWithImgTagAndOnerrorAttribute(t *testing.T) {
+	str := "simple text message with a <img alt=\"not found\" src=\"http://evil.site\" onerror=\"alert(secret)\">"
+	expected := "simple text message with a "
+
+	assert.Equal(t, expected, sanitizer.Sanitize(str))
+}

html/templates/error.html

@@ -0,0 +1,69 @@
+<!DOCTYPE html>
+<html class="codeberg-design">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width" />
+    <title>{{.StatusText}}</title>
+
+    <link
+      rel="stylesheet"
+      href="https://design.codeberg.org/design-kit/codeberg.css"
+    />
+    <link
+      rel="stylesheet"
+      href="https://fonts.codeberg.org/dist/inter/Inter%20Web/inter.css"
+    />
+
+    <style>
+      body {
+        margin: 0;
+        padding: 1rem;
+        box-sizing: border-box;
+        width: 100%;
+        min-height: 100vh;
+        display: flex;
+        flex-direction: column;
+        align-items: center;
+        justify-content: center;
+      }
+      code {
+        border-radius: 0.25rem;
+        padding: 0.25rem;
+        background-color: silver;
+      }
+    </style>
+  </head>
+  <body>
+    <svg
+      xmlns="http://www.w3.org/2000/svg"
+      height="10em"
+      viewBox="0 0 24 24"
+      fill="var(--blue-color)"
+    >
+      <path
+        d="M 9 2 C 5.1458514 2 2 5.1458514 2 9 C 2 12.854149 5.1458514 16 9 16 C 10.747998 16 12.345009 15.348024 13.574219 14.28125 L 14 14.707031 L 14 16 L 19.585938 21.585938 C 20.137937 22.137937 21.033938 22.137938 21.585938 21.585938 C 22.137938 21.033938 22.137938 20.137938 21.585938 19.585938 L 16 14 L 14.707031 14 L 14.28125 13.574219 C 15.348024 12.345009 16 10.747998 16 9 C 16 5.1458514 12.854149 2 9 2 z M 9 4 C 11.773268 4 14 6.2267316 14 9 C 14 11.773268 11.773268 14 9 14 C 6.2267316 14 4 11.773268 4 9 C 4 6.2267316 6.2267316 4 9 4 z"
+      />
+    </svg>
+    <h1 class="mb-0 text-primary">{{.StatusText}} ({{.StatusCode}})!</h1>
+    <h5 class="text-center" style="max-width: 25em">
+      <p>Sorry, but this page couldn't be served.</p>
+      <p><b>"{{.Message}}"</b></p>
+      <p>
+        We hope this isn't a problem on our end ;) - Make sure to check the
+        <a
+          href="https://docs.codeberg.org/codeberg-pages/troubleshooting/"
+          target="_blank"
+          >troubleshooting section in the Docs</a
+        >!
+      </p>
+    </h5>
+    <small class="text-muted">
+      <img
+        src="https://design.codeberg.org/logo-kit/icon.svg"
+        class="align-top"
+      />
+      Static pages made easy -
+      <a href="https://codeberg.page">Codeberg Pages</a>
+    </small>
+  </body>
+</html>

integration/get_test.go

@@ -64,7 +64,7 @@ func TestGetContent(t *testing.T) {
 	assert.True(t, getSize(resp.Body) > 100)
 	assert.Len(t, resp.Header.Get("ETag"), 44)
 
-	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
+	// TODO: test get of non cacheable content (content size > fileCacheSizeLimit)
 }
 
 func TestCustomDomain(t *testing.T) {
@@ -109,6 +109,34 @@ func TestCustomDomainRedirects(t *testing.T) {
 	assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location"))
 }
 
+func TestRawCustomDomain(t *testing.T) {
+	log.Println("=== TestRawCustomDomain ===")
+	// test raw domain response for custom domain branch
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "76", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 76, getSize(resp.Body))
+}
+
+func TestRawIndex(t *testing.T) {
+	log.Println("=== TestRawIndex ===")
+	// test raw domain response for index.html
+	resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "597", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 597, getSize(resp.Body))
+}
+
 func TestGetNotFound(t *testing.T) {
 	log.Println("=== TestGetNotFound ===")
 	// test custom not found pages
@@ -123,9 +151,50 @@ func TestGetNotFound(t *testing.T) {
 	assert.EqualValues(t, 37, getSize(resp.Body))
 }
 
+func TestRedirect(t *testing.T) {
+	log.Println("=== TestRedirect ===")
+	// test redirects
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/redirect")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "https://example.com/", resp.Header.Get("Location"))
+}
+
+func TestSPARedirect(t *testing.T) {
+	log.Println("=== TestSPARedirect ===")
+	// test SPA redirects
+	url := "https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/app/aqdjw"
+	resp, err := getTestHTTPSClient().Get(url)
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, url, resp.Request.URL.String())
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "258", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 258, getSize(resp.Body))
+}
+
+func TestSplatRedirect(t *testing.T) {
+	log.Println("=== TestSplatRedirect ===")
+	// test splat redirects
+	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/articles/qfopefe")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "/posts/qfopefe", resp.Header.Get("Location"))
+}
+
 func TestFollowSymlink(t *testing.T) {
 	log.Printf("=== TestFollowSymlink ===\n")
 
+	// file symlink
 	resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
 	assert.NoError(t, err)
 	if !assert.NotNil(t, resp) {
@@ -137,6 +206,16 @@ func TestFollowSymlink(t *testing.T) {
 	body := getBytes(resp.Body)
 	assert.EqualValues(t, 4, len(body))
 	assert.EqualValues(t, "abc\n", string(body))
+
+	// relative file links (../index.html file in this case)
+	resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/dir_aim/some/")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusOK, resp.StatusCode)
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "an index\n", string(getBytes(resp.Body)))
 }
 
 func TestLFSSupport(t *testing.T) {
@@ -165,6 +244,18 @@ func TestGetOptions(t *testing.T) {
 	assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
 }
 
+func TestHttpRedirect(t *testing.T) {
+	log.Println("=== TestHttpRedirect ===")
+	resp, err := getTestHTTPSClient().Get("http://mock-pages.codeberg-test.org:8880/README.md")
+	assert.NoError(t, err)
+	if !assert.NotNil(t, resp) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location"))
+}
+
 func getTestHTTPSClient() *http.Client {
 	cookieJar, _ := cookiejar.New(nil)
 	return &http.Client{

integration/main_test.go

@@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
 	}
 	defer func() {
 		serverCancel()
-		log.Println("=== TestMain: Server STOPED ===")
+		log.Println("=== TestMain: Server STOPPED ===")
 	}()
 
 	time.Sleep(10 * time.Second)
@@ -39,7 +39,10 @@ func startServer(ctx context.Context) error {
 	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
 	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
 	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master")
 	setEnvIfNotSet("PORT", "4430")
+	setEnvIfNotSet("HTTP_PORT", "8880")
+	setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
 	setEnvIfNotSet("DB_TYPE", "sqlite3")
 
 	app := cli.NewApp()

server/certificates/acme_client.go

@@ -0,0 +1,95 @@
+package certificates
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/go-acme/lego/v4/lego"
+	"github.com/go-acme/lego/v4/providers/dns"
+	"github.com/reugn/equalizer"
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/server/cache"
+)
+
+type AcmeClient struct {
+	legoClient              *lego.Client
+	dnsChallengerLegoClient *lego.Client
+
+	obtainLocks sync.Map
+
+	acmeUseRateLimits bool
+
+	// limiter
+	acmeClientOrderLimit              *equalizer.TokenBucket
+	acmeClientRequestLimit            *equalizer.TokenBucket
+	acmeClientFailLimit               *equalizer.TokenBucket
+	acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
+}
+
+func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) {
+	acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
+	if err != nil {
+		return nil, err
+	}
+
+	acmeClient, err := lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+		if err != nil {
+			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
+		}
+		if enableHTTPServer {
+			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
+			}
+		}
+	}
+
+	mainDomainAcmeClient, err := lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		if dnsProvider == "" {
+			// using mock server, don't use wildcard certs
+			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
+			}
+		} else {
+			// use DNS-Challenge https://go-acme.github.io/lego/dns/
+			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
+			if err != nil {
+				return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err)
+			}
+			if err := mainDomainAcmeClient.Challenge.SetDNS01Provider(provider); err != nil {
+				return nil, fmt.Errorf("can not create DNS-01 provider: %w", err)
+			}
+		}
+	}
+
+	return &AcmeClient{
+		legoClient:              acmeClient,
+		dnsChallengerLegoClient: mainDomainAcmeClient,
+
+		acmeUseRateLimits: acmeUseRateLimits,
+
+		obtainLocks: sync.Map{},
+
+		// limiter
+
+		// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
+		// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
+		acmeClientOrderLimit: equalizer.NewTokenBucket(25, 15*time.Minute),
+		// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
+		acmeClientRequestLimit: equalizer.NewTokenBucket(5, 1*time.Second),
+		// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
+		acmeClientFailLimit: equalizer.NewTokenBucket(5, 1*time.Hour),
+		// checkUserLimit() use this to rate also per user
+		acmeClientCertificateLimitPerUser: map[string]*equalizer.TokenBucket{},
+	}, nil
+}

server/certificates/acme_config.go

@@ -0,0 +1,102 @@
+package certificates
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/go-acme/lego/v4/certcrypto"
+	"github.com/go-acme/lego/v4/lego"
+	"github.com/go-acme/lego/v4/registration"
+	"github.com/rs/zerolog/log"
+)
+
+const challengePath = "/.well-known/acme-challenge/"
+
+func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
+	var myAcmeAccount AcmeAccount
+	var myAcmeConfig *lego.Config
+
+	if account, err := os.ReadFile(configFile); err == nil {
+		log.Info().Msgf("found existing acme account config file '%s'", configFile)
+		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
+			return nil, err
+		}
+		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
+		if err != nil {
+			return nil, err
+		}
+		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+		myAcmeConfig.CADirURL = acmeAPI
+		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+
+		// Validate Config
+		_, err := lego.NewClient(myAcmeConfig)
+		if err != nil {
+			log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate")
+			return nil, fmt.Errorf("acme config validation failed: %w", err)
+		}
+		return myAcmeConfig, nil
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	log.Info().Msgf("no existing acme account config found, try to create a new one")
+
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	myAcmeAccount = AcmeAccount{
+		Email:  acmeMail,
+		Key:    privateKey,
+		KeyPEM: string(certcrypto.PEMEncode(privateKey)),
+	}
+	myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+	myAcmeConfig.CADirURL = acmeAPI
+	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+	tempClient, err := lego.NewClient(myAcmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		// accept terms & log in to EAB
+		if acmeEabKID == "" || acmeEabHmac == "" {
+			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		} else {
+			reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
+				TermsOfServiceAgreed: acmeAcceptTerms,
+				Kid:                  acmeEabKID,
+				HmacEncoded:          acmeEabHmac,
+			})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		}
+
+		if myAcmeAccount.Registration != nil {
+			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
+			if err != nil {
+				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+			log.Info().Msgf("new acme account created. write to config file '%s'", configFile)
+			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
+			if err != nil {
+				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+		}
+	}
+
+	return myAcmeConfig, nil
+}

server/certificates/cached_challengers.go

@@ -0,0 +1,83 @@
+package certificates
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-acme/lego/v4/challenge"
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+)
+
+type AcmeTLSChallengeProvider struct {
+	challengeCache cache.SetGetKey
+}
+
+// make sure AcmeTLSChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeTLSChallengeProvider{}
+
+func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
+	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
+	a.challengeCache.Remove(domain)
+	return nil
+}
+
+type AcmeHTTPChallengeProvider struct {
+	challengeCache cache.SetGetKey
+}
+
+// make sure AcmeHTTPChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeHTTPChallengeProvider{}
+
+func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
+	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
+	a.challengeCache.Remove(domain + "/" + token)
+	return nil
+}
+
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
+	// handle custom-ssl-ports to be added on https redirects
+	portPart := ""
+	if sslPort != 443 {
+		portPart = fmt.Sprintf(":%d", sslPort)
+	}
+
+	return func(w http.ResponseWriter, req *http.Request) {
+		ctx := context.New(w, req)
+		domain := ctx.TrimHostPort()
+
+		// it's an acme request
+		if strings.HasPrefix(ctx.Path(), challengePath) {
+			challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
+			if !ok || challenge == nil {
+				log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
+				ctx.String("no challenge for this token", http.StatusNotFound)
+			}
+			log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
+			ctx.String(challenge.(string))
+			return
+		}
+
+		// it's a normal http request that needs to be redirected
+		u, err := url.Parse(fmt.Sprintf("https://%s%s%s", domain, portPart, ctx.Path()))
+		if err != nil {
+			log.Error().Err(err).Msg("could not craft http to https redirect")
+			ctx.String("", http.StatusInternalServerError)
+		}
+
+		newURL := u.String()
+		log.Debug().Msgf("redirect http to https: %s", newURL)
+		ctx.Redirect(newURL, http.StatusMovedPermanently)
+	}
+}

server/certificates/certificates.go

@@ -2,27 +2,18 @@ package certificates
 
 import (
 	"context"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/go-acme/lego/v4/certcrypto"
 	"github.com/go-acme/lego/v4/certificate"
-	"github.com/go-acme/lego/v4/challenge"
 	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
 	"github.com/go-acme/lego/v4/lego"
-	"github.com/go-acme/lego/v4/providers/dns"
-	"github.com/go-acme/lego/v4/registration"
 	"github.com/reugn/equalizer"
 	"github.com/rs/zerolog/log"
 
@@ -33,33 +24,37 @@ import (
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
+var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix string,
 	giteaClient *gitea.Client,
-	dnsProvider string,
-	acmeUseRateLimits bool,
+	acmeClient *AcmeClient,
+	firstDefaultBranch string,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
 	certDB database.CertDB,
 ) *tls.Config {
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-			sni := strings.ToLower(strings.TrimSpace(info.ServerName))
-			if len(sni) < 1 {
-				return nil, errors.New("missing sni")
+			domain := strings.ToLower(strings.TrimSpace(info.ServerName))
+			if len(domain) < 1 {
+				return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)")
 			}
 
+			// https request init is actually a acme challenge
 			if info.SupportedProtos != nil {
 				for _, proto := range info.SupportedProtos {
 					if proto != tlsalpn01.ACMETLS1Protocol {
 						continue
 					}
+					log.Info().Msgf("Detect ACME-TLS1 challenge for '%s'", domain)
 
-					challenge, ok := challengeCache.Get(sni)
+					challenge, ok := challengeCache.Get(domain)
 					if !ok {
 						return nil, errors.New("no challenge for this domain")
 					}
-					cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+					cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
 					if err != nil {
 						return nil, err
 					}
@@ -69,22 +64,22 @@ func TLSConfig(mainDomainSuffix string,
 
 			targetOwner := ""
 			mayObtainCert := true
-			if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
+			if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
 				// deliver default certificate for the main domain (*.codeberg.page)
-				sni = mainDomainSuffix
+				domain = mainDomainSuffix
 			} else {
 				var targetRepo, targetBranch string
-				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 				if targetOwner == "" {
 					// DNS not set up, return main certificate to redirect to the docs
-					sni = mainDomainSuffix
+					domain = mainDomainSuffix
 				} else {
 					targetOpt := &upstream.Options{
 						TargetOwner:  targetOwner,
 						TargetRepo:   targetRepo,
 						TargetBranch: targetBranch,
 					}
-					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
+					_, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache)
 					if !valid {
 						// We shouldn't obtain a certificate when we cannot check if the
 						// repository has specified this domain in the `.domains` file.
@@ -93,30 +88,34 @@ func TLSConfig(mainDomainSuffix string,
 				}
 			}
 
-			if tlsCertificate, ok := keyCache.Get(sni); ok {
+			if tlsCertificate, ok := keyCache.Get(domain); ok {
 				// we can use an existing certificate object
 				return tlsCertificate.(*tls.Certificate), nil
 			}
 
 			var tlsCertificate *tls.Certificate
 			var err error
-			if tlsCertificate, err = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); err != nil {
-				// request a new certificate
-				if strings.EqualFold(sni, mainDomainSuffix) {
+			if tlsCertificate, err = acmeClient.retrieveCertFromDB(domain, mainDomainSuffix, false, certDB); err != nil {
+				if !errors.Is(err, database.ErrNotFound) {
+					return nil, err
+				}
+				// we could not find a cert in db, request a new certificate
+
+				// first check if we are allowed to obtain a cert for this domain
+				if strings.EqualFold(domain, mainDomainSuffix) {
 					return nil, errors.New("won't request certificate for main domain, something really bad has happened")
 				}
-
 				if !mayObtainCert {
-					return nil, fmt.Errorf("won't request certificate for %q", sni)
+					return nil, fmt.Errorf("won't request certificate for %q", domain)
 				}
 
-				tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+				tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB)
 				if err != nil {
 					return nil, err
 				}
 			}
 
-			if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil {
+			if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil {
 				return nil, err
 			}
 			return tlsCertificate, nil
@@ -141,67 +140,20 @@ func TLSConfig(mainDomainSuffix string,
 	}
 }
 
-func checkUserLimit(user string) error {
-	userLimit, ok := acmeClientCertificateLimitPerUser[user]
+func (c *AcmeClient) checkUserLimit(user string) error {
+	userLimit, ok := c.acmeClientCertificateLimitPerUser[user]
 	if !ok {
-		// Each Codeberg user can only add 10 new domains per day.
+		// Each user can only add 10 new domains per day.
 		userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
-		acmeClientCertificateLimitPerUser[user] = userLimit
+		c.acmeClientCertificateLimitPerUser[user] = userLimit
 	}
 	if !userLimit.Ask() {
-		return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+		return fmt.Errorf("user '%s' error: %w", user, ErrUserRateLimitExceeded)
 	}
 	return nil
 }
 
-var (
-	acmeClient, mainDomainAcmeClient  *lego.Client
-	acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
-)
-
-// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
-// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
-var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
-
-// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
-var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
-
-// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
-var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour)
-
-type AcmeTLSChallengeProvider struct {
-	challengeCache cache.SetGetKey
-}
-
-// make sure AcmeTLSChallengeProvider match Provider interface
-var _ challenge.Provider = AcmeTLSChallengeProvider{}
-
-func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
-	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
-}
-
-func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
-	a.challengeCache.Remove(domain)
-	return nil
-}
-
-type AcmeHTTPChallengeProvider struct {
-	challengeCache cache.SetGetKey
-}
-
-// make sure AcmeHTTPChallengeProvider match Provider interface
-var _ challenge.Provider = AcmeHTTPChallengeProvider{}
-
-func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
-	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
-}
-
-func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
-	a.challengeCache.Remove(domain + "/" + token)
-	return nil
-}
-
-func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (*tls.Certificate, error) {
+func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProvider bool, certDB database.CertDB) (*tls.Certificate, error) {
 	// parse certificate from database
 	res, err := certDB.Get(sni)
 	if err != nil {
@@ -219,7 +171,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 	if !strings.EqualFold(sni, mainDomainSuffix) {
 		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
 		if err != nil {
-			return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err)
+			return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err)
 		}
 
 		// renew certificates 7 days before they expire
@@ -235,7 +187,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 			// TODO: make a queue ?
 			go (func() {
 				res.CSR = nil // acme client doesn't like CSR to be set
-				if _, err := obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB); err != nil {
+				if _, err := c.obtainCert(c.legoClient, []string{sni}, res, "", useDnsProvider, mainDomainSuffix, certDB); err != nil {
 					log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err)
 				}
 			})()
@@ -245,28 +197,26 @@ func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLi
 	return &tlsCertificate, nil
 }
 
-var obtainLocks = sync.Map{}
-
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (*tls.Certificate, error) {
+func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) {
 	name := strings.TrimPrefix(domains[0], "*")
-	if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
+	if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' {
 		domains = domains[1:]
 	}
 
 	// lock to avoid simultaneous requests
-	_, working := obtainLocks.LoadOrStore(name, struct{}{})
+	_, working := c.obtainLocks.LoadOrStore(name, struct{}{})
 	if working {
 		for working {
 			time.Sleep(100 * time.Millisecond)
-			_, working = obtainLocks.Load(name)
+			_, working = c.obtainLocks.Load(name)
 		}
-		cert, err := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+		cert, err := c.retrieveCertFromDB(name, mainDomainSuffix, useDnsProvider, keyDatabase)
 		if err != nil {
 			return nil, fmt.Errorf("certificate failed in synchronous request: %w", err)
 		}
 		return cert, nil
 	}
-	defer obtainLocks.Delete(name)
+	defer c.obtainLocks.Delete(name)
 
 	if acmeClient == nil {
 		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
@@ -276,29 +226,29 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 	var res *certificate.Resource
 	var err error
 	if renew != nil && renew.CertURL != "" {
-		if acmeUseRateLimits {
-			acmeClientRequestLimit.Take()
+		if c.acmeUseRateLimits {
+			c.acmeClientRequestLimit.Take()
 		}
 		log.Debug().Msgf("Renewing certificate for: %v", domains)
 		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
 		if err != nil {
 			log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
-			if acmeUseRateLimits {
-				acmeClientFailLimit.Take()
+			if c.acmeUseRateLimits {
+				c.acmeClientFailLimit.Take()
 			}
 			res = nil
 		}
 	}
 	if res == nil {
 		if user != "" {
-			if err := checkUserLimit(user); err != nil {
+			if err := c.checkUserLimit(user); err != nil {
 				return nil, err
 			}
 		}
 
-		if acmeUseRateLimits {
-			acmeClientOrderLimit.Take()
-			acmeClientRequestLimit.Take()
+		if c.acmeUseRateLimits {
+			c.acmeClientOrderLimit.Take()
+			c.acmeClientRequestLimit.Take()
 		}
 		log.Debug().Msgf("Re-requesting new certificate for %v", domains)
 		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
@@ -306,8 +256,8 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			Bundle:     true,
 			MustStaple: false,
 		})
-		if acmeUseRateLimits && err != nil {
-			acmeClientFailLimit.Take()
+		if c.acmeUseRateLimits && err != nil {
+			c.acmeClientFailLimit.Take()
 		}
 	}
 	if err != nil {
@@ -349,136 +299,15 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 	return &tlsCertificate, nil
 }
 
-func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
-	// TODO: make it a config flag
-	const configFile = "acme-account.json"
-	var myAcmeAccount AcmeAccount
-	var myAcmeConfig *lego.Config
-
-	if account, err := os.ReadFile(configFile); err == nil {
-		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
-			return nil, err
-		}
-		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
-		if err != nil {
-			return nil, err
-		}
-		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-		myAcmeConfig.CADirURL = acmeAPI
-		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-
-		// Validate Config
-		_, err := lego.NewClient(myAcmeConfig)
-		if err != nil {
-			// TODO: should we fail hard instead?
-			log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-		}
-		return myAcmeConfig, nil
-	} else if !os.IsNotExist(err) {
-		return nil, err
-	}
-
-	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	if err != nil {
-		return nil, err
-	}
-	myAcmeAccount = AcmeAccount{
-		Email:  acmeMail,
-		Key:    privateKey,
-		KeyPEM: string(certcrypto.PEMEncode(privateKey)),
-	}
-	myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-	myAcmeConfig.CADirURL = acmeAPI
-	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-	tempClient, err := lego.NewClient(myAcmeConfig)
-	if err != nil {
-		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		// accept terms & log in to EAB
-		if acmeEabKID == "" || acmeEabHmac == "" {
-			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
-			} else {
-				myAcmeAccount.Registration = reg
-			}
-		} else {
-			reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
-				TermsOfServiceAgreed: acmeAcceptTerms,
-				Kid:                  acmeEabKID,
-				HmacEncoded:          acmeEabHmac,
-			})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
-			} else {
-				myAcmeAccount.Registration = reg
-			}
-		}
-
-		if myAcmeAccount.Registration != nil {
-			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
-			if err != nil {
-				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
-				select {}
-			}
-			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
-			if err != nil {
-				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
-				select {}
-			}
-		}
-	}
-
-	return myAcmeConfig, nil
-}
-
-func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+func SetupMainDomainCertificates(mainDomainSuffix string, acmeClient *AcmeClient, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
 	mainCertBytes, err := certDB.Get(mainDomainSuffix)
 	if err != nil && !errors.Is(err, database.ErrNotFound) {
 		return fmt.Errorf("cert database is not working: %w", err)
 	}
 
-	acmeClient, err = lego.NewClient(acmeConfig)
-	if err != nil {
-		log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
-		if err != nil {
-			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
-		}
-		if enableHTTPServer {
-			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
-			}
-		}
-	}
-
-	mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
-	if err != nil {
-		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
-	} else {
-		if dnsProvider == "" {
-			// using mock server, don't use wildcard certs
-			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
-			}
-		} else {
-			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create DNS Challenge provider")
-			}
-			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
-			if err != nil {
-				log.Error().Err(err).Msg("Can't create DNS-01 provider")
-			}
-		}
-	}
-
 	if mainCertBytes == nil {
-		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+		_, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", true, mainDomainSuffix, certDB)
 		if err != nil {
 			log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
 		}
@@ -487,7 +316,7 @@ func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Co
 	return nil
 }
 
-func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *AcmeClient, mainDomainSuffix string, certDB database.CertDB) {
 	for {
 		// delete expired certs that will be invalid until next clean up
 		threshold := time.Now().Add(interval)
@@ -510,14 +339,6 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 				}
 			}
 			log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
-
-			// compact the database
-			msg, err := certDB.Compact()
-			if err != nil {
-				log.Error().Err(err).Msg("Compacting key database failed")
-			} else {
-				log.Debug().Msgf("Compacted key database: %s", msg)
-			}
 		}
 
 		// update main cert
@@ -533,7 +354,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 			} else if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
 				// renew main certificate 30 days before it expires
 				go (func() {
-					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+					_, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", true, mainDomainSuffix, certDB)
 					if err != nil {
 						log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
 					}

server/certificates/mock_test.go

@@ -3,13 +3,16 @@ package certificates
 import (
 	"testing"
 
-	"codeberg.org/codeberg/pages/server/database"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+
+	"codeberg.org/codeberg/pages/server/database"
 )
 
 func TestMockCert(t *testing.T) {
-	db, err := database.NewTmpDB()
-	assert.NoError(t, err)
+	db := database.NewMockCertDB(t)
+	db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil)
+
 	cert, err := mockCert("example.com", "some error msg", "codeberg.page", db)
 	assert.NoError(t, err)
 	if assert.NotEmpty(t, cert) {

server/context/context.go

@@ -48,11 +48,9 @@ func (c *Context) Redirect(uri string, statusCode int) {
 	http.Redirect(c.RespWriter, c.Req, uri, statusCode)
 }
 
-// Path returns requested path.
-//
-// The returned bytes are valid until your request handler returns.
+// Path returns the cleaned requested path.
 func (c *Context) Path() string {
-	return c.Req.URL.Path
+	return utils.CleanPath(c.Req.URL.Path)
 }
 
 func (c *Context) Host() string {

server/database/interface.go

@@ -8,14 +8,15 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+//go:generate go install github.com/vektra/mockery/v2@latest
+//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore
+
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
 	Get(name string) (*certificate.Resource, error)
 	Delete(key string) error
 	Items(page, pageSize int) ([]*Cert, error)
-	// Compact deprecated // TODO: remove in next version
-	Compact() (string, error)
 }
 
 type Cert struct {

server/database/mock.go

@@ -1,54 +1,122 @@
+// Code generated by mockery v2.20.0. DO NOT EDIT.
+
 package database
 
 import (
-	"fmt"
-	"time"
-
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/go-acme/lego/v4/certificate"
+	certificate "github.com/go-acme/lego/v4/certificate"
+	mock "github.com/stretchr/testify/mock"
 )
 
-var _ CertDB = tmpDB{}
-
-type tmpDB struct {
-	intern *mcache.CacheDriver
-	ttl    time.Duration
+// MockCertDB is an autogenerated mock type for the CertDB type
+type MockCertDB struct {
+	mock.Mock
 }
 
-func (p tmpDB) Close() error {
-	_ = p.intern.Close()
-	return nil
-}
+// Close provides a mock function with given fields:
+func (_m *MockCertDB) Close() error {
+	ret := _m.Called()
 
-func (p tmpDB) Put(name string, cert *certificate.Resource) error {
-	return p.intern.Set(name, cert, p.ttl)
-}
-
-func (p tmpDB) Get(name string) (*certificate.Resource, error) {
-	cert, has := p.intern.Get(name)
-	if !has {
-		return nil, fmt.Errorf("cert for %q not found", name)
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
 	}
-	return cert.(*certificate.Resource), nil
+
+	return r0
 }
 
-func (p tmpDB) Delete(key string) error {
-	p.intern.Remove(key)
-	return nil
+// Delete provides a mock function with given fields: key
+func (_m *MockCertDB) Delete(key string) error {
+	ret := _m.Called(key)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(key)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
 }
 
-func (p tmpDB) Compact() (string, error) {
-	p.intern.Truncate()
-	return "Truncate done", nil
+// Get provides a mock function with given fields: name
+func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) {
+	ret := _m.Called(name)
+
+	var r0 *certificate.Resource
+	var r1 error
+	if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok {
+		return rf(name)
+	}
+	if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok {
+		r0 = rf(name)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*certificate.Resource)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(string) error); ok {
+		r1 = rf(name)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
-	return nil, fmt.Errorf("items not implemented for tmpDB")
+// Items provides a mock function with given fields: page, pageSize
+func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) {
+	ret := _m.Called(page, pageSize)
+
+	var r0 []*Cert
+	var r1 error
+	if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok {
+		return rf(page, pageSize)
+	}
+	if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok {
+		r0 = rf(page, pageSize)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*Cert)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(int, int) error); ok {
+		r1 = rf(page, pageSize)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
-func NewTmpDB() (CertDB, error) {
-	return &tmpDB{
-		intern: mcache.New(),
-		ttl:    time.Minute,
-	}, nil
+// Put provides a mock function with given fields: name, cert
+func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error {
+	ret := _m.Called(name, cert)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok {
+		r0 = rf(name, cert)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+type mockConstructorTestingTNewMockCertDB interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB {
+	mock := &MockCertDB{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
 }

server/database/pogreb.go

@@ -1,134 +0,0 @@
-package database
-
-import (
-	"bytes"
-	"context"
-	"encoding/gob"
-	"errors"
-	"fmt"
-	"time"
-
-	"github.com/akrylysov/pogreb"
-	"github.com/akrylysov/pogreb/fs"
-	"github.com/go-acme/lego/v4/certificate"
-	"github.com/rs/zerolog/log"
-)
-
-var _ CertDB = aDB{}
-
-type aDB struct {
-	ctx          context.Context
-	cancel       context.CancelFunc
-	intern       *pogreb.DB
-	syncInterval time.Duration
-}
-
-func (p aDB) Close() error {
-	p.cancel()
-	return p.intern.Sync()
-}
-
-func (p aDB) Put(name string, cert *certificate.Resource) error {
-	var resGob bytes.Buffer
-	if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
-		return err
-	}
-	return p.intern.Put([]byte(name), resGob.Bytes())
-}
-
-func (p aDB) Get(name string) (*certificate.Resource, error) {
-	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get([]byte(name))
-	if err != nil {
-		return nil, err
-	}
-	if resBytes == nil {
-		return nil, nil
-	}
-	if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
-		return nil, err
-	}
-	return cert, nil
-}
-
-func (p aDB) Delete(key string) error {
-	return p.intern.Delete([]byte(key))
-}
-
-func (p aDB) Compact() (string, error) {
-	result, err := p.intern.Compact()
-	if err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("%+v", result), nil
-}
-
-func (p aDB) Items(_, _ int) ([]*Cert, error) {
-	items := make([]*Cert, 0, p.intern.Count())
-	iterator := p.intern.Items()
-	for {
-		key, resBytes, err := iterator.Next()
-		if err != nil {
-			if errors.Is(err, pogreb.ErrIterationDone) {
-				break
-			}
-			return nil, err
-		}
-
-		res := &certificate.Resource{}
-		if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
-			return nil, err
-		}
-
-		cert, err := toCert(string(key), res)
-		if err != nil {
-			return nil, err
-		}
-
-		items = append(items, cert)
-	}
-
-	return items, nil
-}
-
-var _ CertDB = &aDB{}
-
-func (p aDB) sync() {
-	for {
-		err := p.intern.Sync()
-		if err != nil {
-			log.Error().Err(err).Msg("Syncing cert database failed")
-		}
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-time.After(p.syncInterval):
-		}
-	}
-}
-
-func NewPogreb(path string) (CertDB, error) {
-	if path == "" {
-		return nil, fmt.Errorf("path not set")
-	}
-	db, err := pogreb.Open(path, &pogreb.Options{
-		BackgroundSyncInterval:       30 * time.Second,
-		BackgroundCompactionInterval: 6 * time.Hour,
-		FileSystem:                   fs.OSMMap,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	result := &aDB{
-		ctx:          ctx,
-		cancel:       cancel,
-		intern:       db,
-		syncInterval: 5 * time.Minute,
-	}
-
-	go result.sync()
-
-	return result, nil
-}

server/database/xorm.go

@@ -37,7 +37,7 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) {
 	}
 
 	if err := e.Sync2(new(Cert)); err != nil {
-		return nil, fmt.Errorf("cound not sync db model :%w", err)
+		return nil, fmt.Errorf("could not sync db model :%w", err)
 	}
 
 	return &xDB{
@@ -64,7 +64,7 @@ func (x xDB) Put(domain string, cert *certificate.Resource) error {
 	}
 	defer sess.Close()
 
-	if exist, _ := sess.ID(c.Domain).Exist(); exist {
+	if exist, _ := sess.ID(c.Domain).Exist(new(Cert)); exist {
 		if _, err := sess.ID(c.Domain).Update(c); err != nil {
 			return err
 		}
@@ -106,11 +106,6 @@ func (x xDB) Delete(domain string) error {
 	return err
 }
 
-func (x xDB) Compact() (string, error) {
-	// not needed
-	return "", nil
-}
-
 // Items return al certs from db, if pageSize is 0 it does not use limit
 func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
 	// paginated return

server/database/xorm_test.go

@@ -37,7 +37,7 @@ func TestSanitizeWildcardCerts(t *testing.T) {
 	}))
 
 	// update existing cert
-	assert.Error(t, certDB.Put(".wildcard.de", &certificate.Resource{
+	assert.NoError(t, certDB.Put(".wildcard.de", &certificate.Resource{
 		Domain:      "*.wildcard.de",
 		Certificate: localhost_mock_directory_certificate,
 	}))

server/dns/dns.go

@@ -11,9 +11,11 @@ import (
 // lookupCacheTimeout specifies the timeout for the DNS lookup cache.
 var lookupCacheTimeout = 15 * time.Minute
 
+var defaultPagesRepo = "pages"
+
 // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
 // If everything is fine, it returns the target data.
-func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
 	// Get CNAME or TXT
 	var cname string
 	var err error
@@ -50,10 +52,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetG
 		targetBranch = cnameParts[len(cnameParts)-3]
 	}
 	if targetRepo == "" {
-		targetRepo = "pages"
+		targetRepo = defaultPagesRepo
 	}
-	if targetBranch == "" && targetRepo != "pages" {
-		targetBranch = "pages"
+	if targetBranch == "" && targetRepo != defaultPagesRepo {
+		targetBranch = firstDefaultBranch
 	}
 	// if targetBranch is still empty, the caller must find the default branch
 	return

server/gitea/client.go

@@ -17,12 +17,13 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/version"
 )
 
 var ErrorNotFound = errors.New("not found")
 
 const (
-	// cache key prefixe
+	// cache key prefixes
 	branchTimestampCacheKeyPrefix = "branchTime"
 	defaultBranchCacheKeyPrefix   = "defaultBranch"
 	rawContentCacheKeyPrefix      = "rawContent"
@@ -76,7 +77,13 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
 		defaultMimeType = "application/octet-stream"
 	}
 
-	sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
+	sdk, err := gitea.NewClient(
+		giteaRoot,
+		gitea.SetHTTPClient(&stdClient),
+		gitea.SetToken(giteaAPIToken),
+		gitea.SetUserAgent("pages-server/"+version.Version),
+	)
+
 	return &Client{
 		sdkClient:     sdk,
 		responseCache: respCache,
@@ -112,7 +119,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 	if cache, ok := client.responseCache.Get(cacheKey); ok {
 		cache := cache.(FileResponse)
 		cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
-		// TODO: check against some timestamp missmatch?!?
+		// TODO: check against some timestamp mismatch?!?
 		if cache.Exists {
 			if cache.IsSymlink {
 				linkDest := string(cache.Body)
@@ -145,6 +152,10 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 					}
 					linkDest := strings.TrimSpace(string(linkDestBytes))
 
+					// handle relative links
+					// we first remove the link from the path, and make a relative join (resolve parent paths like "/../" too)
+					linkDest = path.Join(path.Dir(resource), linkDest)
+
 					// we store symlink not content to reduce duplicates in cache
 					if err := client.responseCache.Set(cacheKey, FileResponse{
 						Exists:    true,
@@ -168,7 +179,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
 				return reader, resp.Response.Header, resp.StatusCode, err
 			}
 
-			// now we write to cache and respond at the sime time
+			// now we write to cache and respond at the same time
 			fileResp := FileResponse{
 				Exists:   true,
 				ETag:     resp.Header.Get(ETagHeader),
@@ -274,11 +285,11 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
 		return false
 	}
 
-	contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
+	contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64)
 	if err != nil {
 		log.Error().Err(err).Msg("could not parse content length")
 	}
 
 	// if content to big or could not be determined we not cache it
-	return contentLeng > 0 && contentLeng < fileCacheSizeLimit
+	return contentLength > 0 && contentLength < fileCacheSizeLimit
 }

server/handler/handler.go

@@ -10,28 +10,26 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
-	"codeberg.org/codeberg/pages/server/version"
 )
 
 const (
 	headerAccessControlAllowOrigin  = "Access-Control-Allow-Origin"
 	headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
 	defaultPagesRepo                = "pages"
-	defaultPagesBranch              = "pages"
 )
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain string,
 	giteaClient *gitea.Client,
-	rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains []string,
-	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	defaultPagesBranches []string,
+	dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
 		ctx := context.New(w, req)
 
-		ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
+		ctx.RespWriter.Header().Set("Server", "pages-server")
 
 		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
 		ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -66,7 +64,7 @@ func Handler(mainDomainSuffix, rawDomain string,
 		// Block blacklisted paths (like ACME challenges)
 		for _, blacklistedPath := range blacklistedPaths {
 			if strings.HasPrefix(ctx.Path(), blacklistedPath) {
-				html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
+				html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden)
 				return
 			}
 		}
@@ -88,26 +86,28 @@ func Handler(mainDomainSuffix, rawDomain string,
 		pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
 
 		if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
-			log.Debug().Msg("raw domain request detecded")
+			log.Debug().Msg("raw domain request detected")
 			handleRaw(log, ctx, giteaClient,
-				mainDomainSuffix, rawInfoPage,
-				trimmedHost,
-				pathElements,
-				canonicalDomainCache)
-		} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
-			log.Debug().Msg("subdomain request detecded")
-			handleSubDomain(log, ctx, giteaClient,
 				mainDomainSuffix,
 				trimmedHost,
 				pathElements,
-				canonicalDomainCache)
+				canonicalDomainCache, redirectsCache)
+		} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+			log.Debug().Msg("subdomain request detected")
+			handleSubDomain(log, ctx, giteaClient,
+				mainDomainSuffix,
+				defaultPagesBranches,
+				trimmedHost,
+				pathElements,
+				canonicalDomainCache, redirectsCache)
 		} else {
-			log.Debug().Msg("custom domain request detecded")
+			log.Debug().Msg("custom domain request detected")
 			handleCustomDomain(log, ctx, giteaClient,
 				mainDomainSuffix,
 				trimmedHost,
 				pathElements,
-				dnsLookupCache, canonicalDomainCache)
+				defaultPagesBranches[0],
+				dnsLookupCache, canonicalDomainCache, redirectsCache)
 		}
 	}
 }

server/handler/handler_custom_domain.go

@@ -18,10 +18,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 	mainDomainSuffix string,
 	trimmedHost string,
 	pathElements []string,
-	dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	firstDefaultBranch string,
+	dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) {
 	// Serve pages from custom domains
-	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache)
+	targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 	if targetOwner == "" {
 		html.ReturnErrorPage(ctx,
 			"could not obtain repo owner from custom domain",
@@ -52,7 +53,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 			return
 		} else if canonicalDomain != trimmedHost {
 			// only redirect if the target is also a codeberg page!
-			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache)
+			targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
 			if targetOwner != "" {
 				ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
 				return
@@ -63,7 +64,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
 		}
 
 		log.Debug().Msg("tryBranch, now trying upstream 7")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		return
 	}
 

server/handler/handler_raw_domain.go

@@ -16,17 +16,21 @@ import (
 )
 
 func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
-	mainDomainSuffix, rawInfoPage string,
+	mainDomainSuffix string,
 	trimmedHost string,
 	pathElements []string,
-	canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) {
 	// Serve raw content from RawDomain
 	log.Debug().Msg("raw domain")
 
 	if len(pathElements) < 2 {
-		// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
-		ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
+		html.ReturnErrorPage(
+			ctx,
+			"a url in the form of <code>https://{domain}/{owner}/{repo}[/@{branch}]/{path}</code> is required",
+			http.StatusBadRequest,
+		)
+
 		return
 	}
 
@@ -41,7 +45,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
 			TargetPath:   path.Join(pathElements[3:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 			return
 		}
 		log.Debug().Msg("missing branch info")
@@ -58,10 +62,10 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
 		TargetPath:    path.Join(pathElements[2:]...),
 	}, true); works {
 		log.Trace().Msg("tryUpstream: serve raw domain with default branch")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 	} else {
 		html.ReturnErrorPage(ctx,
-			fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
+			fmt.Sprintf("raw domain could not find repo <code>%s/%s</code> or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),
 			http.StatusNotFound)
 	}
 }

server/handler/handler_sub_domain.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/rs/zerolog"
+	"golang.org/x/exp/slices"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
@@ -17,9 +18,10 @@ import (
 
 func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
 	mainDomainSuffix string,
+	defaultPagesBranches []string,
 	trimmedHost string,
 	pathElements []string,
-	canonicalDomainCache cache.SetGetKey,
+	canonicalDomainCache, redirectsCache cache.SetGetKey,
 ) {
 	// Serve pages from subdomains of MainDomainSuffix
 	log.Debug().Msg("main domain suffix")
@@ -51,11 +53,13 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 			TargetPath:    path.Join(pathElements[2:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve with specified repo and branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		} else {
-			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
-				http.StatusFailedDependency)
+			html.ReturnErrorPage(
+				ctx,
+				formatSetBranchNotFoundMessage(pathElements[1][1:], targetOwner, pathElements[0]),
+				http.StatusFailedDependency,
+			)
 		}
 		return
 	}
@@ -63,38 +67,66 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 	// Check if the first directory is a branch for the defaultPagesRepo
 	// example.codeberg.page/@main/index.html
 	if strings.HasPrefix(pathElements[0], "@") {
+		targetBranch := pathElements[0][1:]
+
+		// if the default pages branch can be determined exactly, it does not need to be set
+		if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) {
+			// example.codeberg.org/@pages/... redirects to example.codeberg.org/...
+			ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
+			return
+		}
+
 		log.Debug().Msg("main domain preparations, now trying with specified branch")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
 			TargetRepo:    defaultPagesRepo,
-			TargetBranch:  pathElements[0][1:],
+			TargetBranch:  targetBranch,
 			TargetPath:    path.Join(pathElements[1:]...),
 		}, true); works {
 			log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		} else {
-			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
-				http.StatusFailedDependency)
+			html.ReturnErrorPage(
+				ctx,
+				formatSetBranchNotFoundMessage(targetBranch, targetOwner, defaultPagesRepo),
+				http.StatusFailedDependency,
+			)
 		}
 		return
 	}
 
-	// Check if the first directory is a repo with a defaultPagesRepo branch
-	// example.codeberg.page/myrepo/index.html
-	// example.codeberg.page/pages/... is not allowed here.
-	log.Debug().Msg("main domain preparations, now trying with specified repo")
-	if pathElements[0] != defaultPagesRepo {
+	for _, defaultPagesBranch := range defaultPagesBranches {
+		// Check if the first directory is a repo with a default pages branch
+		// example.codeberg.page/myrepo/index.html
+		// example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here.
+		log.Debug().Msg("main domain preparations, now trying with specified repo")
+		if pathElements[0] != defaultPagesBranch {
+			if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
+				TryIndexPages: true,
+				TargetOwner:   targetOwner,
+				TargetRepo:    pathElements[0],
+				TargetBranch:  defaultPagesBranch,
+				TargetPath:    path.Join(pathElements[1:]...),
+			}, false); works {
+				log.Debug().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
+				return
+			}
+		}
+
+		// Try to use the defaultPagesRepo on an default pages branch
+		// example.codeberg.page/index.html
+		log.Debug().Msg("main domain preparations, now trying with default repo")
 		if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
 			TryIndexPages: true,
 			TargetOwner:   targetOwner,
-			TargetRepo:    pathElements[0],
+			TargetRepo:    defaultPagesRepo,
 			TargetBranch:  defaultPagesBranch,
-			TargetPath:    path.Join(pathElements[1:]...),
+			TargetPath:    path.Join(pathElements...),
 		}, false); works {
-			log.Debug().Msg("tryBranch, now trying upstream 5")
-			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+			log.Debug().Msg("tryBranch, now trying upstream 6")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 			return
 		}
 	}
@@ -109,12 +141,16 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
 		TargetPath:    path.Join(pathElements...),
 	}, false); works {
 		log.Debug().Msg("tryBranch, now trying upstream 6")
-		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
+		tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
 		return
 	}
 
 	// Couldn't find a valid repo/branch
 	html.ReturnErrorPage(ctx,
-		fmt.Sprintf("could not find a valid repository[%s]", targetRepo),
+		fmt.Sprintf("could not find a valid repository or branch for repository: <code>%s</code>", targetRepo),
 		http.StatusNotFound)
 }
+
+func formatSetBranchNotFoundMessage(branch, owner, repo string) string {
+	return fmt.Sprintf("explicitly set branch <code>%q</code> does not exist at <code>%s/%s</code>", branch, owner, repo)
+}

server/handler/handler_test.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"net/http"
 	"net/http/httptest"
 	"testing"
 	"time"
@@ -15,16 +16,17 @@ func TestHandlerPerformance(t *testing.T) {
 	testHandler := Handler(
 		"codeberg.page", "raw.codeberg.org",
 		giteaClient,
-		"https://docs.codeberg.org/pages/raw-content/",
 		[]string{"/.well-known/acme-challenge/"},
 		[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
+		[]string{"pages"},
+		cache.NewKeyValueCache(),
 		cache.NewKeyValueCache(),
 		cache.NewKeyValueCache(),
 	)
 
 	testCase := func(uri string, status int) {
 		t.Run(uri, func(t *testing.T) {
-			req := httptest.NewRequest("GET", uri, nil)
+			req := httptest.NewRequest("GET", uri, http.NoBody)
 			w := httptest.NewRecorder()
 
 			log.Printf("Start: %v\n", time.Now())

server/handler/try.go

@@ -18,9 +18,10 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost string,
 	options *upstream.Options,
 	canonicalDomainCache cache.SetGetKey,
+	redirectsCache cache.SetGetKey,
 ) {
 	// check if a canonical domain exists on a request on MainDomain
-	if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
+	if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
 		canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
 			canonicalPath := ctx.Req.RequestURI
@@ -39,8 +40,8 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
 	options.Host = trimmedHost
 
 	// Try to request the file from the Gitea API
-	if !options.Upstream(ctx, giteaClient) {
-		html.ReturnErrorPage(ctx, "", ctx.StatusCode)
+	if !options.Upstream(ctx, giteaClient, redirectsCache) {
+		html.ReturnErrorPage(ctx, "gitea client failed", ctx.StatusCode)
 	}
 }
 

server/setup.go

@@ -1,27 +0,0 @@
-package server
-
-import (
-	"net/http"
-	"strings"
-
-	"codeberg.org/codeberg/pages/server/cache"
-	"codeberg.org/codeberg/pages/server/context"
-	"codeberg.org/codeberg/pages/server/utils"
-)
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
-	challengePath := "/.well-known/acme-challenge/"
-
-	return func(w http.ResponseWriter, req *http.Request) {
-		ctx := context.New(w, req)
-		if strings.HasPrefix(ctx.Path(), challengePath) {
-			challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
-			if !ok || challenge == nil {
-				ctx.String("no challenge for this token", http.StatusNotFound)
-			}
-			ctx.String(challenge.(string))
-		} else {
-			ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
-		}
-	}
-}

server/upstream/redirects.go

@@ -0,0 +1,117 @@
+package upstream
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/context"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"github.com/rs/zerolog/log"
+)
+
+type Redirect struct {
+	From       string
+	To         string
+	StatusCode int
+}
+
+// redirectsCacheTimeout specifies the timeout for the redirects cache.
+var redirectsCacheTimeout = 10 * time.Minute
+
+const redirectsConfig = "_redirects"
+
+// getRedirects returns redirects specified in the _redirects file.
+func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect {
+	var redirects []Redirect
+	cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
+
+	// Check for cached redirects
+	if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
+		redirects = cachedValue.([]Redirect)
+	} else {
+		// Get _redirects file and parse
+		body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, redirectsConfig)
+		if err == nil {
+			for _, line := range strings.Split(string(body), "\n") {
+				redirectArr := strings.Fields(line)
+
+				// Ignore comments and invalid lines
+				if strings.HasPrefix(line, "#") || len(redirectArr) < 2 {
+					continue
+				}
+
+				// Get redirect status code
+				statusCode := 301
+				if len(redirectArr) == 3 {
+					statusCode, err = strconv.Atoi(redirectArr[2])
+					if err != nil {
+						log.Info().Err(err).Msgf("could not read %s of %s/%s", redirectsConfig, o.TargetOwner, o.TargetRepo)
+					}
+				}
+
+				redirects = append(redirects, Redirect{
+					From:       redirectArr[0],
+					To:         redirectArr[1],
+					StatusCode: statusCode,
+				})
+			}
+		}
+		_ = redirectsCache.Set(cacheKey, redirects, redirectsCacheTimeout)
+	}
+	return redirects
+}
+
+func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) {
+	if len(redirects) > 0 {
+		for _, redirect := range redirects {
+			reqUrl := ctx.Req.RequestURI
+			// remove repo and branch from request url
+			reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo)
+			reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch)
+
+			// check if from url matches request url
+			if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") {
+				// do rewrite if status code is 200
+				if redirect.StatusCode == 200 {
+					o.TargetPath = redirect.To
+					o.Upstream(ctx, giteaClient, redirectsCache)
+					return true
+				} else {
+					ctx.Redirect(redirect.To, redirect.StatusCode)
+					return true
+				}
+			}
+
+			// handle wildcard redirects
+			trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*")
+			if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) {
+				if strings.Contains(redirect.To, ":splat") {
+					splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl))
+					// do rewrite if status code is 200
+					if redirect.StatusCode == 200 {
+						o.TargetPath = splatUrl
+						o.Upstream(ctx, giteaClient, redirectsCache)
+						return true
+					} else {
+						ctx.Redirect(splatUrl, redirect.StatusCode)
+						return true
+					}
+				} else {
+					// do rewrite if status code is 200
+					if redirect.StatusCode == 200 {
+						o.TargetPath = redirect.To
+						o.Upstream(ctx, giteaClient, redirectsCache)
+						return true
+					} else {
+						ctx.Redirect(redirect.To, redirect.StatusCode)
+						return true
+					}
+				}
+			}
+		}
+	}
+
+	return false
+}

server/upstream/upstream.go

@@ -11,6 +11,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/context"
 	"codeberg.org/codeberg/pages/server/gitea"
 )
@@ -52,11 +53,11 @@ type Options struct {
 }
 
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
 
 	if o.TargetOwner == "" || o.TargetRepo == "" {
-		html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
+		html.ReturnErrorPage(ctx, "gitea client: either repo owner or name info is missing", http.StatusBadRequest)
 		return true
 	}
 
@@ -66,7 +67,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		// handle 404
 		if err != nil && errors.Is(err, gitea.ErrorNotFound) || !branchExist {
 			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("branch %q for '%s/%s' not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
+				fmt.Sprintf("branch <code>%q</code> for <code>%s/%s</code> not found", o.TargetBranch, o.TargetOwner, o.TargetRepo),
 				http.StatusNotFound)
 			return true
 		}
@@ -74,7 +75,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		// handle unexpected errors
 		if err != nil {
 			html.ReturnErrorPage(ctx,
-				fmt.Sprintf("could not get timestamp of branch %q: %v", o.TargetBranch, err),
+				fmt.Sprintf("could not get timestamp of branch <code>%q</code>: '%v'", o.TargetBranch, err),
 				http.StatusFailedDependency)
 			return true
 		}
@@ -103,6 +104,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 
 	// Handle not found error
 	if err != nil && errors.Is(err, gitea.ErrorNotFound) {
+		// Get and match redirects
+		redirects := o.getRedirects(giteaClient, redirectsCache)
+		if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) {
+			return true
+		}
+
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -110,7 +117,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaClient) {
+				if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
 					return true
 				}
 			}
@@ -118,7 +125,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaClient) {
+			if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
 				return true
 			}
 		}
@@ -131,11 +138,12 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 			optionsForNotFoundPages.appendTrailingSlash = false
 			for _, notFoundPage := range upstreamNotFoundPages {
 				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
-				if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, redirectsCache) {
 					return true
 				}
 			}
 		}
+
 		return false
 	}
 
@@ -145,16 +153,16 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		var msg string
 
 		if err != nil {
-			msg = "gitea client returned unexpected error"
+			msg = "gitea client: returned unexpected error"
 			log.Error().Err(err).Msg(msg)
-			msg = fmt.Sprintf("%s: %v", msg, err)
+			msg = fmt.Sprintf("%s: '%v'", msg, err)
 		}
 		if reader == nil {
-			msg = "gitea client returned no reader"
+			msg = "gitea client: returned no reader"
 			log.Error().Msg(msg)
 		}
 		if statusCode != http.StatusOK {
-			msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+			msg = fmt.Sprintf("gitea client: couldn't fetch contents: <code>%d - %s</code>", statusCode, http.StatusText(statusCode))
 			log.Error().Msg(msg)
 		}
 
@@ -168,7 +176,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (fin
 		ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
 		return true
 	}
-	if strings.HasSuffix(ctx.Path(), "/index.html") {
+	if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
 		ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
 		return true
 	}

server/utils/utils.go

@@ -1,6 +1,8 @@
 package utils
 
 import (
+	"net/url"
+	"path"
 	"strings"
 )
 
@@ -11,3 +13,15 @@ func TrimHostPort(host string) string {
 	}
 	return host
 }
+
+func CleanPath(uriPath string) string {
+	unescapedPath, _ := url.PathUnescape(uriPath)
+	cleanedPath := path.Join("/", unescapedPath)
+
+	// If the path refers to a directory, add a trailing slash.
+	if !strings.HasSuffix(cleanedPath, "/") && (strings.HasSuffix(unescapedPath, "/") || strings.HasSuffix(unescapedPath, "/.") || strings.HasSuffix(unescapedPath, "/..")) {
+		cleanedPath += "/"
+	}
+
+	return cleanedPath
+}

server/utils/utils_test.go

@@ -11,3 +11,59 @@ func TestTrimHostPort(t *testing.T) {
 	assert.EqualValues(t, "", TrimHostPort(":"))
 	assert.EqualValues(t, "example.com", TrimHostPort("example.com:80"))
 }
+
+// TestCleanPath is mostly copied from fasthttp, to keep the behaviour we had before migrating away from it.
+// Source (MIT licensed): https://github.com/valyala/fasthttp/blob/v1.48.0/uri_test.go#L154
+// Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia, Kirill Danshin, Erik Dubbelboer, FastHTTP Authors
+func TestCleanPath(t *testing.T) {
+	// double slash
+	testURIPathNormalize(t, "/aa//bb", "/aa/bb")
+
+	// triple slash
+	testURIPathNormalize(t, "/x///y/", "/x/y/")
+
+	// multi slashes
+	testURIPathNormalize(t, "/abc//de///fg////", "/abc/de/fg/")
+
+	// encoded slashes
+	testURIPathNormalize(t, "/xxxx%2fyyy%2f%2F%2F", "/xxxx/yyy/")
+
+	// dotdot
+	testURIPathNormalize(t, "/aaa/..", "/")
+
+	// dotdot with trailing slash
+	testURIPathNormalize(t, "/xxx/yyy/../", "/xxx/")
+
+	// multi dotdots
+	testURIPathNormalize(t, "/aaa/bbb/ccc/../../ddd", "/aaa/ddd")
+
+	// dotdots separated by other data
+	testURIPathNormalize(t, "/a/b/../c/d/../e/..", "/a/c/")
+
+	// too many dotdots
+	testURIPathNormalize(t, "/aaa/../../../../xxx", "/xxx")
+	testURIPathNormalize(t, "/../../../../../..", "/")
+	testURIPathNormalize(t, "/../../../../../../", "/")
+
+	// encoded dotdots
+	testURIPathNormalize(t, "/aaa%2Fbbb%2F%2E.%2Fxxx", "/aaa/xxx")
+
+	// double slash with dotdots
+	testURIPathNormalize(t, "/aaa////..//b", "/b")
+
+	// fake dotdot
+	testURIPathNormalize(t, "/aaa/..bbb/ccc/..", "/aaa/..bbb/")
+
+	// single dot
+	testURIPathNormalize(t, "/a/./b/././c/./d.html", "/a/b/c/d.html")
+	testURIPathNormalize(t, "./foo/", "/foo/")
+	testURIPathNormalize(t, "./../.././../../aaa/bbb/../../../././../", "/")
+	testURIPathNormalize(t, "./a/./.././../b/./foo.html", "/b/foo.html")
+}
+
+func testURIPathNormalize(t *testing.T, requestURI, expectedPath string) {
+	cleanedPath := CleanPath(requestURI)
+	if cleanedPath != expectedPath {
+		t.Fatalf("Unexpected path %q. Expected %q. requestURI=%q", cleanedPath, expectedPath, requestURI)
+	}
+}