Add host to handler logging (#123)

- Add the host to the Handler's logging fields, so you don't just see the path, but also which domain was being requested.

Co-authored-by: Gusted <williamzijl7@hotmail.com>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/123
Reviewed-by: 6543 <6543@noreply.codeberg.org>
Co-authored-by: Gusted <gusted@noreply.codeberg.org>
Co-committed-by: Gusted <gusted@noreply.codeberg.org>

.gitignore

@@ -4,3 +4,4 @@ key-database.pogreb/
 acme-account.json
 build/
 vendor/
+pages

.woodpecker.yml

@@ -0,0 +1,108 @@
+branches: main
+
+pipeline:
+  # use vendor to cache dependencies
+  vendor:
+    image: golang:1.18
+    commands:
+      - go mod vendor
+
+  lint:
+    image: golangci/golangci-lint:latest
+    group: compliant
+    pull: true
+    commands:
+      - go version
+      - go install mvdan.cc/gofumpt@latest
+      - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
+      - golangci-lint run --timeout 5m --build-tags integration
+
+  build:
+    group: compliant
+    image: a6543/golang_just
+    commands:
+      - go version
+      - just build
+    when:
+      event: [ "pull_request", "push" ]
+
+  docker-dryrun:
+    group: compliant
+    image: plugins/kaniko
+    settings:
+      dockerfile: Dockerfile
+      no_push: true
+      tags: latest
+    when:
+      event: [ "pull_request", "push" ]
+      path: Dockerfile
+
+  build-tag:
+    group: compliant
+    image: a6543/golang_just
+    commands:
+      - go version
+      - just build-tag ${CI_COMMIT_TAG##v}
+    when:
+      event: [ "tag" ]
+
+  test:
+    group: test
+    image: a6543/golang_just
+    commands:
+      - just test
+
+  integration-tests:
+    group: test
+    image: a6543/golang_just
+    commands:
+      - just integration
+    environment:
+      - ACME_API=https://acme.mock.directory
+      - PAGES_DOMAIN=localhost.mock.directory
+      - RAW_DOMAIN=raw.localhost.mock.directory
+      - PORT=4430
+
+  release:
+    image: plugins/gitea-release
+    settings:
+      base_url: https://codeberg.org
+      file_exists: overwrite
+      files: build/codeberg-pages-server
+      api_key:
+        from_secret: bot_token
+    environment:
+      - DRONE_REPO_OWNER=${CI_REPO_OWNER}
+      - DRONE_REPO_NAME=${CI_REPO_NAME}
+      - DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
+      - DRONE_COMMIT_REF=${CI_COMMIT_REF}
+    when:
+      event: [ "tag" ]
+
+  docker-next:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tags: next
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "push" ]
+
+  docker-tag:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tag: [ latest, "${CI_COMMIT_TAG}" ]
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "tag" ]

Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:alpine as build
+
+WORKDIR /workspace
+
+RUN apk add ca-certificates
+COPY . .
+RUN CGO_ENABLED=0 go build .
+
+FROM scratch
+COPY --from=build /workspace/pages /pages
+COPY --from=build \
+    /etc/ssl/certs/ca-certificates.crt \
+    /etc/ssl/certs/ca-certificates.crt
+    
+ENTRYPOINT ["/pages"]

Justfile

@@ -6,7 +6,44 @@ dev:
     export PAGES_DOMAIN=localhost.mock.directory
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
+    export LOG_LEVEL=trace
     go run .
 
 build:
     CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
+
+build-tag VERSION:
+    CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
+
+lint: tool-golangci tool-gofumpt
+    [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
+    golangci-lint run --timeout 5m --build-tags integration
+
+fmt: tool-gofumpt
+    gofumpt -w --extra .
+
+clean:
+    go clean ./...
+    rm -rf build/
+
+tool-golangci:
+    @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \
+    fi
+
+tool-gofumpt:
+    @hash gofumpt> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install mvdan.cc/gofumpt@latest; \
+    fi
+
+test:
+    go test -race codeberg.org/codeberg/pages/server/...
+
+test-run TEST:
+    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
+
+integration:
+    go test -race -tags integration codeberg.org/codeberg/pages/integration/...
+
+integration-run TEST:
+    go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

README.md

@@ -1,11 +1,66 @@
-## Environment
+# Codeberg Pages
+
+Gitea lacks the ability to host static pages from Git.
+The Codeberg Pages Server addresses this lack by implementing a standalone service
+that connects to Gitea via API.
+It is suitable to be deployed by other Gitea instances, too, to offer static pages hosting to their users.
+
+**End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
+and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
+
+
+## Quickstart
+
+This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
+Mapping custom domains is not static anymore, but can be done with DNS:
+
+1) add a `.domains` text file to your repository, containing the allowed domains, separated by new lines. The
+first line will be the canonical domain/URL; all other occurrences will be redirected to it.
+
+2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
+"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
+If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
+	`www.example.org. IN CNAME main.pages.example.codeberg.page.`
+
+3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
+for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
+record that points to your repo (just like the CNAME record):
+	`example.org IN ALIAS codeberg.page.`
+	`example.org IN TXT main.pages.example.codeberg.page.`
+
+Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
+
+
+## Deployment
+
+**Warning: Some Caveats Apply**  
+> Currently, the deployment requires you to have some knowledge of system administration as well as understanding and building code,
+> so you can eventually edit non-configurable and codeberg-specific settings.
+> In the future, we'll try to reduce these and make hosting Codeberg Pages as easy as setting up Gitea.
+> If you consider using Pages in practice, please consider contacting us first,
+> we'll then try to share some basic steps and document the current usage for admins
+> (might be changing in the current state).
+
+Deploying the software itself is very easy. You can grab a current release binary or build yourself,
+configure the environment as described below, and you are done.
+
+The hard part is about adding **custom domain support** if you intend to use it.
+SSL certificates (request + renewal) is automatically handled by the Pages Server,
+but if you want to run it on a shared IP address (and not a standalone),
+you'll need to configure your reverse proxy not to terminate the TLS connections,
+but forward the requests on the IP level to the Pages Server.
+
+You can check out a proof of concept in the `haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+
+### Environment
 
 - `HOST` & `PORT` (default: `[::]` & `443`): listen address.
 - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
-- `RAW_DOMAIN` (default: `raw.codeberg.org`): domain for raw resources.
+- `RAW_DOMAIN` (default: `raw.codeberg.page`): domain for raw resources.
 - `GITEA_ROOT` (default: `https://codeberg.org`): root of the upstream Gitea instance.
 - `GITEA_API_TOKEN` (default: empty): API token for the Gitea instance to access non-public (e.g. limited) repos.
-- `REDIRECT_RAW_INFO` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
+- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
 - `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).  
   ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
 - `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
@@ -15,3 +70,40 @@
 - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
 - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.  
   See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
+- `DEBUG` (default: false): Set this to true to enable debug logging.
+
+
+## Contributing to the development
+
+The Codeberg team is very open to your contribution.
+Since we are working nicely in a team, it might be hard at times to get started
+(still check out the issues, we always aim to have some things to get you started).
+
+If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
+feel free to ping us in an issue or in a general Matrix chatgroup.
+
+You can also contact the maintainers of this project:
+
+- [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
+- [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
+
+### First steps
+
+The code of this repository is split in several modules.
+While heavy refactoring work is currently undergo, you can easily understand the basic structure:
+The `cmd` folder holds the data necessary for interacting with the service via the cli.
+If you are considering to deploy the service yourself, make sure to check it out.
+The heart of the software lives in the `server` folder and is split in several modules.
+After scanning the code, you should quickly be able to understand their function and start hacking on them.
+
+Again: Feel free to get in touch with us for any questions that might arise.
+Thank you very much.
+
+
+### Test Server
+
+run `just dev`
+now this pages should work:
+ - https://magiclike.localhost.mock.directory:4430/
+ - https://momar.localhost.mock.directory:4430/ci-testing/
+ - https://momar.localhost.mock.directory:4430/pag/@master/

certificates.go

@@ -1,599 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"crypto"
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"crypto/rsa"
-	"crypto/tls"
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/gob"
-	"encoding/json"
-	"encoding/pem"
-	"errors"
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/akrylysov/pogreb/fs"
-	"github.com/go-acme/lego/v4/certificate"
-	"github.com/go-acme/lego/v4/challenge"
-	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
-	"github.com/go-acme/lego/v4/providers/dns"
-	"io/ioutil"
-	"log"
-	"math/big"
-	"os"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/akrylysov/pogreb"
-	"github.com/reugn/equalizer"
-
-	"github.com/go-acme/lego/v4/certcrypto"
-	"github.com/go-acme/lego/v4/lego"
-	"github.com/go-acme/lego/v4/registration"
-)
-
-// tlsConfig contains the configuration for generating, serving and cleaning up Let's Encrypt certificates.
-var tlsConfig = &tls.Config{
-	// check DNS name & get certificate from Let's Encrypt
-	GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
-		sni := strings.ToLower(strings.TrimSpace(info.ServerName))
-		sniBytes := []byte(sni)
-		if len(sni) < 1 {
-			return nil, errors.New("missing sni")
-		}
-
-		if info.SupportedProtos != nil {
-			for _, proto := range info.SupportedProtos {
-				if proto == tlsalpn01.ACMETLS1Protocol {
-					challenge, ok := challengeCache.Get(sni)
-					if !ok {
-						return nil, errors.New("no challenge for this domain")
-					}
-					cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
-					if err != nil {
-						return nil, err
-					}
-					return cert, nil
-				}
-			}
-		}
-
-		targetOwner := ""
-		if bytes.HasSuffix(sniBytes, MainDomainSuffix) || bytes.Equal(sniBytes, MainDomainSuffix[1:]) {
-			// deliver default certificate for the main domain (*.codeberg.page)
-			sniBytes = MainDomainSuffix
-			sni = string(sniBytes)
-		} else {
-			var targetRepo, targetBranch string
-			targetOwner, targetRepo, targetBranch = getTargetFromDNS(sni)
-			if targetOwner == "" {
-				// DNS not set up, return main certificate to redirect to the docs
-				sniBytes = MainDomainSuffix
-				sni = string(sniBytes)
-			} else {
-				_, _ = targetRepo, targetBranch
-				_, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, sni)
-				if !valid {
-					sniBytes = MainDomainSuffix
-					sni = string(sniBytes)
-				}
-			}
-		}
-
-		if tlsCertificate, ok := keyCache.Get(sni); ok {
-			// we can use an existing certificate object
-			return tlsCertificate.(*tls.Certificate), nil
-		}
-
-		var tlsCertificate tls.Certificate
-		var err error
-		var ok bool
-		if tlsCertificate, ok = retrieveCertFromDB(sniBytes); !ok {
-			// request a new certificate
-			if bytes.Equal(sniBytes, MainDomainSuffix) {
-				return nil, errors.New("won't request certificate for main domain, something really bad has happened")
-			}
-
-			tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner)
-			if err != nil {
-				return nil, err
-			}
-		}
-
-		err = keyCache.Set(sni, &tlsCertificate, 15*time.Minute)
-		if err != nil {
-			panic(err)
-		}
-		return &tlsCertificate, nil
-	},
-	PreferServerCipherSuites: true,
-	NextProtos: []string{
-		"http/1.1",
-		tlsalpn01.ACMETLS1Protocol,
-	},
-
-	// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
-	// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
-	MinVersion: tls.VersionTLS12,
-	CipherSuites: []uint16{
-		tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
-		tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
-		tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-		tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
-		tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
-		tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
-	},
-}
-
-var keyCache = mcache.New()
-var keyDatabase, keyDatabaseErr = pogreb.Open("key-database.pogreb", &pogreb.Options{
-	BackgroundSyncInterval:       30 * time.Second,
-	BackgroundCompactionInterval: 6 * time.Hour,
-	FileSystem:                   fs.OSMMap,
-})
-
-func CheckUserLimit(user string) error {
-	userLimit, ok := acmeClientCertificateLimitPerUser[user]
-	if !ok {
-		// Each Codeberg user can only add 10 new domains per day.
-		userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
-		acmeClientCertificateLimitPerUser[user] = userLimit
-	}
-	if !userLimit.Ask() {
-		return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
-	}
-	return nil
-}
-
-var myAcmeAccount AcmeAccount
-var myAcmeConfig *lego.Config
-
-type AcmeAccount struct {
-	Email        string
-	Registration *registration.Resource
-	Key          crypto.PrivateKey `json:"-"`
-	KeyPEM       string            `json:"Key"`
-}
-
-func (u *AcmeAccount) GetEmail() string {
-	return u.Email
-}
-func (u AcmeAccount) GetRegistration() *registration.Resource {
-	return u.Registration
-}
-func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
-	return u.Key
-}
-
-var acmeClient, mainDomainAcmeClient *lego.Client
-var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
-
-// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
-// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
-var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
-
-// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
-var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
-
-var challengeCache = mcache.New()
-
-type AcmeTLSChallengeProvider struct{}
-
-var _ challenge.Provider = AcmeTLSChallengeProvider{}
-
-func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
-	return challengeCache.Set(domain, keyAuth, 1*time.Hour)
-}
-func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
-	challengeCache.Remove(domain)
-	return nil
-}
-
-type AcmeHTTPChallengeProvider struct{}
-
-var _ challenge.Provider = AcmeHTTPChallengeProvider{}
-
-func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
-	return challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
-}
-func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
-	challengeCache.Remove(domain + "/" + token)
-	return nil
-}
-
-func retrieveCertFromDB(sni []byte) (tls.Certificate, bool) {
-	// parse certificate from database
-	res := &certificate.Resource{}
-	if !PogrebGet(keyDatabase, sni, res) {
-		return tls.Certificate{}, false
-	}
-
-	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
-	if err != nil {
-		panic(err)
-	}
-
-	if !bytes.Equal(sni, MainDomainSuffix) {
-		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
-		if err != nil {
-			panic(err)
-		}
-
-		// renew certificates 7 days before they expire
-		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
-			if res.CSR != nil && len(res.CSR) > 0 {
-				// CSR stores the time when the renewal shall be tried again
-				nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
-				if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
-					return tlsCertificate, true
-				}
-			}
-			go (func() {
-				res.CSR = nil // acme client doesn't like CSR to be set
-				tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "")
-				if err != nil {
-					log.Printf("Couldn't renew certificate for %s: %s", sni, err)
-				}
-			})()
-		}
-	}
-
-	return tlsCertificate, true
-}
-
-var obtainLocks = sync.Map{}
-
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string) (tls.Certificate, error) {
-	name := strings.TrimPrefix(domains[0], "*")
-	if os.Getenv("DNS_PROVIDER") == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
-		domains = domains[1:]
-	}
-
-	// lock to avoid simultaneous requests
-	_, working := obtainLocks.LoadOrStore(name, struct{}{})
-	if working {
-		for working {
-			time.Sleep(100 * time.Millisecond)
-			_, working = obtainLocks.Load(name)
-		}
-		cert, ok := retrieveCertFromDB([]byte(name))
-		if !ok {
-			return tls.Certificate{}, errors.New("certificate failed in synchronous request")
-		}
-		return cert, nil
-	}
-	defer obtainLocks.Delete(name)
-
-	if acmeClient == nil {
-		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!"), nil
-	}
-
-	// request actual cert
-	var res *certificate.Resource
-	var err error
-	if renew != nil && renew.CertURL != "" {
-		if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
-			acmeClientRequestLimit.Take()
-		}
-		log.Printf("Renewing certificate for %v", domains)
-		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
-		if err != nil {
-			log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
-			res = nil
-		}
-	}
-	if res == nil {
-		if user != "" {
-			if err := CheckUserLimit(user); err != nil {
-				return tls.Certificate{}, err
-			}
-		}
-
-		if os.Getenv("ACME_USE_RATE_LIMITS") != "false" {
-			acmeClientOrderLimit.Take()
-			acmeClientRequestLimit.Take()
-		}
-		log.Printf("Requesting new certificate for %v", domains)
-		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
-			Domains:    domains,
-			Bundle:     true,
-			MustStaple: false,
-		})
-	}
-	if err != nil {
-		log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
-		if renew != nil && renew.CertURL != "" {
-			tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
-			if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
-				// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
-				renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
-				PogrebPut(keyDatabase, []byte(name), renew)
-				return tlsCertificate, nil
-			}
-		} else {
-			return mockCert(domains[0], err.Error()), err
-		}
-	}
-	log.Printf("Obtained certificate for %v", domains)
-
-	PogrebPut(keyDatabase, []byte(name), res)
-	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
-	if err != nil {
-		return tls.Certificate{}, err
-	}
-	return tlsCertificate, nil
-}
-
-func mockCert(domain string, msg string) tls.Certificate {
-	key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
-	if err != nil {
-		panic(err)
-	}
-
-	template := x509.Certificate{
-		SerialNumber: big.NewInt(1),
-		Subject: pkix.Name{
-			CommonName:   domain,
-			Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
-			OrganizationalUnit: []string{
-				"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
-				"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
-					"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
-				"Error message: " + msg,
-			},
-		},
-
-		// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
-		NotAfter:  time.Now().Add(time.Hour*24*7 + time.Hour*6),
-		NotBefore: time.Now(),
-
-		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
-		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
-		BasicConstraintsValid: true,
-	}
-	certBytes, err := x509.CreateCertificate(
-		rand.Reader,
-		&template,
-		&template,
-		&key.(*rsa.PrivateKey).PublicKey,
-		key,
-	)
-	if err != nil {
-		panic(err)
-	}
-
-	out := &bytes.Buffer{}
-	err = pem.Encode(out, &pem.Block{
-		Bytes: certBytes,
-		Type:  "CERTIFICATE",
-	})
-	if err != nil {
-		panic(err)
-	}
-	outBytes := out.Bytes()
-	res := &certificate.Resource{
-		PrivateKey:        certcrypto.PEMEncode(key),
-		Certificate:       outBytes,
-		IssuerCertificate: outBytes,
-		Domain:            domain,
-	}
-	databaseName := domain
-	if domain == "*"+string(MainDomainSuffix) || domain == string(MainDomainSuffix[1:]) {
-		databaseName = string(MainDomainSuffix)
-	}
-	PogrebPut(keyDatabase, []byte(databaseName), res)
-
-	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
-	if err != nil {
-		panic(err)
-	}
-	return tlsCertificate
-}
-
-func setupCertificates() {
-	if keyDatabaseErr != nil {
-		panic(keyDatabaseErr)
-	}
-
-	if os.Getenv("ACME_ACCEPT_TERMS") != "true" || (os.Getenv("DNS_PROVIDER") == "" && os.Getenv("ACME_API") != "https://acme.mock.directory") {
-		panic(errors.New("you must set ACME_ACCEPT_TERMS and DNS_PROVIDER, unless ACME_API is set to https://acme.mock.directory"))
-	}
-
-	// getting main cert before ACME account so that we can panic here on database failure without hitting rate limits
-	mainCertBytes, err := keyDatabase.Get(MainDomainSuffix)
-	if err != nil {
-		// key database is not working
-		panic(err)
-	}
-
-	if account, err := ioutil.ReadFile("acme-account.json"); err == nil {
-		err = json.Unmarshal(account, &myAcmeAccount)
-		if err != nil {
-			panic(err)
-		}
-		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
-		if err != nil {
-			panic(err)
-		}
-		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-		myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
-		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-		_, err := lego.NewClient(myAcmeConfig)
-		if err != nil {
-			log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
-		}
-	} else if os.IsNotExist(err) {
-		privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-		if err != nil {
-			panic(err)
-		}
-		myAcmeAccount = AcmeAccount{
-			Email:  envOr("ACME_EMAIL", "noreply@example.email"),
-			Key:    privateKey,
-			KeyPEM: string(certcrypto.PEMEncode(privateKey)),
-		}
-		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
-		myAcmeConfig.CADirURL = envOr("ACME_API", "https://acme-v02.api.letsencrypt.org/directory")
-		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
-		tempClient, err := lego.NewClient(myAcmeConfig)
-		if err != nil {
-			log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
-		} else {
-			// accept terms & log in to EAB
-			if os.Getenv("ACME_EAB_KID") == "" || os.Getenv("ACME_EAB_HMAC") == "" {
-				reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true"})
-				if err != nil {
-					log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
-				} else {
-					myAcmeAccount.Registration = reg
-				}
-			} else {
-				reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
-					TermsOfServiceAgreed: os.Getenv("ACME_ACCEPT_TERMS") == "true",
-					Kid:                  os.Getenv("ACME_EAB_KID"),
-					HmacEncoded:          os.Getenv("ACME_EAB_HMAC"),
-				})
-				if err != nil {
-					log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
-				} else {
-					myAcmeAccount.Registration = reg
-				}
-			}
-
-			if myAcmeAccount.Registration != nil {
-				acmeAccountJson, err := json.Marshal(myAcmeAccount)
-				if err != nil {
-					log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
-					select {}
-				}
-				err = ioutil.WriteFile("acme-account.json", acmeAccountJson, 0600)
-				if err != nil {
-					log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
-					select {}
-				}
-			}
-		}
-	} else {
-		panic(err)
-	}
-
-	acmeClient, err = lego.NewClient(myAcmeConfig)
-	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
-	} else {
-		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
-		if err != nil {
-			log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
-		}
-		if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
-			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{})
-			if err != nil {
-				log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
-			}
-		}
-	}
-
-	mainDomainAcmeClient, err = lego.NewClient(myAcmeConfig)
-	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
-	} else {
-		if os.Getenv("DNS_PROVIDER") == "" {
-			// using mock server, don't use wildcard certs
-			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{})
-			if err != nil {
-				log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
-			}
-		} else {
-			provider, err := dns.NewDNSChallengeProviderByName(os.Getenv("DNS_PROVIDER"))
-			if err != nil {
-				log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
-			}
-			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
-			if err != nil {
-				log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
-			}
-		}
-	}
-
-	if mainCertBytes == nil {
-		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, nil, "")
-		if err != nil {
-			log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
-		}
-	}
-
-	go (func() {
-		for {
-			err := keyDatabase.Sync()
-			if err != nil {
-				log.Printf("[ERROR] Syncinc key database failed: %s", err)
-			}
-			time.Sleep(5 * time.Minute)
-		}
-	})()
-	go (func() {
-		for {
-			// clean up expired certs
-			now := time.Now()
-			expiredCertCount := 0
-			keyDatabaseIterator := keyDatabase.Items()
-			key, resBytes, err := keyDatabaseIterator.Next()
-			for err == nil {
-				if !bytes.Equal(key, MainDomainSuffix) {
-					resGob := bytes.NewBuffer(resBytes)
-					resDec := gob.NewDecoder(resGob)
-					res := &certificate.Resource{}
-					err = resDec.Decode(res)
-					if err != nil {
-						panic(err)
-					}
-
-					tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
-					if err != nil || !tlsCertificates[0].NotAfter.After(now) {
-						err := keyDatabase.Delete(key)
-						if err != nil {
-							log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
-						} else {
-							expiredCertCount++
-						}
-					}
-				}
-				key, resBytes, err = keyDatabaseIterator.Next()
-			}
-			log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
-
-			// compact the database
-			result, err := keyDatabase.Compact()
-			if err != nil {
-				log.Printf("[ERROR] Compacting key database failed: %s", err)
-			} else {
-				log.Printf("[INFO] Compacted key database (%+v)", result)
-			}
-
-			// update main cert
-			res := &certificate.Resource{}
-			if !PogrebGet(keyDatabase, MainDomainSuffix, res) {
-				log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
-			} else {
-				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
-
-				// renew main certificate 30 days before it expires
-				if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
-					go (func() {
-						_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(MainDomainSuffix), string(MainDomainSuffix[1:])}, res, "")
-						if err != nil {
-							log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
-						}
-					})()
-				}
-			}
-
-			time.Sleep(12 * time.Hour)
-		}
-	})()
-}

cmd/certs.go

@@ -0,0 +1,72 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/akrylysov/pogreb"
+	"github.com/urfave/cli/v2"
+
+	"codeberg.org/codeberg/pages/server/database"
+)
+
+var Certs = &cli.Command{
+	Name:  "certs",
+	Usage: "manage certs manually",
+	Subcommands: []*cli.Command{
+		{
+			Name:   "list",
+			Usage:  "list all certificates in the database",
+			Action: listCerts,
+		},
+		{
+			Name:   "remove",
+			Usage:  "remove a certificate from the database",
+			Action: removeCert,
+		},
+	},
+}
+
+func listCerts(ctx *cli.Context) error {
+	// TODO: make "key-database.pogreb" set via flag
+	keyDatabase, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
+	}
+
+	items := keyDatabase.Items()
+	for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() {
+		if err != nil {
+			return err
+		}
+		if domain[0] == '.' {
+			fmt.Printf("*")
+		}
+		fmt.Printf("%s\n", domain)
+	}
+	return nil
+}
+
+func removeCert(ctx *cli.Context) error {
+	if ctx.Args().Len() < 1 {
+		return fmt.Errorf("'certs remove' requires at least one domain as an argument")
+	}
+
+	domains := ctx.Args().Slice()
+
+	// TODO: make "key-database.pogreb" set via flag
+	keyDatabase, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
+	}
+
+	for _, domain := range domains {
+		fmt.Printf("Removing domain %s from the database...\n", domain)
+		if err := keyDatabase.Delete(domain); err != nil {
+			return err
+		}
+	}
+	if err := keyDatabase.Close(); err != nil {
+		return err
+	}
+	return nil
+}

cmd/flags.go

@@ -0,0 +1,123 @@
+package cmd
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var ServeFlags = []cli.Flag{
+	// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
+	// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
+	// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
+	&cli.StringFlag{
+		Name:    "pages-domain",
+		Usage:   "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
+		EnvVars: []string{"PAGES_DOMAIN"},
+		Value:   "codeberg.page",
+	},
+	// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
+	&cli.StringFlag{
+		Name:    "gitea-root",
+		Usage:   "specifies the root URL of the Gitea instance, without a trailing slash.",
+		EnvVars: []string{"GITEA_ROOT"},
+		Value:   "https://codeberg.org",
+	},
+	// GiteaApiToken specifies an api token for the Gitea instance
+	&cli.StringFlag{
+		Name:    "gitea-api-token",
+		Usage:   "specifies an api token for the Gitea instance",
+		EnvVars: []string{"GITEA_API_TOKEN"},
+		Value:   "",
+	},
+	// RawDomain specifies the domain from which raw repository content shall be served in the following format:
+	// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
+	// (set to []byte(nil) to disable raw content hosting)
+	&cli.StringFlag{
+		Name:    "raw-domain",
+		Usage:   "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
+		EnvVars: []string{"RAW_DOMAIN"},
+		Value:   "raw.codeberg.page",
+	},
+	// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
+	&cli.StringFlag{
+		Name:    "raw-info-page",
+		Usage:   "will be shown (with a redirect) when trying to access $RAW_DOMAIN directly (or without owner/repo/path)",
+		EnvVars: []string{"RAW_INFO_PAGE"},
+		Value:   "https://docs.codeberg.org/codeberg-pages/raw-content/",
+	},
+
+	// Server
+	&cli.StringFlag{
+		Name:    "host",
+		Usage:   "specifies host of listening address",
+		EnvVars: []string{"HOST"},
+		Value:   "[::]",
+	},
+	&cli.StringFlag{
+		Name:    "port",
+		Usage:   "specifies port of listening address",
+		EnvVars: []string{"PORT"},
+		Value:   "443",
+	},
+	&cli.BoolFlag{
+		Name: "enable-http-server",
+		// TODO: desc
+		EnvVars: []string{"ENABLE_HTTP_SERVER"},
+	},
+	// Server Options
+	&cli.BoolFlag{
+		Name:    "enable-lfs-support",
+		Usage:   "enable lfs support, require gitea v1.17.0 as backend",
+		EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		Name:    "enable-symlink-support",
+		Usage:   "follow symlinks if enabled, require gitea v1.18.0 as backend",
+		EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+		Value:   true,
+	},
+	&cli.StringFlag{
+		Name:    "log-level",
+		Value:   "warn",
+		Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
+		EnvVars: []string{"LOG_LEVEL"},
+	},
+
+	// ACME
+	&cli.StringFlag{
+		Name:    "acme-api-endpoint",
+		EnvVars: []string{"ACME_API"},
+		Value:   "https://acme-v02.api.letsencrypt.org/directory",
+	},
+	&cli.StringFlag{
+		Name:    "acme-email",
+		EnvVars: []string{"ACME_EMAIL"},
+		Value:   "noreply@example.email",
+	},
+	&cli.BoolFlag{
+		Name: "acme-use-rate-limits",
+		// TODO: Usage
+		EnvVars: []string{"ACME_USE_RATE_LIMITS"},
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		Name: "acme-accept-terms",
+		// TODO: Usage
+		EnvVars: []string{"ACME_ACCEPT_TERMS"},
+	},
+	&cli.StringFlag{
+		Name: "acme-eab-kid",
+		// TODO: Usage
+		EnvVars: []string{"ACME_EAB_KID"},
+	},
+	&cli.StringFlag{
+		Name: "acme-eab-hmac",
+		// TODO: Usage
+		EnvVars: []string{"ACME_EAB_HMAC"},
+	},
+	&cli.StringFlag{
+		Name: "dns-provider",
+		// TODO: Usage
+		EnvVars: []string{"DNS_PROVIDER"},
+	},
+}

cmd/main.go

@@ -0,0 +1,156 @@
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+
+	"codeberg.org/codeberg/pages/server"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/certificates"
+	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
+// TODO: make it a flag
+var AllowedCorsDomains = [][]byte{
+	[]byte("fonts.codeberg.org"),
+	[]byte("design.codeberg.org"),
+}
+
+// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
+// TODO: Make it a flag too
+var BlacklistedPaths = [][]byte{
+	[]byte("/.well-known/acme-challenge/"),
+}
+
+// Serve sets up and starts the web server.
+func Serve(ctx *cli.Context) error {
+	// Initalize the logger.
+	logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
+	if err != nil {
+		return err
+	}
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
+
+	giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
+	giteaAPIToken := ctx.String("gitea-api-token")
+	rawDomain := ctx.String("raw-domain")
+	mainDomainSuffix := []byte(ctx.String("pages-domain"))
+	rawInfoPage := ctx.String("raw-info-page")
+	listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
+	enableHTTPServer := ctx.Bool("enable-http-server")
+
+	acmeAPI := ctx.String("acme-api-endpoint")
+	acmeMail := ctx.String("acme-email")
+	acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
+	acmeAcceptTerms := ctx.Bool("acme-accept-terms")
+	acmeEabKID := ctx.String("acme-eab-kid")
+	acmeEabHmac := ctx.String("acme-eab-hmac")
+	dnsProvider := ctx.String("dns-provider")
+	if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
+		return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
+	}
+
+	allowedCorsDomains := AllowedCorsDomains
+	if len(rawDomain) != 0 {
+		allowedCorsDomains = append(allowedCorsDomains, []byte(rawDomain))
+	}
+
+	// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
+	if !bytes.HasPrefix(mainDomainSuffix, []byte{'.'}) {
+		mainDomainSuffix = append([]byte{'.'}, mainDomainSuffix...)
+	}
+
+	keyCache := cache.NewKeyValueCache()
+	challengeCache := cache.NewKeyValueCache()
+	// canonicalDomainCache stores canonical domains
+	canonicalDomainCache := cache.NewKeyValueCache()
+	// dnsLookupCache stores DNS lookups for custom domains
+	dnsLookupCache := cache.NewKeyValueCache()
+	// branchTimestampCache stores branch timestamps for faster cache checking
+	branchTimestampCache := cache.NewKeyValueCache()
+	// fileResponseCache stores responses from the Gitea server
+	// TODO: make this an MRU cache with a size limit
+	fileResponseCache := cache.NewKeyValueCache()
+
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
+	if err != nil {
+		return fmt.Errorf("could not create new gitea client: %v", err)
+	}
+
+	// Create handler based on settings
+	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
+		giteaClient,
+		giteaRoot, rawInfoPage,
+		BlacklistedPaths, allowedCorsDomains,
+		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
+
+	fastServer := server.SetupServer(handler)
+	httpServer := server.SetupHTTPACMEChallengeServer(challengeCache)
+
+	// Setup listener and TLS
+	log.Info().Msgf("Listening on https://%s", listeningAddress)
+	listener, err := net.Listen("tcp", listeningAddress)
+	if err != nil {
+		return fmt.Errorf("couldn't create listener: %v", err)
+	}
+
+	// TODO: make "key-database.pogreb" set via flag
+	certDB, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
+	}
+	defer certDB.Close() //nolint:errcheck    // database has no close ... sync behave like it
+
+	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
+		giteaClient,
+		dnsProvider,
+		acmeUseRateLimits,
+		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
+		certDB))
+
+	acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
+	if err != nil {
+		return err
+	}
+
+	if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
+		return err
+	}
+
+	interval := 12 * time.Hour
+	certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
+	defer cancelCertMaintain()
+	go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
+
+	if enableHTTPServer {
+		go func() {
+			log.Info().Msg("Start HTTP server listening on :80")
+			err := httpServer.ListenAndServe("[::]:80")
+			if err != nil {
+				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
+			}
+		}()
+	}
+
+	// Start the web fastServer
+	log.Info().Msgf("Start listening on %s", listener.Addr())
+	err = fastServer.Serve(listener)
+	if err != nil {
+		log.Panic().Err(err).Msg("Couldn't start fastServer")
+	}
+
+	return nil
+}

debug-stepper/stepper.go

@@ -1,68 +0,0 @@
-package debug_stepper
-
-import (
-	"fmt"
-	"os"
-	"strings"
-	"time"
-)
-
-var Enabled = strings.HasSuffix(os.Args[0], ".test") || os.Getenv("DEBUG") == "1"
-
-var Logger = func(s string, i ...interface{}) {
-	fmt.Printf(s, i...)
-}
-
-type Stepper struct {
-	Name       string
-	Start      time.Time
-	LastStep   time.Time
-	Completion time.Time
-}
-
-func Start(name string) *Stepper {
-	if !Enabled {
-		return nil
-	}
-	t := time.Now()
-	Logger("%s: started at %s\n", name, t.Format(time.RFC3339))
-	return &Stepper{
-		Name:     name,
-		Start:    t,
-		LastStep: t,
-	}
-}
-
-func (s *Stepper) Debug(text string) {
-	if !Enabled {
-		return
-	}
-	t := time.Now()
-	Logger("%s: %s (at %s, %s since last step, %s since start)\n", s.Name, text, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
-}
-
-func (s *Stepper) Step(description string) {
-	if !Enabled {
-		return
-	}
-	if s.Completion != (time.Time{}) {
-		Logger("%s: already completed all tasks.\n")
-		return
-	}
-	t := time.Now()
-	Logger("%s: completed %s at %s (%s)\n", s.Name, description, t.Format(time.RFC3339), t.Sub(s.LastStep).String())
-	s.LastStep = t
-}
-
-func (s *Stepper) Complete() {
-	if !Enabled {
-		return
-	}
-	if s.Completion != (time.Time{}) {
-		Logger("%s: already completed all tasks.\n")
-		return
-	}
-	t := time.Now()
-	Logger("%s: completed all tasks at %s (%s since last step; total time: %s)\n", s.Name, t.Format(time.RFC3339), t.Sub(s.LastStep).String(), t.Sub(s.Start).String())
-	s.Completion = t
-}

domains.go

@@ -1,113 +0,0 @@
-package main
-
-import (
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/valyala/fasthttp"
-	"net"
-	"strings"
-	"time"
-)
-
-// DnsLookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var DnsLookupCacheTimeout = 15 * time.Minute
-
-// dnsLookupCache stores DNS lookups for custom domains
-var dnsLookupCache = mcache.New()
-
-// getTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
-// If everything is fine, it returns the target data.
-func getTargetFromDNS(domain string) (targetOwner, targetRepo, targetBranch string) {
-	// Get CNAME or TXT
-	var cname string
-	var err error
-	if cachedName, ok := dnsLookupCache.Get(domain); ok {
-		cname = cachedName.(string)
-	} else {
-		cname, err = net.LookupCNAME(domain)
-		cname = strings.TrimSuffix(cname, ".")
-		if err != nil || !strings.HasSuffix(cname, string(MainDomainSuffix)) {
-			cname = ""
-			// TODO: check if the A record matches!
-			names, err := net.LookupTXT(domain)
-			if err == nil {
-				for _, name := range names {
-					name = strings.TrimSuffix(name, ".")
-					if strings.HasSuffix(name, string(MainDomainSuffix)) {
-						cname = name
-						break
-					}
-				}
-			}
-		}
-		_ = dnsLookupCache.Set(domain, cname, DnsLookupCacheTimeout)
-	}
-	if cname == "" {
-		return
-	}
-	cnameParts := strings.Split(strings.TrimSuffix(cname, string(MainDomainSuffix)), ".")
-	targetOwner = cnameParts[len(cnameParts)-1]
-	if len(cnameParts) > 1 {
-		targetRepo = cnameParts[len(cnameParts)-2]
-	}
-	if len(cnameParts) > 2 {
-		targetBranch = cnameParts[len(cnameParts)-3]
-	}
-	if targetRepo == "" {
-		targetRepo = "pages"
-	}
-	if targetBranch == "" && targetRepo != "pages" {
-		targetBranch = "pages"
-	}
-	// if targetBranch is still empty, the caller must find the default branch
-	return
-}
-
-// CanonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
-var CanonicalDomainCacheTimeout = 15 * time.Minute
-
-// canonicalDomainCache stores canonical domains
-var canonicalDomainCache = mcache.New()
-
-// checkCanonicalDomain returns the canonical domain specified in the repo (using the file `.canonical-domain`).
-func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain string) (canonicalDomain string, valid bool) {
-	domains := []string{}
-	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
-		domains = cachedValue.([]string)
-		for _, domain := range domains {
-			if domain == actualDomain {
-				valid = true
-				break
-			}
-		}
-	} else {
-		req := fasthttp.AcquireRequest()
-		req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + GiteaApiToken)
-		res := fasthttp.AcquireResponse()
-
-		err := upstreamClient.Do(req, res)
-		if err == nil && res.StatusCode() == fasthttp.StatusOK {
-			for _, domain := range strings.Split(string(res.Body()), "\n") {
-				domain = strings.ToLower(domain)
-				domain = strings.TrimSpace(domain)
-				domain = strings.TrimPrefix(domain, "http://")
-				domain = strings.TrimPrefix(domain, "https://")
-				if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
-					domains = append(domains, domain)
-				}
-				if domain == actualDomain {
-					valid = true
-				}
-			}
-		}
-		domains = append(domains, targetOwner+string(MainDomainSuffix))
-		if domains[len(domains)-1] == actualDomain {
-			valid = true
-		}
-		if targetRepo != "" && targetRepo != "pages" {
-			domains[len(domains)-1] += "/" + targetRepo
-		}
-		_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, CanonicalDomainCacheTimeout)
-	}
-	canonicalDomain = domains[0]
-	return
-}

go.mod

@@ -1,12 +1,128 @@
 module codeberg.org/codeberg/pages
 
-go 1.16
+go 1.18
 
 require (
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
 	github.com/akrylysov/pogreb v0.10.1
 	github.com/go-acme/lego/v4 v4.5.3
+	github.com/joho/godotenv v1.4.0
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
+	github.com/rs/zerolog v1.27.0
+	github.com/stretchr/testify v1.7.0
+	github.com/urfave/cli/v2 v2.3.0
 	github.com/valyala/fasthttp v1.31.0
 	github.com/valyala/fastjson v1.6.3
 )
+
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.19 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
+	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
+	github.com/andybalholm/brotli v1.0.2 // indirect
+	github.com/aws/aws-sdk-go v1.39.0 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
+	github.com/cpu/goacmedns v0.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.6.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
+	github.com/exoscale/egoscale v0.67.0 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gophercloud/gophercloud v0.16.0 // indirect
+	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
+	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
+	github.com/jarcoal/httpmock v1.0.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.7 // indirect
+	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
+	github.com/labbsr0x/goh v1.0.1 // indirect
+	github.com/linode/linodego v0.31.1 // indirect
+	github.com/liquidweb/go-lwApi v0.0.5 // indirect
+	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
+	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
+	github.com/nrdcg/auroradns v1.0.1 // indirect
+	github.com/nrdcg/desec v0.6.0 // indirect
+	github.com/nrdcg/dnspod-go v0.4.0 // indirect
+	github.com/nrdcg/freemyip v0.2.0 // indirect
+	github.com/nrdcg/goinwx v0.8.1 // indirect
+	github.com/nrdcg/namesilo v0.2.1 // indirect
+	github.com/nrdcg/porkbun v0.1.1 // indirect
+	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
+	github.com/ovh/go-ovh v1.1.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.3.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/sacloud/libsacloud v1.36.2 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
+	github.com/softlayer/softlayer-go v1.0.3 // indirect
+	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/transip/gotransip/v6 v6.6.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
+	github.com/vultr/govultr/v2 v2.7.1 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
+	google.golang.org/api v0.20.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
+	google.golang.org/grpc v1.27.1 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go.sum

@@ -95,10 +95,12 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
 github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -148,6 +150,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
 github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b h1:/vQ+oYKu+JoyaMPDsv5FzwuL2wwWBgBbtj/YLCi4LuA=
 github.com/gobs/pretty v0.0.0-20180724170744-09732c25a95b/go.mod h1:Xo4aNUOrJnVruqWQJBtW6+bTBDTniY8yZum5rF3b5jw=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -263,6 +266,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -316,12 +321,15 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -422,6 +430,10 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad h1:WtSUHi5zthjudjI
 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe2Y+6iHTjIq/9HzUq7NII/Nffp0HkFrsAKq4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
+github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sacloud/libsacloud v1.36.2 h1:aosI7clbQ9IU0Hj+3rpk3SKJop5nLPpLThnWCivPqjI=
@@ -429,6 +441,7 @@ github.com/sacloud/libsacloud v1.36.2/go.mod h1:P7YAOVmnIn3DKHqCZcUKYUXmSwGBm3yS
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f h1:WSnaD0/cvbKJgSTYbjAPf4RJXVvNNDAwVm+W8wEmnGE=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -479,6 +492,7 @@ github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4
 github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -589,8 +603,9 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
 golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
 golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -653,8 +668,10 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

handler.go

@@ -1,524 +0,0 @@
-package main
-
-import (
-	"bytes"
-	debug_stepper "codeberg.org/codeberg/pages/debug-stepper"
-	"fmt"
-	"github.com/OrlovEvgeny/go-mcache"
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-	"io"
-	"mime"
-	"path"
-	"strconv"
-	"strings"
-	"time"
-)
-
-// handler handles a single HTTP request to the web server.
-func handler(ctx *fasthttp.RequestCtx) {
-	s := debug_stepper.Start("handler")
-	defer s.Complete()
-
-	ctx.Response.Header.Set("Server", "Codeberg Pages")
-
-	// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
-	ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
-
-	// Enable caching, but require revalidation to reduce confusion
-	ctx.Response.Header.Set("Cache-Control", "must-revalidate")
-
-	trimmedHost := TrimHostPort(ctx.Request.Host())
-
-	// Add HSTS for RawDomain and MainDomainSuffix
-	if hsts := GetHSTSHeader(trimmedHost); hsts != "" {
-		ctx.Response.Header.Set("Strict-Transport-Security", hsts)
-	}
-
-	// Block all methods not required for static pages
-	if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
-		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
-		ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
-		return
-	}
-
-	// Block blacklisted paths (like ACME challenges)
-	for _, blacklistedPath := range BlacklistedPaths {
-		if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
-			returnErrorPage(ctx, fasthttp.StatusForbidden)
-			return
-		}
-	}
-
-	// Allow CORS for specified domains
-	if ctx.IsOptions() {
-		allowCors := false
-		for _, allowedCorsDomain := range AllowedCorsDomains {
-			if bytes.Equal(trimmedHost, allowedCorsDomain) {
-				allowCors = true
-				break
-			}
-		}
-		if allowCors {
-			ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
-			ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
-		}
-		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
-		ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
-		return
-	}
-
-	// Prepare request information to Gitea
-	var targetOwner, targetRepo, targetBranch, targetPath string
-	var targetOptions = &upstreamOptions{
-		ForbiddenMimeTypes: map[string]struct{}{},
-		TryIndexPages:      true,
-	}
-
-	// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
-	// also disallow search indexing and add a Link header to the canonical URL.
-	var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
-		if repo == "" {
-			return false
-		}
-
-		// Check if the branch exists, otherwise treat it as a file path
-		branchTimestampResult := getBranchTimestamp(targetOwner, repo, branch)
-		if branchTimestampResult == nil {
-			// branch doesn't exist
-			return false
-		}
-
-		// Branch exists, use it
-		targetRepo = repo
-		targetPath = strings.Trim(strings.Join(path, "/"), "/")
-		targetBranch = branchTimestampResult.branch
-
-		targetOptions.BranchTimestamp = branchTimestampResult.timestamp
-
-		if canonicalLink != "" {
-			// Hide from search machines & add canonical link
-			ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
-			ctx.Response.Header.Set("Link",
-				strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
-					"; rel=\"canonical\"",
-			)
-		}
-
-		return true
-	}
-
-	// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-	var tryUpstream = func() {
-		// check if a canonical domain exists on a request on MainDomain
-		if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
-			canonicalDomain, _ := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, "")
-			if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(MainDomainSuffix)) {
-				canonicalPath := string(ctx.RequestURI())
-				if targetRepo != "pages" {
-					canonicalPath = "/" + strings.SplitN(canonicalPath, "/", 3)[2]
-				}
-				ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
-				return
-			}
-		}
-
-		// Try to request the file from the Gitea API
-		if !upstream(ctx, targetOwner, targetRepo, targetBranch, targetPath, targetOptions) {
-			returnErrorPage(ctx, ctx.Response.StatusCode())
-		}
-	}
-
-	s.Step("preparations")
-
-	if RawDomain != nil && bytes.Equal(trimmedHost, RawDomain) {
-		// Serve raw content from RawDomain
-		s.Debug("raw domain")
-
-		targetOptions.TryIndexPages = false
-		targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
-		targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
-
-		pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-		if len(pathElements) < 2 {
-			// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
-			ctx.Redirect(RawInfoPage, fasthttp.StatusTemporaryRedirect)
-			return
-		}
-		targetOwner = pathElements[0]
-		targetRepo = pathElements[1]
-
-		// raw.codeberg.org/example/myrepo/@main/index.html
-		if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
-			s.Step("raw domain preparations, now trying with specified branch")
-			if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
-				string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
-			) {
-				s.Step("tryBranch, now trying upstream")
-				tryUpstream()
-				return
-			}
-			s.Debug("missing branch")
-			returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return
-		} else {
-			s.Step("raw domain preparations, now trying with default branch")
-			tryBranch(targetRepo, "", pathElements[2:],
-				string(GiteaRoot)+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
-			)
-			s.Step("tryBranch, now trying upstream")
-			tryUpstream()
-			return
-		}
-
-	} else if bytes.HasSuffix(trimmedHost, MainDomainSuffix) {
-		// Serve pages from subdomains of MainDomainSuffix
-		s.Debug("main domain suffix")
-
-		pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-		targetOwner = string(bytes.TrimSuffix(trimmedHost, MainDomainSuffix))
-		targetRepo = pathElements[0]
-		targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
-
-		if targetOwner == "www" {
-			// www.codeberg.page redirects to codeberg.page
-			ctx.Redirect("https://" + string(MainDomainSuffix[1:]) + string(ctx.Path()), fasthttp.StatusPermanentRedirect)
-			return
-		}
-
-		// Check if the first directory is a repo with the second directory as a branch
-		// example.codeberg.page/myrepo/@main/index.html
-		if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
-			if targetRepo == "pages" {
-				// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
-				ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
-				return
-			}
-
-			s.Step("main domain preparations, now trying with specified repo & branch")
-			if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
-				"/"+pathElements[0]+"/%p",
-			) {
-				s.Step("tryBranch, now trying upstream")
-				tryUpstream()
-			} else {
-				returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			}
-			return
-		}
-
-		// Check if the first directory is a branch for the "pages" repo
-		// example.codeberg.page/@main/index.html
-		if strings.HasPrefix(pathElements[0], "@") {
-			s.Step("main domain preparations, now trying with specified branch")
-			if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
-				s.Step("tryBranch, now trying upstream")
-				tryUpstream()
-			} else {
-				returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			}
-			return
-		}
-
-		// Check if the first directory is a repo with a "pages" branch
-		// example.codeberg.page/myrepo/index.html
-		// example.codeberg.page/pages/... is not allowed here.
-		s.Step("main domain preparations, now trying with specified repo")
-		if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
-			s.Step("tryBranch, now trying upstream")
-			tryUpstream()
-			return
-		}
-
-		// Try to use the "pages" repo on its default branch
-		// example.codeberg.page/index.html
-		s.Step("main domain preparations, now trying with default repo/branch")
-		if tryBranch("pages", "", pathElements, "") {
-			s.Step("tryBranch, now trying upstream")
-			tryUpstream()
-			return
-		}
-
-		// Couldn't find a valid repo/branch
-		returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-		return
-	} else {
-		trimmedHostStr := string(trimmedHost)
-
-		// Serve pages from external domains
-		targetOwner, targetRepo, targetBranch = getTargetFromDNS(trimmedHostStr)
-		if targetOwner == "" {
-			returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return
-		}
-
-		pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
-		canonicalLink := ""
-		if strings.HasPrefix(pathElements[0], "@") {
-			targetBranch = pathElements[0][1:]
-			pathElements = pathElements[1:]
-			canonicalLink = "/%p"
-		}
-
-		// Try to use the given repo on the given branch or the default branch
-		s.Step("custom domain preparations, now trying with details from DNS")
-		if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
-			canonicalDomain, valid := checkCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr)
-			if !valid {
-				returnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
-				return
-			} else if canonicalDomain != trimmedHostStr {
-				// only redirect if the target is also a codeberg page!
-				targetOwner, _, _ = getTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0])
-				if targetOwner != "" {
-					ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
-					return
-				} else {
-					returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-					return
-				}
-			}
-
-			s.Step("tryBranch, now trying upstream")
-			tryUpstream()
-			return
-		} else {
-			returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return
-		}
-	}
-}
-
-// returnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
-// with the provided status code.
-func returnErrorPage(ctx *fasthttp.RequestCtx, code int) {
-	ctx.Response.SetStatusCode(code)
-	ctx.Response.Header.SetContentType("text/html; charset=utf-8")
-	message := fasthttp.StatusMessage(code)
-	if code == fasthttp.StatusMisdirectedRequest {
-		message += " - domain not specified in <code>.domains</code> file"
-	}
-	if code == fasthttp.StatusFailedDependency {
-		message += " - target repo/branch doesn't exist or is private"
-	}
-	ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
-}
-
-// DefaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
-var DefaultBranchCacheTimeout = 15 * time.Minute
-
-// BranchExistanceCacheTimeout specifies the timeout for the branch timestamp & existance cache. It should be shorter
-// than FileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
-// picked up faster, while still allowing the content to be cached longer if nothing changes.
-var BranchExistanceCacheTimeout = 5 * time.Minute
-
-// branchTimestampCache stores branch timestamps for faster cache checking
-var branchTimestampCache = mcache.New()
-
-type branchTimestamp struct {
-	branch    string
-	timestamp time.Time
-}
-
-// FileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
-// on your available memory.
-var FileCacheTimeout = 5 * time.Minute
-
-// FileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-var FileCacheSizeLimit = 1024 * 1024
-
-// fileResponseCache stores responses from the Gitea server
-// TODO: make this an MRU cache with a size limit
-var fileResponseCache = mcache.New()
-
-type fileResponse struct {
-	exists   bool
-	mimeType string
-	body     []byte
-}
-
-// getBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
-// (or nil if the branch doesn't exist)
-func getBranchTimestamp(owner, repo, branch string) *branchTimestamp {
-	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
-		if result == nil {
-			return nil
-		}
-		return result.(*branchTimestamp)
-	}
-	result := &branchTimestamp{}
-	result.branch = branch
-	if branch == "" {
-		// Get default branch
-		var body = make([]byte, 0)
-		status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+GiteaApiToken, 5*time.Second)
-		if err != nil || status != 200 {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, DefaultBranchCacheTimeout)
-			return nil
-		}
-		result.branch = fastjson.GetString(body, "default_branch")
-	}
-
-	var body = make([]byte, 0)
-	status, body, err := fasthttp.GetTimeout(body, string(GiteaRoot)+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+GiteaApiToken, 5*time.Second)
-	if err != nil || status != 200 {
-		return nil
-	}
-
-	result.timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
-	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, BranchExistanceCacheTimeout)
-	return result
-}
-
-var upstreamClient = fasthttp.Client{
-	ReadTimeout:        10 * time.Second,
-	MaxConnDuration:    60 * time.Second,
-	MaxConnWaitTimeout: 1000 * time.Millisecond,
-	MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-}
-
-// upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func upstream(ctx *fasthttp.RequestCtx, targetOwner string, targetRepo string, targetBranch string, targetPath string, options *upstreamOptions) (final bool) {
-	s := debug_stepper.Start("upstream")
-	defer s.Complete()
-
-	if options.ForbiddenMimeTypes == nil {
-		options.ForbiddenMimeTypes = map[string]struct{}{}
-	}
-
-	// Check if the branch exists and when it was modified
-	if options.BranchTimestamp == (time.Time{}) {
-		branch := getBranchTimestamp(targetOwner, targetRepo, targetBranch)
-
-		if branch == nil {
-			returnErrorPage(ctx, fasthttp.StatusFailedDependency)
-			return true
-		}
-		targetBranch = branch.branch
-		options.BranchTimestamp = branch.timestamp
-	}
-
-	if targetOwner == "" || targetRepo == "" || targetBranch == "" {
-		returnErrorPage(ctx, fasthttp.StatusBadRequest)
-		return true
-	}
-
-	// Check if the browser has a cached version
-	if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
-		if !ifModifiedSince.Before(options.BranchTimestamp) {
-			ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
-			return true
-		}
-	}
-	s.Step("preparations")
-
-	// Make a GET request to the upstream URL
-	uri := targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/" + targetPath
-	var req *fasthttp.Request
-	var res *fasthttp.Response
-	var cachedResponse fileResponse
-	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(options.BranchTimestamp.Unix(), 10)); ok {
-		cachedResponse = cachedValue.(fileResponse)
-	} else {
-		req = fasthttp.AcquireRequest()
-		req.SetRequestURI(string(GiteaRoot) + "/api/v1/repos/" + uri + "?access_token=" + GiteaApiToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = upstreamClient.Do(req, res)
-	}
-	s.Step("acquisition")
-
-	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
-		if options.TryIndexPages {
-			// copy the options struct & try if an index page exists
-			optionsForIndexPages := *options
-			optionsForIndexPages.TryIndexPages = false
-			optionsForIndexPages.AppendTrailingSlash = true
-			for _, indexPage := range IndexPages {
-				if upstream(ctx, targetOwner, targetRepo, targetBranch, strings.TrimSuffix(targetPath, "/")+"/"+indexPage, &optionsForIndexPages) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
-						exists: false,
-					}, FileCacheTimeout)
-					return true
-				}
-			}
-		}
-		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
-		if res != nil {
-			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), fileResponse{
-				exists: false,
-			}, FileCacheTimeout)
-		}
-		return false
-	}
-	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
-		returnErrorPage(ctx, fasthttp.StatusInternalServerError)
-		return true
-	}
-
-	// Append trailing slash if missing (for index files)
-	// options.AppendTrailingSlash is only true when looking for index pages
-	if options.AppendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
-		ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
-		return true
-	}
-	s.Step("error handling")
-
-	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(targetPath))
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := options.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if options.DefaultMimeType != "" {
-			mimeType = options.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
-	ctx.Response.Header.SetContentType(mimeType)
-
-	// Everything's okay so far
-	ctx.Response.SetStatusCode(fasthttp.StatusOK)
-	ctx.Response.Header.SetLastModified(options.BranchTimestamp)
-
-	s.Step("response preparations")
-
-	// Write the response body to the original request
-	var cacheBodyWriter bytes.Buffer
-	if res != nil {
-		if res.Header.ContentLength() > FileCacheSizeLimit {
-			err = res.BodyWriteTo(ctx.Response.BodyWriter())
-		} else {
-			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
-		}
-	} else {
-		_, err = ctx.Write(cachedResponse.body)
-	}
-	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
-		returnErrorPage(ctx, fasthttp.StatusInternalServerError)
-		return true
-	}
-	s.Step("response")
-
-	if res != nil {
-		cachedResponse.exists = true
-		cachedResponse.mimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(options.BranchTimestamp.Unix(), 10), cachedResponse, FileCacheTimeout)
-	}
-
-	return true
-}
-
-// upstreamOptions provides various options for the upstream request.
-type upstreamOptions struct {
-	DefaultMimeType     string
-	ForbiddenMimeTypes  map[string]struct{}
-	TryIndexPages       bool
-	AppendTrailingSlash bool
-	BranchTimestamp     time.Time
-}

handler_test.go

@@ -1,53 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/valyala/fasthttp"
-	"testing"
-	"time"
-)
-
-func TestHandlerPerformance(t *testing.T) {
-	ctx := &fasthttp.RequestCtx{
-		Request:  *fasthttp.AcquireRequest(),
-		Response: *fasthttp.AcquireResponse(),
-	}
-	ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
-	fmt.Printf("Start: %v\n", time.Now())
-	start := time.Now()
-	handler(ctx)
-	end := time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
-
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	handler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
-
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	ctx.Request.SetRequestURI("http://example.momar.xyz/")
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	handler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
-}

haproxy-sni/haproxy.cfg

@@ -51,6 +51,7 @@ frontend https_sni_frontend
   ###################################################
   acl use_http_backend req.ssl_sni -i "codeberg.org"
   acl use_http_backend req.ssl_sni -i "join.codeberg.org"
+  # TODO: use this if no SNI exists
   use_backend https_termination_backend if use_http_backend
 
   ############################

helpers.go

@@ -1,56 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"encoding/gob"
-	"github.com/akrylysov/pogreb"
-)
-
-// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
-// string for custom domains.
-func GetHSTSHeader(host []byte) string {
-	if bytes.HasSuffix(host, MainDomainSuffix) || bytes.Equal(host, RawDomain) {
-		return "max-age=63072000; includeSubdomains; preload"
-	} else {
-		return ""
-	}
-}
-
-func TrimHostPort(host []byte) []byte {
-	i := bytes.IndexByte(host, ':')
-	if i >= 0 {
-		return host[:i]
-	}
-	return host
-}
-
-func PogrebPut(db *pogreb.DB, name []byte, obj interface{}) {
-	var resGob bytes.Buffer
-	resEnc := gob.NewEncoder(&resGob)
-	err := resEnc.Encode(obj)
-	if err != nil {
-		panic(err)
-	}
-	err = db.Put(name, resGob.Bytes())
-	if err != nil {
-		panic(err)
-	}
-}
-
-func PogrebGet(db *pogreb.DB, name []byte, obj interface{}) bool {
-	resBytes, err := db.Get(name)
-	if err != nil {
-		panic(err)
-	}
-	if resBytes == nil {
-		return false
-	}
-
-	resGob := bytes.NewBuffer(resBytes)
-	resDec := gob.NewDecoder(resGob)
-	err = resDec.Decode(obj)
-	if err != nil {
-		panic(err)
-	}
-	return true
-}

html/404.html

@@ -21,12 +21,13 @@
     </style>
 </head>
 <body>
-    <i class="fa fa-bug text-primary" style="font-size: 96px;"></i>
+    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
     <h1 class="mb-0 text-primary">
-    	You found a bug!
+		Page not found!
     </h1>
     <h5 class="text-center" style="max-width: 25em;">
-    	Sorry, this page doesn't exist or is inaccessible for other reasons (%status)
+    	Sorry, but this page couldn't be found or is inaccessible (%status).<br/>
+		We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
     </h5>
     <small class="text-muted">
         <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">

html/error.go

@@ -0,0 +1,24 @@
+package html
+
+import (
+	"bytes"
+	"strconv"
+
+	"github.com/valyala/fasthttp"
+)
+
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
+// with the provided status code.
+func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
+	ctx.Response.SetStatusCode(code)
+	ctx.Response.Header.SetContentType("text/html; charset=utf-8")
+	message := fasthttp.StatusMessage(code)
+	if code == fasthttp.StatusMisdirectedRequest {
+		message += " - domain not specified in <code>.domains</code> file"
+	}
+	if code == fasthttp.StatusFailedDependency {
+		message += " - target repo/branch doesn't exist or is private"
+	}
+	// TODO: use template engine?
+	ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
+}

html/html.go

@@ -0,0 +1,6 @@
+package html
+
+import _ "embed"
+
+//go:embed 404.html
+var NotFoundPage []byte

integration/get_test.go

@@ -0,0 +1,143 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetRedirect(t *testing.T) {
+	log.Println("=== TestGetRedirect ===")
+	// test custom domain redirect
+	resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
+	assert.EqualValues(t, 0, getSize(resp.Body))
+}
+
+func TestGetContent(t *testing.T) {
+	log.Println("=== TestGetContent ===")
+	// test get image
+	resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "image/jpeg", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "124635", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 124635, getSize(resp.Body))
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// specify branch
+	resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 1000)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// access branch name contains '/'
+	resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 100)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
+}
+
+func TestCustomDomain(t *testing.T) {
+	log.Println("=== TestCustomDomain ===")
+	resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 106, getSize(resp.Body))
+}
+
+func TestGetNotFound(t *testing.T) {
+	log.Println("=== TestGetNotFound ===")
+	// test custom not found pages
+	resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 37, getSize(resp.Body))
+}
+
+func TestFollowSymlink(t *testing.T) {
+	log.Printf("=== TestFollowSymlink ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+	body := getBytes(resp.Body)
+	assert.EqualValues(t, 4, len(body))
+	assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+	log.Printf("=== TestLFSSupport ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	body := strings.TrimSpace(string(getBytes(resp.Body)))
+	assert.EqualValues(t, 12, len(body))
+	assert.EqualValues(t, "actual value", body)
+}
+
+func getTestHTTPSClient() *http.Client {
+	cookieJar, _ := cookiejar.New(nil)
+	return &http.Client{
+		Jar: cookieJar,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+}
+
+func getBytes(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Bytes()
+}
+
+func getSize(stream io.Reader) int {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Len()
+}

integration/main_test.go

@@ -0,0 +1,62 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"context"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"codeberg.org/codeberg/pages/cmd"
+
+	"github.com/urfave/cli/v2"
+)
+
+func TestMain(m *testing.M) {
+	log.Println("=== TestMain: START Server ===")
+	serverCtx, serverCancel := context.WithCancel(context.Background())
+	if err := startServer(serverCtx); err != nil {
+		log.Fatalf("could not start server: %v", err)
+	}
+	defer func() {
+		serverCancel()
+		log.Println("=== TestMain: Server STOPED ===")
+	}()
+
+	time.Sleep(10 * time.Second)
+
+	os.Exit(m.Run())
+}
+
+func startServer(ctx context.Context) error {
+	args := []string{
+		"--verbose",
+		"--acme-accept-terms", "true",
+	}
+	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
+	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
+	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PORT", "4430")
+
+	app := cli.NewApp()
+	app.Name = "pages-server"
+	app.Action = cmd.Serve
+	app.Flags = cmd.ServeFlags
+
+	go func() {
+		if err := app.RunContext(ctx, args); err != nil {
+			log.Fatalf("run server error: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+func setEnvIfNotSet(key, value string) {
+	if _, set := os.LookupEnv(key); !set {
+		os.Setenv(key, value)
+	}
+}

main.go

@@ -1,159 +1,31 @@
-// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
-//
-// Mapping custom domains is not static anymore, but can be done with DNS:
-//
-// 1) add a "domains.txt" text file to your repository, containing the allowed domains, separated by new lines. The
-// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
-//
-// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
-// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
-//      www.example.org. IN CNAME main.pages.example.codeberg.page.
-//
-// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
-// for "example.org" (if your provider allows ALIAS or similar records):
-//      example.org IN ALIAS codeberg.page.
-//
-// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
 package main
 
 import (
-	"bytes"
-	"crypto/tls"
 	"fmt"
-	"log"
-	"net"
-	"net/http"
 	"os"
-	"time"
 
-	_ "embed"
+	_ "github.com/joho/godotenv/autoload"
+	"github.com/urfave/cli/v2"
 
-	"github.com/valyala/fasthttp"
+	"codeberg.org/codeberg/pages/cmd"
 )
 
-// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
+// can be changed with -X on compile
-// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
+var version = "dev"
-// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
-var MainDomainSuffix = []byte("." + envOr("PAGES_DOMAIN", "codeberg.page"))
 
-// GiteaRoot specifies the root URL of the Gitea instance, without a trailing slash.
-var GiteaRoot = []byte(envOr("GITEA_ROOT", "https://codeberg.org"))
-
-var GiteaApiToken = envOr("GITEA_API_TOKEN", "")
-
-//go:embed 404.html
-var NotFoundPage []byte
-
-// RawDomain specifies the domain from which raw repository content shall be served in the following format:
-// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
-// (set to []byte(nil) to disable raw content hosting)
-var RawDomain = []byte(envOr("RAW_DOMAIN", "raw.codeberg.org"))
-
-// RawInfoPage will be shown (with a redirect) when trying to access RawDomain directly (or without owner/repo/path).
-var RawInfoPage = envOr("REDIRECT_RAW_INFO", "https://docs.codeberg.org/pages/raw-content/")
-
-// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
-var AllowedCorsDomains = [][]byte{
-	RawDomain,
-	[]byte("fonts.codeberg.org"),
-	[]byte("design.codeberg.org"),
-}
-
-// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
-var BlacklistedPaths = [][]byte{
-	[]byte("/.well-known/acme-challenge/"),
-}
-
-// IndexPages lists pages that may be considered as index pages for directories.
-var IndexPages = []string{
-	"index.html",
-}
-
-// main sets up and starts the web server.
 func main() {
-	if len(os.Args) > 1 && os.Args[1] == "--remove-certificate" {
+	app := cli.NewApp()
-		if len(os.Args) < 2 {
+	app.Name = "pages-server"
-			println("--remove-certificate requires at least one domain as an argument")
+	app.Version = version
-			os.Exit(1)
+	app.Usage = "pages server"
-		}
+	app.Action = cmd.Serve
-		if keyDatabaseErr != nil {
+	app.Flags = cmd.ServeFlags
-			panic(keyDatabaseErr)
+	app.Commands = []*cli.Command{
-		}
+		cmd.Certs,
-		for _, domain := range os.Args[2:] {
-			if err := keyDatabase.Delete([]byte(domain)); err != nil {
-				panic(err)
-			}
-		}
-		if err := keyDatabase.Sync(); err != nil {
-			panic(err)
-		}
-		os.Exit(0)
 	}
 
-	// Make sure MainDomain has a trailing dot, and GiteaRoot has no trailing slash
+	if err := app.Run(os.Args); err != nil {
-	if !bytes.HasPrefix(MainDomainSuffix, []byte{'.'}) {
+		_, _ = fmt.Fprintln(os.Stderr, err)
-		MainDomainSuffix = append([]byte{'.'}, MainDomainSuffix...)
+		os.Exit(1)
-	}
-	GiteaRoot = bytes.TrimSuffix(GiteaRoot, []byte{'/'})
-
-	// Use HOST and PORT environment variables to determine listening address
-	address := fmt.Sprintf("%s:%s", envOr("HOST", "[::]"), envOr("PORT", "443"))
-	log.Printf("Listening on https://%s", address)
-
-	// Enable compression by wrapping the handler() method with the compression function provided by FastHTTP
-	compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
-
-	server := &fasthttp.Server{
-		Handler:                      compressedHandler,
-		DisablePreParseMultipartForm: false,
-		MaxRequestBodySize:           0,
-		NoDefaultServerHeader:        true,
-		NoDefaultDate:                true,
-		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
-		Concurrency:                  1024 * 32,        // TODO: adjust bottlenecks for best performance with Gitea!
-		MaxConnsPerIP:                100,
-	}
-
-	// Setup listener and TLS
-	listener, err := net.Listen("tcp", address)
-	if err != nil {
-		log.Fatalf("Couldn't create listener: %s", err)
-	}
-	listener = tls.NewListener(listener, tlsConfig)
-
-	setupCertificates()
-	if os.Getenv("ENABLE_HTTP_SERVER") == "true" {
-		go (func() {
-			challengePath := []byte("/.well-known/acme-challenge/")
-			err := fasthttp.ListenAndServe("[::]:80", func(ctx *fasthttp.RequestCtx) {
-				if bytes.HasPrefix(ctx.Path(), challengePath) {
-					challenge, ok := challengeCache.Get(string(TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
-					if !ok || challenge == nil {
-						ctx.SetStatusCode(http.StatusNotFound)
-						ctx.SetBodyString("no challenge for this token")
-					}
-					ctx.SetBodyString(challenge.(string))
-				} else {
-					ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
-				}
-			})
-			if err != nil {
-				log.Fatalf("Couldn't start HTTP server: %s", err)
-			}
-		})()
-	}
-
-	// Start the web server
-	err = server.Serve(listener)
-	if err != nil {
-		log.Fatalf("Couldn't start server: %s", err)
 	}
 }
-
-// envOr reads an environment variable and returns a default value if it's empty.
-func envOr(env string, or string) string {
-	if v := os.Getenv(env); v != "" {
-		return v
-	}
-	return or
-}

server/cache/interface.go

@@ -0,0 +1,9 @@
+package cache
+
+import "time"
+
+type SetGetKey interface {
+	Set(key string, value interface{}, ttl time.Duration) error
+	Get(key string) (interface{}, bool)
+	Remove(key string)
+}

server/cache/setup.go

@@ -0,0 +1,7 @@
+package cache
+
+import "github.com/OrlovEvgeny/go-mcache"
+
+func NewKeyValueCache() SetGetKey {
+	return mcache.New()
+}

server/certificates/acme_account.go

@@ -0,0 +1,29 @@
+package certificates
+
+import (
+	"crypto"
+
+	"github.com/go-acme/lego/v4/registration"
+)
+
+type AcmeAccount struct {
+	Email        string
+	Registration *registration.Resource
+	Key          crypto.PrivateKey `json:"-"`
+	KeyPEM       string            `json:"Key"`
+}
+
+// make sure AcmeAccount match User interface
+var _ registration.User = &AcmeAccount{}
+
+func (u *AcmeAccount) GetEmail() string {
+	return u.Email
+}
+
+func (u AcmeAccount) GetRegistration() *registration.Resource {
+	return u.Registration
+}
+
+func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
+	return u.Key
+}

server/certificates/certificates.go

@@ -0,0 +1,528 @@
+package certificates
+
+import (
+	"bytes"
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/go-acme/lego/v4/certcrypto"
+	"github.com/go-acme/lego/v4/certificate"
+	"github.com/go-acme/lego/v4/challenge"
+	"github.com/go-acme/lego/v4/challenge/tlsalpn01"
+	"github.com/go-acme/lego/v4/lego"
+	"github.com/go-acme/lego/v4/providers/dns"
+	"github.com/go-acme/lego/v4/registration"
+	"github.com/reugn/equalizer"
+	"github.com/rs/zerolog/log"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/database"
+	dnsutils "codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+)
+
+// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
+func TLSConfig(mainDomainSuffix []byte,
+	giteaClient *gitea.Client,
+	dnsProvider string,
+	acmeUseRateLimits bool,
+	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+	certDB database.CertDB,
+) *tls.Config {
+	return &tls.Config{
+		// check DNS name & get certificate from Let's Encrypt
+		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			sni := strings.ToLower(strings.TrimSpace(info.ServerName))
+			sniBytes := []byte(sni)
+			if len(sni) < 1 {
+				return nil, errors.New("missing sni")
+			}
+
+			if info.SupportedProtos != nil {
+				for _, proto := range info.SupportedProtos {
+					if proto == tlsalpn01.ACMETLS1Protocol {
+						challenge, ok := challengeCache.Get(sni)
+						if !ok {
+							return nil, errors.New("no challenge for this domain")
+						}
+						cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
+						if err != nil {
+							return nil, err
+						}
+						return cert, nil
+					}
+				}
+			}
+
+			targetOwner := ""
+			if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
+				// deliver default certificate for the main domain (*.codeberg.page)
+				sniBytes = mainDomainSuffix
+				sni = string(sniBytes)
+			} else {
+				var targetRepo, targetBranch string
+				targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
+				if targetOwner == "" {
+					// DNS not set up, return main certificate to redirect to the docs
+					sniBytes = mainDomainSuffix
+					sni = string(sniBytes)
+				} else {
+					_, _ = targetRepo, targetBranch
+					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
+					if !valid {
+						sniBytes = mainDomainSuffix
+						sni = string(sniBytes)
+					}
+				}
+			}
+
+			if tlsCertificate, ok := keyCache.Get(sni); ok {
+				// we can use an existing certificate object
+				return tlsCertificate.(*tls.Certificate), nil
+			}
+
+			var tlsCertificate tls.Certificate
+			var err error
+			var ok bool
+			if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
+				// request a new certificate
+				if bytes.Equal(sniBytes, mainDomainSuffix) {
+					return nil, errors.New("won't request certificate for main domain, something really bad has happened")
+				}
+
+				tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			if err := keyCache.Set(sni, &tlsCertificate, 15*time.Minute); err != nil {
+				return nil, err
+			}
+			return &tlsCertificate, nil
+		},
+		PreferServerCipherSuites: true,
+		NextProtos: []string{
+			"http/1.1",
+			tlsalpn01.ACMETLS1Protocol,
+		},
+
+		// generated 2021-07-13, Mozilla Guideline v5.6, Go 1.14.4, intermediate configuration
+		// https://ssl-config.mozilla.org/#server=go&version=1.14.4&config=intermediate&guideline=5.6
+		MinVersion: tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+			tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,
+			tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,
+		},
+	}
+}
+
+func checkUserLimit(user string) error {
+	userLimit, ok := acmeClientCertificateLimitPerUser[user]
+	if !ok {
+		// Each Codeberg user can only add 10 new domains per day.
+		userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
+		acmeClientCertificateLimitPerUser[user] = userLimit
+	}
+	if !userLimit.Ask() {
+		return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
+	}
+	return nil
+}
+
+var (
+	acmeClient, mainDomainAcmeClient  *lego.Client
+	acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+)
+
+// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
+// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
+var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
+
+// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
+var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
+
+type AcmeTLSChallengeProvider struct {
+	challengeCache cache.SetGetKey
+}
+
+// make sure AcmeTLSChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeTLSChallengeProvider{}
+
+func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
+	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
+	a.challengeCache.Remove(domain)
+	return nil
+}
+
+type AcmeHTTPChallengeProvider struct {
+	challengeCache cache.SetGetKey
+}
+
+// make sure AcmeHTTPChallengeProvider match Provider interface
+var _ challenge.Provider = AcmeHTTPChallengeProvider{}
+
+func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
+	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
+}
+
+func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
+	a.challengeCache.Remove(domain + "/" + token)
+	return nil
+}
+
+func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
+	// parse certificate from database
+	res, err := certDB.Get(string(sni))
+	if err != nil {
+		panic(err) // TODO: no panic
+	}
+	if res == nil {
+		return tls.Certificate{}, false
+	}
+
+	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+	if err != nil {
+		panic(err)
+	}
+
+	// TODO: document & put into own function
+	if !bytes.Equal(sni, mainDomainSuffix) {
+		tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
+		if err != nil {
+			panic(err)
+		}
+
+		// renew certificates 7 days before they expire
+		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(7 * 24 * time.Hour)) {
+			// TODO: add ValidUntil to custom res struct
+			if res.CSR != nil && len(res.CSR) > 0 {
+				// CSR stores the time when the renewal shall be tried again
+				nextTryUnix, err := strconv.ParseInt(string(res.CSR), 10, 64)
+				if err == nil && time.Now().Before(time.Unix(nextTryUnix, 0)) {
+					return tlsCertificate, true
+				}
+			}
+			go (func() {
+				res.CSR = nil // acme client doesn't like CSR to be set
+				tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+				if err != nil {
+					log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err)
+				}
+			})()
+		}
+	}
+
+	return tlsCertificate, true
+}
+
+var obtainLocks = sync.Map{}
+
+func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
+	name := strings.TrimPrefix(domains[0], "*")
+	if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
+		domains = domains[1:]
+	}
+
+	// lock to avoid simultaneous requests
+	_, working := obtainLocks.LoadOrStore(name, struct{}{})
+	if working {
+		for working {
+			time.Sleep(100 * time.Millisecond)
+			_, working = obtainLocks.Load(name)
+		}
+		cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+		if !ok {
+			return tls.Certificate{}, errors.New("certificate failed in synchronous request")
+		}
+		return cert, nil
+	}
+	defer obtainLocks.Delete(name)
+
+	if acmeClient == nil {
+		return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", string(mainDomainSuffix), keyDatabase), nil
+	}
+
+	// request actual cert
+	var res *certificate.Resource
+	var err error
+	if renew != nil && renew.CertURL != "" {
+		if acmeUseRateLimits {
+			acmeClientRequestLimit.Take()
+		}
+		log.Debug().Msgf("Renewing certificate for: %v", domains)
+		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
+		if err != nil {
+			log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
+			res = nil
+		}
+	}
+	if res == nil {
+		if user != "" {
+			if err := checkUserLimit(user); err != nil {
+				return tls.Certificate{}, err
+			}
+		}
+
+		if acmeUseRateLimits {
+			acmeClientOrderLimit.Take()
+			acmeClientRequestLimit.Take()
+		}
+		log.Debug().Msgf("Re-requesting new certificate for %v", domains)
+		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
+			Domains:    domains,
+			Bundle:     true,
+			MustStaple: false,
+		})
+	}
+	if err != nil {
+		log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
+		if renew != nil && renew.CertURL != "" {
+			tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
+			if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
+				// avoid sending a mock cert instead of a still valid cert, instead abuse CSR field to store time to try again at
+				renew.CSR = []byte(strconv.FormatInt(time.Now().Add(6*time.Hour).Unix(), 10))
+				if err := keyDatabase.Put(name, renew); err != nil {
+					return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+				}
+				return tlsCertificate, nil
+			}
+		}
+		return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
+	}
+	log.Debug().Msgf("Obtained certificate for %v", domains)
+
+	if err := keyDatabase.Put(name, res); err != nil {
+		return tls.Certificate{}, err
+	}
+	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+	return tlsCertificate, nil
+}
+
+func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
+	const configFile = "acme-account.json"
+	var myAcmeAccount AcmeAccount
+	var myAcmeConfig *lego.Config
+
+	if account, err := os.ReadFile(configFile); err == nil {
+		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
+			return nil, err
+		}
+		myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
+		if err != nil {
+			return nil, err
+		}
+		myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+		myAcmeConfig.CADirURL = acmeAPI
+		myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+
+		// Validate Config
+		_, err := lego.NewClient(myAcmeConfig)
+		if err != nil {
+			// TODO: should we fail hard instead?
+			log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+		}
+		return myAcmeConfig, nil
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	}
+
+	privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return nil, err
+	}
+	myAcmeAccount = AcmeAccount{
+		Email:  acmeMail,
+		Key:    privateKey,
+		KeyPEM: string(certcrypto.PEMEncode(privateKey)),
+	}
+	myAcmeConfig = lego.NewConfig(&myAcmeAccount)
+	myAcmeConfig.CADirURL = acmeAPI
+	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
+	tempClient, err := lego.NewClient(myAcmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		// accept terms & log in to EAB
+		if acmeEabKID == "" || acmeEabHmac == "" {
+			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		} else {
+			reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
+				TermsOfServiceAgreed: acmeAcceptTerms,
+				Kid:                  acmeEabKID,
+				HmacEncoded:          acmeEabHmac,
+			})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
+			} else {
+				myAcmeAccount.Registration = reg
+			}
+		}
+
+		if myAcmeAccount.Registration != nil {
+			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
+			if err != nil {
+				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
+			if err != nil {
+				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
+				select {}
+			}
+		}
+	}
+
+	return myAcmeConfig, nil
+}
+
+func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+	// getting main cert before ACME account so that we can fail here without hitting rate limits
+	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
+	if err != nil {
+		return fmt.Errorf("cert database is not working")
+	}
+
+	acmeClient, err = lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+		if err != nil {
+			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
+		}
+		if enableHTTPServer {
+			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
+			}
+		}
+	}
+
+	mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
+	if err != nil {
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
+	} else {
+		if dnsProvider == "" {
+			// using mock server, don't use wildcard certs
+			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
+			}
+		} else {
+			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create DNS Challenge provider")
+			}
+			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
+			if err != nil {
+				log.Error().Err(err).Msg("Can't create DNS-01 provider")
+			}
+		}
+	}
+
+	if mainCertBytes == nil {
+		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+		if err != nil {
+			log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
+		}
+	}
+
+	return nil
+}
+
+func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+	for {
+		// clean up expired certs
+		now := time.Now()
+		expiredCertCount := 0
+		keyDatabaseIterator := certDB.Items()
+		key, resBytes, err := keyDatabaseIterator.Next()
+		for err == nil {
+			if !bytes.Equal(key, mainDomainSuffix) {
+				resGob := bytes.NewBuffer(resBytes)
+				resDec := gob.NewDecoder(resGob)
+				res := &certificate.Resource{}
+				err = resDec.Decode(res)
+				if err != nil {
+					panic(err)
+				}
+
+				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
+				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
+					err := certDB.Delete(string(key))
+					if err != nil {
+						log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
+					} else {
+						expiredCertCount++
+					}
+				}
+			}
+			key, resBytes, err = keyDatabaseIterator.Next()
+		}
+		log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
+
+		// compact the database
+		msg, err := certDB.Compact()
+		if err != nil {
+			log.Error().Err(err).Msg("Compacting key database failed")
+		} else {
+			log.Debug().Msgf("Compacted key database: %s", msg)
+		}
+
+		// update main cert
+		res, err := certDB.Get(string(mainDomainSuffix))
+		if err != nil {
+			log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
+		} else if res == nil {
+			log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix))
+		} else {
+			tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
+
+			// renew main certificate 30 days before it expires
+			if !tlsCertificates[0].NotAfter.After(time.Now().Add(30 * 24 * time.Hour)) {
+				go (func() {
+					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
+					if err != nil {
+						log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
+					}
+				})()
+			}
+		}
+
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(interval):
+		}
+	}
+}

server/certificates/mock.go

@@ -0,0 +1,86 @@
+package certificates
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"math/big"
+	"time"
+
+	"github.com/go-acme/lego/v4/certcrypto"
+	"github.com/go-acme/lego/v4/certificate"
+
+	"codeberg.org/codeberg/pages/server/database"
+)
+
+func mockCert(domain, msg, mainDomainSuffix string, keyDatabase database.CertDB) tls.Certificate {
+	key, err := certcrypto.GeneratePrivateKey(certcrypto.RSA2048)
+	if err != nil {
+		panic(err)
+	}
+
+	template := x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject: pkix.Name{
+			CommonName:   domain,
+			Organization: []string{"Codeberg Pages Error Certificate (couldn't obtain ACME certificate)"},
+			OrganizationalUnit: []string{
+				"Will not try again for 6 hours to avoid hitting rate limits for your domain.",
+				"Check https://docs.codeberg.org/codeberg-pages/troubleshooting/ for troubleshooting tips, and feel " +
+					"free to create an issue at https://codeberg.org/Codeberg/pages-server if you can't solve it.\n",
+				"Error message: " + msg,
+			},
+		},
+
+		// certificates younger than 7 days are renewed, so this enforces the cert to not be renewed for a 6 hours
+		NotAfter:  time.Now().Add(time.Hour*24*7 + time.Hour*6),
+		NotBefore: time.Now(),
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+	certBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		&template,
+		&template,
+		&key.(*rsa.PrivateKey).PublicKey,
+		key,
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	out := &bytes.Buffer{}
+	err = pem.Encode(out, &pem.Block{
+		Bytes: certBytes,
+		Type:  "CERTIFICATE",
+	})
+	if err != nil {
+		panic(err)
+	}
+	outBytes := out.Bytes()
+	res := &certificate.Resource{
+		PrivateKey:        certcrypto.PEMEncode(key),
+		Certificate:       outBytes,
+		IssuerCertificate: outBytes,
+		Domain:            domain,
+	}
+	databaseName := domain
+	if domain == "*"+mainDomainSuffix || domain == mainDomainSuffix[1:] {
+		databaseName = mainDomainSuffix
+	}
+	if err := keyDatabase.Put(databaseName, res); err != nil {
+		panic(err)
+	}
+
+	tlsCertificate, err := tls.X509KeyPair(res.Certificate, res.PrivateKey)
+	if err != nil {
+		panic(err)
+	}
+	return tlsCertificate
+}

server/certificates/mock_test.go

@@ -0,0 +1,17 @@
+package certificates
+
+import (
+	"testing"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMockCert(t *testing.T) {
+	db, err := database.NewTmpDB()
+	assert.NoError(t, err)
+	cert := mockCert("example.com", "some error msg", "codeberg.page", db)
+	if assert.NotEmpty(t, cert) {
+		assert.NotEmpty(t, cert.Certificate)
+	}
+}

server/database/interface.go

@@ -0,0 +1,15 @@
+package database
+
+import (
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+type CertDB interface {
+	Close() error
+	Put(name string, cert *certificate.Resource) error
+	Get(name string) (*certificate.Resource, error)
+	Delete(key string) error
+	Compact() (string, error)
+	Items() *pogreb.ItemIterator
+}

server/database/mock.go

@@ -0,0 +1,55 @@
+package database
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/OrlovEvgeny/go-mcache"
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+var _ CertDB = tmpDB{}
+
+type tmpDB struct {
+	intern *mcache.CacheDriver
+	ttl    time.Duration
+}
+
+func (p tmpDB) Close() error {
+	_ = p.intern.Close()
+	return nil
+}
+
+func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	return p.intern.Set(name, cert, p.ttl)
+}
+
+func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+	cert, has := p.intern.Get(name)
+	if !has {
+		return nil, fmt.Errorf("cert for '%s' not found", name)
+	}
+	return cert.(*certificate.Resource), nil
+}
+
+func (p tmpDB) Delete(key string) error {
+	p.intern.Remove(key)
+	return nil
+}
+
+func (p tmpDB) Compact() (string, error) {
+	p.intern.Truncate()
+	return "Truncate done", nil
+}
+
+func (p tmpDB) Items() *pogreb.ItemIterator {
+	panic("ItemIterator not implemented for tmpDB")
+}
+
+func NewTmpDB() (CertDB, error) {
+	return &tmpDB{
+		intern: mcache.New(),
+		ttl:    time.Minute,
+	}, nil
+}

server/database/setup.go

@@ -0,0 +1,109 @@
+package database
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"fmt"
+	"time"
+
+	"github.com/akrylysov/pogreb"
+	"github.com/akrylysov/pogreb/fs"
+	"github.com/go-acme/lego/v4/certificate"
+	"github.com/rs/zerolog/log"
+)
+
+var _ CertDB = aDB{}
+
+type aDB struct {
+	ctx          context.Context
+	cancel       context.CancelFunc
+	intern       *pogreb.DB
+	syncInterval time.Duration
+}
+
+func (p aDB) Close() error {
+	p.cancel()
+	return p.intern.Sync()
+}
+
+func (p aDB) Put(name string, cert *certificate.Resource) error {
+	var resGob bytes.Buffer
+	if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
+		return err
+	}
+	return p.intern.Put([]byte(name), resGob.Bytes())
+}
+
+func (p aDB) Get(name string) (*certificate.Resource, error) {
+	cert := &certificate.Resource{}
+	resBytes, err := p.intern.Get([]byte(name))
+	if err != nil {
+		return nil, err
+	}
+	if resBytes == nil {
+		return nil, nil
+	}
+	if err = gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
+		return nil, err
+	}
+	return cert, nil
+}
+
+func (p aDB) Delete(key string) error {
+	return p.intern.Delete([]byte(key))
+}
+
+func (p aDB) Compact() (string, error) {
+	result, err := p.intern.Compact()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%+v", result), nil
+}
+
+func (p aDB) Items() *pogreb.ItemIterator {
+	return p.intern.Items()
+}
+
+var _ CertDB = &aDB{}
+
+func (p aDB) sync() {
+	for {
+		err := p.intern.Sync()
+		if err != nil {
+			log.Error().Err(err).Msg("Syncing cert database failed")
+		}
+		select {
+		case <-p.ctx.Done():
+			return
+		case <-time.After(p.syncInterval):
+		}
+	}
+}
+
+func New(path string) (CertDB, error) {
+	if path == "" {
+		return nil, fmt.Errorf("path not set")
+	}
+	db, err := pogreb.Open(path, &pogreb.Options{
+		BackgroundSyncInterval:       30 * time.Second,
+		BackgroundCompactionInterval: 6 * time.Hour,
+		FileSystem:                   fs.OSMMap,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	result := &aDB{
+		ctx:          ctx,
+		cancel:       cancel,
+		intern:       db,
+		syncInterval: 5 * time.Minute,
+	}
+
+	go result.sync()
+
+	return result, nil
+}

server/dns/const.go

@@ -0,0 +1,6 @@
+package dns
+
+import "time"
+
+// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
+var lookupCacheTimeout = 15 * time.Minute

server/dns/dns.go

@@ -0,0 +1,56 @@
+package dns
+
+import (
+	"net"
+	"strings"
+
+	"codeberg.org/codeberg/pages/server/cache"
+)
+
+// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
+// If everything is fine, it returns the target data.
+func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
+	// Get CNAME or TXT
+	var cname string
+	var err error
+	if cachedName, ok := dnsLookupCache.Get(domain); ok {
+		cname = cachedName.(string)
+	} else {
+		cname, err = net.LookupCNAME(domain)
+		cname = strings.TrimSuffix(cname, ".")
+		if err != nil || !strings.HasSuffix(cname, mainDomainSuffix) {
+			cname = ""
+			// TODO: check if the A record matches!
+			names, err := net.LookupTXT(domain)
+			if err == nil {
+				for _, name := range names {
+					name = strings.TrimSuffix(name, ".")
+					if strings.HasSuffix(name, mainDomainSuffix) {
+						cname = name
+						break
+					}
+				}
+			}
+		}
+		_ = dnsLookupCache.Set(domain, cname, lookupCacheTimeout)
+	}
+	if cname == "" {
+		return
+	}
+	cnameParts := strings.Split(strings.TrimSuffix(cname, mainDomainSuffix), ".")
+	targetOwner = cnameParts[len(cnameParts)-1]
+	if len(cnameParts) > 1 {
+		targetRepo = cnameParts[len(cnameParts)-2]
+	}
+	if len(cnameParts) > 2 {
+		targetBranch = cnameParts[len(cnameParts)-3]
+	}
+	if targetRepo == "" {
+		targetRepo = "pages"
+	}
+	if targetBranch == "" && targetRepo != "pages" {
+		targetBranch = "pages"
+	}
+	// if targetBranch is still empty, the caller must find the default branch
+	return
+}

server/gitea/cache.go

@@ -0,0 +1,12 @@
+package gitea
+
+type FileResponse struct {
+	Exists   bool
+	ETag     []byte
+	MimeType string
+	Body     []byte
+}
+
+func (f FileResponse) IsEmpty() bool {
+	return len(f.Body) != 0
+}

server/gitea/client.go

@@ -0,0 +1,142 @@
+package gitea
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const (
+	giteaAPIRepos         = "/api/v1/repos/"
+	giteaObjectTypeHeader = "X-Gitea-Object-Type"
+)
+
+var ErrorNotFound = errors.New("not found")
+
+type Client struct {
+	giteaRoot      string
+	giteaAPIToken  string
+	fastClient     *fasthttp.Client
+	infoTimeout    time.Duration
+	contentTimeout time.Duration
+
+	followSymlinks bool
+	supportLFS     bool
+}
+
+// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
+func joinURL(baseURL string, paths ...string) string {
+	p := make([]string, 0, len(paths))
+	for i := range paths {
+		path := strings.TrimSpace(paths[i])
+		path = strings.Trim(path, "/")
+		if len(path) != 0 {
+			p = append(p, path)
+		}
+	}
+
+	return baseURL + "/" + strings.Join(p, "/")
+}
+
+func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
+	rootURL, err := url.Parse(giteaRoot)
+	giteaRoot = strings.Trim(rootURL.String(), "/")
+
+	return &Client{
+		giteaRoot:      giteaRoot,
+		giteaAPIToken:  giteaAPIToken,
+		infoTimeout:    5 * time.Second,
+		contentTimeout: 10 * time.Second,
+		fastClient:     getFastHTTPClient(),
+
+		followSymlinks: followSymlinks,
+		supportLFS:     supportLFS,
+	}, err
+}
+
+func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body(), nil
+}
+
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
+	var apiURL string
+	if client.supportLFS {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+	} else {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	}
+	resp, err := client.do(client.contentTimeout, apiURL)
+	if err != nil {
+		return nil, err
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch resp.StatusCode() {
+	case fasthttp.StatusOK:
+		objType := string(resp.Header.Peek(giteaObjectTypeHeader))
+		log.Trace().Msgf("server raw content object: %s", objType)
+		if client.followSymlinks && objType == "symlink" {
+			// TODO: limit to 1000 chars if we switched to std
+			linkDest := strings.TrimSpace(string(resp.Body()))
+			log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+			return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+		}
+
+		return resp, nil
+
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
+	}
+}
+
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}
+
+func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(url)
+	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	err := client.fastClient.DoTimeout(req, res, timeout)
+
+	return res, err
+}

server/gitea/client_test.go

@@ -0,0 +1,23 @@
+package gitea
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoinURL(t *testing.T) {
+	baseURL := ""
+	assert.EqualValues(t, "/", joinURL(baseURL))
+	assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
+
+	baseURL = "http://wwow.url.com"
+	assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
+
+	baseURL = "http://wow.url.com/subpath/2"
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
+}

server/gitea/fasthttp.go

@@ -0,0 +1,15 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -0,0 +1,311 @@
+package server
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+	"github.com/valyala/fasthttp"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+	"codeberg.org/codeberg/pages/server/utils"
+	"codeberg.org/codeberg/pages/server/version"
+)
+
+// Handler handles a single HTTP request to the web server.
+func Handler(mainDomainSuffix, rawDomain []byte,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
+	blacklistedPaths, allowedCorsDomains [][]byte,
+	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) func(ctx *fasthttp.RequestCtx) {
+	return func(ctx *fasthttp.RequestCtx) {
+		log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger()
+
+		ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
+
+		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
+		ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
+
+		// Enable browser caching for up to 10 minutes
+		ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
+
+		trimmedHost := utils.TrimHostPort(ctx.Request.Host())
+
+		// Add HSTS for RawDomain and MainDomainSuffix
+		if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+			ctx.Response.Header.Set("Strict-Transport-Security", hsts)
+		}
+
+		// Block all methods not required for static pages
+		if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
+			ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
+			ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
+			return
+		}
+
+		// Block blacklisted paths (like ACME challenges)
+		for _, blacklistedPath := range blacklistedPaths {
+			if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
+				html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
+				return
+			}
+		}
+
+		// Allow CORS for specified domains
+		allowCors := false
+		for _, allowedCorsDomain := range allowedCorsDomains {
+			if bytes.Equal(trimmedHost, allowedCorsDomain) {
+				allowCors = true
+				break
+			}
+		}
+		if allowCors {
+			ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
+			ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
+		}
+		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
+		if ctx.IsOptions() {
+			ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
+			return
+		}
+
+		// Prepare request information to Gitea
+		var targetOwner, targetRepo, targetBranch, targetPath string
+		targetOptions := &upstream.Options{
+			TryIndexPages: true,
+		}
+
+		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
+		// also disallow search indexing and add a Link header to the canonical URL.
+		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
+			if repo == "" {
+				log.Debug().Msg("tryBranch: repo is empty")
+				return false
+			}
+
+			// Replace "~" to "/" so we can access branch that contains slash character
+			// Branch name cannot contain "~" so doing this is okay
+			branch = strings.ReplaceAll(branch, "~", "/")
+
+			// Check if the branch exists, otherwise treat it as a file path
+			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
+			if branchTimestampResult == nil {
+				log.Debug().Msg("tryBranch: branch doesn't exist")
+				return false
+			}
+
+			// Branch exists, use it
+			targetRepo = repo
+			targetPath = strings.Trim(strings.Join(path, "/"), "/")
+			targetBranch = branchTimestampResult.Branch
+
+			targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
+
+			if canonicalLink != "" {
+				// Hide from search machines & add canonical link
+				ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
+				ctx.Response.Header.Set("Link",
+					strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
+						"; rel=\"canonical\"",
+				)
+			}
+
+			log.Debug().Msg("tryBranch: true")
+			return true
+		}
+
+		log.Debug().Msg("Preparing")
+		if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
+			// Serve raw content from RawDomain
+			log.Debug().Msg("Serving raw domain")
+
+			targetOptions.TryIndexPages = false
+			if targetOptions.ForbiddenMimeTypes == nil {
+				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
+			}
+			targetOptions.ForbiddenMimeTypes["text/html"] = true
+			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
+
+			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+			if len(pathElements) < 2 {
+				// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
+				ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
+				return
+			}
+			targetOwner = pathElements[0]
+			targetRepo = pathElements[1]
+
+			// raw.codeberg.org/example/myrepo/@main/index.html
+			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
+				log.Debug().Msg("Preparing raw domain, now trying with specified branch")
+				if tryBranch(log,
+					targetRepo, pathElements[2][1:], pathElements[3:],
+					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
+				) {
+					log.Info().Msg("tryBranch, now trying upstream 1")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+						canonicalDomainCache, branchTimestampCache, fileResponseCache)
+					return
+				}
+				log.Warn().Msg("Path missed a branch")
+				html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+				return
+			}
+
+			log.Debug().Msg("Preparing raw domain, now trying with default branch")
+			tryBranch(log,
+				targetRepo, "", pathElements[2:],
+				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
+			)
+			log.Info().Msg("tryBranch, now trying upstream 2")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+				canonicalDomainCache, branchTimestampCache, fileResponseCache)
+			return
+
+		} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+			// Serve pages from subdomains of MainDomainSuffix
+			log.Info().Msg("Serve pages from main domain suffix")
+
+			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+			targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
+			targetRepo = pathElements[0]
+			targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
+
+			if targetOwner == "www" {
+				// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
+				ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
+				return
+			}
+
+			// Check if the first directory is a repo with the second directory as a branch
+			// example.codeberg.page/myrepo/@main/index.html
+			if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
+				if targetRepo == "pages" {
+					// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
+					ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
+					return
+				}
+
+				log.Debug().Msg("Preparing main domain, now trying with specified repo & branch")
+				if tryBranch(log,
+					pathElements[0], pathElements[1][1:], pathElements[2:],
+					"/"+pathElements[0]+"/%p",
+				) {
+					log.Info().Msg("tryBranch, now trying upstream 3")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+						canonicalDomainCache, branchTimestampCache, fileResponseCache)
+				} else {
+					log.Warn().Msg("tryBranch: upstream 3 failed")
+					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+				}
+				return
+			}
+
+			// Check if the first directory is a branch for the "pages" repo
+			// example.codeberg.page/@main/index.html
+			if strings.HasPrefix(pathElements[0], "@") {
+				log.Debug().Msg("Preparing main domain, now trying with specified branch")
+				if tryBranch(log,
+					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
+					log.Info().Msg("tryBranch, now trying upstream 4")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+						canonicalDomainCache, branchTimestampCache, fileResponseCache)
+				} else {
+					log.Warn().Msg("tryBranch: upstream 4 failed")
+					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+				}
+				return
+			}
+
+			// Check if the first directory is a repo with a "pages" branch
+			// example.codeberg.page/myrepo/index.html
+			// example.codeberg.page/pages/... is not allowed here.
+			log.Debug().Msg("main domain preparations, now trying with specified repo")
+			if pathElements[0] != "pages" && tryBranch(log,
+				pathElements[0], "pages", pathElements[1:], "") {
+				log.Info().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+					canonicalDomainCache, branchTimestampCache, fileResponseCache)
+				return
+			}
+
+			// Try to use the "pages" repo on its default branch
+			// example.codeberg.page/index.html
+			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
+			if tryBranch(log,
+				"pages", "", pathElements, "") {
+				log.Info().Msg("tryBranch, now trying upstream 6")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+					canonicalDomainCache, branchTimestampCache, fileResponseCache)
+				return
+			}
+
+			// Couldn't find a valid repo/branch
+
+			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+			return
+		} else {
+			trimmedHostStr := string(trimmedHost)
+
+			// Serve pages from external domains
+			targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
+			if targetOwner == "" {
+				html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+				return
+			}
+
+			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+			canonicalLink := ""
+			if strings.HasPrefix(pathElements[0], "@") {
+				targetBranch = pathElements[0][1:]
+				pathElements = pathElements[1:]
+				canonicalLink = "/%p"
+			}
+
+			// Try to use the given repo on the given branch or the default branch
+			log.Debug().Msg("Preparing custom domain, now trying with details from DNS")
+			if tryBranch(log,
+				targetRepo, targetBranch, pathElements, canonicalLink) {
+				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
+				if !valid {
+					log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical")
+					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
+					return
+				} else if canonicalDomain != trimmedHostStr {
+					// only redirect if the target is also a codeberg page!
+					targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
+					if targetOwner != "" {
+						ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
+						return
+					}
+
+					log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
+					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+					return
+				}
+
+				log.Info().Msg("tryBranch, now trying upstream 7 %s")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
+					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
+					canonicalDomainCache, branchTimestampCache, fileResponseCache)
+				return
+			}
+
+			log.Warn().Msg("Couldn't handle request, none of the options succeed")
+			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+			return
+		}
+	}
+}

server/handler_test.go

@@ -0,0 +1,51 @@
+package server
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/valyala/fasthttp"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
+	testHandler := Handler(
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
+		giteaClient,
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
+		[][]byte{[]byte("/.well-known/acme-challenge/")},
+		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
+		cache.NewKeyValueCache(),
+		cache.NewKeyValueCache(),
+		cache.NewKeyValueCache(),
+		cache.NewKeyValueCache(),
+	)
+
+	testCase := func(uri string, status int) {
+		ctx := &fasthttp.RequestCtx{
+			Request:  *fasthttp.AcquireRequest(),
+			Response: *fasthttp.AcquireResponse(),
+		}
+		ctx.Request.SetRequestURI(uri)
+		fmt.Printf("Start: %v\n", time.Now())
+		start := time.Now()
+		testHandler(ctx)
+		end := time.Now()
+		fmt.Printf("Done: %v\n", time.Now())
+		if ctx.Response.StatusCode() != status {
+			t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
+		} else {
+			t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
+		}
+	}
+
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://example.momar.xyz/", 424)       // TODO: expect 200
+	testCase("https://codeberg.page/", 424)           // TODO: expect 200
+}

server/helpers.go

@@ -0,0 +1,15 @@
+package server
+
+import (
+	"bytes"
+)
+
+// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
+// string for custom domains.
+func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
+	if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
+		return "max-age=63072000; includeSubdomains; preload"
+	} else {
+		return ""
+	}
+}

server/setup.go

@@ -0,0 +1,53 @@
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/valyala/fasthttp"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/utils"
+)
+
+type fasthttpLogger struct{}
+
+func (fasthttpLogger) Printf(format string, args ...interface{}) {
+	log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
+}
+
+func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
+	// Enable compression by wrapping the handler with the compression function provided by FastHTTP
+	compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
+
+	return &fasthttp.Server{
+		Handler:                      compressedHandler,
+		DisablePreParseMultipartForm: true,
+		NoDefaultServerHeader:        true,
+		NoDefaultDate:                true,
+		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
+		Logger:                       fasthttpLogger{},
+	}
+}
+
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
+	challengePath := []byte("/.well-known/acme-challenge/")
+
+	return &fasthttp.Server{
+		Handler: func(ctx *fasthttp.RequestCtx) {
+			if bytes.HasPrefix(ctx.Path(), challengePath) {
+				challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
+				if !ok || challenge == nil {
+					ctx.SetStatusCode(http.StatusNotFound)
+					ctx.SetBodyString("no challenge for this token")
+				}
+				ctx.SetBodyString(challenge.(string))
+			} else {
+				ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
+			}
+		},
+	}
+}

server/try.go

@@ -0,0 +1,50 @@
+package server
+
+import (
+	"bytes"
+	"strings"
+
+	"github.com/valyala/fasthttp"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"codeberg.org/codeberg/pages/server/upstream"
+)
+
+// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
+func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
+	mainDomainSuffix, trimmedHost []byte,
+
+	targetOptions *upstream.Options,
+	targetOwner, targetRepo, targetBranch, targetPath string,
+
+	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) {
+	// check if a canonical domain exists on a request on MainDomain
+	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
+		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
+			canonicalPath := string(ctx.RequestURI())
+			if targetRepo != "pages" {
+				path := strings.SplitN(canonicalPath, "/", 3)
+				if len(path) >= 3 {
+					canonicalPath = "/" + path[2]
+				}
+			}
+			ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	targetOptions.TargetOwner = targetOwner
+	targetOptions.TargetRepo = targetRepo
+	targetOptions.TargetBranch = targetBranch
+	targetOptions.TargetPath = targetPath
+	targetOptions.Host = string(trimmedHost)
+
+	// Try to request the file from the Gitea API
+	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
+	}
+}

server/upstream/const.go

@@ -0,0 +1,24 @@
+package upstream
+
+import "time"
+
+// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
+var defaultBranchCacheTimeout = 15 * time.Minute
+
+// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
+// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
+// picked up faster, while still allowing the content to be cached longer if nothing changes.
+var branchExistenceCacheTimeout = 5 * time.Minute
+
+// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
+// on your available memory.
+// TODO: move as option into cache interface
+var fileCacheTimeout = 5 * time.Minute
+
+// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
+var fileCacheSizeLimit = 1024 * 1024
+
+// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
+var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -0,0 +1,50 @@
+package upstream
+
+import (
+	"strings"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
+func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+	var (
+		domains []string
+		valid   bool
+	)
+	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
+		domains = cachedValue.([]string)
+		for _, domain := range domains {
+			if domain == actualDomain {
+				valid = true
+				break
+			}
+		}
+	} else {
+		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
+		if err == nil {
+			for _, domain := range strings.Split(string(body), "\n") {
+				domain = strings.ToLower(domain)
+				domain = strings.TrimSpace(domain)
+				domain = strings.TrimPrefix(domain, "http://")
+				domain = strings.TrimPrefix(domain, "https://")
+				if len(domain) > 0 && !strings.HasPrefix(domain, "#") && !strings.ContainsAny(domain, "\t /") && strings.ContainsRune(domain, '.') {
+					domains = append(domains, domain)
+				}
+				if domain == actualDomain {
+					valid = true
+				}
+			}
+		}
+		domains = append(domains, targetOwner+mainDomainSuffix)
+		if domains[len(domains)-1] == actualDomain {
+			valid = true
+		}
+		if targetRepo != "" && targetRepo != "pages" {
+			domains[len(domains)-1] += "/" + targetRepo
+		}
+		_ = canonicalDomainCache.Set(targetOwner+"/"+targetRepo+"/"+targetBranch, domains, canonicalDomainCacheTimeout)
+	}
+	return domains[0], valid
+}

server/upstream/helper.go

@@ -0,0 +1,76 @@
+package upstream
+
+import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+type branchTimestamp struct {
+	Branch    string
+	Timestamp time.Time
+}
+
+// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
+// (or nil if the branch doesn't exist)
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
+		if result == nil {
+			return nil
+		}
+		return result.(*branchTimestamp)
+	}
+	result := &branchTimestamp{
+		Branch: branch,
+	}
+	if len(branch) == 0 {
+		// Get default branch
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
+		if err != nil {
+			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
+			return nil
+		}
+		result.Branch = defaultBranch
+	}
+
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+	if err != nil {
+		return nil
+	}
+	result.Timestamp = timestamp
+	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
+	return result
+}
+
+func (o *Options) getMimeTypeByExtension() string {
+	if o.ForbiddenMimeTypes == nil {
+		o.ForbiddenMimeTypes = make(map[string]bool)
+	}
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
+	return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
+}

server/upstream/upstream.go

@@ -0,0 +1,213 @@
+package upstream
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/valyala/fasthttp"
+
+	"codeberg.org/codeberg/pages/html"
+	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+)
+
+// upstreamIndexPages lists pages that may be considered as index pages for directories.
+var upstreamIndexPages = []string{
+	"index.html",
+}
+
+// upstreamNotFoundPages lists pages that may be considered as custom 404 Not Found pages.
+var upstreamNotFoundPages = []string{
+	"404.html",
+}
+
+// Options provides various options for the upstream request.
+type Options struct {
+	TargetOwner,
+	TargetRepo,
+	TargetBranch,
+	TargetPath,
+
+	// Used for debugging purposes.
+	Host string
+
+	DefaultMimeType    string
+	ForbiddenMimeTypes map[string]bool
+	TryIndexPages      bool
+	BranchTimestamp    time.Time
+	// internal
+	appendTrailingSlash bool
+	redirectIfExists    string
+}
+
+// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
+
+	// Check if the branch exists and when it was modified
+	if o.BranchTimestamp.IsZero() {
+		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
+
+		if branch == nil {
+			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+			return true
+		}
+		o.TargetBranch = branch.Branch
+		o.BranchTimestamp = branch.Timestamp
+	}
+
+	if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
+		html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
+		return true
+	}
+
+	// Check if the browser has a cached version
+	if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
+		if !ifModifiedSince.Before(o.BranchTimestamp) {
+			ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
+			return true
+		}
+	}
+
+	log.Debug().Msg("Preparing")
+
+	// Make a GET request to the upstream URL
+	uri := o.generateUri()
+	var res *fasthttp.Response
+	var cachedResponse gitea.FileResponse
+	var err error
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
+		cachedResponse = cachedValue.(gitea.FileResponse)
+	} else {
+		res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
+	}
+	log.Debug().Msg("Aquisting")
+
+	// Handle errors
+	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
+		if o.TryIndexPages {
+			// copy the o struct & try if an index page exists
+			optionsForIndexPages := *o
+			optionsForIndexPages.TryIndexPages = false
+			optionsForIndexPages.appendTrailingSlash = true
+			for _, indexPage := range upstreamIndexPages {
+				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
+				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
+					}, fileCacheTimeout)
+					return true
+				}
+			}
+			// compatibility fix for GitHub Pages (/example → /example.html)
+			optionsForIndexPages.appendTrailingSlash = false
+			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
+			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
+			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+					Exists: false,
+				}, fileCacheTimeout)
+				return true
+			}
+		}
+		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+		if o.TryIndexPages {
+			// copy the o struct & try if a not found page exists
+			optionsForNotFoundPages := *o
+			optionsForNotFoundPages.TryIndexPages = false
+			optionsForNotFoundPages.appendTrailingSlash = false
+			for _, notFoundPage := range upstreamNotFoundPages {
+				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
+					}, fileCacheTimeout)
+					return true
+				}
+			}
+		}
+		if res != nil {
+			// Update cache if the request is fresh
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+				Exists: false,
+			}, fileCacheTimeout)
+		}
+		return false
+	}
+	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
+		log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
+		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+		return true
+	}
+
+	// Append trailing slash if missing (for index files), and redirect to fix filenames in general
+	// o.appendTrailingSlash is only true when looking for index pages
+	if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
+		ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
+		return true
+	}
+	if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
+		ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
+		return true
+	}
+	if o.redirectIfExists != "" {
+		ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
+		return true
+	}
+	log.Debug().Msg("Handling error")
+
+	// Set the MIME type
+	mimeType := o.getMimeTypeByExtension()
+	ctx.Response.Header.SetContentType(mimeType)
+
+	// Set ETag
+	if cachedResponse.Exists {
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	} else if res != nil {
+		cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	}
+
+	if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
+		// Everything's okay so far
+		ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	}
+	ctx.Response.Header.SetLastModified(o.BranchTimestamp)
+
+	log.Debug().Msg("Prepare response")
+
+	// Write the response body to the original request
+	var cacheBodyWriter bytes.Buffer
+	if res != nil {
+		if res.Header.ContentLength() > fileCacheSizeLimit {
+			// fasthttp else will set "Content-Length: 0"
+			ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+
+			err = res.BodyWriteTo(ctx.Response.BodyWriter())
+		} else {
+			// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
+			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
+		}
+	} else {
+		_, err = ctx.Write(cachedResponse.Body)
+	}
+	if err != nil {
+		log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
+		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+		return true
+	}
+	log.Debug().Msg("Sending response")
+
+	if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
+		cachedResponse.Exists = true
+		cachedResponse.MimeType = mimeType
+		cachedResponse.Body = cacheBodyWriter.Bytes()
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
+	}
+
+	return true
+}

server/utils/utils.go

@@ -0,0 +1,11 @@
+package utils
+
+import "bytes"
+
+func TrimHostPort(host []byte) []byte {
+	i := bytes.IndexByte(host, ':')
+	if i >= 0 {
+		return host[:i]
+	}
+	return host
+}

server/utils/utils_test.go

@@ -0,0 +1,13 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTrimHostPort(t *testing.T) {
+	assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
+	assert.EqualValues(t, "", TrimHostPort([]byte(":")))
+	assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
+}

server/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string = "dev"